mirror of
https://github.com/paboyle/Grid.git
synced 2026-04-21 11:06:12 +01:00
Compare commits
40 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Guido Cossu
|
2223cf0f4b | ||
|
Guido Cossu
|
56d1308477 | ||
|
Guido Cossu
|
a88ce1a573 | ||
|
Guido Cossu
|
6c9b8e2850 | ||
|
Guido Cossu
|
90b8e57765 | ||
|
Guido Cossu
|
ae6dcd057d | ||
|
Guido Cossu
|
f8721a4388 | ||
|
Guido Cossu
|
38f93f7e12 | ||
|
Guido Cossu
|
7c51dc07f1 | ||
|
Guido Cossu
|
4931505e5a | ||
|
Guido Cossu
|
2099047b06 | ||
|
Guido Cossu
|
0fb307ae17 | ||
|
Guido Cossu
|
2b100d76f2 | ||
|
Guido Cossu
|
04160eaa30 | ||
|
Guido Cossu
|
6cb9d2935f | ||
|
Guido Cossu
|
ac7f845583 | ||
|
Guido Cossu
|
ff057145b4 | ||
|
Guido Cossu
|
9fdb209b7e | ||
|
Guido Cossu
|
2d324f53cc | ||
|
Guido Cossu
|
f250133dc5 | ||
|
Guido Cossu
|
8d61a552e4 | ||
|
Guido Cossu
|
84cac60ca9 | ||
|
Guido Cossu
|
dfd7a2e86e | ||
|
Guido Cossu
|
23666f20c1 | ||
|
Guido Cossu
|
44f33e47d7 | ||
|
Guido Cossu
|
b1bd88cfba | ||
|
Guido Cossu
|
d63110d26f | ||
|
Guido Cossu
|
a170855d84 | ||
|
Guido Cossu
|
416bd9a874 | ||
|
Guido Cossu
|
bc9427c38c | ||
|
Guido Cossu
|
188d9472da | ||
|
Guido Cossu
|
dc6951f43e | ||
|
Guido Cossu
|
2deeef622b | ||
|
Guido Cossu
|
8ec1950718 | ||
|
Guido Cossu
|
dc942d5d56 | ||
|
Guido Cossu
|
b3de0f922a | ||
|
Guido Cossu
|
3c0931cca9 | ||
|
Guido Cossu
|
d75906f517 | ||
|
Guido Cossu
|
c354838200 | ||
|
Guido Cossu
|
3292924e6e |
+6
-116
@@ -1,118 +1,8 @@
|
||||
# Compiled Object files #
|
||||
#########################
|
||||
*.slo
|
||||
*.lo
|
||||
*.o
|
||||
*.obj
|
||||
# Exclude directories
|
||||
_site
|
||||
.sass-cache
|
||||
.jekyll-metadata
|
||||
pdf
|
||||
|
||||
# Editor files #
|
||||
################
|
||||
# Exclude backup files
|
||||
*~
|
||||
*#
|
||||
*.sublime-*
|
||||
|
||||
# Precompiled Headers #
|
||||
#######################
|
||||
*.gch
|
||||
*.pch
|
||||
|
||||
# Compiled Dynamic libraries #
|
||||
##############################
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
|
||||
# Fortran module files #
|
||||
########################
|
||||
*.mod
|
||||
|
||||
# Compiled Static libraries #
|
||||
#############################
|
||||
*.lai
|
||||
*.la
|
||||
*.a
|
||||
*.lib
|
||||
|
||||
# Executables #
|
||||
###############
|
||||
*.exe
|
||||
*.out
|
||||
*.app
|
||||
|
||||
# http://www.gnu.org/software/automake #
|
||||
########################################
|
||||
Makefile.in
|
||||
Makefile
|
||||
Config.h
|
||||
Config.h.in
|
||||
config.log
|
||||
config.status
|
||||
.deps
|
||||
Make.inc
|
||||
eigen.inc
|
||||
Eigen.inc
|
||||
|
||||
# http://www.gnu.org/software/autoconf #
|
||||
########################################
|
||||
autom4te.cache
|
||||
aclocal.m4
|
||||
compile
|
||||
configure
|
||||
depcomp
|
||||
install-sh
|
||||
missing
|
||||
stamp-h1
|
||||
config.sub
|
||||
config.guess
|
||||
INSTALL
|
||||
.dirstamp
|
||||
ltmain.sh
|
||||
|
||||
# Logs and databases #
|
||||
######################
|
||||
*.log
|
||||
*.sql
|
||||
*.sqlite
|
||||
|
||||
# OS generated files #
|
||||
######################
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
.dirstamp
|
||||
|
||||
# build directory #
|
||||
###################
|
||||
build*/*
|
||||
Documentation/_build
|
||||
|
||||
# IDE related files #
|
||||
#####################
|
||||
*.xcodeproj/*
|
||||
build.sh
|
||||
.vscode
|
||||
*.code-workspace
|
||||
|
||||
# Eigen source #
|
||||
################
|
||||
Grid/Eigen
|
||||
Eigen/*
|
||||
|
||||
# libtool macros #
|
||||
##################
|
||||
m4/lt*
|
||||
m4/libtool.m4
|
||||
|
||||
# github pages #
|
||||
################
|
||||
gh-pages/
|
||||
|
||||
# generated sources #
|
||||
#####################
|
||||
Grid/qcd/spin/gamma-gen/*.h
|
||||
Grid/qcd/spin/gamma-gen/*.cc
|
||||
Grid/util/Version.h
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@MacBook-Pro.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
+571
@@ -0,0 +1,571 @@
|
||||
## [3.4.8](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.8)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Improve type readability for larger viewports by bumping up base `font-size`. [#533](https://github.com/mmistakes/minimal-mistakes/issues/533)
|
||||
- Update Portuguese localized UI text. [#541](https://github.com/mmistakes/minimal-mistakes/pull/541)
|
||||
- Add `page.title` and via parameter to Twitter share link. [#538](https://github.com/mmistakes/minimal-mistakes/pull/538)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix Last.fm author profile URL. [#540](https://github.com/mmistakes/minimal-mistakes/pull/540)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Move Brazilian Portuguese localized text under `pt-BR` key.
|
||||
|
||||
## [3.4.7](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.7)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add `layout` based and user-defined class names to `<body>` element for added CSS hooks. [#526](https://github.com/mmistakes/minimal-mistakes/pull/526)
|
||||
- Add simplified Chinese localized UI text. [#532](https://github.com/mmistakes/minimal-mistakes/pull/532)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Remove duplicate include of `base_path` in category-list.html [#522](https://github.com/mmistakes/minimal-mistakes/pull/522)
|
||||
|
||||
## [3.4.6](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.6)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add Italian "comments" related localized UI text. [#514](https://github.com/mmistakes/minimal-mistakes/pull/514)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Disable `compress` HTML layout by default. To enable add `layout: compress` to `_layouts/default.html`.
|
||||
|
||||
## [3.4.5](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.5)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Improve line numbered code block styling when using `{% highlight linenos %}` tag. [#513](https://github.com/mmistakes/minimal-mistakes/issues/513)
|
||||
- Add English fallback to "Follow" button label. [#496](https://github.com/mmistakes/minimal-mistakes/pull/496)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix Firefox alignment issues with code blocks generated with the `{% highlight %}` tag. [#512](https://github.com/mmistakes/minimal-mistakes/issues/512)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Clarified comment for `author.stackoverflow` value used in author sidebar links. [#487](https://github.com/mmistakes/minimal-mistakes/pull/487)
|
||||
- Add list of localized text strings. [#488](https://github.com/mmistakes/minimal-mistakes/pull/488)
|
||||
- Add `{% highlight %}` code block examples to demo site.
|
||||
- Add documentation for using custom sidebar navigation menus. [#476](https://github.com/mmistakes/minimal-mistakes/issues/476)
|
||||
|
||||
## [3.4.4](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.4)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add French "comments" related localized UI text. [#472](https://github.com/mmistakes/minimal-mistakes/pull/472)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Exclude `vendor` in Jekyll config file.
|
||||
- Fix Liquid syntax error for offending parenthesis. [#479](https://github.com/mmistakes/minimal-mistakes/issues/479)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Update gems: `colorator` (1.1.0), `forwardable-extended` (2.6.0), `github-pages` (93), `jekyll` (= 3.2.1), `minima` (= 1.0.1).
|
||||
|
||||
## [3.4.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.3)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Make ["honeypot" `input`](https://github.com/mmistakes/minimal-mistakes/commit/06a8249a69a37dddda7e2a5bfbe32056c1a9a607) in Staticman comment form less obvious to spam bots
|
||||
- Add padding to `.highlight` code blocks to better [align `overflow` scrollbar](https://github.com/mmistakes/minimal-mistakes/commit/e4abec0a6f7f8cff72505ca0754615df294fd5b3) to the bottom.
|
||||
- Add additional image options for Twitter card social sharing meta tags. [#466](https://github.com/mmistakes/minimal-mistakes/pull/466)
|
||||
- Add structured data markup for Staticman comments. [#458](https://github.com/mmistakes/minimal-mistakes/issues/458)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Format `og:locale` tag with `_` instead of `-`. [#462](https://github.com/mmistakes/minimal-mistakes/issues/462)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Add note to docs about using `url: http://localhost:4000` when working locally.
|
||||
|
||||
## [3.4.2](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.2)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Improve UX of static comment forms. [#448](https://github.com/mmistakes/minimal-mistakes/issues/448)
|
||||
|
||||
## [3.4.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.1)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add `staticman.filename` configuration with UNIX timestamp for sorting data files. example ~> `comment-1470943149`.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Don't add `<a>` to author name if URL is blank.
|
||||
|
||||
## [3.4.0](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.4.0)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Support static-based commenting via [Staticman](https://staticman.net/) for sites hosted with GitHub Pages. [#424](https://github.com/mmistakes/minimal-mistakes/issues/424)
|
||||
|
||||
## [3.3.7](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.3.7)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Re-enabled Jekyll plugins in `_config.yml` in case they aren't autoloaded in `Gemfile`. [#417](https://github.com/mmistakes/minimal-mistakes/issues/417)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Fallback to `site.github.url` for use in `{{ base_path }}` when `site.url` is `nil`.
|
||||
- Replace Sass and Autoprefixer `npm` build scripts with [Jekyll's built-in asset support](https://jekyllrb.com/docs/assets/). [#333](https://github.com/mmistakes/minimal-mistakes/issues/333)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Document `site.repository` and its role with [`github-metadata`](https://github.com/jekyll/github-metadata) gem.
|
||||
- Add sample [archive page with content](https://mmistakes.github.io/minimal-mistakes/archive-layout-with-content/) for testing styles on demo site.
|
||||
|
||||
## [3.3.6](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.3.6)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix blank `site.teaser` bug. [#412](https://github.com/mmistakes/minimal-mistakes/issues/412)
|
||||
|
||||
## [3.3.5](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.3.5)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add English default text `site.locale` strings. [#407](https://github.com/mmistakes/minimal-mistakes/issues/407)
|
||||
- Add Portuguese localized UI text. [#411](https://github.com/mmistakes/minimal-mistakes/pull/411)
|
||||
- Add Italian localized UI text. [#409](https://github.com/mmistakes/minimal-mistakes/pull/409)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Remove unused Google AdSense variables in `_config.yml`. [#404](https://github.com/mmistakes/minimal-mistakes/issues/404)
|
||||
- Update `Gemfile` instructions for using `github-pages` vs. native `jekyll` gems.
|
||||
- Disable `gems:` in `_config.yml` and enable plugins with Bundler instead.
|
||||
- Add `repository` to `_config.yml` to suppress GitHub Pages error `Liquid Exception: No repo name found.`
|
||||
|
||||
## [3.3.4](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.3.4)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add support for configurable feed URL to use a service like FeedBurner instead of linking directly to `feed.xml` in `<head>` and the site footer. [#378](https://github.com/mmistakes/minimal-mistakes/issues/378), [#379](https://github.com/mmistakes/minimal-mistakes/pull/379), [#406](https://github.com/mmistakes/minimal-mistakes/pull/406)
|
||||
- Add Turkish localized UI text. [#403](https://github.com/mmistakes/minimal-mistakes/pull/403)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Update gems: `activesupport` (4.2.7), `ffi` (1.9.14), `github-pages` (88), `jekyll-redirect-from` (0.11.0), `jekyll-watch` (1.5.0).
|
||||
|
||||
## [3.3.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.3.3)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Make footer stick to the bottom of the page.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix `gallery` size bug [#402](https://github.com/mmistakes/minimal-mistakes/issues/402)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Set default `lang` to `en`.
|
||||
|
||||
## [3.3.2](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.3.2)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix JavaScript that triggers "sticky" sidebar to avoid layout issues on screen sizes < `1024px`. [#396](https://github.com/mmistakes/minimal-mistakes/issues/396)
|
||||
|
||||
## [3.3.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.3.1)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Enable image popup on < 500px wide screens. [#385](https://github.com/mmistakes/minimal-mistakes/issues/385)
|
||||
- Indicate the relationship between component URLs in a paginated series by applying `rel="prev"` and `rel="next"` to pages that use `site.paginator`. [#253](https://github.com/mmistakes/minimal-mistakes/issues/253)
|
||||
- Improve link posts in archive listings. [#276](https://github.com/mmistakes/minimal-mistakes/issues/276)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Update gems: `github-pages` (86), `ffi` 1.9.13, `jekyll-mentions` 1.1.3, and `rouge` 1.11.1
|
||||
- Fix note about custom sidebar content appearing below author profile. [#388](https://github.com/mmistakes/minimal-mistakes/issues/388)
|
||||
|
||||
## [3.2.13](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.13)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add English default UI text for Canada, Great Britain, and Australia. [#377](https://github.com/mmistakes/minimal-mistakes/issues/377)
|
||||
- Switch default locale from `en-US` to `en`.
|
||||
|
||||
## [3.2.12](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.12)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Remove window width "magic number" from sticky sidebar check in `main.js` for improved flexibility. [#375](https://github.com/mmistakes/minimal-mistakes/pull/375)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix author override conditional where a missing `authors.yml` would show broken sidebar content. Defaults to `site.author`. [#376](https://github.com/mmistakes/minimal-mistakes/pull/376)
|
||||
|
||||
## [3.2.11](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.11)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix disappearing author sidebar links [#372](https://github.com/mmistakes/minimal-mistakes/issues/372)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Update gems: `github-pages` (84), `jekyll-github-metadata` 2.0.2, and `kramdown` 1.11.1
|
||||
- Update vendor JavaScript: jQuery 1.12.4, Stickyfill.js 1.1.4
|
||||
- Update Font Awesome 4.6.3
|
||||
|
||||
## [3.2.10](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.10)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Add `CONTRIBUTING.md`
|
||||
|
||||
## [3.2.9](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.9)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add support for [header overlay images](https://mmistakes.github.io/minimal-mistakes/docs/layouts/#header-overlay) for Open Graph images. [#358](https://github.com/mmistakes/minimal-mistakes/pull/358)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix `Person` typo Schema.org type [#358](https://github.com/mmistakes/minimal-mistakes/pull/358)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Update `github-pages` gem and dependencies.
|
||||
- Remove `minutes_read` to avoid awkward reading time wording [#356](https://github.com/mmistakes/minimal-mistakes/issues/356)
|
||||
|
||||
## [3.2.8](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.8)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Remove `cursor: pointer` that appears on white-space surrounding author side list items and links. [#354](https://github.com/mmistakes/minimal-mistakes/pull/354)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Add contributing information to `README.md`. [#357](https://github.com/mmistakes/minimal-mistakes/issues/357)
|
||||
|
||||
## [3.2.7](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.7)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add French localized UI text. [#346](https://github.com/mmistakes/minimal-mistakes/pull/346)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix branch logic for Yandex and Alexa in `seo.html`. [#348](https://github.com/mmistakes/minimal-mistakes/pull/348)
|
||||
|
||||
## [3.2.6](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.6)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix error `Liquid Exception: divided by 0 in _includes/archive-single.html, included in _layouts/single.html` caused by null `words_per_minute` in `_config.yml`. [#345](https://github.com/mmistakes/minimal-mistakes/pull/345)
|
||||
|
||||
## [3.2.5](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.5)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix link color in hero overlay to be white.
|
||||
- Remove underlines from archive item titles.
|
||||
|
||||
## [3.2.4](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.4)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Improve text alignment of masthead, hero overlay, page footer to be flush left and remove awkward white-space gaps. [#342](https://github.com/mmistakes/minimal-mistakes/issues/342)
|
||||
- Add Spanish localized UI text. [#338](https://github.com/mmistakes/minimal-mistakes/pull/338)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix alignment of icons in author sidebar [#341](https://github.com/mmistakes/minimal-mistakes/issues/341)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Add background color to page footer to set it apart from main content. [#342](https://github.com/mmistakes/minimal-mistakes/issues/342)
|
||||
- Add terms and privacy policy to theme's demo site. [#343](https://github.com/mmistakes/minimal-mistakes/issues/343)
|
||||
- Update screenshots found in theme documentation.
|
||||
|
||||
## [3.2.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.3)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add [Discourse](https://www.discourse.org/) as a commenting provider. [#335](https://github.com/mmistakes/minimal-mistakes/pull/335)
|
||||
|
||||
## [3.2.2](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.2)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add support for image captions in Magnific Popup overlays via the [`gallery`](https://mmistakes.github.io/minimal-mistakes/docs/helpers/#gallery) helper. [#334](https://github.com/mmistakes/minimal-mistakes/issues/334)
|
||||
|
||||
## [3.2.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.1)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Remove need for "double tapping" masthead menu links on iOS devices. [#315](https://github.com/mmistakes/minimal-mistakes/issues/315)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Add `ISSUE_TEMPLATE.md` for improve issue submission process.
|
||||
|
||||
## [3.2.0](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.2.0)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix missing category/tag links in post footer due to possible conflict with `site.tags` and `site.categories`. [#329](https://github.com/mmistakes/minimal-mistakes/issues/329#issuecomment-222375568)
|
||||
|
||||
## [3.1.8](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.8)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix `Liquid Exception: undefined method 'gsub' for nil:NilClass in _layouts/single.html` error when `page.title` is null. `<h1>` element is now conditional if `title: ` is not set for a `page` or collection item. [#312](https://github.com/mmistakes/minimal-mistakes/issues/312)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Remove duplicate `fa-twitter` and `fa-twitter-square` classes from `_utilities.scss`. [#302](https://github.com/mmistakes/minimal-mistakes/issues/302)
|
||||
|
||||
- Document installing additional Jekyll gem dependencies when using `gem "jekyll"` instead of `gem "github-pages"` to avoid any errors on run. [#305](https://github.com/mmistakes/minimal-mistakes/issues/305)
|
||||
|
||||
## [3.1.7](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.7)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add translation key for "Recent Posts" used in home page `index.html`. [#316](https://github.com/mmistakes/minimal-mistakes/pull/316)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Small fix to avoid underlying the whitespace between icons and related text when hovering. [#303](https://github.com/mmistakes/minimal-mistakes/pull/303)
|
||||
|
||||
## [3.1.6](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.6)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Update gem dependencies. Run `bundle` to update `Gemfile.lock`.
|
||||
|
||||
## [3.1.5](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.5)
|
||||
|
||||
### Maintenance
|
||||
|
||||
- Fix `www` and `https` links in author profile include [#293](https://github.com/mmistakes/minimal-mistakes/pull/293)
|
||||
|
||||
## [3.1.4](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.4)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add overlay_filter param to hero headers [#298](https://github.com/mmistakes/minimal-mistakes/pull/298)
|
||||
|
||||
## [3.1.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.3)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Improve `site.locale` documentation [#284](https://github.com/mmistakes/minimal-mistakes/issues/284)
|
||||
- Remove ProTip note about protocol-less `site.url` as it is an anti-pattern [#288](https://github.com/mmistakes/minimal-mistakes/issues/288)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix `og_image` URL in seo.html [#277](https://github.com/mmistakes/minimal-mistakes/issues/277)
|
||||
- Fix `author_profile` toggle when assigned in a `_layout` [#285](https://github.com/mmistakes/minimal-mistakes/issues/285)
|
||||
- Fix typo in `build:all` npm script [#283](https://github.com/mmistakes/minimal-mistakes/pull/283)
|
||||
- Fix URL typo documentation [#287](https://github.com/mmistakes/minimal-mistakes/issues/287)
|
||||
- SEO author bug. If `twitter.username` is set and `author.twitter` is `nil` bad things happen. [#289](https://github.com/mmistakes/minimal-mistakes/issues/289)
|
||||
|
||||
## [3.1.2](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.2)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Explain how to use `nav_list` helper in [documentation](https://mmistakes.github.io/minimal-mistakes/docs/helpers/#navigation-list).
|
||||
- Reduce left/right padding on smaller screens to increase width of main content column.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix alignment issues with related posts [#273](https://github.com/mmistakes/minimal-mistakes/issues/273) and "Follow" button in author profile [#274](https://github.com/mmistakes/minimal-mistakes/issues/274).
|
||||
|
||||
## [3.1.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.1)
|
||||
|
||||
### Bug Fix
|
||||
|
||||
- Fixed reading time bug when `words_per_minute` wasn't set in `_config.yml` [#271](https://github.com/mmistakes/minimal-mistakes/issues/271)
|
||||
|
||||
## [3.1.0](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.1.0)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Updated [Font Awesome](https://fortawesome.github.io/Font-Awesome/whats-new/) to version 4.6.1
|
||||
- Added optional GitHub and Bitbucket links to footer if set on `site.author` in `_config.yml`.
|
||||
|
||||
### Bug Fixes
|
||||
- Fixed Bitbucket URL typo in author sidebar.
|
||||
|
||||
## [3.0.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/3.0.3)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Rebuilt the entire theme: layouts, includes, stylesheets, scripts, you name it.
|
||||
- Refreshed the look and feel while staying true to the original design of the theme (author sidebar/main content).
|
||||
- Replaced grid system with [Susy](http://susy.oddbird.net/).
|
||||
- Replaced Grunt tasks with `npm` scripts.
|
||||
- Removed Google Fonts and replaced with system fonts to improve performance (they can be [added back](https://mmistakes.github.io/minimal-mistakes/docs/stylesheets/) if desired)
|
||||
- Greatly improved [theme documentation](https://mmistakes.github.io/minimal-mistakes/docs/quick-start-guide/).
|
||||
- Increased the amount of sample posts, sample pages, and sample collections to throughly test the theme and edge-cases.
|
||||
- Moved all sample content and assets out of `master` to keep it as clean as possible for forking.
|
||||
- Added new layouts for `splash` pages, archives for [`jekyll-archives`](https://github.com/jekyll/jekyll-archives) if enabled, and [`compress.html`](https://github.com/penibelst/jekyll-compress-html) to improve performance.
|
||||
- Added taxonomy links to posts (tags and categories).
|
||||
- Added optional "reading time" meta data.
|
||||
- Improved Liquid used for Twitter Cards and Open Graph data in `<head>`.
|
||||
- Improved `gallery` include helper and added `feature_row` for use with splash page layout.
|
||||
- Added Keybase.io, author web URI, and Bitbucket optional links to sidebar.
|
||||
- Add `feed.xml` link to footer.
|
||||
- Added a [UI text data file](https://mmistakes.github.io/minimal-mistakes/docs/ui-text/) to easily change all text found in the theme.
|
||||
- Added LinkedIn to optional social share buttons.
|
||||
- Added Facebook, Google+, and custom commenting options in addition to Disqus.
|
||||
- Added optional breadcrumb links.
|
||||
|
||||
## [2.2.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/2.2.1)
|
||||
|
||||
## [2.2.0](https://github.com/mmistakes/minimal-mistakes/releases/tag/2.2.0)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add support for Jekyll 3.0
|
||||
- Minor updates to syntax highlighting CSS and theme documentation
|
||||
|
||||
## [2.1.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/2.1.3)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Cleaner print styles that remove the top navigation, social sharing buttons, and other elements not needed when printed.
|
||||
|
||||
## [2.1.2](https://github.com/mmistakes/minimal-mistakes/releases/tag/2.1.2)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add optional CodePen icon/url to author side bar [#156](https://github.com/mmistakes/minimal-mistakes/pull/156)
|
||||
- Documented Stackoverflow username explanation in `_config.yml` [#157](https://github.com/mmistakes/minimal-mistakes/pull/157)
|
||||
- Simplified Liquid in `post-index.html` to better handle year listings [#166](https://github.com/mmistakes/minimal-mistakes/pull/166)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Cleanup Facebook related Open Graph meta tags [#149](https://github.com/mmistakes/minimal-mistakes/issues/149)
|
||||
- Corrected minor typos [#158](https://github.com/mmistakes/minimal-mistakes/pull/158) [#175](https://github.com/mmistakes/minimal-mistakes/issues/175)
|
||||
|
||||
## [2.1.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/2.1.1)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add optional XING profile link to author sidebar
|
||||
- Include open graph meta tags for feature image (if assigned) [#149](https://github.com/mmistakes/minimal-mistakes/issues/149)
|
||||
- Create an include for feed footer
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Remove http protocol from Google search form on sample 404 page
|
||||
- Only show related posts if there are one or more available
|
||||
- Fix alignment of email address link in author sidebar
|
||||
|
||||
## [2.1.0](https://github.com/mmistakes/minimal-mistakes/releases/tag/2.1.0)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Add optional social sharing buttons ([#42](https://github.com/mmistakes/minimal-mistakes/issues/42))
|
||||
|
||||
|
||||
|
||||
- Add Soundcloud, YouTube ([#95](https://github.com/mmistakes/minimal-mistakes/pull/95)), Flickr ([#119](https://github.com/mmistakes/minimal-mistakes/pull/119)), and Weibo ([#116](https://github.com/mmistakes/minimal-mistakes/pull/116)) icons for use in author sidebar.
|
||||
- Fix typos in posts and documentation and remove references to Less
|
||||
- Include note about Octopress gem being optional
|
||||
- Post author override support extended to the Atom feed ([#71](https://github.com/mmistakes/minimal-mistakes/pull/71))
|
||||
- Only include email address in feed if specified in `_config.yml` or author `_data`
|
||||
- Wrap all page content in `#main` to harmonize article and post index styles ([#86](https://github.com/mmistakes/minimal-mistakes/issues/86))
|
||||
- Include new sample feature images for posts and pages
|
||||
- Table of contents improvements: fix collapse toggle, indent nested elements, show on small screens, and create an `_include` for reusing in posts and pages.
|
||||
- Include note about running Jekyll with `bundle exec` when using Bundler
|
||||
- Fix home page path in top navigation
|
||||
- Remove Google Authorship ([#120](https://github.com/mmistakes/minimal-mistakes/issues/120))
|
||||
- Remove duplicate author content that displayed in `div.article-author-bottom`
|
||||
- Removed unused `_sass/print.scss` styles
|
||||
- Improve comments in `.scss` files
|
||||
|
||||
## [2.0.0](https://github.com/mmistakes/minimal-mistakes/releases/tag/v2.0)
|
||||
|
||||
## [1.3.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/1.3.3)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Added new icons and profile links for Stackoverflow, Dribbble, Pinterest, Foursquare, and Steam to the author bio sidebar.
|
||||
- Cleaned up the Kramdown auto table of contents styling to be more readable
|
||||
- Removed page width specific .less stylesheets and created mixins for easier updating
|
||||
- Removed Modernizr since it wasn't being used
|
||||
- Added pages to sitemap.xml
|
||||
- Added category: to rake new_post task
|
||||
- Minor typographic changes
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Corrected various broken links in README and Theme Setup.
|
||||
|
||||
## [1.3.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/1.3.1)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Cleaned up table of contents styling
|
||||
- Reworked top navigation to be a better experience on small screens. Nav items now display vertically when the menu button is tapped, revealing links with larger touch targets.
|
||||
|
||||
|
||||
|
||||
## [1.2.0](https://github.com/mmistakes/minimal-mistakes/releases/tag/1.2.0)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Table weren't filling the entire width of the content container. They now scale at 100%. Thanks [@dhruvbhatia](https://github.com/dhruvbhatia)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Decreased spacing between Markdown footnotes
|
||||
- Removed dark background on footer
|
||||
- Removed UPPERCASE styling on post titles in the index listing
|
||||
|
||||
## [1.1.4](https://github.com/mmistakes/minimal-mistakes/releases/tag/1.1.4)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix top navigation bug issue ([#10](https://github.com/mmistakes/minimal-mistakes/issues/10)) for real this time. Remember to clear your floats kids.
|
||||
|
||||
## [1.1.3](https://github.com/mmistakes/minimal-mistakes/releases/tag/1.1.3)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix top navigation links that weren't click able on small viewports (Issue [#10](https://github.com/mmistakes/minimal-mistakes/issues/10)).
|
||||
- Remove line wrap from top navigation links that may span multiple lines.
|
||||
|
||||
## [1.1.2](https://github.com/mmistakes/minimal-mistakes/releases/tag/1.1.2)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Added Grunt build script for compiling Less/JavaScript and optimizing image assets.
|
||||
- Added support for large image summary Twitter card.
|
||||
- Stylesheet adjustments
|
||||
|
||||
## [1.1.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/1.1.1)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Removed [Typeplate](http://typeplate.com/) styles. Was [causing issues with newer versions of Less](https://github.com/typeplate/typeplate.github.io/issues/108) and is no longer maintained.
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Added [image attribution](http://mmistakes.github.io/minimal-mistakes/theme-setup/#feature-images) for post and page feature images.
|
||||
- Added [404 page](http://mmistakes.github.io/minimal-mistakes/404.html).
|
||||
- Cleaned up various Less variables to better align with naming conventions used in other MM Jekyll themes.
|
||||
- Removed Chrome Frame references.
|
||||
- Added global CSS3 transitions to text and block elements.
|
||||
- Improved typography in a few places.
|
||||
|
||||
## [1.0.2](https://github.com/mmistakes/minimal-mistakes/releases/tag/v1.0.2)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- Google Analytics, Google Authorship, webmaster verifies, and Twitter card meta are now optional.
|
||||
|
||||
## [1.0.1](https://github.com/mmistakes/minimal-mistakes/releases/tag/v1.0.1)
|
||||
@@ -1,5 +0,0 @@
|
||||
Version : 0.8.0
|
||||
|
||||
- Clang 3.5 and above, ICPC v16 and above, GCC 6.3 and above recommended
|
||||
- MPI and MPI3 comms optimisations for KNL and OPA finished
|
||||
- Half precision comms
|
||||
@@ -0,0 +1,27 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
# Hello! This is where you manage which Jekyll version is used to run.
|
||||
# When you want to use a different version, change it below, save the
|
||||
# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:
|
||||
#
|
||||
# bundle exec jekyll serve
|
||||
#
|
||||
# This will help ensure the proper Jekyll version is running.
|
||||
# Happy Jekylling!
|
||||
|
||||
gem "github-pages", group: :jekyll_plugins
|
||||
|
||||
# If you want to use Jekyll native, uncomment the line below.
|
||||
# To upgrade, run `bundle update`.
|
||||
|
||||
# gem "jekyll"
|
||||
|
||||
gem "wdm", "~> 0.1.0" if Gem.win_platform?
|
||||
|
||||
# If you have any plugins, put them here!
|
||||
group :jekyll_plugins do
|
||||
# gem "jekyll-archives"
|
||||
|
||||
gem 'jekyll-octicons'
|
||||
|
||||
end
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
activesupport (4.2.7)
|
||||
i18n (~> 0.7)
|
||||
json (~> 1.7, >= 1.7.7)
|
||||
minitest (~> 5.1)
|
||||
thread_safe (~> 0.3, >= 0.3.4)
|
||||
tzinfo (~> 1.1)
|
||||
addressable (2.4.0)
|
||||
coffee-script (2.4.1)
|
||||
coffee-script-source
|
||||
execjs
|
||||
coffee-script-source (1.10.0)
|
||||
colorator (1.1.0)
|
||||
ethon (0.9.1)
|
||||
ffi (>= 1.3.0)
|
||||
execjs (2.7.0)
|
||||
faraday (0.9.2)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
ffi (1.9.14)
|
||||
ffi (1.9.14-x64-mingw32)
|
||||
forwardable-extended (2.6.0)
|
||||
gemoji (2.1.0)
|
||||
github-pages (104)
|
||||
activesupport (= 4.2.7)
|
||||
github-pages-health-check (= 1.2.0)
|
||||
jekyll (= 3.3.0)
|
||||
jekyll-avatar (= 0.4.2)
|
||||
jekyll-coffeescript (= 1.0.1)
|
||||
jekyll-feed (= 0.8.0)
|
||||
jekyll-gist (= 1.4.0)
|
||||
jekyll-github-metadata (= 2.2.0)
|
||||
jekyll-mentions (= 1.2.0)
|
||||
jekyll-paginate (= 1.1.0)
|
||||
jekyll-redirect-from (= 0.11.0)
|
||||
jekyll-sass-converter (= 1.3.0)
|
||||
jekyll-seo-tag (= 2.1.0)
|
||||
jekyll-sitemap (= 0.12.0)
|
||||
jekyll-swiss (= 0.4.0)
|
||||
jemoji (= 0.7.0)
|
||||
kramdown (= 1.11.1)
|
||||
liquid (= 3.0.6)
|
||||
listen (= 3.0.6)
|
||||
mercenary (~> 0.3)
|
||||
minima (= 2.0.0)
|
||||
rouge (= 1.11.1)
|
||||
terminal-table (~> 1.4)
|
||||
github-pages-health-check (1.2.0)
|
||||
addressable (~> 2.3)
|
||||
net-dns (~> 0.8)
|
||||
octokit (~> 4.0)
|
||||
public_suffix (~> 1.4)
|
||||
typhoeus (~> 0.7)
|
||||
html-pipeline (2.4.2)
|
||||
activesupport (>= 2)
|
||||
nokogiri (>= 1.4)
|
||||
i18n (0.7.0)
|
||||
jekyll (3.3.0)
|
||||
addressable (~> 2.4)
|
||||
colorator (~> 1.0)
|
||||
jekyll-sass-converter (~> 1.0)
|
||||
jekyll-watch (~> 1.1)
|
||||
kramdown (~> 1.3)
|
||||
liquid (~> 3.0)
|
||||
mercenary (~> 0.3.3)
|
||||
pathutil (~> 0.9)
|
||||
rouge (~> 1.7)
|
||||
safe_yaml (~> 1.0)
|
||||
jekyll-avatar (0.4.2)
|
||||
jekyll (~> 3.0)
|
||||
jekyll-coffeescript (1.0.1)
|
||||
coffee-script (~> 2.2)
|
||||
jekyll-feed (0.8.0)
|
||||
jekyll (~> 3.3)
|
||||
jekyll-gist (1.4.0)
|
||||
octokit (~> 4.2)
|
||||
jekyll-github-metadata (2.2.0)
|
||||
jekyll (~> 3.1)
|
||||
octokit (~> 4.0, != 4.4.0)
|
||||
jekyll-mentions (1.2.0)
|
||||
activesupport (~> 4.0)
|
||||
html-pipeline (~> 2.3)
|
||||
jekyll (~> 3.0)
|
||||
jekyll-octicons (3.0.1)
|
||||
jekyll (~> 3.1)
|
||||
octicons (~> 3.0)
|
||||
jekyll-paginate (1.1.0)
|
||||
jekyll-redirect-from (0.11.0)
|
||||
jekyll (>= 2.0)
|
||||
jekyll-sass-converter (1.3.0)
|
||||
sass (~> 3.2)
|
||||
jekyll-seo-tag (2.1.0)
|
||||
jekyll (~> 3.3)
|
||||
jekyll-sitemap (0.12.0)
|
||||
jekyll (~> 3.3)
|
||||
jekyll-swiss (0.4.0)
|
||||
jekyll-watch (1.5.0)
|
||||
listen (~> 3.0, < 3.1)
|
||||
jemoji (0.7.0)
|
||||
activesupport (~> 4.0)
|
||||
gemoji (~> 2.0)
|
||||
html-pipeline (~> 2.2)
|
||||
jekyll (>= 3.0)
|
||||
json (1.8.3)
|
||||
kramdown (1.11.1)
|
||||
liquid (3.0.6)
|
||||
listen (3.0.6)
|
||||
rb-fsevent (>= 0.9.3)
|
||||
rb-inotify (>= 0.9.7)
|
||||
mercenary (0.3.6)
|
||||
mini_portile2 (2.1.0)
|
||||
minima (2.0.0)
|
||||
minitest (5.9.1)
|
||||
multipart-post (2.0.0)
|
||||
net-dns (0.8.0)
|
||||
nokogiri (1.6.8.1)
|
||||
mini_portile2 (~> 2.1.0)
|
||||
nokogiri (1.6.8.1-x64-mingw32)
|
||||
mini_portile2 (~> 2.1.0)
|
||||
octicons (3.0.1)
|
||||
nokogiri (>= 1.6.3.1)
|
||||
octokit (4.4.1)
|
||||
sawyer (~> 0.7.0, >= 0.5.3)
|
||||
pathutil (0.14.0)
|
||||
forwardable-extended (~> 2.6)
|
||||
public_suffix (1.5.3)
|
||||
rb-fsevent (0.9.8)
|
||||
rb-inotify (0.9.7)
|
||||
ffi (>= 0.5.0)
|
||||
rouge (1.11.1)
|
||||
safe_yaml (1.0.4)
|
||||
sass (3.4.22)
|
||||
sawyer (0.7.0)
|
||||
addressable (>= 2.3.5, < 2.5)
|
||||
faraday (~> 0.8, < 0.10)
|
||||
terminal-table (1.7.3)
|
||||
unicode-display_width (~> 1.1.1)
|
||||
thread_safe (0.3.5)
|
||||
typhoeus (0.8.0)
|
||||
ethon (>= 0.8.0)
|
||||
tzinfo (1.2.2)
|
||||
thread_safe (~> 0.1)
|
||||
unicode-display_width (1.1.1)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
x64-mingw32
|
||||
|
||||
DEPENDENCIES
|
||||
github-pages
|
||||
jekyll-octicons
|
||||
|
||||
BUNDLED WITH
|
||||
1.13.3
|
||||
@@ -1,73 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/DisableWarnings.h
|
||||
|
||||
Copyright (C) 2016
|
||||
|
||||
Author: Guido Cossu <guido.cossu@ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#ifndef DISABLE_WARNINGS_H
|
||||
#define DISABLE_WARNINGS_H
|
||||
|
||||
|
||||
|
||||
#if defined __GNUC__ && __GNUC__>=6
|
||||
#pragma GCC diagnostic ignored "-Wignored-attributes"
|
||||
#endif
|
||||
|
||||
//disables and intel compiler specific warning (in json.hpp)
|
||||
#ifdef __ICC
|
||||
#pragma warning disable 488
|
||||
#endif
|
||||
|
||||
#ifdef __NVCC__
|
||||
//disables nvcc specific warning in json.hpp
|
||||
#pragma clang diagnostic ignored "-Wdeprecated-register"
|
||||
|
||||
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__
|
||||
//disables nvcc specific warning in json.hpp
|
||||
#pragma nv_diag_suppress unsigned_compare_with_zero
|
||||
#pragma nv_diag_suppress cast_to_qualified_type
|
||||
//disables nvcc specific warning in many files
|
||||
#pragma nv_diag_suppress esa_on_defaulted_function_ignored
|
||||
#pragma nv_diag_suppress extra_semicolon
|
||||
#else
|
||||
//disables nvcc specific warning in json.hpp
|
||||
#pragma diag_suppress unsigned_compare_with_zero
|
||||
#pragma diag_suppress cast_to_qualified_type
|
||||
//disables nvcc specific warning in many files
|
||||
#pragma diag_suppress esa_on_defaulted_function_ignored
|
||||
#pragma diag_suppress extra_semicolon
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Disable vectorisation in Eigen on the Power8/9 and PowerPC
|
||||
#ifdef __ALTIVEC__
|
||||
#define EIGEN_DONT_VECTORIZE
|
||||
#endif
|
||||
#ifdef __VSX__
|
||||
#define EIGEN_DONT_VECTORIZE
|
||||
#endif
|
||||
|
||||
#endif
|
||||
-50
@@ -1,50 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Grid.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: azusayamaguchi <ayamaguc@YAMAKAZE.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
//
|
||||
// Grid.h
|
||||
// simd
|
||||
//
|
||||
// Created by Peter Boyle on 09/05/2014.
|
||||
// Copyright (c) 2014 University of Edinburgh. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef GRID_H
|
||||
#define GRID_H
|
||||
|
||||
#include <Grid/GridCore.h>
|
||||
#include <Grid/GridQCDcore.h>
|
||||
#include <Grid/qcd/action/Action.h>
|
||||
#include <Grid/qcd/utils/GaugeFix.h>
|
||||
#include <Grid/qcd/utils/CovariantSmearing.h>
|
||||
#include <Grid/qcd/smearing/Smearing.h>
|
||||
#include <Grid/parallelIO/MetaData.h>
|
||||
#include <Grid/qcd/hmc/HMC_aggregate.h>
|
||||
|
||||
#endif
|
||||
@@ -1,66 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Grid.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: azusayamaguchi <ayamaguc@YAMAKAZE.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
//
|
||||
// Grid.h
|
||||
// simd
|
||||
//
|
||||
// Created by Peter Boyle on 09/05/2014.
|
||||
// Copyright (c) 2014 University of Edinburgh. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef GRID_BASE_H
|
||||
#define GRID_BASE_H
|
||||
|
||||
|
||||
#include <Grid/DisableWarnings.h>
|
||||
#include <Grid/Namespace.h>
|
||||
#include <Grid/GridStd.h>
|
||||
#include <Grid/threads/Pragmas.h>
|
||||
#include <Grid/perfmon/Timer.h>
|
||||
//#include <Grid/perfmon/PerfCount.h>
|
||||
#include <Grid/util/Util.h>
|
||||
#include <Grid/log/Log.h>
|
||||
#include <Grid/perfmon/Tracing.h>
|
||||
#include <Grid/allocator/Allocator.h>
|
||||
#include <Grid/simd/Simd.h>
|
||||
#include <Grid/threads/ThreadReduction.h>
|
||||
#include <Grid/serialisation/Serialisation.h>
|
||||
#include <Grid/util/Sha.h>
|
||||
#include <Grid/communicator/Communicator.h>
|
||||
#include <Grid/cartesian/Cartesian.h>
|
||||
#include <Grid/tensors/Tensors.h>
|
||||
#include <Grid/lattice/Lattice.h>
|
||||
#include <Grid/cshift/Cshift.h>
|
||||
#include <Grid/stencil/Stencil.h>
|
||||
#include <Grid/parallelIO/BinaryIO.h>
|
||||
#include <Grid/algorithms/Algorithms.h>
|
||||
NAMESPACE_CHECK(GridCore)
|
||||
|
||||
#endif
|
||||
@@ -1,44 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Grid.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: azusayamaguchi <ayamaguc@YAMAKAZE.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_QCD_CORE_H
|
||||
#define GRID_QCD_CORE_H
|
||||
|
||||
/////////////////////////
|
||||
// Core Grid QCD headers
|
||||
/////////////////////////
|
||||
#include <Grid/GridCore.h>
|
||||
#include <Grid/qcd/QCD.h>
|
||||
#include <Grid/qcd/spin/Spin.h>
|
||||
#include <Grid/qcd/gparity/Gparity.h>
|
||||
#include <Grid/qcd/utils/Utils.h>
|
||||
#include <Grid/qcd/representations/Representations.h>
|
||||
NAMESPACE_CHECK(GridQCDCore);
|
||||
|
||||
#endif
|
||||
@@ -1,35 +0,0 @@
|
||||
#ifndef GRID_STD_H
|
||||
#define GRID_STD_H
|
||||
|
||||
///////////////////
|
||||
// Std C++ dependencies
|
||||
///////////////////
|
||||
#include <cassert>
|
||||
#include <complex>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <random>
|
||||
#include <functional>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <strings.h>
|
||||
#include <stdio.h>
|
||||
#include <signal.h>
|
||||
#include <ctime>
|
||||
#include <sys/time.h>
|
||||
#include <chrono>
|
||||
#include <zlib.h>
|
||||
|
||||
///////////////////
|
||||
// Grid config
|
||||
///////////////////
|
||||
#include "Config.h"
|
||||
|
||||
#ifdef TOFU
|
||||
#undef GRID_COMMS_THREADS
|
||||
#endif
|
||||
#endif /* GRID_STD_H */
|
||||
@@ -1,75 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#pragma once
|
||||
// Force Eigen to use MKL if Grid has been configured with --enable-mkl
|
||||
#ifdef USE_MKL
|
||||
#define EIGEN_USE_MKL_ALL
|
||||
#endif
|
||||
|
||||
|
||||
#if defined __GNUC__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
||||
#endif
|
||||
|
||||
/* NVCC save and restore compile environment*/
|
||||
#ifdef __NVCC__
|
||||
#pragma push
|
||||
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__
|
||||
#pragma nv_diag_suppress code_is_unreachable
|
||||
#else
|
||||
#pragma diag_suppress code_is_unreachable
|
||||
#endif
|
||||
#pragma push_macro("__CUDA_ARCH__")
|
||||
#pragma push_macro("__NVCC__")
|
||||
#pragma push_macro("__CUDACC__")
|
||||
#undef __CUDA_ARCH__
|
||||
#undef __NVCC__
|
||||
#undef __CUDACC__
|
||||
#define __NVCC__REDEFINE__
|
||||
#endif
|
||||
|
||||
/* SYCL save and restore compile environment*/
|
||||
#ifdef GRID_SYCL
|
||||
#pragma push
|
||||
#pragma push_macro("__SYCL_DEVICE_ONLY__")
|
||||
#undef __SYCL_DEVICE_ONLY__
|
||||
#define EIGEN_DONT_VECTORIZE
|
||||
//#undef EIGEN_USE_SYCL
|
||||
#define __SYCL__REDEFINE__
|
||||
#endif
|
||||
|
||||
/* HIP save and restore compile environment*/
|
||||
#ifdef GRID_HIP
|
||||
#pragma push
|
||||
#pragma push_macro("__HIP_DEVICE_COMPILE__")
|
||||
#endif
|
||||
#define EIGEN_NO_HIP
|
||||
|
||||
#include <Grid/Eigen/Dense>
|
||||
#include <Grid/Eigen/unsupported/CXX11/Tensor>
|
||||
|
||||
/* NVCC restore */
|
||||
#ifdef __NVCC__REDEFINE__
|
||||
#pragma pop_macro("__CUDACC__")
|
||||
#pragma pop_macro("__NVCC__")
|
||||
#pragma pop_macro("__CUDA_ARCH__")
|
||||
#pragma pop
|
||||
#endif
|
||||
|
||||
/*SYCL restore*/
|
||||
#ifdef __SYCL__REDEFINE__
|
||||
#pragma pop_macro("__SYCL_DEVICE_ONLY__")
|
||||
#pragma pop
|
||||
#endif
|
||||
|
||||
/*HIP restore*/
|
||||
#ifdef __HIP__REDEFINE__
|
||||
#pragma pop_macro("__HIP_DEVICE_COMPILE__")
|
||||
#pragma pop
|
||||
#endif
|
||||
|
||||
#if defined __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
#include <Grid/Grid_Eigen_Dense.h>
|
||||
@@ -1,77 +0,0 @@
|
||||
extra_sources=
|
||||
extra_headers=
|
||||
|
||||
if BUILD_COMMS_MPI3
|
||||
extra_sources+=communicator/Communicator_mpi3.cc
|
||||
extra_sources+=communicator/Communicator_base.cc
|
||||
extra_sources+=communicator/SharedMemoryMPI.cc
|
||||
extra_sources+=communicator/SharedMemory.cc
|
||||
endif
|
||||
|
||||
if BUILD_COMMS_NONE
|
||||
extra_sources+=communicator/Communicator_none.cc
|
||||
extra_sources+=communicator/Communicator_base.cc
|
||||
extra_sources+=communicator/SharedMemoryNone.cc
|
||||
extra_sources+=communicator/SharedMemory.cc
|
||||
endif
|
||||
|
||||
if BUILD_HDF5
|
||||
extra_sources+=serialisation/Hdf5IO.cc
|
||||
extra_headers+=serialisation/Hdf5IO.h
|
||||
extra_headers+=serialisation/Hdf5Type.h
|
||||
endif
|
||||
|
||||
|
||||
all: version-cache Version.h
|
||||
|
||||
version-cache:
|
||||
@if [ `git status --porcelain | grep -v '??' | wc -l` -gt 0 ]; then\
|
||||
a="uncommited changes";\
|
||||
else\
|
||||
a="clean";\
|
||||
fi;\
|
||||
echo "`git log -n 1 --format=format:"#define GITHASH \\"%H:%d $$a\\"%n" HEAD`" > vertmp;\
|
||||
if [ -e version-cache ]; then\
|
||||
d=`diff vertmp version-cache`;\
|
||||
if [ "$${d}" != "" ]; then\
|
||||
mv vertmp version-cache;\
|
||||
rm -f Version.h;\
|
||||
fi;\
|
||||
else\
|
||||
mv vertmp version-cache;\
|
||||
rm -f Version.h;\
|
||||
fi;\
|
||||
rm -f vertmp
|
||||
|
||||
Version.h: version-cache
|
||||
cp version-cache Version.h
|
||||
|
||||
.PHONY: version-cache
|
||||
|
||||
#
|
||||
# Libraries
|
||||
#
|
||||
include Make.inc
|
||||
include Eigen.inc
|
||||
|
||||
extra_sources+=$(WILS_FERMION_FILES)
|
||||
extra_sources+=$(STAG_FERMION_FILES)
|
||||
if BUILD_ZMOBIUS
|
||||
extra_sources+=$(ZWILS_FERMION_FILES)
|
||||
endif
|
||||
if BUILD_GPARITY
|
||||
extra_sources+=$(GP_FERMION_FILES)
|
||||
endif
|
||||
if BUILD_FERMION_REPS
|
||||
extra_sources+=$(ADJ_FERMION_FILES)
|
||||
extra_sources+=$(TWOIND_FERMION_FILES)
|
||||
endif
|
||||
|
||||
lib_LIBRARIES = libGrid.a
|
||||
|
||||
CCFILES += $(extra_sources)
|
||||
HFILES += $(extra_headers) Config.h Version.h
|
||||
|
||||
libGrid_a_SOURCES = $(CCFILES)
|
||||
libGrid_adir = $(includedir)/Grid
|
||||
nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) $(eigen_unsupp_files)
|
||||
@@ -1,38 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Namespace.h
|
||||
|
||||
Copyright (C) 2016
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
#include <cassert>
|
||||
|
||||
#define NAMESPACE_BEGIN(A) namespace A {
|
||||
#define NAMESPACE_END(A) }
|
||||
#define GRID_NAMESPACE_BEGIN NAMESPACE_BEGIN(Grid)
|
||||
#define GRID_NAMESPACE_END NAMESPACE_END(Grid)
|
||||
#define NAMESPACE_CHECK(x) struct namespaceTEST##x {}; static_assert(std::is_same<namespaceTEST##x, ::namespaceTEST##x>::value,"Not in :: at" );
|
||||
@@ -1,76 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Algorithms.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_ALGORITHMS_H
|
||||
#define GRID_ALGORITHMS_H
|
||||
|
||||
NAMESPACE_CHECK(algorithms);
|
||||
#include <Grid/algorithms/SparseMatrix.h>
|
||||
#include <Grid/algorithms/LinearOperator.h>
|
||||
#include <Grid/algorithms/Preconditioner.h>
|
||||
NAMESPACE_CHECK(SparseMatrix);
|
||||
|
||||
#include <Grid/algorithms/approx/Zolotarev.h>
|
||||
#include <Grid/algorithms/approx/Chebyshev.h>
|
||||
#include <Grid/algorithms/approx/JacobiPolynomial.h>
|
||||
#include <Grid/algorithms/approx/Remez.h>
|
||||
#include <Grid/algorithms/approx/MultiShiftFunction.h>
|
||||
#include <Grid/algorithms/approx/Forecast.h>
|
||||
#include <Grid/algorithms/approx/RemezGeneral.h>
|
||||
#include <Grid/algorithms/approx/ZMobius.h>
|
||||
NAMESPACE_CHECK(approx);
|
||||
#include <Grid/algorithms/iterative/Deflation.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradient.h>
|
||||
NAMESPACE_CHECK(ConjGrad);
|
||||
#include <Grid/algorithms/iterative/BiCGSTAB.h>
|
||||
NAMESPACE_CHECK(BiCGSTAB);
|
||||
#include <Grid/algorithms/iterative/ConjugateResidual.h>
|
||||
#include <Grid/algorithms/iterative/NormalEquations.h>
|
||||
#include <Grid/algorithms/iterative/SchurRedBlack.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMultiShift.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMixedPrec.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMultiShiftMixedPrec.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMixedPrecBatched.h>
|
||||
#include <Grid/algorithms/iterative/BiCGSTABMixedPrec.h>
|
||||
#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientReliableUpdate.h>
|
||||
#include <Grid/algorithms/iterative/MinimalResidual.h>
|
||||
#include <Grid/algorithms/iterative/GeneralisedMinimalResidual.h>
|
||||
#include <Grid/algorithms/iterative/CommunicationAvoidingGeneralisedMinimalResidual.h>
|
||||
#include <Grid/algorithms/iterative/FlexibleGeneralisedMinimalResidual.h>
|
||||
#include <Grid/algorithms/iterative/FlexibleCommunicationAvoidingGeneralisedMinimalResidual.h>
|
||||
#include <Grid/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h>
|
||||
#include <Grid/algorithms/iterative/ImplicitlyRestartedLanczos.h>
|
||||
#include <Grid/algorithms/iterative/PowerMethod.h>
|
||||
|
||||
NAMESPACE_CHECK(PowerMethod);
|
||||
#include <Grid/algorithms/CoarsenedMatrix.h>
|
||||
NAMESPACE_CHECK(CoarsendMatrix);
|
||||
#include <Grid/algorithms/FFT.h>
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,296 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Cshift.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef _GRID_FFT_H_
|
||||
#define _GRID_FFT_H_
|
||||
|
||||
#ifdef HAVE_FFTW
|
||||
#ifdef USE_MKL
|
||||
#include <fftw/fftw3.h>
|
||||
#else
|
||||
#include <fftw3.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class scalar> struct FFTW { };
|
||||
|
||||
#ifdef HAVE_FFTW
|
||||
template<> struct FFTW<ComplexD> {
|
||||
public:
|
||||
|
||||
typedef fftw_complex FFTW_scalar;
|
||||
typedef fftw_plan FFTW_plan;
|
||||
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, const int *n,int howmany,
|
||||
FFTW_scalar *in, const int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, const int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
return ::fftw_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
||||
}
|
||||
|
||||
static void fftw_flops(const FFTW_plan p,double *add, double *mul, double *fmas){
|
||||
::fftw_flops(p,add,mul,fmas);
|
||||
}
|
||||
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out) {
|
||||
::fftw_execute_dft(p,in,out);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
||||
::fftw_destroy_plan(p);
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct FFTW<ComplexF> {
|
||||
public:
|
||||
|
||||
typedef fftwf_complex FFTW_scalar;
|
||||
typedef fftwf_plan FFTW_plan;
|
||||
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, const int *n,int howmany,
|
||||
FFTW_scalar *in, const int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, const int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
return ::fftwf_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
||||
}
|
||||
|
||||
static void fftw_flops(const FFTW_plan p,double *add, double *mul, double *fmas){
|
||||
::fftwf_flops(p,add,mul,fmas);
|
||||
}
|
||||
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out) {
|
||||
::fftwf_execute_dft(p,in,out);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
||||
::fftwf_destroy_plan(p);
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef FFTW_FORWARD
|
||||
#define FFTW_FORWARD (-1)
|
||||
#define FFTW_BACKWARD (+1)
|
||||
#endif
|
||||
|
||||
class FFT {
|
||||
private:
|
||||
|
||||
GridCartesian *vgrid;
|
||||
GridCartesian *sgrid;
|
||||
|
||||
int Nd;
|
||||
double flops;
|
||||
double flops_call;
|
||||
uint64_t usec;
|
||||
|
||||
Coordinate dimensions;
|
||||
Coordinate processors;
|
||||
Coordinate processor_coor;
|
||||
|
||||
public:
|
||||
|
||||
static const int forward=FFTW_FORWARD;
|
||||
static const int backward=FFTW_BACKWARD;
|
||||
|
||||
double Flops(void) {return flops;}
|
||||
double MFlops(void) {return flops/usec;}
|
||||
double USec(void) {return (double)usec;}
|
||||
|
||||
FFT ( GridCartesian * grid ) :
|
||||
vgrid(grid),
|
||||
Nd(grid->_ndimension),
|
||||
dimensions(grid->_fdimensions),
|
||||
processors(grid->_processors),
|
||||
processor_coor(grid->_processor_coor)
|
||||
{
|
||||
flops=0;
|
||||
usec =0;
|
||||
Coordinate layout(Nd,1);
|
||||
sgrid = new GridCartesian(dimensions,layout,processors,*grid);
|
||||
};
|
||||
|
||||
~FFT ( void) {
|
||||
delete sgrid;
|
||||
}
|
||||
|
||||
template<class vobj>
|
||||
void FFT_dim_mask(Lattice<vobj> &result,const Lattice<vobj> &source,Coordinate mask,int sign){
|
||||
|
||||
conformable(result.Grid(),vgrid);
|
||||
conformable(source.Grid(),vgrid);
|
||||
Lattice<vobj> tmp(vgrid);
|
||||
tmp = source;
|
||||
for(int d=0;d<Nd;d++){
|
||||
if( mask[d] ) {
|
||||
FFT_dim(result,tmp,d,sign);
|
||||
tmp=result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj>
|
||||
void FFT_all_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int sign){
|
||||
Coordinate mask(Nd,1);
|
||||
FFT_dim_mask(result,source,mask,sign);
|
||||
}
|
||||
|
||||
|
||||
template<class vobj>
|
||||
void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int sign){
|
||||
#ifndef HAVE_FFTW
|
||||
assert(0);
|
||||
#else
|
||||
conformable(result.Grid(),vgrid);
|
||||
conformable(source.Grid(),vgrid);
|
||||
|
||||
int L = vgrid->_ldimensions[dim];
|
||||
int G = vgrid->_fdimensions[dim];
|
||||
|
||||
Coordinate layout(Nd,1);
|
||||
Coordinate pencil_gd(vgrid->_fdimensions);
|
||||
|
||||
pencil_gd[dim] = G*processors[dim];
|
||||
|
||||
// Pencil global vol LxLxGxLxL per node
|
||||
GridCartesian pencil_g(pencil_gd,layout,processors,*vgrid);
|
||||
|
||||
// Construct pencils
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
typedef typename sobj::scalar_type scalar;
|
||||
|
||||
Lattice<sobj> pgbuf(&pencil_g);
|
||||
autoView(pgbuf_v , pgbuf, CpuWrite);
|
||||
|
||||
typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
|
||||
typedef typename FFTW<scalar>::FFTW_plan FFTW_plan;
|
||||
|
||||
int Ncomp = sizeof(sobj)/sizeof(scalar);
|
||||
int Nlow = 1;
|
||||
for(int d=0;d<dim;d++){
|
||||
Nlow*=vgrid->_ldimensions[d];
|
||||
}
|
||||
|
||||
int rank = 1; /* 1d transforms */
|
||||
int n[] = {G}; /* 1d transforms of length G */
|
||||
int howmany = Ncomp;
|
||||
int odist,idist,istride,ostride;
|
||||
idist = odist = 1; /* Distance between consecutive FT's */
|
||||
istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */
|
||||
int *inembed = n, *onembed = n;
|
||||
|
||||
scalar div;
|
||||
if ( sign == backward ) div = 1.0/G;
|
||||
else if ( sign == forward ) div = 1.0;
|
||||
else assert(0);
|
||||
|
||||
FFTW_plan p;
|
||||
{
|
||||
FFTW_scalar *in = (FFTW_scalar *)&pgbuf_v[0];
|
||||
FFTW_scalar *out= (FFTW_scalar *)&pgbuf_v[0];
|
||||
p = FFTW<scalar>::fftw_plan_many_dft(rank,n,howmany,
|
||||
in,inembed,
|
||||
istride,idist,
|
||||
out,onembed,
|
||||
ostride, odist,
|
||||
sign,FFTW_ESTIMATE);
|
||||
}
|
||||
|
||||
// Barrel shift and collect global pencil
|
||||
Coordinate lcoor(Nd), gcoor(Nd);
|
||||
result = source;
|
||||
int pc = processor_coor[dim];
|
||||
for(int p=0;p<processors[dim];p++) {
|
||||
{
|
||||
autoView(r_v,result,CpuRead);
|
||||
autoView(p_v,pgbuf,CpuWrite);
|
||||
thread_for(idx, sgrid->lSites(),{
|
||||
Coordinate cbuf(Nd);
|
||||
sobj s;
|
||||
sgrid->LocalIndexToLocalCoor(idx,cbuf);
|
||||
peekLocalSite(s,r_v,cbuf);
|
||||
cbuf[dim]+=((pc+p) % processors[dim])*L;
|
||||
pokeLocalSite(s,p_v,cbuf);
|
||||
});
|
||||
}
|
||||
if (p != processors[dim] - 1) {
|
||||
result = Cshift(result,dim,L);
|
||||
}
|
||||
}
|
||||
|
||||
// Loop over orthog coords
|
||||
int NN=pencil_g.lSites();
|
||||
GridStopWatch timer;
|
||||
timer.Start();
|
||||
thread_for( idx,NN,{
|
||||
Coordinate cbuf(Nd);
|
||||
pencil_g.LocalIndexToLocalCoor(idx, cbuf);
|
||||
if ( cbuf[dim] == 0 ) { // restricts loop to plane at lcoor[dim]==0
|
||||
FFTW_scalar *in = (FFTW_scalar *)&pgbuf_v[idx];
|
||||
FFTW_scalar *out= (FFTW_scalar *)&pgbuf_v[idx];
|
||||
FFTW<scalar>::fftw_execute_dft(p,in,out);
|
||||
}
|
||||
});
|
||||
timer.Stop();
|
||||
|
||||
// performance counting
|
||||
double add,mul,fma;
|
||||
FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
|
||||
flops_call = add+mul+2.0*fma;
|
||||
usec += timer.useconds();
|
||||
flops+= flops_call*NN;
|
||||
|
||||
// writing out result
|
||||
{
|
||||
autoView(pgbuf_v,pgbuf,CpuRead);
|
||||
autoView(result_v,result,CpuWrite);
|
||||
thread_for(idx,sgrid->lSites(),{
|
||||
Coordinate clbuf(Nd), cgbuf(Nd);
|
||||
sobj s;
|
||||
sgrid->LocalIndexToLocalCoor(idx,clbuf);
|
||||
cgbuf = clbuf;
|
||||
cgbuf[dim] = clbuf[dim]+L*pc;
|
||||
peekLocalSite(s,pgbuf_v,cgbuf);
|
||||
pokeLocalSite(s,result_v,clbuf);
|
||||
});
|
||||
}
|
||||
result = result*div;
|
||||
|
||||
// destroying plan
|
||||
FFTW<scalar>::fftw_destroy_plan(p);
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,640 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/LinearOperator.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// LinearOperators Take a something and return a something.
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Hopefully linearity is satisfied and the AdjOp is indeed the Hermitian Conjugateugate (transpose if real):
|
||||
//SBase
|
||||
// i) F(a x + b y) = aF(x) + b F(y).
|
||||
// ii) <x|Op|y> = <y|AdjOp|x>^\ast
|
||||
//
|
||||
// Would be fun to have a test linearity & Herm Conj function!
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class LinearOperatorBase {
|
||||
public:
|
||||
// Support for coarsening to a multigrid
|
||||
virtual void OpDiag (const Field &in, Field &out) = 0; // Abstract base
|
||||
virtual void OpDir (const Field &in, Field &out,int dir,int disp) = 0; // Abstract base
|
||||
virtual void OpDirAll (const Field &in, std::vector<Field> &out) = 0; // Abstract base
|
||||
|
||||
virtual void Op (const Field &in, Field &out) = 0; // Abstract base
|
||||
virtual void AdjOp (const Field &in, Field &out) = 0; // Abstract base
|
||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)=0;
|
||||
virtual void HermOp(const Field &in, Field &out)=0;
|
||||
virtual ~LinearOperatorBase(){};
|
||||
};
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// By sharing the class for Sparse Matrix across multiple operator wrappers, we can share code
|
||||
// between RB and non-RB variants. Sparse matrix is like the fermion action def, and then
|
||||
// the wrappers implement the specialisation of "Op" and "AdjOp" to the cases minimising
|
||||
// replication of code.
|
||||
//
|
||||
// I'm not entirely happy with implementation; to share the Schur code between herm and non-herm
|
||||
// while still having a "OpAndNorm" in the abstract base I had to implement it in both cases
|
||||
// with an assert trap in the non-herm. This isn't right; there must be a better C++ way to
|
||||
// do it, but I fear it required multiple inheritance and mixed in abstract base classes
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// Construct herm op from non-herm matrix
|
||||
////////////////////////////////////////////////////////////////////
|
||||
template<class Matrix,class Field>
|
||||
class MdagMLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
public:
|
||||
MdagMLinearOperator(Matrix &Mat): _Mat(Mat){};
|
||||
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
_Mat.Mdiag(in,out);
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
_Mat.Mdir(in,out,dir,disp);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
_Mat.MdirAll(in,out);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
_Mat.Mdag(in,out);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
_Mat.MdagM(in,out);
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
_Mat.MdagM(in,out);
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// Construct herm op and shift it for mgrid smoother
|
||||
////////////////////////////////////////////////////////////////////
|
||||
template<class Matrix,class Field>
|
||||
class ShiftedMdagMLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
RealD _shift;
|
||||
public:
|
||||
ShiftedMdagMLinearOperator(Matrix &Mat,RealD shift): _Mat(Mat), _shift(shift){};
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
_Mat.Mdiag(in,out);
|
||||
assert(0);
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
_Mat.Mdir(in,out,dir,disp);
|
||||
assert(0);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
assert(0);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
assert(0);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
_Mat.Mdag(in,out);
|
||||
assert(0);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
HermOp(in,out);
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
_Mat.MdagM(in,out);
|
||||
out = out + _shift*in;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// Wrap an already herm matrix
|
||||
////////////////////////////////////////////////////////////////////
|
||||
template<class Matrix,class Field>
|
||||
class HermitianLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
public:
|
||||
HermitianLinearOperator(Matrix &Mat): _Mat(Mat){};
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
_Mat.Mdiag(in,out);
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
_Mat.Mdir(in,out,dir,disp);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
_Mat.MdirAll(in,out);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
HermOp(in,out);
|
||||
ComplexD dot= innerProduct(in,out); n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
}
|
||||
};
|
||||
|
||||
template<class Matrix,class Field>
|
||||
class NonHermitianLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
public:
|
||||
NonHermitianLinearOperator(Matrix &Mat): _Mat(Mat){};
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
_Mat.Mdiag(in,out);
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
_Mat.Mdir(in,out,dir,disp);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
_Mat.MdirAll(in,out);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
_Mat.Mdag(in,out);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
assert(0);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
assert(0);
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// Even Odd Schur decomp operators; there are several
|
||||
// ways to introduce the even odd checkerboarding
|
||||
//////////////////////////////////////////////////////////
|
||||
|
||||
template<class Field>
|
||||
class SchurOperatorBase : public LinearOperatorBase<Field> {
|
||||
public:
|
||||
virtual void Mpc (const Field &in, Field &out) =0;
|
||||
virtual void MpcDag (const Field &in, Field &out) =0;
|
||||
virtual void MpcDagMpc(const Field &in, Field &out) {
|
||||
Field tmp(in.Grid());
|
||||
tmp.Checkerboard() = in.Checkerboard();
|
||||
Mpc(in,tmp);
|
||||
MpcDag(tmp,out);
|
||||
}
|
||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
MpcDagMpc(in,out);
|
||||
ComplexD dot= innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
virtual void HermOp(const Field &in, Field &out){
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
MpcDagMpc(in,out);
|
||||
}
|
||||
void Op (const Field &in, Field &out){
|
||||
Mpc(in,out);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
MpcDag(in,out);
|
||||
}
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
assert(0); // must coarsen the unpreconditioned system
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
assert(0);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
assert(0);
|
||||
};
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
class SchurDiagMooeeOperator : public SchurOperatorBase<Field> {
|
||||
public:
|
||||
Matrix &_Mat;
|
||||
SchurDiagMooeeOperator (Matrix &Mat): _Mat(Mat){};
|
||||
virtual void Mpc (const Field &in, Field &out) {
|
||||
Field tmp(in.Grid());
|
||||
tmp.Checkerboard() = !in.Checkerboard();
|
||||
|
||||
_Mat.Meooe(in,tmp);
|
||||
_Mat.MooeeInv(tmp,out);
|
||||
_Mat.Meooe(out,tmp);
|
||||
_Mat.Mooee(in,out);
|
||||
axpy(out,-1.0,tmp,out);
|
||||
}
|
||||
virtual void MpcDag (const Field &in, Field &out){
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MeooeDag(in,tmp);
|
||||
_Mat.MooeeInvDag(tmp,out);
|
||||
_Mat.MeooeDag(out,tmp);
|
||||
_Mat.MooeeDag(in,out);
|
||||
axpy(out,-1.0,tmp,out);
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
class SchurDiagOneOperator : public SchurOperatorBase<Field> {
|
||||
protected:
|
||||
Matrix &_Mat;
|
||||
public:
|
||||
SchurDiagOneOperator (Matrix &Mat): _Mat(Mat){};
|
||||
|
||||
virtual void Mpc (const Field &in, Field &out) {
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.Meooe(in,out);
|
||||
_Mat.MooeeInv(out,tmp);
|
||||
_Mat.Meooe(tmp,out);
|
||||
_Mat.MooeeInv(out,tmp);
|
||||
axpy(out,-1.0,tmp,in);
|
||||
}
|
||||
virtual void MpcDag (const Field &in, Field &out){
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MooeeInvDag(in,out);
|
||||
_Mat.MeooeDag(out,tmp);
|
||||
_Mat.MooeeInvDag(tmp,out);
|
||||
_Mat.MeooeDag(out,tmp);
|
||||
axpy(out,-1.0,tmp,in);
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
class SchurDiagTwoOperator : public SchurOperatorBase<Field> {
|
||||
protected:
|
||||
Matrix &_Mat;
|
||||
public:
|
||||
SchurDiagTwoOperator (Matrix &Mat): _Mat(Mat){};
|
||||
|
||||
virtual void Mpc (const Field &in, Field &out) {
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MooeeInv(in,out);
|
||||
_Mat.Meooe(out,tmp);
|
||||
_Mat.MooeeInv(tmp,out);
|
||||
_Mat.Meooe(out,tmp);
|
||||
|
||||
axpy(out,-1.0,tmp,in);
|
||||
}
|
||||
virtual void MpcDag (const Field &in, Field &out){
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MeooeDag(in,out);
|
||||
_Mat.MooeeInvDag(out,tmp);
|
||||
_Mat.MeooeDag(tmp,out);
|
||||
_Mat.MooeeInvDag(out,tmp);
|
||||
|
||||
axpy(out,-1.0,tmp,in);
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field>
|
||||
class NonHermitianSchurOperatorBase : public LinearOperatorBase<Field>
|
||||
{
|
||||
public:
|
||||
virtual void Mpc (const Field& in, Field& out) = 0;
|
||||
virtual void MpcDag (const Field& in, Field& out) = 0;
|
||||
virtual void MpcDagMpc(const Field& in, Field& out) {
|
||||
Field tmp(in.Grid());
|
||||
tmp.Checkerboard() = in.Checkerboard();
|
||||
Mpc(in,tmp);
|
||||
MpcDag(tmp,out);
|
||||
}
|
||||
virtual void HermOpAndNorm(const Field& in, Field& out, RealD& n1, RealD& n2) {
|
||||
assert(0);
|
||||
}
|
||||
virtual void HermOp(const Field& in, Field& out) {
|
||||
assert(0);
|
||||
}
|
||||
void Op(const Field& in, Field& out) {
|
||||
Mpc(in, out);
|
||||
}
|
||||
void AdjOp(const Field& in, Field& out) {
|
||||
MpcDag(in, out);
|
||||
}
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag(const Field& in, Field& out) {
|
||||
assert(0); // must coarsen the unpreconditioned system
|
||||
}
|
||||
void OpDir(const Field& in, Field& out, int dir, int disp) {
|
||||
assert(0);
|
||||
}
|
||||
void OpDirAll(const Field& in, std::vector<Field>& out){
|
||||
assert(0);
|
||||
};
|
||||
};
|
||||
|
||||
template<class Matrix, class Field>
|
||||
class NonHermitianSchurDiagMooeeOperator : public NonHermitianSchurOperatorBase<Field>
|
||||
{
|
||||
public:
|
||||
Matrix& _Mat;
|
||||
NonHermitianSchurDiagMooeeOperator(Matrix& Mat): _Mat(Mat){};
|
||||
virtual void Mpc(const Field& in, Field& out) {
|
||||
Field tmp(in.Grid());
|
||||
tmp.Checkerboard() = !in.Checkerboard();
|
||||
|
||||
_Mat.Meooe(in, tmp);
|
||||
_Mat.MooeeInv(tmp, out);
|
||||
_Mat.Meooe(out, tmp);
|
||||
|
||||
_Mat.Mooee(in, out);
|
||||
|
||||
axpy(out, -1.0, tmp, out);
|
||||
}
|
||||
virtual void MpcDag(const Field& in, Field& out) {
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MeooeDag(in, tmp);
|
||||
_Mat.MooeeInvDag(tmp, out);
|
||||
_Mat.MeooeDag(out, tmp);
|
||||
|
||||
_Mat.MooeeDag(in, out);
|
||||
|
||||
axpy(out, -1.0, tmp, out);
|
||||
}
|
||||
};
|
||||
|
||||
template<class Matrix,class Field>
|
||||
class NonHermitianSchurDiagOneOperator : public NonHermitianSchurOperatorBase<Field>
|
||||
{
|
||||
protected:
|
||||
Matrix &_Mat;
|
||||
|
||||
public:
|
||||
NonHermitianSchurDiagOneOperator (Matrix& Mat): _Mat(Mat){};
|
||||
virtual void Mpc(const Field& in, Field& out) {
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.Meooe(in, out);
|
||||
_Mat.MooeeInv(out, tmp);
|
||||
_Mat.Meooe(tmp, out);
|
||||
_Mat.MooeeInv(out, tmp);
|
||||
|
||||
axpy(out, -1.0, tmp, in);
|
||||
}
|
||||
virtual void MpcDag(const Field& in, Field& out) {
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MooeeInvDag(in, out);
|
||||
_Mat.MeooeDag(out, tmp);
|
||||
_Mat.MooeeInvDag(tmp, out);
|
||||
_Mat.MeooeDag(out, tmp);
|
||||
|
||||
axpy(out, -1.0, tmp, in);
|
||||
}
|
||||
};
|
||||
|
||||
template<class Matrix, class Field>
|
||||
class NonHermitianSchurDiagTwoOperator : public NonHermitianSchurOperatorBase<Field>
|
||||
{
|
||||
protected:
|
||||
Matrix& _Mat;
|
||||
|
||||
public:
|
||||
NonHermitianSchurDiagTwoOperator(Matrix& Mat): _Mat(Mat){};
|
||||
|
||||
virtual void Mpc(const Field& in, Field& out) {
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MooeeInv(in, out);
|
||||
_Mat.Meooe(out, tmp);
|
||||
_Mat.MooeeInv(tmp, out);
|
||||
_Mat.Meooe(out, tmp);
|
||||
|
||||
axpy(out, -1.0, tmp, in);
|
||||
}
|
||||
virtual void MpcDag(const Field& in, Field& out) {
|
||||
Field tmp(in.Grid());
|
||||
|
||||
_Mat.MeooeDag(in, out);
|
||||
_Mat.MooeeInvDag(out, tmp);
|
||||
_Mat.MeooeDag(tmp, out);
|
||||
_Mat.MooeeInvDag(out, tmp);
|
||||
|
||||
axpy(out, -1.0, tmp, in);
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Left handed Moo^-1 ; (Moo - Moe Mee^-1 Meo) psi = eta --> ( 1 - Moo^-1 Moe Mee^-1 Meo ) psi = Moo^-1 eta
|
||||
// Right handed Moo^-1 ; (Moo - Moe Mee^-1 Meo) Moo^-1 Moo psi = eta --> ( 1 - Moe Mee^-1 Meo Moo^-1) phi=eta ; psi = Moo^-1 phi
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Matrix,class Field> using SchurDiagOneRH = SchurDiagTwoOperator<Matrix,Field> ;
|
||||
template<class Matrix,class Field> using SchurDiagOneLH = SchurDiagOneOperator<Matrix,Field> ;
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Staggered use
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Matrix,class Field>
|
||||
class SchurStaggeredOperator : public SchurOperatorBase<Field> {
|
||||
protected:
|
||||
Matrix &_Mat;
|
||||
Field tmp;
|
||||
RealD mass;
|
||||
public:
|
||||
SchurStaggeredOperator (Matrix &Mat): _Mat(Mat), tmp(_Mat.RedBlackGrid())
|
||||
{
|
||||
assert( _Mat.isTrivialEE() );
|
||||
mass = _Mat.Mass();
|
||||
}
|
||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
Mpc(in,out);
|
||||
ComplexD dot= innerProduct(in,out);
|
||||
n1 = real(dot);
|
||||
n2 =0.0;
|
||||
}
|
||||
virtual void HermOp(const Field &in, Field &out){
|
||||
Mpc(in,out);
|
||||
// _Mat.Meooe(in,out);
|
||||
// _Mat.Meooe(out,tmp);
|
||||
// axpby(out,-1.0,mass*mass,tmp,in);
|
||||
}
|
||||
virtual void Mpc (const Field &in, Field &out)
|
||||
{
|
||||
Field tmp(in.Grid());
|
||||
Field tmp2(in.Grid());
|
||||
|
||||
// _Mat.Mooee(in,out);
|
||||
// _Mat.Mooee(out,tmp);
|
||||
|
||||
_Mat.Meooe(in,out);
|
||||
_Mat.Meooe(out,tmp);
|
||||
axpby(out,-1.0,mass*mass,tmp,in);
|
||||
}
|
||||
virtual void MpcDag (const Field &in, Field &out){
|
||||
Mpc(in,out);
|
||||
}
|
||||
virtual void MpcDagMpc(const Field &in, Field &out) {
|
||||
assert(0);// Never need with staggered
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Base classes for functions of operators
|
||||
/////////////////////////////////////////////////////////////
|
||||
template<class Field> class OperatorFunction {
|
||||
public:
|
||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) = 0;
|
||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const std::vector<Field> &in,std::vector<Field> &out) {
|
||||
assert(in.size()==out.size());
|
||||
for(int k=0;k<in.size();k++){
|
||||
(*this)(Linop,in[k],out[k]);
|
||||
}
|
||||
};
|
||||
virtual ~OperatorFunction(){};
|
||||
};
|
||||
|
||||
template<class Field> class LinearFunction {
|
||||
public:
|
||||
virtual void operator() (const Field &in, Field &out) = 0;
|
||||
|
||||
virtual void operator() (const std::vector<Field> &in, std::vector<Field> &out)
|
||||
{
|
||||
assert(in.size() == out.size());
|
||||
|
||||
for (unsigned int i = 0; i < in.size(); ++i)
|
||||
{
|
||||
(*this)(in[i], out[i]);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field> class IdentityLinearFunction : public LinearFunction<Field> {
|
||||
public:
|
||||
void operator() (const Field &in, Field &out){
|
||||
out = in;
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Base classes for Multishift solvers for operators
|
||||
/////////////////////////////////////////////////////////////
|
||||
template<class Field> class OperatorMultiFunction {
|
||||
public:
|
||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const Field &in, std::vector<Field> &out) = 0;
|
||||
};
|
||||
|
||||
// FIXME : To think about
|
||||
|
||||
// Chroma functionality list defining LinearOperator
|
||||
/*
|
||||
virtual void operator() (T& chi, const T& psi, enum PlusMinus isign) const = 0;
|
||||
virtual void operator() (T& chi, const T& psi, enum PlusMinus isign, Real epsilon) const
|
||||
virtual const Subset& subset() const = 0;
|
||||
virtual unsigned long nFlops() const { return 0; }
|
||||
virtual void deriv(P& ds_u, const T& chi, const T& psi, enum PlusMinus isign) const
|
||||
class UnprecLinearOperator : public DiffLinearOperator<T,P,Q>
|
||||
const Subset& subset() const {return all;}
|
||||
};
|
||||
*/
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hermitian operator Linear function and operator function
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field>
|
||||
class HermOpOperatorFunction : public OperatorFunction<Field> {
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
||||
Linop.HermOp(in,out);
|
||||
};
|
||||
};
|
||||
|
||||
template<typename Field>
|
||||
class PlainHermOp : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
LinearOperatorBase<Field> &_Linop;
|
||||
|
||||
PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop)
|
||||
{}
|
||||
|
||||
void operator()(const Field& in, Field& out) {
|
||||
_Linop.HermOp(in,out);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Field>
|
||||
class FunctionHermOp : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
OperatorFunction<Field> & _poly;
|
||||
LinearOperatorBase<Field> &_Linop;
|
||||
|
||||
FunctionHermOp(OperatorFunction<Field> & poly,LinearOperatorBase<Field>& linop)
|
||||
: _poly(poly), _Linop(linop) {};
|
||||
|
||||
void operator()(const Field& in, Field& out) {
|
||||
_poly(_Linop,in,out);
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field>
|
||||
class Polynomial : public OperatorFunction<Field> {
|
||||
private:
|
||||
std::vector<RealD> Coeffs;
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
Polynomial(std::vector<RealD> &_Coeffs) : Coeffs(_Coeffs) { };
|
||||
|
||||
// Implement the required interface
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
||||
|
||||
Field AtoN(in.Grid());
|
||||
Field Mtmp(in.Grid());
|
||||
AtoN = in;
|
||||
out = AtoN*Coeffs[0];
|
||||
for(int n=1;n<Coeffs.size();n++){
|
||||
Mtmp = AtoN;
|
||||
Linop.HermOp(Mtmp,AtoN);
|
||||
out=out+AtoN*Coeffs[n];
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,52 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/Preconditioner.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_PRECONDITIONER_H
|
||||
#define GRID_PRECONDITIONER_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class Field> using Preconditioner = LinearFunction<Field> ;
|
||||
|
||||
/*
|
||||
template<class Field> class Preconditioner : public LinearFunction<Field> {
|
||||
using LinearFunction<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field & psi)=0;
|
||||
};
|
||||
*/
|
||||
|
||||
template<class Field> class TrivialPrecon : public Preconditioner<Field> {
|
||||
public:
|
||||
using Preconditioner<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field & psi){
|
||||
psi = src;
|
||||
}
|
||||
TrivialPrecon(void){};
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,81 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/SparseMatrix.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_ALGORITHM_SPARSE_MATRIX_H
|
||||
#define GRID_ALGORITHM_SPARSE_MATRIX_H
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Interface defining what I expect of a general sparse matrix, such as a Fermion action
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class SparseMatrixBase {
|
||||
public:
|
||||
virtual GridBase *Grid(void) =0;
|
||||
// Full checkerboar operations
|
||||
virtual void M (const Field &in, Field &out)=0;
|
||||
virtual void Mdag (const Field &in, Field &out)=0;
|
||||
virtual void MdagM(const Field &in, Field &out) {
|
||||
Field tmp (in.Grid());
|
||||
M(in,tmp);
|
||||
Mdag(tmp,out);
|
||||
}
|
||||
virtual void Mdiag (const Field &in, Field &out)=0;
|
||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp)=0;
|
||||
virtual void MdirAll (const Field &in, std::vector<Field> &out)=0;
|
||||
virtual ~SparseMatrixBase() {};
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Interface augmented by a red black sparse matrix, such as a Fermion action
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class CheckerBoardedSparseMatrixBase : public SparseMatrixBase<Field> {
|
||||
public:
|
||||
virtual GridBase *RedBlackGrid(void)=0;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Query the even even properties to make algorithmic decisions
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
virtual RealD Mass(void) { return 0.0; };
|
||||
virtual int ConstEE(void) { return 1; }; // Disable assumptions unless overridden
|
||||
virtual int isTrivialEE(void) { return 0; }; // by a derived class that knows better
|
||||
|
||||
// half checkerboard operaions
|
||||
virtual void Meooe (const Field &in, Field &out)=0;
|
||||
virtual void Mooee (const Field &in, Field &out)=0;
|
||||
virtual void MooeeInv (const Field &in, Field &out)=0;
|
||||
|
||||
virtual void MeooeDag (const Field &in, Field &out)=0;
|
||||
virtual void MooeeDag (const Field &in, Field &out)=0;
|
||||
virtual void MooeeInvDag (const Field &in, Field &out)=0;
|
||||
virtual ~CheckerBoardedSparseMatrixBase() {};
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,395 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/approx/Chebyshev.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christoph Lehner <clehner@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CHEBYSHEV_H
|
||||
#define GRID_CHEBYSHEV_H
|
||||
|
||||
#include <Grid/algorithms/LinearOperator.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
struct ChebyParams : Serializable {
|
||||
GRID_SERIALIZABLE_CLASS_MEMBERS(ChebyParams,
|
||||
RealD, alpha,
|
||||
RealD, beta,
|
||||
int, Npoly);
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Generic Chebyshev approximations
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field>
|
||||
class Chebyshev : public OperatorFunction<Field> {
|
||||
private:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
std::vector<RealD> Coeffs;
|
||||
int order;
|
||||
RealD hi;
|
||||
RealD lo;
|
||||
|
||||
public:
|
||||
void csv(std::ostream &out){
|
||||
RealD diff = hi-lo;
|
||||
RealD delta = diff*1.0e-9;
|
||||
for (RealD x=lo; x<hi; x+=delta) {
|
||||
delta*=1.1;
|
||||
RealD f = approx(x);
|
||||
out<< x<<" "<<f<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Convenience for plotting the approximation
|
||||
void PlotApprox(std::ostream &out) {
|
||||
out<<"Polynomial approx ["<<lo<<","<<hi<<"]"<<std::endl;
|
||||
for(RealD x=lo;x<hi;x+=(hi-lo)/50.0){
|
||||
out <<x<<"\t"<<approx(x)<<std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
Chebyshev(){};
|
||||
Chebyshev(ChebyParams p){ Init(p.alpha,p.beta,p.Npoly);};
|
||||
Chebyshev(RealD _lo,RealD _hi,int _order, RealD (* func)(RealD) ) {Init(_lo,_hi,_order,func);};
|
||||
Chebyshev(RealD _lo,RealD _hi,int _order) {Init(_lo,_hi,_order);};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// c.f. numerical recipes "chebft"/"chebev". This is sec 5.8 "Chebyshev approximation".
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// CJ: the one we need for Lanczos
|
||||
void Init(RealD _lo,RealD _hi,int _order)
|
||||
{
|
||||
lo=_lo;
|
||||
hi=_hi;
|
||||
order=_order;
|
||||
|
||||
if(order < 2) exit(-1);
|
||||
Coeffs.resize(order);
|
||||
Coeffs.assign(0.,order);
|
||||
Coeffs[order-1] = 1.;
|
||||
};
|
||||
|
||||
// PB - more efficient low pass drops high modes above the low as 1/x uses all Chebyshev's.
|
||||
// Similar kick effect below the threshold as Lanczos filter approach
|
||||
void InitLowPass(RealD _lo,RealD _hi,int _order)
|
||||
{
|
||||
lo=_lo;
|
||||
hi=_hi;
|
||||
order=_order;
|
||||
|
||||
if(order < 2) exit(-1);
|
||||
Coeffs.resize(order);
|
||||
for(int j=0;j<order;j++){
|
||||
RealD k=(order-1.0);
|
||||
RealD s=std::cos( j*M_PI*(k+0.5)/order );
|
||||
Coeffs[j] = s * 2.0/order;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
void Init(RealD _lo,RealD _hi,int _order, RealD (* func)(RealD))
|
||||
{
|
||||
lo=_lo;
|
||||
hi=_hi;
|
||||
order=_order;
|
||||
|
||||
if(order < 2) exit(-1);
|
||||
Coeffs.resize(order);
|
||||
for(int j=0;j<order;j++){
|
||||
RealD s=0;
|
||||
for(int k=0;k<order;k++){
|
||||
RealD y=std::cos(M_PI*(k+0.5)/order);
|
||||
RealD x=0.5*(y*(hi-lo)+(hi+lo));
|
||||
RealD f=func(x);
|
||||
s=s+f*std::cos( j*M_PI*(k+0.5)/order );
|
||||
}
|
||||
Coeffs[j] = s * 2.0/order;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
void JacksonSmooth(void){
|
||||
RealD M=order;
|
||||
RealD alpha = M_PI/(M+2);
|
||||
RealD lmax = std::cos(alpha);
|
||||
RealD sumUsq =0;
|
||||
std::vector<RealD> U(M);
|
||||
std::vector<RealD> a(M);
|
||||
std::vector<RealD> g(M);
|
||||
for(int n=0;n<=M;n++){
|
||||
U[n] = std::sin((n+1)*std::acos(lmax))/std::sin(std::acos(lmax));
|
||||
sumUsq += U[n]*U[n];
|
||||
}
|
||||
sumUsq = std::sqrt(sumUsq);
|
||||
|
||||
for(int i=1;i<=M;i++){
|
||||
a[i] = U[i]/sumUsq;
|
||||
}
|
||||
g[0] = 1.0;
|
||||
for(int m=1;m<=M;m++){
|
||||
g[m] = 0;
|
||||
for(int i=0;i<=M-m;i++){
|
||||
g[m]+= a[i]*a[m+i];
|
||||
}
|
||||
}
|
||||
for(int m=1;m<=M;m++){
|
||||
Coeffs[m]*=g[m];
|
||||
}
|
||||
}
|
||||
RealD approx(RealD x) // Convenience for plotting the approximation
|
||||
{
|
||||
RealD Tn;
|
||||
RealD Tnm;
|
||||
RealD Tnp;
|
||||
|
||||
RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
||||
|
||||
RealD T0=1;
|
||||
RealD T1=y;
|
||||
|
||||
RealD sum;
|
||||
sum = 0.5*Coeffs[0]*T0;
|
||||
sum+= Coeffs[1]*T1;
|
||||
|
||||
Tn =T1;
|
||||
Tnm=T0;
|
||||
for(int i=2;i<order;i++){
|
||||
Tnp=2*y*Tn-Tnm;
|
||||
Tnm=Tn;
|
||||
Tn =Tnp;
|
||||
sum+= Tn*Coeffs[i];
|
||||
}
|
||||
return sum;
|
||||
};
|
||||
|
||||
RealD approxD(RealD x)
|
||||
{
|
||||
RealD Un;
|
||||
RealD Unm;
|
||||
RealD Unp;
|
||||
|
||||
RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
||||
|
||||
RealD U0=1;
|
||||
RealD U1=2*y;
|
||||
|
||||
RealD sum;
|
||||
sum = Coeffs[1]*U0;
|
||||
sum+= Coeffs[2]*U1*2.0;
|
||||
|
||||
Un =U1;
|
||||
Unm=U0;
|
||||
for(int i=2;i<order-1;i++){
|
||||
Unp=2*y*Un-Unm;
|
||||
Unm=Un;
|
||||
Un =Unp;
|
||||
sum+= Un*Coeffs[i+1]*(i+1.0);
|
||||
}
|
||||
return sum/(0.5*(hi-lo));
|
||||
};
|
||||
|
||||
RealD approxInv(RealD z, RealD x0, int maxiter, RealD resid) {
|
||||
RealD x = x0;
|
||||
RealD eps;
|
||||
|
||||
int i;
|
||||
for (i=0;i<maxiter;i++) {
|
||||
eps = approx(x) - z;
|
||||
if (fabs(eps / z) < resid)
|
||||
return x;
|
||||
x = x - eps / approxD(x);
|
||||
}
|
||||
|
||||
return std::numeric_limits<double>::quiet_NaN();
|
||||
}
|
||||
|
||||
// Implement the required interface
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
||||
|
||||
GridBase *grid=in.Grid();
|
||||
|
||||
int vol=grid->gSites();
|
||||
typedef typename Field::vector_type vector_type;
|
||||
|
||||
Field T0(grid); T0 = in;
|
||||
Field T1(grid);
|
||||
Field T2(grid);
|
||||
Field y(grid);
|
||||
|
||||
Field *Tnm = &T0;
|
||||
Field *Tn = &T1;
|
||||
Field *Tnp = &T2;
|
||||
|
||||
// Tn=T1 = (xscale M + mscale)in
|
||||
RealD xscale = 2.0/(hi-lo);
|
||||
RealD mscale = -(hi+lo)/(hi-lo);
|
||||
Linop.HermOp(T0,y);
|
||||
axpby(T1,xscale,mscale,y,in);
|
||||
|
||||
// sum = .5 c[0] T0 + c[1] T1
|
||||
// out = ()*T0 + Coeffs[1]*T1;
|
||||
axpby(out,0.5*Coeffs[0],Coeffs[1],T0,T1);
|
||||
for(int n=2;n<order;n++){
|
||||
|
||||
Linop.HermOp(*Tn,y);
|
||||
axpby(y,xscale,mscale,y,(*Tn));
|
||||
axpby(*Tnp,2.0,-1.0,y,(*Tnm));
|
||||
if ( Coeffs[n] != 0.0) {
|
||||
axpy(out,Coeffs[n],*Tnp,out);
|
||||
}
|
||||
|
||||
// Cycle pointers to avoid copies
|
||||
Field *swizzle = Tnm;
|
||||
Tnm =Tn;
|
||||
Tn =Tnp;
|
||||
Tnp =swizzle;
|
||||
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template<class Field>
|
||||
class ChebyshevLanczos : public Chebyshev<Field> {
|
||||
private:
|
||||
std::vector<RealD> Coeffs;
|
||||
int order;
|
||||
RealD alpha;
|
||||
RealD beta;
|
||||
RealD mu;
|
||||
|
||||
public:
|
||||
ChebyshevLanczos(RealD _alpha,RealD _beta,RealD _mu,int _order) :
|
||||
alpha(_alpha),
|
||||
beta(_beta),
|
||||
mu(_mu)
|
||||
{
|
||||
order=_order;
|
||||
Coeffs.resize(order);
|
||||
for(int i=0;i<_order;i++){
|
||||
Coeffs[i] = 0.0;
|
||||
}
|
||||
Coeffs[order-1]=1.0;
|
||||
};
|
||||
|
||||
void csv(std::ostream &out){
|
||||
for (RealD x=-1.2*alpha; x<1.2*alpha; x+=(2.0*alpha)/10000) {
|
||||
RealD f = approx(x);
|
||||
out<< x<<" "<<f<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
RealD approx(RealD xx) // Convenience for plotting the approximation
|
||||
{
|
||||
RealD Tn;
|
||||
RealD Tnm;
|
||||
RealD Tnp;
|
||||
Real aa = alpha * alpha;
|
||||
Real bb = beta * beta;
|
||||
|
||||
RealD x = ( 2.0 * (xx-mu)*(xx-mu) - (aa+bb) ) / (aa-bb);
|
||||
|
||||
RealD y= x;
|
||||
|
||||
RealD T0=1;
|
||||
RealD T1=y;
|
||||
|
||||
RealD sum;
|
||||
sum = 0.5*Coeffs[0]*T0;
|
||||
sum+= Coeffs[1]*T1;
|
||||
|
||||
Tn =T1;
|
||||
Tnm=T0;
|
||||
for(int i=2;i<order;i++){
|
||||
Tnp=2*y*Tn-Tnm;
|
||||
Tnm=Tn;
|
||||
Tn =Tnp;
|
||||
sum+= Tn*Coeffs[i];
|
||||
}
|
||||
return sum;
|
||||
};
|
||||
|
||||
// shift_Multiply in Rudy's code
|
||||
void AminusMuSq(LinearOperatorBase<Field> &Linop, const Field &in, Field &out)
|
||||
{
|
||||
GridBase *grid=in.Grid();
|
||||
Field tmp(grid);
|
||||
|
||||
RealD aa= alpha*alpha;
|
||||
RealD bb= beta * beta;
|
||||
|
||||
Linop.HermOp(in,out);
|
||||
out = out - mu*in;
|
||||
|
||||
Linop.HermOp(out,tmp);
|
||||
tmp = tmp - mu * out;
|
||||
|
||||
out = (2.0/ (aa-bb) ) * tmp - ((aa+bb)/(aa-bb))*in;
|
||||
};
|
||||
// Implement the required interface
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
||||
|
||||
GridBase *grid=in.Grid();
|
||||
|
||||
int vol=grid->gSites();
|
||||
|
||||
Field T0(grid); T0 = in;
|
||||
Field T1(grid);
|
||||
Field T2(grid);
|
||||
Field y(grid);
|
||||
|
||||
Field *Tnm = &T0;
|
||||
Field *Tn = &T1;
|
||||
Field *Tnp = &T2;
|
||||
|
||||
// Tn=T1 = (xscale M )*in
|
||||
AminusMuSq(Linop,T0,T1);
|
||||
|
||||
// sum = .5 c[0] T0 + c[1] T1
|
||||
out = (0.5*Coeffs[0])*T0 + Coeffs[1]*T1;
|
||||
for(int n=2;n<order;n++){
|
||||
|
||||
AminusMuSq(Linop,*Tn,y);
|
||||
|
||||
*Tnp=2.0*y-(*Tnm);
|
||||
|
||||
out=out+Coeffs[n]* (*Tnp);
|
||||
|
||||
// Cycle pointers to avoid copies
|
||||
Field *swizzle = Tnm;
|
||||
Tnm =Tn;
|
||||
Tn =Tnp;
|
||||
Tnp =swizzle;
|
||||
|
||||
}
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,152 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/approx/Forecast.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: David Murphy <dmurphy@phys.columbia.edu>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#ifndef INCLUDED_FORECAST_H
|
||||
#define INCLUDED_FORECAST_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
// Abstract base class.
|
||||
// Takes a matrix (Mat), a source (phi), and a vector of Fields (chi)
|
||||
// and returns a forecasted solution to the system D*psi = phi (psi).
|
||||
template<class Matrix, class Field>
|
||||
class Forecast
|
||||
{
|
||||
public:
|
||||
virtual Field operator()(Matrix &Mat, const Field& phi, const std::vector<Field>& chi) = 0;
|
||||
};
|
||||
|
||||
// Implementation of Brower et al.'s chronological inverter (arXiv:hep-lat/9509012),
|
||||
// used to forecast solutions across poles of the EOFA heatbath.
|
||||
//
|
||||
// Modified from CPS (cps_pp/src/util/dirac_op/d_op_base/comsrc/minresext.C)
|
||||
template<class Matrix, class Field>
|
||||
class ChronoForecast : public Forecast<Matrix,Field>
|
||||
{
|
||||
public:
|
||||
Field operator()(Matrix &Mat, const Field& phi, const std::vector<Field>& prev_solns)
|
||||
{
|
||||
int degree = prev_solns.size();
|
||||
Field chi(phi); // forecasted solution
|
||||
|
||||
// Trivial cases
|
||||
if(degree == 0){ chi = Zero(); return chi; }
|
||||
else if(degree == 1){ return prev_solns[0]; }
|
||||
|
||||
// RealD dot;
|
||||
ComplexD xp;
|
||||
Field r(phi); // residual
|
||||
Field Mv(phi);
|
||||
std::vector<Field> v(prev_solns); // orthonormalized previous solutions
|
||||
std::vector<Field> MdagMv(degree,phi);
|
||||
|
||||
// Array to hold the matrix elements
|
||||
std::vector<std::vector<ComplexD>> G(degree, std::vector<ComplexD>(degree));
|
||||
|
||||
// Solution and source vectors
|
||||
std::vector<ComplexD> a(degree);
|
||||
std::vector<ComplexD> b(degree);
|
||||
|
||||
// Orthonormalize the vector basis
|
||||
for(int i=0; i<degree; i++){
|
||||
v[i] *= 1.0/std::sqrt(norm2(v[i]));
|
||||
for(int j=i+1; j<degree; j++){ v[j] -= innerProduct(v[i],v[j]) * v[i]; }
|
||||
}
|
||||
|
||||
// Perform sparse matrix multiplication and construct rhs
|
||||
for(int i=0; i<degree; i++){
|
||||
b[i] = innerProduct(v[i],phi);
|
||||
Mat.M(v[i],Mv);
|
||||
Mat.Mdag(Mv,MdagMv[i]);
|
||||
G[i][i] = innerProduct(v[i],MdagMv[i]);
|
||||
}
|
||||
|
||||
// Construct the matrix
|
||||
for(int j=0; j<degree; j++){
|
||||
for(int k=j+1; k<degree; k++){
|
||||
G[j][k] = innerProduct(v[j],MdagMv[k]);
|
||||
G[k][j] = conjugate(G[j][k]);
|
||||
}}
|
||||
|
||||
// Gauss-Jordan elimination with partial pivoting
|
||||
for(int i=0; i<degree; i++){
|
||||
|
||||
// Perform partial pivoting
|
||||
int k = i;
|
||||
for(int j=i+1; j<degree; j++){ if(abs(G[j][j]) > abs(G[k][k])){ k = j; } }
|
||||
if(k != i){
|
||||
xp = b[k];
|
||||
b[k] = b[i];
|
||||
b[i] = xp;
|
||||
for(int j=0; j<degree; j++){
|
||||
xp = G[k][j];
|
||||
G[k][j] = G[i][j];
|
||||
G[i][j] = xp;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert matrix to upper triangular form
|
||||
for(int j=i+1; j<degree; j++){
|
||||
xp = G[j][i]/G[i][i];
|
||||
b[j] -= xp * b[i];
|
||||
for(int k=0; k<degree; k++){ G[j][k] -= xp*G[i][k]; }
|
||||
}
|
||||
}
|
||||
|
||||
// Use Gaussian elimination to solve equations and calculate initial guess
|
||||
chi = Zero();
|
||||
r = phi;
|
||||
for(int i=degree-1; i>=0; i--){
|
||||
a[i] = 0.0;
|
||||
for(int j=i+1; j<degree; j++){ a[i] += G[i][j] * a[j]; }
|
||||
a[i] = (b[i]-a[i])/G[i][i];
|
||||
chi += a[i]*v[i];
|
||||
r -= a[i]*MdagMv[i];
|
||||
}
|
||||
|
||||
RealD true_r(0.0);
|
||||
ComplexD tmp;
|
||||
for(int i=0; i<degree; i++){
|
||||
tmp = -b[i];
|
||||
for(int j=0; j<degree; j++){ tmp += G[i][j]*a[j]; }
|
||||
tmp = conjugate(tmp)*tmp;
|
||||
true_r += std::sqrt(tmp.real());
|
||||
}
|
||||
|
||||
RealD error = std::sqrt(norm2(r)/norm2(phi));
|
||||
std::cout << GridLogMessage << "ChronoForecast: |res|/|src| = " << error << std::endl;
|
||||
|
||||
return chi;
|
||||
};
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,129 +0,0 @@
|
||||
#ifndef GRID_JACOBIPOLYNOMIAL_H
|
||||
#define GRID_JACOBIPOLYNOMIAL_H
|
||||
|
||||
#include <Grid/algorithms/LinearOperator.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class Field>
|
||||
class JacobiPolynomial : public OperatorFunction<Field> {
|
||||
private:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
int order;
|
||||
RealD hi;
|
||||
RealD lo;
|
||||
RealD alpha;
|
||||
RealD beta;
|
||||
|
||||
public:
|
||||
void csv(std::ostream &out){
|
||||
csv(out,lo,hi);
|
||||
}
|
||||
void csv(std::ostream &out,RealD llo,RealD hhi){
|
||||
RealD diff = hhi-llo;
|
||||
RealD delta = diff*1.0e-5;
|
||||
for (RealD x=llo-delta; x<=hhi; x+=delta) {
|
||||
RealD f = approx(x);
|
||||
out<< x<<" "<<f <<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
JacobiPolynomial(){};
|
||||
JacobiPolynomial(RealD _lo,RealD _hi,int _order,RealD _alpha, RealD _beta)
|
||||
{
|
||||
lo=_lo;
|
||||
hi=_hi;
|
||||
alpha=_alpha;
|
||||
beta=_beta;
|
||||
order=_order;
|
||||
};
|
||||
|
||||
RealD approx(RealD x) // Convenience for plotting the approximation
|
||||
{
|
||||
RealD Tn;
|
||||
RealD Tnm;
|
||||
RealD Tnp;
|
||||
|
||||
RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
||||
|
||||
RealD T0=1.0;
|
||||
RealD T1=(alpha-beta)*0.5+(alpha+beta+2.0)*0.5*y;
|
||||
|
||||
Tn =T1;
|
||||
Tnm=T0;
|
||||
for(int n=2;n<=order;n++){
|
||||
RealD cnp = 2.0*n*(n+alpha+beta)*(2.0*n-2.0+alpha+beta);
|
||||
RealD cny = (2.0*n-2.0+alpha+beta)*(2.0*n-1.0+alpha+beta)*(2.0*n+alpha+beta);
|
||||
RealD cn1 = (2.0*n+alpha+beta-1.0)*(alpha*alpha-beta*beta);
|
||||
RealD cnm = - 2.0*(n+alpha-1.0)*(n+beta-1.0)*(2.0*n+alpha+beta);
|
||||
Tnp= ( cny * y *Tn + cn1 * Tn + cnm * Tnm )/ cnp;
|
||||
Tnm=Tn;
|
||||
Tn =Tnp;
|
||||
}
|
||||
return Tnp;
|
||||
};
|
||||
|
||||
// Implement the required interface
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
||||
GridBase *grid=in.Grid();
|
||||
|
||||
int vol=grid->gSites();
|
||||
|
||||
Field T0(grid);
|
||||
Field T1(grid);
|
||||
Field T2(grid);
|
||||
Field y(grid);
|
||||
|
||||
|
||||
Field *Tnm = &T0;
|
||||
Field *Tn = &T1;
|
||||
Field *Tnp = &T2;
|
||||
|
||||
// RealD T0=1.0;
|
||||
T0=in;
|
||||
|
||||
// RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
||||
// = x * 2/(hi-lo) - (hi+lo)/(hi-lo)
|
||||
Linop.HermOp(T0,y);
|
||||
RealD xscale = 2.0/(hi-lo);
|
||||
RealD mscale = -(hi+lo)/(hi-lo);
|
||||
Linop.HermOp(T0,y);
|
||||
y=y*xscale+in*mscale;
|
||||
|
||||
// RealD T1=(alpha-beta)*0.5+(alpha+beta+2.0)*0.5*y;
|
||||
RealD halfAmB = (alpha-beta)*0.5;
|
||||
RealD halfApBp2= (alpha+beta+2.0)*0.5;
|
||||
T1 = halfAmB * in + halfApBp2*y;
|
||||
|
||||
for(int n=2;n<=order;n++){
|
||||
|
||||
Linop.HermOp(*Tn,y);
|
||||
y=xscale*y+mscale*(*Tn);
|
||||
|
||||
RealD cnp = 2.0*n*(n+alpha+beta)*(2.0*n-2.0+alpha+beta);
|
||||
RealD cny = (2.0*n-2.0+alpha+beta)*(2.0*n-1.0+alpha+beta)*(2.0*n+alpha+beta);
|
||||
RealD cn1 = (2.0*n+alpha+beta-1.0)*(alpha*alpha-beta*beta);
|
||||
RealD cnm = - 2.0*(n+alpha-1.0)*(n+beta-1.0)*(2.0*n+alpha+beta);
|
||||
|
||||
// Tnp= ( cny * y *Tn + cn1 * Tn + cnm * Tnm )/ cnp;
|
||||
cny=cny/cnp;
|
||||
cn1=cn1/cnp;
|
||||
cn1=cn1/cnp;
|
||||
cnm=cnm/cnp;
|
||||
|
||||
*Tnp=cny*y + cn1 *(*Tn) + cnm * (*Tnm);
|
||||
|
||||
// Cycle pointers to avoid copies
|
||||
Field *swizzle = Tnm;
|
||||
Tnm =Tn;
|
||||
Tn =Tnp;
|
||||
Tnp =swizzle;
|
||||
}
|
||||
out=*Tnp;
|
||||
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,57 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/approx/MultiShiftFunction.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
double MultiShiftFunction::approx(double x)
|
||||
{
|
||||
double a = norm;
|
||||
for(int n=0;n<poles.size();n++){
|
||||
a = a + residues[n]/(x+poles[n]);
|
||||
}
|
||||
return a;
|
||||
}
|
||||
void MultiShiftFunction::gnuplot(std::ostream &out)
|
||||
{
|
||||
out<<"f(x) = "<<norm<<"";
|
||||
for(int n=0;n<poles.size();n++){
|
||||
out<<"+("<<residues[n]<<"/(x+"<<poles[n]<<"))";
|
||||
}
|
||||
out<<";"<<std::endl;
|
||||
}
|
||||
void MultiShiftFunction::csv(std::ostream &out)
|
||||
{
|
||||
for (double x=lo; x<hi; x*=1.05) {
|
||||
double f = approx(x);
|
||||
double r = sqrt(x);
|
||||
out<< x<<","<<r<<","<<f<<","<<r-f<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,67 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/approx/MultiShiftFunction.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef MULTI_SHIFT_FUNCTION
|
||||
#define MULTI_SHIFT_FUNCTION
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
class MultiShiftFunction {
|
||||
public:
|
||||
int order;
|
||||
std::vector<RealD> poles;
|
||||
std::vector<RealD> residues;
|
||||
std::vector<RealD> tolerances;
|
||||
RealD norm;
|
||||
RealD lo,hi;
|
||||
|
||||
MultiShiftFunction(int n,RealD _lo,RealD _hi): poles(n), residues(n), lo(_lo), hi(_hi) {;};
|
||||
RealD approx(RealD x);
|
||||
void csv(std::ostream &out);
|
||||
void gnuplot(std::ostream &out);
|
||||
|
||||
void Init(AlgRemez & remez,double tol,bool inverse)
|
||||
{
|
||||
order=remez.getDegree();
|
||||
tolerances.resize(remez.getDegree(),tol);
|
||||
poles.resize(remez.getDegree());
|
||||
residues.resize(remez.getDegree());
|
||||
remez.getBounds(lo,hi);
|
||||
if ( inverse ) remez.getIPFE (&residues[0],&poles[0],&norm);
|
||||
else remez.getPFE (&residues[0],&poles[0],&norm);
|
||||
}
|
||||
// Allow deferred initialisation
|
||||
MultiShiftFunction(void){};
|
||||
MultiShiftFunction(AlgRemez & remez,double tol,bool inverse)
|
||||
{
|
||||
Init(remez,tol,inverse);
|
||||
}
|
||||
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,80 +0,0 @@
|
||||
-----------------------------------------------------------------------------------
|
||||
|
||||
PAB. Took Mike Clark's AlgRemez from GitHub and (modified a little) include.
|
||||
This is open source and license and readme and comments are preserved consistent
|
||||
with the license. Mike, thankyou!
|
||||
-----------------------------------------------------------------------------------
|
||||
-----------------------------------------------------------------------------------
|
||||
AlgRemez
|
||||
|
||||
The archive downloadable here contains an implementation of the Remez
|
||||
algorithm which calculates optimal rational (and polynomial)
|
||||
approximations to the nth root over a given spectral range. The Remez
|
||||
algorithm, although in principle is extremely straightforward to
|
||||
program, is quite difficult to get completely correct, e.g., the Maple
|
||||
implementation of the algorithm does not always converge to the
|
||||
correct answer.
|
||||
|
||||
To use this algorithm you need to install GMP, the GNU Multiple
|
||||
Precision Library, and when configuring the install, you must include
|
||||
the --enable-mpfr option (see the GMP manual for more details). You
|
||||
also have to edit the Makefile for AlgRemez appropriately for your
|
||||
system, namely to point corrrectly to the location of the GMP library.
|
||||
|
||||
The simple main program included with this archive invokes the
|
||||
AlgRemez class to calculate an approximation given by command line
|
||||
arguments. It is invoked by the following
|
||||
|
||||
./test y z n d lambda_low lambda_high precision,
|
||||
|
||||
where the function to be approximated is f(x) = x^(y/z), with degree
|
||||
(n,d) over the spectral range [lambda_low, lambda_high], using
|
||||
precision digits of precision in the arithmetic. So an example would
|
||||
be
|
||||
|
||||
./test 1 2 5 5 0.0004 64 40
|
||||
|
||||
which corresponds to constructing a rational approximation to the
|
||||
square root function, with degree (5,5) over the range [0.0004,64]
|
||||
with 40 digits of precision used for the arithmetic. The parameters y
|
||||
and z must be positive, the approximation to f(x) = x^(-y/z) is simply
|
||||
the inverse of the approximation to f(x) = x^(y/z). After the
|
||||
approximation has been constructed, the roots and poles of the
|
||||
rational function are found, and then the partial fraction expansion
|
||||
of both the rational function and it's inverse are found, the results
|
||||
of which are output to a file called "approx.dat". In addition, the
|
||||
error function of the approximation is output to "error.dat", where it
|
||||
can be checked that the resultant approximation satisfies Chebychev's
|
||||
criterion, namely all error maxima are equal in magnitude, and
|
||||
adjacent maxima are oppostie in sign. There are some caveats here
|
||||
however, the optimal polynomial approximation has complex roots, and
|
||||
the root finding implemented here cannot (yet) handle complex roots.
|
||||
In addition, the partial fraction expansion of rational approximations
|
||||
is only found for the case n = d, i.e., the degree of numerator
|
||||
polynomial equals that of the denominator polynomial. The convention
|
||||
for the partial fraction expansion is that polar shifts are always
|
||||
written added to x, not subtracted.
|
||||
|
||||
To do list
|
||||
|
||||
1. Include an exponential dampening factor in the function to be
|
||||
approximated. This may sound trivial to implement, but for some
|
||||
parameters, the algorithm seems to breakdown. Also, the roots in the
|
||||
rational approximation sometimes become complex, which currently
|
||||
breaks the stupidly simple root finding code.
|
||||
|
||||
2. Make the algorithm faster - it's too slow when running on qcdoc.
|
||||
|
||||
3. Add complex root finding.
|
||||
|
||||
4. Add more options for error minimisation - currently the code
|
||||
minimises the relative error, should add options for absolute error,
|
||||
and other norms.
|
||||
|
||||
There will be a forthcoming publication concerning the results
|
||||
generated by this software, but in the meantime, if you use this
|
||||
software, please cite it as
|
||||
"M.A. Clark and A.D. Kennedy, https://github.com/mikeaclark/AlgRemez, 2005".
|
||||
|
||||
If you have any problems using the software, then please email scientist.mike@gmail.com.
|
||||
|
||||
@@ -1,759 +0,0 @@
|
||||
/*
|
||||
|
||||
Mike Clark - 25th May 2005
|
||||
|
||||
alg_remez.C
|
||||
|
||||
AlgRemez is an implementation of the Remez algorithm, which in this
|
||||
case is used for generating the optimal nth root rational
|
||||
approximation.
|
||||
|
||||
Note this class requires the gnu multiprecision (GNU MP) library.
|
||||
|
||||
*/
|
||||
|
||||
#include<math.h>
|
||||
#include<stdio.h>
|
||||
#include<stdlib.h>
|
||||
#include<string>
|
||||
#include<iostream>
|
||||
#include<iomanip>
|
||||
#include<cassert>
|
||||
|
||||
#include<Grid/algorithms/approx/Remez.h>
|
||||
|
||||
// Constructor
|
||||
AlgRemez::AlgRemez(double lower, double upper, long precision)
|
||||
{
|
||||
prec = precision;
|
||||
bigfloat::setDefaultPrecision(prec);
|
||||
|
||||
apstrt = lower;
|
||||
apend = upper;
|
||||
apwidt = apend - apstrt;
|
||||
|
||||
std::cout<<"Approximation bounds are ["<<apstrt<<","<<apend<<"]\n";
|
||||
std::cout<<"Precision of arithmetic is "<<precision<<std::endl;
|
||||
|
||||
alloc = 0;
|
||||
n = 0;
|
||||
d = 0;
|
||||
|
||||
foundRoots = 0;
|
||||
|
||||
// Only require the approximation spread to be less than 1 ulp
|
||||
tolerance = 1e-15;
|
||||
}
|
||||
|
||||
// Destructor
|
||||
AlgRemez::~AlgRemez()
|
||||
{
|
||||
if (alloc) {
|
||||
delete [] param;
|
||||
delete [] roots;
|
||||
delete [] poles;
|
||||
delete [] xx;
|
||||
delete [] mm;
|
||||
delete [] a_power;
|
||||
delete [] a;
|
||||
}
|
||||
}
|
||||
|
||||
// Free memory and reallocate as necessary
|
||||
void AlgRemez::allocate(int num_degree, int den_degree)
|
||||
{
|
||||
// Arrays have previously been allocated, deallocate first, then allocate
|
||||
if (alloc) {
|
||||
delete [] param;
|
||||
delete [] roots;
|
||||
delete [] poles;
|
||||
delete [] xx;
|
||||
delete [] mm;
|
||||
}
|
||||
|
||||
// Note use of new and delete in memory allocation - cannot run on qcdsp
|
||||
param = new bigfloat[num_degree+den_degree+1];
|
||||
roots = new bigfloat[num_degree];
|
||||
poles = new bigfloat[den_degree];
|
||||
xx = new bigfloat[num_degree+den_degree+3];
|
||||
mm = new bigfloat[num_degree+den_degree+2];
|
||||
|
||||
if (!alloc) {
|
||||
// The coefficients of the sum in the exponential
|
||||
a = new bigfloat[SUM_MAX];
|
||||
a_power = new int[SUM_MAX];
|
||||
}
|
||||
|
||||
alloc = 1;
|
||||
}
|
||||
|
||||
// Reset the bounds of the approximation
|
||||
void AlgRemez::setBounds(double lower, double upper)
|
||||
{
|
||||
apstrt = lower;
|
||||
apend = upper;
|
||||
apwidt = apend - apstrt;
|
||||
}
|
||||
|
||||
// Generate the rational approximation x^(pnum/pden)
|
||||
double AlgRemez::generateApprox(int degree, unsigned long pnum,
|
||||
unsigned long pden)
|
||||
{
|
||||
return generateApprox(degree, degree, pnum, pden);
|
||||
}
|
||||
|
||||
double AlgRemez::generateApprox(int num_degree, int den_degree,
|
||||
unsigned long pnum, unsigned long pden)
|
||||
{
|
||||
double *a_param = 0;
|
||||
int *a_pow = 0;
|
||||
return generateApprox(num_degree, den_degree, pnum, pden, 0, a_param, a_pow);
|
||||
}
|
||||
|
||||
// Generate the rational approximation x^(pnum/pden)
|
||||
double AlgRemez::generateApprox(int num_degree, int den_degree,
|
||||
unsigned long pnum, unsigned long pden,
|
||||
int a_len, double *a_param, int *a_pow)
|
||||
{
|
||||
std::cout<<"Degree of the approximation is ("<<num_degree<<","<<den_degree<<")\n";
|
||||
std::cout<<"Approximating the function x^("<<pnum<<"/"<<pden<<")\n";
|
||||
|
||||
// Reallocate arrays, since degree has changed
|
||||
if (num_degree != n || den_degree != d) allocate(num_degree,den_degree);
|
||||
|
||||
assert(a_len<=SUM_MAX);
|
||||
|
||||
step = new bigfloat[num_degree+den_degree+2];
|
||||
|
||||
a_length = a_len;
|
||||
for (int j=0; j<a_len; j++) {
|
||||
a[j]= a_param[j];
|
||||
a_power[j] = a_pow[j];
|
||||
}
|
||||
|
||||
power_num = pnum;
|
||||
power_den = pden;
|
||||
spread = 1.0e37;
|
||||
iter = 0;
|
||||
|
||||
n = num_degree;
|
||||
d = den_degree;
|
||||
neq = n + d + 1;
|
||||
|
||||
initialGuess();
|
||||
stpini(step);
|
||||
|
||||
while (spread > tolerance) { //iterate until convergance
|
||||
|
||||
if (iter++%100==0)
|
||||
std::cout<<"Iteration " <<iter-1<<" spread "<<(double)spread<<" delta "<<(double)delta<<std::endl;
|
||||
|
||||
equations();
|
||||
if (delta < tolerance) {
|
||||
std::cout<<"Delta too small, try increasing precision\n";
|
||||
assert(0);
|
||||
};
|
||||
assert( delta>= tolerance);
|
||||
|
||||
search(step);
|
||||
}
|
||||
|
||||
int sign;
|
||||
double error = (double)getErr(mm[0],&sign);
|
||||
std::cout<<"Converged at "<<iter<<" iterations; error = "<<error<<std::endl;
|
||||
|
||||
// Once the approximation has been generated, calculate the roots
|
||||
if(!root()) {
|
||||
std::cout<<"Root finding failed\n";
|
||||
} else {
|
||||
foundRoots = 1;
|
||||
}
|
||||
|
||||
delete [] step;
|
||||
|
||||
// Return the maximum error in the approximation
|
||||
return error;
|
||||
}
|
||||
|
||||
// Return the partial fraction expansion of the approximation x^(pnum/pden)
|
||||
int AlgRemez::getPFE(double *Res, double *Pole, double *Norm) {
|
||||
|
||||
if (n!=d) {
|
||||
std::cout<<"Cannot handle case: Numerator degree neq Denominator degree\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!alloc) {
|
||||
std::cout<<"Approximation not yet generated\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!foundRoots) {
|
||||
std::cout<<"Roots not found, so PFE cannot be taken\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
bigfloat *r = new bigfloat[n];
|
||||
bigfloat *p = new bigfloat[d];
|
||||
|
||||
for (int i=0; i<n; i++) r[i] = roots[i];
|
||||
for (int i=0; i<d; i++) p[i] = poles[i];
|
||||
|
||||
// Perform a partial fraction expansion
|
||||
pfe(r, p, norm);
|
||||
|
||||
// Convert to double and return
|
||||
*Norm = (double)norm;
|
||||
for (int i=0; i<n; i++) Res[i] = (double)r[i];
|
||||
for (int i=0; i<d; i++) Pole[i] = (double)p[i];
|
||||
|
||||
delete [] r;
|
||||
delete [] p;
|
||||
|
||||
// Where the smallest shift is located
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Return the partial fraction expansion of the approximation x^(-pnum/pden)
|
||||
int AlgRemez::getIPFE(double *Res, double *Pole, double *Norm) {
|
||||
|
||||
if (n!=d) {
|
||||
std::cout<<"Cannot handle case: Numerator degree neq Denominator degree\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!alloc) {
|
||||
std::cout<<"Approximation not yet generated\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!foundRoots) {
|
||||
std::cout<<"Roots not found, so PFE cannot be taken\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
bigfloat *r = new bigfloat[d];
|
||||
bigfloat *p = new bigfloat[n];
|
||||
|
||||
// Want the inverse function
|
||||
for (int i=0; i<n; i++) {
|
||||
r[i] = poles[i];
|
||||
p[i] = roots[i];
|
||||
}
|
||||
|
||||
// Perform a partial fraction expansion
|
||||
pfe(r, p, (bigfloat)1l/norm);
|
||||
|
||||
// Convert to double and return
|
||||
*Norm = (double)((bigfloat)1l/(norm));
|
||||
for (int i=0; i<n; i++) {
|
||||
Res[i] = (double)r[i];
|
||||
Pole[i] = (double)p[i];
|
||||
}
|
||||
|
||||
delete [] r;
|
||||
delete [] p;
|
||||
|
||||
// Where the smallest shift is located
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Initial values of maximal and minimal errors
|
||||
void AlgRemez::initialGuess() {
|
||||
|
||||
// Supply initial guesses for solution points
|
||||
long ncheb = neq; // Degree of Chebyshev error estimate
|
||||
bigfloat a, r;
|
||||
|
||||
// Find ncheb+1 extrema of Chebyshev polynomial
|
||||
|
||||
a = ncheb;
|
||||
mm[0] = apstrt;
|
||||
for (long i = 1; i < ncheb; i++) {
|
||||
r = 0.5 * (1 - cos((M_PI * i)/(double) a));
|
||||
//r *= sqrt_bf(r);
|
||||
r = (exp((double)r)-1.0)/(exp(1.0)-1.0);
|
||||
mm[i] = apstrt + r * apwidt;
|
||||
}
|
||||
mm[ncheb] = apend;
|
||||
|
||||
a = 2.0 * ncheb;
|
||||
for (long i = 0; i <= ncheb; i++) {
|
||||
r = 0.5 * (1 - cos(M_PI * (2*i+1)/(double) a));
|
||||
//r *= sqrt_bf(r); // Squeeze to low end of interval
|
||||
r = (exp((double)r)-1.0)/(exp(1.0)-1.0);
|
||||
xx[i] = apstrt + r * apwidt;
|
||||
}
|
||||
}
|
||||
|
||||
// Initialise step sizes
|
||||
void AlgRemez::stpini(bigfloat *step) {
|
||||
xx[neq+1] = apend;
|
||||
delta = 0.25;
|
||||
step[0] = xx[0] - apstrt;
|
||||
for (int i = 1; i < neq; i++) step[i] = xx[i] - xx[i-1];
|
||||
step[neq] = step[neq-1];
|
||||
}
|
||||
|
||||
// Search for error maxima and minima
|
||||
void AlgRemez::search(bigfloat *step) {
|
||||
bigfloat a, q, xm, ym, xn, yn, xx0, xx1;
|
||||
int i, meq, emsign, ensign, steps;
|
||||
|
||||
meq = neq + 1;
|
||||
bigfloat *yy = new bigfloat[meq];
|
||||
|
||||
bigfloat eclose = 1.0e30;
|
||||
bigfloat farther = 0l;
|
||||
|
||||
xx0 = apstrt;
|
||||
|
||||
for (i = 0; i < meq; i++) {
|
||||
steps = 0;
|
||||
xx1 = xx[i]; // Next zero
|
||||
if (i == meq-1) xx1 = apend;
|
||||
xm = mm[i];
|
||||
ym = getErr(xm,&emsign);
|
||||
q = step[i];
|
||||
xn = xm + q;
|
||||
if (xn < xx0 || xn >= xx1) { // Cannot skip over adjacent boundaries
|
||||
q = -q;
|
||||
xn = xm;
|
||||
yn = ym;
|
||||
ensign = emsign;
|
||||
} else {
|
||||
yn = getErr(xn,&ensign);
|
||||
if (yn < ym) {
|
||||
q = -q;
|
||||
xn = xm;
|
||||
yn = ym;
|
||||
ensign = emsign;
|
||||
}
|
||||
}
|
||||
|
||||
while(yn >= ym) { // March until error becomes smaller.
|
||||
if (++steps > 10) break;
|
||||
ym = yn;
|
||||
xm = xn;
|
||||
emsign = ensign;
|
||||
a = xm + q;
|
||||
if (a == xm || a <= xx0 || a >= xx1) break;// Must not skip over the zeros either side.
|
||||
xn = a;
|
||||
yn = getErr(xn,&ensign);
|
||||
}
|
||||
|
||||
mm[i] = xm; // Position of maximum
|
||||
yy[i] = ym; // Value of maximum
|
||||
|
||||
if (eclose > ym) eclose = ym;
|
||||
if (farther < ym) farther = ym;
|
||||
|
||||
xx0 = xx1; // Walk to next zero.
|
||||
} // end of search loop
|
||||
|
||||
q = (farther - eclose); // Decrease step size if error spread increased
|
||||
if (eclose != 0.0) q /= eclose; // Relative error spread
|
||||
if (q >= spread) delta *= 0.5; // Spread is increasing; decrease step size
|
||||
spread = q;
|
||||
|
||||
for (i = 0; i < neq; i++) {
|
||||
q = yy[i+1];
|
||||
if (q != 0.0) q = yy[i] / q - (bigfloat)1l;
|
||||
else q = 0.0625;
|
||||
if (q > (bigfloat)0.25) q = 0.25;
|
||||
q *= mm[i+1] - mm[i];
|
||||
step[i] = q * delta;
|
||||
}
|
||||
step[neq] = step[neq-1];
|
||||
|
||||
for (i = 0; i < neq; i++) { // Insert new locations for the zeros.
|
||||
xm = xx[i] - step[i];
|
||||
if (xm <= apstrt) continue;
|
||||
if (xm >= apend) continue;
|
||||
if (xm <= mm[i]) xm = (bigfloat)0.5 * (mm[i] + xx[i]);
|
||||
if (xm >= mm[i+1]) xm = (bigfloat)0.5 * (mm[i+1] + xx[i]);
|
||||
xx[i] = xm;
|
||||
}
|
||||
|
||||
delete [] yy;
|
||||
}
|
||||
|
||||
// Solve the equations
|
||||
void AlgRemez::equations(void) {
|
||||
bigfloat x, y, z;
|
||||
int i, j, ip;
|
||||
bigfloat *aa;
|
||||
|
||||
bigfloat *AA = new bigfloat[(neq)*(neq)];
|
||||
bigfloat *BB = new bigfloat[neq];
|
||||
|
||||
for (i = 0; i < neq; i++) { // set up the equations for solution by simq()
|
||||
ip = neq * i; // offset to 1st element of this row of matrix
|
||||
x = xx[i]; // the guess for this row
|
||||
y = func(x); // right-hand-side vector
|
||||
|
||||
z = (bigfloat)1l;
|
||||
aa = AA+ip;
|
||||
for (j = 0; j <= n; j++) {
|
||||
*aa++ = z;
|
||||
z *= x;
|
||||
}
|
||||
|
||||
z = (bigfloat)1l;
|
||||
for (j = 0; j < d; j++) {
|
||||
*aa++ = -y * z;
|
||||
z *= x;
|
||||
}
|
||||
BB[i] = y * z; // Right hand side vector
|
||||
}
|
||||
|
||||
// Solve the simultaneous linear equations.
|
||||
if (simq(AA, BB, param, neq)) {
|
||||
std::cout<<"simq failed\n";
|
||||
exit(0);
|
||||
}
|
||||
|
||||
delete [] AA;
|
||||
delete [] BB;
|
||||
|
||||
}
|
||||
|
||||
// Evaluate the rational form P(x)/Q(x) using coefficients
|
||||
// from the solution vector param
|
||||
bigfloat AlgRemez::approx(const bigfloat x) {
|
||||
bigfloat yn, yd;
|
||||
int i;
|
||||
|
||||
// Work backwards toward the constant term.
|
||||
yn = param[n]; // Highest order numerator coefficient
|
||||
for (i = n-1; i >= 0; i--) yn = x * yn + param[i];
|
||||
yd = x + param[n+d]; // Highest degree coefficient = 1.0
|
||||
for (i = n+d-1; i > n; i--) yd = x * yd + param[i];
|
||||
|
||||
return(yn/yd);
|
||||
}
|
||||
|
||||
// Compute size and sign of the approximation error at x
|
||||
bigfloat AlgRemez::getErr(bigfloat x, int *sign) {
|
||||
bigfloat e, f;
|
||||
|
||||
f = func(x);
|
||||
e = approx(x) - f;
|
||||
if (f != 0) e /= f;
|
||||
if (e < (bigfloat)0.0) {
|
||||
*sign = -1;
|
||||
e = -e;
|
||||
}
|
||||
else *sign = 1;
|
||||
|
||||
return(e);
|
||||
}
|
||||
|
||||
// Calculate function required for the approximation.
|
||||
bigfloat AlgRemez::func(const bigfloat x) {
|
||||
|
||||
bigfloat z = (bigfloat)power_num / (bigfloat)power_den;
|
||||
bigfloat y;
|
||||
|
||||
if (x == (bigfloat)1.0) y = (bigfloat)1.0;
|
||||
else y = pow_bf(x,z);
|
||||
|
||||
if (a_length > 0) {
|
||||
bigfloat sum = 0l;
|
||||
for (int j=0; j<a_length; j++) sum += a[j]*pow_bf(x,a_power[j]);
|
||||
return y * exp_bf(sum);
|
||||
} else {
|
||||
return y;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Solve the system AX=B
|
||||
int AlgRemez::simq(bigfloat A[], bigfloat B[], bigfloat X[], int n) {
|
||||
|
||||
int i, j, ij, ip, ipj, ipk, ipn;
|
||||
int idxpiv, iback;
|
||||
int k, kp, kp1, kpk, kpn;
|
||||
int nip, nkp, nm1;
|
||||
bigfloat em, q, rownrm, big, size, pivot, sum;
|
||||
bigfloat *aa;
|
||||
|
||||
// simq() work vector
|
||||
int *IPS = new int[(neq) * sizeof(int)];
|
||||
|
||||
nm1 = n - 1;
|
||||
// Initialize IPS and X
|
||||
|
||||
ij = 0;
|
||||
for (i = 0; i < n; i++) {
|
||||
IPS[i] = i;
|
||||
rownrm = 0.0;
|
||||
for(j = 0; j < n; j++) {
|
||||
q = abs_bf(A[ij]);
|
||||
if(rownrm < q) rownrm = q;
|
||||
++ij;
|
||||
}
|
||||
if (rownrm == (bigfloat)0l) {
|
||||
std::cout<<"simq rownrm=0\n";
|
||||
delete [] IPS;
|
||||
return(1);
|
||||
}
|
||||
X[i] = (bigfloat)1.0 / rownrm;
|
||||
}
|
||||
|
||||
for (k = 0; k < nm1; k++) {
|
||||
big = 0.0;
|
||||
idxpiv = 0;
|
||||
|
||||
for (i = k; i < n; i++) {
|
||||
ip = IPS[i];
|
||||
ipk = n*ip + k;
|
||||
size = abs_bf(A[ipk]) * X[ip];
|
||||
if (size > big) {
|
||||
big = size;
|
||||
idxpiv = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (big == (bigfloat)0l) {
|
||||
std::cout<<"simq big=0\n";
|
||||
delete [] IPS;
|
||||
return(2);
|
||||
}
|
||||
if (idxpiv != k) {
|
||||
j = IPS[k];
|
||||
IPS[k] = IPS[idxpiv];
|
||||
IPS[idxpiv] = j;
|
||||
}
|
||||
kp = IPS[k];
|
||||
kpk = n*kp + k;
|
||||
pivot = A[kpk];
|
||||
kp1 = k+1;
|
||||
for (i = kp1; i < n; i++) {
|
||||
ip = IPS[i];
|
||||
ipk = n*ip + k;
|
||||
em = -A[ipk] / pivot;
|
||||
A[ipk] = -em;
|
||||
nip = n*ip;
|
||||
nkp = n*kp;
|
||||
aa = A+nkp+kp1;
|
||||
for (j = kp1; j < n; j++) {
|
||||
ipj = nip + j;
|
||||
A[ipj] = A[ipj] + em * *aa++;
|
||||
}
|
||||
}
|
||||
}
|
||||
kpn = n * IPS[n-1] + n - 1; // last element of IPS[n] th row
|
||||
if (A[kpn] == (bigfloat)0l) {
|
||||
std::cout<<"simq A[kpn]=0\n";
|
||||
delete [] IPS;
|
||||
return(3);
|
||||
}
|
||||
|
||||
|
||||
ip = IPS[0];
|
||||
X[0] = B[ip];
|
||||
for (i = 1; i < n; i++) {
|
||||
ip = IPS[i];
|
||||
ipj = n * ip;
|
||||
sum = 0.0;
|
||||
for (j = 0; j < i; j++) {
|
||||
sum += A[ipj] * X[j];
|
||||
++ipj;
|
||||
}
|
||||
X[i] = B[ip] - sum;
|
||||
}
|
||||
|
||||
ipn = n * IPS[n-1] + n - 1;
|
||||
X[n-1] = X[n-1] / A[ipn];
|
||||
|
||||
for (iback = 1; iback < n; iback++) {
|
||||
//i goes (n-1),...,1
|
||||
i = nm1 - iback;
|
||||
ip = IPS[i];
|
||||
nip = n*ip;
|
||||
sum = 0.0;
|
||||
aa = A+nip+i+1;
|
||||
for (j= i + 1; j < n; j++)
|
||||
sum += *aa++ * X[j];
|
||||
X[i] = (X[i] - sum) / A[nip+i];
|
||||
}
|
||||
|
||||
delete [] IPS;
|
||||
return(0);
|
||||
}
|
||||
|
||||
// Calculate the roots of the approximation
|
||||
int AlgRemez::root() {
|
||||
|
||||
long i,j;
|
||||
bigfloat x,dx=0.05;
|
||||
bigfloat upper=1, lower=-100000;
|
||||
bigfloat tol = 1e-20;
|
||||
|
||||
bigfloat *poly = new bigfloat[neq+1];
|
||||
|
||||
// First find the numerator roots
|
||||
for (i=0; i<=n; i++) poly[i] = param[i];
|
||||
|
||||
for (i=n-1; i>=0; i--) {
|
||||
roots[i] = rtnewt(poly,i+1,lower,upper,tol);
|
||||
if (roots[i] == 0.0) {
|
||||
std::cout<<"Failure to converge on root "<<i+1<<"/"<<n<<"\n";
|
||||
return 0;
|
||||
}
|
||||
poly[0] = -poly[0]/roots[i];
|
||||
for (j=1; j<=i; j++) poly[j] = (poly[j-1] - poly[j])/roots[i];
|
||||
}
|
||||
|
||||
// Now find the denominator roots
|
||||
poly[d] = 1l;
|
||||
for (i=0; i<d; i++) poly[i] = param[n+1+i];
|
||||
|
||||
for (i=d-1; i>=0; i--) {
|
||||
poles[i]=rtnewt(poly,i+1,lower,upper,tol);
|
||||
if (poles[i] == 0.0) {
|
||||
std::cout<<"Failure to converge on pole "<<i+1<<"/"<<d<<"\n";
|
||||
return 0;
|
||||
}
|
||||
poly[0] = -poly[0]/poles[i];
|
||||
for (j=1; j<=i; j++) poly[j] = (poly[j-1] - poly[j])/poles[i];
|
||||
}
|
||||
|
||||
norm = param[n];
|
||||
|
||||
delete [] poly;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Evaluate the polynomial
|
||||
bigfloat AlgRemez::polyEval(bigfloat x, bigfloat *poly, long size) {
|
||||
bigfloat f = poly[size];
|
||||
for (int i=size-1; i>=0; i--) f = f*x + poly[i];
|
||||
return f;
|
||||
}
|
||||
|
||||
// Evaluate the differential of the polynomial
|
||||
bigfloat AlgRemez::polyDiff(bigfloat x, bigfloat *poly, long size) {
|
||||
bigfloat df = (bigfloat)size*poly[size];
|
||||
for (int i=size-1; i>0; i--) df = df*x + (bigfloat)i*poly[i];
|
||||
return df;
|
||||
}
|
||||
|
||||
|
||||
// Newton's method to calculate roots
|
||||
bigfloat AlgRemez::rtnewt(bigfloat *poly, long i, bigfloat x1,
|
||||
bigfloat x2, bigfloat xacc) {
|
||||
int j;
|
||||
bigfloat df, dx, f, rtn;
|
||||
|
||||
rtn=(bigfloat)0.5*(x1+x2);
|
||||
for (j=1; j<=JMAX;j++) {
|
||||
f = polyEval(rtn, poly, i);
|
||||
df = polyDiff(rtn, poly, i);
|
||||
dx = f/df;
|
||||
rtn -= dx;
|
||||
if (abs_bf(dx) < xacc) return rtn;
|
||||
}
|
||||
std::cout<<"Maximum number of iterations exceeded in rtnewt\n";
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// Evaluate the partial fraction expansion of the rational function
|
||||
// with res roots and poles poles. Result is overwritten on input
|
||||
// arrays.
|
||||
void AlgRemez::pfe(bigfloat *res, bigfloat *poles, bigfloat norm) {
|
||||
int i,j,small;
|
||||
bigfloat temp;
|
||||
bigfloat *numerator = new bigfloat[n];
|
||||
bigfloat *denominator = new bigfloat[d];
|
||||
|
||||
// Construct the polynomials explicitly
|
||||
for (i=1; i<n; i++) {
|
||||
numerator[i] = 0l;
|
||||
denominator[i] = 0l;
|
||||
}
|
||||
numerator[0]=1l;
|
||||
denominator[0]=1l;
|
||||
|
||||
for (j=0; j<n; j++) {
|
||||
for (i=n-1; i>=0; i--) {
|
||||
numerator[i] *= -res[j];
|
||||
denominator[i] *= -poles[j];
|
||||
if (i>0) {
|
||||
numerator[i] += numerator[i-1];
|
||||
denominator[i] += denominator[i-1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to proper fraction form.
|
||||
// Fraction is now in the form 1 + n/d, where O(n)+1=O(d)
|
||||
for (i=0; i<n; i++) numerator[i] -= denominator[i];
|
||||
|
||||
// Find the residues of the partial fraction expansion and absorb the
|
||||
// coefficients.
|
||||
for (i=0; i<n; i++) {
|
||||
res[i] = 0l;
|
||||
for (j=n-1; j>=0; j--) {
|
||||
res[i] = poles[i]*res[i]+numerator[j];
|
||||
}
|
||||
for (j=n-1; j>=0; j--) {
|
||||
if (i!=j) res[i] /= poles[i]-poles[j];
|
||||
}
|
||||
res[i] *= norm;
|
||||
}
|
||||
|
||||
// res now holds the residues
|
||||
j = 0;
|
||||
for (i=0; i<n; i++) poles[i] = -poles[i];
|
||||
|
||||
// Move the ordering of the poles from smallest to largest
|
||||
for (j=0; j<n; j++) {
|
||||
small = j;
|
||||
for (i=j+1; i<n; i++) {
|
||||
if (poles[i] < poles[small]) small = i;
|
||||
}
|
||||
if (small != j) {
|
||||
temp = poles[small];
|
||||
poles[small] = poles[j];
|
||||
poles[j] = temp;
|
||||
temp = res[small];
|
||||
res[small] = res[j];
|
||||
res[j] = temp;
|
||||
}
|
||||
}
|
||||
|
||||
delete [] numerator;
|
||||
delete [] denominator;
|
||||
}
|
||||
|
||||
double AlgRemez::evaluateApprox(double x) {
|
||||
return (double)approx((bigfloat)x);
|
||||
}
|
||||
|
||||
double AlgRemez::evaluateInverseApprox(double x) {
|
||||
return 1.0/(double)approx((bigfloat)x);
|
||||
}
|
||||
|
||||
double AlgRemez::evaluateFunc(double x) {
|
||||
return (double)func((bigfloat)x);
|
||||
}
|
||||
|
||||
double AlgRemez::evaluateInverseFunc(double x) {
|
||||
return 1.0/(double)func((bigfloat)x);
|
||||
}
|
||||
|
||||
void AlgRemez::csv(std::ostream & os)
|
||||
{
|
||||
double lambda_low = apstrt;
|
||||
double lambda_high= apend;
|
||||
for (double x=lambda_low; x<lambda_high; x*=1.05) {
|
||||
double f = evaluateFunc(x);
|
||||
double r = evaluateApprox(x);
|
||||
os<< x<<","<<r<<","<<f<<","<<r-f<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
/*
|
||||
|
||||
Mike Clark - 25th May 2005
|
||||
|
||||
alg_remez.h
|
||||
|
||||
AlgRemez is an implementation of the Remez algorithm, which in this
|
||||
case is used for generating the optimal nth root rational
|
||||
approximation.
|
||||
|
||||
Note this class requires the gnu multiprecision (GNU MP) library.
|
||||
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_ALG_REMEZ_H
|
||||
#define INCLUDED_ALG_REMEZ_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <Grid/GridStd.h>
|
||||
|
||||
#ifdef HAVE_LIBGMP
|
||||
#include "bigfloat.h"
|
||||
#else
|
||||
#include "bigfloat_double.h"
|
||||
#endif
|
||||
|
||||
#define JMAX 10000 //Maximum number of iterations of Newton's approximation
|
||||
#define SUM_MAX 10 // Maximum number of terms in exponential
|
||||
|
||||
/*
|
||||
*Usage examples
|
||||
AlgRemez remez(lambda_low,lambda_high,precision);
|
||||
error = remez.generateApprox(n,d,y,z);
|
||||
remez.getPFE(res,pole,&norm);
|
||||
remez.getIPFE(res,pole,&norm);
|
||||
remez.csv(ostream &os);
|
||||
*/
|
||||
|
||||
class AlgRemez
|
||||
{
|
||||
private:
|
||||
char *cname;
|
||||
|
||||
// The approximation parameters
|
||||
bigfloat *param, *roots, *poles;
|
||||
bigfloat norm;
|
||||
|
||||
// The numerator and denominator degree (n=d)
|
||||
int n, d;
|
||||
|
||||
// The bounds of the approximation
|
||||
bigfloat apstrt, apwidt, apend;
|
||||
|
||||
// the numerator and denominator of the power we are approximating
|
||||
unsigned long power_num;
|
||||
unsigned long power_den;
|
||||
|
||||
// Flag to determine whether the arrays have been allocated
|
||||
int alloc;
|
||||
|
||||
// Flag to determine whether the roots have been found
|
||||
int foundRoots;
|
||||
|
||||
// Variables used to calculate the approximation
|
||||
int nd1, iter;
|
||||
bigfloat *xx, *mm, *step;
|
||||
bigfloat delta, spread, tolerance;
|
||||
|
||||
// The exponential summation coefficients
|
||||
bigfloat *a;
|
||||
int *a_power;
|
||||
int a_length;
|
||||
|
||||
// The number of equations we must solve at each iteration (n+d+1)
|
||||
int neq;
|
||||
|
||||
// The precision of the GNU MP library
|
||||
long prec;
|
||||
|
||||
// Initial values of maximal and minmal errors
|
||||
void initialGuess();
|
||||
|
||||
// Solve the equations
|
||||
void equations();
|
||||
|
||||
// Search for error maxima and minima
|
||||
void search(bigfloat *step);
|
||||
|
||||
// Initialise step sizes
|
||||
void stpini(bigfloat *step);
|
||||
|
||||
// Calculate the roots of the approximation
|
||||
int root();
|
||||
|
||||
// Evaluate the polynomial
|
||||
bigfloat polyEval(bigfloat x, bigfloat *poly, long size);
|
||||
//complex_bf polyEval(complex_bf x, complex_bf *poly, long size);
|
||||
|
||||
// Evaluate the differential of the polynomial
|
||||
bigfloat polyDiff(bigfloat x, bigfloat *poly, long size);
|
||||
//complex_bf polyDiff(complex_bf x, complex_bf *poly, long size);
|
||||
|
||||
// Newton's method to calculate roots
|
||||
bigfloat rtnewt(bigfloat *poly, long i, bigfloat x1, bigfloat x2, bigfloat xacc);
|
||||
//complex_bf rtnewt(complex_bf *poly, long i, bigfloat x1, bigfloat x2, bigfloat xacc);
|
||||
|
||||
// Evaluate the partial fraction expansion of the rational function
|
||||
// with res roots and poles poles. Result is overwritten on input
|
||||
// arrays.
|
||||
void pfe(bigfloat *res, bigfloat* poles, bigfloat norm);
|
||||
|
||||
// Calculate function required for the approximation
|
||||
bigfloat func(bigfloat x);
|
||||
|
||||
// Compute size and sign of the approximation error at x
|
||||
bigfloat getErr(bigfloat x, int *sign);
|
||||
|
||||
// Solve the system AX=B
|
||||
int simq(bigfloat *A, bigfloat *B, bigfloat *X, int n);
|
||||
|
||||
// Free memory and reallocate as necessary
|
||||
void allocate(int num_degree, int den_degree);
|
||||
|
||||
// Evaluate the rational form P(x)/Q(x) using coefficients from the
|
||||
// solution vector param
|
||||
bigfloat approx(bigfloat x);
|
||||
|
||||
public:
|
||||
|
||||
// Constructor
|
||||
AlgRemez(double lower, double upper, long prec);
|
||||
|
||||
// Destructor
|
||||
virtual ~AlgRemez();
|
||||
|
||||
int getDegree(void){
|
||||
assert(n==d);
|
||||
return n;
|
||||
}
|
||||
// Reset the bounds of the approximation
|
||||
void setBounds(double lower, double upper);
|
||||
// Reset the bounds of the approximation
|
||||
void getBounds(double &lower, double &upper) {
|
||||
lower=(double)apstrt;
|
||||
upper=(double)apend;
|
||||
}
|
||||
|
||||
// Generate the rational approximation x^(pnum/pden)
|
||||
double generateApprox(int num_degree, int den_degree,
|
||||
unsigned long power_num, unsigned long power_den,
|
||||
int a_len, double* a_param, int* a_pow);
|
||||
double generateApprox(int num_degree, int den_degree,
|
||||
unsigned long power_num, unsigned long power_den);
|
||||
double generateApprox(int degree, unsigned long power_num,
|
||||
unsigned long power_den);
|
||||
|
||||
// Return the partial fraction expansion of the approximation x^(pnum/pden)
|
||||
int getPFE(double *res, double *pole, double *norm);
|
||||
|
||||
// Return the partial fraction expansion of the approximation x^(-pnum/pden)
|
||||
int getIPFE(double *res, double *pole, double *norm);
|
||||
|
||||
// Evaluate the rational form P(x)/Q(x) using coefficients from the
|
||||
// solution vector param
|
||||
double evaluateApprox(double x);
|
||||
|
||||
// Evaluate the rational form Q(x)/P(x) using coefficients from the
|
||||
// solution vector param
|
||||
double evaluateInverseApprox(double x);
|
||||
|
||||
// Calculate function required for the approximation
|
||||
double evaluateFunc(double x);
|
||||
|
||||
// Calculate inverse function required for the approximation
|
||||
double evaluateInverseFunc(double x);
|
||||
|
||||
// Dump csv of function, approx and error
|
||||
void csv(std::ostream &os);
|
||||
};
|
||||
|
||||
#endif // Include guard
|
||||
|
||||
|
||||
|
||||
@@ -1,473 +0,0 @@
|
||||
#include<math.h>
|
||||
#include<stdio.h>
|
||||
#include<stdlib.h>
|
||||
#include<string>
|
||||
#include<iostream>
|
||||
#include<iomanip>
|
||||
#include<cassert>
|
||||
|
||||
#include<Grid/algorithms/approx/RemezGeneral.h>
|
||||
|
||||
|
||||
// Constructor
|
||||
AlgRemezGeneral::AlgRemezGeneral(double lower, double upper, long precision,
|
||||
bigfloat (*f)(bigfloat x, void *data), void *data): f(f),
|
||||
data(data),
|
||||
prec(precision),
|
||||
apstrt(lower), apend(upper), apwidt(upper - lower),
|
||||
n(0), d(0), pow_n(0), pow_d(0)
|
||||
{
|
||||
bigfloat::setDefaultPrecision(prec);
|
||||
|
||||
std::cout<<"Approximation bounds are ["<<apstrt<<","<<apend<<"]\n";
|
||||
std::cout<<"Precision of arithmetic is "<<precision<<std::endl;
|
||||
}
|
||||
|
||||
//Determine the properties of the numerator and denominator polynomials
|
||||
void AlgRemezGeneral::setupPolyProperties(int num_degree, int den_degree, PolyType num_type_in, PolyType den_type_in){
|
||||
pow_n = num_degree;
|
||||
pow_d = den_degree;
|
||||
|
||||
if(pow_n % 2 == 0 && num_type_in == PolyType::Odd) assert(0);
|
||||
if(pow_n % 2 == 1 && num_type_in == PolyType::Even) assert(0);
|
||||
|
||||
if(pow_d % 2 == 0 && den_type_in == PolyType::Odd) assert(0);
|
||||
if(pow_d % 2 == 1 && den_type_in == PolyType::Even) assert(0);
|
||||
|
||||
num_type = num_type_in;
|
||||
den_type = den_type_in;
|
||||
|
||||
num_pows.resize(pow_n+1);
|
||||
den_pows.resize(pow_d+1);
|
||||
|
||||
int n_in = 0;
|
||||
bool odd = num_type == PolyType::Full || num_type == PolyType::Odd;
|
||||
bool even = num_type == PolyType::Full || num_type == PolyType::Even;
|
||||
for(int i=0;i<=pow_n;i++){
|
||||
num_pows[i] = -1;
|
||||
if(i % 2 == 0 && even) num_pows[i] = n_in++;
|
||||
if(i % 2 == 1 && odd) num_pows[i] = n_in++;
|
||||
}
|
||||
|
||||
std::cout << n_in << " terms in numerator" << std::endl;
|
||||
--n_in; //power is 1 less than the number of terms, eg pow=1 a x^1 + b x^0
|
||||
|
||||
int d_in = 0;
|
||||
odd = den_type == PolyType::Full || den_type == PolyType::Odd;
|
||||
even = den_type == PolyType::Full || den_type == PolyType::Even;
|
||||
for(int i=0;i<=pow_d;i++){
|
||||
den_pows[i] = -1;
|
||||
if(i % 2 == 0 && even) den_pows[i] = d_in++;
|
||||
if(i % 2 == 1 && odd) den_pows[i] = d_in++;
|
||||
}
|
||||
|
||||
std::cout << d_in << " terms in denominator" << std::endl;
|
||||
--d_in;
|
||||
|
||||
n = n_in;
|
||||
d = d_in;
|
||||
}
|
||||
|
||||
//Setup algorithm
|
||||
void AlgRemezGeneral::reinitializeAlgorithm(){
|
||||
spread = 1.0e37;
|
||||
iter = 0;
|
||||
|
||||
neq = n + d + 1; //not +2 because highest-power term in denominator is fixed to 1
|
||||
|
||||
param.resize(neq);
|
||||
yy.resize(neq+1);
|
||||
|
||||
//Initialize linear equation temporaries
|
||||
A.resize(neq*neq);
|
||||
B.resize(neq);
|
||||
IPS.resize(neq);
|
||||
|
||||
//Initialize maximum and minimum errors
|
||||
xx.resize(neq+2);
|
||||
mm.resize(neq+1);
|
||||
initialGuess();
|
||||
|
||||
//Initialize search steps
|
||||
step.resize(neq+1);
|
||||
stpini();
|
||||
}
|
||||
|
||||
double AlgRemezGeneral::generateApprox(const int num_degree, const int den_degree,
|
||||
const PolyType num_type_in, const PolyType den_type_in,
|
||||
const double _tolerance, const int report_freq){
|
||||
//Setup the properties of the polynomial
|
||||
setupPolyProperties(num_degree, den_degree, num_type_in, den_type_in);
|
||||
|
||||
//Setup the algorithm
|
||||
reinitializeAlgorithm();
|
||||
|
||||
bigfloat tolerance = _tolerance;
|
||||
|
||||
//Iterate until convergance
|
||||
while (spread > tolerance) {
|
||||
if (iter++ % report_freq==0)
|
||||
std::cout<<"Iteration " <<iter-1<<" spread "<<(double)spread<<" delta "<<(double)delta << std::endl;
|
||||
|
||||
equations();
|
||||
if (delta < tolerance) {
|
||||
std::cout<<"Iteration " << iter-1 << " delta too small (" << delta << "<" << tolerance << "), try increasing precision\n";
|
||||
assert(0);
|
||||
};
|
||||
assert( delta>= tolerance );
|
||||
|
||||
search();
|
||||
}
|
||||
|
||||
int sign;
|
||||
double error = (double)getErr(mm[0],&sign);
|
||||
std::cout<<"Converged at "<<iter<<" iterations; error = "<<error<<std::endl;
|
||||
|
||||
// Return the maximum error in the approximation
|
||||
return error;
|
||||
}
|
||||
|
||||
|
||||
// Initial values of maximal and minimal errors
|
||||
void AlgRemezGeneral::initialGuess(){
|
||||
// Supply initial guesses for solution points
|
||||
long ncheb = neq; // Degree of Chebyshev error estimate
|
||||
|
||||
// Find ncheb+1 extrema of Chebyshev polynomial
|
||||
bigfloat a = ncheb;
|
||||
bigfloat r;
|
||||
|
||||
mm[0] = apstrt;
|
||||
for (long i = 1; i < ncheb; i++) {
|
||||
r = 0.5 * (1 - cos((M_PI * i)/(double) a));
|
||||
//r *= sqrt_bf(r);
|
||||
r = (exp((double)r)-1.0)/(exp(1.0)-1.0);
|
||||
mm[i] = apstrt + r * apwidt;
|
||||
}
|
||||
mm[ncheb] = apend;
|
||||
|
||||
a = 2.0 * ncheb;
|
||||
for (long i = 0; i <= ncheb; i++) {
|
||||
r = 0.5 * (1 - cos(M_PI * (2*i+1)/(double) a));
|
||||
//r *= sqrt_bf(r); // Squeeze to low end of interval
|
||||
r = (exp((double)r)-1.0)/(exp(1.0)-1.0);
|
||||
xx[i] = apstrt + r * apwidt;
|
||||
}
|
||||
}
|
||||
|
||||
// Initialise step sizes
|
||||
void AlgRemezGeneral::stpini(){
|
||||
xx[neq+1] = apend;
|
||||
delta = 0.25;
|
||||
step[0] = xx[0] - apstrt;
|
||||
for (int i = 1; i < neq; i++) step[i] = xx[i] - xx[i-1];
|
||||
step[neq] = step[neq-1];
|
||||
}
|
||||
|
||||
// Search for error maxima and minima
|
||||
void AlgRemezGeneral::search(){
|
||||
bigfloat a, q, xm, ym, xn, yn, xx1;
|
||||
int emsign, ensign, steps;
|
||||
|
||||
int meq = neq + 1;
|
||||
|
||||
bigfloat eclose = 1.0e30;
|
||||
bigfloat farther = 0l;
|
||||
|
||||
bigfloat xx0 = apstrt;
|
||||
|
||||
for (int i = 0; i < meq; i++) {
|
||||
steps = 0;
|
||||
xx1 = xx[i]; // Next zero
|
||||
if (i == meq-1) xx1 = apend;
|
||||
xm = mm[i];
|
||||
ym = getErr(xm,&emsign);
|
||||
q = step[i];
|
||||
xn = xm + q;
|
||||
if (xn < xx0 || xn >= xx1) { // Cannot skip over adjacent boundaries
|
||||
q = -q;
|
||||
xn = xm;
|
||||
yn = ym;
|
||||
ensign = emsign;
|
||||
} else {
|
||||
yn = getErr(xn,&ensign);
|
||||
if (yn < ym) {
|
||||
q = -q;
|
||||
xn = xm;
|
||||
yn = ym;
|
||||
ensign = emsign;
|
||||
}
|
||||
}
|
||||
|
||||
while(yn >= ym) { // March until error becomes smaller.
|
||||
if (++steps > 10)
|
||||
break;
|
||||
|
||||
ym = yn;
|
||||
xm = xn;
|
||||
emsign = ensign;
|
||||
a = xm + q;
|
||||
if (a == xm || a <= xx0 || a >= xx1)
|
||||
break;// Must not skip over the zeros either side.
|
||||
|
||||
xn = a;
|
||||
yn = getErr(xn,&ensign);
|
||||
}
|
||||
|
||||
mm[i] = xm; // Position of maximum
|
||||
yy[i] = ym; // Value of maximum
|
||||
|
||||
if (eclose > ym) eclose = ym;
|
||||
if (farther < ym) farther = ym;
|
||||
|
||||
xx0 = xx1; // Walk to next zero.
|
||||
} // end of search loop
|
||||
|
||||
q = (farther - eclose); // Decrease step size if error spread increased
|
||||
|
||||
if (eclose != 0.0) q /= eclose; // Relative error spread
|
||||
|
||||
if (q >= spread)
|
||||
delta *= 0.5; // Spread is increasing; decrease step size
|
||||
|
||||
spread = q;
|
||||
|
||||
for (int i = 0; i < neq; i++) {
|
||||
q = yy[i+1];
|
||||
if (q != 0.0) q = yy[i] / q - (bigfloat)1l;
|
||||
else q = 0.0625;
|
||||
if (q > (bigfloat)0.25) q = 0.25;
|
||||
q *= mm[i+1] - mm[i];
|
||||
step[i] = q * delta;
|
||||
}
|
||||
step[neq] = step[neq-1];
|
||||
|
||||
for (int i = 0; i < neq; i++) { // Insert new locations for the zeros.
|
||||
xm = xx[i] - step[i];
|
||||
|
||||
if (xm <= apstrt)
|
||||
continue;
|
||||
|
||||
if (xm >= apend)
|
||||
continue;
|
||||
|
||||
if (xm <= mm[i])
|
||||
xm = (bigfloat)0.5 * (mm[i] + xx[i]);
|
||||
|
||||
if (xm >= mm[i+1])
|
||||
xm = (bigfloat)0.5 * (mm[i+1] + xx[i]);
|
||||
|
||||
xx[i] = xm;
|
||||
}
|
||||
}
|
||||
|
||||
// Solve the equations
|
||||
void AlgRemezGeneral::equations(){
|
||||
bigfloat x, y, z;
|
||||
bigfloat *aa;
|
||||
|
||||
for (int i = 0; i < neq; i++) { // set up the equations for solution by simq()
|
||||
int ip = neq * i; // offset to 1st element of this row of matrix
|
||||
x = xx[i]; // the guess for this row
|
||||
y = func(x); // right-hand-side vector
|
||||
|
||||
z = (bigfloat)1l;
|
||||
aa = A.data()+ip;
|
||||
int t = 0;
|
||||
for (int j = 0; j <= pow_n; j++) {
|
||||
if(num_pows[j] != -1){ *aa++ = z; t++; }
|
||||
z *= x;
|
||||
}
|
||||
assert(t == n+1);
|
||||
|
||||
z = (bigfloat)1l;
|
||||
t = 0;
|
||||
for (int j = 0; j < pow_d; j++) {
|
||||
if(den_pows[j] != -1){ *aa++ = -y * z; t++; }
|
||||
z *= x;
|
||||
}
|
||||
assert(t == d);
|
||||
|
||||
B[i] = y * z; // Right hand side vector
|
||||
}
|
||||
|
||||
// Solve the simultaneous linear equations.
|
||||
if (simq()){
|
||||
std::cout<<"simq failed\n";
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Evaluate the rational form P(x)/Q(x) using coefficients
|
||||
// from the solution vector param
|
||||
bigfloat AlgRemezGeneral::approx(const bigfloat x) const{
|
||||
// Work backwards toward the constant term.
|
||||
int c = n;
|
||||
bigfloat yn = param[c--]; // Highest order numerator coefficient
|
||||
for (int i = pow_n-1; i >= 0; i--) yn = x * yn + (num_pows[i] != -1 ? param[c--] : bigfloat(0l));
|
||||
|
||||
c = n+d;
|
||||
bigfloat yd = 1l; //Highest degree coefficient is 1.0
|
||||
for (int i = pow_d-1; i >= 0; i--) yd = x * yd + (den_pows[i] != -1 ? param[c--] : bigfloat(0l));
|
||||
|
||||
return(yn/yd);
|
||||
}
|
||||
|
||||
// Compute size and sign of the approximation error at x
|
||||
bigfloat AlgRemezGeneral::getErr(bigfloat x, int *sign) const{
|
||||
bigfloat f = func(x);
|
||||
bigfloat e = approx(x) - f;
|
||||
if (f != 0) e /= f;
|
||||
if (e < (bigfloat)0.0) {
|
||||
*sign = -1;
|
||||
e = -e;
|
||||
}
|
||||
else *sign = 1;
|
||||
|
||||
return(e);
|
||||
}
|
||||
|
||||
// Solve the system AX=B
|
||||
int AlgRemezGeneral::simq(){
|
||||
|
||||
int ip, ipj, ipk, ipn;
|
||||
int idxpiv;
|
||||
int kp, kp1, kpk, kpn;
|
||||
int nip, nkp;
|
||||
bigfloat em, q, rownrm, big, size, pivot, sum;
|
||||
bigfloat *aa;
|
||||
bigfloat *X = param.data();
|
||||
|
||||
int n = neq;
|
||||
int nm1 = n - 1;
|
||||
// Initialize IPS and X
|
||||
|
||||
int ij = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
IPS[i] = i;
|
||||
rownrm = 0.0;
|
||||
for(int j = 0; j < n; j++) {
|
||||
q = abs_bf(A[ij]);
|
||||
if(rownrm < q) rownrm = q;
|
||||
++ij;
|
||||
}
|
||||
if (rownrm == (bigfloat)0l) {
|
||||
std::cout<<"simq rownrm=0\n";
|
||||
return(1);
|
||||
}
|
||||
X[i] = (bigfloat)1.0 / rownrm;
|
||||
}
|
||||
|
||||
for (int k = 0; k < nm1; k++) {
|
||||
big = 0.0;
|
||||
idxpiv = 0;
|
||||
|
||||
for (int i = k; i < n; i++) {
|
||||
ip = IPS[i];
|
||||
ipk = n*ip + k;
|
||||
size = abs_bf(A[ipk]) * X[ip];
|
||||
if (size > big) {
|
||||
big = size;
|
||||
idxpiv = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (big == (bigfloat)0l) {
|
||||
std::cout<<"simq big=0\n";
|
||||
return(2);
|
||||
}
|
||||
if (idxpiv != k) {
|
||||
int j = IPS[k];
|
||||
IPS[k] = IPS[idxpiv];
|
||||
IPS[idxpiv] = j;
|
||||
}
|
||||
kp = IPS[k];
|
||||
kpk = n*kp + k;
|
||||
pivot = A[kpk];
|
||||
kp1 = k+1;
|
||||
for (int i = kp1; i < n; i++) {
|
||||
ip = IPS[i];
|
||||
ipk = n*ip + k;
|
||||
em = -A[ipk] / pivot;
|
||||
A[ipk] = -em;
|
||||
nip = n*ip;
|
||||
nkp = n*kp;
|
||||
aa = A.data()+nkp+kp1;
|
||||
for (int j = kp1; j < n; j++) {
|
||||
ipj = nip + j;
|
||||
A[ipj] = A[ipj] + em * *aa++;
|
||||
}
|
||||
}
|
||||
}
|
||||
kpn = n * IPS[n-1] + n - 1; // last element of IPS[n] th row
|
||||
if (A[kpn] == (bigfloat)0l) {
|
||||
std::cout<<"simq A[kpn]=0\n";
|
||||
return(3);
|
||||
}
|
||||
|
||||
|
||||
ip = IPS[0];
|
||||
X[0] = B[ip];
|
||||
for (int i = 1; i < n; i++) {
|
||||
ip = IPS[i];
|
||||
ipj = n * ip;
|
||||
sum = 0.0;
|
||||
for (int j = 0; j < i; j++) {
|
||||
sum += A[ipj] * X[j];
|
||||
++ipj;
|
||||
}
|
||||
X[i] = B[ip] - sum;
|
||||
}
|
||||
|
||||
ipn = n * IPS[n-1] + n - 1;
|
||||
X[n-1] = X[n-1] / A[ipn];
|
||||
|
||||
for (int iback = 1; iback < n; iback++) {
|
||||
//i goes (n-1),...,1
|
||||
int i = nm1 - iback;
|
||||
ip = IPS[i];
|
||||
nip = n*ip;
|
||||
sum = 0.0;
|
||||
aa = A.data()+nip+i+1;
|
||||
for (int j= i + 1; j < n; j++)
|
||||
sum += *aa++ * X[j];
|
||||
X[i] = (X[i] - sum) / A[nip+i];
|
||||
}
|
||||
|
||||
return(0);
|
||||
}
|
||||
|
||||
void AlgRemezGeneral::csv(std::ostream & os) const{
|
||||
os << "Numerator" << std::endl;
|
||||
for(int i=0;i<=pow_n;i++){
|
||||
os << getCoeffNum(i) << "*x^" << i;
|
||||
if(i!=pow_n) os << " + ";
|
||||
}
|
||||
os << std::endl;
|
||||
|
||||
os << "Denominator" << std::endl;
|
||||
for(int i=0;i<=pow_d;i++){
|
||||
os << getCoeffDen(i) << "*x^" << i;
|
||||
if(i!=pow_d) os << " + ";
|
||||
}
|
||||
os << std::endl;
|
||||
|
||||
//For a true minimax solution the errors should all be equal and the signs should oscillate +-+-+- etc
|
||||
int sign;
|
||||
os << "Errors at maxima: coordinate, error, (sign)" << std::endl;
|
||||
for(int i=0;i<neq+1;i++){
|
||||
os << mm[i] << " " << getErr(mm[i],&sign) << " (" << sign << ")" << std::endl;
|
||||
}
|
||||
|
||||
os << "Scan over range:" << std::endl;
|
||||
int npt = 60;
|
||||
bigfloat dlt = (apend - apstrt)/bigfloat(npt-1);
|
||||
|
||||
for (bigfloat x=apstrt; x<=apend; x = x + dlt) {
|
||||
double f = evaluateFunc(x);
|
||||
double r = evaluateApprox(x);
|
||||
os<< x<<","<<r<<","<<f<<","<<r-f<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
/*
|
||||
C.Kelly Jan 2020 based on implementation by M. Clark May 2005
|
||||
|
||||
AlgRemezGeneral is an implementation of the Remez algorithm for approximating an arbitrary function by a rational polynomial
|
||||
It includes optional restriction to odd/even polynomials for the numerator and/or denominator
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_ALG_REMEZ_GENERAL_H
|
||||
#define INCLUDED_ALG_REMEZ_GENERAL_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <Grid/GridStd.h>
|
||||
|
||||
#ifdef HAVE_LIBGMP
|
||||
#include "bigfloat.h"
|
||||
#else
|
||||
#include "bigfloat_double.h"
|
||||
#endif
|
||||
|
||||
|
||||
class AlgRemezGeneral{
|
||||
public:
|
||||
enum PolyType { Even, Odd, Full };
|
||||
|
||||
private:
|
||||
|
||||
// In GSL-style, pass the function as a function pointer. Any data required to evaluate the function is passed in as a void pointer
|
||||
bigfloat (*f)(bigfloat x, void *data);
|
||||
void *data;
|
||||
|
||||
// The approximation parameters
|
||||
std::vector<bigfloat> param;
|
||||
bigfloat norm;
|
||||
|
||||
// The number of non-zero terms in the numerator and denominator
|
||||
int n, d;
|
||||
// The numerator and denominator degree (i.e. the largest power)
|
||||
int pow_n, pow_d;
|
||||
|
||||
// Specify if the numerator and/or denominator are odd/even polynomials
|
||||
PolyType num_type;
|
||||
PolyType den_type;
|
||||
std::vector<int> num_pows; //contains the mapping, with -1 if not present
|
||||
std::vector<int> den_pows;
|
||||
|
||||
// The bounds of the approximation
|
||||
bigfloat apstrt, apwidt, apend;
|
||||
|
||||
// Variables used to calculate the approximation
|
||||
int nd1, iter;
|
||||
std::vector<bigfloat> xx;
|
||||
std::vector<bigfloat> mm;
|
||||
std::vector<bigfloat> step;
|
||||
|
||||
bigfloat delta, spread;
|
||||
|
||||
// Variables used in search
|
||||
std::vector<bigfloat> yy;
|
||||
|
||||
// Variables used in solving linear equations
|
||||
std::vector<bigfloat> A;
|
||||
std::vector<bigfloat> B;
|
||||
std::vector<int> IPS;
|
||||
|
||||
// The number of equations we must solve at each iteration (n+d+1)
|
||||
int neq;
|
||||
|
||||
// The precision of the GNU MP library
|
||||
long prec;
|
||||
|
||||
// Initialize member variables associated with the polynomial's properties
|
||||
void setupPolyProperties(int num_degree, int den_degree, PolyType num_type_in, PolyType den_type_in);
|
||||
|
||||
// Initial values of maximal and minmal errors
|
||||
void initialGuess();
|
||||
|
||||
// Initialise step sizes
|
||||
void stpini();
|
||||
|
||||
// Initialize the algorithm
|
||||
void reinitializeAlgorithm();
|
||||
|
||||
// Solve the equations
|
||||
void equations();
|
||||
|
||||
// Search for error maxima and minima
|
||||
void search();
|
||||
|
||||
// Calculate function required for the approximation
|
||||
inline bigfloat func(bigfloat x) const{
|
||||
return f(x, data);
|
||||
}
|
||||
|
||||
// Compute size and sign of the approximation error at x
|
||||
bigfloat getErr(bigfloat x, int *sign) const;
|
||||
|
||||
// Solve the system AX=B where X = param
|
||||
int simq();
|
||||
|
||||
// Evaluate the rational form P(x)/Q(x) using coefficients from the solution vector param
|
||||
bigfloat approx(bigfloat x) const;
|
||||
|
||||
public:
|
||||
|
||||
AlgRemezGeneral(double lower, double upper, long prec,
|
||||
bigfloat (*f)(bigfloat x, void *data), void *data);
|
||||
|
||||
inline int getDegree(void) const{
|
||||
assert(n==d);
|
||||
return n;
|
||||
}
|
||||
// Reset the bounds of the approximation
|
||||
inline void setBounds(double lower, double upper) {
|
||||
apstrt = lower;
|
||||
apend = upper;
|
||||
apwidt = apend - apstrt;
|
||||
}
|
||||
|
||||
// Get the bounds of the approximation
|
||||
inline void getBounds(double &lower, double &upper) const{
|
||||
lower=(double)apstrt;
|
||||
upper=(double)apend;
|
||||
}
|
||||
|
||||
// Run the algorithm to generate the rational approximation
|
||||
double generateApprox(int num_degree, int den_degree,
|
||||
PolyType num_type, PolyType den_type,
|
||||
const double tolerance = 1e-15, const int report_freq = 1000);
|
||||
|
||||
inline double generateApprox(int num_degree, int den_degree,
|
||||
const double tolerance = 1e-15, const int report_freq = 1000){
|
||||
return generateApprox(num_degree, den_degree, Full, Full, tolerance, report_freq);
|
||||
}
|
||||
|
||||
// Evaluate the rational form P(x)/Q(x) using coefficients from the
|
||||
// solution vector param
|
||||
inline double evaluateApprox(double x) const{
|
||||
return (double)approx((bigfloat)x);
|
||||
}
|
||||
|
||||
// Evaluate the rational form Q(x)/P(x) using coefficients from the solution vector param
|
||||
inline double evaluateInverseApprox(double x) const{
|
||||
return 1.0/(double)approx((bigfloat)x);
|
||||
}
|
||||
|
||||
// Calculate function required for the approximation
|
||||
inline double evaluateFunc(double x) const{
|
||||
return (double)func((bigfloat)x);
|
||||
}
|
||||
|
||||
// Calculate inverse function required for the approximation
|
||||
inline double evaluateInverseFunc(double x) const{
|
||||
return 1.0/(double)func((bigfloat)x);
|
||||
}
|
||||
|
||||
// Dump csv of function, approx and error
|
||||
void csv(std::ostream &os = std::cout) const;
|
||||
|
||||
// Get the coefficient of the term x^i in the numerator
|
||||
inline double getCoeffNum(const int i) const{
|
||||
return num_pows[i] == -1 ? 0. : double(param[num_pows[i]]);
|
||||
}
|
||||
// Get the coefficient of the term x^i in the denominator
|
||||
inline double getCoeffDen(const int i) const{
|
||||
if(i == pow_d) return 1.0;
|
||||
else return den_pows[i] == -1 ? 0. : double(param[den_pows[i]+n+1]);
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,183 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/approx/ZMobius.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#include <Grid/algorithms/approx/ZMobius.h>
|
||||
#include <Grid/algorithms/approx/RemezGeneral.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
NAMESPACE_BEGIN(Approx);
|
||||
|
||||
//Compute the tanh approximation
|
||||
inline double epsilonMobius(const double x, const std::vector<ComplexD> &w){
|
||||
int Ls = w.size();
|
||||
|
||||
ComplexD fxp = 1., fmp = 1.;
|
||||
for(int i=0;i<Ls;i++){
|
||||
fxp = fxp * ( w[i] + x );
|
||||
fmp = fmp * ( w[i] - x );
|
||||
}
|
||||
return ((fxp - fmp)/(fxp + fmp)).real();
|
||||
}
|
||||
inline double epsilonMobius(const double x, const std::vector<RealD> &w){
|
||||
int Ls = w.size();
|
||||
|
||||
double fxp = 1., fmp = 1.;
|
||||
for(int i=0;i<Ls;i++){
|
||||
fxp = fxp * ( w[i] + x );
|
||||
fmp = fmp * ( w[i] - x );
|
||||
}
|
||||
return (fxp - fmp)/(fxp + fmp);
|
||||
}
|
||||
|
||||
|
||||
|
||||
//Compute the tanh approximation in a form suitable for the Remez
|
||||
bigfloat epsilonMobius(bigfloat x, void* data){
|
||||
const std::vector<RealD> &omega = *( (std::vector<RealD> const*)data );
|
||||
bigfloat fxp(1.0);
|
||||
bigfloat fmp(1.0);
|
||||
|
||||
for(int i=0;i<omega.size();i++){
|
||||
fxp = fxp * ( bigfloat(omega[i]) + x);
|
||||
fmp = fmp * ( bigfloat(omega[i]) - x);
|
||||
}
|
||||
return (fxp - fmp)/(fxp + fmp);
|
||||
}
|
||||
|
||||
//Compute the Zmobius Omega parameters suitable for eigenvalue range -lambda_bound <= lambda <= lambda_bound
|
||||
//Note omega_i = 1/(b_i + c_i) where b_i and c_i are the Mobius parameters
|
||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out,
|
||||
const std::vector<RealD> &omega_in, const int Ls_in,
|
||||
const RealD lambda_bound){
|
||||
assert(omega_in.size() == Ls_in);
|
||||
omega_out.resize(Ls_out);
|
||||
|
||||
//Use the Remez algorithm to generate the appropriate rational polynomial
|
||||
//For odd polynomial, to satisfy Haar condition must take either positive or negative half of range (cf https://arxiv.org/pdf/0803.0439.pdf page 6)
|
||||
AlgRemezGeneral remez(0, lambda_bound, 64, &epsilonMobius, (void*)&omega_in);
|
||||
remez.generateApprox(Ls_out-1, Ls_out,AlgRemezGeneral::Odd, AlgRemezGeneral::Even, 1e-15, 100);
|
||||
remez.csv(std::cout);
|
||||
|
||||
//The rational approximation has the form [ f(x) - f(-x) ] / [ f(x) + f(-x) ] where f(x) = \Prod_{i=0}^{L_s-1} ( \omega_i + x )
|
||||
//cf https://academiccommons.columbia.edu/doi/10.7916/D8T72HD7 pg 102
|
||||
//omega_i are therefore the negative of the complex roots of f(x)
|
||||
|
||||
//We can find the roots by recognizing that the eigenvalues of a matrix A are the roots of the characteristic polynomial
|
||||
// \rho(\lambda) = det( A - \lambda I ) where I is the unit matrix
|
||||
//The matrix whose characteristic polynomial is an arbitrary monic polynomial a0 + a1 x + a2 x^2 + ... x^n is the companion matrix
|
||||
// A = | 0 1 0 0 0 .... 0 |
|
||||
// | 0 0 1 0 0 .... 0 |
|
||||
// | : : : : : : |
|
||||
// | 0 0 0 0 0 1
|
||||
// | -a0 -a1 -a2 ... ... -an|
|
||||
|
||||
|
||||
//Note the Remez defines the largest power to have unit coefficient
|
||||
std::vector<RealD> coeffs(Ls_out+1);
|
||||
for(int i=0;i<Ls_out+1;i+=2) coeffs[i] = coeffs[i] = remez.getCoeffDen(i); //even powers
|
||||
for(int i=1;i<Ls_out+1;i+=2) coeffs[i] = coeffs[i] = remez.getCoeffNum(i); //odd powers
|
||||
|
||||
std::vector<std::complex<RealD> > roots(Ls_out);
|
||||
|
||||
//Form the companion matrix
|
||||
Eigen::MatrixXd compn(Ls_out,Ls_out);
|
||||
for(int i=0;i<Ls_out-1;i++) compn(i,0) = 0.;
|
||||
compn(Ls_out - 1, 0) = -coeffs[0];
|
||||
|
||||
for(int j=1;j<Ls_out;j++){
|
||||
for(int i=0;i<Ls_out-1;i++) compn(i,j) = i == j-1 ? 1. : 0.;
|
||||
compn(Ls_out - 1, j) = -coeffs[j];
|
||||
}
|
||||
|
||||
//Eigensolve
|
||||
Eigen::EigenSolver<Eigen::MatrixXd> slv(compn, false);
|
||||
|
||||
const auto & ev = slv.eigenvalues();
|
||||
for(int i=0;i<Ls_out;i++)
|
||||
omega_out[i] = -ev(i);
|
||||
|
||||
//Sort ascending (smallest at start of vector!)
|
||||
std::sort(omega_out.begin(), omega_out.end(),
|
||||
[&](const ComplexD &a, const ComplexD &b){ return a.real() < b.real() || (a.real() == b.real() && a.imag() < b.imag()); });
|
||||
|
||||
//McGlynn thesis pg 122 suggest improved iteration counts if magnitude of omega diminishes towards the center of the 5th dimension
|
||||
std::vector<ComplexD> omega_tmp = omega_out;
|
||||
int s_low=0, s_high=Ls_out-1, ss=0;
|
||||
for(int s_from = Ls_out-1; s_from >= 0; s_from--){ //loop from largest omega
|
||||
int s_to;
|
||||
if(ss % 2 == 0){
|
||||
s_to = s_low++;
|
||||
}else{
|
||||
s_to = s_high--;
|
||||
}
|
||||
omega_out[s_to] = omega_tmp[s_from];
|
||||
++ss;
|
||||
}
|
||||
|
||||
std::cout << "Resulting omega_i:" << std::endl;
|
||||
for(int i=0;i<Ls_out;i++)
|
||||
std::cout << omega_out[i] << std::endl;
|
||||
|
||||
std::cout << "Test result matches the approximate polynomial found by the Remez" << std::endl;
|
||||
std::cout << "<x> <remez approx> <poly approx> <diff poly approx remez approx> <exact> <diff poly approx exact>\n";
|
||||
|
||||
int npt = 60;
|
||||
double dlt = lambda_bound/double(npt-1);
|
||||
|
||||
for (int i =0; i<npt; i++){
|
||||
double x = i*dlt;
|
||||
double r = remez.evaluateApprox(x);
|
||||
double p = epsilonMobius(x, omega_out);
|
||||
double e = epsilonMobius(x, omega_in);
|
||||
|
||||
std::cout << x<< " " << r << " " << p <<" " <<r-p << " " << e << " " << e-p << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//mobius_param = b+c with b-c=1
|
||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound){
|
||||
std::vector<RealD> omega_in(Ls_in, 1./mobius_param);
|
||||
computeZmobiusOmega(omega_out, Ls_out, omega_in, Ls_in, lambda_bound);
|
||||
}
|
||||
|
||||
//ZMobius class takes gamma_i = (b+c) omega_i as its input, where b, c are factored out
|
||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out,
|
||||
const RealD mobius_param_out, const int Ls_out,
|
||||
const RealD mobius_param_in, const int Ls_in,
|
||||
const RealD lambda_bound){
|
||||
computeZmobiusOmega(gamma_out, Ls_out, mobius_param_in, Ls_in, lambda_bound);
|
||||
for(int i=0;i<Ls_out;i++) gamma_out[i] = gamma_out[i] * mobius_param_out;
|
||||
}
|
||||
//Assumes mobius_param_out == mobius_param_in
|
||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound){
|
||||
computeZmobiusGamma(gamma_out, mobius_param, Ls_out, mobius_param, Ls_in, lambda_bound);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Approx);
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,57 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/approx/ZMobius.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_ZMOBIUS_APPROX_H
|
||||
#define GRID_ZMOBIUS_APPROX_H
|
||||
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
NAMESPACE_BEGIN(Approx);
|
||||
|
||||
//Compute the Zmobius Omega parameters suitable for eigenvalue range -lambda_bound <= lambda <= lambda_bound
|
||||
//Note omega_i = 1/(b_i + c_i) where b_i and c_i are the Mobius parameters
|
||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out,
|
||||
const std::vector<RealD> &omega_in, const int Ls_in,
|
||||
const RealD lambda_bound);
|
||||
|
||||
//mobius_param = b+c with b-c=1
|
||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound);
|
||||
|
||||
//ZMobius class takes gamma_i = (b+c) omega_i as its input, where b, c are factored out
|
||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out,
|
||||
const RealD mobius_param_out, const int Ls_out,
|
||||
const RealD mobius_param_in, const int Ls_in,
|
||||
const RealD lambda_bound);
|
||||
|
||||
//Assumes mobius_param_out == mobius_param_in
|
||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound);
|
||||
|
||||
NAMESPACE_END(Approx);
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,730 +0,0 @@
|
||||
/* -*- Mode: C; comment-column: 22; fill-column: 79; compile-command: "gcc -o zolotarev zolotarev.c -ansi -pedantic -lm -DTEST"; -*- */
|
||||
#define VERSION Source Time-stamp: <2015-05-18 16:32:08 neo>
|
||||
|
||||
/* These C routines evalute the optimal rational approximation to the signum
|
||||
* function for epsilon < |x| < 1 using Zolotarev's theorem.
|
||||
*
|
||||
* To obtain reliable results for high degree approximations (large n) it is
|
||||
* necessary to compute using sufficiently high precision arithmetic. To this
|
||||
* end the code has been parameterised to work with the preprocessor names
|
||||
* INTERNAL_PRECISION and PRECISION set to float, double, or long double as
|
||||
* appropriate. INTERNAL_PRECISION is used in computing the Zolotarev
|
||||
* coefficients, which are converted to PRECISION before being returned to the
|
||||
* caller. Presumably even higher precision could be obtained using GMP or
|
||||
* similar package, but bear in mind that rounding errors might also be
|
||||
* significant in evaluating the resulting polynomial. The convergence criteria
|
||||
* have been written in a precision-independent form. */
|
||||
|
||||
#include <math.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#define MAX(a,b) ((a) > (b) ? (a) : (b))
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
#ifndef INTERNAL_PRECISION
|
||||
#define INTERNAL_PRECISION double
|
||||
#endif
|
||||
|
||||
#include "Zolotarev.h"
|
||||
#define ZOLOTAREV_INTERNAL
|
||||
#undef ZOLOTAREV_DATA
|
||||
#define ZOLOTAREV_DATA izd
|
||||
#undef ZPRECISION
|
||||
#define ZPRECISION INTERNAL_PRECISION
|
||||
#include "Zolotarev.h"
|
||||
#undef ZOLOTAREV_INTERNAL
|
||||
|
||||
/* The ANSI standard appears not to know what pi is */
|
||||
|
||||
#ifndef M_PI
|
||||
#define M_PI ((INTERNAL_PRECISION) 3.141592653589793238462643383279502884197\
|
||||
169399375105820974944592307816406286208998628034825342117068)
|
||||
#endif
|
||||
|
||||
#define ZERO ((INTERNAL_PRECISION) 0)
|
||||
#define ONE ((INTERNAL_PRECISION) 1)
|
||||
#define TWO ((INTERNAL_PRECISION) 2)
|
||||
#define THREE ((INTERNAL_PRECISION) 3)
|
||||
#define FOUR ((INTERNAL_PRECISION) 4)
|
||||
#define HALF (ONE/TWO)
|
||||
|
||||
/* The following obscenity seems to be the simplest (?) way to coerce the C
|
||||
* preprocessor to convert the value of a preprocessor token into a string. */
|
||||
|
||||
#define PP2(x) #x
|
||||
#define PP1(a,b,c) a ## b(c)
|
||||
#define STRINGIFY(name) PP1(PP,2,name)
|
||||
|
||||
/* Compute the partial fraction expansion coefficients (alpha) from the
|
||||
* factored form */
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
NAMESPACE_BEGIN(Approx);
|
||||
|
||||
static void construct_partfrac(izd *z) {
|
||||
int dn = z -> dn, dd = z -> dd, type = z -> type;
|
||||
int j, k, da = dd + 1 + type;
|
||||
INTERNAL_PRECISION A = z -> A, *a = z -> a, *ap = z -> ap, *alpha;
|
||||
alpha = (INTERNAL_PRECISION*) malloc(da * sizeof(INTERNAL_PRECISION));
|
||||
for (j = 0; j < dd; j++)
|
||||
for (k = 0, alpha[j] = A; k < dd; k++)
|
||||
alpha[j] *=
|
||||
(k < dn ? ap[j] - a[k] : ONE) / (k == j ? ONE : ap[j] - ap[k]);
|
||||
if(type == 1) /* implicit pole at zero? */
|
||||
for (k = 0, alpha[dd] = A * (dn > dd ? - a[dd] : ONE); k < dd; k++) {
|
||||
alpha[dd] *= a[k] / ap[k];
|
||||
alpha[k] *= (dn > dd ? ap[k] - a[dd] : ONE) / ap[k];
|
||||
}
|
||||
alpha[da-1] = dn == da - 1 ? A : ZERO;
|
||||
z -> alpha = alpha;
|
||||
z -> da = da;
|
||||
return;
|
||||
}
|
||||
|
||||
/* Convert factored polynomial into dense polynomial. The input is the overall
|
||||
* factor A and the roots a[i], such that p = A product(x - a[i], i = 1..d) */
|
||||
|
||||
static INTERNAL_PRECISION *poly_factored_to_dense(INTERNAL_PRECISION A,
|
||||
INTERNAL_PRECISION *a,
|
||||
int d) {
|
||||
INTERNAL_PRECISION *p;
|
||||
int i, j;
|
||||
p = (INTERNAL_PRECISION *) malloc((d + 2) * sizeof(INTERNAL_PRECISION));
|
||||
p[0] = A;
|
||||
for (i = 0; i < d; i++) {
|
||||
p[i+1] = p[i];
|
||||
for (j = i; j > 0; j--) p[j] = p[j-1] - a[i]*p[j];
|
||||
p[0] *= - a[i];
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Convert a rational function of the form R0(x) = x p(x^2)/q(x^2) (type 0) or
|
||||
* R1(x) = p(x^2)/[x q(x^2)] (type 1) into its continued fraction
|
||||
* representation. We assume that 0 <= deg(q) - deg(p) <= 1 for type 0 and 0 <=
|
||||
* deg(p) - deg(q) <= 1 for type 1. On input p and q are in factored form, and
|
||||
* deg(q) = dq, deg(p) = dp. The output is the continued fraction coefficients
|
||||
* beta, where R(x) = beta[0] x + 1/(beta[1] x + 1/(...)).
|
||||
*
|
||||
* The method used is as follows. There are four cases to consider:
|
||||
*
|
||||
* 0.i. Type 0, deg p = deg q
|
||||
*
|
||||
* 0.ii. Type 0, deg p = deg q - 1
|
||||
*
|
||||
* 1.i. Type 1, deg p = deg q
|
||||
*
|
||||
* 1.ii. Type 1, deg p = deg q + 1
|
||||
*
|
||||
* and these are connected by two transformations:
|
||||
*
|
||||
* A. To obtain a continued fraction expansion of type 1 we use a single-step
|
||||
* polynomial division we find beta and r(x) such that p(x) = beta x q(x) +
|
||||
* r(x), with deg(r) = deg(q). This implies that p(x^2) = beta x^2 q(x^2) +
|
||||
* r(x^2), and thus R1(x) = x beta + r(x^2)/(x q(x^2)) = x beta + 1/R0(x)
|
||||
* with R0(x) = x q(x^2)/r(x^2).
|
||||
*
|
||||
* B. A continued fraction expansion of type 0 is obtained in a similar, but
|
||||
* not identical, manner. We use the polynomial division algorithm to compute
|
||||
* the quotient beta and the remainder r that satisfy p(x) = beta q(x) + r(x)
|
||||
* with deg(r) = deg(q) - 1. We thus have x p(x^2) = x beta q(x^2) + x r(x^2),
|
||||
* so R0(x) = x beta + x r(x^2)/q(x^2) = x beta + 1/R1(x) with R1(x) = q(x^2) /
|
||||
* (x r(x^2)).
|
||||
*
|
||||
* Note that the deg(r) must be exactly deg(q) for (A) and deg(q) - 1 for (B)
|
||||
* because p and q have disjoint roots all of multiplicity 1. This means that
|
||||
* the division algorithm requires only a single polynomial subtraction step.
|
||||
*
|
||||
* The transformations between the cases form the following finite state
|
||||
* automaton:
|
||||
*
|
||||
* +------+ +------+ +------+ +------+
|
||||
* | | | | ---(A)---> | | | |
|
||||
* | 0.ii | ---(B)---> | 1.ii | | 0.i | <---(A)--- | 1.i |
|
||||
* | | | | <---(B)--- | | | |
|
||||
* +------+ +------+ +------+ +------+
|
||||
*/
|
||||
|
||||
static INTERNAL_PRECISION *contfrac_A(INTERNAL_PRECISION *,
|
||||
INTERNAL_PRECISION *,
|
||||
INTERNAL_PRECISION *,
|
||||
INTERNAL_PRECISION *, int, int);
|
||||
|
||||
static INTERNAL_PRECISION *contfrac_B(INTERNAL_PRECISION *,
|
||||
INTERNAL_PRECISION *,
|
||||
INTERNAL_PRECISION *,
|
||||
INTERNAL_PRECISION *, int, int);
|
||||
|
||||
static void construct_contfrac(izd *z){
|
||||
INTERNAL_PRECISION *r, A = z -> A, *p = z -> a, *q = z -> ap;
|
||||
int dp = z -> dn, dq = z -> dd, type = z -> type;
|
||||
|
||||
z -> db = 2 * dq + 1 + type;
|
||||
z -> beta = (INTERNAL_PRECISION *)
|
||||
malloc(z -> db * sizeof(INTERNAL_PRECISION));
|
||||
p = poly_factored_to_dense(A, p, dp);
|
||||
q = poly_factored_to_dense(ONE, q, dq);
|
||||
r = (INTERNAL_PRECISION *) malloc((MAX(dp,dq) + 1) *
|
||||
sizeof(INTERNAL_PRECISION));
|
||||
if (type == 0) (void) contfrac_B(z -> beta, p, q, r, dp, dq);
|
||||
else (void) contfrac_A(z -> beta, p, q, r, dp, dq);
|
||||
free(p); free(q); free(r);
|
||||
return;
|
||||
}
|
||||
|
||||
static INTERNAL_PRECISION *contfrac_A(INTERNAL_PRECISION *beta,
|
||||
INTERNAL_PRECISION *p,
|
||||
INTERNAL_PRECISION *q,
|
||||
INTERNAL_PRECISION *r, int dp, int dq) {
|
||||
INTERNAL_PRECISION quot, *rb;
|
||||
int j;
|
||||
|
||||
/* p(x) = x beta q(x) + r(x); dp = dq or dp = dq + 1 */
|
||||
|
||||
quot = dp == dq ? ZERO : p[dp] / q[dq];
|
||||
r[0] = p[0];
|
||||
for (j = 1; j <= dp; j++) r[j] = p[j] - quot * q[j-1];
|
||||
#ifdef DEBUG
|
||||
printf("%s: Continued Fraction form: deg p = %2d, deg q = %2d, beta = %g\n",
|
||||
__FUNCTION__, dp, dq, (float) quot);
|
||||
for (j = 0; j <= dq + 1; j++)
|
||||
printf("\tp[%2d] = %14.6g\tq[%2d] = %14.6g\tr[%2d] = %14.6g\n",
|
||||
j, (float) (j > dp ? ZERO : p[j]),
|
||||
j, (float) (j == 0 ? ZERO : q[j-1]),
|
||||
j, (float) (j == dp ? ZERO : r[j]));
|
||||
#endif /* DEBUG */
|
||||
*(rb = contfrac_B(beta, q, r, p, dq, dq)) = quot;
|
||||
return rb + 1;
|
||||
}
|
||||
|
||||
static INTERNAL_PRECISION *contfrac_B(INTERNAL_PRECISION *beta,
|
||||
INTERNAL_PRECISION *p,
|
||||
INTERNAL_PRECISION *q,
|
||||
INTERNAL_PRECISION *r, int dp, int dq) {
|
||||
INTERNAL_PRECISION quot, *rb;
|
||||
int j;
|
||||
|
||||
/* p(x) = beta q(x) + r(x); dp = dq or dp = dq - 1 */
|
||||
|
||||
quot = dp == dq ? p[dp] / q[dq] : ZERO;
|
||||
for (j = 0; j < dq; j++) r[j] = p[j] - quot * q[j];
|
||||
#ifdef DEBUG
|
||||
printf("%s: Continued Fraction form: deg p = %2d, deg q = %2d, beta = %g\n",
|
||||
__FUNCTION__, dp, dq, (float) quot);
|
||||
for (j = 0; j <= dq; j++)
|
||||
printf("\tp[%2d] = %14.6g\tq[%2d] = %14.6g\tr[%2d] = %14.6g\n",
|
||||
j, (float) (j > dp ? ZERO : p[j]),
|
||||
j, (float) q[j],
|
||||
j, (float) (j == dq ? ZERO : r[j]));
|
||||
#endif /* DEBUG */
|
||||
*(rb = dq > 0 ? contfrac_A(beta, q, r, p, dq, dq-1) : beta) = quot;
|
||||
return rb + 1;
|
||||
}
|
||||
|
||||
/* The global variable U is used to hold the argument u throughout the AGM
|
||||
* recursion. The global variables F and K are set in the innermost
|
||||
* instantiation of the recursive function AGM to the values of the elliptic
|
||||
* integrals F(u,k) and K(k) respectively. They must be made thread local to
|
||||
* make this code thread-safe in a multithreaded environment. */
|
||||
|
||||
static INTERNAL_PRECISION U, F, K; /* THREAD LOCAL */
|
||||
|
||||
/* Recursive implementation of Gauss' arithmetico-geometric mean, which is the
|
||||
* kernel of the method used to compute the Jacobian elliptic functions
|
||||
* sn(u,k), cn(u,k), and dn(u,k) with parameter k (where 0 < k < 1), as well
|
||||
* as the elliptic integral F(s,k) satisfying F(sn(u,k)) = u and the complete
|
||||
* elliptic integral K(k).
|
||||
*
|
||||
* The algorithm used is a recursive implementation of the Gauss (Landen)
|
||||
* transformation.
|
||||
*
|
||||
* The function returns the value of sn(u,k'), where k' is the dual parameter,
|
||||
* and also sets the values of the global variables F and K. The latter is
|
||||
* used to determine the sign of cn(u,k').
|
||||
*
|
||||
* The algorithm is deemed to have converged when b ceases to increase. This
|
||||
* works whatever INTERNAL_PRECISION is specified. */
|
||||
|
||||
static INTERNAL_PRECISION AGM(INTERNAL_PRECISION a,
|
||||
INTERNAL_PRECISION b,
|
||||
INTERNAL_PRECISION s) {
|
||||
static INTERNAL_PRECISION pb = -ONE;
|
||||
INTERNAL_PRECISION c, d, xi;
|
||||
|
||||
if (b <= pb) {
|
||||
pb = -ONE;
|
||||
F = asin(s) / a; /* Here, a is the AGM */
|
||||
K = M_PI / (TWO * a);
|
||||
return sin(U * a);
|
||||
}
|
||||
pb = b;
|
||||
c = a - b;
|
||||
d = a + b;
|
||||
xi = AGM(HALF*d, sqrt(a*b), ONE + c*c == ONE ?
|
||||
HALF*s*d/a : (a - sqrt(a*a - s*s*c*d))/(c*s));
|
||||
return 2*a*xi / (d + c*xi*xi);
|
||||
}
|
||||
|
||||
/* Computes sn(u,k), cn(u,k), dn(u,k), F(u,k), and K(k). It is essentially a
|
||||
* wrapper for the routine AGM. The sign of cn(u,k) is defined to be -1 if
|
||||
* K(k) < u < 3*K(k) and +1 otherwise, and thus sign is computed by some quite
|
||||
* unnecessarily obfuscated bit manipulations. */
|
||||
|
||||
static void sncndnFK(INTERNAL_PRECISION u, INTERNAL_PRECISION k,
|
||||
INTERNAL_PRECISION* sn, INTERNAL_PRECISION* cn,
|
||||
INTERNAL_PRECISION* dn, INTERNAL_PRECISION* elF,
|
||||
INTERNAL_PRECISION* elK) {
|
||||
int sgn;
|
||||
U = u;
|
||||
*sn = AGM(ONE, sqrt(ONE - k*k), u);
|
||||
sgn = ((int) (fabs(u) / K)) % 4; /* sgn = 0, 1, 2, 3 */
|
||||
sgn ^= sgn >> 1; /* (sgn & 1) = 0, 1, 1, 0 */
|
||||
sgn = 1 - ((sgn & 1) << 1); /* sgn = 1, -1, -1, 1 */
|
||||
*cn = ((INTERNAL_PRECISION) sgn) * sqrt(ONE - *sn * *sn);
|
||||
*dn = sqrt(ONE - k*k* *sn * *sn);
|
||||
*elF = F;
|
||||
*elK = K;
|
||||
}
|
||||
|
||||
/* Compute the coefficients for the optimal rational approximation R(x) to
|
||||
* sgn(x) of degree n over the interval epsilon < |x| < 1 using Zolotarev's
|
||||
* formula.
|
||||
*
|
||||
* Set type = 0 for the Zolotarev approximation, which is zero at x = 0, and
|
||||
* type = 1 for the approximation which is infinite at x = 0. */
|
||||
|
||||
zolotarev_data* zolotarev(PRECISION epsilon, int n, int type) {
|
||||
INTERNAL_PRECISION A, c, cp, kp, ksq, sn, cn, dn, Kp, Kj, z, z0, t, M, F,
|
||||
l, invlambda, xi, xisq, *tv, s, opl;
|
||||
int m, czero, ts;
|
||||
zolotarev_data *zd;
|
||||
izd *d = (izd*) malloc(sizeof(izd));
|
||||
|
||||
d -> type = type;
|
||||
d -> epsilon = (INTERNAL_PRECISION) epsilon;
|
||||
d -> n = n;
|
||||
d -> dd = n / 2;
|
||||
d -> dn = d -> dd - 1 + n % 2; /* n even: dn = dd - 1, n odd: dn = dd */
|
||||
d -> deg_denom = 2 * d -> dd;
|
||||
d -> deg_num = 2 * d -> dn + 1;
|
||||
|
||||
d -> a = (INTERNAL_PRECISION*) malloc((1 + d -> dn) *
|
||||
sizeof(INTERNAL_PRECISION));
|
||||
d -> ap = (INTERNAL_PRECISION*) malloc(d -> dd *
|
||||
sizeof(INTERNAL_PRECISION));
|
||||
ksq = d -> epsilon * d -> epsilon;
|
||||
kp = sqrt(ONE - ksq);
|
||||
sncndnFK(ZERO, kp, &sn, &cn, &dn, &F, &Kp); /* compute Kp = K(kp) */
|
||||
z0 = TWO * Kp / (INTERNAL_PRECISION) n;
|
||||
M = ONE;
|
||||
A = ONE / d -> epsilon;
|
||||
|
||||
sncndnFK(HALF * z0, kp, &sn, &cn, &dn, &F, &Kj); /* compute xi */
|
||||
xi = ONE / dn;
|
||||
xisq = xi * xi;
|
||||
invlambda = xi;
|
||||
|
||||
for (m = 0; m < d -> dd; m++) {
|
||||
czero = 2 * (m + 1) == n; /* n even and m = dd -1 */
|
||||
|
||||
z = z0 * ((INTERNAL_PRECISION) m + ONE);
|
||||
sncndnFK(z, kp, &sn, &cn, &dn, &F, &Kj);
|
||||
t = cn / sn;
|
||||
c = - t*t;
|
||||
if (!czero) (d -> a)[d -> dn - 1 - m] = ksq / c;
|
||||
|
||||
z = z0 * ((INTERNAL_PRECISION) m + HALF);
|
||||
sncndnFK(z, kp, &sn, &cn, &dn, &F, &Kj);
|
||||
t = cn / sn;
|
||||
cp = - t*t;
|
||||
(d -> ap)[d -> dd - 1 - m] = ksq / cp;
|
||||
|
||||
M *= (ONE - c) / (ONE - cp);
|
||||
A *= (czero ? -ksq : c) * (ONE - cp) / (cp * (ONE - c));
|
||||
invlambda *= (ONE - c*xisq) / (ONE - cp*xisq);
|
||||
}
|
||||
invlambda /= M;
|
||||
d -> A = TWO / (ONE + invlambda) * A;
|
||||
d -> Delta = (invlambda - ONE) / (invlambda + ONE);
|
||||
|
||||
d -> gamma = (INTERNAL_PRECISION*) malloc((1 + d -> n) *
|
||||
sizeof(INTERNAL_PRECISION));
|
||||
l = ONE / invlambda;
|
||||
opl = ONE + l;
|
||||
sncndnFK(sqrt( d -> type == 1
|
||||
? (THREE + l) / (FOUR * opl)
|
||||
: (ONE + THREE*l) / (opl*opl*opl)
|
||||
), sqrt(ONE - l*l), &sn, &cn, &dn, &F, &Kj);
|
||||
s = M * F;
|
||||
for (m = 0; m < d -> n; m++) {
|
||||
sncndnFK(s + TWO*Kp*m/n, kp, &sn, &cn, &dn, &F, &Kj);
|
||||
d -> gamma[m] = d -> epsilon / dn;
|
||||
}
|
||||
|
||||
/* If R(x) is a Zolotarev rational approximation of degree (n,m) with maximum
|
||||
* error Delta, then (1 - Delta^2) / R(x) is also an optimal Chebyshev
|
||||
* approximation of degree (m,n) */
|
||||
|
||||
if (d -> type == 1) {
|
||||
d -> A = (ONE - d -> Delta * d -> Delta) / d -> A;
|
||||
tv = d -> a; d -> a = d -> ap; d -> ap = tv;
|
||||
ts = d -> dn; d -> dn = d -> dd; d -> dd = ts;
|
||||
ts = d -> deg_num; d -> deg_num = d -> deg_denom; d -> deg_denom = ts;
|
||||
}
|
||||
|
||||
construct_partfrac(d);
|
||||
construct_contfrac(d);
|
||||
|
||||
/* Converting everything to PRECISION for external use only */
|
||||
|
||||
zd = (zolotarev_data*) malloc(sizeof(zolotarev_data));
|
||||
zd -> A = (PRECISION) d -> A;
|
||||
zd -> Delta = (PRECISION) d -> Delta;
|
||||
zd -> epsilon = (PRECISION) d -> epsilon;
|
||||
zd -> n = d -> n;
|
||||
zd -> type = d -> type;
|
||||
zd -> dn = d -> dn;
|
||||
zd -> dd = d -> dd;
|
||||
zd -> da = d -> da;
|
||||
zd -> db = d -> db;
|
||||
zd -> deg_num = d -> deg_num;
|
||||
zd -> deg_denom = d -> deg_denom;
|
||||
|
||||
zd -> a = (PRECISION*) malloc(zd -> dn * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dn; m++) zd -> a[m] = (PRECISION) d -> a[m];
|
||||
free(d -> a);
|
||||
|
||||
zd -> ap = (PRECISION*) malloc(zd -> dd * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dd; m++) zd -> ap[m] = (PRECISION) d -> ap[m];
|
||||
free(d -> ap);
|
||||
|
||||
zd -> alpha = (PRECISION*) malloc(zd -> da * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> da; m++) zd -> alpha[m] = (PRECISION) d -> alpha[m];
|
||||
free(d -> alpha);
|
||||
|
||||
zd -> beta = (PRECISION*) malloc(zd -> db * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> db; m++) zd -> beta[m] = (PRECISION) d -> beta[m];
|
||||
free(d -> beta);
|
||||
|
||||
zd -> gamma = (PRECISION*) malloc(zd -> n * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> n; m++) zd -> gamma[m] = (PRECISION) d -> gamma[m];
|
||||
free(d -> gamma);
|
||||
|
||||
free(d);
|
||||
return zd;
|
||||
}
|
||||
|
||||
|
||||
void zolotarev_free(zolotarev_data *zdata)
|
||||
{
|
||||
free(zdata -> a);
|
||||
free(zdata -> ap);
|
||||
free(zdata -> alpha);
|
||||
free(zdata -> beta);
|
||||
free(zdata -> gamma);
|
||||
free(zdata);
|
||||
}
|
||||
|
||||
|
||||
zolotarev_data* higham(PRECISION epsilon, int n) {
|
||||
INTERNAL_PRECISION A, M, c, cp, z, z0, t, epssq;
|
||||
int m, czero;
|
||||
zolotarev_data *zd;
|
||||
izd *d = (izd*) malloc(sizeof(izd));
|
||||
|
||||
d -> type = 0;
|
||||
d -> epsilon = (INTERNAL_PRECISION) epsilon;
|
||||
d -> n = n;
|
||||
d -> dd = n / 2;
|
||||
d -> dn = d -> dd - 1 + n % 2; /* n even: dn = dd - 1, n odd: dn = dd */
|
||||
d -> deg_denom = 2 * d -> dd;
|
||||
d -> deg_num = 2 * d -> dn + 1;
|
||||
|
||||
d -> a = (INTERNAL_PRECISION*) malloc((1 + d -> dn) *
|
||||
sizeof(INTERNAL_PRECISION));
|
||||
d -> ap = (INTERNAL_PRECISION*) malloc(d -> dd *
|
||||
sizeof(INTERNAL_PRECISION));
|
||||
A = (INTERNAL_PRECISION) n;
|
||||
z0 = M_PI / A;
|
||||
A = n % 2 == 0 ? A : ONE / A;
|
||||
M = d -> epsilon * A;
|
||||
epssq = d -> epsilon * d -> epsilon;
|
||||
|
||||
for (m = 0; m < d -> dd; m++) {
|
||||
czero = 2 * (m + 1) == n; /* n even and m = dd - 1*/
|
||||
|
||||
if (!czero) {
|
||||
z = z0 * ((INTERNAL_PRECISION) m + ONE);
|
||||
t = tan(z);
|
||||
c = - t*t;
|
||||
(d -> a)[d -> dn - 1 - m] = c;
|
||||
M *= epssq - c;
|
||||
}
|
||||
|
||||
z = z0 * ((INTERNAL_PRECISION) m + HALF);
|
||||
t = tan(z);
|
||||
cp = - t*t;
|
||||
(d -> ap)[d -> dd - 1 - m] = cp;
|
||||
M /= epssq - cp;
|
||||
}
|
||||
|
||||
d -> gamma = (INTERNAL_PRECISION*) malloc((1 + d -> n) *
|
||||
sizeof(INTERNAL_PRECISION));
|
||||
for (m = 0; m < d -> n; m++) d -> gamma[m] = ONE;
|
||||
|
||||
d -> A = A;
|
||||
d -> Delta = ONE - M;
|
||||
|
||||
construct_partfrac(d);
|
||||
construct_contfrac(d);
|
||||
|
||||
/* Converting everything to PRECISION for external use only */
|
||||
|
||||
zd = (zolotarev_data*) malloc(sizeof(zolotarev_data));
|
||||
zd -> A = (PRECISION) d -> A;
|
||||
zd -> Delta = (PRECISION) d -> Delta;
|
||||
zd -> epsilon = (PRECISION) d -> epsilon;
|
||||
zd -> n = d -> n;
|
||||
zd -> type = d -> type;
|
||||
zd -> dn = d -> dn;
|
||||
zd -> dd = d -> dd;
|
||||
zd -> da = d -> da;
|
||||
zd -> db = d -> db;
|
||||
zd -> deg_num = d -> deg_num;
|
||||
zd -> deg_denom = d -> deg_denom;
|
||||
|
||||
zd -> a = (PRECISION*) malloc(zd -> dn * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dn; m++) zd -> a[m] = (PRECISION) d -> a[m];
|
||||
free(d -> a);
|
||||
|
||||
zd -> ap = (PRECISION*) malloc(zd -> dd * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dd; m++) zd -> ap[m] = (PRECISION) d -> ap[m];
|
||||
free(d -> ap);
|
||||
|
||||
zd -> alpha = (PRECISION*) malloc(zd -> da * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> da; m++) zd -> alpha[m] = (PRECISION) d -> alpha[m];
|
||||
free(d -> alpha);
|
||||
|
||||
zd -> beta = (PRECISION*) malloc(zd -> db * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> db; m++) zd -> beta[m] = (PRECISION) d -> beta[m];
|
||||
free(d -> beta);
|
||||
|
||||
zd -> gamma = (PRECISION*) malloc(zd -> n * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> n; m++) zd -> gamma[m] = (PRECISION) d -> gamma[m];
|
||||
free(d -> gamma);
|
||||
|
||||
free(d);
|
||||
return zd;
|
||||
}
|
||||
|
||||
NAMESPACE_END(Approx);
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#ifdef TEST
|
||||
|
||||
#undef ZERO
|
||||
#define ZERO ((PRECISION) 0)
|
||||
#undef ONE
|
||||
#define ONE ((PRECISION) 1)
|
||||
#undef TWO
|
||||
#define TWO ((PRECISION) 2)
|
||||
|
||||
/* Evaluate the rational approximation R(x) using the factored form */
|
||||
|
||||
static PRECISION zolotarev_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
PRECISION R;
|
||||
|
||||
if (rdata -> type == 0) {
|
||||
R = rdata -> A * x;
|
||||
for (m = 0; m < rdata -> deg_denom/2; m++)
|
||||
R *= (2*(m+1) > rdata -> deg_num ? ONE : x*x - rdata -> a[m]) /
|
||||
(x*x - rdata -> ap[m]);
|
||||
} else {
|
||||
R = rdata -> A / x;
|
||||
for (m = 0; m < rdata -> deg_num/2; m++)
|
||||
R *= (x*x - rdata -> a[m]) /
|
||||
(2*(m+1) > rdata -> deg_denom ? ONE : x*x - rdata -> ap[m]);
|
||||
}
|
||||
return R;
|
||||
}
|
||||
|
||||
/* Evaluate the rational approximation R(x) using the partial fraction form */
|
||||
|
||||
static PRECISION zolotarev_partfrac_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
PRECISION R = rdata -> alpha[rdata -> da - 1];
|
||||
for (m = 0; m < rdata -> dd; m++)
|
||||
R += rdata -> alpha[m] / (x * x - rdata -> ap[m]);
|
||||
if (rdata -> type == 1) R += rdata -> alpha[rdata -> dd] / (x * x);
|
||||
return R * x;
|
||||
}
|
||||
|
||||
/* Evaluate the rational approximation R(x) using continued fraction form.
|
||||
*
|
||||
* If x = 0 and type = 1 then the result should be INF, whereas if x = 0 and
|
||||
* type = 0 then the result should be 0, but division by zero will occur at
|
||||
* intermediate stages of the evaluation. For IEEE implementations with
|
||||
* non-signalling overflow this will work correctly since 1/(1/0) = 1/INF = 0,
|
||||
* but with signalling overflow you will get an error message. */
|
||||
|
||||
static PRECISION zolotarev_contfrac_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
PRECISION R = rdata -> beta[0] * x;
|
||||
for (m = 1; m < rdata -> db; m++) R = rdata -> beta[m] * x + ONE / R;
|
||||
return R;
|
||||
}
|
||||
|
||||
/* Evaluate the rational approximation R(x) using Cayley form */
|
||||
|
||||
static PRECISION zolotarev_cayley_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
PRECISION T;
|
||||
|
||||
T = rdata -> type == 0 ? ONE : -ONE;
|
||||
for (m = 0; m < rdata -> n; m++)
|
||||
T *= (rdata -> gamma[m] - x) / (rdata -> gamma[m] + x);
|
||||
return (ONE - T) / (ONE + T);
|
||||
}
|
||||
|
||||
|
||||
/* Test program. Apart from printing out the parameters for R(x) it produces
|
||||
* the following data files for plotting (unless NPLOT is defined):
|
||||
*
|
||||
* zolotarev-fn is a plot of R(x) for |x| < 1.2. This should look like sgn(x).
|
||||
*
|
||||
* zolotarev-err is a plot of the error |R(x) - sgn(x)| scaled by 1/Delta. This
|
||||
* should oscillate deg_num + deg_denom + 2 times between +1 and -1 over the
|
||||
* domain epsilon <= |x| <= 1.
|
||||
*
|
||||
* If ALLPLOTS is defined then zolotarev-partfrac (zolotarev-contfrac) is a
|
||||
* plot of the difference between the values of R(x) computed using the
|
||||
* factored form and the partial fraction (continued fraction) form, scaled by
|
||||
* 1/Delta. It should be zero everywhere. */
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
int m, n, plotpts = 5000, type = 0;
|
||||
float eps, x, ypferr, ycferr, ycaylerr, maxypferr, maxycferr, maxycaylerr;
|
||||
zolotarev_data *rdata;
|
||||
PRECISION y;
|
||||
FILE *plot_function, *plot_error,
|
||||
*plot_partfrac, *plot_contfrac, *plot_cayley;
|
||||
|
||||
if (argc < 3 || argc > 4) {
|
||||
fprintf(stderr, "Usage: %s epsilon n [type]\n", *argv);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
sscanf(argv[1], "%g", &eps); /* First argument is epsilon */
|
||||
sscanf(argv[2], "%d", &n); /* Second argument is n */
|
||||
if (argc == 4) sscanf(argv[3], "%d", &type); /* Third argument is type */
|
||||
|
||||
if (type < 0 || type > 2) {
|
||||
fprintf(stderr, "%s: type must be 0 (Zolotarev R(0) = 0),\n"
|
||||
"\t\t1 (Zolotarev R(0) = Inf, or 2 (Higham)\n", *argv);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
rdata = type == 2
|
||||
? higham((PRECISION) eps, n)
|
||||
: zolotarev((PRECISION) eps, n, type);
|
||||
|
||||
printf("Zolotarev Test: R(epsilon = %g, n = %d, type = %d)\n\t"
|
||||
STRINGIFY(VERSION) "\n\t" STRINGIFY(HVERSION)
|
||||
"\n\tINTERNAL_PRECISION = " STRINGIFY(INTERNAL_PRECISION)
|
||||
"\tPRECISION = " STRINGIFY(PRECISION)
|
||||
"\n\n\tRational approximation of degree (%d,%d), %s at x = 0\n"
|
||||
"\tDelta = %g (maximum error)\n\n"
|
||||
"\tA = %g (overall factor)\n",
|
||||
(float) rdata -> epsilon, rdata -> n, type,
|
||||
rdata -> deg_num, rdata -> deg_denom,
|
||||
rdata -> type == 1 ? "infinite" : "zero",
|
||||
(float) rdata -> Delta, (float) rdata -> A);
|
||||
for (m = 0; m < MIN(rdata -> dd, rdata -> dn); m++)
|
||||
printf("\ta[%2d] = %14.8g\t\ta'[%2d] = %14.8g\n",
|
||||
m + 1, (float) rdata -> a[m], m + 1, (float) rdata -> ap[m]);
|
||||
if (rdata -> dd > rdata -> dn)
|
||||
printf("\t\t\t\t\ta'[%2d] = %14.8g\n",
|
||||
rdata -> dn + 1, (float) rdata -> ap[rdata -> dn]);
|
||||
if (rdata -> dd < rdata -> dn)
|
||||
printf("\ta[%2d] = %14.8g\n",
|
||||
rdata -> dd + 1, (float) rdata -> a[rdata -> dd]);
|
||||
|
||||
printf("\n\tPartial fraction coefficients\n");
|
||||
printf("\talpha[ 0] = %14.8g\n",
|
||||
(float) rdata -> alpha[rdata -> da - 1]);
|
||||
for (m = 0; m < rdata -> dd; m++)
|
||||
printf("\talpha[%2d] = %14.8g\ta'[%2d] = %14.8g\n",
|
||||
m + 1, (float) rdata -> alpha[m], m + 1, (float) rdata -> ap[m]);
|
||||
if (rdata -> type == 1)
|
||||
printf("\talpha[%2d] = %14.8g\ta'[%2d] = %14.8g\n",
|
||||
rdata -> dd + 1, (float) rdata -> alpha[rdata -> dd],
|
||||
rdata -> dd + 1, (float) ZERO);
|
||||
|
||||
printf("\n\tContinued fraction coefficients\n");
|
||||
for (m = 0; m < rdata -> db; m++)
|
||||
printf("\tbeta[%2d] = %14.8g\n", m, (float) rdata -> beta[m]);
|
||||
|
||||
printf("\n\tCayley transform coefficients\n");
|
||||
for (m = 0; m < rdata -> n; m++)
|
||||
printf("\tgamma[%2d] = %14.8g\n", m, (float) rdata -> gamma[m]);
|
||||
|
||||
#ifndef NPLOT
|
||||
plot_function = fopen("zolotarev-fn.dat", "w");
|
||||
plot_error = fopen("zolotarev-err.dat", "w");
|
||||
#ifdef ALLPLOTS
|
||||
plot_partfrac = fopen("zolotarev-partfrac.dat", "w");
|
||||
plot_contfrac = fopen("zolotarev-contfrac.dat", "w");
|
||||
plot_cayley = fopen("zolotarev-cayley.dat", "w");
|
||||
#endif /* ALLPLOTS */
|
||||
for (m = 0, maxypferr = maxycferr = maxycaylerr = 0.0; m <= plotpts; m++) {
|
||||
x = 2.4 * (float) m / plotpts - 1.2;
|
||||
if (rdata -> type == 0 || fabs(x) * (float) plotpts > 1.0) {
|
||||
/* skip x = 0 for type 1, as R(0) is singular */
|
||||
y = zolotarev_eval((PRECISION) x, rdata);
|
||||
fprintf(plot_function, "%g %g\n", x, (float) y);
|
||||
fprintf(plot_error, "%g %g\n",
|
||||
x, (float)((y - ((x > 0.0 ? ONE : -ONE))) / rdata -> Delta));
|
||||
ypferr = (float)((zolotarev_partfrac_eval((PRECISION) x, rdata) - y)
|
||||
/ rdata -> Delta);
|
||||
ycferr = (float)((zolotarev_contfrac_eval((PRECISION) x, rdata) - y)
|
||||
/ rdata -> Delta);
|
||||
ycaylerr = (float)((zolotarev_cayley_eval((PRECISION) x, rdata) - y)
|
||||
/ rdata -> Delta);
|
||||
if (fabs(x) < 1.0 && fabs(x) > rdata -> epsilon) {
|
||||
maxypferr = MAX(maxypferr, fabs(ypferr));
|
||||
maxycferr = MAX(maxycferr, fabs(ycferr));
|
||||
maxycaylerr = MAX(maxycaylerr, fabs(ycaylerr));
|
||||
}
|
||||
#ifdef ALLPLOTS
|
||||
fprintf(plot_partfrac, "%g %g\n", x, ypferr);
|
||||
fprintf(plot_contfrac, "%g %g\n", x, ycferr);
|
||||
fprintf(plot_cayley, "%g %g\n", x, ycaylerr);
|
||||
#endif /* ALLPLOTS */
|
||||
}
|
||||
}
|
||||
#ifdef ALLPLOTS
|
||||
fclose(plot_cayley);
|
||||
fclose(plot_contfrac);
|
||||
fclose(plot_partfrac);
|
||||
#endif /* ALLPLOTS */
|
||||
fclose(plot_error);
|
||||
fclose(plot_function);
|
||||
|
||||
printf("\n\tMaximum PF error = %g (relative to Delta)\n", maxypferr);
|
||||
printf("\tMaximum CF error = %g (relative to Delta)\n", maxycferr);
|
||||
printf("\tMaximum Cayley error = %g (relative to Delta)\n", maxycaylerr);
|
||||
#endif /* NPLOT */
|
||||
|
||||
free(rdata -> a);
|
||||
free(rdata -> ap);
|
||||
free(rdata -> alpha);
|
||||
free(rdata -> beta);
|
||||
free(rdata -> gamma);
|
||||
free(rdata);
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
#endif /* TEST */
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
/* -*- Mode: C; comment-column: 22; fill-column: 79; -*- */
|
||||
|
||||
#ifdef __cplusplus
|
||||
#include <Grid/Namespace.h>
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
NAMESPACE_BEGIN(Approx);
|
||||
#endif
|
||||
|
||||
#define HVERSION Header Time-stamp: <14-OCT-2004 09:26:51.00 adk@MISSCONTRARY>
|
||||
|
||||
#ifndef ZOLOTAREV_INTERNAL
|
||||
#ifndef PRECISION
|
||||
#define PRECISION double
|
||||
#endif
|
||||
#define ZPRECISION PRECISION
|
||||
#define ZOLOTAREV_DATA zolotarev_data
|
||||
#endif
|
||||
|
||||
/* This struct contains the coefficients which parameterise an optimal rational
|
||||
* approximation to the signum function.
|
||||
*
|
||||
* The parameterisations used are:
|
||||
*
|
||||
* Factored form for type 0 (R0(0) = 0)
|
||||
*
|
||||
* R0(x) = A * x * prod(x^2 - a[j], j = 0 .. dn-1) / prod(x^2 - ap[j], j = 0
|
||||
* .. dd-1),
|
||||
*
|
||||
* where deg_num = 2*dn + 1 and deg_denom = 2*dd.
|
||||
*
|
||||
* Factored form for type 1 (R1(0) = infinity)
|
||||
*
|
||||
* R1(x) = (A / x) * prod(x^2 - a[j], j = 0 .. dn-1) / prod(x^2 - ap[j], j = 0
|
||||
* .. dd-1),
|
||||
*
|
||||
* where deg_num = 2*dn and deg_denom = 2*dd + 1.
|
||||
*
|
||||
* Partial fraction form
|
||||
*
|
||||
* R(x) = alpha[da] * x + sum(alpha[j] * x / (x^2 - ap[j]), j = 0 .. da-1)
|
||||
*
|
||||
* where da = dd for type 0 and da = dd + 1 with ap[dd] = 0 for type 1.
|
||||
*
|
||||
* Continued fraction form
|
||||
*
|
||||
* R(x) = beta[db-1] * x + 1 / (beta[db-2] * x + 1 / (beta[db-3] * x + ...))
|
||||
*
|
||||
* with the final coefficient being beta[0], with d' = 2 * dd + 1 for type 0
|
||||
* and db = 2 * dd + 2 for type 1.
|
||||
*
|
||||
* Cayley form (Chiu's domain wall formulation)
|
||||
*
|
||||
* R(x) = (1 - T(x)) / (1 + T(x))
|
||||
*
|
||||
* where T(x) = prod((x - gamma[j]) / (x + gamma[j]), j = 0 .. n-1)
|
||||
*/
|
||||
|
||||
typedef struct {
|
||||
ZPRECISION *a, /* zeros of numerator, a[0 .. dn-1] */
|
||||
*ap, /* poles (zeros of denominator), ap[0 .. dd-1] */
|
||||
A, /* overall factor */
|
||||
*alpha, /* coefficients of partial fraction, alpha[0 .. da-1] */
|
||||
*beta, /* coefficients of continued fraction, beta[0 .. db-1] */
|
||||
*gamma, /* zeros of numerator of T in Cayley form */
|
||||
Delta, /* maximum error, |R(x) - sgn(x)| <= Delta */
|
||||
epsilon; /* minimum x value, epsilon < |x| < 1 */
|
||||
int n, /* approximation degree */
|
||||
type, /* 0: R(0) = 0, 1: R(0) = infinity */
|
||||
dn, dd, da, db, /* number of elements of a, ap, alpha, and beta */
|
||||
deg_num, /* degree of numerator = deg_denom +/- 1 */
|
||||
deg_denom; /* degree of denominator */
|
||||
} ZOLOTAREV_DATA;
|
||||
|
||||
#ifndef ZOLOTAREV_INTERNAL
|
||||
|
||||
/* zolotarev(epsilon, n, type) returns a pointer to an initialised
|
||||
* zolotarev_data structure. The arguments must satisfy the constraints that
|
||||
* epsilon > 0, n > 0, and type = 0 or 1. */
|
||||
|
||||
ZOLOTAREV_DATA* higham(PRECISION epsilon, int n) ;
|
||||
ZOLOTAREV_DATA* zolotarev(PRECISION epsilon, int n, int type);
|
||||
void zolotarev_free(zolotarev_data *zdata);
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
NAMESPACE_END(Approx);
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,208 +0,0 @@
|
||||
/*
|
||||
Mike Clark - 25th May 2005
|
||||
|
||||
bigfloat.h
|
||||
|
||||
Simple C++ wrapper for multiprecision datatype used by AlgRemez
|
||||
algorithm
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_BIGFLOAT_H
|
||||
#define INCLUDED_BIGFLOAT_H
|
||||
|
||||
#define __GMP_WITHIN_CONFIGURE
|
||||
#include <gmp.h>
|
||||
#include <mpf2mpfr.h>
|
||||
#include <mpfr.h>
|
||||
#undef __GMP_WITHIN_CONFIGURE
|
||||
|
||||
class bigfloat {
|
||||
|
||||
private:
|
||||
|
||||
mpf_t x;
|
||||
|
||||
public:
|
||||
|
||||
bigfloat() { mpf_init(x); }
|
||||
bigfloat(const bigfloat& y) { mpf_init_set(x, y.x); }
|
||||
bigfloat(const unsigned long u) { mpf_init_set_ui(x, u); }
|
||||
bigfloat(const long i) { mpf_init_set_si(x, i); }
|
||||
bigfloat(const int i) {mpf_init_set_si(x,(long)i);}
|
||||
bigfloat(const float d) { mpf_init_set_d(x, (double)d); }
|
||||
bigfloat(const double d) { mpf_init_set_d(x, d); }
|
||||
bigfloat(const char *str) { mpf_init_set_str(x, (char*)str, 10); }
|
||||
~bigfloat(void) { mpf_clear(x); }
|
||||
operator double (void) const { return (double)mpf_get_d(x); }
|
||||
static void setDefaultPrecision(unsigned long dprec) {
|
||||
unsigned long bprec = (unsigned long)(3.321928094 * (double)dprec);
|
||||
mpf_set_default_prec(bprec);
|
||||
}
|
||||
|
||||
void setPrecision(unsigned long dprec) {
|
||||
unsigned long bprec = (unsigned long)(3.321928094 * (double)dprec);
|
||||
mpf_set_prec(x,bprec);
|
||||
}
|
||||
|
||||
unsigned long getPrecision(void) const { return mpf_get_prec(x); }
|
||||
|
||||
unsigned long getDefaultPrecision(void) const { return mpf_get_default_prec(); }
|
||||
|
||||
bigfloat& operator=(const bigfloat& y) {
|
||||
mpf_set(x, y.x);
|
||||
return *this;
|
||||
}
|
||||
|
||||
bigfloat& operator=(const unsigned long y) {
|
||||
mpf_set_ui(x, y);
|
||||
return *this;
|
||||
}
|
||||
|
||||
bigfloat& operator=(const signed long y) {
|
||||
mpf_set_si(x, y);
|
||||
return *this;
|
||||
}
|
||||
|
||||
bigfloat& operator=(const float y) {
|
||||
mpf_set_d(x, (double)y);
|
||||
return *this;
|
||||
}
|
||||
|
||||
bigfloat& operator=(const double y) {
|
||||
mpf_set_d(x, y);
|
||||
return *this;
|
||||
}
|
||||
|
||||
size_t write(void);
|
||||
size_t read(void);
|
||||
|
||||
/* Arithmetic Functions */
|
||||
|
||||
bigfloat& operator+=(const bigfloat& y) { return *this = *this + y; }
|
||||
bigfloat& operator-=(const bigfloat& y) { return *this = *this - y; }
|
||||
bigfloat& operator*=(const bigfloat& y) { return *this = *this * y; }
|
||||
bigfloat& operator/=(const bigfloat& y) { return *this = *this / y; }
|
||||
|
||||
friend bigfloat operator+(const bigfloat& x, const bigfloat& y) {
|
||||
bigfloat a;
|
||||
mpf_add(a.x,x.x,y.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator+(const bigfloat& x, const unsigned long y) {
|
||||
bigfloat a;
|
||||
mpf_add_ui(a.x,x.x,y);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const bigfloat& x, const bigfloat& y) {
|
||||
bigfloat a;
|
||||
mpf_sub(a.x,x.x,y.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const unsigned long x, const bigfloat& y) {
|
||||
bigfloat a;
|
||||
mpf_ui_sub(a.x,x,y.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const bigfloat& x, const unsigned long y) {
|
||||
bigfloat a;
|
||||
mpf_sub_ui(a.x,x.x,y);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const bigfloat& x) {
|
||||
bigfloat a;
|
||||
mpf_neg(a.x,x.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator*(const bigfloat& x, const bigfloat& y) {
|
||||
bigfloat a;
|
||||
mpf_mul(a.x,x.x,y.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator*(const bigfloat& x, const unsigned long y) {
|
||||
bigfloat a;
|
||||
mpf_mul_ui(a.x,x.x,y);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator/(const bigfloat& x, const bigfloat& y){
|
||||
bigfloat a;
|
||||
mpf_div(a.x,x.x,y.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator/(const unsigned long x, const bigfloat& y){
|
||||
bigfloat a;
|
||||
mpf_ui_div(a.x,x,y.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator/(const bigfloat& x, const unsigned long y){
|
||||
bigfloat a;
|
||||
mpf_div_ui(a.x,x.x,y);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat sqrt_bf(const bigfloat& x){
|
||||
bigfloat a;
|
||||
mpf_sqrt(a.x,x.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat sqrt_bf(const unsigned long x){
|
||||
bigfloat a;
|
||||
mpf_sqrt_ui(a.x,x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat abs_bf(const bigfloat& x){
|
||||
bigfloat a;
|
||||
mpf_abs(a.x,x.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat pow_bf(const bigfloat& a, long power) {
|
||||
bigfloat b;
|
||||
mpf_pow_ui(b.x,a.x,power);
|
||||
return b;
|
||||
}
|
||||
|
||||
friend bigfloat pow_bf(const bigfloat& a, bigfloat &power) {
|
||||
bigfloat b;
|
||||
mpfr_pow(b.x,a.x,power.x,GMP_RNDN);
|
||||
return b;
|
||||
}
|
||||
|
||||
friend bigfloat exp_bf(const bigfloat& a) {
|
||||
bigfloat b;
|
||||
mpfr_exp(b.x,a.x,GMP_RNDN);
|
||||
return b;
|
||||
}
|
||||
|
||||
/* Comparison Functions */
|
||||
|
||||
friend int operator>(const bigfloat& x, const bigfloat& y) {
|
||||
int test;
|
||||
test = mpf_cmp(x.x,y.x);
|
||||
if (test > 0) return 1;
|
||||
else return 0;
|
||||
}
|
||||
|
||||
friend int operator<(const bigfloat& x, const bigfloat& y) {
|
||||
int test;
|
||||
test = mpf_cmp(x.x,y.x);
|
||||
if (test < 0) return 1;
|
||||
else return 0;
|
||||
}
|
||||
|
||||
friend int sgn(const bigfloat&);
|
||||
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,195 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/approx/bigfloat_double.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#ifndef INCLUDED_BIGFLOAT_DOUBLE_H
|
||||
#define INCLUDED_BIGFLOAT_DOUBLE_H
|
||||
|
||||
#include <math.h>
|
||||
|
||||
typedef double mfloat;
|
||||
class bigfloat {
|
||||
private:
|
||||
|
||||
mfloat x;
|
||||
|
||||
public:
|
||||
|
||||
bigfloat() { }
|
||||
bigfloat(const bigfloat& y) { x=y.x; }
|
||||
bigfloat(const unsigned long u) { x=u; }
|
||||
bigfloat(const long i) { x=i; }
|
||||
bigfloat(const int i) { x=i;}
|
||||
bigfloat(const float d) { x=d;}
|
||||
bigfloat(const double d) { x=d;}
|
||||
bigfloat(const char *str) { x=std::stod(std::string(str));}
|
||||
~bigfloat(void) { }
|
||||
operator double (void) const { return (double)x; }
|
||||
static void setDefaultPrecision(unsigned long dprec) {
|
||||
}
|
||||
|
||||
void setPrecision(unsigned long dprec) {
|
||||
}
|
||||
|
||||
unsigned long getPrecision(void) const { return 64; }
|
||||
unsigned long getDefaultPrecision(void) const { return 64; }
|
||||
|
||||
bigfloat& operator=(const bigfloat& y) { x=y.x; return *this; }
|
||||
bigfloat& operator=(const unsigned long y) { x=y; return *this; }
|
||||
bigfloat& operator=(const signed long y) { x=y; return *this; }
|
||||
bigfloat& operator=(const float y) { x=y; return *this; }
|
||||
bigfloat& operator=(const double y) { x=y; return *this; }
|
||||
|
||||
size_t write(void);
|
||||
size_t read(void);
|
||||
|
||||
/* Arithmetic Functions */
|
||||
|
||||
bigfloat& operator+=(const bigfloat& y) { return *this = *this + y; }
|
||||
bigfloat& operator-=(const bigfloat& y) { return *this = *this - y; }
|
||||
bigfloat& operator*=(const bigfloat& y) { return *this = *this * y; }
|
||||
bigfloat& operator/=(const bigfloat& y) { return *this = *this / y; }
|
||||
|
||||
friend bigfloat operator+(const bigfloat& x, const bigfloat& y) {
|
||||
bigfloat a;
|
||||
a.x=x.x+y.x;
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator+(const bigfloat& x, const unsigned long y) {
|
||||
bigfloat a;
|
||||
a.x=x.x+y;
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const bigfloat& x, const bigfloat& y) {
|
||||
bigfloat a;
|
||||
a.x=x.x-y.x;
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const unsigned long x, const bigfloat& y) {
|
||||
bigfloat bx(x);
|
||||
return bx-y;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const bigfloat& x, const unsigned long y) {
|
||||
bigfloat by(y);
|
||||
return x-by;
|
||||
}
|
||||
|
||||
friend bigfloat operator-(const bigfloat& x) {
|
||||
bigfloat a;
|
||||
a.x=-x.x;
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator*(const bigfloat& x, const bigfloat& y) {
|
||||
bigfloat a;
|
||||
a.x=x.x*y.x;
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator*(const bigfloat& x, const unsigned long y) {
|
||||
bigfloat a;
|
||||
a.x=x.x*y;
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator/(const bigfloat& x, const bigfloat& y){
|
||||
bigfloat a;
|
||||
a.x=x.x/y.x;
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat operator/(const unsigned long x, const bigfloat& y){
|
||||
bigfloat bx(x);
|
||||
return bx/y;
|
||||
}
|
||||
|
||||
friend bigfloat operator/(const bigfloat& x, const unsigned long y){
|
||||
bigfloat by(y);
|
||||
return x/by;
|
||||
}
|
||||
|
||||
friend bigfloat sqrt_bf(const bigfloat& x){
|
||||
bigfloat a;
|
||||
a.x= sqrt(x.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat sqrt_bf(const unsigned long x){
|
||||
bigfloat a(x);
|
||||
return sqrt_bf(a);
|
||||
}
|
||||
|
||||
friend bigfloat abs_bf(const bigfloat& x){
|
||||
bigfloat a;
|
||||
a.x=fabs(x.x);
|
||||
return a;
|
||||
}
|
||||
|
||||
friend bigfloat pow_bf(const bigfloat& a, long power) {
|
||||
bigfloat b;
|
||||
b.x=pow(a.x,power);
|
||||
return b;
|
||||
}
|
||||
|
||||
friend bigfloat pow_bf(const bigfloat& a, bigfloat &power) {
|
||||
bigfloat b;
|
||||
b.x=pow(a.x,power.x);
|
||||
return b;
|
||||
}
|
||||
|
||||
friend bigfloat exp_bf(const bigfloat& a) {
|
||||
bigfloat b;
|
||||
b.x=exp(a.x);
|
||||
return b;
|
||||
}
|
||||
|
||||
/* Comparison Functions */
|
||||
friend int operator>(const bigfloat& x, const bigfloat& y) {
|
||||
return x.x>y.x;
|
||||
}
|
||||
|
||||
friend int operator<(const bigfloat& x, const bigfloat& y) {
|
||||
return x.x<y.x;
|
||||
}
|
||||
|
||||
friend int sgn(const bigfloat& x) {
|
||||
if ( x.x>=0 ) return 1;
|
||||
else return 0;
|
||||
}
|
||||
|
||||
/* Miscellaneous Functions */
|
||||
|
||||
// friend bigfloat& random(void);
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1,397 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/AdefGeneric.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_ALGORITHMS_ITERATIVE_GENERIC_PCG
|
||||
#define GRID_ALGORITHMS_ITERATIVE_GENERIC_PCG
|
||||
|
||||
/*
|
||||
* Compared to Tang-2009: P=Pleft. P^T = PRight Q=MssInv.
|
||||
* Script A = SolverMatrix
|
||||
* Script P = Preconditioner
|
||||
*
|
||||
* Deflation methods considered
|
||||
* -- Solve P A x = P b [ like Luscher ]
|
||||
* DEF-1 M P A x = M P b [i.e. left precon]
|
||||
* DEF-2 P^T M A x = P^T M b
|
||||
* ADEF-1 Preconditioner = M P + Q [ Q + M + M A Q]
|
||||
* ADEF-2 Preconditioner = P^T M + Q
|
||||
* BNN Preconditioner = P^T M P + Q
|
||||
* BNN2 Preconditioner = M P + P^TM +Q - M P A M
|
||||
*
|
||||
* Implement ADEF-2
|
||||
*
|
||||
* Vstart = P^Tx + Qb
|
||||
* M1 = P^TM + Q
|
||||
* M2=M3=1
|
||||
* Vout = x
|
||||
*/
|
||||
|
||||
// abstract base
|
||||
template<class Field, class CoarseField>
|
||||
class TwoLevelFlexiblePcg : public LinearFunction<Field>
|
||||
{
|
||||
public:
|
||||
int verbose;
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
const int mmax = 5;
|
||||
GridBase *grid;
|
||||
GridBase *coarsegrid;
|
||||
|
||||
LinearOperatorBase<Field> *_Linop
|
||||
OperatorFunction<Field> *_Smoother,
|
||||
LinearFunction<CoarseField> *_CoarseSolver;
|
||||
|
||||
// Need somthing that knows how to get from Coarse to fine and back again
|
||||
|
||||
// more most opertor functions
|
||||
TwoLevelFlexiblePcg(RealD tol,
|
||||
Integer maxit,
|
||||
LinearOperatorBase<Field> *Linop,
|
||||
LinearOperatorBase<Field> *SmootherLinop,
|
||||
OperatorFunction<Field> *Smoother,
|
||||
OperatorFunction<CoarseField> CoarseLinop
|
||||
) :
|
||||
Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
_Linop(Linop),
|
||||
_PreconditionerLinop(PrecLinop),
|
||||
_Preconditioner(Preconditioner)
|
||||
{
|
||||
verbose=0;
|
||||
};
|
||||
|
||||
// The Pcg routine is common to all, but the various matrices differ from derived
|
||||
// implementation to derived implmentation
|
||||
void operator() (const Field &src, Field &psi){
|
||||
void operator() (const Field &src, Field &psi){
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
grid = src.Grid();
|
||||
|
||||
RealD f;
|
||||
RealD rtzp,rtz,a,d,b;
|
||||
RealD rptzp;
|
||||
RealD tn;
|
||||
RealD guess = norm2(psi);
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = ssq*Tolerance*Tolerance;
|
||||
|
||||
/////////////////////////////
|
||||
// Set up history vectors
|
||||
/////////////////////////////
|
||||
std::vector<Field> p (mmax,grid);
|
||||
std::vector<Field> mmp(mmax,grid);
|
||||
std::vector<RealD> pAp(mmax);
|
||||
|
||||
Field x (grid); x = psi;
|
||||
Field z (grid);
|
||||
Field tmp(grid);
|
||||
Field r (grid);
|
||||
Field mu (grid);
|
||||
|
||||
//////////////////////////
|
||||
// x0 = Vstart -- possibly modify guess
|
||||
//////////////////////////
|
||||
x=src;
|
||||
Vstart(x,src);
|
||||
|
||||
// r0 = b -A x0
|
||||
HermOp(x,mmp); // Shouldn't this be something else?
|
||||
axpy (r, -1.0,mmp[0], src); // Recomputes r=src-Ax0
|
||||
|
||||
//////////////////////////////////
|
||||
// Compute z = M1 x
|
||||
//////////////////////////////////
|
||||
M1(r,z,tmp,mp,SmootherMirs);
|
||||
rtzp =real(innerProduct(r,z));
|
||||
|
||||
///////////////////////////////////////
|
||||
// Solve for Mss mu = P A z and set p = z-mu
|
||||
// Def2: p = 1 - Q Az = Pright z
|
||||
// Other algos M2 is trivial
|
||||
///////////////////////////////////////
|
||||
M2(z,p[0]);
|
||||
|
||||
for (int k=0;k<=MaxIterations;k++){
|
||||
|
||||
int peri_k = k % mmax;
|
||||
int peri_kp = (k+1) % mmax;
|
||||
|
||||
rtz=rtzp;
|
||||
d= M3(p[peri_k],mp,mmp[peri_k],tmp);
|
||||
a = rtz/d;
|
||||
|
||||
// Memorise this
|
||||
pAp[peri_k] = d;
|
||||
|
||||
axpy(x,a,p[peri_k],x);
|
||||
RealD rn = axpy_norm(r,-a,mmp[peri_k],r);
|
||||
|
||||
// Compute z = M x
|
||||
M1(r,z,tmp,mp);
|
||||
|
||||
rtzp =real(innerProduct(r,z));
|
||||
|
||||
M2(z,mu); // ADEF-2 this is identity. Axpy possible to eliminate
|
||||
|
||||
p[peri_kp]=p[peri_k];
|
||||
|
||||
// Standard search direction p -> z + b p ; b =
|
||||
b = (rtzp)/rtz;
|
||||
|
||||
int northog;
|
||||
// northog = (peri_kp==0)?1:peri_kp; // This is the fCG(mmax) algorithm
|
||||
northog = (k>mmax-1)?(mmax-1):k; // This is the fCG-Tr(mmax-1) algorithm
|
||||
|
||||
for(int back=0; back < northog; back++){
|
||||
int peri_back = (k-back)%mmax;
|
||||
RealD pbApk= real(innerProduct(mmp[peri_back],p[peri_kp]));
|
||||
RealD beta = -pbApk/pAp[peri_back];
|
||||
axpy(p[peri_kp],beta,p[peri_back],p[peri_kp]);
|
||||
}
|
||||
|
||||
RealD rrn=sqrt(rn/ssq);
|
||||
std::cout<<GridLogMessage<<"TwoLevelfPcg: k= "<<k<<" residual = "<<rrn<<std::endl;
|
||||
|
||||
// Stopping condition
|
||||
if ( rn <= rsq ) {
|
||||
|
||||
HermOp(x,mmp); // Shouldn't this be something else?
|
||||
axpy(tmp,-1.0,src,mmp[0]);
|
||||
|
||||
RealD psinorm = sqrt(norm2(x));
|
||||
RealD srcnorm = sqrt(norm2(src));
|
||||
RealD tmpnorm = sqrt(norm2(tmp));
|
||||
RealD true_residual = tmpnorm/srcnorm;
|
||||
std::cout<<GridLogMessage<<"TwoLevelfPcg: true residual is "<<true_residual<<std::endl;
|
||||
std::cout<<GridLogMessage<<"TwoLevelfPcg: target residual was"<<Tolerance<<std::endl;
|
||||
return k;
|
||||
}
|
||||
}
|
||||
// Non-convergence
|
||||
assert(0);
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
virtual void M(Field & in,Field & out,Field & tmp) {
|
||||
|
||||
}
|
||||
|
||||
virtual void M1(Field & in, Field & out) {// the smoother
|
||||
|
||||
// [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
|
||||
Field tmp(grid);
|
||||
Field Min(grid);
|
||||
|
||||
PcgM(in,Min); // Smoother call
|
||||
|
||||
HermOp(Min,out);
|
||||
axpy(tmp,-1.0,out,in); // tmp = in - A Min
|
||||
|
||||
ProjectToSubspace(tmp,PleftProj);
|
||||
ApplyInverse(PleftProj,PleftMss_proj); // Ass^{-1} [in - A Min]_s
|
||||
PromoteFromSubspace(PleftMss_proj,tmp);// tmp = Q[in - A Min]
|
||||
axpy(out,1.0,Min,tmp); // Min+tmp
|
||||
}
|
||||
|
||||
virtual void M2(const Field & in, Field & out) {
|
||||
out=in;
|
||||
// Must override for Def2 only
|
||||
// case PcgDef2:
|
||||
// Pright(in,out);
|
||||
// break;
|
||||
}
|
||||
|
||||
virtual RealD M3(const Field & p, Field & mmp){
|
||||
double d,dd;
|
||||
HermOpAndNorm(p,mmp,d,dd);
|
||||
return dd;
|
||||
// Must override for Def1 only
|
||||
// case PcgDef1:
|
||||
// d=linop_d->Mprec(p,mmp,tmp,0,1);// Dag no
|
||||
// linop_d->Mprec(mmp,mp,tmp,1);// Dag yes
|
||||
// Pleft(mp,mmp);
|
||||
// d=real(linop_d->inner(p,mmp));
|
||||
}
|
||||
|
||||
virtual void VstartDef2(Field & xconst Field & src){
|
||||
//case PcgDef2:
|
||||
//case PcgAdef2:
|
||||
//case PcgAdef2f:
|
||||
//case PcgV11f:
|
||||
///////////////////////////////////
|
||||
// Choose x_0 such that
|
||||
// x_0 = guess + (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
|
||||
// = [1 - Ass_inv A] Guess + Assinv src
|
||||
// = P^T guess + Assinv src
|
||||
// = Vstart [Tang notation]
|
||||
// This gives:
|
||||
// W^T (src - A x_0) = src_s - A guess_s - r_s
|
||||
// = src_s - (A guess)_s - src_s + (A guess)_s
|
||||
// = 0
|
||||
///////////////////////////////////
|
||||
Field r(grid);
|
||||
Field mmp(grid);
|
||||
|
||||
HermOp(x,mmp);
|
||||
axpy (r, -1.0, mmp, src); // r_{-1} = src - A x
|
||||
ProjectToSubspace(r,PleftProj);
|
||||
ApplyInverseCG(PleftProj,PleftMss_proj); // Ass^{-1} r_s
|
||||
PromoteFromSubspace(PleftMss_proj,mmp);
|
||||
x=x+mmp;
|
||||
|
||||
}
|
||||
|
||||
virtual void Vstart(Field & x,const Field & src){
|
||||
return;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Only Def1 has non-trivial Vout. Override in Def1
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
virtual void Vout (Field & in, Field & out,Field & src){
|
||||
out = in;
|
||||
//case PcgDef1:
|
||||
// //Qb + PT x
|
||||
// ProjectToSubspace(src,PleftProj);
|
||||
// ApplyInverse(PleftProj,PleftMss_proj); // Ass^{-1} r_s
|
||||
// PromoteFromSubspace(PleftMss_proj,tmp);
|
||||
//
|
||||
// Pright(in,out);
|
||||
//
|
||||
// linop_d->axpy(out,tmp,out,1.0);
|
||||
// break;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Pright and Pleft are common to all implementations
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
virtual void Pright(Field & in,Field & out){
|
||||
// P_R = [ 1 0 ]
|
||||
// [ -Mss^-1 Msb 0 ]
|
||||
Field in_sbar(grid);
|
||||
|
||||
ProjectToSubspace(in,PleftProj);
|
||||
PromoteFromSubspace(PleftProj,out);
|
||||
axpy(in_sbar,-1.0,out,in); // in_sbar = in - in_s
|
||||
|
||||
HermOp(in_sbar,out);
|
||||
ProjectToSubspace(out,PleftProj); // Mssbar in_sbar (project)
|
||||
|
||||
ApplyInverse (PleftProj,PleftMss_proj); // Mss^{-1} Mssbar
|
||||
PromoteFromSubspace(PleftMss_proj,out); //
|
||||
|
||||
axpy(out,-1.0,out,in_sbar); // in_sbar - Mss^{-1} Mssbar in_sbar
|
||||
}
|
||||
virtual void Pleft (Field & in,Field & out){
|
||||
// P_L = [ 1 -Mbs Mss^-1]
|
||||
// [ 0 0 ]
|
||||
Field in_sbar(grid);
|
||||
Field tmp2(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
ProjectToSubspace(in,PleftProj);
|
||||
PromoteFromSubspace(PleftProj,out);
|
||||
axpy(in_sbar,-1.0,out,in); // in_sbar = in - in_s
|
||||
|
||||
ApplyInverse(PleftProj,PleftMss_proj); // Mss^{-1} in_s
|
||||
PromoteFromSubspace(PleftMss_proj,out);
|
||||
|
||||
HermOp(out,Mtmp);
|
||||
|
||||
ProjectToSubspace(Mtmp,PleftProj); // Msbar s Mss^{-1}
|
||||
PromoteFromSubspace(PleftProj,tmp2);
|
||||
|
||||
axpy(out,-1.0,tmp2,Mtmp);
|
||||
axpy(out,-1.0,out,in_sbar); // in_sbar - Msbars Mss^{-1} in_s
|
||||
}
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgADef2 : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp){
|
||||
|
||||
}
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp){
|
||||
|
||||
}
|
||||
virtual void M2(Field & in, Field & out){
|
||||
|
||||
}
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp){
|
||||
|
||||
}
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp){
|
||||
|
||||
}
|
||||
}
|
||||
/*
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgAD : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgDef1 : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
virtual void Vout (Field & in, Field & out,Field & src,Field & tmp);
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgDef2 : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgV11: public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
}
|
||||
*/
|
||||
#endif
|
||||
@@ -1,234 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/BiCGSTAB.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: juettner <juettner@soton.ac.uk>
|
||||
Author: David Murphy <djmurphy@mit.edu>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#ifndef GRID_BICGSTAB_H
|
||||
#define GRID_BICGSTAB_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Base classes for iterative processes based on operators
|
||||
// single input vec, single output vec.
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
||||
template <class Field>
|
||||
class BiCGSTAB : public OperatorFunction<Field>
|
||||
{
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
|
||||
BiCGSTAB(RealD tol, Integer maxit, bool err_on_no_conv = true) :
|
||||
Tolerance(tol), MaxIterations(maxit), ErrorOnNoConverge(err_on_no_conv){};
|
||||
|
||||
void operator()(LinearOperatorBase<Field>& Linop, const Field& src, Field& psi)
|
||||
{
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
RealD cp(0), rho(1), rho_prev(0), alpha(1), beta(0), omega(1);
|
||||
RealD a(0), bo(0), b(0), ssq(0);
|
||||
|
||||
Field p(src);
|
||||
Field r(src);
|
||||
Field rhat(src);
|
||||
Field v(src);
|
||||
Field s(src);
|
||||
Field t(src);
|
||||
Field h(src);
|
||||
|
||||
v = Zero();
|
||||
p = Zero();
|
||||
|
||||
// Initial residual computation & set up
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
Linop.Op(psi, v);
|
||||
b = norm2(v);
|
||||
|
||||
r = src - v;
|
||||
rhat = r;
|
||||
a = norm2(r);
|
||||
ssq = norm2(src);
|
||||
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: src " << ssq << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: mp " << b << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: r " << a << std::endl;
|
||||
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
// Check if guess is really REALLY good :)
|
||||
if(a <= rsq){ return; }
|
||||
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: k=0 residual " << a << " target " << rsq << std::endl;
|
||||
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch InnerTimer;
|
||||
GridStopWatch AxpyNormTimer;
|
||||
GridStopWatch LinearCombTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++)
|
||||
{
|
||||
rho_prev = rho;
|
||||
|
||||
LinalgTimer.Start();
|
||||
InnerTimer.Start();
|
||||
ComplexD Crho = innerProduct(rhat,r);
|
||||
InnerTimer.Stop();
|
||||
rho = Crho.real();
|
||||
|
||||
beta = (rho / rho_prev) * (alpha / omega);
|
||||
|
||||
LinearCombTimer.Start();
|
||||
bo = beta * omega;
|
||||
{
|
||||
autoView( p_v , p, AcceleratorWrite);
|
||||
autoView( r_v , r, AcceleratorRead);
|
||||
autoView( v_v , v, AcceleratorRead);
|
||||
accelerator_for(ss, p_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(p_v[ss], beta*p_v(ss) - bo*v_v(ss) + r_v(ss));
|
||||
});
|
||||
}
|
||||
LinearCombTimer.Stop();
|
||||
LinalgTimer.Stop();
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop.Op(p,v);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
InnerTimer.Start();
|
||||
ComplexD Calpha = innerProduct(rhat,v);
|
||||
InnerTimer.Stop();
|
||||
alpha = rho / Calpha.real();
|
||||
|
||||
LinearCombTimer.Start();
|
||||
{
|
||||
autoView( p_v , p, AcceleratorRead);
|
||||
autoView( r_v , r, AcceleratorRead);
|
||||
autoView( v_v , v, AcceleratorRead);
|
||||
autoView( psi_v,psi, AcceleratorRead);
|
||||
autoView( h_v , h, AcceleratorWrite);
|
||||
autoView( s_v , s, AcceleratorWrite);
|
||||
accelerator_for(ss, h_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(h_v[ss], alpha*p_v(ss) + psi_v(ss));
|
||||
});
|
||||
accelerator_for(ss, s_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(s_v[ss], -alpha*v_v(ss) + r_v(ss));
|
||||
});
|
||||
}
|
||||
LinearCombTimer.Stop();
|
||||
LinalgTimer.Stop();
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop.Op(s,t);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
InnerTimer.Start();
|
||||
ComplexD Comega = innerProduct(t,s);
|
||||
InnerTimer.Stop();
|
||||
omega = Comega.real() / norm2(t);
|
||||
|
||||
LinearCombTimer.Start();
|
||||
{
|
||||
autoView( psi_v,psi, AcceleratorWrite);
|
||||
autoView( r_v , r, AcceleratorWrite);
|
||||
autoView( h_v , h, AcceleratorRead);
|
||||
autoView( s_v , s, AcceleratorRead);
|
||||
autoView( t_v , t, AcceleratorRead);
|
||||
accelerator_for(ss, psi_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(psi_v[ss], h_v(ss) + omega * s_v(ss));
|
||||
coalescedWrite(r_v[ss], -omega * t_v(ss) + s_v(ss));
|
||||
});
|
||||
}
|
||||
LinearCombTimer.Stop();
|
||||
|
||||
cp = norm2(r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
std::cout << GridLogIterative << "BiCGSTAB: Iteration " << k << " residual " << sqrt(cp/ssq) << " target " << Tolerance << std::endl;
|
||||
|
||||
// Stopping condition
|
||||
if(cp <= rsq)
|
||||
{
|
||||
SolverTimer.Stop();
|
||||
Linop.Op(psi, v);
|
||||
p = v - src;
|
||||
|
||||
RealD srcnorm = sqrt(norm2(src));
|
||||
RealD resnorm = sqrt(norm2(p));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "BiCGSTAB Converged on iteration " << k << std::endl;
|
||||
std::cout << GridLogMessage << "\tComputed residual " << sqrt(cp/ssq) << std::endl;
|
||||
std::cout << GridLogMessage << "\tTrue residual " << true_residual << std::endl;
|
||||
std::cout << GridLogMessage << "\tTarget " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "Time breakdown " << std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "\tLinalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "\tInner " << InnerTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "\tAxpyNorm " << AxpyNormTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "\tLinearComb " << LinearCombTimer.Elapsed() << std::endl;
|
||||
|
||||
if(ErrorOnNoConverge){ assert(true_residual / Tolerance < 10000.0); }
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "BiCGSTAB did NOT converge" << std::endl;
|
||||
|
||||
if(ErrorOnNoConverge){ assert(0); }
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,159 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/BiCGSTABMixedPrec.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
||||
Author: David Murphy <djmurphy@mit.edu>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#ifndef GRID_BICGSTAB_MIXED_PREC_H
|
||||
#define GRID_BICGSTAB_MIXED_PREC_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
// Mixed precision restarted defect correction BiCGSTAB
|
||||
template<class FieldD, class FieldF, typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0, typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class MixedPrecisionBiCGSTAB : public LinearFunction<FieldD>
|
||||
{
|
||||
public:
|
||||
using LinearFunction<FieldD>::operator();
|
||||
RealD Tolerance;
|
||||
RealD InnerTolerance; // Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||
Integer MaxInnerIterations;
|
||||
Integer MaxOuterIterations;
|
||||
GridBase* SinglePrecGrid; // Grid for single-precision fields
|
||||
RealD OuterLoopNormMult; // Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
|
||||
LinearOperatorBase<FieldF> &Linop_f;
|
||||
LinearOperatorBase<FieldD> &Linop_d;
|
||||
|
||||
Integer TotalInnerIterations; //Number of inner CG iterations
|
||||
Integer TotalOuterIterations; //Number of restarts
|
||||
Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step
|
||||
|
||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
||||
LinearFunction<FieldF> *guesser;
|
||||
|
||||
MixedPrecisionBiCGSTAB(RealD tol, Integer maxinnerit, Integer maxouterit, GridBase* _sp_grid,
|
||||
LinearOperatorBase<FieldF>& _Linop_f, LinearOperatorBase<FieldD>& _Linop_d) :
|
||||
Linop_f(_Linop_f), Linop_d(_Linop_d), Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit),
|
||||
MaxOuterIterations(maxouterit), SinglePrecGrid(_sp_grid), OuterLoopNormMult(100.), guesser(NULL) {};
|
||||
|
||||
void useGuesser(LinearFunction<FieldF>& g){
|
||||
guesser = &g;
|
||||
}
|
||||
|
||||
void operator() (const FieldD& src_d_in, FieldD& sol_d)
|
||||
{
|
||||
TotalInnerIterations = 0;
|
||||
|
||||
GridStopWatch TotalTimer;
|
||||
TotalTimer.Start();
|
||||
|
||||
int cb = src_d_in.Checkerboard();
|
||||
sol_d.Checkerboard() = cb;
|
||||
|
||||
RealD src_norm = norm2(src_d_in);
|
||||
RealD stop = src_norm * Tolerance*Tolerance;
|
||||
|
||||
GridBase* DoublePrecGrid = src_d_in.Grid();
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
tmp_d.Checkerboard() = cb;
|
||||
|
||||
FieldD tmp2_d(DoublePrecGrid);
|
||||
tmp2_d.Checkerboard() = cb;
|
||||
|
||||
FieldD src_d(DoublePrecGrid);
|
||||
src_d = src_d_in; //source for next inner iteration, computed from residual during operation
|
||||
|
||||
RealD inner_tol = InnerTolerance;
|
||||
|
||||
FieldF src_f(SinglePrecGrid);
|
||||
src_f.Checkerboard() = cb;
|
||||
|
||||
FieldF sol_f(SinglePrecGrid);
|
||||
sol_f.Checkerboard() = cb;
|
||||
|
||||
BiCGSTAB<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
||||
CG_f.ErrorOnNoConverge = false;
|
||||
|
||||
GridStopWatch InnerCGtimer;
|
||||
|
||||
GridStopWatch PrecChangeTimer;
|
||||
|
||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
||||
|
||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++)
|
||||
{
|
||||
// Compute double precision rsd and also new RHS vector.
|
||||
Linop_d.Op(sol_d, tmp_d);
|
||||
RealD norm = axpy_norm(src_d, -1., tmp_d, src_d_in); //src_d is residual vector
|
||||
|
||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Outer iteration " << outer_iter << " residual " << norm << " target " << stop << std::endl;
|
||||
|
||||
if(norm < OuterLoopNormMult * stop){
|
||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Outer iteration converged on iteration " << outer_iter << std::endl;
|
||||
break;
|
||||
}
|
||||
while(norm * inner_tol * inner_tol < stop){ inner_tol *= 2; } // inner_tol = sqrt(stop/norm) ??
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(src_f, src_d);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
sol_f = Zero();
|
||||
|
||||
//Optionally improve inner solver guess (eg using known eigenvectors)
|
||||
if(guesser != NULL){ (*guesser)(src_f, sol_f); }
|
||||
|
||||
//Inner CG
|
||||
CG_f.Tolerance = inner_tol;
|
||||
InnerCGtimer.Start();
|
||||
CG_f(Linop_f, src_f, sol_f);
|
||||
InnerCGtimer.Stop();
|
||||
TotalInnerIterations += CG_f.IterationsToComplete;
|
||||
|
||||
//Convert sol back to double and add to double prec solution
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(tmp_d, sol_f);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
axpy(sol_d, 1.0, tmp_d, sol_d);
|
||||
}
|
||||
|
||||
//Final trial CG
|
||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Starting final patch-up double-precision solve" << std::endl;
|
||||
|
||||
BiCGSTAB<FieldD> CG_d(Tolerance, MaxInnerIterations);
|
||||
CG_d(Linop_d, src_d_in, sol_d);
|
||||
TotalFinalStepIterations = CG_d.IterationsToComplete;
|
||||
|
||||
TotalTimer.Stop();
|
||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Inner CG iterations " << TotalInnerIterations << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations << std::endl;
|
||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,698 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/BlockConjugateGradient.h
|
||||
|
||||
Copyright (C) 2017
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
enum BlockCGtype { BlockCG, BlockCGrQ, CGmultiRHS, BlockCGVec, BlockCGrQVec };
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Block conjugate gradient. Dimension zero should be the block direction
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <class Field>
|
||||
class BlockConjugateGradient : public OperatorFunction<Field> {
|
||||
public:
|
||||
|
||||
typedef typename Field::scalar_type scomplex;
|
||||
|
||||
int blockDim ;
|
||||
int Nblock;
|
||||
|
||||
BlockCGtype CGtype;
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
Integer PrintInterval; //GridLogMessages or Iterative
|
||||
RealD TrueResidual;
|
||||
|
||||
BlockConjugateGradient(BlockCGtype cgtype,int _Orthog,RealD tol, Integer maxit, bool err_on_no_conv = true)
|
||||
: Tolerance(tol), CGtype(cgtype), blockDim(_Orthog), MaxIterations(maxit), ErrorOnNoConverge(err_on_no_conv),PrintInterval(100)
|
||||
{};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Thin QR factorisation (google it)
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//Dimensions
|
||||
// R_{ferm x Nblock} = Q_{ferm x Nblock} x C_{Nblock x Nblock} -> ferm x Nblock
|
||||
//
|
||||
// Rdag R = m_rr = Herm = L L^dag <-- Cholesky decomposition (LLT routine in Eigen)
|
||||
//
|
||||
// Q C = R => Q = R C^{-1}
|
||||
//
|
||||
// Want Ident = Q^dag Q = C^{-dag} R^dag R C^{-1} = C^{-dag} L L^dag C^{-1} = 1_{Nblock x Nblock}
|
||||
//
|
||||
// Set C = L^{dag}, and then Q^dag Q = ident
|
||||
//
|
||||
// Checks:
|
||||
// Cdag C = Rdag R ; passes.
|
||||
// QdagQ = 1 ; passes
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void ThinQRfact (Eigen::MatrixXcd &m_rr,
|
||||
Eigen::MatrixXcd &C,
|
||||
Eigen::MatrixXcd &Cinv,
|
||||
Field & Q,
|
||||
const Field & R)
|
||||
{
|
||||
int Orthog = blockDim; // First dimension is block dim; this is an assumption
|
||||
sliceInnerProductMatrix(m_rr,R,R,Orthog);
|
||||
|
||||
// Force manifest hermitian to avoid rounding related
|
||||
m_rr = 0.5*(m_rr+m_rr.adjoint());
|
||||
|
||||
Eigen::MatrixXcd L = m_rr.llt().matrixL();
|
||||
|
||||
C = L.adjoint();
|
||||
Cinv = C.inverse();
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Q = R C^{-1}
|
||||
//
|
||||
// Q_j = R_i Cinv(i,j)
|
||||
//
|
||||
// NB maddMatrix conventions are Right multiplication X[j] a[j,i] already
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
sliceMulMatrix(Q,Cinv,R,Orthog);
|
||||
}
|
||||
// see comments above
|
||||
void ThinQRfact (Eigen::MatrixXcd &m_rr,
|
||||
Eigen::MatrixXcd &C,
|
||||
Eigen::MatrixXcd &Cinv,
|
||||
std::vector<Field> & Q,
|
||||
const std::vector<Field> & R)
|
||||
{
|
||||
InnerProductMatrix(m_rr,R,R);
|
||||
|
||||
m_rr = 0.5*(m_rr+m_rr.adjoint());
|
||||
|
||||
Eigen::MatrixXcd L = m_rr.llt().matrixL();
|
||||
|
||||
C = L.adjoint();
|
||||
Cinv = C.inverse();
|
||||
|
||||
MulMatrix(Q,Cinv,R);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Call one of several implementations
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &Src, Field &Psi)
|
||||
{
|
||||
if ( CGtype == BlockCGrQ ) {
|
||||
BlockCGrQsolve(Linop,Src,Psi);
|
||||
} else if (CGtype == CGmultiRHS ) {
|
||||
CGmultiRHSsolve(Linop,Src,Psi);
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
virtual void operator()(LinearOperatorBase<Field> &Linop, const std::vector<Field> &Src, std::vector<Field> &Psi)
|
||||
{
|
||||
if ( CGtype == BlockCGrQVec ) {
|
||||
BlockCGrQsolveVec(Linop,Src,Psi);
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// BlockCGrQ implementation:
|
||||
//--------------------------
|
||||
// X is guess/Solution
|
||||
// B is RHS
|
||||
// Solve A X_i = B_i ; i refers to Nblock index
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
|
||||
{
|
||||
int Orthog = blockDim; // First dimension is block dim; this is an assumption
|
||||
Nblock = B.Grid()->_fdimensions[Orthog];
|
||||
/* FAKE */
|
||||
Nblock=8;
|
||||
std::cout<<GridLogMessage<<" Block Conjugate Gradient : Orthog "<<Orthog<<" Nblock "<<Nblock<<std::endl;
|
||||
|
||||
X.Checkerboard() = B.Checkerboard();
|
||||
conformable(X, B);
|
||||
|
||||
Field tmp(B);
|
||||
Field Q(B);
|
||||
Field D(B);
|
||||
Field Z(B);
|
||||
Field AD(B);
|
||||
|
||||
Eigen::MatrixXcd m_DZ = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_M = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_rr = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
|
||||
Eigen::MatrixXcd m_C = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_Cinv = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_S = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_Sinv = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
|
||||
Eigen::MatrixXcd m_tmp = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_tmp1 = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
|
||||
// Initial residual computation & set up
|
||||
std::vector<RealD> residuals(Nblock);
|
||||
std::vector<RealD> ssq(Nblock);
|
||||
|
||||
sliceNorm(ssq,B,Orthog);
|
||||
RealD sssum=0;
|
||||
for(int b=0;b<Nblock;b++) sssum+=ssq[b];
|
||||
|
||||
sliceNorm(residuals,B,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
sliceNorm(residuals,X,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
/************************************************************************
|
||||
* Block conjugate gradient rQ (Sebastien Birk Thesis, after Dubrulle 2001)
|
||||
************************************************************************
|
||||
* Dimensions:
|
||||
*
|
||||
* X,B==(Nferm x Nblock)
|
||||
* A==(Nferm x Nferm)
|
||||
*
|
||||
* Nferm = Nspin x Ncolour x Ncomplex x Nlattice_site
|
||||
*
|
||||
* QC = R = B-AX, D = Q ; QC => Thin QR factorisation (google it)
|
||||
* for k:
|
||||
* Z = AD
|
||||
* M = [D^dag Z]^{-1}
|
||||
* X = X + D MC
|
||||
* QS = Q - ZM
|
||||
* D = Q + D S^dag
|
||||
* C = S C
|
||||
*/
|
||||
///////////////////////////////////////
|
||||
// Initial block: initial search dir is guess
|
||||
///////////////////////////////////////
|
||||
std::cout << GridLogMessage<<"BlockCGrQ algorithm initialisation " <<std::endl;
|
||||
|
||||
//1. QC = R = B-AX, D = Q ; QC => Thin QR factorisation (google it)
|
||||
Linop.HermOp(X, AD);
|
||||
tmp = B - AD;
|
||||
|
||||
ThinQRfact (m_rr, m_C, m_Cinv, Q, tmp);
|
||||
D=Q;
|
||||
|
||||
std::cout << GridLogMessage<<"BlockCGrQ computed initial residual and QR fact " <<std::endl;
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
///////////////////////////////////////
|
||||
GridStopWatch sliceInnerTimer;
|
||||
GridStopWatch sliceMaddTimer;
|
||||
GridStopWatch QRTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
|
||||
//3. Z = AD
|
||||
MatrixTimer.Start();
|
||||
Linop.HermOp(D, Z);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
//4. M = [D^dag Z]^{-1}
|
||||
sliceInnerTimer.Start();
|
||||
sliceInnerProductMatrix(m_DZ,D,Z,Orthog);
|
||||
sliceInnerTimer.Stop();
|
||||
m_M = m_DZ.inverse();
|
||||
|
||||
//5. X = X + D MC
|
||||
m_tmp = m_M * m_C;
|
||||
sliceMaddTimer.Start();
|
||||
sliceMaddMatrix(X,m_tmp, D,X,Orthog);
|
||||
sliceMaddTimer.Stop();
|
||||
|
||||
//6. QS = Q - ZM
|
||||
sliceMaddTimer.Start();
|
||||
sliceMaddMatrix(tmp,m_M,Z,Q,Orthog,-1.0);
|
||||
sliceMaddTimer.Stop();
|
||||
QRTimer.Start();
|
||||
ThinQRfact (m_rr, m_S, m_Sinv, Q, tmp);
|
||||
QRTimer.Stop();
|
||||
|
||||
//7. D = Q + D S^dag
|
||||
m_tmp = m_S.adjoint();
|
||||
|
||||
sliceMaddTimer.Start();
|
||||
sliceMaddMatrix(D,m_tmp,D,Q,Orthog);
|
||||
sliceMaddTimer.Stop();
|
||||
|
||||
//8. C = S C
|
||||
m_C = m_S*m_C;
|
||||
|
||||
/*********************
|
||||
* convergence monitor
|
||||
*********************
|
||||
*/
|
||||
m_rr = m_C.adjoint() * m_C;
|
||||
|
||||
RealD max_resid=0;
|
||||
RealD rrsum=0;
|
||||
RealD rr;
|
||||
|
||||
for(int b=0;b<Nblock;b++) {
|
||||
rrsum+=real(m_rr(b,b));
|
||||
rr = real(m_rr(b,b))/ssq[b];
|
||||
if ( rr > max_resid ) max_resid = rr;
|
||||
}
|
||||
|
||||
std::cout << GridLogIterative << "\titeration "<<k<<" rr_sum "<<rrsum<<" ssq_sum "<< sssum
|
||||
<<" ave "<<std::sqrt(rrsum/sssum) << " max "<< max_resid <<std::endl;
|
||||
|
||||
if ( max_resid < Tolerance*Tolerance ) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
std::cout << GridLogMessage<<"BlockCGrQ converged in "<<k<<" iterations"<<std::endl;
|
||||
|
||||
for(int b=0;b<Nblock;b++){
|
||||
std::cout << GridLogMessage<< "\t\tblock "<<b<<" computed resid "
|
||||
<< std::sqrt(real(m_rr(b,b))/ssq[b])<<std::endl;
|
||||
}
|
||||
std::cout << GridLogMessage<<"\tMax residual is "<<std::sqrt(max_resid)<<std::endl;
|
||||
|
||||
Linop.HermOp(X, AD);
|
||||
AD = AD-B;
|
||||
TrueResidual = std::sqrt(norm2(AD)/norm2(B));
|
||||
std::cout << GridLogMessage <<"\tTrue residual is " << TrueResidual <<std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tInnerProd " << sliceInnerTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMaddMatrix " << sliceMaddTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tThinQRfact " << QRTimer.Elapsed() <<std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
std::cout << GridLogMessage << "BlockConjugateGradient(rQ) did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// multiRHS conjugate gradient. Dimension zero should be the block direction
|
||||
// Use this for spread out across nodes
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void CGmultiRHSsolve(LinearOperatorBase<Field> &Linop, const Field &Src, Field &Psi)
|
||||
{
|
||||
int Orthog = blockDim; // First dimension is block dim
|
||||
Nblock = Src.Grid()->_fdimensions[Orthog];
|
||||
|
||||
std::cout<<GridLogMessage<<"MultiRHS Conjugate Gradient : Orthog "<<Orthog<<" Nblock "<<Nblock<<std::endl;
|
||||
|
||||
Psi.Checkerboard() = Src.Checkerboard();
|
||||
conformable(Psi, Src);
|
||||
|
||||
Field P(Src);
|
||||
Field AP(Src);
|
||||
Field R(Src);
|
||||
|
||||
std::vector<ComplexD> v_pAp(Nblock);
|
||||
std::vector<RealD> v_rr (Nblock);
|
||||
std::vector<RealD> v_rr_inv(Nblock);
|
||||
std::vector<RealD> v_alpha(Nblock);
|
||||
std::vector<RealD> v_beta(Nblock);
|
||||
|
||||
// Initial residual computation & set up
|
||||
std::vector<RealD> residuals(Nblock);
|
||||
std::vector<RealD> ssq(Nblock);
|
||||
|
||||
sliceNorm(ssq,Src,Orthog);
|
||||
RealD sssum=0;
|
||||
for(int b=0;b<Nblock;b++) sssum+=ssq[b];
|
||||
|
||||
sliceNorm(residuals,Src,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
sliceNorm(residuals,Psi,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
// Initial search dir is guess
|
||||
Linop.HermOp(Psi, AP);
|
||||
|
||||
R = Src - AP;
|
||||
P = R;
|
||||
sliceNorm(v_rr,R,Orthog);
|
||||
|
||||
GridStopWatch sliceInnerTimer;
|
||||
GridStopWatch sliceMaddTimer;
|
||||
GridStopWatch sliceNormTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
|
||||
RealD rrsum=0;
|
||||
for(int b=0;b<Nblock;b++) rrsum+=real(v_rr[b]);
|
||||
|
||||
std::cout << GridLogIterative << "\titeration "<<k<<" rr_sum "<<rrsum<<" ssq_sum "<< sssum
|
||||
<<" / "<<std::sqrt(rrsum/sssum) <<std::endl;
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop.HermOp(P, AP);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
// Alpha
|
||||
sliceInnerTimer.Start();
|
||||
sliceInnerProductVector(v_pAp,P,AP,Orthog);
|
||||
sliceInnerTimer.Stop();
|
||||
for(int b=0;b<Nblock;b++){
|
||||
v_alpha[b] = v_rr[b]/real(v_pAp[b]);
|
||||
}
|
||||
|
||||
// Psi, R update
|
||||
sliceMaddTimer.Start();
|
||||
sliceMaddVector(Psi,v_alpha, P,Psi,Orthog); // add alpha * P to psi
|
||||
sliceMaddVector(R ,v_alpha,AP, R,Orthog,-1.0);// sub alpha * AP to resid
|
||||
sliceMaddTimer.Stop();
|
||||
|
||||
// Beta
|
||||
for(int b=0;b<Nblock;b++){
|
||||
v_rr_inv[b] = 1.0/v_rr[b];
|
||||
}
|
||||
sliceNormTimer.Start();
|
||||
sliceNorm(v_rr,R,Orthog);
|
||||
sliceNormTimer.Stop();
|
||||
for(int b=0;b<Nblock;b++){
|
||||
v_beta[b] = v_rr_inv[b] *v_rr[b];
|
||||
}
|
||||
|
||||
// Search update
|
||||
sliceMaddTimer.Start();
|
||||
sliceMaddVector(P,v_beta,P,R,Orthog);
|
||||
sliceMaddTimer.Stop();
|
||||
|
||||
/*********************
|
||||
* convergence monitor
|
||||
*********************
|
||||
*/
|
||||
RealD max_resid=0;
|
||||
for(int b=0;b<Nblock;b++){
|
||||
RealD rr = v_rr[b]/ssq[b];
|
||||
if ( rr > max_resid ) max_resid = rr;
|
||||
}
|
||||
|
||||
if ( max_resid < Tolerance*Tolerance ) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
std::cout << GridLogMessage<<"MultiRHS solver converged in " <<k<<" iterations"<<std::endl;
|
||||
for(int b=0;b<Nblock;b++){
|
||||
std::cout << GridLogMessage<< "\t\tBlock "<<b<<" computed resid "<< std::sqrt(v_rr[b]/ssq[b])<<std::endl;
|
||||
}
|
||||
std::cout << GridLogMessage<<"\tMax residual is "<<std::sqrt(max_resid)<<std::endl;
|
||||
|
||||
Linop.HermOp(Psi, AP);
|
||||
AP = AP-Src;
|
||||
TrueResidual = std::sqrt(norm2(AP)/norm2(Src));
|
||||
std::cout <<GridLogMessage << "\tTrue residual is " << TrueResidual <<std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tInnerProd " << sliceInnerTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tNorm " << sliceNormTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMaddMatrix " << sliceMaddTimer.Elapsed() <<std::endl;
|
||||
|
||||
|
||||
IterationsToComplete = k;
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
std::cout << GridLogMessage << "MultiRHSConjugateGradient did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
|
||||
void InnerProductMatrix(Eigen::MatrixXcd &m , const std::vector<Field> &X, const std::vector<Field> &Y){
|
||||
for(int b=0;b<Nblock;b++){
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
m(b,bp) = innerProduct(X[b],Y[bp]);
|
||||
}}
|
||||
}
|
||||
void MaddMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X,const std::vector<Field> &Y,RealD scale=1.0){
|
||||
// Should make this cache friendly with site outermost, parallel_for
|
||||
// Deal with case AP aliases with either Y or X
|
||||
std::vector<Field> tmp(Nblock,X[0]);
|
||||
for(int b=0;b<Nblock;b++){
|
||||
tmp[b] = Y[b];
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
tmp[b] = tmp[b] + scomplex(scale*m(bp,b))*X[bp];
|
||||
}
|
||||
}
|
||||
for(int b=0;b<Nblock;b++){
|
||||
AP[b] = tmp[b];
|
||||
}
|
||||
}
|
||||
void MulMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X){
|
||||
// Should make this cache friendly with site outermost, parallel_for
|
||||
for(int b=0;b<Nblock;b++){
|
||||
AP[b] = Zero();
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
AP[b] += scomplex(m(bp,b))*X[bp];
|
||||
}
|
||||
}
|
||||
}
|
||||
double normv(const std::vector<Field> &P){
|
||||
double nn = 0.0;
|
||||
for(int b=0;b<Nblock;b++) {
|
||||
nn+=norm2(P[b]);
|
||||
}
|
||||
return nn;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// BlockCGrQvec implementation:
|
||||
//--------------------------
|
||||
// X is guess/Solution
|
||||
// B is RHS
|
||||
// Solve A X_i = B_i ; i refers to Nblock index
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
void BlockCGrQsolveVec(LinearOperatorBase<Field> &Linop, const std::vector<Field> &B, std::vector<Field> &X)
|
||||
{
|
||||
Nblock = B.size();
|
||||
assert(Nblock == X.size());
|
||||
|
||||
std::cout<<GridLogMessage<<" Block Conjugate Gradient Vec rQ : Nblock "<<Nblock<<std::endl;
|
||||
|
||||
for(int b=0;b<Nblock;b++){
|
||||
X[b].Checkerboard() = B[b].Checkerboard();
|
||||
conformable(X[b], B[b]);
|
||||
conformable(X[b], X[0]);
|
||||
}
|
||||
|
||||
Field Fake(B[0]);
|
||||
|
||||
std::vector<Field> tmp(Nblock,Fake);
|
||||
std::vector<Field> Q(Nblock,Fake);
|
||||
std::vector<Field> D(Nblock,Fake);
|
||||
std::vector<Field> Z(Nblock,Fake);
|
||||
std::vector<Field> AD(Nblock,Fake);
|
||||
|
||||
Eigen::MatrixXcd m_DZ = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_M = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_rr = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
|
||||
Eigen::MatrixXcd m_C = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_Cinv = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_S = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_Sinv = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
||||
|
||||
Eigen::MatrixXcd m_tmp = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
Eigen::MatrixXcd m_tmp1 = Eigen::MatrixXcd::Identity(Nblock,Nblock);
|
||||
|
||||
// Initial residual computation & set up
|
||||
std::vector<RealD> residuals(Nblock);
|
||||
std::vector<RealD> ssq(Nblock);
|
||||
|
||||
RealD sssum=0;
|
||||
for(int b=0;b<Nblock;b++){ ssq[b] = norm2(B[b]);}
|
||||
for(int b=0;b<Nblock;b++) sssum+=ssq[b];
|
||||
|
||||
for(int b=0;b<Nblock;b++){ residuals[b] = norm2(B[b]);}
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
for(int b=0;b<Nblock;b++){ residuals[b] = norm2(X[b]);}
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
/************************************************************************
|
||||
* Block conjugate gradient rQ (Sebastien Birk Thesis, after Dubrulle 2001)
|
||||
************************************************************************
|
||||
* Dimensions:
|
||||
*
|
||||
* X,B==(Nferm x Nblock)
|
||||
* A==(Nferm x Nferm)
|
||||
*
|
||||
* Nferm = Nspin x Ncolour x Ncomplex x Nlattice_site
|
||||
*
|
||||
* QC = R = B-AX, D = Q ; QC => Thin QR factorisation (google it)
|
||||
* for k:
|
||||
* Z = AD
|
||||
* M = [D^dag Z]^{-1}
|
||||
* X = X + D MC
|
||||
* QS = Q - ZM
|
||||
* D = Q + D S^dag
|
||||
* C = S C
|
||||
*/
|
||||
///////////////////////////////////////
|
||||
// Initial block: initial search dir is guess
|
||||
///////////////////////////////////////
|
||||
std::cout << GridLogMessage<<"BlockCGrQvec algorithm initialisation " <<std::endl;
|
||||
|
||||
//1. QC = R = B-AX, D = Q ; QC => Thin QR factorisation (google it)
|
||||
for(int b=0;b<Nblock;b++) {
|
||||
Linop.HermOp(X[b], AD[b]);
|
||||
tmp[b] = B[b] - AD[b];
|
||||
}
|
||||
|
||||
ThinQRfact (m_rr, m_C, m_Cinv, Q, tmp);
|
||||
|
||||
for(int b=0;b<Nblock;b++) D[b]=Q[b];
|
||||
|
||||
std::cout << GridLogMessage<<"BlockCGrQ vec computed initial residual and QR fact " <<std::endl;
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
///////////////////////////////////////
|
||||
GridStopWatch sliceInnerTimer;
|
||||
GridStopWatch sliceMaddTimer;
|
||||
GridStopWatch QRTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
|
||||
//3. Z = AD
|
||||
MatrixTimer.Start();
|
||||
for(int b=0;b<Nblock;b++) Linop.HermOp(D[b], Z[b]);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
//4. M = [D^dag Z]^{-1}
|
||||
sliceInnerTimer.Start();
|
||||
InnerProductMatrix(m_DZ,D,Z);
|
||||
sliceInnerTimer.Stop();
|
||||
m_M = m_DZ.inverse();
|
||||
|
||||
//5. X = X + D MC
|
||||
m_tmp = m_M * m_C;
|
||||
sliceMaddTimer.Start();
|
||||
MaddMatrix(X,m_tmp, D,X);
|
||||
sliceMaddTimer.Stop();
|
||||
|
||||
//6. QS = Q - ZM
|
||||
sliceMaddTimer.Start();
|
||||
MaddMatrix(tmp,m_M,Z,Q,-1.0);
|
||||
sliceMaddTimer.Stop();
|
||||
QRTimer.Start();
|
||||
ThinQRfact (m_rr, m_S, m_Sinv, Q, tmp);
|
||||
QRTimer.Stop();
|
||||
|
||||
//7. D = Q + D S^dag
|
||||
m_tmp = m_S.adjoint();
|
||||
sliceMaddTimer.Start();
|
||||
MaddMatrix(D,m_tmp,D,Q);
|
||||
sliceMaddTimer.Stop();
|
||||
|
||||
//8. C = S C
|
||||
m_C = m_S*m_C;
|
||||
|
||||
/*********************
|
||||
* convergence monitor
|
||||
*********************
|
||||
*/
|
||||
m_rr = m_C.adjoint() * m_C;
|
||||
|
||||
RealD max_resid=0;
|
||||
RealD rrsum=0;
|
||||
RealD rr;
|
||||
|
||||
for(int b=0;b<Nblock;b++) {
|
||||
rrsum+=real(m_rr(b,b));
|
||||
rr = real(m_rr(b,b))/ssq[b];
|
||||
if ( rr > max_resid ) max_resid = rr;
|
||||
}
|
||||
|
||||
std::cout << GridLogIterative << "\t Block Iteration "<<k<<" ave resid "<< std::sqrt(rrsum/sssum) << " max "<< std::sqrt(max_resid) <<std::endl;
|
||||
|
||||
if ( max_resid < Tolerance*Tolerance ) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
std::cout << GridLogMessage<<"BlockCGrQ converged in "<<k<<" iterations"<<std::endl;
|
||||
|
||||
for(int b=0;b<Nblock;b++){
|
||||
std::cout << GridLogMessage<< "\t\tblock "<<b<<" computed resid "<< std::sqrt(real(m_rr(b,b))/ssq[b])<<std::endl;
|
||||
}
|
||||
std::cout << GridLogMessage<<"\tMax residual is "<<std::sqrt(max_resid)<<std::endl;
|
||||
|
||||
for(int b=0;b<Nblock;b++) Linop.HermOp(X[b], AD[b]);
|
||||
for(int b=0;b<Nblock;b++) AD[b] = AD[b]-B[b];
|
||||
TrueResidual = std::sqrt(normv(AD)/normv(B));
|
||||
std::cout << GridLogMessage << "\tTrue residual is " << TrueResidual <<std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tInnerProd " << sliceInnerTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMaddMatrix " << sliceMaddTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tThinQRfact " << QRTimer.Elapsed() <<std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
std::cout << GridLogMessage << "BlockConjugateGradient(rQ) did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/CommunicationAvoidingGeneralisedMinimalResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
#define GRID_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
|
||||
namespace Grid {
|
||||
|
||||
template<class Field>
|
||||
class CommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an assert when CAGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
|
||||
Integer MaxIterations;
|
||||
Integer RestartLength;
|
||||
Integer MaxNumberOfRestarts;
|
||||
Integer IterationCount; // Number of iterations the CAGMRES took to finish,
|
||||
// filled in upon completion
|
||||
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch QrTimer;
|
||||
GridStopWatch CompSolutionTimer;
|
||||
|
||||
Eigen::MatrixXcd H;
|
||||
|
||||
std::vector<ComplexD> y;
|
||||
std::vector<ComplexD> gamma;
|
||||
std::vector<ComplexD> c;
|
||||
std::vector<ComplexD> s;
|
||||
|
||||
CommunicationAvoidingGeneralisedMinimalResidual(RealD tol,
|
||||
Integer maxit,
|
||||
Integer restart_length,
|
||||
bool err_on_no_conv = true)
|
||||
: Tolerance(tol)
|
||||
, MaxIterations(maxit)
|
||||
, RestartLength(restart_length)
|
||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
||||
, ErrorOnNoConverge(err_on_no_conv)
|
||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
||||
, y(RestartLength + 1, 0.)
|
||||
, gamma(RestartLength + 1, 0.)
|
||||
, c(RestartLength + 1, 0.)
|
||||
, s(RestartLength + 1, 0.) {};
|
||||
|
||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
||||
|
||||
std::cout << GridLogWarning << "This algorithm currently doesn't differ from regular GMRES" << std::endl;
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
Field r(src.Grid());
|
||||
|
||||
std::cout << std::setprecision(4) << std::scientific;
|
||||
std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: src " << ssq << std::endl;
|
||||
|
||||
MatrixTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
QrTimer.Reset();
|
||||
CompSolutionTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
IterationCount = 0;
|
||||
|
||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
||||
|
||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
LinOp.Op(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
|
||||
RealD srcnorm = sqrt(ssq);
|
||||
RealD resnorm = sqrt(norm2(r));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "CommunicationAvoidingGeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
||||
<< " computed residual " << sqrt(cp / ssq)
|
||||
<< " true residual " << true_residual
|
||||
<< " target " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "CommunicationAvoidingGeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
|
||||
RealD cp = 0;
|
||||
|
||||
Field w(src.Grid());
|
||||
Field r(src.Grid());
|
||||
|
||||
// this should probably be made a class member so that it is only allocated once, not in every restart
|
||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(psi, w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
r = src - w;
|
||||
|
||||
gamma[0] = sqrt(norm2(r));
|
||||
|
||||
ComplexD scale = 1.0/gamma[0];
|
||||
v[0] = scale * r;
|
||||
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for (int i=0; i<RestartLength; i++) {
|
||||
|
||||
IterationCount++;
|
||||
|
||||
arnoldiStep(LinOp, v, w, i);
|
||||
|
||||
qrUpdate(i);
|
||||
|
||||
cp = norm(gamma[i+1]);
|
||||
|
||||
std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: Iteration " << IterationCount
|
||||
<< " residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
||||
|
||||
computeSolution(v, psi, i);
|
||||
|
||||
return cp;
|
||||
}
|
||||
}
|
||||
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, Field &w, int iter) {
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(v[iter], w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
for (int i = 0; i <= iter; ++i) {
|
||||
H(iter, i) = innerProduct(v[i], w);
|
||||
w = w - ComplexD(H(iter, i)) * v[i];
|
||||
}
|
||||
|
||||
H(iter, iter + 1) = sqrt(norm2(w));
|
||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
|
||||
void qrUpdate(int iter) {
|
||||
|
||||
QrTimer.Start();
|
||||
for (int i = 0; i < iter ; ++i) {
|
||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
||||
H(iter, i + 1) = tmp;
|
||||
}
|
||||
|
||||
// Compute new Givens Rotation
|
||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
||||
c[iter] = H(iter, iter) / nu;
|
||||
s[iter] = H(iter, iter + 1) / nu;
|
||||
|
||||
// Apply new Givens rotation
|
||||
H(iter, iter) = nu;
|
||||
H(iter, iter + 1) = 0.;
|
||||
|
||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
||||
QrTimer.Stop();
|
||||
}
|
||||
|
||||
void computeSolution(std::vector<Field> const &v, Field &psi, int iter) {
|
||||
|
||||
CompSolutionTimer.Start();
|
||||
for (int i = iter; i >= 0; i--) {
|
||||
y[i] = gamma[i];
|
||||
for (int k = i + 1; k <= iter; k++)
|
||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
||||
y[i] = y[i] / ComplexD(H(i, i));
|
||||
}
|
||||
|
||||
for (int i = 0; i <= iter; i++)
|
||||
psi = psi + v[i] * y[i];
|
||||
CompSolutionTimer.Stop();
|
||||
}
|
||||
};
|
||||
}
|
||||
#endif
|
||||
@@ -1,218 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradient.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_GRADIENT_H
|
||||
#define GRID_CONJUGATE_GRADIENT_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Base classes for iterative processes based on operators
|
||||
// single input vec, single output vec.
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
||||
template <class Field>
|
||||
class ConjugateGradient : public OperatorFunction<Field> {
|
||||
public:
|
||||
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
RealD TrueResidual;
|
||||
|
||||
ConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
|
||||
: Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
ErrorOnNoConverge(err_on_no_conv){};
|
||||
|
||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
|
||||
|
||||
GRID_TRACE("ConjugateGradient");
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
|
||||
conformable(psi, src);
|
||||
|
||||
RealD cp, c, a, d, b, ssq, qq;
|
||||
//RealD b_pred;
|
||||
|
||||
Field p(src);
|
||||
Field mmp(src);
|
||||
Field r(src);
|
||||
|
||||
// Initial residual computation & set up
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
Linop.HermOpAndNorm(psi, mmp, d, b);
|
||||
|
||||
r = src - mmp;
|
||||
p = r;
|
||||
|
||||
a = norm2(p);
|
||||
cp = a;
|
||||
ssq = norm2(src);
|
||||
|
||||
// Handle trivial case of zero src
|
||||
if (ssq == 0.){
|
||||
psi = Zero();
|
||||
IterationsToComplete = 1;
|
||||
TrueResidual = 0.;
|
||||
return;
|
||||
}
|
||||
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: src " << ssq << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: mp " << d << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: mmp " << b << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: cp,r " << cp << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: p " << a << std::endl;
|
||||
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
// Check if guess is really REALLY good :)
|
||||
if (cp <= rsq) {
|
||||
TrueResidual = std::sqrt(a/ssq);
|
||||
std::cout << GridLogMessage << "ConjugateGradient guess is converged already " << std::endl;
|
||||
IterationsToComplete = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
std::cout << GridLogIterative << std::setprecision(8)
|
||||
<< "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch InnerTimer;
|
||||
GridStopWatch AxpyNormTimer;
|
||||
GridStopWatch LinearCombTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
|
||||
RealD usecs = -usecond();
|
||||
SolverTimer.Start();
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
|
||||
GridStopWatch IterationTimer;
|
||||
IterationTimer.Start();
|
||||
c = cp;
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop.HermOp(p, mmp);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
InnerTimer.Start();
|
||||
ComplexD dc = innerProduct(p,mmp);
|
||||
InnerTimer.Stop();
|
||||
d = dc.real();
|
||||
a = c / d;
|
||||
|
||||
AxpyNormTimer.Start();
|
||||
cp = axpy_norm(r, -a, mmp, r);
|
||||
AxpyNormTimer.Stop();
|
||||
b = cp / c;
|
||||
|
||||
LinearCombTimer.Start();
|
||||
{
|
||||
autoView( psi_v , psi, AcceleratorWrite);
|
||||
autoView( p_v , p, AcceleratorWrite);
|
||||
autoView( r_v , r, AcceleratorWrite);
|
||||
accelerator_for(ss,p_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(psi_v[ss], a * p_v(ss) + psi_v(ss));
|
||||
coalescedWrite(p_v[ss] , b * p_v(ss) + r_v (ss));
|
||||
});
|
||||
}
|
||||
LinearCombTimer.Stop();
|
||||
LinalgTimer.Stop();
|
||||
|
||||
IterationTimer.Stop();
|
||||
if ( (k % 500) == 0 ) {
|
||||
std::cout << GridLogMessage << "ConjugateGradient: Iteration " << k
|
||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << std::endl;
|
||||
} else {
|
||||
std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
|
||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << " took " << IterationTimer.Elapsed() << std::endl;
|
||||
}
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
usecs +=usecond();
|
||||
SolverTimer.Stop();
|
||||
Linop.HermOpAndNorm(psi, mmp, d, qq);
|
||||
p = mmp - src;
|
||||
GridBase *grid = src.Grid();
|
||||
RealD DwfFlops = (1452. )*grid->gSites()*4*k
|
||||
+ (8+4+8+4+4)*12*grid->gSites()*k; // CG linear algebra
|
||||
RealD srcnorm = std::sqrt(norm2(src));
|
||||
RealD resnorm = std::sqrt(norm2(p));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
std::cout << GridLogMessage << "ConjugateGradient Converged on iteration " << k
|
||||
<< "\tComputed residual " << std::sqrt(cp / ssq)
|
||||
<< "\tTrue residual " << true_residual
|
||||
<< "\tTarget " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "Time breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tInner " << InnerTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
||||
|
||||
std::cout << GridLogDebug << "\tMobius flop rate " << DwfFlops/ usecs<< " Gflops " <<std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
|
||||
|
||||
IterationsToComplete = k;
|
||||
TrueResidual = true_residual;
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Failed. Calculate true residual before giving up
|
||||
Linop.HermOpAndNorm(psi, mmp, d, qq);
|
||||
p = mmp - src;
|
||||
|
||||
TrueResidual = sqrt(norm2(p)/ssq);
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradient did NOT converge "<<k<<" / "<< MaxIterations<< std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,170 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMixedPrec.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_GRADIENT_MIXED_PREC_H
|
||||
#define GRID_CONJUGATE_GRADIENT_MIXED_PREC_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//Mixed precision restarted defect correction CG
|
||||
template<class FieldD,class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class MixedPrecisionConjugateGradient : public LinearFunction<FieldD> {
|
||||
public:
|
||||
using LinearFunction<FieldD>::operator();
|
||||
RealD Tolerance;
|
||||
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||
Integer MaxInnerIterations;
|
||||
Integer MaxOuterIterations;
|
||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
||||
RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
|
||||
LinearOperatorBase<FieldF> &Linop_f;
|
||||
LinearOperatorBase<FieldD> &Linop_d;
|
||||
|
||||
Integer TotalInnerIterations; //Number of inner CG iterations
|
||||
Integer TotalOuterIterations; //Number of restarts
|
||||
Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step
|
||||
RealD TrueResidual;
|
||||
|
||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
||||
LinearFunction<FieldF> *guesser;
|
||||
|
||||
MixedPrecisionConjugateGradient(RealD tol,
|
||||
Integer maxinnerit,
|
||||
Integer maxouterit,
|
||||
GridBase* _sp_grid,
|
||||
LinearOperatorBase<FieldF> &_Linop_f,
|
||||
LinearOperatorBase<FieldD> &_Linop_d) :
|
||||
Linop_f(_Linop_f), Linop_d(_Linop_d),
|
||||
Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), SinglePrecGrid(_sp_grid),
|
||||
OuterLoopNormMult(100.), guesser(NULL){ };
|
||||
|
||||
void useGuesser(LinearFunction<FieldF> &g){
|
||||
guesser = &g;
|
||||
}
|
||||
|
||||
void operator() (const FieldD &src_d_in, FieldD &sol_d){
|
||||
std::cout << GridLogMessage << "MixedPrecisionConjugateGradient: Starting mixed precision CG with outer tolerance " << Tolerance << " and inner tolerance " << InnerTolerance << std::endl;
|
||||
TotalInnerIterations = 0;
|
||||
|
||||
GridStopWatch TotalTimer;
|
||||
TotalTimer.Start();
|
||||
|
||||
int cb = src_d_in.Checkerboard();
|
||||
sol_d.Checkerboard() = cb;
|
||||
|
||||
RealD src_norm = norm2(src_d_in);
|
||||
RealD stop = src_norm * Tolerance*Tolerance;
|
||||
|
||||
GridBase* DoublePrecGrid = src_d_in.Grid();
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
tmp_d.Checkerboard() = cb;
|
||||
|
||||
FieldD tmp2_d(DoublePrecGrid);
|
||||
tmp2_d.Checkerboard() = cb;
|
||||
|
||||
FieldD src_d(DoublePrecGrid);
|
||||
src_d = src_d_in; //source for next inner iteration, computed from residual during operation
|
||||
|
||||
RealD inner_tol = InnerTolerance;
|
||||
|
||||
FieldF src_f(SinglePrecGrid);
|
||||
src_f.Checkerboard() = cb;
|
||||
|
||||
FieldF sol_f(SinglePrecGrid);
|
||||
sol_f.Checkerboard() = cb;
|
||||
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting initial inner CG with tolerance " << inner_tol << std::endl;
|
||||
ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
||||
CG_f.ErrorOnNoConverge = false;
|
||||
|
||||
GridStopWatch InnerCGtimer;
|
||||
|
||||
GridStopWatch PrecChangeTimer;
|
||||
|
||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
||||
|
||||
precisionChangeWorkspace pc_wk_sp_to_dp(DoublePrecGrid, SinglePrecGrid);
|
||||
precisionChangeWorkspace pc_wk_dp_to_sp(SinglePrecGrid, DoublePrecGrid);
|
||||
|
||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
|
||||
//Compute double precision rsd and also new RHS vector.
|
||||
Linop_d.HermOp(sol_d, tmp_d);
|
||||
RealD norm = axpy_norm(src_d, -1., tmp_d, src_d_in); //src_d is residual vector
|
||||
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " <<outer_iter<<" residual "<< norm<< " target "<< stop<<std::endl;
|
||||
|
||||
if(norm < OuterLoopNormMult * stop){
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration converged on iteration " <<outer_iter <<std::endl;
|
||||
break;
|
||||
}
|
||||
while(norm * inner_tol * inner_tol < stop) inner_tol *= 2; // inner_tol = sqrt(stop/norm) ??
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(src_f, src_d, pc_wk_dp_to_sp);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
sol_f = Zero();
|
||||
|
||||
//Optionally improve inner solver guess (eg using known eigenvectors)
|
||||
if(guesser != NULL)
|
||||
(*guesser)(src_f, sol_f);
|
||||
|
||||
//Inner CG
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " << outer_iter << " starting inner CG with tolerance " << inner_tol << std::endl;
|
||||
CG_f.Tolerance = inner_tol;
|
||||
InnerCGtimer.Start();
|
||||
CG_f(Linop_f, src_f, sol_f);
|
||||
InnerCGtimer.Stop();
|
||||
TotalInnerIterations += CG_f.IterationsToComplete;
|
||||
|
||||
//Convert sol back to double and add to double prec solution
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(tmp_d, sol_f, pc_wk_sp_to_dp);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
axpy(sol_d, 1.0, tmp_d, sol_d);
|
||||
}
|
||||
|
||||
//Final trial CG
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting final patch-up double-precision solve"<<std::endl;
|
||||
|
||||
ConjugateGradient<FieldD> CG_d(Tolerance, MaxInnerIterations);
|
||||
CG_d(Linop_d, src_d_in, sol_d);
|
||||
TotalFinalStepIterations = CG_d.IterationsToComplete;
|
||||
TrueResidual = CG_d.TrueResidual;
|
||||
|
||||
TotalTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Inner CG iterations " << TotalInnerIterations << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations << std::endl;
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,213 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMixedPrecBatched.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Raoul Hodgson <raoul.hodgson@ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_GRADIENT_MIXED_PREC_BATCHED_H
|
||||
#define GRID_CONJUGATE_GRADIENT_MIXED_PREC_BATCHED_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//Mixed precision restarted defect correction CG
|
||||
template<class FieldD,class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class MixedPrecisionConjugateGradientBatched : public LinearFunction<FieldD> {
|
||||
public:
|
||||
using LinearFunction<FieldD>::operator();
|
||||
RealD Tolerance;
|
||||
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||
Integer MaxInnerIterations;
|
||||
Integer MaxOuterIterations;
|
||||
Integer MaxPatchupIterations;
|
||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
||||
RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
|
||||
LinearOperatorBase<FieldF> &Linop_f;
|
||||
LinearOperatorBase<FieldD> &Linop_d;
|
||||
|
||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
||||
LinearFunction<FieldF> *guesser;
|
||||
bool updateResidual;
|
||||
|
||||
MixedPrecisionConjugateGradientBatched(RealD tol,
|
||||
Integer maxinnerit,
|
||||
Integer maxouterit,
|
||||
Integer maxpatchit,
|
||||
GridBase* _sp_grid,
|
||||
LinearOperatorBase<FieldF> &_Linop_f,
|
||||
LinearOperatorBase<FieldD> &_Linop_d,
|
||||
bool _updateResidual=true) :
|
||||
Linop_f(_Linop_f), Linop_d(_Linop_d),
|
||||
Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), MaxPatchupIterations(maxpatchit), SinglePrecGrid(_sp_grid),
|
||||
OuterLoopNormMult(100.), guesser(NULL), updateResidual(_updateResidual) { };
|
||||
|
||||
void useGuesser(LinearFunction<FieldF> &g){
|
||||
guesser = &g;
|
||||
}
|
||||
|
||||
void operator() (const FieldD &src_d_in, FieldD &sol_d){
|
||||
std::vector<FieldD> srcs_d_in{src_d_in};
|
||||
std::vector<FieldD> sols_d{sol_d};
|
||||
|
||||
(*this)(srcs_d_in,sols_d);
|
||||
|
||||
sol_d = sols_d[0];
|
||||
}
|
||||
|
||||
void operator() (const std::vector<FieldD> &src_d_in, std::vector<FieldD> &sol_d){
|
||||
assert(src_d_in.size() == sol_d.size());
|
||||
int NBatch = src_d_in.size();
|
||||
|
||||
std::cout << GridLogMessage << "NBatch = " << NBatch << std::endl;
|
||||
|
||||
Integer TotalOuterIterations = 0; //Number of restarts
|
||||
std::vector<Integer> TotalInnerIterations(NBatch,0); //Number of inner CG iterations
|
||||
std::vector<Integer> TotalFinalStepIterations(NBatch,0); //Number of CG iterations in final patch-up step
|
||||
|
||||
GridStopWatch TotalTimer;
|
||||
TotalTimer.Start();
|
||||
|
||||
GridStopWatch InnerCGtimer;
|
||||
GridStopWatch PrecChangeTimer;
|
||||
|
||||
int cb = src_d_in[0].Checkerboard();
|
||||
|
||||
std::vector<RealD> src_norm;
|
||||
std::vector<RealD> norm;
|
||||
std::vector<RealD> stop;
|
||||
|
||||
GridBase* DoublePrecGrid = src_d_in[0].Grid();
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
tmp_d.Checkerboard() = cb;
|
||||
|
||||
FieldD tmp2_d(DoublePrecGrid);
|
||||
tmp2_d.Checkerboard() = cb;
|
||||
|
||||
std::vector<FieldD> src_d;
|
||||
std::vector<FieldF> src_f;
|
||||
std::vector<FieldF> sol_f;
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
sol_d[i].Checkerboard() = cb;
|
||||
|
||||
src_norm.push_back(norm2(src_d_in[i]));
|
||||
norm.push_back(0.);
|
||||
stop.push_back(src_norm[i] * Tolerance*Tolerance);
|
||||
|
||||
src_d.push_back(src_d_in[i]); //source for next inner iteration, computed from residual during operation
|
||||
|
||||
src_f.push_back(SinglePrecGrid);
|
||||
src_f[i].Checkerboard() = cb;
|
||||
|
||||
sol_f.push_back(SinglePrecGrid);
|
||||
sol_f[i].Checkerboard() = cb;
|
||||
}
|
||||
|
||||
RealD inner_tol = InnerTolerance;
|
||||
|
||||
ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
||||
CG_f.ErrorOnNoConverge = false;
|
||||
|
||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
||||
|
||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout << GridLogMessage << "Outer iteration " << outer_iter << std::endl;
|
||||
|
||||
bool allConverged = true;
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
//Compute double precision rsd and also new RHS vector.
|
||||
Linop_d.HermOp(sol_d[i], tmp_d);
|
||||
norm[i] = axpy_norm(src_d[i], -1., tmp_d, src_d_in[i]); //src_d is residual vector
|
||||
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Outer iteration " << outer_iter <<" solve " << i << " residual "<< norm[i] << " target "<< stop[i] <<std::endl;
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(src_f[i], src_d[i]);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
sol_f[i] = Zero();
|
||||
|
||||
if(norm[i] > OuterLoopNormMult * stop[i]) {
|
||||
allConverged = false;
|
||||
}
|
||||
}
|
||||
if (allConverged) break;
|
||||
|
||||
if (updateResidual) {
|
||||
RealD normMax = *std::max_element(std::begin(norm), std::end(norm));
|
||||
RealD stopMax = *std::max_element(std::begin(stop), std::end(stop));
|
||||
while( normMax * inner_tol * inner_tol < stopMax) inner_tol *= 2; // inner_tol = sqrt(stop/norm) ??
|
||||
CG_f.Tolerance = inner_tol;
|
||||
}
|
||||
|
||||
//Optionally improve inner solver guess (eg using known eigenvectors)
|
||||
if(guesser != NULL) {
|
||||
(*guesser)(src_f, sol_f);
|
||||
}
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
//Inner CG
|
||||
InnerCGtimer.Start();
|
||||
CG_f(Linop_f, src_f[i], sol_f[i]);
|
||||
InnerCGtimer.Stop();
|
||||
TotalInnerIterations[i] += CG_f.IterationsToComplete;
|
||||
|
||||
//Convert sol back to double and add to double prec solution
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(tmp_d, sol_f[i]);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
axpy(sol_d[i], 1.0, tmp_d, sol_d[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//Final trial CG
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Starting final patch-up double-precision solve"<<std::endl;
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
ConjugateGradient<FieldD> CG_d(Tolerance, MaxPatchupIterations);
|
||||
CG_d(Linop_d, src_d_in[i], sol_d[i]);
|
||||
TotalFinalStepIterations[i] += CG_d.IterationsToComplete;
|
||||
}
|
||||
|
||||
TotalTimer.Stop();
|
||||
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: solve " << i << " Inner CG iterations " << TotalInnerIterations[i] << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations[i] << std::endl;
|
||||
}
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,346 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_MULTI_SHIFT_GRADIENT_H
|
||||
#define GRID_CONJUGATE_MULTI_SHIFT_GRADIENT_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Base classes for iterative processes based on operators
|
||||
// single input vec, single output vec.
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
||||
template<class Field>
|
||||
class ConjugateGradientMultiShift : public OperatorMultiFunction<Field>,
|
||||
public OperatorFunction<Field>
|
||||
{
|
||||
public:
|
||||
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
// RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
||||
int verbose;
|
||||
MultiShiftFunction shifts;
|
||||
std::vector<RealD> TrueResidualShift;
|
||||
|
||||
ConjugateGradientMultiShift(Integer maxit, const MultiShiftFunction &_shifts) :
|
||||
MaxIterations(maxit),
|
||||
shifts(_shifts)
|
||||
{
|
||||
verbose=1;
|
||||
IterationsToCompleteShift.resize(_shifts.order);
|
||||
TrueResidualShift.resize(_shifts.order);
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &src, Field &psi)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
int nshift = shifts.order;
|
||||
std::vector<Field> results(nshift,grid);
|
||||
(*this)(Linop,src,results,psi);
|
||||
}
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector<Field> &results, Field &psi)
|
||||
{
|
||||
int nshift = shifts.order;
|
||||
|
||||
(*this)(Linop,src,results);
|
||||
|
||||
psi = shifts.norm*src;
|
||||
for(int i=0;i<nshift;i++){
|
||||
psi = psi + shifts.residues[i]*results[i];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector<Field> &psi)
|
||||
{
|
||||
GRID_TRACE("ConjugateGradientMultiShift");
|
||||
|
||||
GridBase *grid = src.Grid();
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Convenience references to the info stored in "MultiShiftFunction"
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
int nshift = shifts.order;
|
||||
|
||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
||||
std::vector<RealD> alpha(nshift,1.0);
|
||||
std::vector<Field> ps(nshift,grid);// Search directions
|
||||
|
||||
assert(psi.size()==nshift);
|
||||
assert(mass.size()==nshift);
|
||||
assert(mresidual.size()==nshift);
|
||||
|
||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
||||
RealD bs[nshift];
|
||||
RealD rsq[nshift];
|
||||
RealD z[nshift][2];
|
||||
int converged[nshift];
|
||||
|
||||
const int primary =0;
|
||||
|
||||
//Primary shift fields CG iteration
|
||||
RealD a,b,c,d;
|
||||
RealD cp,bp,qq; //prev
|
||||
|
||||
// Matrix mult fields
|
||||
Field r(grid);
|
||||
Field p(grid);
|
||||
Field tmp(grid);
|
||||
Field mmp(grid);
|
||||
|
||||
// Check lightest mass
|
||||
for(int s=0;s<nshift;s++){
|
||||
assert( mass[s]>= mass[primary] );
|
||||
converged[s]=0;
|
||||
}
|
||||
|
||||
// Wire guess to zero
|
||||
// Residuals "r" are src
|
||||
// First search direction "p" is also src
|
||||
cp = norm2(src);
|
||||
|
||||
// Handle trivial case of zero src.
|
||||
if( cp == 0. ){
|
||||
for(int s=0;s<nshift;s++){
|
||||
psi[s] = Zero();
|
||||
IterationsToCompleteShift[s] = 1;
|
||||
TrueResidualShift[s] = 0.;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShift: shift "<<s
|
||||
<<" target resid "<<rsq[s]<<std::endl;
|
||||
ps[s] = src;
|
||||
}
|
||||
// r and p for primary
|
||||
r=src;
|
||||
p=src;
|
||||
|
||||
//MdagM+m[0]
|
||||
Linop.HermOpAndNorm(p,mmp,d,qq);
|
||||
axpy(mmp,mass[0],p,mmp);
|
||||
RealD rn = norm2(p);
|
||||
d += rn*mass[0];
|
||||
|
||||
// have verified that inner product of
|
||||
// p and mmp is equal to d after this since
|
||||
// the d computation is tricky
|
||||
// qq = real(innerProduct(p,mmp));
|
||||
// std::cout<<GridLogMessage << "debug equal ? qq "<<qq<<" d "<< d<<std::endl;
|
||||
|
||||
b = -cp /d;
|
||||
|
||||
// Set up the various shift variables
|
||||
int iz=0;
|
||||
z[0][1-iz] = 1.0;
|
||||
z[0][iz] = 1.0;
|
||||
bs[0] = b;
|
||||
for(int s=1;s<nshift;s++){
|
||||
z[s][1-iz] = 1.0;
|
||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
||||
bs[s] = b*z[s][iz];
|
||||
}
|
||||
|
||||
// r += b[0] A.p[0]
|
||||
// c= norm(r)
|
||||
c=axpy_norm(r,b,mmp,r);
|
||||
|
||||
for(int s=0;s<nshift;s++) {
|
||||
axpby(psi[s],0.,-bs[s]*alpha[s],src,src);
|
||||
}
|
||||
|
||||
std::cout << GridLogIterative << "ConjugateGradientMultiShift: initial rn (|src|^2) =" << rn << " qq (|MdagM src|^2) =" << qq << " d ( dot(src, [MdagM + m_0]src) ) =" << d << " c=" << c << std::endl;
|
||||
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
///////////////////////////////////////
|
||||
GridStopWatch AXPYTimer;
|
||||
GridStopWatch ShiftTimer;
|
||||
GridStopWatch QRTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
// Iteration loop
|
||||
int k;
|
||||
|
||||
for (k=1;k<=MaxIterations;k++){
|
||||
|
||||
a = c /cp;
|
||||
AXPYTimer.Start();
|
||||
axpy(p,a,p,r);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
// Note to self - direction ps is iterated seperately
|
||||
// for each shift. Does not appear to have any scope
|
||||
// for avoiding linear algebra in "single" case.
|
||||
//
|
||||
// However SAME r is used. Could load "r" and update
|
||||
// ALL ps[s]. 2/3 Bandwidth saving
|
||||
// New Kernel: Load r, vector of coeffs, vector of pointers ps
|
||||
AXPYTimer.Start();
|
||||
for(int s=0;s<nshift;s++){
|
||||
if ( ! converged[s] ) {
|
||||
if (s==0){
|
||||
axpy(ps[s],a,ps[s],r);
|
||||
} else{
|
||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
||||
axpby(ps[s],z[s][iz],as,r,ps[s]);
|
||||
}
|
||||
}
|
||||
}
|
||||
AXPYTimer.Stop();
|
||||
|
||||
cp=c;
|
||||
MatrixTimer.Start();
|
||||
//Linop.HermOpAndNorm(p,mmp,d,qq); // d is used
|
||||
// The below is faster on KNL
|
||||
Linop.HermOp(p,mmp);
|
||||
d=real(innerProduct(p,mmp));
|
||||
|
||||
MatrixTimer.Stop();
|
||||
|
||||
AXPYTimer.Start();
|
||||
axpy(mmp,mass[0],p,mmp);
|
||||
AXPYTimer.Stop();
|
||||
RealD rn = norm2(p);
|
||||
d += rn*mass[0];
|
||||
|
||||
bp=b;
|
||||
b=-cp/d;
|
||||
|
||||
AXPYTimer.Start();
|
||||
c=axpy_norm(r,b,mmp,r);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
// Toggle the recurrence history
|
||||
bs[0] = b;
|
||||
iz = 1-iz;
|
||||
ShiftTimer.Start();
|
||||
for(int s=1;s<nshift;s++){
|
||||
if((!converged[s])){
|
||||
RealD z0 = z[s][1-iz];
|
||||
RealD z1 = z[s][iz];
|
||||
z[s][iz] = z0*z1*bp
|
||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
||||
}
|
||||
}
|
||||
ShiftTimer.Stop();
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
int ss = s;
|
||||
// Scope for optimisation here in case of "single".
|
||||
// Could load psi[0] and pull all ps[s] in.
|
||||
// if ( single ) ss=primary;
|
||||
// Bandwith saving in single case is Ls * 3 -> 2+Ls, so ~ 3x saving
|
||||
// Pipelined CG gain:
|
||||
//
|
||||
// New Kernel: Load r, vector of coeffs, vector of pointers ps
|
||||
// New Kernel: Load psi[0], vector of coeffs, vector of pointers ps
|
||||
// If can predict the coefficient bs then we can fuse these and avoid write reread cyce
|
||||
// on ps[s].
|
||||
// Before: 3 x npole + 3 x npole
|
||||
// After : 2 x npole (ps[s]) => 3x speed up of multishift CG.
|
||||
|
||||
if( (!converged[s]) ) {
|
||||
axpy(psi[ss],-bs[s]*alpha[s],ps[s],psi[ss]);
|
||||
}
|
||||
}
|
||||
|
||||
// Convergence checks
|
||||
int all_converged = 1;
|
||||
for(int s=0;s<nshift;s++){
|
||||
|
||||
if ( (!converged[s]) ){
|
||||
IterationsToCompleteShift[s] = k;
|
||||
|
||||
RealD css = c * z[s][iz]* z[s][iz];
|
||||
|
||||
if(css<rsq[s]){
|
||||
if ( ! converged[s] )
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShift k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
||||
converged[s]=1;
|
||||
} else {
|
||||
all_converged=0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if ( all_converged ){
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
|
||||
std::cout<<GridLogMessage<< "CGMultiShift: All shifts have converged iteration "<<k<<std::endl;
|
||||
std::cout<<GridLogMessage<< "CGMultiShift: Checking solutions"<<std::endl;
|
||||
|
||||
// Check answers
|
||||
for(int s=0; s < nshift; s++) {
|
||||
Linop.HermOpAndNorm(psi[s],mmp,d,qq);
|
||||
axpy(tmp,mass[s],psi[s],mmp);
|
||||
axpy(r,-alpha[s],src,tmp);
|
||||
RealD rn = norm2(r);
|
||||
RealD cn = norm2(src);
|
||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
||||
std::cout<<GridLogMessage<<"CGMultiShift: shift["<<s<<"] true residual "<< TrueResidualShift[s] <<std::endl;
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
// ugly hack
|
||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
||||
// assert(0);
|
||||
}
|
||||
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,373 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christopher Kelly <ckelly@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//CK 2020: A variant of the multi-shift conjugate gradient with the matrix multiplication in single precision.
|
||||
//The residual is stored in single precision, but the search directions and solution are stored in double precision.
|
||||
//Every update_freq iterations the residual is corrected in double precision.
|
||||
//For safety the a final regular CG is applied to clean up if necessary
|
||||
|
||||
//PB Pure single, then double fixup
|
||||
|
||||
template<class FieldD, class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class ConjugateGradientMultiShiftMixedPrecCleanup : public OperatorMultiFunction<FieldD>,
|
||||
public OperatorFunction<FieldD>
|
||||
{
|
||||
public:
|
||||
|
||||
using OperatorFunction<FieldD>::operator();
|
||||
|
||||
RealD Tolerance;
|
||||
Integer MaxIterationsMshift;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
||||
int verbose;
|
||||
MultiShiftFunction shifts;
|
||||
std::vector<RealD> TrueResidualShift;
|
||||
|
||||
int ReliableUpdateFreq; //number of iterations between reliable updates
|
||||
|
||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
||||
LinearOperatorBase<FieldF> &Linop_f; //single precision
|
||||
|
||||
ConjugateGradientMultiShiftMixedPrecCleanup(Integer maxit, const MultiShiftFunction &_shifts,
|
||||
GridBase* _SinglePrecGrid, LinearOperatorBase<FieldF> &_Linop_f,
|
||||
int _ReliableUpdateFreq) :
|
||||
MaxIterationsMshift(maxit), shifts(_shifts), SinglePrecGrid(_SinglePrecGrid), Linop_f(_Linop_f), ReliableUpdateFreq(_ReliableUpdateFreq),
|
||||
MaxIterations(20000)
|
||||
{
|
||||
verbose=1;
|
||||
IterationsToCompleteShift.resize(_shifts.order);
|
||||
TrueResidualShift.resize(_shifts.order);
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, FieldD &psi)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
int nshift = shifts.order;
|
||||
std::vector<FieldD> results(nshift,grid);
|
||||
(*this)(Linop,src,results,psi);
|
||||
}
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, std::vector<FieldD> &results, FieldD &psi)
|
||||
{
|
||||
int nshift = shifts.order;
|
||||
|
||||
(*this)(Linop,src,results);
|
||||
|
||||
psi = shifts.norm*src;
|
||||
for(int i=0;i<nshift;i++){
|
||||
psi = psi + shifts.residues[i]*results[i];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop_d, const FieldD &src_d, std::vector<FieldD> &psi_d)
|
||||
{
|
||||
GRID_TRACE("ConjugateGradientMultiShiftMixedPrecCleanup");
|
||||
GridBase *DoublePrecGrid = src_d.Grid();
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Convenience references to the info stored in "MultiShiftFunction"
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
int nshift = shifts.order;
|
||||
|
||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
||||
std::vector<RealD> alpha(nshift,1.0);
|
||||
|
||||
//Double precision search directions
|
||||
FieldD p_d(DoublePrecGrid);
|
||||
std::vector<FieldF> ps_f (nshift, SinglePrecGrid);// Search directions (single precision)
|
||||
std::vector<FieldF> psi_f(nshift, SinglePrecGrid);// solutions (single precision)
|
||||
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
FieldD r_d(DoublePrecGrid);
|
||||
FieldF r_f(SinglePrecGrid);
|
||||
FieldD mmp_d(DoublePrecGrid);
|
||||
|
||||
assert(psi_d.size()==nshift);
|
||||
assert(mass.size()==nshift);
|
||||
assert(mresidual.size()==nshift);
|
||||
|
||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
||||
RealD bs[nshift];
|
||||
RealD rsq[nshift];
|
||||
RealD rsqf[nshift];
|
||||
RealD z[nshift][2];
|
||||
int converged[nshift];
|
||||
|
||||
const int primary =0;
|
||||
|
||||
//Primary shift fields CG iteration
|
||||
RealD a,b,c,d;
|
||||
RealD cp,bp,qq; //prev
|
||||
|
||||
// Matrix mult fields
|
||||
FieldF p_f(SinglePrecGrid);
|
||||
FieldF mmp_f(SinglePrecGrid);
|
||||
|
||||
// Check lightest mass
|
||||
for(int s=0;s<nshift;s++){
|
||||
assert( mass[s]>= mass[primary] );
|
||||
converged[s]=0;
|
||||
}
|
||||
|
||||
// Wire guess to zero
|
||||
// Residuals "r" are src
|
||||
// First search direction "p" is also src
|
||||
cp = norm2(src_d);
|
||||
|
||||
// Handle trivial case of zero src.
|
||||
if( cp == 0. ){
|
||||
for(int s=0;s<nshift;s++){
|
||||
psi_d[s] = Zero();
|
||||
psi_f[s] = Zero();
|
||||
IterationsToCompleteShift[s] = 1;
|
||||
TrueResidualShift[s] = 0.;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
||||
rsqf[s] =rsq[s];
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: shift "<< s <<" target resid "<<rsq[s]<<std::endl;
|
||||
// ps_d[s] = src_d;
|
||||
precisionChangeFast(ps_f[s],src_d);
|
||||
}
|
||||
// r and p for primary
|
||||
p_d = src_d; //primary copy --- make this a reference to ps_d to save axpys
|
||||
r_d = p_d;
|
||||
|
||||
//MdagM+m[0]
|
||||
precisionChangeFast(p_f,p_d);
|
||||
Linop_f.HermOpAndNorm(p_f,mmp_f,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
precisionChangeFast(tmp_d,mmp_f);
|
||||
Linop_d.HermOpAndNorm(p_d,mmp_d,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
tmp_d = tmp_d - mmp_d;
|
||||
std::cout << " Testing operators match "<<norm2(mmp_d)<<" f "<<norm2(mmp_f)<<" diff "<< norm2(tmp_d)<<std::endl;
|
||||
// assert(norm2(tmp_d)< 1.0e-4);
|
||||
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
b = -cp /d;
|
||||
|
||||
// Set up the various shift variables
|
||||
int iz=0;
|
||||
z[0][1-iz] = 1.0;
|
||||
z[0][iz] = 1.0;
|
||||
bs[0] = b;
|
||||
for(int s=1;s<nshift;s++){
|
||||
z[s][1-iz] = 1.0;
|
||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
||||
bs[s] = b*z[s][iz];
|
||||
}
|
||||
|
||||
// r += b[0] A.p[0]
|
||||
// c= norm(r)
|
||||
c=axpy_norm(r_d,b,mmp_d,r_d);
|
||||
|
||||
for(int s=0;s<nshift;s++) {
|
||||
axpby(psi_d[s],0.,-bs[s]*alpha[s],src_d,src_d);
|
||||
precisionChangeFast(psi_f[s],psi_d[s]);
|
||||
}
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
///////////////////////////////////////
|
||||
GridStopWatch AXPYTimer, ShiftTimer, QRTimer, MatrixTimer, SolverTimer, PrecChangeTimer, CleanupTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
|
||||
// Iteration loop
|
||||
int k;
|
||||
|
||||
for (k=1;k<=MaxIterationsMshift;k++){
|
||||
|
||||
a = c /cp;
|
||||
AXPYTimer.Start();
|
||||
axpy(p_d,a,p_d,r_d);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChangeFast(r_f, r_d);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
AXPYTimer.Start();
|
||||
for(int s=0;s<nshift;s++){
|
||||
if ( ! converged[s] ) {
|
||||
if (s==0){
|
||||
axpy(ps_f[s],a,ps_f[s],r_f);
|
||||
} else{
|
||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
||||
axpby(ps_f[s],z[s][iz],as,r_f,ps_f[s]);
|
||||
}
|
||||
}
|
||||
}
|
||||
AXPYTimer.Stop();
|
||||
|
||||
cp=c;
|
||||
PrecChangeTimer.Start();
|
||||
precisionChangeFast(p_f, p_d); //get back single prec search direction for linop
|
||||
PrecChangeTimer.Stop();
|
||||
MatrixTimer.Start();
|
||||
Linop_f.HermOp(p_f,mmp_f);
|
||||
MatrixTimer.Stop();
|
||||
PrecChangeTimer.Start();
|
||||
precisionChangeFast(mmp_d, mmp_f); // From Float to Double
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
d=real(innerProduct(p_d,mmp_d));
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
bp=b;
|
||||
b=-cp/d;
|
||||
|
||||
// Toggle the recurrence history
|
||||
bs[0] = b;
|
||||
iz = 1-iz;
|
||||
ShiftTimer.Start();
|
||||
for(int s=1;s<nshift;s++){
|
||||
if((!converged[s])){
|
||||
RealD z0 = z[s][1-iz];
|
||||
RealD z1 = z[s][iz];
|
||||
z[s][iz] = z0*z1*bp
|
||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
||||
}
|
||||
}
|
||||
ShiftTimer.Stop();
|
||||
|
||||
//Update single precision solutions
|
||||
AXPYTimer.Start();
|
||||
for(int s=0;s<nshift;s++){
|
||||
int ss = s;
|
||||
if( (!converged[s]) ) {
|
||||
axpy(psi_f[ss],-bs[s]*alpha[s],ps_f[s],psi_f[ss]);
|
||||
}
|
||||
}
|
||||
c = axpy_norm(r_d,b,mmp_d,r_d);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
// Convergence checks
|
||||
int all_converged = 1;
|
||||
for(int s=0;s<nshift;s++){
|
||||
|
||||
if ( (!converged[s]) ){
|
||||
IterationsToCompleteShift[s] = k;
|
||||
|
||||
RealD css = c * z[s][iz]* z[s][iz];
|
||||
|
||||
if(css<rsqf[s]){
|
||||
if ( ! converged[s] )
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
||||
converged[s]=1;
|
||||
} else {
|
||||
all_converged=0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if ( all_converged || k == MaxIterationsMshift-1){
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
precisionChangeFast(psi_d[s],psi_f[s]);
|
||||
}
|
||||
|
||||
|
||||
if ( all_converged ){
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: All shifts have converged iteration "<<k<<std::endl;
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: Checking solutions"<<std::endl;
|
||||
} else {
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: Not all shifts have converged iteration "<<k<<std::endl;
|
||||
}
|
||||
|
||||
// Check answers
|
||||
for(int s=0; s < nshift; s++) {
|
||||
Linop_d.HermOpAndNorm(psi_d[s],mmp_d,d,qq);
|
||||
axpy(tmp_d,mass[s],psi_d[s],mmp_d);
|
||||
axpy(r_d,-alpha[s],src_d,tmp_d);
|
||||
RealD rn = norm2(r_d);
|
||||
RealD cn = norm2(src_d);
|
||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: shift["<<s<<"] true residual "<< TrueResidualShift[s] << " target " << mresidual[s] << std::endl;
|
||||
|
||||
//If we have not reached the desired tolerance, do a (mixed precision) CG cleanup
|
||||
if(rn >= rsq[s]){
|
||||
CleanupTimer.Start();
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: performing cleanup step for shift " << s << std::endl;
|
||||
|
||||
//Setup linear operators for final cleanup
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldD> Linop_shift_d(Linop_d, mass[s]);
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldF> Linop_shift_f(Linop_f, mass[s]);
|
||||
|
||||
MixedPrecisionConjugateGradient<FieldD,FieldF> cg(mresidual[s], MaxIterations, MaxIterations, SinglePrecGrid, Linop_shift_f, Linop_shift_d);
|
||||
cg(src_d, psi_d[s]);
|
||||
|
||||
TrueResidualShift[s] = cg.TrueResidual;
|
||||
CleanupTimer.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientMultiShiftMixedPrecCleanup: Time Breakdown for body"<<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tPrecision Change " << PrecChangeTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tFinal Cleanup " << CleanupTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver+Cleanup " << SolverTimer.Elapsed() + CleanupTimer.Elapsed() << std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
||||
assert(0);
|
||||
}
|
||||
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,416 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christopher Kelly <ckelly@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_GRADIENT_MULTI_SHIFT_MIXEDPREC_H
|
||||
#define GRID_CONJUGATE_GRADIENT_MULTI_SHIFT_MIXEDPREC_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//CK 2020: A variant of the multi-shift conjugate gradient with the matrix multiplication in single precision.
|
||||
//The residual is stored in single precision, but the search directions and solution are stored in double precision.
|
||||
//Every update_freq iterations the residual is corrected in double precision.
|
||||
|
||||
//For safety the a final regular CG is applied to clean up if necessary
|
||||
|
||||
//Linop to add shift to input linop, used in cleanup CG
|
||||
namespace ConjugateGradientMultiShiftMixedPrecSupport{
|
||||
template<typename Field>
|
||||
class ShiftedLinop: public LinearOperatorBase<Field>{
|
||||
public:
|
||||
LinearOperatorBase<Field> &linop_base;
|
||||
RealD shift;
|
||||
|
||||
ShiftedLinop(LinearOperatorBase<Field> &_linop_base, RealD _shift): linop_base(_linop_base), shift(_shift){}
|
||||
|
||||
void OpDiag (const Field &in, Field &out){ assert(0); }
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp){ assert(0); }
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){ assert(0); }
|
||||
|
||||
void Op (const Field &in, Field &out){ assert(0); }
|
||||
void AdjOp (const Field &in, Field &out){ assert(0); }
|
||||
|
||||
void HermOp(const Field &in, Field &out){
|
||||
linop_base.HermOp(in, out);
|
||||
axpy(out, shift, in, out);
|
||||
}
|
||||
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
HermOp(in,out);
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
template<class FieldD, class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class ConjugateGradientMultiShiftMixedPrec : public OperatorMultiFunction<FieldD>,
|
||||
public OperatorFunction<FieldD>
|
||||
{
|
||||
public:
|
||||
|
||||
using OperatorFunction<FieldD>::operator();
|
||||
|
||||
RealD Tolerance;
|
||||
Integer MaxIterationsMshift;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
||||
int verbose;
|
||||
MultiShiftFunction shifts;
|
||||
std::vector<RealD> TrueResidualShift;
|
||||
|
||||
int ReliableUpdateFreq; //number of iterations between reliable updates
|
||||
|
||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
||||
LinearOperatorBase<FieldF> &Linop_f; //single precision
|
||||
|
||||
ConjugateGradientMultiShiftMixedPrec(Integer maxit, const MultiShiftFunction &_shifts,
|
||||
GridBase* _SinglePrecGrid, LinearOperatorBase<FieldF> &_Linop_f,
|
||||
int _ReliableUpdateFreq) :
|
||||
MaxIterationsMshift(maxit), shifts(_shifts), SinglePrecGrid(_SinglePrecGrid), Linop_f(_Linop_f), ReliableUpdateFreq(_ReliableUpdateFreq),
|
||||
MaxIterations(20000)
|
||||
{
|
||||
verbose=1;
|
||||
IterationsToCompleteShift.resize(_shifts.order);
|
||||
TrueResidualShift.resize(_shifts.order);
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, FieldD &psi)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
int nshift = shifts.order;
|
||||
std::vector<FieldD> results(nshift,grid);
|
||||
(*this)(Linop,src,results,psi);
|
||||
}
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, std::vector<FieldD> &results, FieldD &psi)
|
||||
{
|
||||
int nshift = shifts.order;
|
||||
|
||||
(*this)(Linop,src,results);
|
||||
|
||||
psi = shifts.norm*src;
|
||||
for(int i=0;i<nshift;i++){
|
||||
psi = psi + shifts.residues[i]*results[i];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop_d, const FieldD &src_d, std::vector<FieldD> &psi_d)
|
||||
{
|
||||
GRID_TRACE("ConjugateGradientMultiShiftMixedPrec");
|
||||
GridBase *DoublePrecGrid = src_d.Grid();
|
||||
|
||||
precisionChangeWorkspace pc_wk_s_to_d(DoublePrecGrid,SinglePrecGrid);
|
||||
precisionChangeWorkspace pc_wk_d_to_s(SinglePrecGrid,DoublePrecGrid);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Convenience references to the info stored in "MultiShiftFunction"
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
int nshift = shifts.order;
|
||||
|
||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
||||
std::vector<RealD> alpha(nshift,1.0);
|
||||
|
||||
//Double precision search directions
|
||||
FieldD p_d(DoublePrecGrid);
|
||||
std::vector<FieldD> ps_d(nshift, DoublePrecGrid);// Search directions (double precision)
|
||||
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
FieldD r_d(DoublePrecGrid);
|
||||
FieldD mmp_d(DoublePrecGrid);
|
||||
|
||||
assert(psi_d.size()==nshift);
|
||||
assert(mass.size()==nshift);
|
||||
assert(mresidual.size()==nshift);
|
||||
|
||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
||||
RealD bs[nshift];
|
||||
RealD rsq[nshift];
|
||||
RealD rsqf[nshift];
|
||||
RealD z[nshift][2];
|
||||
int converged[nshift];
|
||||
|
||||
const int primary =0;
|
||||
|
||||
//Primary shift fields CG iteration
|
||||
RealD a,b,c,d;
|
||||
RealD cp,bp,qq; //prev
|
||||
|
||||
// Matrix mult fields
|
||||
FieldF p_f(SinglePrecGrid);
|
||||
FieldF mmp_f(SinglePrecGrid);
|
||||
|
||||
// Check lightest mass
|
||||
for(int s=0;s<nshift;s++){
|
||||
assert( mass[s]>= mass[primary] );
|
||||
converged[s]=0;
|
||||
}
|
||||
|
||||
// Wire guess to zero
|
||||
// Residuals "r" are src
|
||||
// First search direction "p" is also src
|
||||
cp = norm2(src_d);
|
||||
|
||||
// Handle trivial case of zero src.
|
||||
if( cp == 0. ){
|
||||
for(int s=0;s<nshift;s++){
|
||||
psi_d[s] = Zero();
|
||||
IterationsToCompleteShift[s] = 1;
|
||||
TrueResidualShift[s] = 0.;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
||||
rsqf[s] =rsq[s];
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: shift "<< s <<" target resid "<<rsq[s]<<std::endl;
|
||||
ps_d[s] = src_d;
|
||||
}
|
||||
// r and p for primary
|
||||
p_d = src_d; //primary copy --- make this a reference to ps_d to save axpys
|
||||
r_d = p_d;
|
||||
|
||||
//MdagM+m[0]
|
||||
precisionChange(p_f, p_d, pc_wk_d_to_s);
|
||||
|
||||
Linop_f.HermOpAndNorm(p_f,mmp_f,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
precisionChange(tmp_d, mmp_f, pc_wk_s_to_d);
|
||||
Linop_d.HermOpAndNorm(p_d,mmp_d,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
tmp_d = tmp_d - mmp_d;
|
||||
std::cout << " Testing operators match "<<norm2(mmp_d)<<" f "<<norm2(mmp_f)<<" diff "<< norm2(tmp_d)<<std::endl;
|
||||
// assert(norm2(tmp_d)< 1.0e-4);
|
||||
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
b = -cp /d;
|
||||
|
||||
// Set up the various shift variables
|
||||
int iz=0;
|
||||
z[0][1-iz] = 1.0;
|
||||
z[0][iz] = 1.0;
|
||||
bs[0] = b;
|
||||
for(int s=1;s<nshift;s++){
|
||||
z[s][1-iz] = 1.0;
|
||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
||||
bs[s] = b*z[s][iz];
|
||||
}
|
||||
|
||||
// r += b[0] A.p[0]
|
||||
// c= norm(r)
|
||||
c=axpy_norm(r_d,b,mmp_d,r_d);
|
||||
|
||||
for(int s=0;s<nshift;s++) {
|
||||
axpby(psi_d[s],0.,-bs[s]*alpha[s],src_d,src_d);
|
||||
}
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
///////////////////////////////////////
|
||||
GridStopWatch AXPYTimer, ShiftTimer, QRTimer, MatrixTimer, SolverTimer, PrecChangeTimer, CleanupTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
|
||||
// Iteration loop
|
||||
int k;
|
||||
|
||||
for (k=1;k<=MaxIterationsMshift;k++){
|
||||
|
||||
a = c /cp;
|
||||
AXPYTimer.Start();
|
||||
axpy(p_d,a,p_d,r_d);
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
if ( ! converged[s] ) {
|
||||
if (s==0){
|
||||
axpy(ps_d[s],a,ps_d[s],r_d);
|
||||
} else{
|
||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
||||
axpby(ps_d[s],z[s][iz],as,r_d,ps_d[s]);
|
||||
}
|
||||
}
|
||||
}
|
||||
AXPYTimer.Stop();
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(p_f, p_d, pc_wk_d_to_s); //get back single prec search direction for linop
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
cp=c;
|
||||
MatrixTimer.Start();
|
||||
Linop_f.HermOp(p_f,mmp_f);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(mmp_d, mmp_f, pc_wk_s_to_d); // From Float to Double
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
AXPYTimer.Start();
|
||||
d=real(innerProduct(p_d,mmp_d));
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
AXPYTimer.Stop();
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
bp=b;
|
||||
b=-cp/d;
|
||||
|
||||
// Toggle the recurrence history
|
||||
bs[0] = b;
|
||||
iz = 1-iz;
|
||||
ShiftTimer.Start();
|
||||
for(int s=1;s<nshift;s++){
|
||||
if((!converged[s])){
|
||||
RealD z0 = z[s][1-iz];
|
||||
RealD z1 = z[s][iz];
|
||||
z[s][iz] = z0*z1*bp
|
||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
||||
}
|
||||
}
|
||||
ShiftTimer.Stop();
|
||||
|
||||
//Update double precision solutions
|
||||
AXPYTimer.Start();
|
||||
for(int s=0;s<nshift;s++){
|
||||
int ss = s;
|
||||
if( (!converged[s]) ) {
|
||||
axpy(psi_d[ss],-bs[s]*alpha[s],ps_d[s],psi_d[ss]);
|
||||
}
|
||||
}
|
||||
|
||||
//Perform reliable update if necessary; otherwise update residual from single-prec mmp
|
||||
c = axpy_norm(r_d,b,mmp_d,r_d);
|
||||
|
||||
AXPYTimer.Stop();
|
||||
|
||||
if(k % ReliableUpdateFreq == 0){
|
||||
RealD c_old = c;
|
||||
//Replace r with true residual
|
||||
MatrixTimer.Start();
|
||||
Linop_d.HermOp(psi_d[0],mmp_d);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
AXPYTimer.Start();
|
||||
axpy(mmp_d,mass[0],psi_d[0],mmp_d);
|
||||
|
||||
c = axpy_norm(r_d, -1.0, mmp_d, src_d);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec k="<<k<< ", replaced |r|^2 = "<<c_old <<" with |r|^2 = "<<c<<std::endl;
|
||||
}
|
||||
|
||||
// Convergence checks
|
||||
int all_converged = 1;
|
||||
for(int s=0;s<nshift;s++){
|
||||
|
||||
if ( (!converged[s]) ){
|
||||
IterationsToCompleteShift[s] = k;
|
||||
|
||||
RealD css = c * z[s][iz]* z[s][iz];
|
||||
|
||||
if(css<rsqf[s]){
|
||||
if ( ! converged[s] )
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
||||
converged[s]=1;
|
||||
} else {
|
||||
all_converged=0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if ( all_converged || k == MaxIterationsMshift-1){
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
if ( all_converged ){
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: All shifts have converged iteration "<<k<<std::endl;
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: Checking solutions"<<std::endl;
|
||||
} else {
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: Not all shifts have converged iteration "<<k<<std::endl;
|
||||
}
|
||||
|
||||
// Check answers
|
||||
for(int s=0; s < nshift; s++) {
|
||||
Linop_d.HermOpAndNorm(psi_d[s],mmp_d,d,qq);
|
||||
axpy(tmp_d,mass[s],psi_d[s],mmp_d);
|
||||
axpy(r_d,-alpha[s],src_d,tmp_d);
|
||||
RealD rn = norm2(r_d);
|
||||
RealD cn = norm2(src_d);
|
||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: shift["<<s<<"] true residual "<< TrueResidualShift[s] << " target " << mresidual[s] << std::endl;
|
||||
|
||||
//If we have not reached the desired tolerance, do a (mixed precision) CG cleanup
|
||||
if(rn >= rsq[s]){
|
||||
CleanupTimer.Start();
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: performing cleanup step for shift " << s << std::endl;
|
||||
|
||||
//Setup linear operators for final cleanup
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldD> Linop_shift_d(Linop_d, mass[s]);
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldF> Linop_shift_f(Linop_f, mass[s]);
|
||||
|
||||
MixedPrecisionConjugateGradient<FieldD,FieldF> cg(mresidual[s], MaxIterations, MaxIterations, SinglePrecGrid, Linop_shift_f, Linop_shift_d);
|
||||
cg(src_d, psi_d[s]);
|
||||
|
||||
TrueResidualShift[s] = cg.TrueResidual;
|
||||
CleanupTimer.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientMultiShiftMixedPrec: Time Breakdown for body"<<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tPrecision Change " << PrecChangeTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tFinal Cleanup " << CleanupTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver+Cleanup " << SolverTimer.Elapsed() + CleanupTimer.Elapsed() << std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
||||
assert(0);
|
||||
}
|
||||
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,277 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_GRADIENT_RELIABLE_UPDATE_H
|
||||
#define GRID_CONJUGATE_GRADIENT_RELIABLE_UPDATE_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class FieldD,class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class ConjugateGradientReliableUpdate : public LinearFunction<FieldD> {
|
||||
public:
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
Integer ReliableUpdatesPerformed;
|
||||
|
||||
bool DoFinalCleanup; //Final DP cleanup, defaults to true
|
||||
Integer IterationsToCleanup; //Final DP cleanup step iterations
|
||||
|
||||
LinearOperatorBase<FieldF> &Linop_f;
|
||||
LinearOperatorBase<FieldD> &Linop_d;
|
||||
GridBase* SinglePrecGrid;
|
||||
RealD Delta; //reliable update parameter. A reliable update is performed when the residual drops by a factor of Delta relative to its value at the last update
|
||||
|
||||
//Optional ability to switch to a different linear operator once the tolerance reaches a certain point. Useful for single/half -> single/single
|
||||
LinearOperatorBase<FieldF> *Linop_fallback;
|
||||
RealD fallback_transition_tol;
|
||||
|
||||
|
||||
ConjugateGradientReliableUpdate(RealD tol, Integer maxit, RealD _delta, GridBase* _sp_grid, LinearOperatorBase<FieldF> &_Linop_f, LinearOperatorBase<FieldD> &_Linop_d, bool err_on_no_conv = true)
|
||||
: Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
Delta(_delta),
|
||||
Linop_f(_Linop_f),
|
||||
Linop_d(_Linop_d),
|
||||
SinglePrecGrid(_sp_grid),
|
||||
ErrorOnNoConverge(err_on_no_conv),
|
||||
DoFinalCleanup(true),
|
||||
Linop_fallback(NULL)
|
||||
{
|
||||
assert(Delta > 0. && Delta < 1. && "Expect 0 < Delta < 1");
|
||||
};
|
||||
|
||||
void setFallbackLinop(LinearOperatorBase<FieldF> &_Linop_fallback, const RealD _fallback_transition_tol){
|
||||
Linop_fallback = &_Linop_fallback;
|
||||
fallback_transition_tol = _fallback_transition_tol;
|
||||
}
|
||||
|
||||
void operator()(const FieldD &src, FieldD &psi) {
|
||||
GRID_TRACE("ConjugateGradientReliableUpdate");
|
||||
LinearOperatorBase<FieldF> *Linop_f_use = &Linop_f;
|
||||
bool using_fallback = false;
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
RealD cp, c, a, d, b, ssq, qq, b_pred;
|
||||
|
||||
FieldD p(src);
|
||||
FieldD mmp(src);
|
||||
FieldD r(src);
|
||||
|
||||
// Initial residual computation & set up
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
Linop_d.HermOpAndNorm(psi, mmp, d, b);
|
||||
|
||||
r = src - mmp;
|
||||
p = r;
|
||||
|
||||
a = norm2(p);
|
||||
cp = a;
|
||||
ssq = norm2(src);
|
||||
|
||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: src " << ssq << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: mp " << d << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: mmp " << b << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: cp,r " << cp << std::endl;
|
||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: p " << a << std::endl;
|
||||
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
// Check if guess is really REALLY good :)
|
||||
if (cp <= rsq) {
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate guess was REALLY good\n";
|
||||
std::cout << GridLogMessage << "\tComputed residual " << std::sqrt(cp / ssq)<<std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
//Single prec initialization
|
||||
precisionChangeWorkspace pc_wk_sp_to_dp(src.Grid(), SinglePrecGrid);
|
||||
precisionChangeWorkspace pc_wk_dp_to_sp(SinglePrecGrid, src.Grid());
|
||||
|
||||
FieldF r_f(SinglePrecGrid);
|
||||
r_f.Checkerboard() = r.Checkerboard();
|
||||
precisionChange(r_f, r, pc_wk_dp_to_sp);
|
||||
|
||||
FieldF psi_f(r_f);
|
||||
psi_f = Zero();
|
||||
|
||||
FieldF p_f(r_f);
|
||||
FieldF mmp_f(r_f);
|
||||
|
||||
RealD MaxResidSinceLastRelUp = cp; //initial residual
|
||||
|
||||
std::cout << GridLogIterative << std::setprecision(4)
|
||||
<< "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
GridStopWatch PrecChangeTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
int k = 0;
|
||||
int l = 0;
|
||||
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
c = cp;
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop_f_use->HermOpAndNorm(p_f, mmp_f, d, qq);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
a = c / d;
|
||||
b_pred = a * (a * qq - d) / c;
|
||||
|
||||
cp = axpy_norm(r_f, -a, mmp_f, r_f);
|
||||
b = cp / c;
|
||||
|
||||
// Fuse these loops ; should be really easy
|
||||
psi_f = a * p_f + psi_f;
|
||||
//p_f = p_f * b + r_f;
|
||||
|
||||
LinalgTimer.Stop();
|
||||
|
||||
std::cout << GridLogIterative << "ConjugateGradientReliableUpdate: Iteration " << k
|
||||
<< " residual " << cp << " target " << rsq << std::endl;
|
||||
std::cout << GridLogDebug << "a = "<< a << " b_pred = "<< b_pred << " b = "<< b << std::endl;
|
||||
std::cout << GridLogDebug << "qq = "<< qq << " d = "<< d << " c = "<< c << std::endl;
|
||||
|
||||
if(cp > MaxResidSinceLastRelUp){
|
||||
std::cout << GridLogIterative << "ConjugateGradientReliableUpdate: updating MaxResidSinceLastRelUp : " << MaxResidSinceLastRelUp << " -> " << cp << std::endl;
|
||||
MaxResidSinceLastRelUp = cp;
|
||||
}
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
//Although not written in the paper, I assume that I have to add on the final solution
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(mmp, psi_f, pc_wk_sp_to_dp);
|
||||
PrecChangeTimer.Stop();
|
||||
psi = psi + mmp;
|
||||
|
||||
|
||||
SolverTimer.Stop();
|
||||
Linop_d.HermOpAndNorm(psi, mmp, d, qq);
|
||||
p = mmp - src;
|
||||
|
||||
RealD srcnorm = std::sqrt(norm2(src));
|
||||
RealD resnorm = std::sqrt(norm2(p));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate Converged on iteration " << k << " after " << l << " reliable updates" << std::endl;
|
||||
std::cout << GridLogMessage << "\tComputed residual " << std::sqrt(cp / ssq)<<std::endl;
|
||||
std::cout << GridLogMessage << "\tTrue residual " << true_residual<<std::endl;
|
||||
std::cout << GridLogMessage << "\tTarget " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "Time breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tPrecChange " << PrecChangeTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tPrecChange avg time " << PrecChangeTimer.Elapsed()/(2*l+1) <<std::endl;
|
||||
|
||||
|
||||
IterationsToComplete = k;
|
||||
ReliableUpdatesPerformed = l;
|
||||
|
||||
if(DoFinalCleanup){
|
||||
//Do a final CG to cleanup
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate performing final cleanup.\n";
|
||||
ConjugateGradient<FieldD> CG(Tolerance,MaxIterations);
|
||||
CG.ErrorOnNoConverge = ErrorOnNoConverge;
|
||||
CG(Linop_d,src,psi);
|
||||
IterationsToCleanup = CG.IterationsToComplete;
|
||||
}
|
||||
else if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate complete.\n";
|
||||
return;
|
||||
}
|
||||
else if(cp < Delta * MaxResidSinceLastRelUp) { //reliable update
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate "
|
||||
<< cp << "(residual) < " << Delta << "(Delta) * " << MaxResidSinceLastRelUp << "(MaxResidSinceLastRelUp) on iteration " << k << " : performing reliable update\n";
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(mmp, psi_f, pc_wk_sp_to_dp);
|
||||
PrecChangeTimer.Stop();
|
||||
psi = psi + mmp;
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop_d.HermOpAndNorm(psi, mmp, d, qq);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
r = src - mmp;
|
||||
|
||||
psi_f = Zero();
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(r_f, r, pc_wk_dp_to_sp);
|
||||
PrecChangeTimer.Stop();
|
||||
cp = norm2(r);
|
||||
MaxResidSinceLastRelUp = cp;
|
||||
|
||||
b = cp/c;
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate new residual " << cp << std::endl;
|
||||
|
||||
l = l+1;
|
||||
}
|
||||
|
||||
p_f = p_f * b + r_f; //update search vector after reliable update appears to help convergence
|
||||
|
||||
if(!using_fallback && Linop_fallback != NULL && cp < fallback_transition_tol){
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate switching to fallback linear operator on iteration " << k << " at residual " << cp << std::endl;
|
||||
Linop_f_use = Linop_fallback;
|
||||
using_fallback = true;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate did NOT converge"
|
||||
<< std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
ReliableUpdatesPerformed = l;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
|
||||
#endif
|
||||
@@ -1,113 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_RESIDUAL_H
|
||||
#define GRID_CONJUGATE_RESIDUAL_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Base classes for iterative processes based on operators
|
||||
// single input vec, single output vec.
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
||||
template<class Field>
|
||||
class ConjugateResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
int verbose;
|
||||
|
||||
ConjugateResidual(RealD tol,Integer maxit) : Tolerance(tol), MaxIterations(maxit) {
|
||||
verbose=0;
|
||||
};
|
||||
|
||||
void operator() (LinearOperatorBase<Field> &Linop,const Field &src, Field &psi){
|
||||
|
||||
RealD a, b; // c, d;
|
||||
RealD cp, ssq,rsq;
|
||||
|
||||
RealD rAr, rAAr, rArp;
|
||||
RealD pAp, pAAp;
|
||||
|
||||
GridBase *grid = src.Grid();
|
||||
psi=Zero();
|
||||
Field r(grid), p(grid), Ap(grid), Ar(grid);
|
||||
|
||||
r=src;
|
||||
p=src;
|
||||
|
||||
Linop.HermOpAndNorm(p,Ap,pAp,pAAp);
|
||||
Linop.HermOpAndNorm(r,Ar,rAr,rAAr);
|
||||
|
||||
cp =norm2(r);
|
||||
ssq=norm2(src);
|
||||
rsq=Tolerance*Tolerance*ssq;
|
||||
|
||||
if (verbose) std::cout<<GridLogMessage<<"ConjugateResidual: iteration " <<0<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
||||
|
||||
for(int k=1;k<MaxIterations;k++){
|
||||
|
||||
a = rAr/pAAp;
|
||||
|
||||
axpy(psi,a,p,psi);
|
||||
|
||||
cp = axpy_norm(r,-a,Ap,r);
|
||||
|
||||
rArp=rAr;
|
||||
|
||||
Linop.HermOpAndNorm(r,Ar,rAr,rAAr);
|
||||
|
||||
b =rAr/rArp;
|
||||
|
||||
axpy(p,b,p,r);
|
||||
pAAp=axpy_norm(Ap,b,Ap,Ar);
|
||||
|
||||
if(verbose) std::cout<<GridLogMessage<<"ConjugateResidual: iteration " <<k<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
||||
|
||||
if(cp<rsq) {
|
||||
Linop.HermOp(psi,Ap);
|
||||
axpy(r,-1.0,src,Ap);
|
||||
RealD true_resid = norm2(r)/ssq;
|
||||
std::cout<<GridLogMessage<<"ConjugateResidual: Converged on iteration " <<k
|
||||
<< " computed residual "<<std::sqrt(cp/ssq)
|
||||
<< " true residual "<<std::sqrt(true_resid)
|
||||
<< " target " <<Tolerance <<std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::cout<<GridLogMessage<<"ConjugateResidual did NOT converge"<<std::endl;
|
||||
assert(0);
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,157 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_DEFLATION_H
|
||||
#define GRID_DEFLATION_H
|
||||
|
||||
namespace Grid {
|
||||
|
||||
template<class Field>
|
||||
class ZeroGuesser: public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field &guess) { guess = Zero(); };
|
||||
};
|
||||
template<class Field>
|
||||
class DoNothingGuesser: public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field &guess) { };
|
||||
};
|
||||
template<class Field>
|
||||
class SourceGuesser: public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field &guess) { guess = src; };
|
||||
};
|
||||
|
||||
////////////////////////////////
|
||||
// Fine grid deflation
|
||||
////////////////////////////////
|
||||
template<class Field>
|
||||
class DeflatedGuesser: public LinearFunction<Field> {
|
||||
private:
|
||||
const std::vector<Field> &evec;
|
||||
const std::vector<RealD> &eval;
|
||||
const unsigned int N;
|
||||
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
|
||||
DeflatedGuesser(const std::vector<Field> & _evec,const std::vector<RealD> & _eval)
|
||||
: DeflatedGuesser(_evec, _eval, _evec.size())
|
||||
{}
|
||||
|
||||
DeflatedGuesser(const std::vector<Field> & _evec, const std::vector<RealD> & _eval, const unsigned int _N)
|
||||
: evec(_evec), eval(_eval), N(_N)
|
||||
{
|
||||
assert(evec.size()==eval.size());
|
||||
assert(N <= evec.size());
|
||||
}
|
||||
|
||||
virtual void operator()(const Field &src,Field &guess) {
|
||||
guess = Zero();
|
||||
for (int i=0;i<N;i++) {
|
||||
const Field& tmp = evec[i];
|
||||
axpy(guess,TensorRemove(innerProduct(tmp,src)) / eval[i],tmp,guess);
|
||||
}
|
||||
guess.Checkerboard() = src.Checkerboard();
|
||||
}
|
||||
};
|
||||
|
||||
template<class FineField, class CoarseField>
|
||||
class LocalCoherenceDeflatedGuesser: public LinearFunction<FineField> {
|
||||
private:
|
||||
const std::vector<FineField> &subspace;
|
||||
const std::vector<CoarseField> &evec_coarse;
|
||||
const std::vector<RealD> &eval_coarse;
|
||||
public:
|
||||
|
||||
using LinearFunction<FineField>::operator();
|
||||
LocalCoherenceDeflatedGuesser(const std::vector<FineField> &_subspace,
|
||||
const std::vector<CoarseField> &_evec_coarse,
|
||||
const std::vector<RealD> &_eval_coarse)
|
||||
: subspace(_subspace),
|
||||
evec_coarse(_evec_coarse),
|
||||
eval_coarse(_eval_coarse)
|
||||
{
|
||||
}
|
||||
|
||||
void operator()(const FineField &src,FineField &guess) {
|
||||
int N = (int)evec_coarse.size();
|
||||
CoarseField src_coarse(evec_coarse[0].Grid());
|
||||
CoarseField guess_coarse(evec_coarse[0].Grid()); guess_coarse = Zero();
|
||||
blockProject(src_coarse,src,subspace);
|
||||
for (int i=0;i<N;i++) {
|
||||
const CoarseField & tmp = evec_coarse[i];
|
||||
axpy(guess_coarse,TensorRemove(innerProduct(tmp,src_coarse)) / eval_coarse[i],tmp,guess_coarse);
|
||||
}
|
||||
blockPromote(guess_coarse,guess,subspace);
|
||||
guess.Checkerboard() = src.Checkerboard();
|
||||
};
|
||||
|
||||
void operator()(const std::vector<FineField> &src,std::vector<FineField> &guess) {
|
||||
int Nevec = (int)evec_coarse.size();
|
||||
int Nsrc = (int)src.size();
|
||||
// make temp variables
|
||||
std::vector<CoarseField> src_coarse(Nsrc,evec_coarse[0].Grid());
|
||||
std::vector<CoarseField> guess_coarse(Nsrc,evec_coarse[0].Grid());
|
||||
//Preporcessing
|
||||
std::cout << GridLogMessage << "Start BlockProject for loop" << std::endl;
|
||||
for (int j=0;j<Nsrc;j++)
|
||||
{
|
||||
guess_coarse[j] = Zero();
|
||||
std::cout << GridLogMessage << "BlockProject iter: " << j << std::endl;
|
||||
blockProject(src_coarse[j],src[j],subspace);
|
||||
}
|
||||
//deflation set up for eigen vector batchsize 1 and source batch size equal number of sources
|
||||
std::cout << GridLogMessage << "Start ProjectAccum for loop" << std::endl;
|
||||
for (int i=0;i<Nevec;i++)
|
||||
{
|
||||
std::cout << GridLogMessage << "ProjectAccum Nvec: " << i << std::endl;
|
||||
const CoarseField & tmp = evec_coarse[i];
|
||||
for (int j=0;j<Nsrc;j++)
|
||||
{
|
||||
axpy(guess_coarse[j],TensorRemove(innerProduct(tmp,src_coarse[j])) / eval_coarse[i],tmp,guess_coarse[j]);
|
||||
}
|
||||
}
|
||||
//postprocessing
|
||||
std::cout << GridLogMessage << "Start BlockPromote for loop" << std::endl;
|
||||
for (int j=0;j<Nsrc;j++)
|
||||
{
|
||||
std::cout << GridLogMessage << "BlockProject iter: " << j << std::endl;
|
||||
blockPromote(guess_coarse[j],guess[j],subspace);
|
||||
guess[j].Checkerboard() = src[j].Checkerboard();
|
||||
}
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
||||
}
|
||||
#endif
|
||||
@@ -1,258 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/FlexibleCommunicationAvoidingGeneralisedMinimalResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_FLEXIBLE_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
#define GRID_FLEXIBLE_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
|
||||
namespace Grid {
|
||||
|
||||
template<class Field>
|
||||
class FlexibleCommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an assert when FCAGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
|
||||
Integer MaxIterations;
|
||||
Integer RestartLength;
|
||||
Integer MaxNumberOfRestarts;
|
||||
Integer IterationCount; // Number of iterations the FCAGMRES took to finish,
|
||||
// filled in upon completion
|
||||
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch PrecTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch QrTimer;
|
||||
GridStopWatch CompSolutionTimer;
|
||||
|
||||
Eigen::MatrixXcd H;
|
||||
|
||||
std::vector<ComplexD> y;
|
||||
std::vector<ComplexD> gamma;
|
||||
std::vector<ComplexD> c;
|
||||
std::vector<ComplexD> s;
|
||||
|
||||
LinearFunction<Field> &Preconditioner;
|
||||
|
||||
FlexibleCommunicationAvoidingGeneralisedMinimalResidual(RealD tol,
|
||||
Integer maxit,
|
||||
LinearFunction<Field> &Prec,
|
||||
Integer restart_length,
|
||||
bool err_on_no_conv = true)
|
||||
: Tolerance(tol)
|
||||
, MaxIterations(maxit)
|
||||
, RestartLength(restart_length)
|
||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
||||
, ErrorOnNoConverge(err_on_no_conv)
|
||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
||||
, y(RestartLength + 1, 0.)
|
||||
, gamma(RestartLength + 1, 0.)
|
||||
, c(RestartLength + 1, 0.)
|
||||
, s(RestartLength + 1, 0.)
|
||||
, Preconditioner(Prec) {};
|
||||
|
||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
||||
|
||||
std::cout << GridLogWarning << "This algorithm currently doesn't differ from regular FGMRES" << std::endl;
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
Field r(src.Grid());
|
||||
|
||||
std::cout << std::setprecision(4) << std::scientific;
|
||||
std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: src " << ssq << std::endl;
|
||||
|
||||
PrecTimer.Reset();
|
||||
MatrixTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
QrTimer.Reset();
|
||||
CompSolutionTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
IterationCount = 0;
|
||||
|
||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
||||
|
||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
LinOp.Op(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
|
||||
RealD srcnorm = sqrt(ssq);
|
||||
RealD resnorm = sqrt(norm2(r));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
||||
<< " computed residual " << sqrt(cp / ssq)
|
||||
<< " true residual " << true_residual
|
||||
<< " target " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Precon " << PrecTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
|
||||
RealD cp = 0;
|
||||
|
||||
Field w(src.Grid());
|
||||
Field r(src.Grid());
|
||||
|
||||
// these should probably be made class members so that they are only allocated once, not in every restart
|
||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
||||
std::vector<Field> z(RestartLength + 1, src.Grid()); for (auto &elem : z) elem = Zero();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(psi, w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
r = src - w;
|
||||
|
||||
gamma[0] = sqrt(norm2(r));
|
||||
|
||||
v[0] = (1. / gamma[0]) * r;
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for (int i=0; i<RestartLength; i++) {
|
||||
|
||||
IterationCount++;
|
||||
|
||||
arnoldiStep(LinOp, v, z, w, i);
|
||||
|
||||
qrUpdate(i);
|
||||
|
||||
cp = norm(gamma[i+1]);
|
||||
|
||||
std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: Iteration " << IterationCount
|
||||
<< " residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
||||
|
||||
computeSolution(z, psi, i);
|
||||
|
||||
return cp;
|
||||
}
|
||||
}
|
||||
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, std::vector<Field> &z, Field &w, int iter) {
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(v[iter], z[iter]);
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(z[iter], w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
for (int i = 0; i <= iter; ++i) {
|
||||
H(iter, i) = innerProduct(v[i], w);
|
||||
w = w - ComplexD(H(iter, i)) * v[i];
|
||||
}
|
||||
|
||||
H(iter, iter + 1) = sqrt(norm2(w));
|
||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
|
||||
void qrUpdate(int iter) {
|
||||
|
||||
QrTimer.Start();
|
||||
for (int i = 0; i < iter ; ++i) {
|
||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
||||
H(iter, i + 1) = tmp;
|
||||
}
|
||||
|
||||
// Compute new Givens Rotation
|
||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
||||
c[iter] = H(iter, iter) / nu;
|
||||
s[iter] = H(iter, iter + 1) / nu;
|
||||
|
||||
// Apply new Givens rotation
|
||||
H(iter, iter) = nu;
|
||||
H(iter, iter + 1) = 0.;
|
||||
|
||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
||||
QrTimer.Stop();
|
||||
}
|
||||
|
||||
void computeSolution(std::vector<Field> const &z, Field &psi, int iter) {
|
||||
|
||||
CompSolutionTimer.Start();
|
||||
for (int i = iter; i >= 0; i--) {
|
||||
y[i] = gamma[i];
|
||||
for (int k = i + 1; k <= iter; k++)
|
||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
||||
y[i] = y[i] / ComplexD(H(i, i));
|
||||
}
|
||||
|
||||
for (int i = 0; i <= iter; i++)
|
||||
psi = psi + z[i] * y[i];
|
||||
CompSolutionTimer.Stop();
|
||||
}
|
||||
};
|
||||
}
|
||||
#endif
|
||||
@@ -1,256 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/FlexibleGeneralisedMinimalResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
#define GRID_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
|
||||
namespace Grid {
|
||||
|
||||
template<class Field>
|
||||
class FlexibleGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an assert when FGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
|
||||
Integer MaxIterations;
|
||||
Integer RestartLength;
|
||||
Integer MaxNumberOfRestarts;
|
||||
Integer IterationCount; // Number of iterations the FGMRES took to finish,
|
||||
// filled in upon completion
|
||||
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch PrecTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch QrTimer;
|
||||
GridStopWatch CompSolutionTimer;
|
||||
|
||||
Eigen::MatrixXcd H;
|
||||
|
||||
std::vector<ComplexD> y;
|
||||
std::vector<ComplexD> gamma;
|
||||
std::vector<ComplexD> c;
|
||||
std::vector<ComplexD> s;
|
||||
|
||||
LinearFunction<Field> &Preconditioner;
|
||||
|
||||
FlexibleGeneralisedMinimalResidual(RealD tol,
|
||||
Integer maxit,
|
||||
LinearFunction<Field> &Prec,
|
||||
Integer restart_length,
|
||||
bool err_on_no_conv = true)
|
||||
: Tolerance(tol)
|
||||
, MaxIterations(maxit)
|
||||
, RestartLength(restart_length)
|
||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
||||
, ErrorOnNoConverge(err_on_no_conv)
|
||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
||||
, y(RestartLength + 1, 0.)
|
||||
, gamma(RestartLength + 1, 0.)
|
||||
, c(RestartLength + 1, 0.)
|
||||
, s(RestartLength + 1, 0.)
|
||||
, Preconditioner(Prec) {};
|
||||
|
||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
Field r(src.Grid());
|
||||
|
||||
std::cout << std::setprecision(4) << std::scientific;
|
||||
std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: src " << ssq << std::endl;
|
||||
|
||||
PrecTimer.Reset();
|
||||
MatrixTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
QrTimer.Reset();
|
||||
CompSolutionTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
IterationCount = 0;
|
||||
|
||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
||||
|
||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
LinOp.Op(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
|
||||
RealD srcnorm = sqrt(ssq);
|
||||
RealD resnorm = sqrt(norm2(r));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "FlexibleGeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
||||
<< " computed residual " << sqrt(cp / ssq)
|
||||
<< " true residual " << true_residual
|
||||
<< " target " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Precon " << PrecTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "FGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "FlexibleGeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
|
||||
RealD cp = 0;
|
||||
|
||||
Field w(src.Grid());
|
||||
Field r(src.Grid());
|
||||
|
||||
// these should probably be made class members so that they are only allocated once, not in every restart
|
||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
||||
std::vector<Field> z(RestartLength + 1, src.Grid()); for (auto &elem : z) elem = Zero();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(psi, w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
r = src - w;
|
||||
|
||||
gamma[0] = sqrt(norm2(r));
|
||||
|
||||
v[0] = (1. / gamma[0]) * r;
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for (int i=0; i<RestartLength; i++) {
|
||||
|
||||
IterationCount++;
|
||||
|
||||
arnoldiStep(LinOp, v, z, w, i);
|
||||
|
||||
qrUpdate(i);
|
||||
|
||||
cp = norm(gamma[i+1]);
|
||||
|
||||
std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: Iteration " << IterationCount
|
||||
<< " residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
||||
|
||||
computeSolution(z, psi, i);
|
||||
|
||||
return cp;
|
||||
}
|
||||
}
|
||||
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, std::vector<Field> &z, Field &w, int iter) {
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(v[iter], z[iter]);
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(z[iter], w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
for (int i = 0; i <= iter; ++i) {
|
||||
H(iter, i) = innerProduct(v[i], w);
|
||||
w = w - ComplexD(H(iter, i)) * v[i];
|
||||
}
|
||||
|
||||
H(iter, iter + 1) = sqrt(norm2(w));
|
||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
|
||||
void qrUpdate(int iter) {
|
||||
|
||||
QrTimer.Start();
|
||||
for (int i = 0; i < iter ; ++i) {
|
||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
||||
H(iter, i + 1) = tmp;
|
||||
}
|
||||
|
||||
// Compute new Givens Rotation
|
||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
||||
c[iter] = H(iter, iter) / nu;
|
||||
s[iter] = H(iter, iter + 1) / nu;
|
||||
|
||||
// Apply new Givens rotation
|
||||
H(iter, iter) = nu;
|
||||
H(iter, iter + 1) = 0.;
|
||||
|
||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
||||
QrTimer.Stop();
|
||||
}
|
||||
|
||||
void computeSolution(std::vector<Field> const &z, Field &psi, int iter) {
|
||||
|
||||
CompSolutionTimer.Start();
|
||||
for (int i = iter; i >= 0; i--) {
|
||||
y[i] = gamma[i];
|
||||
for (int k = i + 1; k <= iter; k++)
|
||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
||||
y[i] = y[i] / ComplexD(H(i, i));
|
||||
}
|
||||
|
||||
for (int i = 0; i <= iter; i++)
|
||||
psi = psi + z[i] * y[i];
|
||||
CompSolutionTimer.Stop();
|
||||
}
|
||||
};
|
||||
}
|
||||
#endif
|
||||
@@ -1,244 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/GeneralisedMinimalResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
#define GRID_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
|
||||
namespace Grid {
|
||||
|
||||
template<class Field>
|
||||
class GeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an assert when GMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
|
||||
Integer MaxIterations;
|
||||
Integer RestartLength;
|
||||
Integer MaxNumberOfRestarts;
|
||||
Integer IterationCount; // Number of iterations the GMRES took to finish,
|
||||
// filled in upon completion
|
||||
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch QrTimer;
|
||||
GridStopWatch CompSolutionTimer;
|
||||
|
||||
Eigen::MatrixXcd H;
|
||||
|
||||
std::vector<ComplexD> y;
|
||||
std::vector<ComplexD> gamma;
|
||||
std::vector<ComplexD> c;
|
||||
std::vector<ComplexD> s;
|
||||
|
||||
GeneralisedMinimalResidual(RealD tol,
|
||||
Integer maxit,
|
||||
Integer restart_length,
|
||||
bool err_on_no_conv = true)
|
||||
: Tolerance(tol)
|
||||
, MaxIterations(maxit)
|
||||
, RestartLength(restart_length)
|
||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
||||
, ErrorOnNoConverge(err_on_no_conv)
|
||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
||||
, y(RestartLength + 1, 0.)
|
||||
, gamma(RestartLength + 1, 0.)
|
||||
, c(RestartLength + 1, 0.)
|
||||
, s(RestartLength + 1, 0.) {};
|
||||
|
||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
Field r(src.Grid());
|
||||
|
||||
std::cout << std::setprecision(4) << std::scientific;
|
||||
std::cout << GridLogIterative << "GeneralisedMinimalResidual: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << "GeneralisedMinimalResidual: src " << ssq << std::endl;
|
||||
|
||||
MatrixTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
QrTimer.Reset();
|
||||
CompSolutionTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
IterationCount = 0;
|
||||
|
||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
||||
|
||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
LinOp.Op(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
|
||||
RealD srcnorm = sqrt(ssq);
|
||||
RealD resnorm = sqrt(norm2(r));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "GeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
||||
<< " computed residual " << sqrt(cp / ssq)
|
||||
<< " true residual " << true_residual
|
||||
<< " target " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "GMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "GMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "GMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "GMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "GMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "GeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
|
||||
RealD cp = 0;
|
||||
|
||||
Field w(src.Grid());
|
||||
Field r(src.Grid());
|
||||
|
||||
// this should probably be made a class member so that it is only allocated once, not in every restart
|
||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(psi, w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
r = src - w;
|
||||
|
||||
gamma[0] = sqrt(norm2(r));
|
||||
|
||||
v[0] = (1. / gamma[0]) * r;
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for (int i=0; i<RestartLength; i++) {
|
||||
|
||||
IterationCount++;
|
||||
|
||||
arnoldiStep(LinOp, v, w, i);
|
||||
|
||||
qrUpdate(i);
|
||||
|
||||
cp = norm(gamma[i+1]);
|
||||
|
||||
std::cout << GridLogIterative << "GeneralisedMinimalResidual: Iteration " << IterationCount
|
||||
<< " residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
||||
|
||||
computeSolution(v, psi, i);
|
||||
|
||||
return cp;
|
||||
}
|
||||
}
|
||||
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, Field &w, int iter) {
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(v[iter], w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
for (int i = 0; i <= iter; ++i) {
|
||||
H(iter, i) = innerProduct(v[i], w);
|
||||
w = w - ComplexD(H(iter, i)) * v[i];
|
||||
}
|
||||
|
||||
H(iter, iter + 1) = sqrt(norm2(w));
|
||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
|
||||
void qrUpdate(int iter) {
|
||||
|
||||
QrTimer.Start();
|
||||
for (int i = 0; i < iter ; ++i) {
|
||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
||||
H(iter, i + 1) = tmp;
|
||||
}
|
||||
|
||||
// Compute new Givens Rotation
|
||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
||||
c[iter] = H(iter, iter) / nu;
|
||||
s[iter] = H(iter, iter + 1) / nu;
|
||||
|
||||
// Apply new Givens rotation
|
||||
H(iter, iter) = nu;
|
||||
H(iter, iter + 1) = 0.;
|
||||
|
||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
||||
QrTimer.Stop();
|
||||
}
|
||||
|
||||
void computeSolution(std::vector<Field> const &v, Field &psi, int iter) {
|
||||
|
||||
CompSolutionTimer.Start();
|
||||
for (int i = iter; i >= 0; i--) {
|
||||
y[i] = gamma[i];
|
||||
for (int k = i + 1; k <= iter; k++)
|
||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
||||
y[i] = y[i] / ComplexD(H(i, i));
|
||||
}
|
||||
|
||||
for (int i = 0; i <= iter; i++)
|
||||
psi = psi + v[i] * y[i];
|
||||
CompSolutionTimer.Stop();
|
||||
}
|
||||
};
|
||||
}
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,736 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Chulwoo Jung <chulwoo@bnl.gov>
|
||||
Author: Christoph Lehner <clehner@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_BIRL_H
|
||||
#define GRID_BIRL_H
|
||||
|
||||
#include <string.h> //memset
|
||||
//#include <zlib.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Implicitly restarted lanczos
|
||||
/////////////////////////////////////////////////////////////
|
||||
template<class Field> class ImplicitlyRestartedLanczosTester
|
||||
{
|
||||
public:
|
||||
virtual int TestConvergence(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox)=0;
|
||||
virtual int ReconstructEval(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox)=0;
|
||||
};
|
||||
|
||||
enum IRLdiagonalisation {
|
||||
IRLdiagonaliseWithDSTEGR,
|
||||
IRLdiagonaliseWithQR,
|
||||
IRLdiagonaliseWithEigen
|
||||
};
|
||||
|
||||
template<class Field> class ImplicitlyRestartedLanczosHermOpTester : public ImplicitlyRestartedLanczosTester<Field>
|
||||
{
|
||||
public:
|
||||
|
||||
LinearFunction<Field> &_HermOp;
|
||||
ImplicitlyRestartedLanczosHermOpTester(LinearFunction<Field> &HermOp) : _HermOp(HermOp) { };
|
||||
int ReconstructEval(int j,RealD resid,Field &B, RealD &eval,RealD evalMaxApprox)
|
||||
{
|
||||
return TestConvergence(j,resid,B,eval,evalMaxApprox);
|
||||
}
|
||||
int TestConvergence(int j,RealD eresid,Field &B, RealD &eval,RealD evalMaxApprox)
|
||||
{
|
||||
Field v(B);
|
||||
RealD eval_poly = eval;
|
||||
// Apply operator
|
||||
_HermOp(B,v);
|
||||
|
||||
RealD vnum = real(innerProduct(B,v)); // HermOp.
|
||||
RealD vden = norm2(B);
|
||||
RealD vv0 = norm2(v);
|
||||
eval = vnum/vden;
|
||||
v -= eval*B;
|
||||
|
||||
RealD vv = norm2(v) / ::pow(evalMaxApprox,2.0);
|
||||
|
||||
std::cout.precision(13);
|
||||
std::cout<<GridLogIRL << "[" << std::setw(3)<<j<<"] "
|
||||
<<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
|
||||
<<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
|
||||
<<std::endl;
|
||||
|
||||
int conv=0;
|
||||
if( (vv<eresid*eresid) ) conv = 1;
|
||||
|
||||
return conv;
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field>
|
||||
class ImplicitlyRestartedLanczos {
|
||||
private:
|
||||
const RealD small = 1.0e-8;
|
||||
int MaxIter;
|
||||
int MinRestart; // Minimum number of restarts; only check for convergence after
|
||||
int Nstop; // Number of evecs checked for convergence
|
||||
int Nk; // Number of converged sought
|
||||
// int Np; // Np -- Number of spare vecs in krylov space // == Nm - Nk
|
||||
int Nm; // Nm -- total number of vectors
|
||||
IRLdiagonalisation diagonalisation;
|
||||
int orth_period;
|
||||
|
||||
RealD OrthoTime;
|
||||
RealD eresid, betastp;
|
||||
////////////////////////////////
|
||||
// Embedded objects
|
||||
////////////////////////////////
|
||||
LinearFunction<Field> &_PolyOp;
|
||||
LinearFunction<Field> &_HermOp;
|
||||
ImplicitlyRestartedLanczosTester<Field> &_Tester;
|
||||
// Default tester provided (we need a ref to something in default case)
|
||||
ImplicitlyRestartedLanczosHermOpTester<Field> SimpleTester;
|
||||
/////////////////////////
|
||||
// Constructor
|
||||
/////////////////////////
|
||||
|
||||
public:
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// PAB:
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// Too many options & knobs.
|
||||
// Eliminate:
|
||||
// orth_period
|
||||
// betastp
|
||||
// MinRestart
|
||||
//
|
||||
// Do we really need orth_period
|
||||
// What is the theoretical basis & guarantees of betastp ?
|
||||
// Nstop=Nk viable?
|
||||
// MinRestart avoidable with new convergence test?
|
||||
// Could cut to PolyOp, HermOp, Tester, Nk, Nm, resid, maxiter (+diagonalisation)
|
||||
// HermOp could be eliminated if we dropped the Power method for max eval.
|
||||
// -- also: The eval, eval2, eval2_copy stuff is still unnecessarily unclear
|
||||
//////////////////////////////////////////////////////////////////
|
||||
ImplicitlyRestartedLanczos(LinearFunction<Field> & PolyOp,
|
||||
LinearFunction<Field> & HermOp,
|
||||
ImplicitlyRestartedLanczosTester<Field> & Tester,
|
||||
int _Nstop, // sought vecs
|
||||
int _Nk, // sought vecs
|
||||
int _Nm, // spare vecs
|
||||
RealD _eresid, // resid in lmdue deficit
|
||||
int _MaxIter, // Max iterations
|
||||
RealD _betastp=0.0, // if beta(k) < betastp: converged
|
||||
int _MinRestart=0, int _orth_period = 1,
|
||||
IRLdiagonalisation _diagonalisation= IRLdiagonaliseWithEigen) :
|
||||
SimpleTester(HermOp), _PolyOp(PolyOp), _HermOp(HermOp), _Tester(Tester),
|
||||
Nstop(_Nstop) , Nk(_Nk), Nm(_Nm),
|
||||
eresid(_eresid), betastp(_betastp),
|
||||
MaxIter(_MaxIter) , MinRestart(_MinRestart),
|
||||
orth_period(_orth_period), diagonalisation(_diagonalisation) { };
|
||||
|
||||
ImplicitlyRestartedLanczos(LinearFunction<Field> & PolyOp,
|
||||
LinearFunction<Field> & HermOp,
|
||||
int _Nstop, // sought vecs
|
||||
int _Nk, // sought vecs
|
||||
int _Nm, // spare vecs
|
||||
RealD _eresid, // resid in lmdue deficit
|
||||
int _MaxIter, // Max iterations
|
||||
RealD _betastp=0.0, // if beta(k) < betastp: converged
|
||||
int _MinRestart=0, int _orth_period = 1,
|
||||
IRLdiagonalisation _diagonalisation= IRLdiagonaliseWithEigen) :
|
||||
SimpleTester(HermOp), _PolyOp(PolyOp), _HermOp(HermOp), _Tester(SimpleTester),
|
||||
Nstop(_Nstop) , Nk(_Nk), Nm(_Nm),
|
||||
eresid(_eresid), betastp(_betastp),
|
||||
MaxIter(_MaxIter) , MinRestart(_MinRestart),
|
||||
orth_period(_orth_period), diagonalisation(_diagonalisation) { };
|
||||
|
||||
////////////////////////////////
|
||||
// Helpers
|
||||
////////////////////////////////
|
||||
template<typename T> static RealD normalise(T& v)
|
||||
{
|
||||
RealD nn = norm2(v);
|
||||
nn = std::sqrt(nn);
|
||||
v = v * (1.0/nn);
|
||||
return nn;
|
||||
}
|
||||
|
||||
void orthogonalize(Field& w, std::vector<Field>& evec,int k)
|
||||
{
|
||||
OrthoTime-=usecond()/1e6;
|
||||
basisOrthogonalize(evec,w,k);
|
||||
normalise(w);
|
||||
OrthoTime+=usecond()/1e6;
|
||||
}
|
||||
|
||||
/* Rudy Arthur's thesis pp.137
|
||||
------------------------
|
||||
Require: M > K P = M − K †
|
||||
Compute the factorization AVM = VM HM + fM eM
|
||||
repeat
|
||||
Q=I
|
||||
for i = 1,...,P do
|
||||
QiRi =HM −θiI Q = QQi
|
||||
H M = Q †i H M Q i
|
||||
end for
|
||||
βK =HM(K+1,K) σK =Q(M,K)
|
||||
r=vK+1βK +rσK
|
||||
VK =VM(1:M)Q(1:M,1:K)
|
||||
HK =HM(1:K,1:K)
|
||||
→AVK =VKHK +fKe†K † Extend to an M = K + P step factorization AVM = VMHM + fMeM
|
||||
until convergence
|
||||
*/
|
||||
void calc(std::vector<RealD>& eval, std::vector<Field>& evec, const Field& src, int& Nconv, bool reverse=false)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
assert(grid == evec[0].Grid());
|
||||
|
||||
// GridLogIRL.TimingMode(1);
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
std::cout << GridLogIRL <<" ImplicitlyRestartedLanczos::calc() starting iteration 0 / "<< MaxIter<< std::endl;
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
std::cout << GridLogIRL <<" -- seek Nk = " << Nk <<" vectors"<< std::endl;
|
||||
std::cout << GridLogIRL <<" -- accept Nstop = " << Nstop <<" vectors"<< std::endl;
|
||||
std::cout << GridLogIRL <<" -- total Nm = " << Nm <<" vectors"<< std::endl;
|
||||
std::cout << GridLogIRL <<" -- size of eval = " << eval.size() << std::endl;
|
||||
std::cout << GridLogIRL <<" -- size of evec = " << evec.size() << std::endl;
|
||||
if ( diagonalisation == IRLdiagonaliseWithDSTEGR ) {
|
||||
std::cout << GridLogIRL << "Diagonalisation is DSTEGR "<<std::endl;
|
||||
} else if ( diagonalisation == IRLdiagonaliseWithQR ) {
|
||||
std::cout << GridLogIRL << "Diagonalisation is QR "<<std::endl;
|
||||
} else if ( diagonalisation == IRLdiagonaliseWithEigen ) {
|
||||
std::cout << GridLogIRL << "Diagonalisation is Eigen "<<std::endl;
|
||||
}
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
|
||||
assert(Nm <= evec.size() && Nm <= eval.size());
|
||||
|
||||
// quickly get an idea of the largest eigenvalue to more properly normalize the residuum
|
||||
RealD evalMaxApprox = 0.0;
|
||||
{
|
||||
auto src_n = src;
|
||||
auto tmp = src;
|
||||
std::cout << GridLogIRL << " IRL source norm " << norm2(src) << std::endl;
|
||||
const int _MAX_ITER_IRL_MEVAPP_ = 50;
|
||||
for (int i=0;i<_MAX_ITER_IRL_MEVAPP_;i++) {
|
||||
normalise(src_n);
|
||||
_HermOp(src_n,tmp);
|
||||
// std::cout << GridLogMessage<< tmp<<std::endl; exit(0);
|
||||
// std::cout << GridLogIRL << " _HermOp " << norm2(tmp) << std::endl;
|
||||
RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
|
||||
RealD vden = norm2(src_n);
|
||||
RealD na = vnum/vden;
|
||||
if (fabs(evalMaxApprox/na - 1.0) < 0.0001)
|
||||
i=_MAX_ITER_IRL_MEVAPP_;
|
||||
evalMaxApprox = na;
|
||||
std::cout << GridLogIRL << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
|
||||
src_n = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<RealD> lme(Nm);
|
||||
std::vector<RealD> lme2(Nm);
|
||||
std::vector<RealD> eval2(Nm);
|
||||
std::vector<RealD> eval2_copy(Nm);
|
||||
Eigen::MatrixXd Qt = Eigen::MatrixXd::Zero(Nm,Nm);
|
||||
|
||||
Field f(grid);
|
||||
Field v(grid);
|
||||
int k1 = 1;
|
||||
int k2 = Nk;
|
||||
RealD beta_k;
|
||||
|
||||
Nconv = 0;
|
||||
|
||||
// Set initial vector
|
||||
evec[0] = src;
|
||||
normalise(evec[0]);
|
||||
|
||||
// Initial Nk steps
|
||||
OrthoTime=0.;
|
||||
for(int k=0; k<Nk; ++k) step(eval,lme,evec,f,Nm,k);
|
||||
std::cout<<GridLogIRL <<"Initial "<< Nk <<"steps done "<<std::endl;
|
||||
std::cout<<GridLogIRL <<"Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
|
||||
|
||||
//////////////////////////////////
|
||||
// Restarting loop begins
|
||||
//////////////////////////////////
|
||||
int iter;
|
||||
for(iter = 0; iter<MaxIter; ++iter){
|
||||
|
||||
OrthoTime=0.;
|
||||
|
||||
std::cout<< GridLogMessage <<" **********************"<< std::endl;
|
||||
std::cout<< GridLogMessage <<" Restart iteration = "<< iter << std::endl;
|
||||
std::cout<< GridLogMessage <<" **********************"<< std::endl;
|
||||
|
||||
std::cout<<GridLogIRL <<" running "<<Nm-Nk <<" steps: "<<std::endl;
|
||||
for(int k=Nk; k<Nm; ++k) step(eval,lme,evec,f,Nm,k);
|
||||
f *= lme[Nm-1];
|
||||
|
||||
std::cout<<GridLogIRL <<" "<<Nm-Nk <<" steps done "<<std::endl;
|
||||
std::cout<<GridLogIRL <<"Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
|
||||
|
||||
//////////////////////////////////
|
||||
// getting eigenvalues
|
||||
//////////////////////////////////
|
||||
for(int k=0; k<Nm; ++k){
|
||||
eval2[k] = eval[k+k1-1];
|
||||
lme2[k] = lme[k+k1-1];
|
||||
}
|
||||
Qt = Eigen::MatrixXd::Identity(Nm,Nm);
|
||||
diagonalize(eval2,lme2,Nm,Nm,Qt,grid);
|
||||
std::cout<<GridLogIRL <<" diagonalized "<<std::endl;
|
||||
|
||||
//////////////////////////////////
|
||||
// sorting
|
||||
//////////////////////////////////
|
||||
eval2_copy = eval2;
|
||||
std::partial_sort(eval2.begin(),eval2.begin()+Nm,eval2.end(),std::greater<RealD>());
|
||||
std::cout<<GridLogIRL <<" evals sorted "<<std::endl;
|
||||
const int chunk=8;
|
||||
for(int io=0; io<k2;io+=chunk){
|
||||
std::cout<<GridLogIRL << "eval "<< std::setw(3) << io ;
|
||||
for(int ii=0;ii<chunk;ii++){
|
||||
if ( (io+ii)<k2 )
|
||||
std::cout<< " "<< std::setw(12)<< eval2[io+ii];
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
//////////////////////////////////
|
||||
// Implicitly shifted QR transformations
|
||||
//////////////////////////////////
|
||||
Qt = Eigen::MatrixXd::Identity(Nm,Nm);
|
||||
for(int ip=k2; ip<Nm; ++ip){
|
||||
QR_decomp(eval,lme,Nm,Nm,Qt,eval2[ip],k1,Nm);
|
||||
}
|
||||
std::cout<<GridLogIRL <<"QR decomposed "<<std::endl;
|
||||
|
||||
assert(k2<Nm); assert(k2<Nm); assert(k1>0);
|
||||
|
||||
basisRotate(evec,Qt,k1-1,k2+1,0,Nm,Nm); /// big constraint on the basis
|
||||
std::cout<<GridLogIRL <<"basisRotated by Qt *"<<k1-1<<","<<k2+1<<")"<<std::endl;
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Compressed vector f and beta(k2)
|
||||
////////////////////////////////////////////////////
|
||||
f *= Qt(k2-1,Nm-1);
|
||||
f += lme[k2-1] * evec[k2];
|
||||
beta_k = norm2(f);
|
||||
beta_k = std::sqrt(beta_k);
|
||||
std::cout<<GridLogIRL<<" beta(k) = "<<beta_k<<std::endl;
|
||||
|
||||
RealD betar = 1.0/beta_k;
|
||||
evec[k2] = betar * f;
|
||||
lme[k2-1] = beta_k;
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Convergence test
|
||||
////////////////////////////////////////////////////
|
||||
for(int k=0; k<Nm; ++k){
|
||||
eval2[k] = eval[k];
|
||||
lme2[k] = lme[k];
|
||||
}
|
||||
Qt = Eigen::MatrixXd::Identity(Nm,Nm);
|
||||
diagonalize(eval2,lme2,Nk,Nm,Qt,grid);
|
||||
std::cout<<GridLogIRL <<" Diagonalized "<<std::endl;
|
||||
|
||||
Nconv = 0;
|
||||
if (iter >= MinRestart) {
|
||||
|
||||
std::cout << GridLogIRL << "Test convergence: rotate subset of vectors to test convergence " << std::endl;
|
||||
|
||||
Field B(grid); B.Checkerboard() = evec[0].Checkerboard();
|
||||
|
||||
// power of two search pattern; not every evalue in eval2 is assessed.
|
||||
int allconv =1;
|
||||
for(int jj = 1; jj<=Nstop; jj*=2){
|
||||
int j = Nstop-jj;
|
||||
RealD e = eval2_copy[j]; // Discard the evalue
|
||||
basisRotateJ(B,evec,Qt,j,0,Nk,Nm);
|
||||
if( !_Tester.TestConvergence(j,eresid,B,e,evalMaxApprox) ) {
|
||||
allconv=0;
|
||||
}
|
||||
}
|
||||
// Do evec[0] for good measure
|
||||
{
|
||||
int j=0;
|
||||
RealD e = eval2_copy[0];
|
||||
basisRotateJ(B,evec,Qt,j,0,Nk,Nm);
|
||||
if( !_Tester.TestConvergence(j,eresid,B,e,evalMaxApprox) ) allconv=0;
|
||||
}
|
||||
if ( allconv ) Nconv = Nstop;
|
||||
|
||||
// test if we converged, if so, terminate
|
||||
std::cout<<GridLogIRL<<" #modes converged: >= "<<Nconv<<"/"<<Nstop<<std::endl;
|
||||
// if( Nconv>=Nstop || beta_k < betastp){
|
||||
if( Nconv>=Nstop){
|
||||
goto converged;
|
||||
}
|
||||
|
||||
} else {
|
||||
std::cout << GridLogIRL << "iter < MinRestart: do not yet test for convergence\n";
|
||||
} // end of iter loop
|
||||
}
|
||||
|
||||
std::cout<<GridLogError<<"\n NOT converged.\n";
|
||||
abort();
|
||||
|
||||
converged:
|
||||
{
|
||||
Field B(grid); B.Checkerboard() = evec[0].Checkerboard();
|
||||
basisRotate(evec,Qt,0,Nk,0,Nk,Nm);
|
||||
std::cout << GridLogIRL << " Rotated basis"<<std::endl;
|
||||
Nconv=0;
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Full final convergence test; unconditionally applied
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
for(int j = 0; j<=Nk; j++){
|
||||
B=evec[j];
|
||||
if( _Tester.ReconstructEval(j,eresid,B,eval2[j],evalMaxApprox) ) {
|
||||
Nconv++;
|
||||
}
|
||||
}
|
||||
|
||||
if ( Nconv < Nstop )
|
||||
std::cout << GridLogIRL << "Nconv ("<<Nconv<<") < Nstop ("<<Nstop<<")"<<std::endl;
|
||||
|
||||
eval=eval2;
|
||||
|
||||
//Keep only converged
|
||||
eval.resize(Nconv);// Nstop?
|
||||
evec.resize(Nconv,grid);// Nstop?
|
||||
basisSortInPlace(evec,eval,reverse);
|
||||
|
||||
}
|
||||
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
std::cout << GridLogIRL << "ImplicitlyRestartedLanczos CONVERGED ; Summary :\n";
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
std::cout << GridLogIRL << " -- Iterations = "<< iter << "\n";
|
||||
std::cout << GridLogIRL << " -- beta(k) = "<< beta_k << "\n";
|
||||
std::cout << GridLogIRL << " -- Nconv = "<< Nconv << "\n";
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
}
|
||||
|
||||
private:
|
||||
/* Saad PP. 195
|
||||
1. Choose an initial vector v1 of 2-norm unity. Set β1 ≡ 0, v0 ≡ 0
|
||||
2. For k = 1,2,...,m Do:
|
||||
3. wk:=Avk - b_k v_{k-1}
|
||||
4. ak:=(wk,vk) //
|
||||
5. wk:=wk-akvk // wk orthog vk
|
||||
6. bk+1 := ||wk||_2. If b_k+1 = 0 then Stop
|
||||
7. vk+1 := wk/b_k+1
|
||||
8. EndDo
|
||||
*/
|
||||
void step(std::vector<RealD>& lmd,
|
||||
std::vector<RealD>& lme,
|
||||
std::vector<Field>& evec,
|
||||
Field& w,int Nm,int k)
|
||||
{
|
||||
std::cout<<GridLogIRL << "Lanczos step " <<k<<std::endl;
|
||||
const RealD tiny = 1.0e-20;
|
||||
assert( k< Nm );
|
||||
|
||||
GridStopWatch gsw_op,gsw_o;
|
||||
|
||||
Field& evec_k = evec[k];
|
||||
|
||||
_PolyOp(evec_k,w); std::cout<<GridLogIRL << "PolyOp" <<std::endl;
|
||||
|
||||
if(k>0) w -= lme[k-1] * evec[k-1];
|
||||
|
||||
ComplexD zalph = innerProduct(evec_k,w);
|
||||
RealD alph = real(zalph);
|
||||
|
||||
w = w - alph * evec_k;
|
||||
|
||||
RealD beta = normalise(w);
|
||||
|
||||
lmd[k] = alph;
|
||||
lme[k] = beta;
|
||||
|
||||
if ( (k>0) && ( (k % orth_period) == 0 )) {
|
||||
std::cout<<GridLogIRL << "Orthogonalising " <<k<<std::endl;
|
||||
orthogonalize(w,evec,k); // orthonormalise
|
||||
std::cout<<GridLogIRL << "Orthogonalised " <<k<<std::endl;
|
||||
}
|
||||
|
||||
if(k < Nm-1) evec[k+1] = w;
|
||||
|
||||
std::cout<<GridLogIRL << "alpha[" << k << "] = " << zalph << " beta[" << k << "] = "<<beta<<std::endl;
|
||||
if ( beta < tiny )
|
||||
std::cout<<GridLogIRL << " beta is tiny "<<beta<<std::endl;
|
||||
|
||||
std::cout<<GridLogIRL << "Lanczos step complete " <<k<<std::endl;
|
||||
}
|
||||
|
||||
void diagonalize_Eigen(std::vector<RealD>& lmd, std::vector<RealD>& lme,
|
||||
int Nk, int Nm,
|
||||
Eigen::MatrixXd & Qt, // Nm x Nm
|
||||
GridBase *grid)
|
||||
{
|
||||
Eigen::MatrixXd TriDiag = Eigen::MatrixXd::Zero(Nk,Nk);
|
||||
|
||||
for(int i=0;i<Nk;i++) TriDiag(i,i) = lmd[i];
|
||||
for(int i=0;i<Nk-1;i++) TriDiag(i,i+1) = lme[i];
|
||||
for(int i=0;i<Nk-1;i++) TriDiag(i+1,i) = lme[i];
|
||||
|
||||
Eigen::SelfAdjointEigenSolver<Eigen::MatrixXd> eigensolver(TriDiag);
|
||||
|
||||
for (int i = 0; i < Nk; i++) {
|
||||
lmd[Nk-1-i] = eigensolver.eigenvalues()(i);
|
||||
}
|
||||
for (int i = 0; i < Nk; i++) {
|
||||
for (int j = 0; j < Nk; j++) {
|
||||
Qt(Nk-1-i,j) = eigensolver.eigenvectors()(j,i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// File could end here if settle on Eigen ??? !!!
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
void QR_decomp(std::vector<RealD>& lmd, // Nm
|
||||
std::vector<RealD>& lme, // Nm
|
||||
int Nk, int Nm, // Nk, Nm
|
||||
Eigen::MatrixXd& Qt, // Nm x Nm matrix
|
||||
RealD Dsh, int kmin, int kmax)
|
||||
{
|
||||
int k = kmin-1;
|
||||
RealD x;
|
||||
|
||||
RealD Fden = 1.0/hypot(lmd[k]-Dsh,lme[k]);
|
||||
RealD c = ( lmd[k] -Dsh) *Fden;
|
||||
RealD s = -lme[k] *Fden;
|
||||
|
||||
RealD tmpa1 = lmd[k];
|
||||
RealD tmpa2 = lmd[k+1];
|
||||
RealD tmpb = lme[k];
|
||||
|
||||
lmd[k] = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb;
|
||||
lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb;
|
||||
lme[k] = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb;
|
||||
x =-s*lme[k+1];
|
||||
lme[k+1] = c*lme[k+1];
|
||||
|
||||
for(int i=0; i<Nk; ++i){
|
||||
RealD Qtmp1 = Qt(k,i);
|
||||
RealD Qtmp2 = Qt(k+1,i);
|
||||
Qt(k,i) = c*Qtmp1 - s*Qtmp2;
|
||||
Qt(k+1,i)= s*Qtmp1 + c*Qtmp2;
|
||||
}
|
||||
|
||||
// Givens transformations
|
||||
for(int k = kmin; k < kmax-1; ++k){
|
||||
|
||||
RealD Fden = 1.0/hypot(x,lme[k-1]);
|
||||
RealD c = lme[k-1]*Fden;
|
||||
RealD s = - x*Fden;
|
||||
|
||||
RealD tmpa1 = lmd[k];
|
||||
RealD tmpa2 = lmd[k+1];
|
||||
RealD tmpb = lme[k];
|
||||
|
||||
lmd[k] = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb;
|
||||
lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb;
|
||||
lme[k] = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb;
|
||||
lme[k-1] = c*lme[k-1] -s*x;
|
||||
|
||||
if(k != kmax-2){
|
||||
x = -s*lme[k+1];
|
||||
lme[k+1] = c*lme[k+1];
|
||||
}
|
||||
|
||||
for(int i=0; i<Nk; ++i){
|
||||
RealD Qtmp1 = Qt(k,i);
|
||||
RealD Qtmp2 = Qt(k+1,i);
|
||||
Qt(k,i) = c*Qtmp1 -s*Qtmp2;
|
||||
Qt(k+1,i) = s*Qtmp1 +c*Qtmp2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void diagonalize(std::vector<RealD>& lmd, std::vector<RealD>& lme,
|
||||
int Nk, int Nm,
|
||||
Eigen::MatrixXd & Qt,
|
||||
GridBase *grid)
|
||||
{
|
||||
Qt = Eigen::MatrixXd::Identity(Nm,Nm);
|
||||
if ( diagonalisation == IRLdiagonaliseWithDSTEGR ) {
|
||||
diagonalize_lapack(lmd,lme,Nk,Nm,Qt,grid);
|
||||
} else if ( diagonalisation == IRLdiagonaliseWithQR ) {
|
||||
diagonalize_QR(lmd,lme,Nk,Nm,Qt,grid);
|
||||
} else if ( diagonalisation == IRLdiagonaliseWithEigen ) {
|
||||
diagonalize_Eigen(lmd,lme,Nk,Nm,Qt,grid);
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef USE_LAPACK
|
||||
void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
|
||||
double *vl, double *vu, int *il, int *iu, double *abstol,
|
||||
int *m, double *w, double *z, int *ldz, int *isuppz,
|
||||
double *work, int *lwork, int *iwork, int *liwork,
|
||||
int *info);
|
||||
#endif
|
||||
|
||||
void diagonalize_lapack(std::vector<RealD>& lmd,
|
||||
std::vector<RealD>& lme,
|
||||
int Nk, int Nm,
|
||||
Eigen::MatrixXd& Qt,
|
||||
GridBase *grid)
|
||||
{
|
||||
#ifdef USE_LAPACK
|
||||
const int size = Nm;
|
||||
int NN = Nk;
|
||||
double evals_tmp[NN];
|
||||
double evec_tmp[NN][NN];
|
||||
memset(evec_tmp[0],0,sizeof(double)*NN*NN);
|
||||
double DD[NN];
|
||||
double EE[NN];
|
||||
for (int i = 0; i< NN; i++) {
|
||||
for (int j = i - 1; j <= i + 1; j++) {
|
||||
if ( j < NN && j >= 0 ) {
|
||||
if (i==j) DD[i] = lmd[i];
|
||||
if (i==j) evals_tmp[i] = lmd[i];
|
||||
if (j==(i-1)) EE[j] = lme[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
int evals_found;
|
||||
int lwork = ( (18*NN) > (1+4*NN+NN*NN)? (18*NN):(1+4*NN+NN*NN)) ;
|
||||
int liwork = 3+NN*10 ;
|
||||
int iwork[liwork];
|
||||
double work[lwork];
|
||||
int isuppz[2*NN];
|
||||
char jobz = 'V'; // calculate evals & evecs
|
||||
char range = 'I'; // calculate all evals
|
||||
// char range = 'A'; // calculate all evals
|
||||
char uplo = 'U'; // refer to upper half of original matrix
|
||||
char compz = 'I'; // Compute eigenvectors of tridiagonal matrix
|
||||
int ifail[NN];
|
||||
int info;
|
||||
int total = grid->_Nprocessors;
|
||||
int node = grid->_processor;
|
||||
int interval = (NN/total)+1;
|
||||
double vl = 0.0, vu = 0.0;
|
||||
int il = interval*node+1 , iu = interval*(node+1);
|
||||
if (iu > NN) iu=NN;
|
||||
double tol = 0.0;
|
||||
if (1) {
|
||||
memset(evals_tmp,0,sizeof(double)*NN);
|
||||
if ( il <= NN){
|
||||
LAPACK_dstegr(&jobz, &range, &NN,
|
||||
(double*)DD, (double*)EE,
|
||||
&vl, &vu, &il, &iu, // these four are ignored if second parameteris 'A'
|
||||
&tol, // tolerance
|
||||
&evals_found, evals_tmp, (double*)evec_tmp, &NN,
|
||||
isuppz,
|
||||
work, &lwork, iwork, &liwork,
|
||||
&info);
|
||||
for (int i = iu-1; i>= il-1; i--){
|
||||
evals_tmp[i] = evals_tmp[i - (il-1)];
|
||||
if (il>1) evals_tmp[i-(il-1)]=0.;
|
||||
for (int j = 0; j< NN; j++){
|
||||
evec_tmp[i][j] = evec_tmp[i - (il-1)][j];
|
||||
if (il>1) evec_tmp[i-(il-1)][j]=0.;
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
grid->GlobalSumVector(evals_tmp,NN);
|
||||
grid->GlobalSumVector((double*)evec_tmp,NN*NN);
|
||||
}
|
||||
}
|
||||
// Safer to sort instead of just reversing it,
|
||||
// but the document of the routine says evals are sorted in increasing order.
|
||||
// qr gives evals in decreasing order.
|
||||
for(int i=0;i<NN;i++){
|
||||
lmd [NN-1-i]=evals_tmp[i];
|
||||
for(int j=0;j<NN;j++){
|
||||
Qt((NN-1-i),j)=evec_tmp[i][j];
|
||||
}
|
||||
}
|
||||
#else
|
||||
assert(0);
|
||||
#endif
|
||||
}
|
||||
|
||||
void diagonalize_QR(std::vector<RealD>& lmd, std::vector<RealD>& lme,
|
||||
int Nk, int Nm,
|
||||
Eigen::MatrixXd & Qt,
|
||||
GridBase *grid)
|
||||
{
|
||||
int QRiter = 100*Nm;
|
||||
int kmin = 1;
|
||||
int kmax = Nk;
|
||||
|
||||
// (this should be more sophisticated)
|
||||
for(int iter=0; iter<QRiter; ++iter){
|
||||
|
||||
// determination of 2x2 leading submatrix
|
||||
RealD dsub = lmd[kmax-1]-lmd[kmax-2];
|
||||
RealD dd = std::sqrt(dsub*dsub + 4.0*lme[kmax-2]*lme[kmax-2]);
|
||||
RealD Dsh = 0.5*(lmd[kmax-2]+lmd[kmax-1] +dd*(dsub/fabs(dsub)));
|
||||
// (Dsh: shift)
|
||||
|
||||
// transformation
|
||||
QR_decomp(lmd,lme,Nk,Nm,Qt,Dsh,kmin,kmax); // Nk, Nm
|
||||
|
||||
// Convergence criterion (redef of kmin and kamx)
|
||||
for(int j=kmax-1; j>= kmin; --j){
|
||||
RealD dds = fabs(lmd[j-1])+fabs(lmd[j]);
|
||||
if(fabs(lme[j-1])+dds > dds){
|
||||
kmax = j+1;
|
||||
goto continued;
|
||||
}
|
||||
}
|
||||
QRiter = iter;
|
||||
return;
|
||||
|
||||
continued:
|
||||
for(int j=0; j<kmax-1; ++j){
|
||||
RealD dds = fabs(lmd[j])+fabs(lmd[j+1]);
|
||||
if(fabs(lme[j])+dds > dds){
|
||||
kmin = j+1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::cout << GridLogError << "[QL method] Error - Too many iteration: "<<QRiter<<"\n";
|
||||
abort();
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,454 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/LocalCoherenceLanczos.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Christoph Lehner <clehner@bnl.gov>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_LOCAL_COHERENCE_IRL_H
|
||||
#define GRID_LOCAL_COHERENCE_IRL_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
struct LanczosParams : Serializable {
|
||||
public:
|
||||
GRID_SERIALIZABLE_CLASS_MEMBERS(LanczosParams,
|
||||
ChebyParams, Cheby,/*Chebyshev*/
|
||||
int, Nstop, /*Vecs in Lanczos must converge Nstop < Nk < Nm*/
|
||||
int, Nk, /*Vecs in Lanczos seek converge*/
|
||||
int, Nm, /*Total vecs in Lanczos include restart*/
|
||||
RealD, resid, /*residual*/
|
||||
int, MaxIt,
|
||||
RealD, betastp, /* ? */
|
||||
int, MinRes); // Must restart
|
||||
};
|
||||
|
||||
//This class is the input parameter class for some testing programs
|
||||
struct LocalCoherenceLanczosParams : Serializable {
|
||||
public:
|
||||
GRID_SERIALIZABLE_CLASS_MEMBERS(LocalCoherenceLanczosParams,
|
||||
bool, saveEvecs,
|
||||
bool, doFine,
|
||||
bool, doFineRead,
|
||||
bool, doCoarse,
|
||||
bool, doCoarseRead,
|
||||
LanczosParams, FineParams,
|
||||
LanczosParams, CoarseParams,
|
||||
ChebyParams, Smoother,
|
||||
RealD , coarse_relax_tol,
|
||||
std::vector<int>, blockSize,
|
||||
std::string, config,
|
||||
std::vector < ComplexD >, omega,
|
||||
RealD, mass,
|
||||
RealD, M5);
|
||||
};
|
||||
|
||||
// Duplicate functionality; ProjectedFunctionHermOp could be used with the trivial function
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class ProjectedHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
|
||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj> FineField;
|
||||
|
||||
LinearOperatorBase<FineField> &_Linop;
|
||||
std::vector<FineField> &subspace;
|
||||
|
||||
ProjectedHermOp(LinearOperatorBase<FineField>& linop, std::vector<FineField> & _subspace) :
|
||||
_Linop(linop), subspace(_subspace)
|
||||
{
|
||||
assert(subspace.size() >0);
|
||||
};
|
||||
|
||||
void operator()(const CoarseField& in, CoarseField& out) {
|
||||
GridBase *FineGrid = subspace[0].Grid();
|
||||
int checkerboard = subspace[0].Checkerboard();
|
||||
|
||||
FineField fin (FineGrid); fin.Checkerboard()= checkerboard;
|
||||
FineField fout(FineGrid); fout.Checkerboard() = checkerboard;
|
||||
|
||||
blockPromote(in,fin,subspace); std::cout<<GridLogIRL<<"ProjectedHermop : Promote to fine"<<std::endl;
|
||||
_Linop.HermOp(fin,fout); std::cout<<GridLogIRL<<"ProjectedHermop : HermOp (fine) "<<std::endl;
|
||||
blockProject(out,fout,subspace); std::cout<<GridLogIRL<<"ProjectedHermop : Project to coarse "<<std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class ProjectedFunctionHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
|
||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj> FineField;
|
||||
|
||||
|
||||
OperatorFunction<FineField> & _poly;
|
||||
LinearOperatorBase<FineField> &_Linop;
|
||||
std::vector<FineField> &subspace;
|
||||
|
||||
ProjectedFunctionHermOp(OperatorFunction<FineField> & poly,
|
||||
LinearOperatorBase<FineField>& linop,
|
||||
std::vector<FineField> & _subspace) :
|
||||
_poly(poly),
|
||||
_Linop(linop),
|
||||
subspace(_subspace)
|
||||
{ };
|
||||
|
||||
void operator()(const CoarseField& in, CoarseField& out) {
|
||||
|
||||
GridBase *FineGrid = subspace[0].Grid();
|
||||
int checkerboard = subspace[0].Checkerboard();
|
||||
|
||||
FineField fin (FineGrid); fin.Checkerboard() =checkerboard;
|
||||
FineField fout(FineGrid);fout.Checkerboard() =checkerboard;
|
||||
|
||||
blockPromote(in,fin,subspace); std::cout<<GridLogIRL<<"ProjectedFunctionHermop : Promote to fine"<<std::endl;
|
||||
_poly(_Linop,fin,fout); std::cout<<GridLogIRL<<"ProjectedFunctionHermop : Poly "<<std::endl;
|
||||
blockProject(out,fout,subspace); std::cout<<GridLogIRL<<"ProjectedFunctionHermop : Project to coarse "<<std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class ImplicitlyRestartedLanczosSmoothedTester : public ImplicitlyRestartedLanczosTester<Lattice<iVector<CComplex,nbasis > > >
|
||||
{
|
||||
public:
|
||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj> FineField;
|
||||
|
||||
LinearFunction<CoarseField> & _Poly;
|
||||
OperatorFunction<FineField> & _smoother;
|
||||
LinearOperatorBase<FineField> &_Linop;
|
||||
RealD _coarse_relax_tol;
|
||||
std::vector<FineField> &_subspace;
|
||||
|
||||
int _largestEvalIdxForReport; //The convergence of the LCL is based on the evals of the coarse grid operator, not those of the underlying fine grid operator
|
||||
//As a result we do not know what the eval range of the fine operator is until the very end, making tuning the Cheby bounds very difficult
|
||||
//To work around this issue, every restart we separately reconstruct the fine operator eval for the lowest and highest evec and print these
|
||||
//out alongside the evals of the coarse operator. To do so we need to know the index of the largest eval (i.e. Nstop-1)
|
||||
//NOTE: If largestEvalIdxForReport=-1 (default) then this is not performed
|
||||
|
||||
ImplicitlyRestartedLanczosSmoothedTester(LinearFunction<CoarseField> &Poly,
|
||||
OperatorFunction<FineField> &smoother,
|
||||
LinearOperatorBase<FineField> &Linop,
|
||||
std::vector<FineField> &subspace,
|
||||
RealD coarse_relax_tol=5.0e3,
|
||||
int largestEvalIdxForReport=-1)
|
||||
: _smoother(smoother), _Linop(Linop), _Poly(Poly), _subspace(subspace),
|
||||
_coarse_relax_tol(coarse_relax_tol), _largestEvalIdxForReport(largestEvalIdxForReport)
|
||||
{ };
|
||||
|
||||
//evalMaxApprox: approximation of largest eval of the fine Chebyshev operator (suitably wrapped by block projection)
|
||||
int TestConvergence(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
|
||||
{
|
||||
CoarseField v(B);
|
||||
RealD eval_poly = eval;
|
||||
|
||||
// Apply operator
|
||||
_Poly(B,v);
|
||||
|
||||
RealD vnum = real(innerProduct(B,v)); // HermOp.
|
||||
RealD vden = norm2(B);
|
||||
RealD vv0 = norm2(v);
|
||||
eval = vnum/vden;
|
||||
v -= eval*B;
|
||||
|
||||
RealD vv = norm2(v) / ::pow(evalMaxApprox,2.0);
|
||||
|
||||
std::cout.precision(13);
|
||||
std::cout<<GridLogIRL << "[" << std::setw(3)<<j<<"] "
|
||||
<<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
|
||||
<<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
|
||||
<<std::endl;
|
||||
|
||||
if(_largestEvalIdxForReport != -1 && (j==0 || j==_largestEvalIdxForReport)){
|
||||
std::cout<<GridLogIRL << "Estimating true eval of fine grid operator for eval idx " << j << std::endl;
|
||||
RealD tmp_eval;
|
||||
ReconstructEval(j,eresid,B,tmp_eval,1.0); //don't use evalMaxApprox of coarse operator! (cf below)
|
||||
}
|
||||
|
||||
int conv=0;
|
||||
if( (vv<eresid*eresid) ) conv = 1;
|
||||
return conv;
|
||||
}
|
||||
|
||||
//This function is called at the end of the coarse grid Lanczos. It promotes the coarse eigenvector 'B' to the fine grid,
|
||||
//applies a smoother to the result then computes the computes the *fine grid* eigenvalue (output as 'eval').
|
||||
|
||||
//evalMaxApprox should be the approximation of the largest eval of the fine Hermop. However when this function is called by IRL it actually passes the largest eval of the *Chebyshev* operator (as this is the max approx used for the TestConvergence above)
|
||||
//As the largest eval of the Chebyshev is typically several orders of magnitude larger this makes the convergence test pass even when it should not.
|
||||
//We therefore ignore evalMaxApprox here and use a value of 1.0 (note this value is already used by TestCoarse)
|
||||
int ReconstructEval(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
|
||||
{
|
||||
evalMaxApprox = 1.0; //cf above
|
||||
GridBase *FineGrid = _subspace[0].Grid();
|
||||
int checkerboard = _subspace[0].Checkerboard();
|
||||
FineField fB(FineGrid);fB.Checkerboard() =checkerboard;
|
||||
FineField fv(FineGrid);fv.Checkerboard() =checkerboard;
|
||||
|
||||
blockPromote(B,fv,_subspace);
|
||||
|
||||
_smoother(_Linop,fv,fB);
|
||||
|
||||
RealD eval_poly = eval;
|
||||
_Linop.HermOp(fB,fv);
|
||||
|
||||
RealD vnum = real(innerProduct(fB,fv)); // HermOp.
|
||||
RealD vden = norm2(fB);
|
||||
RealD vv0 = norm2(fv);
|
||||
eval = vnum/vden;
|
||||
fv -= eval*fB;
|
||||
RealD vv = norm2(fv) / ::pow(evalMaxApprox,2.0);
|
||||
if ( j > nbasis ) eresid = eresid*_coarse_relax_tol;
|
||||
|
||||
std::cout.precision(13);
|
||||
std::cout<<GridLogIRL << "[" << std::setw(3)<<j<<"] "
|
||||
<<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
|
||||
<<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv << " target " << eresid*eresid
|
||||
<<std::endl;
|
||||
if( (vv<eresid*eresid) ) return 1;
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////
|
||||
// Make serializable Lanczos params
|
||||
////////////////////////////////////////////
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class LocalCoherenceLanczos
|
||||
{
|
||||
public:
|
||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||
typedef Lattice<Fobj> FineField;
|
||||
|
||||
protected:
|
||||
GridBase *_CoarseGrid;
|
||||
GridBase *_FineGrid;
|
||||
int _checkerboard;
|
||||
LinearOperatorBase<FineField> & _FineOp;
|
||||
|
||||
std::vector<RealD> &evals_fine;
|
||||
std::vector<RealD> &evals_coarse;
|
||||
std::vector<FineField> &subspace;
|
||||
std::vector<CoarseField> &evec_coarse;
|
||||
|
||||
private:
|
||||
std::vector<RealD> _evals_fine;
|
||||
std::vector<RealD> _evals_coarse;
|
||||
std::vector<FineField> _subspace;
|
||||
std::vector<CoarseField> _evec_coarse;
|
||||
|
||||
public:
|
||||
|
||||
LocalCoherenceLanczos(GridBase *FineGrid,
|
||||
GridBase *CoarseGrid,
|
||||
LinearOperatorBase<FineField> &FineOp,
|
||||
int checkerboard) :
|
||||
_CoarseGrid(CoarseGrid),
|
||||
_FineGrid(FineGrid),
|
||||
_FineOp(FineOp),
|
||||
_checkerboard(checkerboard),
|
||||
evals_fine (_evals_fine),
|
||||
evals_coarse(_evals_coarse),
|
||||
subspace (_subspace),
|
||||
evec_coarse(_evec_coarse)
|
||||
{
|
||||
evals_fine.resize(0);
|
||||
evals_coarse.resize(0);
|
||||
};
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Alternate constructore, external storage for use by Hadrons module
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
LocalCoherenceLanczos(GridBase *FineGrid,
|
||||
GridBase *CoarseGrid,
|
||||
LinearOperatorBase<FineField> &FineOp,
|
||||
int checkerboard,
|
||||
std::vector<FineField> &ext_subspace,
|
||||
std::vector<CoarseField> &ext_coarse,
|
||||
std::vector<RealD> &ext_eval_fine,
|
||||
std::vector<RealD> &ext_eval_coarse
|
||||
) :
|
||||
_CoarseGrid(CoarseGrid),
|
||||
_FineGrid(FineGrid),
|
||||
_FineOp(FineOp),
|
||||
_checkerboard(checkerboard),
|
||||
evals_fine (ext_eval_fine),
|
||||
evals_coarse(ext_eval_coarse),
|
||||
subspace (ext_subspace),
|
||||
evec_coarse (ext_coarse)
|
||||
{
|
||||
evals_fine.resize(0);
|
||||
evals_coarse.resize(0);
|
||||
};
|
||||
|
||||
//The block inner product is the inner product on the fine grid locally summed over the blocks
|
||||
//to give a Lattice<Scalar> on the coarse grid. This function orthnormalizes the fine-grid subspace
|
||||
//vectors under the block inner product. This step must be performed after computing the fine grid
|
||||
//eigenvectors and before computing the coarse grid eigenvectors.
|
||||
void Orthogonalise(void ) {
|
||||
CoarseScalar InnerProd(_CoarseGrid);
|
||||
std::cout << GridLogMessage <<" Gramm-Schmidt pass 1"<<std::endl;
|
||||
blockOrthogonalise(InnerProd,subspace);
|
||||
std::cout << GridLogMessage <<" Gramm-Schmidt pass 2"<<std::endl;
|
||||
blockOrthogonalise(InnerProd,subspace);
|
||||
};
|
||||
|
||||
template<typename T> static RealD normalise(T& v)
|
||||
{
|
||||
RealD nn = norm2(v);
|
||||
nn = ::sqrt(nn);
|
||||
v = v * (1.0/nn);
|
||||
return nn;
|
||||
}
|
||||
/*
|
||||
void fakeFine(void)
|
||||
{
|
||||
int Nk = nbasis;
|
||||
subspace.resize(Nk,_FineGrid);
|
||||
subspace[0]=1.0;
|
||||
subspace[0].Checkerboard()=_checkerboard;
|
||||
normalise(subspace[0]);
|
||||
PlainHermOp<FineField> Op(_FineOp);
|
||||
for(int k=1;k<Nk;k++){
|
||||
subspace[k].Checkerboard()=_checkerboard;
|
||||
Op(subspace[k-1],subspace[k]);
|
||||
normalise(subspace[k]);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
void testFine(RealD resid)
|
||||
{
|
||||
assert(evals_fine.size() == nbasis);
|
||||
assert(subspace.size() == nbasis);
|
||||
PlainHermOp<FineField> Op(_FineOp);
|
||||
ImplicitlyRestartedLanczosHermOpTester<FineField> SimpleTester(Op);
|
||||
for(int k=0;k<nbasis;k++){
|
||||
assert(SimpleTester.ReconstructEval(k,resid,subspace[k],evals_fine[k],1.0)==1);
|
||||
}
|
||||
}
|
||||
|
||||
//While this method serves to check the coarse eigenvectors, it also recomputes the eigenvalues from the smoothed reconstructed eigenvectors
|
||||
//hence the smoother can be tuned after running the coarse Lanczos by using a different smoother here
|
||||
void testCoarse(RealD resid,ChebyParams cheby_smooth,RealD relax)
|
||||
{
|
||||
assert(evals_fine.size() == nbasis);
|
||||
assert(subspace.size() == nbasis);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// create a smoother and see if we can get a cheap convergence test and smooth inside the IRL
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Chebyshev<FineField> ChebySmooth(cheby_smooth);
|
||||
ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (ChebySmooth,_FineOp,subspace);
|
||||
ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,subspace,relax);
|
||||
|
||||
for(int k=0;k<evec_coarse.size();k++){
|
||||
if ( k < nbasis ) {
|
||||
assert(ChebySmoothTester.ReconstructEval(k,resid,evec_coarse[k],evals_coarse[k],1.0)==1);
|
||||
} else {
|
||||
assert(ChebySmoothTester.ReconstructEval(k,resid*relax,evec_coarse[k],evals_coarse[k],1.0)==1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void calcFine(ChebyParams cheby_parms,int Nstop,int Nk,int Nm,RealD resid,
|
||||
RealD MaxIt, RealD betastp, int MinRes)
|
||||
{
|
||||
assert(nbasis<=Nm);
|
||||
Chebyshev<FineField> Cheby(cheby_parms);
|
||||
FunctionHermOp<FineField> ChebyOp(Cheby,_FineOp);
|
||||
PlainHermOp<FineField> Op(_FineOp);
|
||||
|
||||
evals_fine.resize(Nm);
|
||||
subspace.resize(Nm,_FineGrid);
|
||||
|
||||
ImplicitlyRestartedLanczos<FineField> IRL(ChebyOp,Op,Nstop,Nk,Nm,resid,MaxIt,betastp,MinRes);
|
||||
|
||||
FineField src(_FineGrid);
|
||||
typedef typename FineField::scalar_type Scalar;
|
||||
// src=1.0;
|
||||
src=Scalar(1.0);
|
||||
src.Checkerboard() = _checkerboard;
|
||||
|
||||
int Nconv;
|
||||
IRL.calc(evals_fine,subspace,src,Nconv,false);
|
||||
|
||||
// Shrink down to number saved
|
||||
assert(Nstop>=nbasis);
|
||||
assert(Nconv>=nbasis);
|
||||
evals_fine.resize(nbasis);
|
||||
subspace.resize(nbasis,_FineGrid);
|
||||
}
|
||||
|
||||
|
||||
//cheby_op: Parameters of the fine grid Chebyshev polynomial used for the Lanczos acceleration
|
||||
//cheby_smooth: Parameters of a separate Chebyshev polynomial used after the Lanczos has completed to smooth out high frequency noise in the reconstructed fine grid eigenvectors prior to computing the eigenvalue
|
||||
//relax: Reconstructed eigenvectors (post smoothing) are naturally not as precise as true eigenvectors. This factor acts as a multiplier on the stopping condition when determining whether the results satisfy the user provided stopping condition
|
||||
void calcCoarse(ChebyParams cheby_op,ChebyParams cheby_smooth,RealD relax,
|
||||
int Nstop, int Nk, int Nm,RealD resid,
|
||||
RealD MaxIt, RealD betastp, int MinRes)
|
||||
{
|
||||
Chebyshev<FineField> Cheby(cheby_op); //Chebyshev of fine operator on fine grid
|
||||
ProjectedHermOp<Fobj,CComplex,nbasis> Op(_FineOp,subspace); //Fine operator on coarse grid with intermediate fine grid conversion
|
||||
ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (Cheby,_FineOp,subspace); //Chebyshev of fine operator on coarse grid with intermediate fine grid conversion
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// create a smoother and see if we can get a cheap convergence test and smooth inside the IRL
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Chebyshev<FineField> ChebySmooth(cheby_smooth); //lower order Chebyshev of fine operator on fine grid used to smooth regenerated eigenvectors
|
||||
ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,subspace,relax,Nstop-1);
|
||||
|
||||
evals_coarse.resize(Nm);
|
||||
evec_coarse.resize(Nm,_CoarseGrid);
|
||||
|
||||
CoarseField src(_CoarseGrid); src=1.0;
|
||||
|
||||
//Note the "tester" here is also responsible for generating the fine grid eigenvalues which are output into the "evals_coarse" array
|
||||
ImplicitlyRestartedLanczos<CoarseField> IRL(ChebyOp,ChebyOp,ChebySmoothTester,Nstop,Nk,Nm,resid,MaxIt,betastp,MinRes);
|
||||
int Nconv=0;
|
||||
IRL.calc(evals_coarse,evec_coarse,src,Nconv,false);
|
||||
assert(Nconv>=Nstop);
|
||||
evals_coarse.resize(Nstop);
|
||||
evec_coarse.resize (Nstop,_CoarseGrid);
|
||||
for (int i=0;i<Nstop;i++){
|
||||
std::cout << i << " Coarse eval = " << evals_coarse[i] << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
//Get the fine eigenvector 'i' by reconstruction
|
||||
void getFineEvecEval(FineField &evec, RealD &eval, const int i) const{
|
||||
blockPromote(evec_coarse[i],evec,subspace);
|
||||
eval = evals_coarse[i];
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,157 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/MinimalResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_MINIMAL_RESIDUAL_H
|
||||
#define GRID_MINIMAL_RESIDUAL_H
|
||||
|
||||
namespace Grid {
|
||||
|
||||
template<class Field> class MinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // throw an assert when the MR fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
RealD overRelaxParam;
|
||||
Integer IterationsToComplete; // Number of iterations the MR took to finish.
|
||||
// Filled in upon completion
|
||||
|
||||
MinimalResidual(RealD tol, Integer maxit, Real ovrelparam = 1.0, bool err_on_no_conv = true)
|
||||
: Tolerance(tol), MaxIterations(maxit), overRelaxParam(ovrelparam), ErrorOnNoConverge(err_on_no_conv){};
|
||||
|
||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
ComplexD a, c;
|
||||
RealD d;
|
||||
|
||||
Field Mr(src);
|
||||
Field r(src);
|
||||
|
||||
// Initial residual computation & set up
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
Linop.Op(psi, Mr);
|
||||
|
||||
r = src - Mr;
|
||||
|
||||
RealD cp = norm2(r);
|
||||
|
||||
std::cout << std::setprecision(4) << std::scientific;
|
||||
std::cout << GridLogIterative << "MinimalResidual: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << "MinimalResidual: src " << ssq << std::endl;
|
||||
std::cout << GridLogIterative << "MinimalResidual: cp,r " << cp << std::endl;
|
||||
|
||||
if (cp <= rsq) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::cout << GridLogIterative << "MinimalResidual: k=0 residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop.Op(r, Mr);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
c = innerProduct(Mr, r);
|
||||
|
||||
d = norm2(Mr);
|
||||
|
||||
a = c / d;
|
||||
|
||||
a = a * overRelaxParam;
|
||||
|
||||
psi = psi + r * a;
|
||||
|
||||
r = r - Mr * a;
|
||||
|
||||
cp = norm2(r);
|
||||
|
||||
LinalgTimer.Stop();
|
||||
|
||||
std::cout << GridLogIterative << "MinimalResidual: Iteration " << k
|
||||
<< " residual " << cp << " target " << rsq << std::endl;
|
||||
std::cout << GridLogDebug << "a = " << a << " c = " << c << " d = " << d << std::endl;
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
SolverTimer.Stop();
|
||||
|
||||
Linop.Op(psi, Mr);
|
||||
r = src - Mr;
|
||||
|
||||
RealD srcnorm = sqrt(ssq);
|
||||
RealD resnorm = sqrt(norm2(r));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "MinimalResidual Converged on iteration " << k
|
||||
<< " computed residual " << sqrt(cp / ssq)
|
||||
<< " true residual " << true_residual
|
||||
<< " target " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "MR Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MR Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MR Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
assert(true_residual / Tolerance < 10000.0);
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "MinimalResidual did NOT converge"
|
||||
<< std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
assert(0);
|
||||
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
};
|
||||
} // namespace Grid
|
||||
#endif
|
||||
@@ -1,276 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_MIXED_PRECISION_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
#define GRID_MIXED_PRECISION_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
||||
|
||||
namespace Grid {
|
||||
|
||||
template<class FieldD, class FieldF, typename std::enable_if<getPrecision<FieldD>::value == 2, int>::type = 0, typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class MixedPrecisionFlexibleGeneralisedMinimalResidual : public OperatorFunction<FieldD> {
|
||||
public:
|
||||
|
||||
using OperatorFunction<FieldD>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an assert when MPFGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
|
||||
Integer MaxIterations;
|
||||
Integer RestartLength;
|
||||
Integer MaxNumberOfRestarts;
|
||||
Integer IterationCount; // Number of iterations the MPFGMRES took to finish,
|
||||
// filled in upon completion
|
||||
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch PrecTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch QrTimer;
|
||||
GridStopWatch CompSolutionTimer;
|
||||
GridStopWatch ChangePrecTimer;
|
||||
|
||||
Eigen::MatrixXcd H;
|
||||
|
||||
std::vector<ComplexD> y;
|
||||
std::vector<ComplexD> gamma;
|
||||
std::vector<ComplexD> c;
|
||||
std::vector<ComplexD> s;
|
||||
|
||||
GridBase* SinglePrecGrid;
|
||||
|
||||
LinearFunction<FieldF> &Preconditioner;
|
||||
|
||||
MixedPrecisionFlexibleGeneralisedMinimalResidual(RealD tol,
|
||||
Integer maxit,
|
||||
GridBase * sp_grid,
|
||||
LinearFunction<FieldF> &Prec,
|
||||
Integer restart_length,
|
||||
bool err_on_no_conv = true)
|
||||
: Tolerance(tol)
|
||||
, MaxIterations(maxit)
|
||||
, RestartLength(restart_length)
|
||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
||||
, ErrorOnNoConverge(err_on_no_conv)
|
||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
||||
, y(RestartLength + 1, 0.)
|
||||
, gamma(RestartLength + 1, 0.)
|
||||
, c(RestartLength + 1, 0.)
|
||||
, s(RestartLength + 1, 0.)
|
||||
, SinglePrecGrid(sp_grid)
|
||||
, Preconditioner(Prec) {};
|
||||
|
||||
void operator()(LinearOperatorBase<FieldD> &LinOp, const FieldD &src, FieldD &psi) {
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
|
||||
FieldD r(src.Grid());
|
||||
|
||||
std::cout << std::setprecision(4) << std::scientific;
|
||||
std::cout << GridLogIterative << "MPFGMRES: guess " << guess << std::endl;
|
||||
std::cout << GridLogIterative << "MPFGMRES: src " << ssq << std::endl;
|
||||
|
||||
PrecTimer.Reset();
|
||||
MatrixTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
QrTimer.Reset();
|
||||
CompSolutionTimer.Reset();
|
||||
ChangePrecTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
IterationCount = 0;
|
||||
|
||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
||||
|
||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
LinOp.Op(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
|
||||
RealD srcnorm = sqrt(ssq);
|
||||
RealD resnorm = sqrt(norm2(r));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "MPFGMRES: Converged on iteration " << IterationCount
|
||||
<< " computed residual " << sqrt(cp / ssq)
|
||||
<< " true residual " << true_residual
|
||||
<< " target " << Tolerance << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Precon " << PrecTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: PrecChange " << ChangePrecTimer.Elapsed() << std::endl;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "MPFGMRES did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<FieldD> &LinOp, const FieldD &src, FieldD &psi, RealD rsq) {
|
||||
|
||||
RealD cp = 0;
|
||||
|
||||
FieldD w(src.Grid());
|
||||
FieldD r(src.Grid());
|
||||
|
||||
// these should probably be made class members so that they are only allocated once, not in every restart
|
||||
std::vector<FieldD> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
||||
std::vector<FieldD> z(RestartLength + 1, src.Grid()); for (auto &elem : z) elem = Zero();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(psi, w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
r = src - w;
|
||||
|
||||
gamma[0] = sqrt(norm2(r));
|
||||
|
||||
v[0] = (1. / gamma[0]) * r;
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for (int i=0; i<RestartLength; i++) {
|
||||
|
||||
IterationCount++;
|
||||
|
||||
arnoldiStep(LinOp, v, z, w, i);
|
||||
|
||||
qrUpdate(i);
|
||||
|
||||
cp = norm(gamma[i+1]);
|
||||
|
||||
std::cout << GridLogIterative << "MPFGMRES: Iteration " << IterationCount
|
||||
<< " residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
||||
|
||||
computeSolution(z, psi, i);
|
||||
|
||||
return cp;
|
||||
}
|
||||
}
|
||||
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
void arnoldiStep(LinearOperatorBase<FieldD> &LinOp, std::vector<FieldD> &v, std::vector<FieldD> &z, FieldD &w, int iter) {
|
||||
|
||||
FieldF v_f(SinglePrecGrid);
|
||||
FieldF z_f(SinglePrecGrid);
|
||||
|
||||
ChangePrecTimer.Start();
|
||||
precisionChange(v_f, v[iter]);
|
||||
precisionChange(z_f, z[iter]);
|
||||
ChangePrecTimer.Stop();
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(v_f, z_f);
|
||||
PrecTimer.Stop();
|
||||
|
||||
ChangePrecTimer.Start();
|
||||
precisionChange(z[iter], z_f);
|
||||
ChangePrecTimer.Stop();
|
||||
|
||||
MatrixTimer.Start();
|
||||
LinOp.Op(z[iter], w);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
for (int i = 0; i <= iter; ++i) {
|
||||
H(iter, i) = innerProduct(v[i], w);
|
||||
w = w - ComplexD(H(iter, i)) * v[i];
|
||||
}
|
||||
|
||||
H(iter, iter + 1) = sqrt(norm2(w));
|
||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
|
||||
void qrUpdate(int iter) {
|
||||
|
||||
QrTimer.Start();
|
||||
for (int i = 0; i < iter ; ++i) {
|
||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
||||
H(iter, i + 1) = tmp;
|
||||
}
|
||||
|
||||
// Compute new Givens Rotation
|
||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
||||
c[iter] = H(iter, iter) / nu;
|
||||
s[iter] = H(iter, iter + 1) / nu;
|
||||
|
||||
// Apply new Givens rotation
|
||||
H(iter, iter) = nu;
|
||||
H(iter, iter + 1) = 0.;
|
||||
|
||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
||||
QrTimer.Stop();
|
||||
}
|
||||
|
||||
void computeSolution(std::vector<FieldD> const &z, FieldD &psi, int iter) {
|
||||
|
||||
CompSolutionTimer.Start();
|
||||
for (int i = iter; i >= 0; i--) {
|
||||
y[i] = gamma[i];
|
||||
for (int k = i + 1; k <= iter; k++)
|
||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
||||
y[i] = y[i] / ComplexD(H(i, i));
|
||||
}
|
||||
|
||||
for (int i = 0; i <= iter; i++)
|
||||
psi = psi + z[i] * y[i];
|
||||
CompSolutionTimer.Stop();
|
||||
}
|
||||
};
|
||||
}
|
||||
#endif
|
||||
@@ -1,112 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/NormalEquations.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_NORMAL_EQUATIONS_H
|
||||
#define GRID_NORMAL_EQUATIONS_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Take a matrix and form an NE solver calling a Herm solver
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class NormalEquations {
|
||||
private:
|
||||
SparseMatrixBase<Field> & _Matrix;
|
||||
OperatorFunction<Field> & _HermitianSolver;
|
||||
LinearFunction<Field> & _Guess;
|
||||
public:
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Wrap the usual normal equations trick
|
||||
/////////////////////////////////////////////////////
|
||||
NormalEquations(SparseMatrixBase<Field> &Matrix, OperatorFunction<Field> &HermitianSolver,
|
||||
LinearFunction<Field> &Guess)
|
||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
||||
|
||||
void operator() (const Field &in, Field &out){
|
||||
|
||||
Field src(in.Grid());
|
||||
Field tmp(in.Grid());
|
||||
|
||||
MdagMLinearOperator<SparseMatrixBase<Field>,Field> MdagMOp(_Matrix);
|
||||
_Matrix.Mdag(in,src);
|
||||
_Guess(src,out);
|
||||
_HermitianSolver(MdagMOp,src,out); // Mdag M out = Mdag in
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field> class HPDSolver {
|
||||
private:
|
||||
LinearOperatorBase<Field> & _Matrix;
|
||||
OperatorFunction<Field> & _HermitianSolver;
|
||||
LinearFunction<Field> & _Guess;
|
||||
public:
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Wrap the usual normal equations trick
|
||||
/////////////////////////////////////////////////////
|
||||
HPDSolver(LinearOperatorBase<Field> &Matrix,
|
||||
OperatorFunction<Field> &HermitianSolver,
|
||||
LinearFunction<Field> &Guess)
|
||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
||||
|
||||
void operator() (const Field &in, Field &out){
|
||||
|
||||
_Guess(in,out);
|
||||
_HermitianSolver(_Matrix,in,out); // Mdag M out = Mdag in
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template<class Field> class MdagMSolver {
|
||||
private:
|
||||
SparseMatrixBase<Field> & _Matrix;
|
||||
OperatorFunction<Field> & _HermitianSolver;
|
||||
LinearFunction<Field> & _Guess;
|
||||
public:
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Wrap the usual normal equations trick
|
||||
/////////////////////////////////////////////////////
|
||||
MdagMSolver(SparseMatrixBase<Field> &Matrix, OperatorFunction<Field> &HermitianSolver,
|
||||
LinearFunction<Field> &Guess)
|
||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
||||
|
||||
void operator() (const Field &in, Field &out){
|
||||
|
||||
MdagMLinearOperator<SparseMatrixBase<Field>,Field> MdagMOp(_Matrix);
|
||||
_Guess(in,out);
|
||||
|
||||
_HermitianSolver(MdagMOp,in,out); // Mdag M out = Mdag in
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,47 +0,0 @@
|
||||
#pragma once
|
||||
namespace Grid {
|
||||
template<class Field> class PowerMethod
|
||||
{
|
||||
public:
|
||||
|
||||
template<typename T> static RealD normalise(T& v)
|
||||
{
|
||||
RealD nn = norm2(v);
|
||||
nn = sqrt(nn);
|
||||
v = v * (1.0/nn);
|
||||
return nn;
|
||||
}
|
||||
|
||||
RealD operator()(LinearOperatorBase<Field> &HermOp, const Field &src)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
|
||||
// quickly get an idea of the largest eigenvalue to more properly normalize the residuum
|
||||
RealD evalMaxApprox = 0.0;
|
||||
auto src_n = src;
|
||||
auto tmp = src;
|
||||
const int _MAX_ITER_EST_ = 50;
|
||||
|
||||
for (int i=0;i<_MAX_ITER_EST_;i++) {
|
||||
|
||||
normalise(src_n);
|
||||
HermOp.HermOp(src_n,tmp);
|
||||
RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
|
||||
RealD vden = norm2(src_n);
|
||||
RealD na = vnum/vden;
|
||||
|
||||
std::cout << GridLogIterative << "PowerMethod: Current approximation of largest eigenvalue " << na << std::endl;
|
||||
|
||||
if ( (fabs(evalMaxApprox/na - 1.0) < 0.001) || (i==_MAX_ITER_EST_-1) ) {
|
||||
evalMaxApprox = na;
|
||||
std::cout << GridLogMessage << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
|
||||
return evalMaxApprox;
|
||||
}
|
||||
evalMaxApprox = na;
|
||||
src_n = tmp;
|
||||
}
|
||||
assert(0);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/PrecConjugateResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_PREC_CONJUGATE_RESIDUAL_H
|
||||
#define GRID_PREC_CONJUGATE_RESIDUAL_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Base classes for iterative processes based on operators
|
||||
// single input vec, single output vec.
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
||||
template<class Field>
|
||||
class PrecConjugateResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
int verbose;
|
||||
LinearFunction<Field> &Preconditioner;
|
||||
|
||||
PrecConjugateResidual(RealD tol,Integer maxit,LinearFunction<Field> &Prec) : Tolerance(tol), MaxIterations(maxit), Preconditioner(Prec)
|
||||
{
|
||||
verbose=1;
|
||||
};
|
||||
|
||||
void operator() (LinearOperatorBase<Field> &Linop,const Field &src, Field &psi){
|
||||
|
||||
RealD a, b, c, d;
|
||||
RealD cp, ssq,rsq;
|
||||
|
||||
RealD rAr, rAAr, rArp;
|
||||
RealD pAp, pAAp;
|
||||
|
||||
GridBase *grid = src.Grid();
|
||||
Field r(grid), p(grid), Ap(grid), Ar(grid), z(grid);
|
||||
|
||||
psi=zero;
|
||||
r = src;
|
||||
Preconditioner(r,p);
|
||||
|
||||
|
||||
|
||||
Linop.HermOpAndNorm(p,Ap,pAp,pAAp);
|
||||
Ar=Ap;
|
||||
rAr=pAp;
|
||||
rAAr=pAAp;
|
||||
|
||||
cp =norm2(r);
|
||||
ssq=norm2(src);
|
||||
rsq=Tolerance*Tolerance*ssq;
|
||||
|
||||
if (verbose) std::cout<<GridLogMessage<<"PrecConjugateResidual: iteration " <<0<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
||||
|
||||
for(int k=0;k<MaxIterations;k++){
|
||||
|
||||
|
||||
Preconditioner(Ap,z);
|
||||
RealD rq= real(innerProduct(Ap,z));
|
||||
|
||||
a = rAr/rq;
|
||||
|
||||
axpy(psi,a,p,psi);
|
||||
cp = axpy_norm(r,-a,z,r);
|
||||
|
||||
rArp=rAr;
|
||||
|
||||
Linop.HermOpAndNorm(r,Ar,rAr,rAAr);
|
||||
|
||||
b =rAr/rArp;
|
||||
|
||||
axpy(p,b,p,r);
|
||||
pAAp=axpy_norm(Ap,b,Ap,Ar);
|
||||
|
||||
if(verbose) std::cout<<GridLogMessage<<"PrecConjugateResidual: iteration " <<k<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
||||
|
||||
if(cp<rsq) {
|
||||
Linop.HermOp(psi,Ap);
|
||||
axpy(r,-1.0,src,Ap);
|
||||
RealD true_resid = norm2(r)/ssq;
|
||||
std::cout<<GridLogMessage<<"PrecConjugateResidual: Converged on iteration " <<k
|
||||
<< " computed residual "<<sqrt(cp/ssq)
|
||||
<< " true residual "<<sqrt(true_resid)
|
||||
<< " target " <<Tolerance <<std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::cout<<GridLogMessage<<"PrecConjugateResidual did NOT converge"<<std::endl;
|
||||
assert(0);
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,239 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/PrecGeneralisedConjugateResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_PREC_GCR_H
|
||||
#define GRID_PREC_GCR_H
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//VPGCR Abe and Zhang, 2005.
|
||||
//INTERNATIONAL JOURNAL OF NUMERICAL ANALYSIS AND MODELING
|
||||
//Computing and Information Volume 2, Number 2, Pages 147-161
|
||||
//NB. Likely not original reference since they are focussing on a preconditioner variant.
|
||||
// but VPGCR was nicely written up in their paper
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#define GCRLogLevel std::cout << GridLogMessage <<std::string(level,'\t')<< " Level "<<level<<" "
|
||||
|
||||
template<class Field>
|
||||
class PrecGeneralisedConjugateResidual : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
int verbose;
|
||||
int mmax;
|
||||
int nstep;
|
||||
int steps;
|
||||
int level;
|
||||
GridStopWatch PrecTimer;
|
||||
GridStopWatch MatTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
|
||||
LinearFunction<Field> &Preconditioner;
|
||||
LinearOperatorBase<Field> &Linop;
|
||||
|
||||
void Level(int lv) { level=lv; };
|
||||
|
||||
PrecGeneralisedConjugateResidual(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax,int _nstep) :
|
||||
Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
Linop(_Linop),
|
||||
Preconditioner(Prec),
|
||||
mmax(_mmax),
|
||||
nstep(_nstep)
|
||||
{
|
||||
level=1;
|
||||
verbose=1;
|
||||
};
|
||||
|
||||
void operator() (const Field &src, Field &psi){
|
||||
|
||||
psi=Zero();
|
||||
RealD cp, ssq,rsq;
|
||||
ssq=norm2(src);
|
||||
rsq=Tolerance*Tolerance*ssq;
|
||||
|
||||
Field r(src.Grid());
|
||||
|
||||
PrecTimer.Reset();
|
||||
MatTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
steps=0;
|
||||
for(int k=0;k<MaxIterations;k++){
|
||||
|
||||
cp=GCRnStep(src,psi,rsq);
|
||||
|
||||
GCRLogLevel <<"PGCR("<<mmax<<","<<nstep<<") "<< steps <<" steps cp = "<<cp<<" target "<<rsq <<std::endl;
|
||||
|
||||
if(cp<rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
Linop.HermOp(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
RealD tr = norm2(r);
|
||||
GCRLogLevel<<"PGCR: Converged on iteration " <<steps
|
||||
<< " computed residual "<<sqrt(cp/ssq)
|
||||
<< " true residual " <<sqrt(tr/ssq)
|
||||
<< " target " <<Tolerance <<std::endl;
|
||||
|
||||
GCRLogLevel<<"PGCR Time elapsed: Total "<< SolverTimer.Elapsed() <<std::endl;
|
||||
/*
|
||||
GCRLogLevel<<"PGCR Time elapsed: Precon "<< PrecTimer.Elapsed() <<std::endl;
|
||||
GCRLogLevel<<"PGCR Time elapsed: Matrix "<< MatTimer.Elapsed() <<std::endl;
|
||||
GCRLogLevel<<"PGCR Time elapsed: Linalg "<< LinalgTimer.Elapsed() <<std::endl;
|
||||
*/
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
GCRLogLevel<<"Variable Preconditioned GCR did not converge"<<std::endl;
|
||||
// assert(0);
|
||||
}
|
||||
|
||||
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
||||
|
||||
RealD cp;
|
||||
RealD a, b;
|
||||
RealD zAz, zAAz;
|
||||
RealD rq;
|
||||
|
||||
GridBase *grid = src.Grid();
|
||||
|
||||
Field r(grid);
|
||||
Field z(grid);
|
||||
Field tmp(grid);
|
||||
Field ttmp(grid);
|
||||
Field Az(grid);
|
||||
|
||||
////////////////////////////////
|
||||
// history for flexible orthog
|
||||
////////////////////////////////
|
||||
std::vector<Field> q(mmax,grid);
|
||||
std::vector<Field> p(mmax,grid);
|
||||
std::vector<RealD> qq(mmax);
|
||||
|
||||
GCRLogLevel<< "PGCR nStep("<<nstep<<")"<<std::endl;
|
||||
|
||||
//////////////////////////////////
|
||||
// initial guess x0 is taken as nonzero.
|
||||
// r0=src-A x0 = src
|
||||
//////////////////////////////////
|
||||
MatTimer.Start();
|
||||
Linop.HermOpAndNorm(psi,Az,zAz,zAAz);
|
||||
MatTimer.Stop();
|
||||
|
||||
|
||||
LinalgTimer.Start();
|
||||
r=src-Az;
|
||||
LinalgTimer.Stop();
|
||||
GCRLogLevel<< "PGCR true residual r = src - A psi "<<norm2(r) <<std::endl;
|
||||
|
||||
/////////////////////
|
||||
// p = Prec(r)
|
||||
/////////////////////
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(r,z);
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatTimer.Start();
|
||||
Linop.HermOpAndNorm(z,Az,zAz,zAAz);
|
||||
MatTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
//p[0],q[0],qq[0]
|
||||
p[0]= z;
|
||||
q[0]= Az;
|
||||
qq[0]= zAAz;
|
||||
|
||||
cp =norm2(r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for(int k=0;k<nstep;k++){
|
||||
|
||||
steps++;
|
||||
|
||||
int kp = k+1;
|
||||
int peri_k = k %mmax;
|
||||
int peri_kp= kp%mmax;
|
||||
|
||||
LinalgTimer.Start();
|
||||
rq= real(innerProduct(r,q[peri_k])); // what if rAr not real?
|
||||
a = rq/qq[peri_k];
|
||||
|
||||
axpy(psi,a,p[peri_k],psi);
|
||||
|
||||
cp = axpy_norm(r,-a,q[peri_k],r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
GCRLogLevel<< "PGCR step["<<steps<<"] resid " << cp << " target " <<rsq<<std::endl;
|
||||
|
||||
if((k==nstep-1)||(cp<rsq)){
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(r,z);// solve Az = r
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatTimer.Start();
|
||||
Linop.HermOpAndNorm(z,Az,zAz,zAAz);
|
||||
MatTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
q[peri_kp]=Az;
|
||||
p[peri_kp]=z;
|
||||
|
||||
int northog = ((kp)>(mmax-1))?(mmax-1):(kp); // if more than mmax done, we orthog all mmax history.
|
||||
for(int back=0;back<northog;back++){
|
||||
|
||||
int peri_back=(k-back)%mmax; assert((k-back)>=0);
|
||||
|
||||
b=-real(innerProduct(q[peri_back],Az))/qq[peri_back];
|
||||
p[peri_kp]=p[peri_kp]+b*p[peri_back];
|
||||
q[peri_kp]=q[peri_kp]+b*q[peri_back];
|
||||
|
||||
}
|
||||
qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
assert(0); // never reached
|
||||
return cp;
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,242 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/PrecGeneralisedConjugateResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_PREC_GCR_NON_HERM_H
|
||||
#define GRID_PREC_GCR_NON_HERM_H
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//VPGCR Abe and Zhang, 2005.
|
||||
//INTERNATIONAL JOURNAL OF NUMERICAL ANALYSIS AND MODELING
|
||||
//Computing and Information Volume 2, Number 2, Pages 147-161
|
||||
//NB. Likely not original reference since they are focussing on a preconditioner variant.
|
||||
// but VPGCR was nicely written up in their paper
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#define GCRLogLevel std::cout << GridLogMessage <<std::string(level,'\t')<< " Level "<<level<<" "
|
||||
|
||||
template<class Field>
|
||||
class PrecGeneralisedConjugateResidualNonHermitian : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
int verbose;
|
||||
int mmax;
|
||||
int nstep;
|
||||
int steps;
|
||||
int level;
|
||||
GridStopWatch PrecTimer;
|
||||
GridStopWatch MatTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
|
||||
LinearFunction<Field> &Preconditioner;
|
||||
LinearOperatorBase<Field> &Linop;
|
||||
|
||||
void Level(int lv) { level=lv; };
|
||||
|
||||
PrecGeneralisedConjugateResidualNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax,int _nstep) :
|
||||
Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
Linop(_Linop),
|
||||
Preconditioner(Prec),
|
||||
mmax(_mmax),
|
||||
nstep(_nstep)
|
||||
{
|
||||
level=1;
|
||||
verbose=1;
|
||||
};
|
||||
|
||||
void operator() (const Field &src, Field &psi){
|
||||
|
||||
psi=Zero();
|
||||
RealD cp, ssq,rsq;
|
||||
ssq=norm2(src);
|
||||
rsq=Tolerance*Tolerance*ssq;
|
||||
|
||||
Field r(src.Grid());
|
||||
|
||||
PrecTimer.Reset();
|
||||
MatTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
steps=0;
|
||||
for(int k=0;k<MaxIterations;k++){
|
||||
|
||||
cp=GCRnStep(src,psi,rsq);
|
||||
|
||||
GCRLogLevel <<"PGCR("<<mmax<<","<<nstep<<") "<< steps <<" steps cp = "<<cp<<" target "<<rsq <<std::endl;
|
||||
|
||||
if(cp<rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
Linop.Op(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
RealD tr = norm2(r);
|
||||
GCRLogLevel<<"PGCR: Converged on iteration " <<steps
|
||||
<< " computed residual "<<sqrt(cp/ssq)
|
||||
<< " true residual " <<sqrt(tr/ssq)
|
||||
<< " target " <<Tolerance <<std::endl;
|
||||
|
||||
GCRLogLevel<<"PGCR Time elapsed: Total "<< SolverTimer.Elapsed() <<std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
GCRLogLevel<<"Variable Preconditioned GCR did not converge"<<std::endl;
|
||||
// assert(0);
|
||||
}
|
||||
|
||||
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
||||
|
||||
RealD cp;
|
||||
ComplexD a, b;
|
||||
// ComplexD zAz;
|
||||
RealD zAAz;
|
||||
ComplexD rq;
|
||||
|
||||
GridBase *grid = src.Grid();
|
||||
|
||||
Field r(grid);
|
||||
Field z(grid);
|
||||
Field tmp(grid);
|
||||
Field ttmp(grid);
|
||||
Field Az(grid);
|
||||
|
||||
////////////////////////////////
|
||||
// history for flexible orthog
|
||||
////////////////////////////////
|
||||
std::vector<Field> q(mmax,grid);
|
||||
std::vector<Field> p(mmax,grid);
|
||||
std::vector<RealD> qq(mmax);
|
||||
|
||||
GCRLogLevel<< "PGCR nStep("<<nstep<<")"<<std::endl;
|
||||
|
||||
//////////////////////////////////
|
||||
// initial guess x0 is taken as nonzero.
|
||||
// r0=src-A x0 = src
|
||||
//////////////////////////////////
|
||||
MatTimer.Start();
|
||||
Linop.Op(psi,Az);
|
||||
// zAz = innerProduct(Az,psi);
|
||||
zAAz= norm2(Az);
|
||||
MatTimer.Stop();
|
||||
|
||||
|
||||
LinalgTimer.Start();
|
||||
r=src-Az;
|
||||
LinalgTimer.Stop();
|
||||
GCRLogLevel<< "PGCR true residual r = src - A psi "<<norm2(r) <<std::endl;
|
||||
|
||||
/////////////////////
|
||||
// p = Prec(r)
|
||||
/////////////////////
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(r,z);
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatTimer.Start();
|
||||
Linop.Op(z,Az);
|
||||
MatTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
// zAz = innerProduct(Az,psi);
|
||||
zAAz= norm2(Az);
|
||||
|
||||
//p[0],q[0],qq[0]
|
||||
p[0]= z;
|
||||
q[0]= Az;
|
||||
qq[0]= zAAz;
|
||||
|
||||
cp =norm2(r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for(int k=0;k<nstep;k++){
|
||||
|
||||
steps++;
|
||||
|
||||
int kp = k+1;
|
||||
int peri_k = k %mmax;
|
||||
int peri_kp= kp%mmax;
|
||||
|
||||
LinalgTimer.Start();
|
||||
rq= innerProduct(q[peri_k],r); // what if rAr not real?
|
||||
a = rq/qq[peri_k];
|
||||
|
||||
axpy(psi,a,p[peri_k],psi);
|
||||
|
||||
cp = axpy_norm(r,-a,q[peri_k],r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
GCRLogLevel<< "PGCR step["<<steps<<"] resid " << cp << " target " <<rsq<<std::endl;
|
||||
|
||||
if((k==nstep-1)||(cp<rsq)){
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(r,z);// solve Az = r
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatTimer.Start();
|
||||
Linop.Op(z,Az);
|
||||
MatTimer.Stop();
|
||||
// zAz = innerProduct(Az,psi);
|
||||
zAAz= norm2(Az);
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
q[peri_kp]=Az;
|
||||
p[peri_kp]=z;
|
||||
|
||||
int northog = ((kp)>(mmax-1))?(mmax-1):(kp); // if more than mmax done, we orthog all mmax history.
|
||||
for(int back=0;back<northog;back++){
|
||||
|
||||
int peri_back=(k-back)%mmax; assert((k-back)>=0);
|
||||
|
||||
b=-real(innerProduct(q[peri_back],Az))/qq[peri_back];
|
||||
p[peri_kp]=p[peri_kp]+b*p[peri_back];
|
||||
q[peri_kp]=q[peri_kp]+b*q[peri_back];
|
||||
|
||||
}
|
||||
qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
assert(0); // never reached
|
||||
return cp;
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,371 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithmsf/iterative/QuasiMinimalResidual.h
|
||||
|
||||
Copyright (C) 2019
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class Field>
|
||||
RealD innerG5ProductReal(Field &l, Field &r)
|
||||
{
|
||||
Gamma G5(Gamma::Algebra::Gamma5);
|
||||
Field tmp(l.Grid());
|
||||
// tmp = G5*r;
|
||||
G5R5(tmp,r);
|
||||
ComplexD ip =innerProduct(l,tmp);
|
||||
std::cout << "innerProductRealG5R5 "<<ip<<std::endl;
|
||||
return ip.real();
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class QuasiMinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge;
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
Integer IterationCount;
|
||||
|
||||
QuasiMinimalResidual(RealD tol,
|
||||
Integer maxit,
|
||||
bool err_on_no_conv = true)
|
||||
: Tolerance(tol)
|
||||
, MaxIterations(maxit)
|
||||
, ErrorOnNoConverge(err_on_no_conv)
|
||||
{};
|
||||
|
||||
#if 1
|
||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &b, Field &x)
|
||||
{
|
||||
RealD resid;
|
||||
IterationCount=0;
|
||||
|
||||
RealD rho, rho_1, xi, gamma, gamma_1, theta, theta_1;
|
||||
RealD eta, delta, ep, beta;
|
||||
|
||||
GridBase *Grid = b.Grid();
|
||||
Field r(Grid), d(Grid), s(Grid);
|
||||
Field v(Grid), w(Grid), y(Grid), z(Grid);
|
||||
Field v_tld(Grid), w_tld(Grid), y_tld(Grid), z_tld(Grid);
|
||||
Field p(Grid), q(Grid), p_tld(Grid);
|
||||
|
||||
Real normb = norm2(b);
|
||||
|
||||
LinOp.Op(x,r); r = b - r;
|
||||
|
||||
assert(normb> 0.0);
|
||||
|
||||
resid = norm2(r)/normb;
|
||||
if (resid <= Tolerance) {
|
||||
return;
|
||||
}
|
||||
|
||||
v_tld = r;
|
||||
y = v_tld;
|
||||
rho = norm2(y);
|
||||
|
||||
// Take Gamma5 conjugate
|
||||
// Gamma G5(Gamma::Algebra::Gamma5);
|
||||
// G5R5(w_tld,r);
|
||||
// w_tld = G5* v_tld;
|
||||
w_tld=v_tld;
|
||||
z = w_tld;
|
||||
xi = norm2(z);
|
||||
|
||||
gamma = 1.0;
|
||||
eta = -1.0;
|
||||
theta = 0.0;
|
||||
|
||||
for (int i = 1; i <= MaxIterations; i++) {
|
||||
|
||||
// Breakdown tests
|
||||
assert( rho != 0.0);
|
||||
assert( xi != 0.0);
|
||||
|
||||
v = (1. / rho) * v_tld;
|
||||
y = (1. / rho) * y;
|
||||
|
||||
w = (1. / xi) * w_tld;
|
||||
z = (1. / xi) * z;
|
||||
|
||||
ComplexD Zdelta = innerProduct(z, y); // Complex?
|
||||
std::cout << "Zdelta "<<Zdelta<<std::endl;
|
||||
delta = Zdelta.real();
|
||||
|
||||
y_tld = y;
|
||||
z_tld = z;
|
||||
|
||||
if (i > 1) {
|
||||
p = y_tld - (xi * delta / ep) * p;
|
||||
q = z_tld - (rho * delta / ep) * q;
|
||||
} else {
|
||||
p = y_tld;
|
||||
q = z_tld;
|
||||
}
|
||||
|
||||
LinOp.Op(p,p_tld); // p_tld = A * p;
|
||||
ComplexD Zep = innerProduct(q, p_tld);
|
||||
ep=Zep.real();
|
||||
std::cout << "Zep "<<Zep <<std::endl;
|
||||
// Complex Audit
|
||||
assert(abs(ep)>0);
|
||||
|
||||
beta = ep / delta;
|
||||
assert(abs(beta)>0);
|
||||
|
||||
v_tld = p_tld - beta * v;
|
||||
y = v_tld;
|
||||
|
||||
rho_1 = rho;
|
||||
rho = norm2(y);
|
||||
LinOp.AdjOp(q,w_tld);
|
||||
w_tld = w_tld - beta * w;
|
||||
z = w_tld;
|
||||
|
||||
xi = norm2(z);
|
||||
|
||||
gamma_1 = gamma;
|
||||
theta_1 = theta;
|
||||
|
||||
theta = rho / (gamma_1 * beta);
|
||||
gamma = 1.0 / sqrt(1.0 + theta * theta);
|
||||
std::cout << "theta "<<theta<<std::endl;
|
||||
std::cout << "gamma "<<gamma<<std::endl;
|
||||
|
||||
assert(abs(gamma)> 0.0);
|
||||
|
||||
eta = -eta * rho_1 * gamma* gamma / (beta * gamma_1 * gamma_1);
|
||||
|
||||
if (i > 1) {
|
||||
d = eta * p + (theta_1 * theta_1 * gamma * gamma) * d;
|
||||
s = eta * p_tld + (theta_1 * theta_1 * gamma * gamma) * s;
|
||||
} else {
|
||||
d = eta * p;
|
||||
s = eta * p_tld;
|
||||
}
|
||||
|
||||
x =x+d; // update approximation vector
|
||||
r =r-s; // compute residual
|
||||
|
||||
if ((resid = norm2(r) / normb) <= Tolerance) {
|
||||
return;
|
||||
}
|
||||
std::cout << "Iteration "<<i<<" resid " << resid<<std::endl;
|
||||
}
|
||||
assert(0);
|
||||
return; // no convergence
|
||||
}
|
||||
#else
|
||||
// QMRg5 SMP thesis
|
||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &b, Field &x)
|
||||
{
|
||||
// Real scalars
|
||||
GridBase *grid = b.Grid();
|
||||
|
||||
Field r(grid);
|
||||
Field p_m(grid), p_m_minus_1(grid), p_m_minus_2(grid);
|
||||
Field v_m(grid), v_m_minus_1(grid), v_m_plus_1(grid);
|
||||
Field tmp(grid);
|
||||
|
||||
RealD w;
|
||||
RealD z1, z2;
|
||||
RealD delta_m, delta_m_minus_1;
|
||||
RealD c_m_plus_1, c_m, c_m_minus_1;
|
||||
RealD s_m_plus_1, s_m, s_m_minus_1;
|
||||
RealD alpha, beta, gamma, epsilon;
|
||||
RealD mu, nu, rho, theta, xi, chi;
|
||||
RealD mod2r, mod2b;
|
||||
RealD tau2, target2;
|
||||
|
||||
mod2b=norm2(b);
|
||||
|
||||
/////////////////////////
|
||||
// Initial residual
|
||||
/////////////////////////
|
||||
LinOp.Op(x,tmp);
|
||||
r = b - tmp;
|
||||
|
||||
/////////////////////////
|
||||
// \mu = \rho = |r_0|
|
||||
/////////////////////////
|
||||
mod2r = norm2(r);
|
||||
rho = sqrt( mod2r);
|
||||
mu=rho;
|
||||
|
||||
std::cout << "QuasiMinimalResidual rho "<< rho<<std::endl;
|
||||
/////////////////////////
|
||||
// Zero negative history
|
||||
/////////////////////////
|
||||
v_m_plus_1 = Zero();
|
||||
v_m_minus_1 = Zero();
|
||||
p_m_minus_1 = Zero();
|
||||
p_m_minus_2 = Zero();
|
||||
|
||||
// v0
|
||||
v_m = (1.0/rho)*r;
|
||||
|
||||
/////////////////////////
|
||||
// Initial coeffs
|
||||
/////////////////////////
|
||||
delta_m_minus_1 = 1.0;
|
||||
c_m_minus_1 = 1.0;
|
||||
c_m = 1.0;
|
||||
s_m_minus_1 = 0.0;
|
||||
s_m = 0.0;
|
||||
|
||||
/////////////////////////
|
||||
// Set up convergence check
|
||||
/////////////////////////
|
||||
tau2 = mod2r;
|
||||
target2 = mod2b * Tolerance*Tolerance;
|
||||
|
||||
for(int iter = 0 ; iter < MaxIterations; iter++){
|
||||
|
||||
/////////////////////////
|
||||
// \delta_m = (v_m, \gamma_5 v_m)
|
||||
/////////////////////////
|
||||
delta_m = innerG5ProductReal(v_m,v_m);
|
||||
std::cout << "QuasiMinimalResidual delta_m "<< delta_m<<std::endl;
|
||||
|
||||
/////////////////////////
|
||||
// tmp = A v_m
|
||||
/////////////////////////
|
||||
LinOp.Op(v_m,tmp);
|
||||
|
||||
/////////////////////////
|
||||
// \alpha = (v_m, \gamma_5 temp) / \delta_m
|
||||
/////////////////////////
|
||||
alpha = innerG5ProductReal(v_m,tmp);
|
||||
alpha = alpha/delta_m ;
|
||||
std::cout << "QuasiMinimalResidual alpha "<< alpha<<std::endl;
|
||||
|
||||
/////////////////////////
|
||||
// \beta = \rho \delta_m / \delta_{m-1}
|
||||
/////////////////////////
|
||||
beta = rho * delta_m / delta_m_minus_1;
|
||||
std::cout << "QuasiMinimalResidual beta "<< beta<<std::endl;
|
||||
|
||||
/////////////////////////
|
||||
// \tilde{v}_{m+1} = temp - \alpha v_m - \beta v_{m-1}
|
||||
/////////////////////////
|
||||
v_m_plus_1 = tmp - alpha*v_m - beta*v_m_minus_1;
|
||||
|
||||
///////////////////////////////
|
||||
// \rho = || \tilde{v}_{m+1} ||
|
||||
///////////////////////////////
|
||||
rho = sqrt( norm2(v_m_plus_1) );
|
||||
std::cout << "QuasiMinimalResidual rho "<< rho<<std::endl;
|
||||
|
||||
///////////////////////////////
|
||||
// v_{m+1} = \tilde{v}_{m+1}
|
||||
///////////////////////////////
|
||||
v_m_plus_1 = (1.0 / rho) * v_m_plus_1;
|
||||
|
||||
////////////////////////////////
|
||||
// QMR recurrence coefficients.
|
||||
////////////////////////////////
|
||||
theta = s_m_minus_1 * beta;
|
||||
gamma = c_m_minus_1 * beta;
|
||||
epsilon = c_m * gamma + s_m * alpha;
|
||||
xi = -s_m * gamma + c_m * alpha;
|
||||
nu = sqrt( xi*xi + rho*rho );
|
||||
c_m_plus_1 = fabs(xi) / nu;
|
||||
if ( xi == 0.0 ) {
|
||||
s_m_plus_1 = 1.0;
|
||||
} else {
|
||||
s_m_plus_1 = c_m_plus_1 * rho / xi;
|
||||
}
|
||||
chi = c_m_plus_1 * xi + s_m_plus_1 * rho;
|
||||
|
||||
std::cout << "QuasiMinimalResidual coeffs "<< theta <<" "<<gamma<<" "<< epsilon<<" "<< xi<<" "<< nu<<std::endl;
|
||||
std::cout << "QuasiMinimalResidual coeffs "<< chi <<std::endl;
|
||||
|
||||
////////////////////////////////
|
||||
//p_m=(v_m - \epsilon p_{m-1} - \theta p_{m-2}) / \chi
|
||||
////////////////////////////////
|
||||
p_m = (1.0/chi) * v_m - (epsilon/chi) * p_m_minus_1 - (theta/chi) * p_m_minus_2;
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// \psi = \psi + c_{m+1} \mu p_m
|
||||
////////////////////////////////////////////////////////////////
|
||||
x = x + ( c_m_plus_1 * mu ) * p_m;
|
||||
|
||||
////////////////////////////////////////
|
||||
//
|
||||
////////////////////////////////////////
|
||||
mu = -s_m_plus_1 * mu;
|
||||
delta_m_minus_1 = delta_m;
|
||||
c_m_minus_1 = c_m;
|
||||
c_m = c_m_plus_1;
|
||||
s_m_minus_1 = s_m;
|
||||
s_m = s_m_plus_1;
|
||||
|
||||
////////////////////////////////////
|
||||
// Could use pointer swizzle games.
|
||||
////////////////////////////////////
|
||||
v_m_minus_1 = v_m;
|
||||
v_m = v_m_plus_1;
|
||||
p_m_minus_2 = p_m_minus_1;
|
||||
p_m_minus_1 = p_m;
|
||||
|
||||
|
||||
/////////////////////////////////////
|
||||
// Convergence checks
|
||||
/////////////////////////////////////
|
||||
z1 = RealD(iter+1.0);
|
||||
z2 = z1 + 1.0;
|
||||
tau2 = tau2 *( z2 / z1 ) * s_m * s_m;
|
||||
std::cout << " QuasiMinimumResidual iteration "<< iter<<std::endl;
|
||||
std::cout << " QuasiMinimumResidual tau bound "<< tau2<<std::endl;
|
||||
|
||||
// Compute true residual
|
||||
mod2r = tau2;
|
||||
if ( 1 || (tau2 < (100.0 * target2)) ) {
|
||||
LinOp.Op(x,tmp);
|
||||
r = b - tmp;
|
||||
mod2r = norm2(r);
|
||||
std::cout << " QuasiMinimumResidual true residual is "<< mod2r<<std::endl;
|
||||
}
|
||||
|
||||
|
||||
if ( mod2r < target2 ) {
|
||||
|
||||
std::cout << " QuasiMinimumResidual has converged"<<std::endl;
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
#endif
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,651 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/SchurRedBlack.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_SCHUR_RED_BLACK_H
|
||||
#define GRID_SCHUR_RED_BLACK_H
|
||||
|
||||
|
||||
/*
|
||||
* Red black Schur decomposition
|
||||
*
|
||||
* M = (Mee Meo) = (1 0 ) (Mee 0 ) (1 Mee^{-1} Meo)
|
||||
* (Moe Moo) (Moe Mee^-1 1 ) (0 Moo-Moe Mee^-1 Meo) (0 1 )
|
||||
* = L D U
|
||||
*
|
||||
* L^-1 = (1 0 )
|
||||
* (-MoeMee^{-1} 1 )
|
||||
* L^{dag} = ( 1 Mee^{-dag} Moe^{dag} )
|
||||
* ( 0 1 )
|
||||
* L^{-d} = ( 1 -Mee^{-dag} Moe^{dag} )
|
||||
* ( 0 1 )
|
||||
*
|
||||
* U^-1 = (1 -Mee^{-1} Meo)
|
||||
* (0 1 )
|
||||
* U^{dag} = ( 1 0)
|
||||
* (Meo^dag Mee^{-dag} 1)
|
||||
* U^{-dag} = ( 1 0)
|
||||
* (-Meo^dag Mee^{-dag} 1)
|
||||
***********************
|
||||
* M psi = eta
|
||||
***********************
|
||||
*Odd
|
||||
* i) D_oo psi_o = L^{-1} eta_o
|
||||
* eta_o' = (D_oo)^dag (eta_o - Moe Mee^{-1} eta_e)
|
||||
*
|
||||
* Wilson:
|
||||
* (D_oo)^{\dag} D_oo psi_o = (D_oo)^dag L^{-1} eta_o
|
||||
* Stag:
|
||||
* D_oo psi_o = L^{-1} eta = (eta_o - Moe Mee^{-1} eta_e)
|
||||
*
|
||||
* L^-1 eta_o= (1 0 ) (e
|
||||
* (-MoeMee^{-1} 1 )
|
||||
*
|
||||
*Even
|
||||
* ii) Mee psi_e + Meo psi_o = src_e
|
||||
*
|
||||
* => sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
*
|
||||
*
|
||||
* TODO: Other options:
|
||||
*
|
||||
* a) change checkerboards for Schur e<->o
|
||||
*
|
||||
* Left precon by Moo^-1
|
||||
* b) Doo^{dag} M_oo^-dag Moo^-1 Doo psi_0 = (D_oo)^dag M_oo^-dag Moo^-1 L^{-1} eta_o
|
||||
* eta_o' = (D_oo)^dag M_oo^-dag Moo^-1 (eta_o - Moe Mee^{-1} eta_e)
|
||||
*
|
||||
* Right precon by Moo^-1
|
||||
* c) M_oo^-dag Doo^{dag} Doo Moo^-1 phi_0 = M_oo^-dag (D_oo)^dag L^{-1} eta_o
|
||||
* eta_o' = M_oo^-dag (D_oo)^dag (eta_o - Moe Mee^{-1} eta_e)
|
||||
* psi_o = M_oo^-1 phi_o
|
||||
* TODO: Deflation
|
||||
*/
|
||||
namespace Grid {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Use base class to share code
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Take a matrix and form a Red Black solver calling a Herm solver
|
||||
// Use of RB info prevents making SchurRedBlackSolve conform to standard interface
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class SchurRedBlackBase {
|
||||
protected:
|
||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
||||
OperatorFunction<Field> & _HermitianRBSolver;
|
||||
int CBfactorise;
|
||||
bool subGuess;
|
||||
bool useSolnAsInitGuess; // if true user-supplied solution vector is used as initial guess for solver
|
||||
public:
|
||||
|
||||
SchurRedBlackBase(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
||||
const bool _solnAsInitGuess = false) :
|
||||
_HermitianRBSolver(HermitianRBSolver),
|
||||
useSolnAsInitGuess(_solnAsInitGuess)
|
||||
{
|
||||
CBfactorise = 0;
|
||||
subtractGuess(initSubGuess);
|
||||
};
|
||||
void subtractGuess(const bool initSubGuess)
|
||||
{
|
||||
subGuess = initSubGuess;
|
||||
}
|
||||
bool isSubtractGuess(void)
|
||||
{
|
||||
return subGuess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Shared code
|
||||
/////////////////////////////////////////////////////////////
|
||||
void operator() (Matrix & _Matrix,const Field &in, Field &out){
|
||||
ZeroGuesser<Field> guess;
|
||||
(*this)(_Matrix,in,out,guess);
|
||||
}
|
||||
void operator()(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &out)
|
||||
{
|
||||
ZeroGuesser<Field> guess;
|
||||
(*this)(_Matrix,in,out,guess);
|
||||
}
|
||||
|
||||
void RedBlackSource(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &src_o)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
Field tmp(grid);
|
||||
int nblock = in.size();
|
||||
for(int b=0;b<nblock;b++){
|
||||
RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
||||
}
|
||||
}
|
||||
// James can write his own deflated guesser
|
||||
// with optimised code for the inner products
|
||||
// RedBlackSolveSplitGrid();
|
||||
// RedBlackSolve(_Matrix,src_o,sol_o);
|
||||
|
||||
void RedBlackSolution(Matrix &_Matrix, const std::vector<Field> &in, const std::vector<Field> &sol_o, std::vector<Field> &out)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
Field tmp(grid);
|
||||
int nblock = in.size();
|
||||
for(int b=0;b<nblock;b++) {
|
||||
pickCheckerboard(Even,tmp,in[b]);
|
||||
RedBlackSolution(_Matrix,sol_o[b],tmp,out[b]);
|
||||
}
|
||||
}
|
||||
|
||||
template<class Guesser>
|
||||
void operator()(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &out,Guesser &guess)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
int nblock = in.size();
|
||||
|
||||
std::vector<Field> src_o(nblock,grid);
|
||||
std::vector<Field> sol_o(nblock,grid);
|
||||
|
||||
std::vector<Field> guess_save;
|
||||
|
||||
Field resid(fgrid);
|
||||
Field tmp(grid);
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// Prepare RedBlack source
|
||||
////////////////////////////////////////////////
|
||||
RedBlackSource(_Matrix,in,src_o);
|
||||
// for(int b=0;b<nblock;b++){
|
||||
// RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
||||
// }
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// Make the guesses
|
||||
////////////////////////////////////////////////
|
||||
if ( subGuess ) guess_save.resize(nblock,grid);
|
||||
|
||||
|
||||
if(useSolnAsInitGuess) {
|
||||
for(int b=0;b<nblock;b++){
|
||||
pickCheckerboard(Odd, sol_o[b], out[b]);
|
||||
}
|
||||
} else {
|
||||
guess(src_o, sol_o);
|
||||
}
|
||||
|
||||
if ( subGuess ) {
|
||||
for(int b=0;b<nblock;b++){
|
||||
guess_save[b] = sol_o[b];
|
||||
}
|
||||
}
|
||||
//////////////////////////////////////////////////////////////
|
||||
// Call the block solver
|
||||
//////////////////////////////////////////////////////////////
|
||||
std::cout<<GridLogMessage << "SchurRedBlackBase calling the solver for "<<nblock<<" RHS" <<std::endl;
|
||||
RedBlackSolve(_Matrix,src_o,sol_o);
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// A2A boolean behavioural control & reconstruct other checkerboard
|
||||
////////////////////////////////////////////////
|
||||
for(int b=0;b<nblock;b++) {
|
||||
|
||||
if (subGuess) sol_o[b] = sol_o[b] - guess_save[b];
|
||||
|
||||
///////// Needs even source //////////////
|
||||
pickCheckerboard(Even,tmp,in[b]);
|
||||
RedBlackSolution(_Matrix,sol_o[b],tmp,out[b]);
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Check unprec residual if possible
|
||||
/////////////////////////////////////////////////
|
||||
if ( ! subGuess ) {
|
||||
_Matrix.M(out[b],resid);
|
||||
resid = resid-in[b];
|
||||
RealD ns = norm2(in[b]);
|
||||
RealD nr = norm2(resid);
|
||||
|
||||
std::cout<<GridLogMessage<< "SchurRedBlackBase solver true unprec resid["<<b<<"] "<<std::sqrt(nr/ns) << std::endl;
|
||||
} else {
|
||||
std::cout<<GridLogMessage<< "SchurRedBlackBase Guess subtracted after solve["<<b<<"] " << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
template<class Guesser>
|
||||
void operator() (Matrix & _Matrix,const Field &in, Field &out,Guesser &guess){
|
||||
|
||||
// FIXME CGdiagonalMee not implemented virtual function
|
||||
// FIXME use CBfactorise to control schur decomp
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
Field resid(fgrid);
|
||||
Field src_o(grid);
|
||||
Field src_e(grid);
|
||||
Field sol_o(grid);
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// RedBlack source
|
||||
////////////////////////////////////////////////
|
||||
RedBlackSource(_Matrix,in,src_e,src_o);
|
||||
|
||||
////////////////////////////////
|
||||
// Construct the guess
|
||||
////////////////////////////////
|
||||
if(useSolnAsInitGuess) {
|
||||
pickCheckerboard(Odd, sol_o, out);
|
||||
} else {
|
||||
guess(src_o,sol_o);
|
||||
}
|
||||
|
||||
Field guess_save(grid);
|
||||
guess_save = sol_o;
|
||||
|
||||
//////////////////////////////////////////////////////////////
|
||||
// Call the red-black solver
|
||||
//////////////////////////////////////////////////////////////
|
||||
RedBlackSolve(_Matrix,src_o,sol_o);
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// Fionn A2A boolean behavioural control
|
||||
////////////////////////////////////////////////
|
||||
if (subGuess) sol_o= sol_o-guess_save;
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// RedBlack solution needs the even source
|
||||
///////////////////////////////////////////////////
|
||||
RedBlackSolution(_Matrix,sol_o,src_e,out);
|
||||
|
||||
// Verify the unprec residual
|
||||
if ( ! subGuess ) {
|
||||
_Matrix.M(out,resid);
|
||||
resid = resid-in;
|
||||
RealD ns = norm2(in);
|
||||
RealD nr = norm2(resid);
|
||||
|
||||
std::cout<<GridLogMessage << "SchurRedBlackBase solver true unprec resid "<< std::sqrt(nr/ns) << std::endl;
|
||||
} else {
|
||||
std::cout << GridLogMessage << "SchurRedBlackBase Guess subtracted after solve." << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Override in derived.
|
||||
/////////////////////////////////////////////////////////////
|
||||
virtual void RedBlackSource (Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o) =0;
|
||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol) =0;
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o) =0;
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)=0;
|
||||
|
||||
};
|
||||
|
||||
template<class Field> class SchurRedBlackStaggeredSolve : public SchurRedBlackBase<Field> {
|
||||
public:
|
||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
||||
|
||||
SchurRedBlackStaggeredSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
||||
const bool _solnAsInitGuess = false)
|
||||
: SchurRedBlackBase<Field> (HermitianRBSolver,initSubGuess,_solnAsInitGuess)
|
||||
{
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// Override RedBlack specialisation
|
||||
//////////////////////////////////////////////////////
|
||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
pickCheckerboard(Even,src_e,src);
|
||||
pickCheckerboard(Odd ,src_o,src);
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); assert( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; assert( tmp.Checkerboard() ==Odd);
|
||||
|
||||
_Matrix.Mooee(tmp,src_o); // Extra factor of "m" in source from dumb choice of matrix norm.
|
||||
}
|
||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e_c,Field &sol)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field sol_e(grid);
|
||||
Field src_e(grid);
|
||||
|
||||
src_e = src_e_c; // Const correctness
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
src_e = src_e-tmp; assert( src_e.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(src_e,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
|
||||
setCheckerboard(sol,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o); assert( sol_o.Checkerboard() ==Odd );
|
||||
}
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
||||
{
|
||||
SchurStaggeredOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); assert(sol_o.Checkerboard()==Odd);
|
||||
};
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
||||
{
|
||||
SchurStaggeredOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
||||
}
|
||||
};
|
||||
template<class Field> using SchurRedBlackStagSolve = SchurRedBlackStaggeredSolve<Field>;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Site diagonal has Mooee on it.
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class SchurRedBlackDiagMooeeSolve : public SchurRedBlackBase<Field> {
|
||||
public:
|
||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
||||
|
||||
SchurRedBlackDiagMooeeSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
||||
const bool _solnAsInitGuess = false)
|
||||
: SchurRedBlackBase<Field> (HermitianRBSolver,initSubGuess,_solnAsInitGuess) {};
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// Override RedBlack specialisation
|
||||
//////////////////////////////////////////////////////
|
||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
pickCheckerboard(Even,src_e,src);
|
||||
pickCheckerboard(Odd ,src_o,src);
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); assert( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; assert( tmp.Checkerboard() ==Odd);
|
||||
|
||||
// get the right MpcDag
|
||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
_HermOpEO.MpcDag(tmp,src_o); assert(src_o.Checkerboard() ==Odd);
|
||||
|
||||
}
|
||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field sol_e(grid);
|
||||
Field src_e_i(grid);
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
src_e_i = src_e-tmp; assert( src_e_i.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(src_e_i,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
|
||||
setCheckerboard(sol,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o); assert( sol_o.Checkerboard() ==Odd );
|
||||
}
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
||||
{
|
||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); assert(sol_o.Checkerboard()==Odd);
|
||||
};
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
||||
{
|
||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field> class NonHermitianSchurRedBlackDiagMooeeSolve : public SchurRedBlackBase<Field>
|
||||
{
|
||||
public:
|
||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
||||
|
||||
NonHermitianSchurRedBlackDiagMooeeSolve(OperatorFunction<Field>& RBSolver, const bool initSubGuess = false,
|
||||
const bool _solnAsInitGuess = false)
|
||||
: SchurRedBlackBase<Field>(RBSolver, initSubGuess, _solnAsInitGuess) {};
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// Override RedBlack specialisation
|
||||
//////////////////////////////////////////////////////
|
||||
virtual void RedBlackSource(Matrix& _Matrix, const Field& src, Field& src_e, Field& src_o)
|
||||
{
|
||||
GridBase* grid = _Matrix.RedBlackGrid();
|
||||
GridBase* fgrid = _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
pickCheckerboard(Even, src_e, src);
|
||||
pickCheckerboard(Odd , src_o, src);
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e, tmp); assert( tmp.Checkerboard() == Even );
|
||||
_Matrix.Meooe (tmp, Mtmp); assert( Mtmp.Checkerboard() == Odd );
|
||||
src_o -= Mtmp; assert( src_o.Checkerboard() == Odd );
|
||||
}
|
||||
|
||||
virtual void RedBlackSolution(Matrix& _Matrix, const Field& sol_o, const Field& src_e, Field& sol)
|
||||
{
|
||||
GridBase* grid = _Matrix.RedBlackGrid();
|
||||
GridBase* fgrid = _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field sol_e(grid);
|
||||
Field src_e_i(grid);
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o, tmp); assert( tmp.Checkerboard() == Even );
|
||||
src_e_i = src_e - tmp; assert( src_e_i.Checkerboard() == Even );
|
||||
_Matrix.MooeeInv(src_e_i, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
|
||||
setCheckerboard(sol, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
setCheckerboard(sol, sol_o); assert( sol_o.Checkerboard() == Odd );
|
||||
}
|
||||
|
||||
virtual void RedBlackSolve(Matrix& _Matrix, const Field& src_o, Field& sol_o)
|
||||
{
|
||||
NonHermitianSchurDiagMooeeOperator<Matrix,Field> _OpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o); assert(sol_o.Checkerboard() == Odd);
|
||||
}
|
||||
|
||||
virtual void RedBlackSolve(Matrix& _Matrix, const std::vector<Field>& src_o, std::vector<Field>& sol_o)
|
||||
{
|
||||
NonHermitianSchurDiagMooeeOperator<Matrix,Field> _OpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o);
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Site diagonal is identity, right preconditioned by Mee^inv
|
||||
// ( 1 - Meo Moo^inv Moe Mee^inv ) phi =( 1 - Meo Moo^inv Moe Mee^inv ) Mee psi = = eta = eta
|
||||
//=> psi = MeeInv phi
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class SchurRedBlackDiagTwoSolve : public SchurRedBlackBase<Field> {
|
||||
public:
|
||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Wrap the usual normal equations Schur trick
|
||||
/////////////////////////////////////////////////////
|
||||
SchurRedBlackDiagTwoSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
||||
const bool _solnAsInitGuess = false)
|
||||
: SchurRedBlackBase<Field>(HermitianRBSolver,initSubGuess,_solnAsInitGuess) {};
|
||||
|
||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
|
||||
Field tmp(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
pickCheckerboard(Even,src_e,src);
|
||||
pickCheckerboard(Odd ,src_o,src);
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); assert( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; assert( tmp.Checkerboard() ==Odd);
|
||||
|
||||
// get the right MpcDag
|
||||
_HermOpEO.MpcDag(tmp,src_o); assert(src_o.Checkerboard() ==Odd);
|
||||
}
|
||||
|
||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
Field sol_o_i(grid);
|
||||
Field tmp(grid);
|
||||
Field sol_e(grid);
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// MooeeInv due to pecond
|
||||
////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(sol_o,tmp);
|
||||
sol_o_i = tmp;
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o_i,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
tmp = src_e-tmp; assert( src_e.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(tmp,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
|
||||
setCheckerboard(sol,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o_i); assert( sol_o_i.Checkerboard() ==Odd );
|
||||
};
|
||||
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
||||
{
|
||||
SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
||||
};
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
||||
{
|
||||
SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field> class NonHermitianSchurRedBlackDiagTwoSolve : public SchurRedBlackBase<Field>
|
||||
{
|
||||
public:
|
||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Wrap the usual normal equations Schur trick
|
||||
/////////////////////////////////////////////////////
|
||||
NonHermitianSchurRedBlackDiagTwoSolve(OperatorFunction<Field>& RBSolver, const bool initSubGuess = false,
|
||||
const bool _solnAsInitGuess = false)
|
||||
: SchurRedBlackBase<Field>(RBSolver, initSubGuess, _solnAsInitGuess) {};
|
||||
|
||||
virtual void RedBlackSource(Matrix& _Matrix, const Field& src, Field& src_e, Field& src_o)
|
||||
{
|
||||
GridBase* grid = _Matrix.RedBlackGrid();
|
||||
GridBase* fgrid = _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
pickCheckerboard(Even, src_e, src);
|
||||
pickCheckerboard(Odd , src_o, src);
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e, tmp); assert( tmp.Checkerboard() == Even );
|
||||
_Matrix.Meooe (tmp, Mtmp); assert( Mtmp.Checkerboard() == Odd );
|
||||
src_o -= Mtmp; assert( src_o.Checkerboard() == Odd );
|
||||
}
|
||||
|
||||
virtual void RedBlackSolution(Matrix& _Matrix, const Field& sol_o, const Field& src_e, Field& sol)
|
||||
{
|
||||
GridBase* grid = _Matrix.RedBlackGrid();
|
||||
GridBase* fgrid = _Matrix.Grid();
|
||||
|
||||
Field sol_o_i(grid);
|
||||
Field tmp(grid);
|
||||
Field sol_e(grid);
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// MooeeInv due to pecond
|
||||
////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(sol_o, tmp);
|
||||
sol_o_i = tmp;
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o_i, tmp); assert( tmp.Checkerboard() == Even );
|
||||
tmp = src_e - tmp; assert( src_e.Checkerboard() == Even );
|
||||
_Matrix.MooeeInv(tmp, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
|
||||
setCheckerboard(sol, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
setCheckerboard(sol, sol_o_i); assert( sol_o_i.Checkerboard() == Odd );
|
||||
};
|
||||
|
||||
virtual void RedBlackSolve(Matrix& _Matrix, const Field& src_o, Field& sol_o)
|
||||
{
|
||||
NonHermitianSchurDiagTwoOperator<Matrix,Field> _OpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o);
|
||||
};
|
||||
|
||||
virtual void RedBlackSolve(Matrix& _Matrix, const std::vector<Field>& src_o, std::vector<Field>& sol_o)
|
||||
{
|
||||
NonHermitianSchurDiagTwoOperator<Matrix,Field> _OpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,183 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/AlignedAllocator.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<typename _Tp>
|
||||
class alignedAllocator {
|
||||
public:
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef _Tp* pointer;
|
||||
typedef const _Tp* const_pointer;
|
||||
typedef _Tp& reference;
|
||||
typedef const _Tp& const_reference;
|
||||
typedef _Tp value_type;
|
||||
|
||||
template<typename _Tp1> struct rebind { typedef alignedAllocator<_Tp1> other; };
|
||||
alignedAllocator() throw() { }
|
||||
alignedAllocator(const alignedAllocator&) throw() { }
|
||||
template<typename _Tp1> alignedAllocator(const alignedAllocator<_Tp1>&) throw() { }
|
||||
~alignedAllocator() throw() { }
|
||||
pointer address(reference __x) const { return &__x; }
|
||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
||||
|
||||
pointer allocate(size_type __n, const void* _p= 0)
|
||||
{
|
||||
size_type bytes = __n*sizeof(_Tp);
|
||||
profilerAllocate(bytes);
|
||||
_Tp *ptr = (_Tp*) MemoryManager::CpuAllocate(bytes);
|
||||
assert( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deallocate(pointer __p, size_type __n)
|
||||
{
|
||||
size_type bytes = __n * sizeof(_Tp);
|
||||
profilerFree(bytes);
|
||||
MemoryManager::CpuFree((void *)__p,bytes);
|
||||
}
|
||||
|
||||
// FIXME: hack for the copy constructor: it must be avoided to avoid single thread loop
|
||||
void construct(pointer __p, const _Tp& __val) { assert(0);};
|
||||
void construct(pointer __p) { };
|
||||
void destroy(pointer __p) { };
|
||||
};
|
||||
template<typename _Tp> inline bool operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
|
||||
template<typename _Tp> inline bool operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Unified virtual memory
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
template<typename _Tp>
|
||||
class uvmAllocator {
|
||||
public:
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef _Tp* pointer;
|
||||
typedef const _Tp* const_pointer;
|
||||
typedef _Tp& reference;
|
||||
typedef const _Tp& const_reference;
|
||||
typedef _Tp value_type;
|
||||
|
||||
template<typename _Tp1> struct rebind { typedef uvmAllocator<_Tp1> other; };
|
||||
uvmAllocator() throw() { }
|
||||
uvmAllocator(const uvmAllocator&) throw() { }
|
||||
template<typename _Tp1> uvmAllocator(const uvmAllocator<_Tp1>&) throw() { }
|
||||
~uvmAllocator() throw() { }
|
||||
pointer address(reference __x) const { return &__x; }
|
||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
||||
|
||||
pointer allocate(size_type __n, const void* _p= 0)
|
||||
{
|
||||
size_type bytes = __n*sizeof(_Tp);
|
||||
profilerAllocate(bytes);
|
||||
_Tp *ptr = (_Tp*) MemoryManager::SharedAllocate(bytes);
|
||||
assert( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deallocate(pointer __p, size_type __n)
|
||||
{
|
||||
size_type bytes = __n * sizeof(_Tp);
|
||||
profilerFree(bytes);
|
||||
MemoryManager::SharedFree((void *)__p,bytes);
|
||||
}
|
||||
|
||||
void construct(pointer __p, const _Tp& __val) { new((void *)__p) _Tp(__val); };
|
||||
void construct(pointer __p) { };
|
||||
void destroy(pointer __p) { };
|
||||
};
|
||||
template<typename _Tp> inline bool operator==(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return true; }
|
||||
template<typename _Tp> inline bool operator!=(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return false; }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Device memory
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
template<typename _Tp>
|
||||
class devAllocator {
|
||||
public:
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef _Tp* pointer;
|
||||
typedef const _Tp* const_pointer;
|
||||
typedef _Tp& reference;
|
||||
typedef const _Tp& const_reference;
|
||||
typedef _Tp value_type;
|
||||
|
||||
template<typename _Tp1> struct rebind { typedef devAllocator<_Tp1> other; };
|
||||
devAllocator() throw() { }
|
||||
devAllocator(const devAllocator&) throw() { }
|
||||
template<typename _Tp1> devAllocator(const devAllocator<_Tp1>&) throw() { }
|
||||
~devAllocator() throw() { }
|
||||
pointer address(reference __x) const { return &__x; }
|
||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
||||
|
||||
pointer allocate(size_type __n, const void* _p= 0)
|
||||
{
|
||||
size_type bytes = __n*sizeof(_Tp);
|
||||
profilerAllocate(bytes);
|
||||
_Tp *ptr = (_Tp*) MemoryManager::AcceleratorAllocate(bytes);
|
||||
assert( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deallocate(pointer __p, size_type __n)
|
||||
{
|
||||
size_type bytes = __n * sizeof(_Tp);
|
||||
profilerFree(bytes);
|
||||
MemoryManager::AcceleratorFree((void *)__p,bytes);
|
||||
}
|
||||
void construct(pointer __p, const _Tp& __val) { };
|
||||
void construct(pointer __p) { };
|
||||
void destroy(pointer __p) { };
|
||||
};
|
||||
template<typename _Tp> inline bool operator==(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return true; }
|
||||
template<typename _Tp> inline bool operator!=(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return false; }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Template typedefs
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
// Cshift on device
|
||||
template<class T> using cshiftAllocator = devAllocator<T>;
|
||||
#else
|
||||
// Cshift on host
|
||||
template<class T> using cshiftAllocator = std::allocator<T>;
|
||||
#endif
|
||||
|
||||
template<class T> using Vector = std::vector<T,uvmAllocator<T> >;
|
||||
template<class T> using stencilVector = std::vector<T,alignedAllocator<T> >;
|
||||
template<class T> using commVector = std::vector<T,devAllocator<T> >;
|
||||
template<class T> using cshiftVector = std::vector<T,cshiftAllocator<T> >;
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
#pragma once
|
||||
#include <Grid/allocator/MemoryStats.h>
|
||||
#include <Grid/allocator/MemoryManager.h>
|
||||
#include <Grid/allocator/AlignedAllocator.h>
|
||||
@@ -1,324 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/*Allocation types, saying which pointer cache should be used*/
|
||||
#define Cpu (0)
|
||||
#define CpuHuge (1)
|
||||
#define CpuSmall (2)
|
||||
#define Acc (3)
|
||||
#define AccHuge (4)
|
||||
#define AccSmall (5)
|
||||
#define Shared (6)
|
||||
#define SharedHuge (7)
|
||||
#define SharedSmall (8)
|
||||
#undef GRID_MM_VERBOSE
|
||||
uint64_t total_shared;
|
||||
uint64_t total_device;
|
||||
uint64_t total_host;;
|
||||
void MemoryManager::PrintBytes(void)
|
||||
{
|
||||
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
||||
std::cout << " MemoryManager : PrintBytes "<<std::endl;
|
||||
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
||||
std::cout << " MemoryManager : "<<(total_shared>>20)<<" shared Mbytes "<<std::endl;
|
||||
std::cout << " MemoryManager : "<<(total_device>>20)<<" accelerator Mbytes "<<std::endl;
|
||||
std::cout << " MemoryManager : "<<(total_host>>20) <<" cpu Mbytes "<<std::endl;
|
||||
uint64_t cacheBytes;
|
||||
cacheBytes = CacheBytes[Cpu];
|
||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" cpu cache Mbytes "<<std::endl;
|
||||
cacheBytes = CacheBytes[Acc];
|
||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" acc cache Mbytes "<<std::endl;
|
||||
cacheBytes = CacheBytes[Shared];
|
||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" shared cache Mbytes "<<std::endl;
|
||||
|
||||
#ifdef GRID_CUDA
|
||||
cuda_mem();
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
uint64_t MemoryManager::DeviceCacheBytes() { return CacheBytes[Acc] + CacheBytes[AccHuge] + CacheBytes[AccSmall]; }
|
||||
uint64_t MemoryManager::HostCacheBytes() { return CacheBytes[Cpu] + CacheBytes[CpuHuge] + CacheBytes[CpuSmall]; }
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Data tables for recently freed pooiniter caches
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
MemoryManager::AllocationCacheEntry MemoryManager::Entries[MemoryManager::NallocType][MemoryManager::NallocCacheMax];
|
||||
int MemoryManager::Victim[MemoryManager::NallocType];
|
||||
int MemoryManager::Ncache[MemoryManager::NallocType] = { 2, 0, 8, 8, 0, 16, 8, 0, 16 };
|
||||
uint64_t MemoryManager::CacheBytes[MemoryManager::NallocType];
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Actual allocation and deallocation utils
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
void *MemoryManager::AcceleratorAllocate(size_t bytes)
|
||||
{
|
||||
total_device+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Acc);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocDevice(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"AcceleratorAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::AcceleratorFree (void *ptr,size_t bytes)
|
||||
{
|
||||
total_device-=bytes;
|
||||
void *__freeme = Insert(ptr,bytes,Acc);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeDevice(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"AcceleratorFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
void *MemoryManager::SharedAllocate(size_t bytes)
|
||||
{
|
||||
total_shared+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Shared);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocShared(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"SharedAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::SharedFree (void *ptr,size_t bytes)
|
||||
{
|
||||
total_shared-=bytes;
|
||||
void *__freeme = Insert(ptr,bytes,Shared);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeShared(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"SharedFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
#ifdef GRID_UVM
|
||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
||||
{
|
||||
total_host+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocShared(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
||||
{
|
||||
total_host-=bytes;
|
||||
NotifyDeletion(_ptr);
|
||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeShared(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
#else
|
||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
||||
{
|
||||
total_host+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocCpu(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
||||
{
|
||||
total_host-=bytes;
|
||||
NotifyDeletion(_ptr);
|
||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeCpu(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
//////////////////////////////////////////
|
||||
// call only once
|
||||
//////////////////////////////////////////
|
||||
void MemoryManager::Init(void)
|
||||
{
|
||||
|
||||
char * str;
|
||||
int Nc;
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_LARGE");
|
||||
if ( str ) {
|
||||
Nc = atoi(str);
|
||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
||||
Ncache[Cpu]=Nc;
|
||||
Ncache[Acc]=Nc;
|
||||
Ncache[Shared]=Nc;
|
||||
}
|
||||
}
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_HUGE");
|
||||
if ( str ) {
|
||||
Nc = atoi(str);
|
||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
||||
Ncache[CpuHuge]=Nc;
|
||||
Ncache[AccHuge]=Nc;
|
||||
Ncache[SharedHuge]=Nc;
|
||||
}
|
||||
}
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_SMALL");
|
||||
if ( str ) {
|
||||
Nc = atoi(str);
|
||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
||||
Ncache[CpuSmall]=Nc;
|
||||
Ncache[AccSmall]=Nc;
|
||||
Ncache[SharedSmall]=Nc;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void MemoryManager::InitMessage(void) {
|
||||
|
||||
#ifndef GRID_UVM
|
||||
std::cout << GridLogMessage << "MemoryManager Cache "<< MemoryManager::DeviceMaxBytes <<" bytes "<<std::endl;
|
||||
#endif
|
||||
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() setting up"<<std::endl;
|
||||
#ifdef ALLOCATION_CACHE
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent host allocations: SMALL "<<Ncache[CpuSmall]<<" LARGE "<<Ncache[Cpu]<<" HUGE "<<Ncache[CpuHuge]<<std::endl;
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent device allocations: SMALL "<<Ncache[AccSmall]<<" LARGE "<<Ncache[Acc]<<" Huge "<<Ncache[AccHuge]<<std::endl;
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent shared allocations: SMALL "<<Ncache[SharedSmall]<<" LARGE "<<Ncache[Shared]<<" Huge "<<Ncache[SharedHuge]<<std::endl;
|
||||
#endif
|
||||
|
||||
#ifdef GRID_UVM
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Unified memory space"<<std::endl;
|
||||
#ifdef GRID_CUDA
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMallocManaged"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMallocManaged"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_SYCL
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_shared"<<std::endl;
|
||||
#endif
|
||||
#else
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory"<<std::endl;
|
||||
#ifdef GRID_CUDA
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMalloc"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMalloc"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_SYCL
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_device"<<std::endl;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
void *MemoryManager::Insert(void *ptr,size_t bytes,int type)
|
||||
{
|
||||
#ifdef ALLOCATION_CACHE
|
||||
int cache;
|
||||
if (bytes < GRID_ALLOC_SMALL_LIMIT) cache = type + 2;
|
||||
else if (bytes >= GRID_ALLOC_HUGE_LIMIT) cache = type + 1;
|
||||
else cache = type;
|
||||
|
||||
return Insert(ptr,bytes,Entries[cache],Ncache[cache],Victim[cache],CacheBytes[cache]);
|
||||
#else
|
||||
return ptr;
|
||||
#endif
|
||||
}
|
||||
|
||||
void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim, uint64_t &cacheBytes)
|
||||
{
|
||||
#ifdef GRID_OMP
|
||||
assert(omp_in_parallel()==0);
|
||||
#endif
|
||||
|
||||
if (ncache == 0) return ptr;
|
||||
|
||||
void * ret = NULL;
|
||||
int v = -1;
|
||||
|
||||
for(int e=0;e<ncache;e++) {
|
||||
if ( entries[e].valid==0 ) {
|
||||
v=e;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ( v==-1 ) {
|
||||
v=victim;
|
||||
victim = (victim+1)%ncache;
|
||||
}
|
||||
|
||||
if ( entries[v].valid ) {
|
||||
ret = entries[v].address;
|
||||
cacheBytes -= entries[v].bytes;
|
||||
entries[v].valid = 0;
|
||||
entries[v].address = NULL;
|
||||
entries[v].bytes = 0;
|
||||
}
|
||||
|
||||
entries[v].address=ptr;
|
||||
entries[v].bytes =bytes;
|
||||
entries[v].valid =1;
|
||||
cacheBytes += bytes;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void *MemoryManager::Lookup(size_t bytes,int type)
|
||||
{
|
||||
#ifdef ALLOCATION_CACHE
|
||||
int cache;
|
||||
if (bytes < GRID_ALLOC_SMALL_LIMIT) cache = type + 2;
|
||||
else if (bytes >= GRID_ALLOC_HUGE_LIMIT) cache = type + 1;
|
||||
else cache = type;
|
||||
|
||||
return Lookup(bytes,Entries[cache],Ncache[cache],CacheBytes[cache]);
|
||||
#else
|
||||
return NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
void *MemoryManager::Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t & cacheBytes)
|
||||
{
|
||||
#ifdef GRID_OMP
|
||||
assert(omp_in_parallel()==0);
|
||||
#endif
|
||||
for(int e=0;e<ncache;e++){
|
||||
if ( entries[e].valid && ( entries[e].bytes == bytes ) ) {
|
||||
entries[e].valid = 0;
|
||||
cacheBytes -= entries[e].bytes;
|
||||
return entries[e].address;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/MemoryManager.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
#include <list>
|
||||
#include <unordered_map>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
// Move control to configure.ac and Config.h?
|
||||
|
||||
#define GRID_ALLOC_SMALL_LIMIT (4096)
|
||||
#define GRID_ALLOC_HUGE_LIMIT (2147483648)
|
||||
|
||||
#define STRINGIFY(x) #x
|
||||
#define TOSTRING(x) STRINGIFY(x)
|
||||
#define FILE_LINE __FILE__ ":" TOSTRING(__LINE__)
|
||||
#define AUDIT(a) MemoryManager::Audit(FILE_LINE)
|
||||
|
||||
/*Pinning pages is costly*/
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Advise the LatticeAccelerator class
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
enum ViewAdvise {
|
||||
AdviseDefault = 0x0, // Regular data
|
||||
AdviseInfrequentUse = 0x1 // Advise that the data is used infrequently. This can
|
||||
// significantly influence performance of bulk storage.
|
||||
|
||||
// AdviseTransient = 0x2, // Data will mostly be read. On some architectures
|
||||
// enables read-only copies of memory to be kept on
|
||||
// host and device.
|
||||
|
||||
// AdviseAcceleratorWriteDiscard = 0x4 // Field will be written in entirety on device
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// View Access Mode
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
enum ViewMode {
|
||||
AcceleratorRead = 0x01,
|
||||
AcceleratorWrite = 0x02,
|
||||
AcceleratorWriteDiscard = 0x04,
|
||||
CpuRead = 0x08,
|
||||
CpuWrite = 0x10,
|
||||
CpuWriteDiscard = 0x10 // same for now
|
||||
};
|
||||
|
||||
struct MemoryStatus {
|
||||
uint64_t DeviceBytes;
|
||||
uint64_t DeviceLRUBytes;
|
||||
uint64_t DeviceMaxBytes;
|
||||
uint64_t HostToDeviceBytes;
|
||||
uint64_t DeviceToHostBytes;
|
||||
uint64_t HostToDeviceXfer;
|
||||
uint64_t DeviceToHostXfer;
|
||||
uint64_t DeviceEvictions;
|
||||
uint64_t DeviceDestroy;
|
||||
uint64_t DeviceAllocCacheBytes;
|
||||
uint64_t HostAllocCacheBytes;
|
||||
};
|
||||
|
||||
|
||||
class MemoryManager {
|
||||
private:
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// For caching recently freed allocations
|
||||
////////////////////////////////////////////////////////////
|
||||
typedef struct {
|
||||
void *address;
|
||||
size_t bytes;
|
||||
int valid;
|
||||
} AllocationCacheEntry;
|
||||
|
||||
static const int NallocCacheMax=128;
|
||||
static const int NallocType=9;
|
||||
static AllocationCacheEntry Entries[NallocType][NallocCacheMax];
|
||||
static int Victim[NallocType];
|
||||
static int Ncache[NallocType];
|
||||
static uint64_t CacheBytes[NallocType];
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Free pool
|
||||
/////////////////////////////////////////////////
|
||||
static void *Insert(void *ptr,size_t bytes,int type) ;
|
||||
static void *Lookup(size_t bytes,int type) ;
|
||||
static void *Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim,uint64_t &cbytes) ;
|
||||
static void *Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t &cbytes) ;
|
||||
|
||||
public:
|
||||
static void PrintBytes(void);
|
||||
static void Audit(std::string s);
|
||||
static void Init(void);
|
||||
static void InitMessage(void);
|
||||
static void *AcceleratorAllocate(size_t bytes);
|
||||
static void AcceleratorFree (void *ptr,size_t bytes);
|
||||
static void *SharedAllocate(size_t bytes);
|
||||
static void SharedFree (void *ptr,size_t bytes);
|
||||
static void *CpuAllocate(size_t bytes);
|
||||
static void CpuFree (void *ptr,size_t bytes);
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// Footprint tracking
|
||||
////////////////////////////////////////////////////////
|
||||
static uint64_t DeviceBytes;
|
||||
static uint64_t DeviceLRUBytes;
|
||||
static uint64_t DeviceMaxBytes;
|
||||
static uint64_t HostToDeviceBytes;
|
||||
static uint64_t DeviceToHostBytes;
|
||||
static uint64_t HostToDeviceXfer;
|
||||
static uint64_t DeviceToHostXfer;
|
||||
static uint64_t DeviceEvictions;
|
||||
static uint64_t DeviceDestroy;
|
||||
|
||||
static uint64_t DeviceCacheBytes();
|
||||
static uint64_t HostCacheBytes();
|
||||
|
||||
static MemoryStatus GetFootprint(void) {
|
||||
MemoryStatus stat;
|
||||
stat.DeviceBytes = DeviceBytes;
|
||||
stat.DeviceLRUBytes = DeviceLRUBytes;
|
||||
stat.DeviceMaxBytes = DeviceMaxBytes;
|
||||
stat.HostToDeviceBytes = HostToDeviceBytes;
|
||||
stat.DeviceToHostBytes = DeviceToHostBytes;
|
||||
stat.HostToDeviceXfer = HostToDeviceXfer;
|
||||
stat.DeviceToHostXfer = DeviceToHostXfer;
|
||||
stat.DeviceEvictions = DeviceEvictions;
|
||||
stat.DeviceDestroy = DeviceDestroy;
|
||||
stat.DeviceAllocCacheBytes = DeviceCacheBytes();
|
||||
stat.HostAllocCacheBytes = HostCacheBytes();
|
||||
return stat;
|
||||
};
|
||||
|
||||
private:
|
||||
#ifndef GRID_UVM
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Data tables for ViewCache
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
typedef std::list<uint64_t> LRU_t;
|
||||
typedef typename LRU_t::iterator LRUiterator;
|
||||
typedef struct {
|
||||
int LRU_valid;
|
||||
LRUiterator LRU_entry;
|
||||
uint64_t CpuPtr;
|
||||
uint64_t AccPtr;
|
||||
size_t bytes;
|
||||
uint32_t transient;
|
||||
uint32_t state;
|
||||
uint32_t accLock;
|
||||
uint32_t cpuLock;
|
||||
} AcceleratorViewEntry;
|
||||
|
||||
typedef std::unordered_map<uint64_t,AcceleratorViewEntry> AccViewTable_t;
|
||||
typedef typename AccViewTable_t::iterator AccViewTableIterator ;
|
||||
|
||||
static AccViewTable_t AccViewTable;
|
||||
static LRU_t LRU;
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Device motion
|
||||
/////////////////////////////////////////////////
|
||||
static void Create(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
static void EvictVictims(uint64_t bytes); // Frees up <bytes>
|
||||
static void Evict(AcceleratorViewEntry &AccCache);
|
||||
static void Flush(AcceleratorViewEntry &AccCache);
|
||||
static void Clone(AcceleratorViewEntry &AccCache);
|
||||
static void AccDiscard(AcceleratorViewEntry &AccCache);
|
||||
static void CpuDiscard(AcceleratorViewEntry &AccCache);
|
||||
|
||||
// static void LRUupdate(AcceleratorViewEntry &AccCache);
|
||||
static void LRUinsert(AcceleratorViewEntry &AccCache);
|
||||
static void LRUremove(AcceleratorViewEntry &AccCache);
|
||||
|
||||
// manage entries in the table
|
||||
static int EntryPresent(uint64_t CpuPtr);
|
||||
static void EntryCreate(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
static void EntryErase (uint64_t CpuPtr);
|
||||
static AccViewTableIterator EntryLookup(uint64_t CpuPtr);
|
||||
static void EntrySet (uint64_t CpuPtr,AcceleratorViewEntry &entry);
|
||||
|
||||
static void AcceleratorViewClose(uint64_t AccPtr);
|
||||
static uint64_t AcceleratorViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
static void CpuViewClose(uint64_t Ptr);
|
||||
static uint64_t CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
#endif
|
||||
static void NotifyDeletion(void * CpuPtr);
|
||||
|
||||
public:
|
||||
static void Print(void);
|
||||
static void PrintAll(void);
|
||||
static void PrintState( void* CpuPtr);
|
||||
static int isOpen (void* CpuPtr);
|
||||
static void ViewClose(void* CpuPtr,ViewMode mode);
|
||||
static void *ViewOpen (void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@@ -1,601 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#ifndef GRID_UVM
|
||||
|
||||
#warning "Using explicit device memory copies"
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#define MAXLINE 512
|
||||
static char print_buffer [ MAXLINE ];
|
||||
|
||||
#define mprintf(...) snprintf (print_buffer,MAXLINE, __VA_ARGS__ ); std::cout << GridLogMemory << print_buffer;
|
||||
#define dprintf(...) snprintf (print_buffer,MAXLINE, __VA_ARGS__ ); std::cout << GridLogMemory << print_buffer;
|
||||
//#define dprintf(...)
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// For caching copies of data on device
|
||||
////////////////////////////////////////////////////////////
|
||||
MemoryManager::AccViewTable_t MemoryManager::AccViewTable;
|
||||
MemoryManager::LRU_t MemoryManager::LRU;
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// Footprint tracking
|
||||
////////////////////////////////////////////////////////
|
||||
uint64_t MemoryManager::DeviceBytes;
|
||||
uint64_t MemoryManager::DeviceLRUBytes;
|
||||
uint64_t MemoryManager::DeviceMaxBytes = 1024*1024*128;
|
||||
uint64_t MemoryManager::HostToDeviceBytes;
|
||||
uint64_t MemoryManager::DeviceToHostBytes;
|
||||
uint64_t MemoryManager::HostToDeviceXfer;
|
||||
uint64_t MemoryManager::DeviceToHostXfer;
|
||||
uint64_t MemoryManager::DeviceEvictions;
|
||||
uint64_t MemoryManager::DeviceDestroy;
|
||||
|
||||
////////////////////////////////////
|
||||
// Priority ordering for unlocked entries
|
||||
// Empty
|
||||
// CpuDirty
|
||||
// Consistent
|
||||
// AccDirty
|
||||
////////////////////////////////////
|
||||
#define Empty (0x0) /*Entry unoccupied */
|
||||
#define CpuDirty (0x1) /*CPU copy is golden, Acc buffer MAY not be allocated*/
|
||||
#define Consistent (0x2) /*ACC copy AND CPU copy are valid */
|
||||
#define AccDirty (0x4) /*ACC copy is golden */
|
||||
#define EvictNext (0x8) /*Priority for eviction*/
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Mechanics of data table maintenance
|
||||
/////////////////////////////////////////////////
|
||||
int MemoryManager::EntryPresent(uint64_t CpuPtr)
|
||||
{
|
||||
if(AccViewTable.empty()) return 0;
|
||||
|
||||
auto count = AccViewTable.count(CpuPtr); assert((count==0)||(count==1));
|
||||
return count;
|
||||
}
|
||||
void MemoryManager::EntryCreate(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
||||
{
|
||||
assert(!EntryPresent(CpuPtr));
|
||||
AcceleratorViewEntry AccCache;
|
||||
AccCache.CpuPtr = CpuPtr;
|
||||
AccCache.AccPtr = (uint64_t)NULL;
|
||||
AccCache.bytes = bytes;
|
||||
AccCache.state = CpuDirty;
|
||||
AccCache.LRU_valid=0;
|
||||
AccCache.transient=0;
|
||||
AccCache.accLock=0;
|
||||
AccCache.cpuLock=0;
|
||||
AccViewTable[CpuPtr] = AccCache;
|
||||
}
|
||||
MemoryManager::AccViewTableIterator MemoryManager::EntryLookup(uint64_t CpuPtr)
|
||||
{
|
||||
assert(EntryPresent(CpuPtr));
|
||||
auto AccCacheIterator = AccViewTable.find(CpuPtr);
|
||||
assert(AccCacheIterator!=AccViewTable.end());
|
||||
return AccCacheIterator;
|
||||
}
|
||||
void MemoryManager::EntryErase(uint64_t CpuPtr)
|
||||
{
|
||||
auto AccCache = EntryLookup(CpuPtr);
|
||||
AccViewTable.erase(CpuPtr);
|
||||
}
|
||||
void MemoryManager::LRUinsert(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
assert(AccCache.LRU_valid==0);
|
||||
if (AccCache.transient) {
|
||||
LRU.push_back(AccCache.CpuPtr);
|
||||
AccCache.LRU_entry = --LRU.end();
|
||||
} else {
|
||||
LRU.push_front(AccCache.CpuPtr);
|
||||
AccCache.LRU_entry = LRU.begin();
|
||||
}
|
||||
AccCache.LRU_valid = 1;
|
||||
DeviceLRUBytes+=AccCache.bytes;
|
||||
}
|
||||
void MemoryManager::LRUremove(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
assert(AccCache.LRU_valid==1);
|
||||
LRU.erase(AccCache.LRU_entry);
|
||||
AccCache.LRU_valid = 0;
|
||||
DeviceLRUBytes-=AccCache.bytes;
|
||||
}
|
||||
/////////////////////////////////////////////////
|
||||
// Accelerator cache motion & consistency logic
|
||||
/////////////////////////////////////////////////
|
||||
void MemoryManager::AccDiscard(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
///////////////////////////////////////////////////////////
|
||||
// Remove from Accelerator, remove entry, without flush
|
||||
// Cannot be locked. If allocated Must be in LRU pool.
|
||||
///////////////////////////////////////////////////////////
|
||||
assert(AccCache.state!=Empty);
|
||||
|
||||
mprintf("MemoryManager: Discard(%lx) %lx\n",(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr);
|
||||
assert(AccCache.accLock==0);
|
||||
assert(AccCache.cpuLock==0);
|
||||
assert(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
if(AccCache.AccPtr) {
|
||||
AcceleratorFree((void *)AccCache.AccPtr,AccCache.bytes);
|
||||
DeviceDestroy++;
|
||||
DeviceBytes -=AccCache.bytes;
|
||||
LRUremove(AccCache);
|
||||
AccCache.AccPtr=(uint64_t) NULL;
|
||||
dprintf("MemoryManager: Free(%lx) LRU %ld Total %ld\n",(uint64_t)AccCache.AccPtr,DeviceLRUBytes,DeviceBytes);
|
||||
}
|
||||
uint64_t CpuPtr = AccCache.CpuPtr;
|
||||
EntryErase(CpuPtr);
|
||||
}
|
||||
|
||||
void MemoryManager::Evict(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// Make CPU consistent, remove from Accelerator, remove from LRU, LEAVE CPU only entry
|
||||
// Cannot be acclocked. If allocated must be in LRU pool.
|
||||
//
|
||||
// Nov 2022... Felix issue: Allocating two CpuPtrs, can have an entry in LRU-q with CPUlock.
|
||||
// and require to evict the AccPtr copy. Eviction was a mistake in CpuViewOpen
|
||||
// but there is a weakness where CpuLock entries are attempted for erase
|
||||
// Take these OUT LRU queue when CPU locked?
|
||||
// Cannot take out the table as cpuLock data is important.
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
assert(AccCache.state!=Empty);
|
||||
|
||||
mprintf("MemoryManager: Evict cpu %lx acc %lx cpuLock %ld accLock %ld\n",
|
||||
(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr,
|
||||
(uint64_t)AccCache.cpuLock,(uint64_t)AccCache.accLock);
|
||||
if (AccCache.accLock!=0) return;
|
||||
if (AccCache.cpuLock!=0) return;
|
||||
if(AccCache.state==AccDirty) {
|
||||
Flush(AccCache);
|
||||
}
|
||||
if(AccCache.AccPtr) {
|
||||
AcceleratorFree((void *)AccCache.AccPtr,AccCache.bytes);
|
||||
LRUremove(AccCache);
|
||||
AccCache.AccPtr=(uint64_t)NULL;
|
||||
AccCache.state=CpuDirty; // CPU primary now
|
||||
DeviceBytes -=AccCache.bytes;
|
||||
dprintf("MemoryManager: Free(%lx) footprint now %ld \n",(uint64_t)AccCache.AccPtr,DeviceBytes);
|
||||
}
|
||||
// uint64_t CpuPtr = AccCache.CpuPtr;
|
||||
DeviceEvictions++;
|
||||
// EntryErase(CpuPtr);
|
||||
}
|
||||
void MemoryManager::Flush(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
assert(AccCache.state==AccDirty);
|
||||
assert(AccCache.cpuLock==0);
|
||||
assert(AccCache.accLock==0);
|
||||
assert(AccCache.AccPtr!=(uint64_t)NULL);
|
||||
assert(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
acceleratorCopyFromDevice((void *)AccCache.AccPtr,(void *)AccCache.CpuPtr,AccCache.bytes);
|
||||
mprintf("MemoryManager: Flush %lx -> %lx\n",(uint64_t)AccCache.AccPtr,(uint64_t)AccCache.CpuPtr); fflush(stdout);
|
||||
DeviceToHostBytes+=AccCache.bytes;
|
||||
DeviceToHostXfer++;
|
||||
AccCache.state=Consistent;
|
||||
}
|
||||
void MemoryManager::Clone(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
assert(AccCache.state==CpuDirty);
|
||||
assert(AccCache.cpuLock==0);
|
||||
assert(AccCache.accLock==0);
|
||||
assert(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
if(AccCache.AccPtr==(uint64_t)NULL){
|
||||
AccCache.AccPtr=(uint64_t)AcceleratorAllocate(AccCache.bytes);
|
||||
DeviceBytes+=AccCache.bytes;
|
||||
}
|
||||
mprintf("MemoryManager: Clone %lx <- %lx\n",(uint64_t)AccCache.AccPtr,(uint64_t)AccCache.CpuPtr); fflush(stdout);
|
||||
acceleratorCopyToDevice((void *)AccCache.CpuPtr,(void *)AccCache.AccPtr,AccCache.bytes);
|
||||
HostToDeviceBytes+=AccCache.bytes;
|
||||
HostToDeviceXfer++;
|
||||
AccCache.state=Consistent;
|
||||
}
|
||||
|
||||
void MemoryManager::CpuDiscard(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
assert(AccCache.state!=Empty);
|
||||
assert(AccCache.cpuLock==0);
|
||||
assert(AccCache.accLock==0);
|
||||
assert(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
if(AccCache.AccPtr==(uint64_t)NULL){
|
||||
AccCache.AccPtr=(uint64_t)AcceleratorAllocate(AccCache.bytes);
|
||||
DeviceBytes+=AccCache.bytes;
|
||||
}
|
||||
AccCache.state=AccDirty;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// View management
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
void MemoryManager::ViewClose(void* Ptr,ViewMode mode)
|
||||
{
|
||||
if( (mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard) ){
|
||||
dprintf("AcceleratorViewClose %lx\n",(uint64_t)Ptr);
|
||||
AcceleratorViewClose((uint64_t)Ptr);
|
||||
} else if( (mode==CpuRead)||(mode==CpuWrite)){
|
||||
CpuViewClose((uint64_t)Ptr);
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
void *MemoryManager::ViewOpen(void* _CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
||||
{
|
||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
||||
if( (mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard) ){
|
||||
dprintf("AcceleratorViewOpen %lx\n",(uint64_t)CpuPtr);
|
||||
return (void *) AcceleratorViewOpen(CpuPtr,bytes,mode,hint);
|
||||
} else if( (mode==CpuRead)||(mode==CpuWrite)){
|
||||
return (void *)CpuViewOpen(CpuPtr,bytes,mode,hint);
|
||||
} else {
|
||||
assert(0);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
void MemoryManager::EvictVictims(uint64_t bytes)
|
||||
{
|
||||
assert(bytes<DeviceMaxBytes);
|
||||
while(bytes+DeviceLRUBytes > DeviceMaxBytes){
|
||||
if ( DeviceLRUBytes > 0){
|
||||
assert(LRU.size()>0);
|
||||
uint64_t victim = LRU.back(); // From the LRU
|
||||
auto AccCacheIterator = EntryLookup(victim);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
Evict(AccCache);
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
uint64_t MemoryManager::AcceleratorViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Find if present, otherwise get or force an empty
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
if ( EntryPresent(CpuPtr)==0 ){
|
||||
EntryCreate(CpuPtr,bytes,mode,hint);
|
||||
}
|
||||
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
if (!AccCache.AccPtr) {
|
||||
EvictVictims(bytes);
|
||||
}
|
||||
assert((mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard));
|
||||
|
||||
assert(AccCache.cpuLock==0); // Programming error
|
||||
|
||||
if(AccCache.state!=Empty) {
|
||||
dprintf("ViewOpen found entry %lx %lx : %ld %ld accLock %ld\n",
|
||||
(uint64_t)AccCache.CpuPtr,
|
||||
(uint64_t)CpuPtr,
|
||||
(uint64_t)AccCache.bytes,
|
||||
(uint64_t)bytes,
|
||||
(uint64_t)AccCache.accLock);
|
||||
assert(AccCache.CpuPtr == CpuPtr);
|
||||
assert(AccCache.bytes ==bytes);
|
||||
}
|
||||
/*
|
||||
* State transitions and actions
|
||||
*
|
||||
* Action State StateNext Flush Clone
|
||||
*
|
||||
* AccRead Empty Consistent - Y
|
||||
* AccWrite Empty AccDirty - Y
|
||||
* AccRead CpuDirty Consistent - Y
|
||||
* AccWrite CpuDirty AccDirty - Y
|
||||
* AccRead Consistent Consistent - -
|
||||
* AccWrite Consistent AccDirty - -
|
||||
* AccRead AccDirty AccDirty - -
|
||||
* AccWrite AccDirty AccDirty - -
|
||||
*/
|
||||
if(AccCache.state==Empty) {
|
||||
assert(AccCache.LRU_valid==0);
|
||||
AccCache.CpuPtr = CpuPtr;
|
||||
AccCache.AccPtr = (uint64_t)NULL;
|
||||
AccCache.bytes = bytes;
|
||||
AccCache.state = CpuDirty; // Cpu starts primary
|
||||
if(mode==AcceleratorWriteDiscard){
|
||||
CpuDiscard(AccCache);
|
||||
AccCache.state = AccDirty; // Empty + AcceleratorWrite=> AccDirty
|
||||
} else if(mode==AcceleratorWrite){
|
||||
Clone(AccCache);
|
||||
AccCache.state = AccDirty; // Empty + AcceleratorWrite=> AccDirty
|
||||
} else {
|
||||
Clone(AccCache);
|
||||
AccCache.state = Consistent; // Empty + AccRead => Consistent
|
||||
}
|
||||
AccCache.accLock= 1;
|
||||
dprintf("Copied Empty entry into device accLock= %d\n",AccCache.accLock);
|
||||
} else if(AccCache.state==CpuDirty ){
|
||||
if(mode==AcceleratorWriteDiscard) {
|
||||
CpuDiscard(AccCache);
|
||||
AccCache.state = AccDirty; // CpuDirty + AcceleratorWrite=> AccDirty
|
||||
} else if(mode==AcceleratorWrite) {
|
||||
Clone(AccCache);
|
||||
AccCache.state = AccDirty; // CpuDirty + AcceleratorWrite=> AccDirty
|
||||
} else {
|
||||
Clone(AccCache);
|
||||
AccCache.state = Consistent; // CpuDirty + AccRead => Consistent
|
||||
}
|
||||
AccCache.accLock++;
|
||||
dprintf("CpuDirty entry into device ++accLock= %d\n",AccCache.accLock);
|
||||
} else if(AccCache.state==Consistent) {
|
||||
if((mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard))
|
||||
AccCache.state = AccDirty; // Consistent + AcceleratorWrite=> AccDirty
|
||||
else
|
||||
AccCache.state = Consistent; // Consistent + AccRead => Consistent
|
||||
AccCache.accLock++;
|
||||
dprintf("Consistent entry into device ++accLock= %d\n",AccCache.accLock);
|
||||
} else if(AccCache.state==AccDirty) {
|
||||
if((mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard))
|
||||
AccCache.state = AccDirty; // AccDirty + AcceleratorWrite=> AccDirty
|
||||
else
|
||||
AccCache.state = AccDirty; // AccDirty + AccRead => AccDirty
|
||||
AccCache.accLock++;
|
||||
dprintf("AccDirty entry ++accLock= %d\n",AccCache.accLock);
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
|
||||
assert(AccCache.accLock>0);
|
||||
// If view is opened on device must remove from LRU
|
||||
if(AccCache.LRU_valid==1){
|
||||
// must possibly remove from LRU as now locked on GPU
|
||||
dprintf("AccCache entry removed from LRU \n");
|
||||
LRUremove(AccCache);
|
||||
}
|
||||
|
||||
int transient =hint;
|
||||
AccCache.transient= transient? EvictNext : 0;
|
||||
|
||||
return AccCache.AccPtr;
|
||||
}
|
||||
////////////////////////////////////
|
||||
// look up & decrement lock count
|
||||
////////////////////////////////////
|
||||
void MemoryManager::AcceleratorViewClose(uint64_t CpuPtr)
|
||||
{
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
|
||||
assert(AccCache.cpuLock==0);
|
||||
assert(AccCache.accLock>0);
|
||||
|
||||
AccCache.accLock--;
|
||||
// Move to LRU queue if not locked and close on device
|
||||
if(AccCache.accLock==0) {
|
||||
dprintf("AccleratorViewClose %lx AccLock decremented to %ld move to LRU queue\n",(uint64_t)CpuPtr,(uint64_t)AccCache.accLock);
|
||||
LRUinsert(AccCache);
|
||||
} else {
|
||||
dprintf("AccleratorViewClose %lx AccLock decremented to %ld\n",(uint64_t)CpuPtr,(uint64_t)AccCache.accLock);
|
||||
}
|
||||
}
|
||||
void MemoryManager::CpuViewClose(uint64_t CpuPtr)
|
||||
{
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
|
||||
assert(AccCache.cpuLock>0);
|
||||
assert(AccCache.accLock==0);
|
||||
|
||||
AccCache.cpuLock--;
|
||||
}
|
||||
/*
|
||||
* Action State StateNext Flush Clone
|
||||
*
|
||||
* CpuRead Empty CpuDirty - -
|
||||
* CpuWrite Empty CpuDirty - -
|
||||
* CpuRead CpuDirty CpuDirty - -
|
||||
* CpuWrite CpuDirty CpuDirty - -
|
||||
* CpuRead Consistent Consistent - -
|
||||
* CpuWrite Consistent CpuDirty - -
|
||||
* CpuRead AccDirty Consistent Y -
|
||||
* CpuWrite AccDirty CpuDirty Y -
|
||||
*/
|
||||
uint64_t MemoryManager::CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise transient)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Find if present, otherwise get or force an empty
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
if ( EntryPresent(CpuPtr)==0 ){
|
||||
EntryCreate(CpuPtr,bytes,mode,transient);
|
||||
}
|
||||
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
|
||||
// CPU doesn't need to free space
|
||||
// if (!AccCache.AccPtr) {
|
||||
// EvictVictims(bytes);
|
||||
// }
|
||||
|
||||
assert((mode==CpuRead)||(mode==CpuWrite));
|
||||
assert(AccCache.accLock==0); // Programming error
|
||||
|
||||
if(AccCache.state!=Empty) {
|
||||
assert(AccCache.CpuPtr == CpuPtr);
|
||||
assert(AccCache.bytes==bytes);
|
||||
}
|
||||
|
||||
if(AccCache.state==Empty) {
|
||||
AccCache.CpuPtr = CpuPtr;
|
||||
AccCache.AccPtr = (uint64_t)NULL;
|
||||
AccCache.bytes = bytes;
|
||||
AccCache.state = CpuDirty; // Empty + CpuRead/CpuWrite => CpuDirty
|
||||
AccCache.accLock= 0;
|
||||
AccCache.cpuLock= 1;
|
||||
} else if(AccCache.state==CpuDirty ){
|
||||
// AccPtr dont care, deferred allocate
|
||||
AccCache.state = CpuDirty; // CpuDirty +CpuRead/CpuWrite => CpuDirty
|
||||
AccCache.cpuLock++;
|
||||
} else if(AccCache.state==Consistent) {
|
||||
assert(AccCache.AccPtr != (uint64_t)NULL);
|
||||
if(mode==CpuWrite)
|
||||
AccCache.state = CpuDirty; // Consistent +CpuWrite => CpuDirty
|
||||
else
|
||||
AccCache.state = Consistent; // Consistent +CpuRead => Consistent
|
||||
AccCache.cpuLock++;
|
||||
} else if(AccCache.state==AccDirty) {
|
||||
assert(AccCache.AccPtr != (uint64_t)NULL);
|
||||
Flush(AccCache);
|
||||
if(mode==CpuWrite) AccCache.state = CpuDirty; // AccDirty +CpuWrite => CpuDirty, Flush
|
||||
else AccCache.state = Consistent; // AccDirty +CpuRead => Consistent, Flush
|
||||
AccCache.cpuLock++;
|
||||
} else {
|
||||
assert(0); // should be unreachable
|
||||
}
|
||||
|
||||
AccCache.transient= transient? EvictNext : 0;
|
||||
|
||||
return AccCache.CpuPtr;
|
||||
}
|
||||
void MemoryManager::NotifyDeletion(void *_ptr)
|
||||
{
|
||||
// Look up in ViewCache
|
||||
uint64_t ptr = (uint64_t)_ptr;
|
||||
if(EntryPresent(ptr)) {
|
||||
auto e = EntryLookup(ptr);
|
||||
AccDiscard(e->second);
|
||||
}
|
||||
}
|
||||
void MemoryManager::Print(void)
|
||||
{
|
||||
PrintBytes();
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
std::cout << GridLogMessage << "Memory Manager " << std::endl;
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
std::cout << GridLogMessage << DeviceBytes << " bytes allocated on device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceLRUBytes<< " bytes evictable on device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceMaxBytes<< " bytes max on device " << std::endl;
|
||||
std::cout << GridLogMessage << HostToDeviceXfer << " transfers to device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceToHostXfer << " transfers from device " << std::endl;
|
||||
std::cout << GridLogMessage << HostToDeviceBytes<< " bytes transfered to device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceToHostBytes<< " bytes transfered from device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceEvictions << " Evictions from device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceDestroy << " Destroyed vectors on device " << std::endl;
|
||||
std::cout << GridLogMessage << AccViewTable.size()<< " vectors " << LRU.size()<<" evictable"<< std::endl;
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
}
|
||||
void MemoryManager::PrintAll(void)
|
||||
{
|
||||
Print();
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
for(auto it=AccViewTable.begin();it!=AccViewTable.end();it++){
|
||||
auto &AccCache = it->second;
|
||||
|
||||
std::string str;
|
||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
||||
|
||||
std::cout << GridLogMessage << "0x"<<std::hex<<AccCache.CpuPtr<<std::dec
|
||||
<< "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
||||
<< "\t" << AccCache.cpuLock
|
||||
<< "\t" << AccCache.accLock
|
||||
<< "\t" << AccCache.LRU_valid<<std::endl;
|
||||
}
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
|
||||
};
|
||||
int MemoryManager::isOpen (void* _CpuPtr)
|
||||
{
|
||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
||||
if ( EntryPresent(CpuPtr) ){
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
return AccCache.cpuLock+AccCache.accLock;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
void MemoryManager::Audit(std::string s)
|
||||
{
|
||||
uint64_t CpuBytes=0;
|
||||
uint64_t AccBytes=0;
|
||||
uint64_t LruBytes1=0;
|
||||
uint64_t LruBytes2=0;
|
||||
uint64_t LruCnt=0;
|
||||
|
||||
std::cout << " Memory Manager::Audit() from "<<s<<std::endl;
|
||||
for(auto it=LRU.begin();it!=LRU.end();it++){
|
||||
uint64_t cpuPtr = *it;
|
||||
assert(EntryPresent(cpuPtr));
|
||||
auto AccCacheIterator = EntryLookup(cpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
LruBytes2+=AccCache.bytes;
|
||||
assert(AccCache.LRU_valid==1);
|
||||
assert(AccCache.LRU_entry==it);
|
||||
}
|
||||
std::cout << " Memory Manager::Audit() LRU queue matches table entries "<<std::endl;
|
||||
|
||||
for(auto it=AccViewTable.begin();it!=AccViewTable.end();it++){
|
||||
auto &AccCache = it->second;
|
||||
|
||||
std::string str;
|
||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
||||
|
||||
CpuBytes+=AccCache.bytes;
|
||||
if( AccCache.AccPtr ) AccBytes+=AccCache.bytes;
|
||||
if( AccCache.LRU_valid ) LruBytes1+=AccCache.bytes;
|
||||
if( AccCache.LRU_valid ) LruCnt++;
|
||||
|
||||
if ( AccCache.cpuLock || AccCache.accLock ) {
|
||||
assert(AccCache.LRU_valid==0);
|
||||
|
||||
std::cout << GridLogError << s<< "\n\t 0x"<<std::hex<<AccCache.CpuPtr<<std::dec
|
||||
<< "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
||||
<< "\t cpuLock " << AccCache.cpuLock
|
||||
<< "\t accLock " << AccCache.accLock
|
||||
<< "\t LRUvalid " << AccCache.LRU_valid<<std::endl;
|
||||
}
|
||||
|
||||
assert( AccCache.cpuLock== 0 ) ;
|
||||
assert( AccCache.accLock== 0 ) ;
|
||||
}
|
||||
std::cout << " Memory Manager::Audit() no locked table entries "<<std::endl;
|
||||
assert(LruBytes1==LruBytes2);
|
||||
assert(LruBytes1==DeviceLRUBytes);
|
||||
std::cout << " Memory Manager::Audit() evictable bytes matches sum over table "<<std::endl;
|
||||
assert(AccBytes==DeviceBytes);
|
||||
std::cout << " Memory Manager::Audit() device bytes matches sum over table "<<std::endl;
|
||||
assert(LruCnt == LRU.size());
|
||||
std::cout << " Memory Manager::Audit() LRU entry count matches "<<std::endl;
|
||||
|
||||
}
|
||||
|
||||
void MemoryManager::PrintState(void* _CpuPtr)
|
||||
{
|
||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
||||
|
||||
if ( EntryPresent(CpuPtr) ){
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
std::string str;
|
||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
||||
if ( AccCache.state==EvictNext) str = std::string("EvictNext");
|
||||
|
||||
std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tx"<<std::hex<<AccCache.CpuPtr<<std::dec
|
||||
<< "\tx"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
||||
<< "\t" << AccCache.cpuLock
|
||||
<< "\t" << AccCache.accLock
|
||||
<< "\t" << AccCache.LRU_valid<<std::endl;
|
||||
|
||||
} else {
|
||||
std::cout << GridLogMessage << "No Entry in AccCache table." << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,31 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#ifdef GRID_UVM
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// View management is 1:1 address space mapping
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
uint64_t MemoryManager::DeviceBytes;
|
||||
uint64_t MemoryManager::DeviceLRUBytes;
|
||||
uint64_t MemoryManager::DeviceMaxBytes = 1024*1024*128;
|
||||
uint64_t MemoryManager::HostToDeviceBytes;
|
||||
uint64_t MemoryManager::DeviceToHostBytes;
|
||||
uint64_t MemoryManager::HostToDeviceXfer;
|
||||
uint64_t MemoryManager::DeviceToHostXfer;
|
||||
uint64_t MemoryManager::DeviceEvictions;
|
||||
uint64_t MemoryManager::DeviceDestroy;
|
||||
|
||||
void MemoryManager::Audit(std::string s){};
|
||||
void MemoryManager::ViewClose(void* AccPtr,ViewMode mode){};
|
||||
void *MemoryManager::ViewOpen(void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint){ return CpuPtr; };
|
||||
int MemoryManager::isOpen (void* CpuPtr) { return 0;}
|
||||
void MemoryManager::PrintState(void* CpuPtr)
|
||||
{
|
||||
std::cout << GridLogMessage << "Host<->Device memory movement not currently managed by Grid." << std::endl;
|
||||
};
|
||||
void MemoryManager::Print(void){};
|
||||
void MemoryManager::PrintAll(void){};
|
||||
void MemoryManager::NotifyDeletion(void *ptr){};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,67 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
MemoryStats *MemoryProfiler::stats = nullptr;
|
||||
bool MemoryProfiler::debug = false;
|
||||
|
||||
void check_huge_pages(void *Buf,uint64_t BYTES)
|
||||
{
|
||||
#ifdef __linux__
|
||||
int fd = open("/proc/self/pagemap", O_RDONLY);
|
||||
assert(fd >= 0);
|
||||
const int page_size = 4096;
|
||||
uint64_t virt_pfn = (uint64_t)Buf / page_size;
|
||||
off_t offset = sizeof(uint64_t) * virt_pfn;
|
||||
uint64_t npages = (BYTES + page_size-1) / page_size;
|
||||
uint64_t pagedata[npages];
|
||||
uint64_t ret = lseek(fd, offset, SEEK_SET);
|
||||
assert(ret == offset);
|
||||
ret = ::read(fd, pagedata, sizeof(uint64_t)*npages);
|
||||
assert(ret == sizeof(uint64_t) * npages);
|
||||
int nhugepages = npages / 512;
|
||||
int n4ktotal, nnothuge;
|
||||
n4ktotal = 0;
|
||||
nnothuge = 0;
|
||||
for (int i = 0; i < nhugepages; ++i) {
|
||||
uint64_t baseaddr = (pagedata[i*512] & 0x7fffffffffffffULL) * page_size;
|
||||
for (int j = 0; j < 512; ++j) {
|
||||
uint64_t pageaddr = (pagedata[i*512+j] & 0x7fffffffffffffULL) * page_size;
|
||||
++n4ktotal;
|
||||
if (pageaddr != baseaddr + j * page_size)
|
||||
++nnothuge;
|
||||
}
|
||||
}
|
||||
int rank = CartesianCommunicator::RankWorld();
|
||||
printf("rank %d Allocated %d 4k pages, %d not in huge pages\n", rank, n4ktotal, nnothuge);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string sizeString(const size_t bytes)
|
||||
{
|
||||
constexpr unsigned int bufSize = 256;
|
||||
const char *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
|
||||
char buf[256];
|
||||
size_t s = 0;
|
||||
double count = bytes;
|
||||
|
||||
while (count >= 1024 && s < 7)
|
||||
{
|
||||
s++;
|
||||
count /= 1024;
|
||||
}
|
||||
if (count - floor(count) == 0.0)
|
||||
{
|
||||
snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
|
||||
}
|
||||
else
|
||||
{
|
||||
snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
|
||||
}
|
||||
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/MemoryStats.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
std::string sizeString(size_t bytes);
|
||||
|
||||
struct MemoryStats
|
||||
{
|
||||
size_t totalAllocated{0}, maxAllocated{0},
|
||||
currentlyAllocated{0}, totalFreed{0};
|
||||
};
|
||||
|
||||
class MemoryProfiler
|
||||
{
|
||||
public:
|
||||
static MemoryStats *stats;
|
||||
static bool debug;
|
||||
};
|
||||
|
||||
#define memString(bytes) std::to_string(bytes) + " (" + sizeString(bytes) + ")"
|
||||
#define profilerDebugPrint \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
std::cout << GridLogDebug << "[Memory debug] Stats " << MemoryProfiler::stats << std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] total : " << memString(s->totalAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] max : " << memString(s->maxAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] current: " << memString(s->currentlyAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] freed : " << memString(s->totalFreed) \
|
||||
<< std::endl; \
|
||||
}
|
||||
|
||||
#define profilerAllocate(bytes) \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
s->totalAllocated += (bytes); \
|
||||
s->currentlyAllocated += (bytes); \
|
||||
s->maxAllocated = std::max(s->maxAllocated, s->currentlyAllocated); \
|
||||
} \
|
||||
if (MemoryProfiler::debug) \
|
||||
{ \
|
||||
std::cout << GridLogDebug << "[Memory debug] allocating " << memString(bytes) << std::endl; \
|
||||
profilerDebugPrint; \
|
||||
}
|
||||
|
||||
#define profilerFree(bytes) \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
s->totalFreed += (bytes); \
|
||||
s->currentlyAllocated -= (bytes); \
|
||||
} \
|
||||
if (MemoryProfiler::debug) \
|
||||
{ \
|
||||
std::cout << GridLogDebug << "[Memory debug] freeing " << memString(bytes) << std::endl; \
|
||||
profilerDebugPrint; \
|
||||
}
|
||||
|
||||
void check_huge_pages(void *Buf,uint64_t BYTES);
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Cartesian.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CARTESIAN_H
|
||||
#define GRID_CARTESIAN_H
|
||||
|
||||
#include <Grid/cartesian/Cartesian_base.h>
|
||||
#include <Grid/cartesian/Cartesian_full.h>
|
||||
#include <Grid/cartesian/Cartesian_red_black.h>
|
||||
|
||||
#endif
|
||||
@@ -1,291 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/cartesian/Cartesian_base.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Guido Cossu <guido.cossu@ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CARTESIAN_BASE_H
|
||||
#define GRID_CARTESIAN_BASE_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Commicator provides information on the processor grid
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// unsigned long _ndimension;
|
||||
// Coordinate _processors; // processor grid
|
||||
// int _processor; // linear processor rank
|
||||
// Coordinate _processor_coor; // linear processor rank
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
class GridBase : public CartesianCommunicator , public GridThread {
|
||||
|
||||
public:
|
||||
int dummy;
|
||||
// Give Lattice access
|
||||
template<class object> friend class Lattice;
|
||||
|
||||
GridBase(const Coordinate & processor_grid) : CartesianCommunicator(processor_grid) { LocallyPeriodic=0;};
|
||||
|
||||
GridBase(const Coordinate & processor_grid,
|
||||
const CartesianCommunicator &parent,
|
||||
int &split_rank)
|
||||
: CartesianCommunicator(processor_grid,parent,split_rank) {LocallyPeriodic=0;};
|
||||
|
||||
GridBase(const Coordinate & processor_grid,
|
||||
const CartesianCommunicator &parent)
|
||||
: CartesianCommunicator(processor_grid,parent,dummy) {LocallyPeriodic=0;};
|
||||
|
||||
virtual ~GridBase() = default;
|
||||
|
||||
// Physics Grid information.
|
||||
Coordinate _simd_layout;// Which dimensions get relayed out over simd lanes.
|
||||
Coordinate _fdimensions;// (full) Global dimensions of array prior to cb removal
|
||||
Coordinate _gdimensions;// Global dimensions of array after cb removal
|
||||
Coordinate _ldimensions;// local dimensions of array with processor images removed
|
||||
Coordinate _rdimensions;// Reduced local dimensions with simd lane images and processor images removed
|
||||
Coordinate _ostride; // Outer stride for each dimension
|
||||
Coordinate _istride; // Inner stride i.e. within simd lane
|
||||
int _osites; // _isites*_osites = product(dimensions).
|
||||
int _isites;
|
||||
int _fsites; // _isites*_osites = product(dimensions).
|
||||
int _gsites;
|
||||
Coordinate _slice_block;// subslice information
|
||||
Coordinate _slice_stride;
|
||||
Coordinate _slice_nblock;
|
||||
|
||||
Coordinate _lstart; // local start of array in gcoors _processor_coor[d]*_ldimensions[d]
|
||||
Coordinate _lend ; // local end of array in gcoors _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1
|
||||
|
||||
bool _isCheckerBoarded;
|
||||
int LocallyPeriodic;
|
||||
Coordinate _checker_dim_mask;
|
||||
|
||||
public:
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Checkerboarding interface is virtual and overridden by
|
||||
// GridCartesian / GridRedBlackCartesian
|
||||
////////////////////////////////////////////////////////////////
|
||||
virtual int CheckerBoarded(int dim)=0;
|
||||
virtual int CheckerBoard(const Coordinate &site)=0;
|
||||
virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0;
|
||||
virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0;
|
||||
virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int cb)=0;
|
||||
virtual int CheckerBoardFromOindex (int Oindex)=0;
|
||||
virtual int CheckerBoardFromOindexTable (int Oindex)=0;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Local layout calculations
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// These routines are key. Subdivide the linearised cartesian index into
|
||||
// "inner" index identifying which simd lane of object<vFcomplex> is associated with coord
|
||||
// "outer" index identifying which element of _odata in class "Lattice" is associated with coord.
|
||||
//
|
||||
// Compared to, say, Blitz++ we simply need to store BOTH an inner stride and an outer
|
||||
// stride per dimension. The cost of evaluating the indexing information is doubled for an n-dimensional
|
||||
// coordinate. Note, however, for data parallel operations the "inner" indexing cost is not paid and all
|
||||
// lanes are operated upon simultaneously.
|
||||
|
||||
virtual int oIndex(Coordinate &coor)
|
||||
{
|
||||
int idx=0;
|
||||
// Works with either global or local coordinates
|
||||
for(int d=0;d<_ndimension;d++) idx+=_ostride[d]*(coor[d]%_rdimensions[d]);
|
||||
return idx;
|
||||
}
|
||||
virtual int iIndex(Coordinate &lcoor)
|
||||
{
|
||||
int idx=0;
|
||||
for(int d=0;d<_ndimension;d++) idx+=_istride[d]*(lcoor[d]/_rdimensions[d]);
|
||||
return idx;
|
||||
}
|
||||
inline int oIndexReduced(Coordinate &ocoor)
|
||||
{
|
||||
int idx=0;
|
||||
// ocoor is already reduced so can eliminate the modulo operation
|
||||
// for fast indexing and inline the routine
|
||||
for(int d=0;d<_ndimension;d++) idx+=_ostride[d]*ocoor[d];
|
||||
return idx;
|
||||
}
|
||||
inline void oCoorFromOindex (Coordinate& coor,int Oindex){
|
||||
Lexicographic::CoorFromIndex(coor,Oindex,_rdimensions);
|
||||
}
|
||||
|
||||
inline void InOutCoorToLocalCoor (Coordinate &ocoor, Coordinate &icoor, Coordinate &lcoor) {
|
||||
lcoor.resize(_ndimension);
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
lcoor[d] = ocoor[d] + _rdimensions[d] * icoor[d];
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// SIMD lane addressing
|
||||
//////////////////////////////////////////////////////////
|
||||
inline void iCoorFromIindex(Coordinate &coor,int lane)
|
||||
{
|
||||
Lexicographic::CoorFromIndex(coor,lane,_simd_layout);
|
||||
}
|
||||
|
||||
inline int PermuteDim(int dimension){
|
||||
return _simd_layout[dimension]>1;
|
||||
}
|
||||
inline int PermuteType(int dimension){
|
||||
int permute_type=0;
|
||||
//
|
||||
// Best way to encode this would be to present a mask
|
||||
// for which simd dimensions are rotated, and the rotation
|
||||
// size. If there is only one simd dimension rotated, this is just
|
||||
// a permute.
|
||||
//
|
||||
// Cases: PermuteType == 1,2,4,8
|
||||
// Distance should be either 0,1,2..
|
||||
//
|
||||
if ( _simd_layout[dimension] > 2 ) {
|
||||
for(int d=0;d<_ndimension;d++){
|
||||
if ( d != dimension ) assert ( (_simd_layout[d]==1) );
|
||||
}
|
||||
permute_type = RotateBit; // How to specify distance; this is not just direction.
|
||||
return permute_type;
|
||||
}
|
||||
|
||||
for(int d=_ndimension-1;d>dimension;d--){
|
||||
if (_simd_layout[d]>1 ) permute_type++;
|
||||
}
|
||||
return permute_type;
|
||||
}
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Array sizing queries
|
||||
////////////////////////////////////////////////////////////////
|
||||
|
||||
inline int iSites(void) const { return _isites; };
|
||||
inline int Nsimd(void) const { return _isites; };// Synonymous with iSites
|
||||
inline int oSites(void) const { return _osites; };
|
||||
inline int lSites(void) const { return _isites*_osites; };
|
||||
inline int gSites(void) const { return _isites*_osites*_Nprocessors; };
|
||||
inline int Nd (void) const { return _ndimension;};
|
||||
|
||||
inline const Coordinate LocalStarts(void) { return _lstart; };
|
||||
inline const Coordinate &FullDimensions(void) { return _fdimensions;};
|
||||
inline const Coordinate &GlobalDimensions(void) { return _gdimensions;};
|
||||
inline const Coordinate &LocalDimensions(void) { return _ldimensions;};
|
||||
inline const Coordinate &VirtualLocalDimensions(void) { return _ldimensions;};
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Utility to print the full decomposition details
|
||||
////////////////////////////////////////////////////////////////
|
||||
|
||||
void show_decomposition(){
|
||||
std::cout << GridLogMessage << "\tFull Dimensions : " << _fdimensions << std::endl;
|
||||
std::cout << GridLogMessage << "\tSIMD layout : " << _simd_layout << std::endl;
|
||||
std::cout << GridLogMessage << "\tGlobal Dimensions : " << _gdimensions << std::endl;
|
||||
std::cout << GridLogMessage << "\tLocal Dimensions : " << _ldimensions << std::endl;
|
||||
std::cout << GridLogMessage << "\tReduced Dimensions : " << _rdimensions << std::endl;
|
||||
std::cout << GridLogMessage << "\tOuter strides : " << _ostride << std::endl;
|
||||
std::cout << GridLogMessage << "\tInner strides : " << _istride << std::endl;
|
||||
std::cout << GridLogMessage << "\tiSites : " << _isites << std::endl;
|
||||
std::cout << GridLogMessage << "\toSites : " << _osites << std::endl;
|
||||
std::cout << GridLogMessage << "\tlSites : " << lSites() << std::endl;
|
||||
std::cout << GridLogMessage << "\tgSites : " << gSites() << std::endl;
|
||||
std::cout << GridLogMessage << "\tNd : " << _ndimension << std::endl;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Global addressing
|
||||
////////////////////////////////////////////////////////////////
|
||||
void GlobalIndexToGlobalCoor(int gidx,Coordinate &gcoor){
|
||||
assert(gidx< gSites());
|
||||
Lexicographic::CoorFromIndex(gcoor,gidx,_gdimensions);
|
||||
}
|
||||
void LocalIndexToLocalCoor(int lidx,Coordinate &lcoor){
|
||||
assert(lidx<lSites());
|
||||
Lexicographic::CoorFromIndex(lcoor,lidx,_ldimensions);
|
||||
}
|
||||
void GlobalCoorToGlobalIndex(const Coordinate & gcoor,int & gidx){
|
||||
gidx=0;
|
||||
int mult=1;
|
||||
for(int mu=0;mu<_ndimension;mu++) {
|
||||
gidx+=mult*gcoor[mu];
|
||||
mult*=_gdimensions[mu];
|
||||
}
|
||||
}
|
||||
void GlobalCoorToProcessorCoorLocalCoor(Coordinate &pcoor,Coordinate &lcoor,const Coordinate &gcoor)
|
||||
{
|
||||
pcoor.resize(_ndimension);
|
||||
lcoor.resize(_ndimension);
|
||||
for(int mu=0;mu<_ndimension;mu++){
|
||||
int _fld = _fdimensions[mu]/_processors[mu];
|
||||
pcoor[mu] = gcoor[mu]/_fld;
|
||||
lcoor[mu] = gcoor[mu]%_fld;
|
||||
}
|
||||
}
|
||||
void GlobalCoorToRankIndex(int &rank, int &o_idx, int &i_idx ,const Coordinate &gcoor)
|
||||
{
|
||||
Coordinate pcoor;
|
||||
Coordinate lcoor;
|
||||
GlobalCoorToProcessorCoorLocalCoor(pcoor,lcoor,gcoor);
|
||||
rank = RankFromProcessorCoor(pcoor);
|
||||
/*
|
||||
Coordinate cblcoor(lcoor);
|
||||
for(int d=0;d<cblcoor.size();d++){
|
||||
if( this->CheckerBoarded(d) ) {
|
||||
cblcoor[d] = lcoor[d]/2;
|
||||
}
|
||||
}
|
||||
*/
|
||||
i_idx= iIndex(lcoor);
|
||||
o_idx= oIndex(lcoor);
|
||||
}
|
||||
|
||||
void RankIndexToGlobalCoor(int rank, int o_idx, int i_idx , Coordinate &gcoor)
|
||||
{
|
||||
gcoor.resize(_ndimension);
|
||||
Coordinate coor(_ndimension);
|
||||
|
||||
ProcessorCoorFromRank(rank,coor);
|
||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] = _ldimensions[mu]*coor[mu];
|
||||
|
||||
iCoorFromIindex(coor,i_idx);
|
||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] += _rdimensions[mu]*coor[mu];
|
||||
|
||||
oCoorFromOindex (coor,o_idx);
|
||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] += coor[mu];
|
||||
|
||||
}
|
||||
void RankIndexCbToFullGlobalCoor(int rank, int o_idx, int i_idx, int cb,Coordinate &fcoor)
|
||||
{
|
||||
RankIndexToGlobalCoor(rank,o_idx,i_idx ,fcoor);
|
||||
if(CheckerBoarded(0)){
|
||||
fcoor[0] = fcoor[0]*2+cb;
|
||||
}
|
||||
}
|
||||
void ProcessorCoorLocalCoorToGlobalCoor(Coordinate &Pcoor,Coordinate &Lcoor,Coordinate &gcoor)
|
||||
{
|
||||
gcoor.resize(_ndimension);
|
||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] = Pcoor[mu]*_ldimensions[mu]+Lcoor[mu];
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,178 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/cartesian/Cartesian_full.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CARTESIAN_FULL_H
|
||||
#define GRID_CARTESIAN_FULL_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Grid Support.
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
class GridCartesian: public GridBase {
|
||||
|
||||
public:
|
||||
int dummy;
|
||||
Coordinate _checker_dim_mask;
|
||||
virtual int CheckerBoardFromOindexTable (int Oindex) {
|
||||
return 0;
|
||||
}
|
||||
virtual int CheckerBoardFromOindex (int Oindex)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
virtual int CheckerBoarded(int dim){
|
||||
return 0;
|
||||
}
|
||||
virtual int CheckerBoard(const Coordinate &site){
|
||||
return 0;
|
||||
}
|
||||
virtual int CheckerBoardDestination(int cb,int shift,int dim){
|
||||
return 0;
|
||||
}
|
||||
virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift, int ocb){
|
||||
return shift;
|
||||
}
|
||||
virtual int CheckerBoardShift(int source_cb,int dim,int shift, int osite){
|
||||
return shift;
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
// Constructor takes a parent grid and possibly subdivides communicator.
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
GridCartesian(const Coordinate &dimensions,
|
||||
const Coordinate &simd_layout,
|
||||
const Coordinate &processor_grid,
|
||||
const GridCartesian &parent) : GridBase(processor_grid,parent,dummy)
|
||||
{
|
||||
Init(dimensions,simd_layout,processor_grid);
|
||||
}
|
||||
GridCartesian(const Coordinate &dimensions,
|
||||
const Coordinate &simd_layout,
|
||||
const Coordinate &processor_grid,
|
||||
const GridCartesian &parent,int &split_rank) : GridBase(processor_grid,parent,split_rank)
|
||||
{
|
||||
Init(dimensions,simd_layout,processor_grid);
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
// Construct from comm world
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
GridCartesian(const Coordinate &dimensions,
|
||||
const Coordinate &simd_layout,
|
||||
const Coordinate &processor_grid) : GridBase(processor_grid)
|
||||
{
|
||||
Init(dimensions,simd_layout,processor_grid);
|
||||
}
|
||||
|
||||
virtual ~GridCartesian() = default;
|
||||
|
||||
void Init(const Coordinate &dimensions,
|
||||
const Coordinate &simd_layout,
|
||||
const Coordinate &processor_grid)
|
||||
{
|
||||
///////////////////////
|
||||
// Grid information
|
||||
///////////////////////
|
||||
_isCheckerBoarded = false;
|
||||
_ndimension = dimensions.size();
|
||||
|
||||
_fdimensions.resize(_ndimension);
|
||||
_gdimensions.resize(_ndimension);
|
||||
_ldimensions.resize(_ndimension);
|
||||
_rdimensions.resize(_ndimension);
|
||||
_simd_layout.resize(_ndimension);
|
||||
_checker_dim_mask.resize(_ndimension);;
|
||||
_lstart.resize(_ndimension);
|
||||
_lend.resize(_ndimension);
|
||||
|
||||
_ostride.resize(_ndimension);
|
||||
_istride.resize(_ndimension);
|
||||
|
||||
_fsites = _gsites = _osites = _isites = 1;
|
||||
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
_checker_dim_mask[d]=0;
|
||||
|
||||
_fdimensions[d] = dimensions[d]; // Global dimensions
|
||||
_gdimensions[d] = _fdimensions[d]; // Global dimensions
|
||||
_simd_layout[d] = simd_layout[d];
|
||||
_fsites = _fsites * _fdimensions[d];
|
||||
_gsites = _gsites * _gdimensions[d];
|
||||
|
||||
// Use a reduced simd grid
|
||||
_ldimensions[d] = _gdimensions[d] / _processors[d]; //local dimensions
|
||||
//std::cout << _ldimensions[d] << " " << _gdimensions[d] << " " << _processors[d] << std::endl;
|
||||
assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
||||
|
||||
_rdimensions[d] = _ldimensions[d] / _simd_layout[d]; //overdecomposition
|
||||
assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
||||
|
||||
_lstart[d] = _processor_coor[d] * _ldimensions[d];
|
||||
_lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
|
||||
_osites *= _rdimensions[d];
|
||||
_isites *= _simd_layout[d];
|
||||
|
||||
// Addressing support
|
||||
if (d == 0)
|
||||
{
|
||||
_ostride[d] = 1;
|
||||
_istride[d] = 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
_ostride[d] = _ostride[d - 1] * _rdimensions[d - 1];
|
||||
_istride[d] = _istride[d - 1] * _simd_layout[d - 1];
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////
|
||||
// subplane information
|
||||
///////////////////////
|
||||
_slice_block.resize(_ndimension);
|
||||
_slice_stride.resize(_ndimension);
|
||||
_slice_nblock.resize(_ndimension);
|
||||
|
||||
int block = 1;
|
||||
int nblock = 1;
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
nblock *= _rdimensions[d];
|
||||
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
nblock /= _rdimensions[d];
|
||||
_slice_block[d] = block;
|
||||
_slice_stride[d] = _ostride[d] * _rdimensions[d];
|
||||
_slice_nblock[d] = nblock;
|
||||
block = block * _rdimensions[d];
|
||||
}
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,305 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/cartesian/Cartesian_red_black.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CARTESIAN_RED_BLACK_H
|
||||
#define GRID_CARTESIAN_RED_BLACK_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
static const int CbRed =0;
|
||||
static const int CbBlack=1;
|
||||
static const int Even =CbRed;
|
||||
static const int Odd =CbBlack;
|
||||
|
||||
accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex,const Coordinate &rdim,const Coordinate &chk_dim_msk)
|
||||
{
|
||||
int nd=rdim.size();
|
||||
Coordinate coor(nd);
|
||||
|
||||
Lexicographic::CoorFromIndex(coor,oindex,rdim);
|
||||
|
||||
int linear=0;
|
||||
for(int d=0;d<nd;d++){
|
||||
if(chk_dim_msk[d])
|
||||
linear=linear+coor[d];
|
||||
}
|
||||
return (linear&0x1);
|
||||
}
|
||||
|
||||
|
||||
// Specialise this for red black grids storing half the data like a chess board.
|
||||
class GridRedBlackCartesian : public GridBase
|
||||
{
|
||||
public:
|
||||
// Coordinate _checker_dim_mask;
|
||||
int _checker_dim;
|
||||
std::vector<int> _checker_board;
|
||||
|
||||
virtual int CheckerBoarded(int dim){
|
||||
if( dim==_checker_dim) return 1;
|
||||
else return 0;
|
||||
}
|
||||
virtual int CheckerBoard(const Coordinate &site){
|
||||
int linear=0;
|
||||
assert(site.size()==_ndimension);
|
||||
for(int d=0;d<_ndimension;d++){
|
||||
if(_checker_dim_mask[d])
|
||||
linear=linear+site[d];
|
||||
}
|
||||
return (linear&0x1);
|
||||
}
|
||||
|
||||
// Depending on the cb of site, we toggle source cb.
|
||||
// for block #b, element #e = (b, e)
|
||||
// we need
|
||||
virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int ocb){
|
||||
if(dim != _checker_dim) return shift;
|
||||
|
||||
int fulldim =_fdimensions[dim];
|
||||
shift = (shift+fulldim)%fulldim;
|
||||
|
||||
// Probably faster with table lookup;
|
||||
// or by looping over x,y,z and multiply rather than computing checkerboard.
|
||||
|
||||
if ( (source_cb+ocb)&1 ) {
|
||||
return (shift)/2;
|
||||
} else {
|
||||
return (shift+1)/2;
|
||||
}
|
||||
}
|
||||
virtual int CheckerBoardFromOindexTable (int Oindex) {
|
||||
return _checker_board[Oindex];
|
||||
}
|
||||
virtual int CheckerBoardFromOindex (int Oindex)
|
||||
{
|
||||
Coordinate ocoor;
|
||||
oCoorFromOindex(ocoor,Oindex);
|
||||
return CheckerBoard(ocoor);
|
||||
}
|
||||
virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite){
|
||||
|
||||
if(dim != _checker_dim) return shift;
|
||||
|
||||
int ocb=CheckerBoardFromOindex(osite);
|
||||
|
||||
return CheckerBoardShiftForCB(source_cb,dim,shift,ocb);
|
||||
}
|
||||
|
||||
virtual int CheckerBoardDestination(int source_cb,int shift,int dim){
|
||||
if ( _checker_dim_mask[dim] ) {
|
||||
// If _fdimensions[checker_dim] is odd, then shifting by 1 in other dims
|
||||
// does NOT cause a parity hop.
|
||||
int add=(dim==_checker_dim) ? 0 : _fdimensions[_checker_dim];
|
||||
if ( (shift+add) &0x1) {
|
||||
return 1-source_cb;
|
||||
} else {
|
||||
return source_cb;
|
||||
}
|
||||
} else {
|
||||
return source_cb;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Create Redblack from original grid; require full grid pointer ?
|
||||
////////////////////////////////////////////////////////////
|
||||
GridRedBlackCartesian(const GridBase *base) : GridBase(base->_processors,*base)
|
||||
{
|
||||
int dims = base->_ndimension;
|
||||
Coordinate checker_dim_mask(dims,1);
|
||||
int checker_dim = 0;
|
||||
Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim);
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Create redblack from original grid, with non-trivial checker dim mask
|
||||
////////////////////////////////////////////////////////////
|
||||
GridRedBlackCartesian(const GridBase *base,
|
||||
const Coordinate &checker_dim_mask,
|
||||
int checker_dim
|
||||
) : GridBase(base->_processors,*base)
|
||||
{
|
||||
Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim) ;
|
||||
}
|
||||
|
||||
virtual ~GridRedBlackCartesian() = default;
|
||||
|
||||
void Init(const Coordinate &dimensions,
|
||||
const Coordinate &simd_layout,
|
||||
const Coordinate &processor_grid,
|
||||
const Coordinate &checker_dim_mask,
|
||||
int checker_dim)
|
||||
{
|
||||
|
||||
_isCheckerBoarded = true;
|
||||
_checker_dim = checker_dim;
|
||||
assert(checker_dim_mask[checker_dim] == 1);
|
||||
_ndimension = dimensions.size();
|
||||
assert(checker_dim_mask.size() == _ndimension);
|
||||
assert(processor_grid.size() == _ndimension);
|
||||
assert(simd_layout.size() == _ndimension);
|
||||
|
||||
_fdimensions.resize(_ndimension);
|
||||
_gdimensions.resize(_ndimension);
|
||||
_ldimensions.resize(_ndimension);
|
||||
_rdimensions.resize(_ndimension);
|
||||
_simd_layout.resize(_ndimension);
|
||||
_lstart.resize(_ndimension);
|
||||
_lend.resize(_ndimension);
|
||||
|
||||
_ostride.resize(_ndimension);
|
||||
_istride.resize(_ndimension);
|
||||
|
||||
_fsites = _gsites = _osites = _isites = 1;
|
||||
|
||||
_checker_dim_mask = checker_dim_mask;
|
||||
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
_fdimensions[d] = dimensions[d];
|
||||
_gdimensions[d] = _fdimensions[d];
|
||||
_fsites = _fsites * _fdimensions[d];
|
||||
_gsites = _gsites * _gdimensions[d];
|
||||
|
||||
if (d == _checker_dim)
|
||||
{
|
||||
assert((_gdimensions[d] & 0x1) == 0);
|
||||
_gdimensions[d] = _gdimensions[d] / 2; // Remove a checkerboard
|
||||
_gsites /= 2;
|
||||
}
|
||||
_ldimensions[d] = _gdimensions[d] / _processors[d];
|
||||
assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
||||
_lstart[d] = _processor_coor[d] * _ldimensions[d];
|
||||
_lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
|
||||
|
||||
// Use a reduced simd grid
|
||||
_simd_layout[d] = simd_layout[d];
|
||||
_rdimensions[d] = _ldimensions[d] / _simd_layout[d]; // this is not checking if this is integer
|
||||
assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
||||
assert(_rdimensions[d] > 0);
|
||||
|
||||
// all elements of a simd vector must have same checkerboard.
|
||||
// If Ls vectorised, this must still be the case; e.g. dwf rb5d
|
||||
if (_simd_layout[d] > 1)
|
||||
{
|
||||
if (checker_dim_mask[d])
|
||||
{
|
||||
assert((_rdimensions[d] & 0x1) == 0);
|
||||
}
|
||||
}
|
||||
|
||||
_osites *= _rdimensions[d];
|
||||
_isites *= _simd_layout[d];
|
||||
|
||||
// Addressing support
|
||||
if (d == 0)
|
||||
{
|
||||
_ostride[d] = 1;
|
||||
_istride[d] = 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
_ostride[d] = _ostride[d - 1] * _rdimensions[d - 1];
|
||||
_istride[d] = _istride[d - 1] * _simd_layout[d - 1];
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// subplane information
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
_slice_block.resize(_ndimension);
|
||||
_slice_stride.resize(_ndimension);
|
||||
_slice_nblock.resize(_ndimension);
|
||||
|
||||
int block = 1;
|
||||
int nblock = 1;
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
nblock *= _rdimensions[d];
|
||||
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
nblock /= _rdimensions[d];
|
||||
_slice_block[d] = block;
|
||||
_slice_stride[d] = _ostride[d] * _rdimensions[d];
|
||||
_slice_nblock[d] = nblock;
|
||||
block = block * _rdimensions[d];
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// Create a checkerboard lookup table
|
||||
////////////////////////////////////////////////
|
||||
int rvol = 1;
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
rvol = rvol * _rdimensions[d];
|
||||
}
|
||||
_checker_board.resize(rvol);
|
||||
for (int osite = 0; osite < _osites; osite++)
|
||||
{
|
||||
_checker_board[osite] = CheckerBoardFromOindex(osite);
|
||||
}
|
||||
};
|
||||
|
||||
protected:
|
||||
virtual int oIndex(Coordinate &coor)
|
||||
{
|
||||
int idx = 0;
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
if (d == _checker_dim)
|
||||
{
|
||||
idx += _ostride[d] * ((coor[d] / 2) % _rdimensions[d]);
|
||||
}
|
||||
else
|
||||
{
|
||||
idx += _ostride[d] * (coor[d] % _rdimensions[d]);
|
||||
}
|
||||
}
|
||||
return idx;
|
||||
};
|
||||
|
||||
virtual int iIndex(Coordinate &lcoor)
|
||||
{
|
||||
int idx = 0;
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
if (d == _checker_dim)
|
||||
{
|
||||
idx += _istride[d] * (lcoor[d] / (2 * _rdimensions[d]));
|
||||
}
|
||||
else
|
||||
{
|
||||
idx += _istride[d] * (lcoor[d] / _rdimensions[d]);
|
||||
}
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,35 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Communicator.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_COMMUNICATOR_H
|
||||
#define GRID_COMMUNICATOR_H
|
||||
|
||||
#include <Grid/util/Coordinate.h>
|
||||
#include <Grid/communicator/SharedMemory.h>
|
||||
#include <Grid/communicator/Communicator_base.h>
|
||||
|
||||
#endif
|
||||
@@ -1,79 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/Communicator_none.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/GridCore.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <limits.h>
|
||||
#include <sys/mman.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
bool Stencil_force_mpi = true;
|
||||
|
||||
///////////////////////////////////////////////////////////////
|
||||
// Info that is setup once and indept of cartesian layout
|
||||
///////////////////////////////////////////////////////////////
|
||||
CartesianCommunicator::CommunicatorPolicy_t
|
||||
CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent;
|
||||
int CartesianCommunicator::nCommThreads = -1;
|
||||
|
||||
/////////////////////////////////
|
||||
// Grid information queries
|
||||
/////////////////////////////////
|
||||
int CartesianCommunicator::Dimensions(void) { return _ndimension; };
|
||||
int CartesianCommunicator::IsBoss(void) { return _processor==0; };
|
||||
int CartesianCommunicator::BossRank(void) { return 0; };
|
||||
int CartesianCommunicator::ThisRank(void) { return _processor; };
|
||||
const Coordinate & CartesianCommunicator::ThisProcessorCoor(void) { return _processor_coor; };
|
||||
const Coordinate & CartesianCommunicator::ProcessorGrid(void) { return _processors; };
|
||||
int CartesianCommunicator::ProcessorCount(void) { return _Nprocessors; };
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// very VERY rarely (Log, serial RNG) we need world without a grid
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void CartesianCommunicator::GlobalSum(ComplexF &c)
|
||||
{
|
||||
GlobalSumVector((float *)&c,2);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(ComplexF *c,int N)
|
||||
{
|
||||
GlobalSumVector((float *)c,2*N);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(ComplexD &c)
|
||||
{
|
||||
GlobalSumVector((double *)&c,2);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
|
||||
{
|
||||
GlobalSumVector((double *)c,2*N);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/Communicator_base.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_COMMUNICATOR_BASE_H
|
||||
#define GRID_COMMUNICATOR_BASE_H
|
||||
|
||||
///////////////////////////////////
|
||||
// Processor layout information
|
||||
///////////////////////////////////
|
||||
#include <Grid/communicator/SharedMemory.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
extern bool Stencil_force_mpi ;
|
||||
|
||||
class CartesianCommunicator : public SharedMemory {
|
||||
|
||||
public:
|
||||
|
||||
////////////////////////////////////////////
|
||||
// Policies
|
||||
////////////////////////////////////////////
|
||||
enum CommunicatorPolicy_t { CommunicatorPolicyConcurrent, CommunicatorPolicySequential };
|
||||
static CommunicatorPolicy_t CommunicatorPolicy;
|
||||
static void SetCommunicatorPolicy(CommunicatorPolicy_t policy ) { CommunicatorPolicy = policy; }
|
||||
static int nCommThreads;
|
||||
|
||||
////////////////////////////////////////////
|
||||
// Communicator should know nothing of the physics grid, only processor grid.
|
||||
////////////////////////////////////////////
|
||||
int _Nprocessors; // How many in all
|
||||
int _processor; // linear processor rank
|
||||
unsigned long _ndimension;
|
||||
Coordinate _shm_processors; // Which dimensions get relayed out over processors lanes.
|
||||
Coordinate _processors; // Which dimensions get relayed out over processors lanes.
|
||||
Coordinate _processor_coor; // linear processor coordinate
|
||||
static Grid_MPI_Comm communicator_world;
|
||||
Grid_MPI_Comm communicator;
|
||||
std::vector<Grid_MPI_Comm> communicator_halo;
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// Must call in Grid startup
|
||||
////////////////////////////////////////////////
|
||||
static void Init(int *argc, char ***argv);
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// Constructors to sub-divide a parent communicator
|
||||
// and default to comm world
|
||||
////////////////////////////////////////////////
|
||||
CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank);
|
||||
CartesianCommunicator(const Coordinate &pdimensions_in);
|
||||
virtual ~CartesianCommunicator();
|
||||
|
||||
private:
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// Private initialise from an MPI communicator
|
||||
// Can use after an MPI_Comm_split, but hidden from user so private
|
||||
////////////////////////////////////////////////
|
||||
void InitFromMPICommunicator(const Coordinate &processors, Grid_MPI_Comm communicator_base);
|
||||
|
||||
public:
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Wraps MPI_Cart routines, or implements equivalent on other impls
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
void ShiftedRanks(int dim,int shift,int & source, int & dest);
|
||||
int RankFromProcessorCoor(Coordinate &coor);
|
||||
void ProcessorCoorFromRank(int rank,Coordinate &coor);
|
||||
|
||||
int Dimensions(void) ;
|
||||
int IsBoss(void) ;
|
||||
int BossRank(void) ;
|
||||
int ThisRank(void) ;
|
||||
const Coordinate & ThisProcessorCoor(void) ;
|
||||
const Coordinate & ShmGrid(void) { return _shm_processors; } ;
|
||||
const Coordinate & ProcessorGrid(void) ;
|
||||
int ProcessorCount(void) ;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// very VERY rarely (Log, serial RNG) we need world without a grid
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
static int RankWorld(void) ;
|
||||
static void BroadcastWorld(int root,void* data, int bytes);
|
||||
static void BarrierWorld(void);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Reduction
|
||||
////////////////////////////////////////////////////////////
|
||||
void GlobalMax(RealD &);
|
||||
void GlobalMax(RealF &);
|
||||
void GlobalSum(RealF &);
|
||||
void GlobalSumVector(RealF *,int N);
|
||||
void GlobalSum(RealD &);
|
||||
void GlobalSumVector(RealD *,int N);
|
||||
void GlobalSum(uint32_t &);
|
||||
void GlobalSum(uint64_t &);
|
||||
void GlobalSumVector(uint64_t*,int N);
|
||||
void GlobalSum(ComplexF &c);
|
||||
void GlobalSumVector(ComplexF *c,int N);
|
||||
void GlobalSum(ComplexD &c);
|
||||
void GlobalSumVector(ComplexD *c,int N);
|
||||
void GlobalXOR(uint32_t &);
|
||||
void GlobalXOR(uint64_t &);
|
||||
|
||||
template<class obj> void GlobalSum(obj &o){
|
||||
typedef typename obj::scalar_type scalar_type;
|
||||
int words = sizeof(obj)/sizeof(scalar_type);
|
||||
scalar_type * ptr = (scalar_type *)& o; // Safe alias
|
||||
GlobalSumVector(ptr,words);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Face exchange, buffer swap in translational invariant way
|
||||
////////////////////////////////////////////////////////////
|
||||
void SendToRecvFrom(void *xmit,
|
||||
int xmit_to_rank,
|
||||
void *recv,
|
||||
int recv_from_rank,
|
||||
int bytes);
|
||||
|
||||
double StencilSendToRecvFrom(void *xmit,
|
||||
int xmit_to_rank,int do_xmit,
|
||||
void *recv,
|
||||
int recv_from_rank,int do_recv,
|
||||
int bytes,int dir);
|
||||
|
||||
double StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int xmit_to_rank,int do_xmit,
|
||||
void *recv,
|
||||
int recv_from_rank,int do_recv,
|
||||
int xbytes,int rbytes,int dir);
|
||||
|
||||
|
||||
void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int i);
|
||||
void StencilBarrier(void);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Barrier
|
||||
////////////////////////////////////////////////////////////
|
||||
void Barrier(void);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Broadcast a buffer and composite larger
|
||||
////////////////////////////////////////////////////////////
|
||||
void Broadcast(int root,void* data, int bytes);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// All2All down one dimension
|
||||
////////////////////////////////////////////////////////////
|
||||
template<class T> void AllToAll(int dim,std::vector<T> &in, std::vector<T> &out){
|
||||
assert(dim>=0);
|
||||
assert(dim<_ndimension);
|
||||
assert(in.size()==out.size());
|
||||
int numnode = _processors[dim];
|
||||
uint64_t bytes=sizeof(T);
|
||||
uint64_t words=in.size()/numnode;
|
||||
assert(numnode * words == in.size());
|
||||
assert(words < (1ULL<<31));
|
||||
AllToAll(dim,(void *)&in[0],(void *)&out[0],words,bytes);
|
||||
}
|
||||
void AllToAll(int dim ,void *in,void *out,uint64_t words,uint64_t bytes);
|
||||
void AllToAll(void *in,void *out,uint64_t words ,uint64_t bytes);
|
||||
|
||||
template<class obj> void Broadcast(int root,obj &data)
|
||||
{
|
||||
Broadcast(root,(void *)&data,sizeof(data));
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,483 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/Communicator_mpi.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/GridCore.h>
|
||||
#include <Grid/communicator/SharedMemory.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
Grid_MPI_Comm CartesianCommunicator::communicator_world;
|
||||
|
||||
////////////////////////////////////////////
|
||||
// First initialise of comms system
|
||||
////////////////////////////////////////////
|
||||
void CartesianCommunicator::Init(int *argc, char ***argv)
|
||||
{
|
||||
|
||||
int flag;
|
||||
int provided;
|
||||
|
||||
MPI_Initialized(&flag); // needed to coexist with other libs apparently
|
||||
if ( !flag ) {
|
||||
|
||||
#ifndef GRID_COMMS_THREADS
|
||||
nCommThreads=1;
|
||||
// wrong results here too
|
||||
// For now: comms-overlap leads to wrong results in Benchmark_wilson even on single node MPI runs
|
||||
// other comms schemes are ok
|
||||
MPI_Init_thread(argc,argv,MPI_THREAD_SERIALIZED,&provided);
|
||||
#else
|
||||
MPI_Init_thread(argc,argv,MPI_THREAD_MULTIPLE,&provided);
|
||||
#endif
|
||||
//If only 1 comms thread we require any threading mode other than SINGLE, but for multiple comms threads we need MULTIPLE
|
||||
if( (nCommThreads == 1) && (provided == MPI_THREAD_SINGLE) ) {
|
||||
assert(0);
|
||||
}
|
||||
|
||||
if( (nCommThreads > 1) && (provided != MPI_THREAD_MULTIPLE) ) {
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Never clean up as done once.
|
||||
MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
|
||||
|
||||
Grid_quiesce_nodes();
|
||||
GlobalSharedMemory::Init(communicator_world);
|
||||
GlobalSharedMemory::SharedMemoryAllocate(
|
||||
GlobalSharedMemory::MAX_MPI_SHM_BYTES,
|
||||
GlobalSharedMemory::Hugepages);
|
||||
Grid_unquiesce_nodes();
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// Use cartesian communicators now even in MPI3
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
|
||||
{
|
||||
int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest);
|
||||
assert(ierr==0);
|
||||
}
|
||||
int CartesianCommunicator::RankFromProcessorCoor(Coordinate &coor)
|
||||
{
|
||||
int rank;
|
||||
int ierr=MPI_Cart_rank (communicator, &coor[0], &rank);
|
||||
assert(ierr==0);
|
||||
return rank;
|
||||
}
|
||||
void CartesianCommunicator::ProcessorCoorFromRank(int rank, Coordinate &coor)
|
||||
{
|
||||
coor.resize(_ndimension);
|
||||
int ierr=MPI_Cart_coords (communicator, rank, _ndimension,&coor[0]);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Initialises from communicator_world
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
||||
{
|
||||
MPI_Comm optimal_comm;
|
||||
////////////////////////////////////////////////////
|
||||
// Remap using the shared memory optimising routine
|
||||
// The remap creates a comm which must be freed
|
||||
////////////////////////////////////////////////////
|
||||
GlobalSharedMemory::OptimalCommunicator (processors,optimal_comm,_shm_processors);
|
||||
InitFromMPICommunicator(processors,optimal_comm);
|
||||
SetCommunicator(optimal_comm);
|
||||
///////////////////////////////////////////////////
|
||||
// Free the temp communicator
|
||||
///////////////////////////////////////////////////
|
||||
MPI_Comm_free(&optimal_comm);
|
||||
}
|
||||
|
||||
//////////////////////////////////
|
||||
// Try to subdivide communicator
|
||||
//////////////////////////////////
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank)
|
||||
{
|
||||
_ndimension = processors.size(); assert(_ndimension>=1);
|
||||
int parent_ndimension = parent._ndimension; assert(_ndimension >= parent._ndimension);
|
||||
Coordinate parent_processor_coor(_ndimension,0);
|
||||
Coordinate parent_processors (_ndimension,1);
|
||||
Coordinate shm_processors (_ndimension,1);
|
||||
// Can make 5d grid from 4d etc...
|
||||
int pad = _ndimension-parent_ndimension;
|
||||
for(int d=0;d<parent_ndimension;d++){
|
||||
parent_processor_coor[pad+d]=parent._processor_coor[d];
|
||||
parent_processors [pad+d]=parent._processors[d];
|
||||
shm_processors [pad+d]=parent._shm_processors[d];
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// split the communicator
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// int Nparent = parent._processors ;
|
||||
int Nparent;
|
||||
MPI_Comm_size(parent.communicator,&Nparent);
|
||||
|
||||
int childsize=1;
|
||||
for(int d=0;d<processors.size();d++) {
|
||||
childsize *= processors[d];
|
||||
}
|
||||
int Nchild = Nparent/childsize;
|
||||
assert (childsize * Nchild == Nparent);
|
||||
|
||||
Coordinate ccoor(_ndimension); // coor within subcommunicator
|
||||
Coordinate scoor(_ndimension); // coor of split within parent
|
||||
Coordinate ssize(_ndimension); // coor of split within parent
|
||||
|
||||
for(int d=0;d<_ndimension;d++){
|
||||
ccoor[d] = parent_processor_coor[d] % processors[d];
|
||||
scoor[d] = parent_processor_coor[d] / processors[d];
|
||||
ssize[d] = parent_processors[d] / processors[d];
|
||||
if ( processors[d] < shm_processors[d] ) shm_processors[d] = processors[d]; // subnode splitting.
|
||||
}
|
||||
|
||||
// rank within subcomm ; srank is rank of subcomm within blocks of subcomms
|
||||
int crank;
|
||||
// Mpi uses the reverse Lexico convention to us; so reversed routines called
|
||||
Lexicographic::IndexFromCoorReversed(ccoor,crank,processors); // processors is the split grid dimensions
|
||||
Lexicographic::IndexFromCoorReversed(scoor,srank,ssize); // ssize is the number of split grids
|
||||
|
||||
MPI_Comm comm_split;
|
||||
if ( Nchild > 1 ) {
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Split the communicator
|
||||
////////////////////////////////////////////////////////////////
|
||||
int ierr= MPI_Comm_split(parent.communicator,srank,crank,&comm_split);
|
||||
assert(ierr==0);
|
||||
|
||||
} else {
|
||||
srank = 0;
|
||||
int ierr = MPI_Comm_dup (parent.communicator,&comm_split);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Set up from the new split communicator
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
InitFromMPICommunicator(processors,comm_split);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Take the right SHM buffers
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
SetCommunicator(comm_split);
|
||||
|
||||
///////////////////////////////////////////////
|
||||
// Free the temp communicator
|
||||
///////////////////////////////////////////////
|
||||
MPI_Comm_free(&comm_split);
|
||||
|
||||
if(0){
|
||||
std::cout << " ndim " <<_ndimension<<" " << parent._ndimension << std::endl;
|
||||
for(int d=0;d<processors.size();d++){
|
||||
std::cout << d<< " " << _processor_coor[d] <<" " << ccoor[d]<<std::endl;
|
||||
}
|
||||
}
|
||||
for(int d=0;d<processors.size();d++){
|
||||
assert(_processor_coor[d] == ccoor[d] );
|
||||
}
|
||||
}
|
||||
|
||||
void CartesianCommunicator::InitFromMPICommunicator(const Coordinate &processors, MPI_Comm communicator_base)
|
||||
{
|
||||
////////////////////////////////////////////////////
|
||||
// Creates communicator, and the communicator_halo
|
||||
////////////////////////////////////////////////////
|
||||
_ndimension = processors.size();
|
||||
_processor_coor.resize(_ndimension);
|
||||
|
||||
/////////////////////////////////
|
||||
// Count the requested nodes
|
||||
/////////////////////////////////
|
||||
_Nprocessors=1;
|
||||
_processors = processors;
|
||||
for(int i=0;i<_ndimension;i++){
|
||||
_Nprocessors*=_processors[i];
|
||||
}
|
||||
|
||||
Coordinate periodic(_ndimension,1);
|
||||
MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],0,&communicator);
|
||||
MPI_Comm_rank(communicator,&_processor);
|
||||
MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
|
||||
|
||||
if ( 0 && (communicator_base != communicator_world) ) {
|
||||
std::cout << "InitFromMPICommunicator Cartesian communicator created with a non-world communicator"<<std::endl;
|
||||
std::cout << " new communicator rank "<<_processor<< " coor ["<<_ndimension<<"] ";
|
||||
for(int d=0;d<_processors.size();d++){
|
||||
std::cout << _processor_coor[d]<<" ";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
int Size;
|
||||
MPI_Comm_size(communicator,&Size);
|
||||
|
||||
communicator_halo.resize (2*_ndimension);
|
||||
for(int i=0;i<_ndimension*2;i++){
|
||||
MPI_Comm_dup(communicator,&communicator_halo[i]);
|
||||
}
|
||||
assert(Size==_Nprocessors);
|
||||
}
|
||||
|
||||
CartesianCommunicator::~CartesianCommunicator()
|
||||
{
|
||||
int MPI_is_finalised;
|
||||
MPI_Finalized(&MPI_is_finalised);
|
||||
if (communicator && !MPI_is_finalised) {
|
||||
MPI_Comm_free(&communicator);
|
||||
for(int i=0;i<communicator_halo.size();i++){
|
||||
MPI_Comm_free(&communicator_halo[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(uint32_t &u){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(uint64_t &u){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(uint64_t* u,int N){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,u,N,MPI_UINT64_T,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalXOR(uint32_t &u){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalXOR(uint64_t &u){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalMax(float &f)
|
||||
{
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_MAX,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalMax(double &d)
|
||||
{
|
||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_MAX,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(float &f){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(float *f,int N)
|
||||
{
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(double &d)
|
||||
{
|
||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(double *d,int N)
|
||||
{
|
||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
// Basic Halo comms primitive
|
||||
void CartesianCommunicator::SendToRecvFrom(void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
int bytes)
|
||||
{
|
||||
std::vector<CommsRequest_t> reqs(0);
|
||||
unsigned long xcrc = crc32(0L, Z_NULL, 0);
|
||||
unsigned long rcrc = crc32(0L, Z_NULL, 0);
|
||||
|
||||
int myrank = _processor;
|
||||
int ierr;
|
||||
|
||||
// Enforce no UVM in comms, device or host OK
|
||||
assert(acceleratorIsCommunicable(xmit));
|
||||
assert(acceleratorIsCommunicable(recv));
|
||||
|
||||
// Give the CPU to MPI immediately; can use threads to overlap optionally
|
||||
// printf("proc %d SendToRecvFrom %d bytes Sendrecv \n",_processor,bytes);
|
||||
ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,dest,myrank,
|
||||
recv,bytes,MPI_CHAR,from, from,
|
||||
communicator,MPI_STATUS_IGNORE);
|
||||
assert(ierr==0);
|
||||
|
||||
// xcrc = crc32(xcrc,(unsigned char *)xmit,bytes);
|
||||
// rcrc = crc32(rcrc,(unsigned char *)recv,bytes);
|
||||
// printf("proc %d SendToRecvFrom %d bytes xcrc %lx rcrc %lx\n",_processor,bytes,xcrc,rcrc); fflush
|
||||
}
|
||||
// Basic Halo comms primitive
|
||||
double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
|
||||
int dest, int dox,
|
||||
void *recv,
|
||||
int from, int dor,
|
||||
int bytes,int dir)
|
||||
{
|
||||
std::vector<CommsRequest_t> list;
|
||||
double offbytes = StencilSendToRecvFromBegin(list,xmit,dest,dox,recv,from,dor,bytes,bytes,dir);
|
||||
StencilSendToRecvFromComplete(list,dir);
|
||||
return offbytes;
|
||||
}
|
||||
|
||||
double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int dest,int dox,
|
||||
void *recv,
|
||||
int from,int dor,
|
||||
int xbytes,int rbytes,int dir)
|
||||
{
|
||||
int ncomm =communicator_halo.size();
|
||||
int commdir=dir%ncomm;
|
||||
|
||||
MPI_Request xrq;
|
||||
MPI_Request rrq;
|
||||
|
||||
int ierr;
|
||||
int gdest = ShmRanks[dest];
|
||||
int gfrom = ShmRanks[from];
|
||||
int gme = ShmRanks[_processor];
|
||||
|
||||
assert(dest != _processor);
|
||||
assert(from != _processor);
|
||||
assert(gme == ShmRank);
|
||||
double off_node_bytes=0.0;
|
||||
int tag;
|
||||
|
||||
if ( dor ) {
|
||||
if ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
tag= dir+from*32;
|
||||
ierr=MPI_Irecv(recv, rbytes, MPI_CHAR,from,tag,communicator_halo[commdir],&rrq);
|
||||
assert(ierr==0);
|
||||
list.push_back(rrq);
|
||||
off_node_bytes+=rbytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (dox) {
|
||||
if ( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
tag= dir+_processor*32;
|
||||
ierr =MPI_Isend(xmit, xbytes, MPI_CHAR,dest,tag,communicator_halo[commdir],&xrq);
|
||||
assert(ierr==0);
|
||||
list.push_back(xrq);
|
||||
off_node_bytes+=xbytes;
|
||||
} else {
|
||||
void *shm = (void *) this->ShmBufferTranslate(dest,recv);
|
||||
assert(shm!=NULL);
|
||||
acceleratorCopyDeviceToDeviceAsynch(xmit,shm,xbytes);
|
||||
}
|
||||
}
|
||||
|
||||
return off_node_bytes;
|
||||
}
|
||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
|
||||
{
|
||||
int nreq=list.size();
|
||||
|
||||
if (nreq==0) return;
|
||||
|
||||
std::vector<MPI_Status> status(nreq);
|
||||
int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
|
||||
assert(ierr==0);
|
||||
list.resize(0);
|
||||
}
|
||||
void CartesianCommunicator::StencilBarrier(void)
|
||||
{
|
||||
MPI_Barrier (ShmComm);
|
||||
}
|
||||
//void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
|
||||
//{
|
||||
//}
|
||||
void CartesianCommunicator::Barrier(void)
|
||||
{
|
||||
int ierr = MPI_Barrier(communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
|
||||
{
|
||||
int ierr=MPI_Bcast(data,
|
||||
bytes,
|
||||
MPI_BYTE,
|
||||
root,
|
||||
communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
int CartesianCommunicator::RankWorld(void){
|
||||
int r;
|
||||
MPI_Comm_rank(communicator_world,&r);
|
||||
return r;
|
||||
}
|
||||
void CartesianCommunicator::BarrierWorld(void){
|
||||
int ierr = MPI_Barrier(communicator_world);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
|
||||
{
|
||||
int ierr= MPI_Bcast(data,
|
||||
bytes,
|
||||
MPI_BYTE,
|
||||
root,
|
||||
communicator_world);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words,uint64_t bytes)
|
||||
{
|
||||
Coordinate row(_ndimension,1);
|
||||
assert(dim>=0 && dim<_ndimension);
|
||||
|
||||
// Split the communicator
|
||||
row[dim] = _processors[dim];
|
||||
|
||||
int me;
|
||||
CartesianCommunicator Comm(row,*this,me);
|
||||
Comm.AllToAll(in,out,words,bytes);
|
||||
}
|
||||
void CartesianCommunicator::AllToAll(void *in,void *out,uint64_t words,uint64_t bytes)
|
||||
{
|
||||
// MPI is a pain and uses "int" arguments
|
||||
// 64*64*64*128*16 == 500Million elements of data.
|
||||
// When 24*4 bytes multiples get 50x 10^9 >>> 2x10^9 Y2K bug.
|
||||
// (Turns up on 32^3 x 64 Gparity too)
|
||||
MPI_Datatype object;
|
||||
int iwords;
|
||||
int ibytes;
|
||||
iwords = words;
|
||||
ibytes = bytes;
|
||||
assert(words == iwords); // safe to cast to int ?
|
||||
assert(bytes == ibytes); // safe to cast to int ?
|
||||
MPI_Type_contiguous(ibytes,MPI_BYTE,&object);
|
||||
MPI_Type_commit(&object);
|
||||
MPI_Alltoall(in,iwords,object,out,iwords,object,communicator);
|
||||
MPI_Type_free(&object);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,141 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/Communicator_none.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Info that is setup once and indept of cartesian layout
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Grid_MPI_Comm CartesianCommunicator::communicator_world;
|
||||
|
||||
void CartesianCommunicator::Init(int *argc, char *** arv)
|
||||
{
|
||||
GlobalSharedMemory::Init(communicator_world);
|
||||
GlobalSharedMemory::SharedMemoryAllocate(
|
||||
GlobalSharedMemory::MAX_MPI_SHM_BYTES,
|
||||
GlobalSharedMemory::Hugepages);
|
||||
}
|
||||
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank)
|
||||
: CartesianCommunicator(processors)
|
||||
{
|
||||
_shm_processors = Coordinate(processors.size(),1);
|
||||
srank=0;
|
||||
SetCommunicator(communicator_world);
|
||||
}
|
||||
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
||||
{
|
||||
_shm_processors = Coordinate(processors.size(),1);
|
||||
_processors = processors;
|
||||
_ndimension = processors.size(); assert(_ndimension>=1);
|
||||
_processor_coor.resize(_ndimension);
|
||||
|
||||
// Require 1^N processor grid for fake
|
||||
_Nprocessors=1;
|
||||
_processor = 0;
|
||||
for(int d=0;d<_ndimension;d++) {
|
||||
assert(_processors[d]==1);
|
||||
_processor_coor[d] = 0;
|
||||
}
|
||||
SetCommunicator(communicator_world);
|
||||
}
|
||||
|
||||
CartesianCommunicator::~CartesianCommunicator(){}
|
||||
|
||||
void CartesianCommunicator::GlobalMax(float &){}
|
||||
void CartesianCommunicator::GlobalMax(double &){}
|
||||
void CartesianCommunicator::GlobalSum(float &){}
|
||||
void CartesianCommunicator::GlobalSumVector(float *,int N){}
|
||||
void CartesianCommunicator::GlobalSum(double &){}
|
||||
void CartesianCommunicator::GlobalSumVector(double *,int N){}
|
||||
void CartesianCommunicator::GlobalSum(uint32_t &){}
|
||||
void CartesianCommunicator::GlobalSum(uint64_t &){}
|
||||
void CartesianCommunicator::GlobalSumVector(uint64_t *,int N){}
|
||||
void CartesianCommunicator::GlobalXOR(uint32_t &){}
|
||||
void CartesianCommunicator::GlobalXOR(uint64_t &){}
|
||||
|
||||
|
||||
// Basic Halo comms primitive -- should never call in single node
|
||||
void CartesianCommunicator::SendToRecvFrom(void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
int bytes)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words,uint64_t bytes)
|
||||
{
|
||||
bcopy(in,out,bytes*words);
|
||||
}
|
||||
void CartesianCommunicator::AllToAll(void *in,void *out,uint64_t words,uint64_t bytes)
|
||||
{
|
||||
bcopy(in,out,bytes*words);
|
||||
}
|
||||
|
||||
int CartesianCommunicator::RankWorld(void){return 0;}
|
||||
void CartesianCommunicator::Barrier(void){}
|
||||
void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {}
|
||||
void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { }
|
||||
void CartesianCommunicator::BarrierWorld(void) { }
|
||||
int CartesianCommunicator::RankFromProcessorCoor(Coordinate &coor) { return 0;}
|
||||
void CartesianCommunicator::ProcessorCoorFromRank(int rank, Coordinate &coor){ coor = _processor_coor; }
|
||||
void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
|
||||
{
|
||||
source =0;
|
||||
dest=0;
|
||||
}
|
||||
|
||||
double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
|
||||
int xmit_to_rank,int dox,
|
||||
void *recv,
|
||||
int recv_from_rank,int dor,
|
||||
int bytes, int dir)
|
||||
{
|
||||
return 2.0*bytes;
|
||||
}
|
||||
double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int xmit_to_rank,int dox,
|
||||
void *recv,
|
||||
int recv_from_rank,int dor,
|
||||
int xbytes,int rbytes, int dir)
|
||||
{
|
||||
return xbytes+rbytes;
|
||||
}
|
||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
|
||||
{
|
||||
}
|
||||
|
||||
void CartesianCommunicator::StencilBarrier(void){};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/SharedMemory.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
// static data
|
||||
|
||||
int GlobalSharedMemory::HPEhypercube = 1;
|
||||
uint64_t GlobalSharedMemory::MAX_MPI_SHM_BYTES = 1024LL*1024LL*1024LL;
|
||||
int GlobalSharedMemory::Hugepages = 0;
|
||||
int GlobalSharedMemory::_ShmSetup;
|
||||
int GlobalSharedMemory::_ShmAlloc;
|
||||
uint64_t GlobalSharedMemory::_ShmAllocBytes;
|
||||
|
||||
std::vector<void *> GlobalSharedMemory::WorldShmCommBufs;
|
||||
|
||||
Grid_MPI_Comm GlobalSharedMemory::WorldShmComm;
|
||||
int GlobalSharedMemory::WorldShmRank;
|
||||
int GlobalSharedMemory::WorldShmSize;
|
||||
std::vector<int> GlobalSharedMemory::WorldShmRanks;
|
||||
|
||||
Grid_MPI_Comm GlobalSharedMemory::WorldComm;
|
||||
int GlobalSharedMemory::WorldSize;
|
||||
int GlobalSharedMemory::WorldRank;
|
||||
|
||||
int GlobalSharedMemory::WorldNodes;
|
||||
int GlobalSharedMemory::WorldNode;
|
||||
|
||||
void GlobalSharedMemory::SharedMemoryFree(void)
|
||||
{
|
||||
assert(_ShmAlloc);
|
||||
assert(_ShmAllocBytes>0);
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
munmap(WorldShmCommBufs[r],_ShmAllocBytes);
|
||||
}
|
||||
_ShmAlloc = 0;
|
||||
_ShmAllocBytes = 0;
|
||||
}
|
||||
/////////////////////////////////
|
||||
// Alloc, free shmem region
|
||||
/////////////////////////////////
|
||||
void *SharedMemory::ShmBufferMalloc(size_t bytes){
|
||||
// bytes = (bytes+sizeof(vRealD))&(~(sizeof(vRealD)-1));// align up bytes
|
||||
void *ptr = (void *)heap_top;
|
||||
heap_top += bytes;
|
||||
heap_bytes+= bytes;
|
||||
if (heap_bytes >= heap_size) {
|
||||
std::cout<< " ShmBufferMalloc exceeded shared heap size -- try increasing with --shm <MB> flag" <<std::endl;
|
||||
std::cout<< " Parameter specified in units of MB (megabytes) " <<std::endl;
|
||||
std::cout<< " Current alloc is " << (bytes/(1024*1024)) <<"MB"<<std::endl;
|
||||
std::cout<< " Current bytes is " << (heap_bytes/(1024*1024)) <<"MB"<<std::endl;
|
||||
std::cout<< " Current heap is " << (heap_size/(1024*1024)) <<"MB"<<std::endl;
|
||||
assert(heap_bytes<heap_size);
|
||||
}
|
||||
//std::cerr << "ShmBufferMalloc "<<std::hex<< ptr<<" - "<<((uint64_t)ptr+bytes)<<std::dec<<std::endl;
|
||||
return ptr;
|
||||
}
|
||||
void SharedMemory::ShmBufferFreeAll(void) {
|
||||
heap_top =(size_t)ShmBufferSelf();
|
||||
heap_bytes=0;
|
||||
}
|
||||
void *SharedMemory::ShmBufferSelf(void)
|
||||
{
|
||||
//std::cerr << "ShmBufferSelf "<<ShmRank<<" "<<std::hex<< ShmCommBufs[ShmRank] <<std::dec<<std::endl;
|
||||
return ShmCommBufs[ShmRank];
|
||||
}
|
||||
static inline int divides(int a,int b)
|
||||
{
|
||||
return ( b == ( (b/a)*a ) );
|
||||
}
|
||||
void GlobalSharedMemory::GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Allow user to configure through environment variable
|
||||
////////////////////////////////////////////////////////////////
|
||||
char* str = getenv(("GRID_SHM_DIMS_" + std::to_string(ShmDims.size())).c_str());
|
||||
if ( str ) {
|
||||
std::vector<int> IntShmDims;
|
||||
GridCmdOptionIntVector(std::string(str),IntShmDims);
|
||||
assert(IntShmDims.size() == WorldDims.size());
|
||||
long ShmSize = 1;
|
||||
for (int dim=0;dim<WorldDims.size();dim++) {
|
||||
ShmSize *= (ShmDims[dim] = IntShmDims[dim]);
|
||||
assert(divides(ShmDims[dim],WorldDims[dim]));
|
||||
}
|
||||
assert(ShmSize == WorldShmSize);
|
||||
return;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Powers of 2,3,5 only in prime decomposition for now
|
||||
////////////////////////////////////////////////////////////////
|
||||
int ndimension = WorldDims.size();
|
||||
ShmDims=Coordinate(ndimension,1);
|
||||
|
||||
std::vector<int> primes({2,3,5});
|
||||
|
||||
int dim = 0;
|
||||
int last_dim = ndimension - 1;
|
||||
int AutoShmSize = 1;
|
||||
while(AutoShmSize != WorldShmSize) {
|
||||
int p;
|
||||
for(p=0;p<primes.size();p++) {
|
||||
int prime=primes[p];
|
||||
if ( divides(prime,WorldDims[dim]/ShmDims[dim])
|
||||
&& divides(prime,WorldShmSize/AutoShmSize) ) {
|
||||
AutoShmSize*=prime;
|
||||
ShmDims[dim]*=prime;
|
||||
last_dim = dim;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (p == primes.size() && last_dim == dim) {
|
||||
std::cerr << "GlobalSharedMemory::GetShmDims failed" << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
dim=(dim+1) %ndimension;
|
||||
}
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/SharedMemory.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
#if defined (GRID_COMMS_MPI3)
|
||||
#include <mpi.h>
|
||||
#endif
|
||||
#include <semaphore.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <limits.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/ipc.h>
|
||||
#include <sys/shm.h>
|
||||
#include <sys/mman.h>
|
||||
#include <zlib.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#if defined (GRID_COMMS_MPI3)
|
||||
typedef MPI_Comm Grid_MPI_Comm;
|
||||
typedef MPI_Request CommsRequest_t;
|
||||
#else
|
||||
typedef int CommsRequest_t;
|
||||
typedef int Grid_MPI_Comm;
|
||||
#endif
|
||||
|
||||
class GlobalSharedMemory {
|
||||
private:
|
||||
static const int MAXLOG2RANKSPERNODE = 16;
|
||||
|
||||
|
||||
// Init once lock on the buffer allocation
|
||||
static int _ShmSetup;
|
||||
static int _ShmAlloc;
|
||||
static uint64_t _ShmAllocBytes;
|
||||
|
||||
public:
|
||||
///////////////////////////////////////
|
||||
// HPE 8600 hypercube optimisation
|
||||
///////////////////////////////////////
|
||||
static int HPEhypercube;
|
||||
|
||||
static int ShmSetup(void) { return _ShmSetup; }
|
||||
static int ShmAlloc(void) { return _ShmAlloc; }
|
||||
static uint64_t ShmAllocBytes(void) { return _ShmAllocBytes; }
|
||||
static uint64_t MAX_MPI_SHM_BYTES;
|
||||
static int Hugepages;
|
||||
|
||||
static std::vector<void *> WorldShmCommBufs;
|
||||
|
||||
static Grid_MPI_Comm WorldComm;
|
||||
static int WorldRank;
|
||||
static int WorldSize;
|
||||
|
||||
static Grid_MPI_Comm WorldShmComm;
|
||||
static int WorldShmRank;
|
||||
static int WorldShmSize;
|
||||
|
||||
static int WorldNodes;
|
||||
static int WorldNode;
|
||||
|
||||
static std::vector<int> WorldShmRanks;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Create an optimal reordered communicator that makes MPI_Cart_create get it right
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
static void Init(Grid_MPI_Comm comm); // Typically MPI_COMM_WORLD
|
||||
// Turns MPI_COMM_WORLD into right layout for Cartesian
|
||||
static void OptimalCommunicator (const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims);
|
||||
static void OptimalCommunicatorHypercube (const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims);
|
||||
static void OptimalCommunicatorSharedMemory(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims);
|
||||
static void GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims);
|
||||
///////////////////////////////////////////////////
|
||||
// Provide shared memory facilities off comm world
|
||||
///////////////////////////////////////////////////
|
||||
static void SharedMemoryAllocate(uint64_t bytes, int flags);
|
||||
static void SharedMemoryFree(void);
|
||||
static void SharedMemoryCopy(void *dest,void *src,size_t bytes);
|
||||
static void SharedMemoryZero(void *dest,size_t bytes);
|
||||
|
||||
};
|
||||
|
||||
//////////////////////////////
|
||||
// one per communicator
|
||||
//////////////////////////////
|
||||
class SharedMemory
|
||||
{
|
||||
private:
|
||||
static const int MAXLOG2RANKSPERNODE = 16;
|
||||
|
||||
size_t heap_top;
|
||||
size_t heap_bytes;
|
||||
size_t heap_size;
|
||||
|
||||
protected:
|
||||
|
||||
Grid_MPI_Comm ShmComm; // for barriers
|
||||
int ShmRank;
|
||||
int ShmSize;
|
||||
std::vector<void *> ShmCommBufs;
|
||||
std::vector<int> ShmRanks;// Mapping comm ranks to Shm ranks
|
||||
|
||||
public:
|
||||
SharedMemory() {};
|
||||
~SharedMemory();
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
// set the buffers & sizes
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
void SetCommunicator(Grid_MPI_Comm comm);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// For this instance ; disjoint buffer sets between splits if split grid
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
void ShmBarrier(void);
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// Call on any instance
|
||||
///////////////////////////////////////////////////
|
||||
void SharedMemoryTest(void);
|
||||
|
||||
void *ShmBufferSelf(void);
|
||||
void *ShmBuffer (int rank);
|
||||
void *ShmBufferTranslate(int rank,void * local_p);
|
||||
void *ShmBufferMalloc(size_t bytes);
|
||||
void ShmBufferFreeAll(void) ;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Make info on Nodes & ranks and Shared memory available
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
int NodeCount(void) { return GlobalSharedMemory::WorldNodes;};
|
||||
int RankCount(void) { return GlobalSharedMemory::WorldSize;};
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,920 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/SharedMemory.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christoph Lehner <christoph@lhnr.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#include <Grid/GridCore.h>
|
||||
#include <pwd.h>
|
||||
#include <syscall.h>
|
||||
|
||||
#ifdef GRID_CUDA
|
||||
#include <cuda_runtime_api.h>
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
#include <hip/hip_runtime_api.h>
|
||||
#endif
|
||||
#ifdef GRID_SYCL
|
||||
#define GRID_SYCL_LEVEL_ZERO_IPC
|
||||
#endif
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
#define header "SharedMemoryMpi: "
|
||||
/*Construct from an MPI communicator*/
|
||||
void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
{
|
||||
assert(_ShmSetup==0);
|
||||
WorldComm = comm;
|
||||
MPI_Comm_rank(WorldComm,&WorldRank);
|
||||
MPI_Comm_size(WorldComm,&WorldSize);
|
||||
// WorldComm, WorldSize, WorldRank
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Split into groups that can share memory
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
#ifndef GRID_MPI3_SHM_NONE
|
||||
MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&WorldShmComm);
|
||||
#else
|
||||
MPI_Comm_split(comm, WorldRank, 0, &WorldShmComm);
|
||||
#endif
|
||||
|
||||
MPI_Comm_rank(WorldShmComm ,&WorldShmRank);
|
||||
MPI_Comm_size(WorldShmComm ,&WorldShmSize);
|
||||
|
||||
if ( WorldRank == 0) {
|
||||
std::cout << header " World communicator of size " <<WorldSize << std::endl;
|
||||
std::cout << header " Node communicator of size " <<WorldShmSize << std::endl;
|
||||
}
|
||||
// WorldShmComm, WorldShmSize, WorldShmRank
|
||||
|
||||
// WorldNodes
|
||||
WorldNodes = WorldSize/WorldShmSize;
|
||||
assert( (WorldNodes * WorldShmSize) == WorldSize );
|
||||
|
||||
|
||||
// FIXME: Check all WorldShmSize are the same ?
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// find world ranks in our SHM group (i.e. which ranks are on our node)
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
MPI_Group WorldGroup, ShmGroup;
|
||||
MPI_Comm_group (WorldComm, &WorldGroup);
|
||||
MPI_Comm_group (WorldShmComm, &ShmGroup);
|
||||
|
||||
std::vector<int> world_ranks(WorldSize); for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
|
||||
|
||||
WorldShmRanks.resize(WorldSize);
|
||||
MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &WorldShmRanks[0]);
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// Identify who is in my group and nominate the leader
|
||||
///////////////////////////////////////////////////////////////////
|
||||
int g=0;
|
||||
std::vector<int> MyGroup;
|
||||
MyGroup.resize(WorldShmSize);
|
||||
for(int rank=0;rank<WorldSize;rank++){
|
||||
if(WorldShmRanks[rank]!=MPI_UNDEFINED){
|
||||
assert(g<WorldShmSize);
|
||||
MyGroup[g++] = rank;
|
||||
}
|
||||
}
|
||||
|
||||
std::sort(MyGroup.begin(),MyGroup.end(),std::less<int>());
|
||||
int myleader = MyGroup[0];
|
||||
|
||||
std::vector<int> leaders_1hot(WorldSize,0);
|
||||
std::vector<int> leaders_group(WorldNodes,0);
|
||||
leaders_1hot [ myleader ] = 1;
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// global sum leaders over comm world
|
||||
///////////////////////////////////////////////////////////////////
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,WorldComm);
|
||||
assert(ierr==0);
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// find the group leaders world rank
|
||||
///////////////////////////////////////////////////////////////////
|
||||
int group=0;
|
||||
for(int l=0;l<WorldSize;l++){
|
||||
if(leaders_1hot[l]){
|
||||
leaders_group[group++] = l;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// Identify the node of the group in which I (and my leader) live
|
||||
///////////////////////////////////////////////////////////////////
|
||||
WorldNode=-1;
|
||||
for(int g=0;g<WorldNodes;g++){
|
||||
if (myleader == leaders_group[g]){
|
||||
WorldNode=g;
|
||||
}
|
||||
}
|
||||
assert(WorldNode!=-1);
|
||||
_ShmSetup=1;
|
||||
}
|
||||
// Gray encode support
|
||||
int BinaryToGray (int binary) {
|
||||
int gray = (binary>>1)^binary;
|
||||
return gray;
|
||||
}
|
||||
int Log2Size(int TwoToPower,int MAXLOG2)
|
||||
{
|
||||
int log2size = -1;
|
||||
for(int i=0;i<=MAXLOG2;i++){
|
||||
if ( (0x1<<i) == TwoToPower ) {
|
||||
log2size = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return log2size;
|
||||
}
|
||||
void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// Look and see if it looks like an HPE 8600 based on hostname conventions
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
const int namelen = _POSIX_HOST_NAME_MAX;
|
||||
char name[namelen];
|
||||
int R;
|
||||
int I;
|
||||
int N;
|
||||
gethostname(name,namelen);
|
||||
int nscan = sscanf(name,"r%di%dn%d",&R,&I,&N) ;
|
||||
|
||||
if(nscan==3 && HPEhypercube ) OptimalCommunicatorHypercube(processors,optimal_comm,SHM);
|
||||
else OptimalCommunicatorSharedMemory(processors,optimal_comm,SHM);
|
||||
}
|
||||
|
||||
void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Assert power of two shm_size.
|
||||
////////////////////////////////////////////////////////////////
|
||||
int log2size = Log2Size(WorldShmSize,MAXLOG2RANKSPERNODE);
|
||||
assert(log2size != -1);
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Identify the hypercube coordinate of this node using hostname
|
||||
////////////////////////////////////////////////////////////////
|
||||
// n runs 0...7 9...16 18...25 27...34 (8*4) 5 bits
|
||||
// i runs 0..7 3 bits
|
||||
// r runs 0..3 2 bits
|
||||
// 2^10 = 1024 nodes
|
||||
const int maxhdim = 10;
|
||||
std::vector<int> HyperCubeCoords(maxhdim,0);
|
||||
std::vector<int> RootHyperCubeCoords(maxhdim,0);
|
||||
int R;
|
||||
int I;
|
||||
int N;
|
||||
const int namelen = _POSIX_HOST_NAME_MAX;
|
||||
char name[namelen];
|
||||
|
||||
// Parse ICE-XA hostname to get hypercube location
|
||||
gethostname(name,namelen);
|
||||
int nscan = sscanf(name,"r%di%dn%d",&R,&I,&N) ;
|
||||
assert(nscan==3);
|
||||
|
||||
int nlo = N%9;
|
||||
int nhi = N/9;
|
||||
uint32_t hypercoor = (R<<8)|(I<<5)|(nhi<<3)|nlo ;
|
||||
uint32_t rootcoor = hypercoor;
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// Print debug info
|
||||
//////////////////////////////////////////////////////////////////
|
||||
for(int d=0;d<maxhdim;d++){
|
||||
HyperCubeCoords[d] = (hypercoor>>d)&0x1;
|
||||
}
|
||||
|
||||
std::string hname(name);
|
||||
// std::cout << "hostname "<<hname<<std::endl;
|
||||
// std::cout << "R " << R << " I " << I << " N "<< N
|
||||
// << " hypercoor 0x"<<std::hex<<hypercoor<<std::dec<<std::endl;
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// broadcast node 0's base coordinate for this partition.
|
||||
//////////////////////////////////////////////////////////////////
|
||||
MPI_Bcast(&rootcoor, sizeof(rootcoor), MPI_BYTE, 0, WorldComm);
|
||||
hypercoor=hypercoor-rootcoor;
|
||||
assert(hypercoor<WorldSize);
|
||||
assert(hypercoor>=0);
|
||||
|
||||
//////////////////////////////////////
|
||||
// Printing
|
||||
//////////////////////////////////////
|
||||
for(int d=0;d<maxhdim;d++){
|
||||
HyperCubeCoords[d] = (hypercoor>>d)&0x1;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Identify subblock of ranks on node spreading across dims
|
||||
// in a maximally symmetrical way
|
||||
////////////////////////////////////////////////////////////////
|
||||
int ndimension = processors.size();
|
||||
Coordinate processor_coor(ndimension);
|
||||
Coordinate WorldDims = processors;
|
||||
Coordinate ShmDims (ndimension); Coordinate NodeDims (ndimension);
|
||||
Coordinate ShmCoor (ndimension); Coordinate NodeCoor (ndimension); Coordinate WorldCoor(ndimension);
|
||||
Coordinate HyperCoor(ndimension);
|
||||
|
||||
GetShmDims(WorldDims,ShmDims);
|
||||
SHM = ShmDims;
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish torus of processes and nodes with sub-blockings
|
||||
////////////////////////////////////////////////////////////////
|
||||
for(int d=0;d<ndimension;d++){
|
||||
NodeDims[d] = WorldDims[d]/ShmDims[d];
|
||||
}
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Map Hcube according to physical lattice
|
||||
// must partition. Loop over dims and find out who would join.
|
||||
////////////////////////////////////////////////////////////////
|
||||
int hcoor = hypercoor;
|
||||
for(int d=0;d<ndimension;d++){
|
||||
int bits = Log2Size(NodeDims[d],MAXLOG2RANKSPERNODE);
|
||||
int msk = (0x1<<bits)-1;
|
||||
HyperCoor[d]=hcoor & msk;
|
||||
HyperCoor[d]=BinaryToGray(HyperCoor[d]); // Space filling curve magic
|
||||
hcoor = hcoor >> bits;
|
||||
}
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Check processor counts match
|
||||
////////////////////////////////////////////////////////////////
|
||||
int Nprocessors=1;
|
||||
for(int i=0;i<ndimension;i++){
|
||||
Nprocessors*=processors[i];
|
||||
}
|
||||
assert(WorldSize==Nprocessors);
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish mapping between lexico physics coord and WorldRank
|
||||
////////////////////////////////////////////////////////////////
|
||||
int rank;
|
||||
|
||||
Lexicographic::CoorFromIndexReversed(NodeCoor,WorldNode ,NodeDims);
|
||||
|
||||
for(int d=0;d<ndimension;d++) NodeCoor[d]=HyperCoor[d];
|
||||
|
||||
Lexicographic::CoorFromIndexReversed(ShmCoor ,WorldShmRank,ShmDims);
|
||||
for(int d=0;d<ndimension;d++) WorldCoor[d] = NodeCoor[d]*ShmDims[d]+ShmCoor[d];
|
||||
Lexicographic::IndexFromCoorReversed(WorldCoor,rank,WorldDims);
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Build the new communicator
|
||||
/////////////////////////////////////////////////////////////////
|
||||
int ierr= MPI_Comm_split(WorldComm,0,rank,&optimal_comm);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Identify subblock of ranks on node spreading across dims
|
||||
// in a maximally symmetrical way
|
||||
////////////////////////////////////////////////////////////////
|
||||
int ndimension = processors.size();
|
||||
Coordinate processor_coor(ndimension);
|
||||
Coordinate WorldDims = processors; Coordinate ShmDims(ndimension); Coordinate NodeDims (ndimension);
|
||||
Coordinate ShmCoor(ndimension); Coordinate NodeCoor(ndimension); Coordinate WorldCoor(ndimension);
|
||||
|
||||
GetShmDims(WorldDims,ShmDims);
|
||||
SHM=ShmDims;
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish torus of processes and nodes with sub-blockings
|
||||
////////////////////////////////////////////////////////////////
|
||||
for(int d=0;d<ndimension;d++){
|
||||
NodeDims[d] = WorldDims[d]/ShmDims[d];
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Check processor counts match
|
||||
////////////////////////////////////////////////////////////////
|
||||
int Nprocessors=1;
|
||||
for(int i=0;i<ndimension;i++){
|
||||
Nprocessors*=processors[i];
|
||||
}
|
||||
assert(WorldSize==Nprocessors);
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish mapping between lexico physics coord and WorldRank
|
||||
////////////////////////////////////////////////////////////////
|
||||
int rank;
|
||||
|
||||
Lexicographic::CoorFromIndexReversed(NodeCoor,WorldNode ,NodeDims);
|
||||
Lexicographic::CoorFromIndexReversed(ShmCoor ,WorldShmRank,ShmDims);
|
||||
for(int d=0;d<ndimension;d++) WorldCoor[d] = NodeCoor[d]*ShmDims[d]+ShmCoor[d];
|
||||
Lexicographic::IndexFromCoorReversed(WorldCoor,rank,WorldDims);
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Build the new communicator
|
||||
/////////////////////////////////////////////////////////////////
|
||||
int ierr= MPI_Comm_split(WorldComm,0,rank,&optimal_comm);
|
||||
assert(ierr==0);
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// SHMGET
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef GRID_MPI3_SHMGET
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " shmget implementation "<<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
std::vector<int> shmids(WorldShmSize);
|
||||
|
||||
if ( WorldShmRank == 0 ) {
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
size_t size = bytes;
|
||||
key_t key = IPC_PRIVATE;
|
||||
int flags = IPC_CREAT | SHM_R | SHM_W;
|
||||
#ifdef SHM_HUGETLB
|
||||
if (Hugepages) flags|=SHM_HUGETLB;
|
||||
#endif
|
||||
if ((shmids[r]= shmget(key,size, flags)) ==-1) {
|
||||
int errsv = errno;
|
||||
printf("Errno %d\n",errsv);
|
||||
printf("key %d\n",key);
|
||||
printf("size %ld\n",size);
|
||||
printf("flags %d\n",flags);
|
||||
perror("shmget");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(WorldShmComm);
|
||||
MPI_Bcast(&shmids[0],WorldShmSize*sizeof(int),MPI_BYTE,0,WorldShmComm);
|
||||
MPI_Barrier(WorldShmComm);
|
||||
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
WorldShmCommBufs[r] = (uint64_t *)shmat(shmids[r], NULL,0);
|
||||
if (WorldShmCommBufs[r] == (uint64_t *)-1) {
|
||||
perror("Shared memory attach failure");
|
||||
shmctl(shmids[r], IPC_RMID, NULL);
|
||||
exit(2);
|
||||
}
|
||||
}
|
||||
MPI_Barrier(WorldShmComm);
|
||||
///////////////////////////////////
|
||||
// Mark for clean up
|
||||
///////////////////////////////////
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
shmctl(shmids[r], IPC_RMID,(struct shmid_ds *)NULL);
|
||||
}
|
||||
MPI_Barrier(WorldShmComm);
|
||||
|
||||
_ShmAlloc=1;
|
||||
_ShmAllocBytes = bytes;
|
||||
}
|
||||
#endif
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hugetlbfs mapping intended
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if defined(GRID_CUDA) ||defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
|
||||
//if defined(GRID_SYCL)
|
||||
#if 0
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
void * ShmCommBuf ;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the pointer array for shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Each MPI rank should allocate our own buffer
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||
|
||||
if (ShmCommBuf == (void *)NULL ) {
|
||||
std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||
|
||||
SharedMemoryZero(ShmCommBuf,bytes);
|
||||
|
||||
assert(WorldShmSize == 1);
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
WorldShmCommBufs[r] = ShmCommBuf;
|
||||
}
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(GRID_CUDA) ||defined(GRID_HIP) ||defined(GRID_SYCL)
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
void * ShmCommBuf ;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the pointer array for shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// TODO/FIXME : NOT ALL NVLINK BOARDS have full Peer to peer connectivity.
|
||||
// The annoyance is that they have partial peer 2 peer. This occurs on the 8 GPU blades.
|
||||
// e.g. DGX1, supermicro board,
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// cudaDeviceGetP2PAttribute(&perfRank, cudaDevP2PAttrPerformanceRank, device1, device2);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Each MPI rank should allocate our own buffer
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||
if (ShmCommBuf == (void *)NULL ) {
|
||||
std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if ( WorldRank == 0 ){
|
||||
std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf << " - "<<(bytes-1+(uint64_t)ShmCommBuf) <<std::dec<<" for comms buffers " <<std::endl;
|
||||
}
|
||||
SharedMemoryZero(ShmCommBuf,bytes);
|
||||
std::cout<< "Setting up IPC"<<std::endl;
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Loop over ranks/gpu's on our node
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
|
||||
#ifndef GRID_MPI3_SHM_NONE
|
||||
//////////////////////////////////////////////////
|
||||
// If it is me, pass around the IPC access key
|
||||
//////////////////////////////////////////////////
|
||||
void * thisBuf = ShmCommBuf;
|
||||
if(!Stencil_force_mpi) {
|
||||
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||
typedef struct { int fd; pid_t pid ; } clone_mem_t;
|
||||
|
||||
auto zeDevice = cl::sycl::get_native<cl::sycl::backend::level_zero>(theGridAccelerator->get_device());
|
||||
auto zeContext = cl::sycl::get_native<cl::sycl::backend::level_zero>(theGridAccelerator->get_context());
|
||||
|
||||
ze_ipc_mem_handle_t ihandle;
|
||||
clone_mem_t handle;
|
||||
|
||||
if ( r==WorldShmRank ) {
|
||||
auto err = zeMemGetIpcHandle(zeContext,ShmCommBuf,&ihandle);
|
||||
if ( err != ZE_RESULT_SUCCESS ) {
|
||||
std::cout << "SharedMemoryMPI.cc zeMemGetIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
} else {
|
||||
std::cout << "SharedMemoryMPI.cc zeMemGetIpcHandle succeeded for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
}
|
||||
memcpy((void *)&handle.fd,(void *)&ihandle,sizeof(int));
|
||||
handle.pid = getpid();
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_CUDA
|
||||
cudaIpcMemHandle_t handle;
|
||||
if ( r==WorldShmRank ) {
|
||||
auto err = cudaIpcGetMemHandle(&handle,ShmCommBuf);
|
||||
if ( err != cudaSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc cudaIpcGetMemHandle failed for rank" << r <<" "<<cudaGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
hipIpcMemHandle_t handle;
|
||||
if ( r==WorldShmRank ) {
|
||||
auto err = hipIpcGetMemHandle(&handle,ShmCommBuf);
|
||||
if ( err != hipSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc hipIpcGetMemHandle failed for rank" << r <<" "<<hipGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Share this IPC handle across the Shm Comm
|
||||
//////////////////////////////////////////////////
|
||||
{
|
||||
int ierr=MPI_Bcast(&handle,
|
||||
sizeof(handle),
|
||||
MPI_BYTE,
|
||||
r,
|
||||
WorldShmComm);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////
|
||||
// If I am not the source, overwrite thisBuf with remote buffer
|
||||
///////////////////////////////////////////////////////////////
|
||||
|
||||
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||
if ( r!=WorldShmRank ) {
|
||||
thisBuf = nullptr;
|
||||
std::cout<<"mapping seeking remote pid/fd "
|
||||
<<handle.pid<<"/"
|
||||
<<handle.fd<<std::endl;
|
||||
|
||||
int pidfd = syscall(SYS_pidfd_open,handle.pid,0);
|
||||
std::cout<<"Using IpcHandle pidfd "<<pidfd<<"\n";
|
||||
// int myfd = syscall(SYS_pidfd_getfd,pidfd,handle.fd,0);
|
||||
int myfd = syscall(438,pidfd,handle.fd,0);
|
||||
|
||||
std::cout<<"Using IpcHandle myfd "<<myfd<<"\n";
|
||||
|
||||
memcpy((void *)&ihandle,(void *)&myfd,sizeof(int));
|
||||
|
||||
auto err = zeMemOpenIpcHandle(zeContext,zeDevice,ihandle,0,&thisBuf);
|
||||
if ( err != ZE_RESULT_SUCCESS ) {
|
||||
std::cout << "SharedMemoryMPI.cc "<<zeContext<<" "<<zeDevice<<std::endl;
|
||||
std::cout << "SharedMemoryMPI.cc zeMemOpenIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
} else {
|
||||
std::cout << "SharedMemoryMPI.cc zeMemOpenIpcHandle succeeded for rank "<<r<<std::endl;
|
||||
std::cout << "SharedMemoryMPI.cc zeMemOpenIpcHandle pointer is "<<std::hex<<thisBuf<<std::dec<<std::endl;
|
||||
}
|
||||
assert(thisBuf!=nullptr);
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_CUDA
|
||||
if ( r!=WorldShmRank ) {
|
||||
auto err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
|
||||
if ( err != cudaSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc cudaIpcOpenMemHandle failed for rank" << r <<" "<<cudaGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
if ( r!=WorldShmRank ) {
|
||||
auto err = hipIpcOpenMemHandle(&thisBuf,handle,hipIpcMemLazyEnablePeerAccess);
|
||||
if ( err != hipSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc hipIpcOpenMemHandle failed for rank" << r <<" "<<hipGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
///////////////////////////////////////////////////////////////
|
||||
// Save a copy of the device buffers
|
||||
///////////////////////////////////////////////////////////////
|
||||
}
|
||||
WorldShmCommBufs[r] = thisBuf;
|
||||
#else
|
||||
WorldShmCommBufs[r] = ShmCommBuf;
|
||||
#endif
|
||||
}
|
||||
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
}
|
||||
#endif
|
||||
|
||||
#else
|
||||
#ifdef GRID_MPI3_SHMMMAP
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " MMAP implementation "<< GRID_SHM_PATH <<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hugetlbfs and others map filesystems as mappable huge pages
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
char shm_name [NAME_MAX];
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
|
||||
sprintf(shm_name,GRID_SHM_PATH "/Grid_mpi3_shm_%d_%d",WorldNode,r);
|
||||
int fd=open(shm_name,O_RDWR|O_CREAT,0666);
|
||||
if ( fd == -1) {
|
||||
printf("open %s failed\n",shm_name);
|
||||
perror("open hugetlbfs");
|
||||
exit(0);
|
||||
}
|
||||
int mmap_flag = MAP_SHARED ;
|
||||
#ifdef MAP_POPULATE
|
||||
mmap_flag|=MAP_POPULATE;
|
||||
#endif
|
||||
#ifdef MAP_HUGETLB
|
||||
if ( flags ) mmap_flag |= MAP_HUGETLB;
|
||||
#endif
|
||||
void *ptr = (void *) mmap(NULL, bytes, PROT_READ | PROT_WRITE, mmap_flag,fd, 0);
|
||||
if ( ptr == (void *)MAP_FAILED ) {
|
||||
printf("mmap %s failed\n",shm_name);
|
||||
perror("failed mmap"); assert(0);
|
||||
}
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
close(fd);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
// std::cout << header "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
}
|
||||
_ShmAlloc=1;
|
||||
_ShmAllocBytes = bytes;
|
||||
};
|
||||
#endif // MMAP
|
||||
|
||||
#ifdef GRID_MPI3_SHM_NONE
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " MMAP anonymous implementation "<<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hugetlbf and others map filesystems as mappable huge pages
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
char shm_name [NAME_MAX];
|
||||
assert(WorldShmSize == 1);
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
|
||||
int fd=-1;
|
||||
int mmap_flag = MAP_SHARED |MAP_ANONYMOUS ;
|
||||
#ifdef MAP_POPULATE
|
||||
mmap_flag|=MAP_POPULATE;
|
||||
#endif
|
||||
#ifdef MAP_HUGETLB
|
||||
if ( flags ) mmap_flag |= MAP_HUGETLB;
|
||||
#endif
|
||||
void *ptr = (void *) mmap(NULL, bytes, PROT_READ | PROT_WRITE, mmap_flag,fd, 0);
|
||||
if ( ptr == (void *)MAP_FAILED ) {
|
||||
printf("mmap %s failed\n",shm_name);
|
||||
perror("failed mmap"); assert(0);
|
||||
}
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
close(fd);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
// std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
}
|
||||
_ShmAlloc=1;
|
||||
_ShmAllocBytes = bytes;
|
||||
};
|
||||
#endif // MMAP
|
||||
|
||||
#ifdef GRID_MPI3_SHMOPEN
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// POSIX SHMOPEN ; as far as I know Linux does not allow EXPLICIT HugePages with this case
|
||||
// tmpfs (Larry Meadows says) does not support explicit huge page, and this is used for
|
||||
// the posix shm virtual file system
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " SHMOPEN implementation "<<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
|
||||
char shm_name [NAME_MAX];
|
||||
if ( WorldShmRank == 0 ) {
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
|
||||
size_t size = bytes;
|
||||
|
||||
struct passwd *pw = getpwuid (getuid());
|
||||
sprintf(shm_name,"/Grid_%s_mpi3_shm_%d_%d",pw->pw_name,WorldNode,r);
|
||||
|
||||
shm_unlink(shm_name);
|
||||
int fd=shm_open(shm_name,O_RDWR|O_CREAT,0666);
|
||||
if ( fd < 0 ) { perror("failed shm_open"); assert(0); }
|
||||
ftruncate(fd, size);
|
||||
|
||||
int mmap_flag = MAP_SHARED;
|
||||
#ifdef MAP_POPULATE
|
||||
mmap_flag |= MAP_POPULATE;
|
||||
#endif
|
||||
#ifdef MAP_HUGETLB
|
||||
if (flags) mmap_flag |= MAP_HUGETLB;
|
||||
#endif
|
||||
void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, mmap_flag, fd, 0);
|
||||
|
||||
if ( ptr == (void * )MAP_FAILED ) {
|
||||
perror("failed mmap");
|
||||
assert(0);
|
||||
}
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
close(fd);
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Barrier(WorldShmComm);
|
||||
|
||||
if ( WorldShmRank != 0 ) {
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
|
||||
size_t size = bytes ;
|
||||
|
||||
struct passwd *pw = getpwuid (getuid());
|
||||
sprintf(shm_name,"/Grid_%s_mpi3_shm_%d_%d",pw->pw_name,WorldNode,r);
|
||||
|
||||
int fd=shm_open(shm_name,O_RDWR,0666);
|
||||
if ( fd<0 ) { perror("failed shm_open"); assert(0); }
|
||||
|
||||
void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
if ( ptr == MAP_FAILED ) { perror("failed mmap"); assert(0); }
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
|
||||
close(fd);
|
||||
}
|
||||
}
|
||||
_ShmAlloc=1;
|
||||
_ShmAllocBytes = bytes;
|
||||
}
|
||||
#endif
|
||||
#endif // End NVCC case for GPU device buffers
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
// Routines accessing shared memory should route through for GPU safety
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
void GlobalSharedMemory::SharedMemoryZero(void *dest,size_t bytes)
|
||||
{
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
acceleratorMemSet(dest,0,bytes);
|
||||
#else
|
||||
bzero(dest,bytes);
|
||||
#endif
|
||||
}
|
||||
void GlobalSharedMemory::SharedMemoryCopy(void *dest,void *src,size_t bytes)
|
||||
{
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
acceleratorCopyToDevice(src,dest,bytes);
|
||||
#else
|
||||
bcopy(src,dest,bytes);
|
||||
#endif
|
||||
}
|
||||
////////////////////////////////////////////////////////
|
||||
// Global shared functionality finished
|
||||
// Now move to per communicator functionality
|
||||
////////////////////////////////////////////////////////
|
||||
void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
{
|
||||
int rank, size;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&size);
|
||||
ShmRanks.resize(size);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Split into groups that can share memory
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
#ifndef GRID_MPI3_SHM_NONE
|
||||
MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm);
|
||||
#else
|
||||
MPI_Comm_split(comm, rank, 0, &ShmComm);
|
||||
#endif
|
||||
MPI_Comm_rank(ShmComm ,&ShmRank);
|
||||
MPI_Comm_size(ShmComm ,&ShmSize);
|
||||
ShmCommBufs.resize(ShmSize);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Map ShmRank to WorldShmRank and use the right buffer
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
assert (GlobalSharedMemory::ShmAlloc()==1);
|
||||
heap_size = GlobalSharedMemory::ShmAllocBytes();
|
||||
for(int r=0;r<ShmSize;r++){
|
||||
|
||||
uint32_t wsr = (r==ShmRank) ? GlobalSharedMemory::WorldShmRank : 0 ;
|
||||
|
||||
MPI_Allreduce(MPI_IN_PLACE,&wsr,1,MPI_UINT32_T,MPI_SUM,ShmComm);
|
||||
|
||||
ShmCommBufs[r] = GlobalSharedMemory::WorldShmCommBufs[wsr];
|
||||
}
|
||||
ShmBufferFreeAll();
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// find comm ranks in our SHM group (i.e. which ranks are on our node)
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
MPI_Group FullGroup, ShmGroup;
|
||||
MPI_Comm_group (comm , &FullGroup);
|
||||
MPI_Comm_group (ShmComm, &ShmGroup);
|
||||
|
||||
std::vector<int> ranks(size); for(int r=0;r<size;r++) ranks[r]=r;
|
||||
MPI_Group_translate_ranks (FullGroup,size,&ranks[0],ShmGroup, &ShmRanks[0]);
|
||||
|
||||
#ifdef GRID_SHM_FORCE_MPI
|
||||
// Hide the shared memory path between ranks
|
||||
{
|
||||
for(int r=0;r<size;r++){
|
||||
if ( r!=rank ) {
|
||||
ShmRanks[r] = MPI_UNDEFINED;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
//SharedMemoryTest();
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// On node barrier
|
||||
//////////////////////////////////////////////////////////////////
|
||||
void SharedMemory::ShmBarrier(void)
|
||||
{
|
||||
MPI_Barrier (ShmComm);
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Test the shared memory is working
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void SharedMemory::SharedMemoryTest(void)
|
||||
{
|
||||
ShmBarrier();
|
||||
uint64_t check[3];
|
||||
uint64_t magic = 0x5A5A5A;
|
||||
if ( ShmRank == 0 ) {
|
||||
for(uint64_t r=0;r<ShmSize;r++){
|
||||
check[0]=GlobalSharedMemory::WorldNode;
|
||||
check[1]=r;
|
||||
check[2]=magic;
|
||||
GlobalSharedMemory::SharedMemoryCopy( ShmCommBufs[r], check, 3*sizeof(uint64_t));
|
||||
}
|
||||
}
|
||||
ShmBarrier();
|
||||
for(uint64_t r=0;r<ShmSize;r++){
|
||||
ShmBarrier();
|
||||
GlobalSharedMemory::SharedMemoryCopy(check,ShmCommBufs[r], 3*sizeof(uint64_t));
|
||||
ShmBarrier();
|
||||
assert(check[0]==GlobalSharedMemory::WorldNode);
|
||||
assert(check[1]==r);
|
||||
assert(check[2]==magic);
|
||||
ShmBarrier();
|
||||
}
|
||||
}
|
||||
|
||||
void *SharedMemory::ShmBuffer(int rank)
|
||||
{
|
||||
int gpeer = ShmRanks[rank];
|
||||
if (gpeer == MPI_UNDEFINED){
|
||||
return NULL;
|
||||
} else {
|
||||
return ShmCommBufs[gpeer];
|
||||
}
|
||||
}
|
||||
void *SharedMemory::ShmBufferTranslate(int rank,void * local_p)
|
||||
{
|
||||
int gpeer = ShmRanks[rank];
|
||||
assert(gpeer!=ShmRank); // never send to self
|
||||
if (gpeer == MPI_UNDEFINED){
|
||||
return NULL;
|
||||
} else {
|
||||
uint64_t offset = (uint64_t)local_p - (uint64_t)ShmCommBufs[ShmRank];
|
||||
uint64_t remote = (uint64_t)ShmCommBufs[gpeer]+offset;
|
||||
return (void *) remote;
|
||||
}
|
||||
}
|
||||
SharedMemory::~SharedMemory()
|
||||
{
|
||||
int MPI_is_finalised; MPI_Finalized(&MPI_is_finalised);
|
||||
if ( !MPI_is_finalised ) {
|
||||
MPI_Comm_free(&ShmComm);
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/SharedMemory.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
#define header "SharedMemoryNone: "
|
||||
|
||||
/*Construct from an MPI communicator*/
|
||||
void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
{
|
||||
assert(_ShmSetup==0);
|
||||
WorldComm = 0;
|
||||
WorldRank = 0;
|
||||
WorldSize = 1;
|
||||
WorldShmComm = 0 ;
|
||||
WorldShmRank = 0 ;
|
||||
WorldShmSize = 1 ;
|
||||
WorldNodes = 1 ;
|
||||
WorldNode = 0 ;
|
||||
WorldShmRanks.resize(WorldSize); WorldShmRanks[0] = 0;
|
||||
WorldShmCommBufs.resize(1);
|
||||
_ShmSetup=1;
|
||||
}
|
||||
|
||||
void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
{
|
||||
optimal_comm = WorldComm;
|
||||
SHM = Coordinate(processors.size(),1);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hugetlbfs mapping intended, use anonymous mmap
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if 1
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " GPU implementation "<<std::endl;
|
||||
void * ShmCommBuf ;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Each MPI rank should allocate our own buffer
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||
|
||||
if (ShmCommBuf == (void *)NULL ) {
|
||||
std::cerr << " SharedMemoryNone.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if ( WorldRank == 0 ){
|
||||
std::cout << WorldRank << header " SharedMemoryNone.cc acceleratorAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||
}
|
||||
SharedMemoryZero(ShmCommBuf,bytes);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Loop over ranks/gpu's on our node
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
WorldShmCommBufs[0] = ShmCommBuf;
|
||||
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
}
|
||||
#else
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
void * ShmCommBuf ;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
int mmap_flag =0;
|
||||
#ifdef MAP_ANONYMOUS
|
||||
mmap_flag = mmap_flag| MAP_SHARED | MAP_ANONYMOUS;
|
||||
#endif
|
||||
#ifdef MAP_ANON
|
||||
mmap_flag = mmap_flag| MAP_SHARED | MAP_ANON;
|
||||
#endif
|
||||
#ifdef MAP_HUGETLB
|
||||
if ( flags ) mmap_flag |= MAP_HUGETLB;
|
||||
#endif
|
||||
ShmCommBuf =(void *) mmap(NULL, bytes, PROT_READ | PROT_WRITE, mmap_flag, -1, 0);
|
||||
if (ShmCommBuf == (void *)MAP_FAILED) {
|
||||
perror("mmap failed ");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
#ifdef MADV_HUGEPAGE
|
||||
if (!Hugepages ) madvise(ShmCommBuf,bytes,MADV_HUGEPAGE);
|
||||
#endif
|
||||
bzero(ShmCommBuf,bytes);
|
||||
WorldShmCommBufs[0] = ShmCommBuf;
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
};
|
||||
#endif
|
||||
void GlobalSharedMemory::SharedMemoryZero(void *dest,size_t bytes)
|
||||
{
|
||||
acceleratorMemSet(dest,0,bytes);
|
||||
}
|
||||
void GlobalSharedMemory::SharedMemoryCopy(void *dest,void *src,size_t bytes)
|
||||
{
|
||||
acceleratorCopyToDevice(src,dest,bytes);
|
||||
}
|
||||
////////////////////////////////////////////////////////
|
||||
// Global shared functionality finished
|
||||
// Now move to per communicator functionality
|
||||
////////////////////////////////////////////////////////
|
||||
void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
{
|
||||
assert(GlobalSharedMemory::ShmAlloc()==1);
|
||||
ShmRanks.resize(1);
|
||||
ShmCommBufs.resize(1);
|
||||
ShmRanks[0] = 0;
|
||||
ShmRank = 0;
|
||||
ShmSize = 1;
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Map ShmRank to WorldShmRank and use the right buffer
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
ShmCommBufs[0] = GlobalSharedMemory::WorldShmCommBufs[0];
|
||||
heap_size = GlobalSharedMemory::ShmAllocBytes();
|
||||
ShmBufferFreeAll();
|
||||
return;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// On node barrier
|
||||
//////////////////////////////////////////////////////////////////
|
||||
void SharedMemory::ShmBarrier(void){ return ; }
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Test the shared memory is working
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void SharedMemory::SharedMemoryTest(void) { return; }
|
||||
|
||||
void *SharedMemory::ShmBuffer(int rank)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
void *SharedMemory::ShmBufferTranslate(int rank,void * local_p)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
SharedMemory::~SharedMemory()
|
||||
{};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Cshift.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef _GRID_CSHIFT_H_
|
||||
#define _GRID_CSHIFT_H_
|
||||
|
||||
#include <Grid/cshift/Cshift_common.h>
|
||||
|
||||
#ifdef GRID_COMMS_NONE
|
||||
#include <Grid/cshift/Cshift_none.h>
|
||||
#endif
|
||||
|
||||
#ifdef GRID_COMMS_MPI
|
||||
#include <Grid/cshift/Cshift_mpi.h>
|
||||
#endif
|
||||
|
||||
#ifdef GRID_COMMS_MPI3
|
||||
#include <Grid/cshift/Cshift_mpi.h>
|
||||
#endif
|
||||
|
||||
#ifdef GRID_COMMS_MPIT
|
||||
#include <Grid/cshift/Cshift_mpi.h>
|
||||
#endif
|
||||
|
||||
#ifdef GRID_COMMS_SHMEM
|
||||
#include <Grid/cshift/Cshift_mpi.h> // uses same implementation of communicator
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
||||
auto Cshift(const Expression &expr,int dim,int shift) -> decltype(closure(expr))
|
||||
{
|
||||
return Cshift(closure(expr),dim,shift);
|
||||
}
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,536 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/cshift/Cshift_common.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
extern Vector<std::pair<int,int> > Cshift_table;
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// Gather for when there is no need to SIMD split
|
||||
///////////////////////////////////////////////////////////////////
|
||||
template<class vobj> void
|
||||
Gather_plane_simple (const Lattice<vobj> &rhs,cshiftVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
cbmask = 0x3;
|
||||
}
|
||||
|
||||
int so=plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int ent = 0;
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
int stride=rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if ( cbmask == 0x3 ) {
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o = n*stride;
|
||||
int bo = n*e2;
|
||||
Cshift_table[ent++] = std::pair<int,int>(off+bo+b,so+o+b);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
int bo=0;
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o = n*stride;
|
||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);
|
||||
if ( ocb &cbmask ) {
|
||||
Cshift_table[ent++]=std::pair<int,int> (off+bo++,so+o+b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
auto buffer_p = & buffer[0];
|
||||
auto table = &Cshift_table[0];
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
||||
coalescedWrite(buffer_p[table[i].first],coalescedRead(rhs_v[table[i].second]));
|
||||
});
|
||||
#else
|
||||
autoView(rhs_v , rhs, CpuRead);
|
||||
thread_for(i,ent,{
|
||||
buffer_p[table[i].first]=rhs_v[table[i].second];
|
||||
});
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// Gather for when there *is* need to SIMD split
|
||||
///////////////////////////////////////////////////////////////////
|
||||
template<class vobj> void
|
||||
Gather_plane_extract(const Lattice<vobj> &rhs,
|
||||
ExtractPointerArray<typename vobj::scalar_object> pointers,
|
||||
int dimension,int plane,int cbmask)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
cbmask = 0x3;
|
||||
}
|
||||
|
||||
int so = plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
|
||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int n1=rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if ( cbmask ==0x3){
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
int o = n*n1;
|
||||
int offset = b+n*e2;
|
||||
|
||||
vobj temp =rhs_v[so+o+b];
|
||||
extract<vobj>(temp,pointers,offset);
|
||||
});
|
||||
#else
|
||||
autoView(rhs_v , rhs, CpuRead);
|
||||
thread_for2d(n,e1,b,e2,{
|
||||
int o = n*n1;
|
||||
int offset = b+n*e2;
|
||||
|
||||
vobj temp =rhs_v[so+o+b];
|
||||
extract<vobj>(temp,pointers,offset);
|
||||
});
|
||||
#endif
|
||||
} else {
|
||||
Coordinate rdim=rhs.Grid()->_rdimensions;
|
||||
Coordinate cdm =rhs.Grid()->_checker_dim_mask;
|
||||
std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
|
||||
Coordinate coor;
|
||||
|
||||
int o=n*n1;
|
||||
int oindex = o+b;
|
||||
|
||||
int cb = RedBlackCheckerBoardFromOindex(oindex, rdim, cdm);
|
||||
|
||||
int ocb=1<<cb;
|
||||
int offset = b+n*e2;
|
||||
|
||||
if ( ocb & cbmask ) {
|
||||
vobj temp =rhs_v[so+o+b];
|
||||
extract<vobj>(temp,pointers,offset);
|
||||
}
|
||||
});
|
||||
#else
|
||||
autoView(rhs_v , rhs, CpuRead);
|
||||
thread_for2d(n,e1,b,e2,{
|
||||
|
||||
Coordinate coor;
|
||||
|
||||
int o=n*n1;
|
||||
int oindex = o+b;
|
||||
|
||||
int cb = RedBlackCheckerBoardFromOindex(oindex, rdim, cdm);
|
||||
|
||||
int ocb=1<<cb;
|
||||
int offset = b+n*e2;
|
||||
|
||||
if ( ocb & cbmask ) {
|
||||
vobj temp =rhs_v[so+o+b];
|
||||
extract<vobj>(temp,pointers,offset);
|
||||
}
|
||||
});
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// Scatter for when there is no need to SIMD split
|
||||
//////////////////////////////////////////////////////
|
||||
template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,cshiftVector<vobj> &buffer, int dimension,int plane,int cbmask)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
cbmask=0x3;
|
||||
}
|
||||
|
||||
int so = plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
|
||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int stride=rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
int ent =0;
|
||||
|
||||
if ( cbmask ==0x3 ) {
|
||||
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*rhs.Grid()->_slice_stride[dimension];
|
||||
int bo =n*rhs.Grid()->_slice_block[dimension];
|
||||
Cshift_table[ent++] = std::pair<int,int>(so+o+b,bo+b);
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
int bo=0;
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*rhs.Grid()->_slice_stride[dimension];
|
||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);// Could easily be a table lookup
|
||||
if ( ocb & cbmask ) {
|
||||
Cshift_table[ent++]=std::pair<int,int> (so+o+b,bo++);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto buffer_p = & buffer[0];
|
||||
auto table = &Cshift_table[0];
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView( rhs_v, rhs, AcceleratorWrite);
|
||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
||||
coalescedWrite(rhs_v[table[i].first],coalescedRead(buffer_p[table[i].second]));
|
||||
});
|
||||
#else
|
||||
autoView( rhs_v, rhs, CpuWrite);
|
||||
thread_for(i,ent,{
|
||||
rhs_v[table[i].first]=buffer_p[table[i].second];
|
||||
});
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// Scatter for when there *is* need to SIMD split
|
||||
//////////////////////////////////////////////////////
|
||||
template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerArray<typename vobj::scalar_object> pointers,int dimension,int plane,int cbmask)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
cbmask=0x3;
|
||||
}
|
||||
|
||||
int so = plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
|
||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
|
||||
if(cbmask ==0x3 ) {
|
||||
int _slice_stride = rhs.Grid()->_slice_stride[dimension];
|
||||
int _slice_block = rhs.Grid()->_slice_block[dimension];
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView( rhs_v , rhs, AcceleratorWrite);
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
int o = n*_slice_stride;
|
||||
int offset = b+n*_slice_block;
|
||||
merge(rhs_v[so+o+b],pointers,offset);
|
||||
});
|
||||
#else
|
||||
autoView( rhs_v , rhs, CpuWrite);
|
||||
thread_for2d(n,e1,b,e2,{
|
||||
int o = n*_slice_stride;
|
||||
int offset = b+n*_slice_block;
|
||||
merge(rhs_v[so+o+b],pointers,offset);
|
||||
});
|
||||
#endif
|
||||
} else {
|
||||
|
||||
// Case of SIMD split AND checker dim cannot currently be hit, except in
|
||||
// Test_cshift_red_black code.
|
||||
std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
||||
std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
|
||||
assert(0); // This will fail if hit on GPU
|
||||
autoView( rhs_v, rhs, CpuWrite);
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o = n*rhs.Grid()->_slice_stride[dimension];
|
||||
int offset = b+n*rhs.Grid()->_slice_block[dimension];
|
||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);
|
||||
if ( ocb&cbmask ) {
|
||||
merge(rhs_v[so+o+b],pointers,offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if (defined(GRID_CUDA) || defined(GRID_HIP)) && defined(ACCELERATOR_CSHIFT)
|
||||
|
||||
template <typename T>
|
||||
T iDivUp(T a, T b) // Round a / b to nearest higher integer value
|
||||
{ return (a % b != 0) ? (a / b + 1) : (a / b); }
|
||||
|
||||
template <typename T>
|
||||
__global__ void populate_Cshift_table(T* vector, T lo, T ro, T e1, T e2, T stride)
|
||||
{
|
||||
int idx = blockIdx.x*blockDim.x + threadIdx.x;
|
||||
if (idx >= e1*e2) return;
|
||||
|
||||
int n, b, o;
|
||||
|
||||
n = idx / e2;
|
||||
b = idx % e2;
|
||||
o = n*stride + b;
|
||||
|
||||
vector[2*idx + 0] = lo + o;
|
||||
vector[2*idx + 1] = ro + o;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// local to node block strided copies
|
||||
//////////////////////////////////////////////////////
|
||||
|
||||
template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs, int dimension,int lplane,int rplane,int cbmask)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
cbmask=0x3;
|
||||
}
|
||||
|
||||
int ro = rplane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
int lo = lplane*lhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
|
||||
int e1=rhs.Grid()->_slice_nblock[dimension]; // clearly loop invariant for icpc
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int stride = rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
int ent=0;
|
||||
|
||||
if(cbmask == 0x3 ){
|
||||
#if (defined(GRID_CUDA) || defined(GRID_HIP)) && defined(ACCELERATOR_CSHIFT)
|
||||
ent = e1*e2;
|
||||
dim3 blockSize(acceleratorThreads());
|
||||
dim3 gridSize(iDivUp((unsigned int)ent, blockSize.x));
|
||||
populate_Cshift_table<<<gridSize, blockSize>>>(&Cshift_table[0].first, lo, ro, e1, e2, stride);
|
||||
accelerator_barrier();
|
||||
#else
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*stride+b;
|
||||
Cshift_table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*stride+b;
|
||||
int ocb=1<<lhs.Grid()->CheckerBoardFromOindex(o);
|
||||
if ( ocb&cbmask ) {
|
||||
Cshift_table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto table = &Cshift_table[0];
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
autoView(lhs_v , lhs, AcceleratorWrite);
|
||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
||||
coalescedWrite(lhs_v[table[i].first],coalescedRead(rhs_v[table[i].second]));
|
||||
});
|
||||
#else
|
||||
autoView(rhs_v , rhs, CpuRead);
|
||||
autoView(lhs_v , lhs, CpuWrite);
|
||||
thread_for(i,ent,{
|
||||
lhs_v[table[i].first]=rhs_v[table[i].second];
|
||||
});
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj> void Copy_plane_permute(Lattice<vobj>& lhs,const Lattice<vobj> &rhs, int dimension,int lplane,int rplane,int cbmask,int permute_type)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
cbmask=0x3;
|
||||
}
|
||||
|
||||
int ro = rplane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
int lo = lplane*lhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
||||
|
||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
||||
int e2=rhs.Grid()->_slice_block [dimension];
|
||||
int stride = rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
int ent=0;
|
||||
|
||||
if ( cbmask == 0x3 ) {
|
||||
#if (defined(GRID_CUDA) || defined(GRID_HIP)) && defined(ACCELERATOR_CSHIFT)
|
||||
ent = e1*e2;
|
||||
dim3 blockSize(acceleratorThreads());
|
||||
dim3 gridSize(iDivUp((unsigned int)ent, blockSize.x));
|
||||
populate_Cshift_table<<<gridSize, blockSize>>>(&Cshift_table[0].first, lo, ro, e1, e2, stride);
|
||||
accelerator_barrier();
|
||||
#else
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*stride;
|
||||
Cshift_table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
||||
}}
|
||||
#endif
|
||||
} else {
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*stride;
|
||||
int ocb=1<<lhs.Grid()->CheckerBoardFromOindex(o+b);
|
||||
if ( ocb&cbmask ) Cshift_table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
||||
}}
|
||||
}
|
||||
|
||||
{
|
||||
auto table = &Cshift_table[0];
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView( rhs_v, rhs, AcceleratorRead);
|
||||
autoView( lhs_v, lhs, AcceleratorWrite);
|
||||
accelerator_for(i,ent,1,{
|
||||
permute(lhs_v[table[i].first],rhs_v[table[i].second],permute_type);
|
||||
});
|
||||
#else
|
||||
autoView( rhs_v, rhs, CpuRead);
|
||||
autoView( lhs_v, lhs, CpuWrite);
|
||||
thread_for(i,ent,{
|
||||
permute(lhs_v[table[i].first],rhs_v[table[i].second],permute_type);
|
||||
});
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// Local to node Cshift
|
||||
//////////////////////////////////////////////////////
|
||||
template<class vobj> void Cshift_local(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
int sshift[2];
|
||||
|
||||
sshift[0] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Even);
|
||||
sshift[1] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Odd);
|
||||
|
||||
if ( sshift[0] == sshift[1] ) {
|
||||
Cshift_local(ret,rhs,dimension,shift,0x3);
|
||||
} else {
|
||||
Cshift_local(ret,rhs,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
|
||||
Cshift_local(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_local(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
||||
{
|
||||
GridBase *grid = rhs.Grid();
|
||||
int fd = grid->_fdimensions[dimension];
|
||||
int rd = grid->_rdimensions[dimension];
|
||||
int ld = grid->_ldimensions[dimension];
|
||||
int gd = grid->_gdimensions[dimension];
|
||||
int ly = grid->_simd_layout[dimension];
|
||||
|
||||
// Map to always positive shift modulo global full dimension.
|
||||
shift = (shift+fd)%fd;
|
||||
|
||||
// the permute type
|
||||
ret.Checkerboard() = grid->CheckerBoardDestination(rhs.Checkerboard(),shift,dimension);
|
||||
int permute_dim =grid->PermuteDim(dimension);
|
||||
int permute_type=grid->PermuteType(dimension);
|
||||
int permute_type_dist;
|
||||
|
||||
for(int x=0;x<rd;x++){
|
||||
|
||||
// int o = 0;
|
||||
int bo = x * grid->_ostride[dimension];
|
||||
int cb= (cbmask==0x2)? Odd : Even;
|
||||
|
||||
int sshift = grid->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
||||
int sx = (x+sshift)%rd;
|
||||
|
||||
// wrap is whether sshift > rd.
|
||||
// num is sshift mod rd.
|
||||
//
|
||||
// shift 7
|
||||
//
|
||||
// XoXo YcYc
|
||||
// oXoX cYcY
|
||||
// XoXo YcYc
|
||||
// oXoX cYcY
|
||||
//
|
||||
// sshift --
|
||||
//
|
||||
// XX YY ; 3
|
||||
// XX YY ; 0
|
||||
// XX YY ; 3
|
||||
// XX YY ; 0
|
||||
//
|
||||
int permute_slice=0;
|
||||
if(permute_dim){
|
||||
int wrap = sshift/rd; wrap=wrap % ly;
|
||||
int num = sshift%rd;
|
||||
|
||||
if ( x< rd-num ) permute_slice=wrap;
|
||||
else permute_slice = (wrap+1)%ly;
|
||||
|
||||
if ( (ly>2) && (permute_slice) ) {
|
||||
assert(permute_type & RotateBit);
|
||||
permute_type_dist = permute_type|permute_slice;
|
||||
} else {
|
||||
permute_type_dist = permute_type;
|
||||
}
|
||||
}
|
||||
|
||||
if ( permute_slice ) Copy_plane_permute(ret,rhs,dimension,x,sx,cbmask,permute_type_dist);
|
||||
else Copy_plane(ret,rhs,dimension,x,sx,cbmask);
|
||||
|
||||
}
|
||||
}
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,467 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/cshift/Cshift_mpi.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef _GRID_CSHIFT_MPI_H_
|
||||
#define _GRID_CSHIFT_MPI_H_
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
|
||||
Lattice<vobj> ret(rhs.Grid());
|
||||
|
||||
int fd = rhs.Grid()->_fdimensions[dimension];
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
// Map to always positive shift modulo global full dimension.
|
||||
shift = (shift+fd)%fd;
|
||||
|
||||
ret.Checkerboard() = rhs.Grid()->CheckerBoardDestination(rhs.Checkerboard(),shift,dimension);
|
||||
|
||||
// the permute type
|
||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
||||
int splice_dim = rhs.Grid()->_simd_layout[dimension]>1 && (comm_dim);
|
||||
|
||||
|
||||
if ( !comm_dim ) {
|
||||
//std::cout << "CSHIFT: Cshift_local" <<std::endl;
|
||||
Cshift_local(ret,rhs,dimension,shift); // Handles checkerboarding
|
||||
} else if ( splice_dim ) {
|
||||
//std::cout << "CSHIFT: Cshift_comms_simd call - splice_dim = " << splice_dim << " shift " << shift << " dimension = " << dimension << std::endl;
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift);
|
||||
} else {
|
||||
//std::cout << "CSHIFT: Cshift_comms" <<std::endl;
|
||||
Cshift_comms(ret,rhs,dimension,shift);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_comms(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
int sshift[2];
|
||||
|
||||
sshift[0] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Even);
|
||||
sshift[1] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Odd);
|
||||
|
||||
// std::cout << "Cshift_comms dim "<<dimension<<"cb "<<rhs.Checkerboard()<<"shift "<<shift<<" sshift " << sshift[0]<<" "<<sshift[1]<<std::endl;
|
||||
if ( sshift[0] == sshift[1] ) {
|
||||
// std::cout << "Single pass Cshift_comms" <<std::endl;
|
||||
Cshift_comms(ret,rhs,dimension,shift,0x3);
|
||||
} else {
|
||||
// std::cout << "Two pass Cshift_comms" <<std::endl;
|
||||
Cshift_comms(ret,rhs,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
|
||||
Cshift_comms(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_comms_simd(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
int sshift[2];
|
||||
|
||||
sshift[0] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Even);
|
||||
sshift[1] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Odd);
|
||||
|
||||
//std::cout << "Cshift_comms_simd dim "<<dimension<<"cb "<<rhs.checkerboard<<"shift "<<shift<<" sshift " << sshift[0]<<" "<<sshift[1]<<std::endl;
|
||||
if ( sshift[0] == sshift[1] ) {
|
||||
//std::cout << "Single pass Cshift_comms" <<std::endl;
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x3);
|
||||
} else {
|
||||
//std::cout << "Two pass Cshift_comms" <<std::endl;
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
|
||||
}
|
||||
}
|
||||
#define ACCELERATOR_CSHIFT_NO_COPY
|
||||
#ifdef ACCELERATOR_CSHIFT_NO_COPY
|
||||
template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
||||
{
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
|
||||
GridBase *grid=rhs.Grid();
|
||||
Lattice<vobj> temp(rhs.Grid());
|
||||
|
||||
int fd = rhs.Grid()->_fdimensions[dimension];
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
int pd = rhs.Grid()->_processors[dimension];
|
||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
||||
assert(simd_layout==1);
|
||||
assert(comm_dim==1);
|
||||
assert(shift>=0);
|
||||
assert(shift<fd);
|
||||
|
||||
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
||||
static cshiftVector<vobj> send_buf; send_buf.resize(buffer_size);
|
||||
static cshiftVector<vobj> recv_buf; recv_buf.resize(buffer_size);
|
||||
|
||||
int cb= (cbmask==0x2)? Odd : Even;
|
||||
int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
||||
|
||||
for(int x=0;x<rd;x++){
|
||||
|
||||
int sx = (x+sshift)%rd;
|
||||
int comm_proc = ((x+sshift)/rd)%pd;
|
||||
|
||||
if (comm_proc==0) {
|
||||
|
||||
Copy_plane(ret,rhs,dimension,x,sx,cbmask);
|
||||
|
||||
} else {
|
||||
|
||||
int words = buffer_size;
|
||||
if (cbmask != 0x3) words=words>>1;
|
||||
|
||||
int bytes = words * sizeof(vobj);
|
||||
|
||||
Gather_plane_simple (rhs,send_buf,dimension,sx,cbmask);
|
||||
|
||||
// int rank = grid->_processor;
|
||||
int recv_from_rank;
|
||||
int xmit_to_rank;
|
||||
grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
|
||||
|
||||
grid->Barrier();
|
||||
|
||||
grid->SendToRecvFrom((void *)&send_buf[0],
|
||||
xmit_to_rank,
|
||||
(void *)&recv_buf[0],
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
|
||||
grid->Barrier();
|
||||
|
||||
Scatter_plane_simple (ret,recv_buf,dimension,x,cbmask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
||||
{
|
||||
GridBase *grid=rhs.Grid();
|
||||
const int Nsimd = grid->Nsimd();
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename vobj::scalar_object scalar_object;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
|
||||
int fd = grid->_fdimensions[dimension];
|
||||
int rd = grid->_rdimensions[dimension];
|
||||
int ld = grid->_ldimensions[dimension];
|
||||
int pd = grid->_processors[dimension];
|
||||
int simd_layout = grid->_simd_layout[dimension];
|
||||
int comm_dim = grid->_processors[dimension] >1 ;
|
||||
|
||||
//std::cout << "Cshift_comms_simd dim "<< dimension << " fd "<<fd<<" rd "<<rd
|
||||
// << " ld "<<ld<<" pd " << pd<<" simd_layout "<<simd_layout
|
||||
// << " comm_dim " << comm_dim << " cbmask " << cbmask <<std::endl;
|
||||
|
||||
assert(comm_dim==1);
|
||||
assert(simd_layout==2);
|
||||
assert(shift>=0);
|
||||
assert(shift<fd);
|
||||
|
||||
int permute_type=grid->PermuteType(dimension);
|
||||
|
||||
///////////////////////////////////////////////
|
||||
// Simd direction uses an extract/merge pair
|
||||
///////////////////////////////////////////////
|
||||
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
||||
// int words = sizeof(vobj)/sizeof(vector_type);
|
||||
|
||||
static std::vector<cshiftVector<scalar_object> > send_buf_extract; send_buf_extract.resize(Nsimd);
|
||||
static std::vector<cshiftVector<scalar_object> > recv_buf_extract; recv_buf_extract.resize(Nsimd);
|
||||
scalar_object * recv_buf_extract_mpi;
|
||||
scalar_object * send_buf_extract_mpi;
|
||||
|
||||
for(int s=0;s<Nsimd;s++){
|
||||
send_buf_extract[s].resize(buffer_size);
|
||||
recv_buf_extract[s].resize(buffer_size);
|
||||
}
|
||||
|
||||
int bytes = buffer_size*sizeof(scalar_object);
|
||||
|
||||
ExtractPointerArray<scalar_object> pointers(Nsimd); //
|
||||
ExtractPointerArray<scalar_object> rpointers(Nsimd); // received pointers
|
||||
|
||||
///////////////////////////////////////////
|
||||
// Work out what to send where
|
||||
///////////////////////////////////////////
|
||||
int cb = (cbmask==0x2)? Odd : Even;
|
||||
int sshift= grid->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
||||
|
||||
// loop over outer coord planes orthog to dim
|
||||
for(int x=0;x<rd;x++){
|
||||
|
||||
// FIXME call local permute copy if none are offnode.
|
||||
for(int i=0;i<Nsimd;i++){
|
||||
pointers[i] = &send_buf_extract[i][0];
|
||||
}
|
||||
int sx = (x+sshift)%rd;
|
||||
Gather_plane_extract(rhs,pointers,dimension,sx,cbmask);
|
||||
|
||||
for(int i=0;i<Nsimd;i++){
|
||||
|
||||
int inner_bit = (Nsimd>>(permute_type+1));
|
||||
int ic= (i&inner_bit)? 1:0;
|
||||
|
||||
int my_coor = rd*ic + x;
|
||||
int nbr_coor = my_coor+sshift;
|
||||
int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
|
||||
|
||||
int nbr_ic = (nbr_coor%ld)/rd; // inner coord of peer
|
||||
int nbr_ox = (nbr_coor%rd); // outer coord of peer
|
||||
int nbr_lane = (i&(~inner_bit));
|
||||
|
||||
int recv_from_rank;
|
||||
int xmit_to_rank;
|
||||
|
||||
if (nbr_ic) nbr_lane|=inner_bit;
|
||||
|
||||
assert (sx == nbr_ox);
|
||||
|
||||
if(nbr_proc){
|
||||
grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank);
|
||||
|
||||
grid->Barrier();
|
||||
|
||||
send_buf_extract_mpi = &send_buf_extract[nbr_lane][0];
|
||||
recv_buf_extract_mpi = &recv_buf_extract[i][0];
|
||||
grid->SendToRecvFrom((void *)send_buf_extract_mpi,
|
||||
xmit_to_rank,
|
||||
(void *)recv_buf_extract_mpi,
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
|
||||
grid->Barrier();
|
||||
|
||||
rpointers[i] = &recv_buf_extract[i][0];
|
||||
} else {
|
||||
rpointers[i] = &send_buf_extract[nbr_lane][0];
|
||||
}
|
||||
|
||||
}
|
||||
Scatter_plane_merge(ret,rpointers,dimension,x,cbmask);
|
||||
}
|
||||
|
||||
}
|
||||
#else
|
||||
template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
||||
{
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
|
||||
GridBase *grid=rhs.Grid();
|
||||
Lattice<vobj> temp(rhs.Grid());
|
||||
|
||||
int fd = rhs.Grid()->_fdimensions[dimension];
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
int pd = rhs.Grid()->_processors[dimension];
|
||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
||||
assert(simd_layout==1);
|
||||
assert(comm_dim==1);
|
||||
assert(shift>=0);
|
||||
assert(shift<fd);
|
||||
|
||||
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
||||
static cshiftVector<vobj> send_buf_v; send_buf_v.resize(buffer_size);
|
||||
static cshiftVector<vobj> recv_buf_v; recv_buf_v.resize(buffer_size);
|
||||
vobj *send_buf;
|
||||
vobj *recv_buf;
|
||||
{
|
||||
grid->ShmBufferFreeAll();
|
||||
size_t bytes = buffer_size*sizeof(vobj);
|
||||
send_buf=(vobj *)grid->ShmBufferMalloc(bytes);
|
||||
recv_buf=(vobj *)grid->ShmBufferMalloc(bytes);
|
||||
}
|
||||
|
||||
int cb= (cbmask==0x2)? Odd : Even;
|
||||
int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
||||
|
||||
for(int x=0;x<rd;x++){
|
||||
|
||||
int sx = (x+sshift)%rd;
|
||||
int comm_proc = ((x+sshift)/rd)%pd;
|
||||
|
||||
if (comm_proc==0) {
|
||||
|
||||
Copy_plane(ret,rhs,dimension,x,sx,cbmask);
|
||||
|
||||
} else {
|
||||
|
||||
int words = buffer_size;
|
||||
if (cbmask != 0x3) words=words>>1;
|
||||
|
||||
int bytes = words * sizeof(vobj);
|
||||
|
||||
Gather_plane_simple (rhs,send_buf_v,dimension,sx,cbmask);
|
||||
|
||||
// int rank = grid->_processor;
|
||||
int recv_from_rank;
|
||||
int xmit_to_rank;
|
||||
grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
|
||||
|
||||
|
||||
grid->Barrier();
|
||||
|
||||
acceleratorCopyDeviceToDevice((void *)&send_buf_v[0],(void *)&send_buf[0],bytes);
|
||||
grid->SendToRecvFrom((void *)&send_buf[0],
|
||||
xmit_to_rank,
|
||||
(void *)&recv_buf[0],
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
acceleratorCopyDeviceToDevice((void *)&recv_buf[0],(void *)&recv_buf_v[0],bytes);
|
||||
|
||||
grid->Barrier();
|
||||
|
||||
Scatter_plane_simple (ret,recv_buf_v,dimension,x,cbmask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
||||
{
|
||||
GridBase *grid=rhs.Grid();
|
||||
const int Nsimd = grid->Nsimd();
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename vobj::scalar_object scalar_object;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
|
||||
int fd = grid->_fdimensions[dimension];
|
||||
int rd = grid->_rdimensions[dimension];
|
||||
int ld = grid->_ldimensions[dimension];
|
||||
int pd = grid->_processors[dimension];
|
||||
int simd_layout = grid->_simd_layout[dimension];
|
||||
int comm_dim = grid->_processors[dimension] >1 ;
|
||||
|
||||
//std::cout << "Cshift_comms_simd dim "<< dimension << " fd "<<fd<<" rd "<<rd
|
||||
// << " ld "<<ld<<" pd " << pd<<" simd_layout "<<simd_layout
|
||||
// << " comm_dim " << comm_dim << " cbmask " << cbmask <<std::endl;
|
||||
|
||||
assert(comm_dim==1);
|
||||
assert(simd_layout==2);
|
||||
assert(shift>=0);
|
||||
assert(shift<fd);
|
||||
|
||||
int permute_type=grid->PermuteType(dimension);
|
||||
|
||||
///////////////////////////////////////////////
|
||||
// Simd direction uses an extract/merge pair
|
||||
///////////////////////////////////////////////
|
||||
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
||||
// int words = sizeof(vobj)/sizeof(vector_type);
|
||||
|
||||
static std::vector<cshiftVector<scalar_object> > send_buf_extract; send_buf_extract.resize(Nsimd);
|
||||
static std::vector<cshiftVector<scalar_object> > recv_buf_extract; recv_buf_extract.resize(Nsimd);
|
||||
scalar_object * recv_buf_extract_mpi;
|
||||
scalar_object * send_buf_extract_mpi;
|
||||
{
|
||||
size_t bytes = sizeof(scalar_object)*buffer_size;
|
||||
grid->ShmBufferFreeAll();
|
||||
send_buf_extract_mpi = (scalar_object *)grid->ShmBufferMalloc(bytes);
|
||||
recv_buf_extract_mpi = (scalar_object *)grid->ShmBufferMalloc(bytes);
|
||||
}
|
||||
for(int s=0;s<Nsimd;s++){
|
||||
send_buf_extract[s].resize(buffer_size);
|
||||
recv_buf_extract[s].resize(buffer_size);
|
||||
}
|
||||
|
||||
int bytes = buffer_size*sizeof(scalar_object);
|
||||
|
||||
ExtractPointerArray<scalar_object> pointers(Nsimd); //
|
||||
ExtractPointerArray<scalar_object> rpointers(Nsimd); // received pointers
|
||||
|
||||
///////////////////////////////////////////
|
||||
// Work out what to send where
|
||||
///////////////////////////////////////////
|
||||
int cb = (cbmask==0x2)? Odd : Even;
|
||||
int sshift= grid->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
||||
|
||||
// loop over outer coord planes orthog to dim
|
||||
for(int x=0;x<rd;x++){
|
||||
|
||||
// FIXME call local permute copy if none are offnode.
|
||||
for(int i=0;i<Nsimd;i++){
|
||||
pointers[i] = &send_buf_extract[i][0];
|
||||
}
|
||||
int sx = (x+sshift)%rd;
|
||||
Gather_plane_extract(rhs,pointers,dimension,sx,cbmask);
|
||||
|
||||
for(int i=0;i<Nsimd;i++){
|
||||
|
||||
int inner_bit = (Nsimd>>(permute_type+1));
|
||||
int ic= (i&inner_bit)? 1:0;
|
||||
|
||||
int my_coor = rd*ic + x;
|
||||
int nbr_coor = my_coor+sshift;
|
||||
int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
|
||||
|
||||
int nbr_ic = (nbr_coor%ld)/rd; // inner coord of peer
|
||||
int nbr_ox = (nbr_coor%rd); // outer coord of peer
|
||||
int nbr_lane = (i&(~inner_bit));
|
||||
|
||||
int recv_from_rank;
|
||||
int xmit_to_rank;
|
||||
|
||||
if (nbr_ic) nbr_lane|=inner_bit;
|
||||
|
||||
assert (sx == nbr_ox);
|
||||
|
||||
if(nbr_proc){
|
||||
grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank);
|
||||
|
||||
grid->Barrier();
|
||||
|
||||
acceleratorCopyDeviceToDevice((void *)&send_buf_extract[nbr_lane][0],(void *)send_buf_extract_mpi,bytes);
|
||||
grid->SendToRecvFrom((void *)send_buf_extract_mpi,
|
||||
xmit_to_rank,
|
||||
(void *)recv_buf_extract_mpi,
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
acceleratorCopyDeviceToDevice((void *)recv_buf_extract_mpi,(void *)&recv_buf_extract[i][0],bytes);
|
||||
|
||||
grid->Barrier();
|
||||
rpointers[i] = &recv_buf_extract[i][0];
|
||||
} else {
|
||||
rpointers[i] = &send_buf_extract[nbr_lane][0];
|
||||
}
|
||||
|
||||
}
|
||||
Scatter_plane_merge(ret,rpointers,dimension,x,cbmask);
|
||||
}
|
||||
|
||||
}
|
||||
#endif
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,40 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/cshift/Cshift_none.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef _GRID_CSHIFT_NONE_H_
|
||||
#define _GRID_CSHIFT_NONE_H_
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
Lattice<vobj> ret(rhs.Grid());
|
||||
ret.Checkerboard() = rhs.Grid()->CheckerBoardDestination(rhs.Checkerboard(),shift,dimension);
|
||||
Cshift_local(ret,rhs,dimension,shift);
|
||||
return ret;
|
||||
}
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,4 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
Vector<std::pair<int,int> > Cshift_table;
|
||||
NAMESPACE_END(Grid);
|
||||
-22091
File diff suppressed because it is too large
Load Diff
@@ -1,49 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Lattice.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
#include <Grid/lattice/Lattice_view.h>
|
||||
#include <Grid/lattice/Lattice_base.h>
|
||||
#include <Grid/lattice/Lattice_conformable.h>
|
||||
#include <Grid/lattice/Lattice_ET.h>
|
||||
#include <Grid/lattice/Lattice_arith.h>
|
||||
#include <Grid/lattice/Lattice_trace.h>
|
||||
#include <Grid/lattice/Lattice_transpose.h>
|
||||
#include <Grid/lattice/Lattice_local.h>
|
||||
#include <Grid/lattice/Lattice_reduction.h>
|
||||
#include <Grid/lattice/Lattice_peekpoke.h>
|
||||
#include <Grid/lattice/Lattice_reality.h>
|
||||
#include <Grid/lattice/Lattice_real_imag.h>
|
||||
#include <Grid/lattice/Lattice_comparison_utils.h>
|
||||
#include <Grid/lattice/Lattice_comparison.h>
|
||||
#include <Grid/lattice/Lattice_coordinate.h>
|
||||
//#include <Grid/lattice/Lattice_where.h>
|
||||
#include <Grid/lattice/Lattice_rng.h>
|
||||
#include <Grid/lattice/Lattice_unary.h>
|
||||
#include <Grid/lattice/Lattice_transfer.h>
|
||||
#include <Grid/lattice/Lattice_basis.h>
|
||||
#include <Grid/lattice/Lattice_crc.h>
|
||||
@@ -1,530 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/lattice/Lattice_ET.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: neo <cossu@post.kek.jp>
|
||||
Author: Christoph Lehner <christoph@lhnr.de
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_LATTICE_ET_H
|
||||
#define GRID_LATTICE_ET_H
|
||||
|
||||
#include <iostream>
|
||||
#include <tuple>
|
||||
#include <typeinfo>
|
||||
#include <vector>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Predicated where support
|
||||
////////////////////////////////////////////////////
|
||||
#ifdef GRID_SIMT
|
||||
// drop to scalar in SIMT; cleaner in fact
|
||||
template <class iobj, class vobj, class robj>
|
||||
accelerator_inline vobj predicatedWhere(const iobj &predicate,
|
||||
const vobj &iftrue,
|
||||
const robj &iffalse)
|
||||
{
|
||||
Integer mask = TensorRemove(predicate);
|
||||
typename std::remove_const<vobj>::type ret= iffalse;
|
||||
if (mask) ret=iftrue;
|
||||
return ret;
|
||||
}
|
||||
#else
|
||||
template <class iobj, class vobj, class robj>
|
||||
accelerator_inline vobj predicatedWhere(const iobj &predicate,
|
||||
const vobj &iftrue,
|
||||
const robj &iffalse)
|
||||
{
|
||||
typename std::remove_const<vobj>::type ret;
|
||||
|
||||
typedef typename vobj::scalar_object scalar_object;
|
||||
// typedef typename vobj::scalar_type scalar_type;
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
|
||||
const int Nsimd = vobj::vector_type::Nsimd();
|
||||
|
||||
ExtractBuffer<Integer> mask(Nsimd);
|
||||
ExtractBuffer<scalar_object> truevals(Nsimd);
|
||||
ExtractBuffer<scalar_object> falsevals(Nsimd);
|
||||
|
||||
extract(iftrue, truevals);
|
||||
extract(iffalse, falsevals);
|
||||
extract<vInteger, Integer>(TensorRemove(predicate), mask);
|
||||
|
||||
for (int s = 0; s < Nsimd; s++) {
|
||||
if (mask[s]) falsevals[s] = truevals[s];
|
||||
}
|
||||
|
||||
merge(ret, falsevals);
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
//Specialization of getVectorType for lattices
|
||||
/////////////////////////////////////////////////////
|
||||
template<typename T>
|
||||
struct getVectorType<Lattice<T> >{
|
||||
typedef typename Lattice<T>::vector_object type;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////
|
||||
//-- recursive evaluation of expressions; --
|
||||
// handle leaves of syntax tree
|
||||
///////////////////////////////////////////////////
|
||||
template<class sobj,
|
||||
typename std::enable_if<!is_lattice<sobj>::value&&!is_lattice_expr<sobj>::value,sobj>::type * = nullptr>
|
||||
accelerator_inline
|
||||
sobj eval(const uint64_t ss, const sobj &arg)
|
||||
{
|
||||
return arg;
|
||||
}
|
||||
template <class lobj> accelerator_inline
|
||||
auto eval(const uint64_t ss, const LatticeView<lobj> &arg) -> decltype(arg(ss))
|
||||
{
|
||||
return arg(ss);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////
|
||||
//-- recursive evaluation of expressions; --
|
||||
// whole vector return, used only for expression return type inference
|
||||
///////////////////////////////////////////////////
|
||||
template<class sobj> accelerator_inline
|
||||
sobj vecEval(const uint64_t ss, const sobj &arg)
|
||||
{
|
||||
return arg;
|
||||
}
|
||||
template <class lobj> accelerator_inline
|
||||
const lobj & vecEval(const uint64_t ss, const LatticeView<lobj> &arg)
|
||||
{
|
||||
return arg[ss];
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// handle nodes in syntax tree- eval one operand
|
||||
// vecEval needed (but never called as all expressions offloaded) to infer the return type
|
||||
// in SIMT contexts of closure.
|
||||
///////////////////////////////////////////////////
|
||||
template <typename Op, typename T1> accelerator_inline
|
||||
auto vecEval(const uint64_t ss, const LatticeUnaryExpression<Op, T1> &expr)
|
||||
-> decltype(expr.op.func( vecEval(ss, expr.arg1)))
|
||||
{
|
||||
return expr.op.func( vecEval(ss, expr.arg1) );
|
||||
}
|
||||
// vecEval two operands
|
||||
template <typename Op, typename T1, typename T2> accelerator_inline
|
||||
auto vecEval(const uint64_t ss, const LatticeBinaryExpression<Op, T1, T2> &expr)
|
||||
-> decltype(expr.op.func( vecEval(ss,expr.arg1),vecEval(ss,expr.arg2)))
|
||||
{
|
||||
return expr.op.func( vecEval(ss,expr.arg1), vecEval(ss,expr.arg2) );
|
||||
}
|
||||
// vecEval three operands
|
||||
template <typename Op, typename T1, typename T2, typename T3> accelerator_inline
|
||||
auto vecEval(const uint64_t ss, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
||||
-> decltype(expr.op.func(vecEval(ss, expr.arg1), vecEval(ss, expr.arg2), vecEval(ss, expr.arg3)))
|
||||
{
|
||||
return expr.op.func(vecEval(ss, expr.arg1), vecEval(ss, expr.arg2), vecEval(ss, expr.arg3));
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// handle nodes in syntax tree- eval one operand coalesced
|
||||
///////////////////////////////////////////////////
|
||||
template <typename Op, typename T1> accelerator_inline
|
||||
auto eval(const uint64_t ss, const LatticeUnaryExpression<Op, T1> &expr)
|
||||
-> decltype(expr.op.func( eval(ss, expr.arg1)))
|
||||
{
|
||||
return expr.op.func( eval(ss, expr.arg1) );
|
||||
}
|
||||
// eval two operands
|
||||
template <typename Op, typename T1, typename T2> accelerator_inline
|
||||
auto eval(const uint64_t ss, const LatticeBinaryExpression<Op, T1, T2> &expr)
|
||||
-> decltype(expr.op.func( eval(ss,expr.arg1),eval(ss,expr.arg2)))
|
||||
{
|
||||
return expr.op.func( eval(ss,expr.arg1), eval(ss,expr.arg2) );
|
||||
}
|
||||
// eval three operands
|
||||
template <typename Op, typename T1, typename T2, typename T3> accelerator_inline
|
||||
auto eval(const uint64_t ss, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
||||
-> decltype(expr.op.func(eval(ss, expr.arg1),
|
||||
eval(ss, expr.arg2),
|
||||
eval(ss, expr.arg3)))
|
||||
{
|
||||
#ifdef GRID_SIMT
|
||||
// Handles Nsimd (vInteger) != Nsimd(ComplexD)
|
||||
typedef decltype(vecEval(ss, expr.arg2)) rvobj;
|
||||
typedef typename std::remove_reference<rvobj>::type vobj;
|
||||
|
||||
const int Nsimd = vobj::vector_type::Nsimd();
|
||||
|
||||
auto vpred = vecEval(ss,expr.arg1);
|
||||
|
||||
ExtractBuffer<Integer> mask(Nsimd);
|
||||
extract<vInteger, Integer>(TensorRemove(vpred), mask);
|
||||
|
||||
int s = acceleratorSIMTlane(Nsimd);
|
||||
return expr.op.func(mask[s],
|
||||
eval(ss, expr.arg2),
|
||||
eval(ss, expr.arg3));
|
||||
#else
|
||||
return expr.op.func(eval(ss, expr.arg1),
|
||||
eval(ss, expr.arg2),
|
||||
eval(ss, expr.arg3));
|
||||
#endif
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Obtain the grid from an expression, ensuring conformable. This must follow a
|
||||
// tree recursion; must retain grid pointer in the LatticeView class which sucks
|
||||
// Use a different method, and make it void *.
|
||||
// Perhaps a conformable method.
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
accelerator_inline void GridFromExpression(GridBase *&grid, const T1 &lat) // Lattice leaf
|
||||
{
|
||||
lat.Conformable(grid);
|
||||
}
|
||||
|
||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
accelerator_inline
|
||||
void GridFromExpression(GridBase *&grid,const T1 ¬lat) // non-lattice leaf
|
||||
{}
|
||||
|
||||
template <typename Op, typename T1>
|
||||
accelerator_inline
|
||||
void GridFromExpression(GridBase *&grid,const LatticeUnaryExpression<Op, T1> &expr)
|
||||
{
|
||||
GridFromExpression(grid, expr.arg1); // recurse
|
||||
}
|
||||
|
||||
template <typename Op, typename T1, typename T2>
|
||||
accelerator_inline
|
||||
void GridFromExpression(GridBase *&grid, const LatticeBinaryExpression<Op, T1, T2> &expr)
|
||||
{
|
||||
GridFromExpression(grid, expr.arg1); // recurse
|
||||
GridFromExpression(grid, expr.arg2);
|
||||
}
|
||||
template <typename Op, typename T1, typename T2, typename T3>
|
||||
accelerator_inline
|
||||
void GridFromExpression(GridBase *&grid, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
||||
{
|
||||
GridFromExpression(grid, expr.arg1); // recurse
|
||||
GridFromExpression(grid, expr.arg2); // recurse
|
||||
GridFromExpression(grid, expr.arg3); // recurse
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Obtain the CB from an expression, ensuring conformable. This must follow a
|
||||
// tree recursion
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
inline void CBFromExpression(int &cb, const T1 &lat) // Lattice leaf
|
||||
{
|
||||
if ((cb == Odd) || (cb == Even)) {
|
||||
assert(cb == lat.Checkerboard());
|
||||
}
|
||||
cb = lat.Checkerboard();
|
||||
}
|
||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
inline void CBFromExpression(int &cb, const T1 ¬lat) {} // non-lattice leaf
|
||||
template <typename Op, typename T1> inline
|
||||
void CBFromExpression(int &cb,const LatticeUnaryExpression<Op, T1> &expr)
|
||||
{
|
||||
CBFromExpression(cb, expr.arg1); // recurse AST
|
||||
}
|
||||
template <typename Op, typename T1, typename T2> inline
|
||||
void CBFromExpression(int &cb,const LatticeBinaryExpression<Op, T1, T2> &expr)
|
||||
{
|
||||
CBFromExpression(cb, expr.arg1); // recurse AST
|
||||
CBFromExpression(cb, expr.arg2); // recurse AST
|
||||
}
|
||||
template <typename Op, typename T1, typename T2, typename T3>
|
||||
inline void CBFromExpression(int &cb, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
||||
{
|
||||
CBFromExpression(cb, expr.arg1); // recurse AST
|
||||
CBFromExpression(cb, expr.arg2); // recurse AST
|
||||
CBFromExpression(cb, expr.arg3); // recurse AST
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// ViewOpen
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
inline void ExpressionViewOpen(T1 &lat) // Lattice leaf
|
||||
{
|
||||
lat.ViewOpen(AcceleratorRead);
|
||||
}
|
||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
inline void ExpressionViewOpen(T1 ¬lat) {}
|
||||
|
||||
template <typename Op, typename T1> inline
|
||||
void ExpressionViewOpen(LatticeUnaryExpression<Op, T1> &expr)
|
||||
{
|
||||
ExpressionViewOpen(expr.arg1); // recurse AST
|
||||
}
|
||||
|
||||
template <typename Op, typename T1, typename T2> inline
|
||||
void ExpressionViewOpen(LatticeBinaryExpression<Op, T1, T2> &expr)
|
||||
{
|
||||
ExpressionViewOpen(expr.arg1); // recurse AST
|
||||
ExpressionViewOpen(expr.arg2); // rrecurse AST
|
||||
}
|
||||
template <typename Op, typename T1, typename T2, typename T3>
|
||||
inline void ExpressionViewOpen(LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
||||
{
|
||||
ExpressionViewOpen(expr.arg1); // recurse AST
|
||||
ExpressionViewOpen(expr.arg2); // recurse AST
|
||||
ExpressionViewOpen(expr.arg3); // recurse AST
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// ViewClose
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
inline void ExpressionViewClose( T1 &lat) // Lattice leaf
|
||||
{
|
||||
lat.ViewClose();
|
||||
}
|
||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
||||
inline void ExpressionViewClose(T1 ¬lat) {}
|
||||
|
||||
template <typename Op, typename T1> inline
|
||||
void ExpressionViewClose(LatticeUnaryExpression<Op, T1> &expr)
|
||||
{
|
||||
ExpressionViewClose(expr.arg1); // recurse AST
|
||||
}
|
||||
template <typename Op, typename T1, typename T2> inline
|
||||
void ExpressionViewClose(LatticeBinaryExpression<Op, T1, T2> &expr)
|
||||
{
|
||||
ExpressionViewClose(expr.arg1); // recurse AST
|
||||
ExpressionViewClose(expr.arg2); // recurse AST
|
||||
}
|
||||
template <typename Op, typename T1, typename T2, typename T3>
|
||||
inline void ExpressionViewClose(LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
||||
{
|
||||
ExpressionViewClose(expr.arg1); // recurse AST
|
||||
ExpressionViewClose(expr.arg2); // recurse AST
|
||||
ExpressionViewClose(expr.arg3); // recurse AST
|
||||
}
|
||||
|
||||
////////////////////////////////////////////
|
||||
// Unary operators and funcs
|
||||
////////////////////////////////////////////
|
||||
#define GridUnopClass(name, ret) \
|
||||
struct name { \
|
||||
template<class _arg> static auto accelerator_inline func(const _arg a) -> decltype(ret) { return ret; } \
|
||||
};
|
||||
|
||||
GridUnopClass(UnarySub, -a);
|
||||
GridUnopClass(UnaryNot, Not(a));
|
||||
GridUnopClass(UnaryTrace, trace(a));
|
||||
GridUnopClass(UnaryTranspose, transpose(a));
|
||||
GridUnopClass(UnaryTa, Ta(a));
|
||||
GridUnopClass(UnaryProjectOnGroup, ProjectOnGroup(a));
|
||||
GridUnopClass(UnaryTimesI, timesI(a));
|
||||
GridUnopClass(UnaryTimesMinusI, timesMinusI(a));
|
||||
GridUnopClass(UnaryAbs, abs(a));
|
||||
GridUnopClass(UnarySqrt, sqrt(a));
|
||||
GridUnopClass(UnarySin, sin(a));
|
||||
GridUnopClass(UnaryCos, cos(a));
|
||||
GridUnopClass(UnaryAsin, asin(a));
|
||||
GridUnopClass(UnaryAcos, acos(a));
|
||||
GridUnopClass(UnaryLog, log(a));
|
||||
GridUnopClass(UnaryExp, exp(a));
|
||||
|
||||
////////////////////////////////////////////
|
||||
// Binary operators
|
||||
////////////////////////////////////////////
|
||||
#define GridBinOpClass(name, combination) \
|
||||
struct name { \
|
||||
template <class _left, class _right> \
|
||||
static auto accelerator_inline \
|
||||
func(const _left &lhs, const _right &rhs) \
|
||||
-> decltype(combination) const \
|
||||
{ \
|
||||
return combination; \
|
||||
} \
|
||||
};
|
||||
|
||||
GridBinOpClass(BinaryAdd, lhs + rhs);
|
||||
GridBinOpClass(BinarySub, lhs - rhs);
|
||||
GridBinOpClass(BinaryMul, lhs *rhs);
|
||||
GridBinOpClass(BinaryDiv, lhs /rhs);
|
||||
GridBinOpClass(BinaryAnd, lhs &rhs);
|
||||
GridBinOpClass(BinaryOr, lhs | rhs);
|
||||
GridBinOpClass(BinaryAndAnd, lhs &&rhs);
|
||||
GridBinOpClass(BinaryOrOr, lhs || rhs);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Trinary conditional op
|
||||
////////////////////////////////////////////////////
|
||||
#define GridTrinOpClass(name, combination) \
|
||||
struct name { \
|
||||
template <class _predicate,class _left, class _right> \
|
||||
static auto accelerator_inline \
|
||||
func(const _predicate &pred, const _left &lhs, const _right &rhs) \
|
||||
-> decltype(combination) const \
|
||||
{ \
|
||||
return combination; \
|
||||
} \
|
||||
};
|
||||
|
||||
GridTrinOpClass(TrinaryWhere,
|
||||
(predicatedWhere<
|
||||
typename std::remove_reference<_predicate>::type,
|
||||
typename std::remove_reference<_left>::type,
|
||||
typename std::remove_reference<_right>::type>(pred, lhs,rhs)));
|
||||
|
||||
////////////////////////////////////////////
|
||||
// Operator syntactical glue
|
||||
////////////////////////////////////////////
|
||||
#define GRID_UNOP(name) name
|
||||
#define GRID_BINOP(name) name
|
||||
#define GRID_TRINOP(name) name
|
||||
|
||||
#define GRID_DEF_UNOP(op, name) \
|
||||
template <typename T1, typename std::enable_if<is_lattice<T1>::value||is_lattice_expr<T1>::value,T1>::type * = nullptr> \
|
||||
inline auto op(const T1 &arg) ->decltype(LatticeUnaryExpression<GRID_UNOP(name),T1>(GRID_UNOP(name)(), arg)) \
|
||||
{ \
|
||||
return LatticeUnaryExpression<GRID_UNOP(name),T1>(GRID_UNOP(name)(), arg); \
|
||||
}
|
||||
|
||||
#define GRID_BINOP_LEFT(op, name) \
|
||||
template <typename T1, typename T2, \
|
||||
typename std::enable_if<is_lattice<T1>::value||is_lattice_expr<T1>::value,T1>::type * = nullptr> \
|
||||
inline auto op(const T1 &lhs, const T2 &rhs) \
|
||||
->decltype(LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs,rhs)) \
|
||||
{ \
|
||||
return LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs,rhs);\
|
||||
}
|
||||
|
||||
#define GRID_BINOP_RIGHT(op, name) \
|
||||
template <typename T1, typename T2, \
|
||||
typename std::enable_if<!is_lattice<T1>::value&&!is_lattice_expr<T1>::value,T1>::type * = nullptr, \
|
||||
typename std::enable_if< is_lattice<T2>::value|| is_lattice_expr<T2>::value,T2>::type * = nullptr> \
|
||||
inline auto op(const T1 &lhs, const T2 &rhs) \
|
||||
->decltype(LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs, rhs)) \
|
||||
{ \
|
||||
return LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs, rhs); \
|
||||
}
|
||||
|
||||
#define GRID_DEF_BINOP(op, name) \
|
||||
GRID_BINOP_LEFT(op, name); \
|
||||
GRID_BINOP_RIGHT(op, name);
|
||||
|
||||
#define GRID_DEF_TRINOP(op, name) \
|
||||
template <typename T1, typename T2, typename T3> \
|
||||
inline auto op(const T1 &pred, const T2 &lhs, const T3 &rhs) \
|
||||
->decltype(LatticeTrinaryExpression<GRID_TRINOP(name),T1,T2,T3>(GRID_TRINOP(name)(),pred, lhs, rhs)) \
|
||||
{ \
|
||||
return LatticeTrinaryExpression<GRID_TRINOP(name),T1,T2,T3>(GRID_TRINOP(name)(),pred, lhs, rhs); \
|
||||
}
|
||||
|
||||
////////////////////////
|
||||
// Operator definitions
|
||||
////////////////////////
|
||||
GRID_DEF_UNOP(operator-, UnarySub);
|
||||
GRID_DEF_UNOP(Not, UnaryNot);
|
||||
GRID_DEF_UNOP(operator!, UnaryNot);
|
||||
//GRID_DEF_UNOP(adj, UnaryAdj);
|
||||
//GRID_DEF_UNOP(conjugate, UnaryConj);
|
||||
GRID_DEF_UNOP(trace, UnaryTrace);
|
||||
GRID_DEF_UNOP(transpose, UnaryTranspose);
|
||||
GRID_DEF_UNOP(Ta, UnaryTa);
|
||||
GRID_DEF_UNOP(ProjectOnGroup, UnaryProjectOnGroup);
|
||||
GRID_DEF_UNOP(timesI, UnaryTimesI);
|
||||
GRID_DEF_UNOP(timesMinusI, UnaryTimesMinusI);
|
||||
GRID_DEF_UNOP(abs, UnaryAbs); // abs overloaded in cmath C++98; DON'T do the
|
||||
// abs-fabs-dabs-labs thing
|
||||
GRID_DEF_UNOP(sqrt, UnarySqrt);
|
||||
GRID_DEF_UNOP(sin, UnarySin);
|
||||
GRID_DEF_UNOP(cos, UnaryCos);
|
||||
GRID_DEF_UNOP(asin, UnaryAsin);
|
||||
GRID_DEF_UNOP(acos, UnaryAcos);
|
||||
GRID_DEF_UNOP(log, UnaryLog);
|
||||
GRID_DEF_UNOP(exp, UnaryExp);
|
||||
|
||||
GRID_DEF_BINOP(operator+, BinaryAdd);
|
||||
GRID_DEF_BINOP(operator-, BinarySub);
|
||||
GRID_DEF_BINOP(operator*, BinaryMul);
|
||||
GRID_DEF_BINOP(operator/, BinaryDiv);
|
||||
|
||||
GRID_DEF_BINOP(operator&, BinaryAnd);
|
||||
GRID_DEF_BINOP(operator|, BinaryOr);
|
||||
GRID_DEF_BINOP(operator&&, BinaryAndAnd);
|
||||
GRID_DEF_BINOP(operator||, BinaryOrOr);
|
||||
|
||||
GRID_DEF_TRINOP(where, TrinaryWhere);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Closure convenience to force expression to evaluate
|
||||
/////////////////////////////////////////////////////////////
|
||||
template <class Op, class T1>
|
||||
auto closure(const LatticeUnaryExpression<Op, T1> &expr)
|
||||
-> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1)))>::type >
|
||||
{
|
||||
Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1)))>::type > ret(expr);
|
||||
return ret;
|
||||
}
|
||||
template <class Op, class T1, class T2>
|
||||
auto closure(const LatticeBinaryExpression<Op, T1, T2> &expr)
|
||||
-> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),vecEval(0, expr.arg2)))>::type >
|
||||
{
|
||||
Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),vecEval(0, expr.arg2)))>::type > ret(expr);
|
||||
return ret;
|
||||
}
|
||||
template <class Op, class T1, class T2, class T3>
|
||||
auto closure(const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
||||
-> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),
|
||||
vecEval(0, expr.arg2),
|
||||
vecEval(0, expr.arg3)))>::type >
|
||||
{
|
||||
Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),
|
||||
vecEval(0, expr.arg2),
|
||||
vecEval(0, expr.arg3)))>::type > ret(expr);
|
||||
return ret;
|
||||
}
|
||||
#define EXPRESSION_CLOSURE(function) \
|
||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> \
|
||||
auto function(Expression &expr) -> decltype(function(closure(expr))) \
|
||||
{ \
|
||||
return function(closure(expr)); \
|
||||
}
|
||||
|
||||
|
||||
#undef GRID_UNOP
|
||||
#undef GRID_BINOP
|
||||
#undef GRID_TRINOP
|
||||
|
||||
#undef GRID_DEF_UNOP
|
||||
#undef GRID_DEF_BINOP
|
||||
#undef GRID_DEF_TRINOP
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,274 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/lattice/Lattice_arith.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christoph Lehner <christoph@lhnr.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_LATTICE_ARITH_H
|
||||
#define GRID_LATTICE_ARITH_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// avoid copy back routines for mult, mac, sub, add
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void mult(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("mult");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
autoView( rhs_v , rhs, AcceleratorRead);
|
||||
conformable(ret,rhs);
|
||||
conformable(lhs,rhs);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto lhs_t = lhs_v(ss);
|
||||
auto rhs_t = rhs_v(ss);
|
||||
mult(&tmp,&lhs_t,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void mac(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("mac");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
conformable(ret,rhs);
|
||||
conformable(lhs,rhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
autoView( rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
auto lhs_t=lhs_v(ss);
|
||||
auto rhs_t=rhs_v(ss);
|
||||
auto tmp =ret_v(ss);
|
||||
mac(&tmp,&lhs_t,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void sub(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("sub");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
conformable(ret,rhs);
|
||||
conformable(lhs,rhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
autoView( rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto lhs_t=lhs_v(ss);
|
||||
auto rhs_t=rhs_v(ss);
|
||||
sub(&tmp,&lhs_t,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void add(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("add");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
conformable(ret,rhs);
|
||||
conformable(lhs,rhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
autoView( rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto lhs_t=lhs_v(ss);
|
||||
auto rhs_t=rhs_v(ss);
|
||||
add(&tmp,&lhs_t,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// avoid copy back routines for mult, mac, sub, add
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void mult(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
||||
GRID_TRACE("mult");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
conformable(lhs,ret);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
mult(&tmp,&lhs_v(ss),&rhs);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void mac(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
||||
GRID_TRACE("mac");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
conformable(ret,lhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
auto tmp =ret_v(ss);
|
||||
auto lhs_t=lhs_v(ss);
|
||||
mac(&tmp,&lhs_t,&rhs);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void sub(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
||||
GRID_TRACE("sub");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
conformable(ret,lhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto lhs_t=lhs_v(ss);
|
||||
sub(&tmp,&lhs_t,&rhs);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void add(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
||||
GRID_TRACE("add");
|
||||
ret.Checkerboard() = lhs.Checkerboard();
|
||||
conformable(lhs,ret);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( lhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto lhs_t=lhs_v(ss);
|
||||
add(&tmp,&lhs_t,&rhs);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// avoid copy back routines for mult, mac, sub, add
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void mult(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("mult");
|
||||
ret.Checkerboard() = rhs.Checkerboard();
|
||||
conformable(ret,rhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( rhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto rhs_t=rhs_v(ss);
|
||||
mult(&tmp,&lhs,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void mac(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("mac");
|
||||
ret.Checkerboard() = rhs.Checkerboard();
|
||||
conformable(ret,rhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( rhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
||||
auto tmp =ret_v(ss);
|
||||
auto rhs_t=rhs_v(ss);
|
||||
mac(&tmp,&lhs,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void sub(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("sub");
|
||||
ret.Checkerboard() = rhs.Checkerboard();
|
||||
conformable(ret,rhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( rhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto rhs_t=rhs_v(ss);
|
||||
sub(&tmp,&lhs,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
template<class obj1,class obj2,class obj3> inline
|
||||
void add(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
||||
GRID_TRACE("add");
|
||||
ret.Checkerboard() = rhs.Checkerboard();
|
||||
conformable(ret,rhs);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( rhs_v , lhs, AcceleratorRead);
|
||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
||||
decltype(coalescedRead(obj1())) tmp;
|
||||
auto rhs_t=rhs_v(ss);
|
||||
add(&tmp,&lhs,&rhs_t);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class sobj,class vobj> inline
|
||||
void axpy(Lattice<vobj> &ret,sobj a,const Lattice<vobj> &x,const Lattice<vobj> &y){
|
||||
GRID_TRACE("axpy");
|
||||
ret.Checkerboard() = x.Checkerboard();
|
||||
conformable(ret,x);
|
||||
conformable(x,y);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( x_v , x, AcceleratorRead);
|
||||
autoView( y_v , y, AcceleratorRead);
|
||||
accelerator_for(ss,x_v.size(),vobj::Nsimd(),{
|
||||
auto tmp = a*coalescedRead(x_v[ss])+coalescedRead(y_v[ss]);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
template<class sobj,class vobj> inline
|
||||
void axpby(Lattice<vobj> &ret,sobj a,sobj b,const Lattice<vobj> &x,const Lattice<vobj> &y){
|
||||
GRID_TRACE("axpby");
|
||||
ret.Checkerboard() = x.Checkerboard();
|
||||
conformable(ret,x);
|
||||
conformable(x,y);
|
||||
autoView( ret_v , ret, AcceleratorWrite);
|
||||
autoView( x_v , x, AcceleratorRead);
|
||||
autoView( y_v , y, AcceleratorRead);
|
||||
accelerator_for(ss,x_v.size(),vobj::Nsimd(),{
|
||||
auto tmp = a*x_v(ss)+b*y_v(ss);
|
||||
coalescedWrite(ret_v[ss],tmp);
|
||||
});
|
||||
}
|
||||
|
||||
template<class sobj,class vobj> inline
|
||||
RealD axpy_norm(Lattice<vobj> &ret,sobj a,const Lattice<vobj> &x,const Lattice<vobj> &y)
|
||||
{
|
||||
GRID_TRACE("axpy_norm");
|
||||
return axpy_norm_fast(ret,a,x,y);
|
||||
}
|
||||
template<class sobj,class vobj> inline
|
||||
RealD axpby_norm(Lattice<vobj> &ret,sobj a,sobj b,const Lattice<vobj> &x,const Lattice<vobj> &y)
|
||||
{
|
||||
GRID_TRACE("axpby_norm");
|
||||
return axpby_norm_fast(ret,a,b,x,y);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,382 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/lattice/Lattice_base.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christoph Lehner <christoph@lhnr.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#pragma once
|
||||
|
||||
#define STREAMING_STORES
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
extern int GridCshiftPermuteMap[4][16];
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
// The real lattice class, with normal copy and assignment semantics.
|
||||
// This contains extra (host resident) grid pointer data that may be accessed by host code
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class vobj>
|
||||
class Lattice : public LatticeAccelerator<vobj>
|
||||
{
|
||||
public:
|
||||
GridBase *Grid(void) const { return this->_grid; }
|
||||
///////////////////////////////////////////////////
|
||||
// Member types
|
||||
///////////////////////////////////////////////////
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename vobj::scalar_object scalar_object;
|
||||
typedef vobj vector_object;
|
||||
|
||||
private:
|
||||
void dealloc(void)
|
||||
{
|
||||
if( this->_odata_size ) {
|
||||
alignedAllocator<vobj> alloc;
|
||||
alloc.deallocate(this->_odata,this->_odata_size);
|
||||
this->_odata=nullptr;
|
||||
this->_odata_size=0;
|
||||
}
|
||||
}
|
||||
void resize(uint64_t size)
|
||||
{
|
||||
if ( this->_odata_size != size ) {
|
||||
alignedAllocator<vobj> alloc;
|
||||
|
||||
dealloc();
|
||||
|
||||
this->_odata_size = size;
|
||||
if ( size )
|
||||
this->_odata = alloc.allocate(this->_odata_size);
|
||||
else
|
||||
this->_odata = nullptr;
|
||||
}
|
||||
}
|
||||
public:
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// Can use to make accelerator dirty without copy from host ; useful for temporaries "dont care" prev contents
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
void SetViewMode(ViewMode mode) {
|
||||
LatticeView<vobj> accessor(*( (LatticeAccelerator<vobj> *) this),mode);
|
||||
accessor.ViewClose();
|
||||
}
|
||||
|
||||
// Helper function to print the state of this object in the AccCache
|
||||
void PrintCacheState(void)
|
||||
{
|
||||
MemoryManager::PrintState(this->_odata);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// Return a view object that may be dereferenced in site loops.
|
||||
// The view is trivially copy constructible and may be copied to an accelerator device
|
||||
// in device lambdas
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
LatticeView<vobj> View (ViewMode mode) const
|
||||
{
|
||||
LatticeView<vobj> accessor(*( (LatticeAccelerator<vobj> *) this),mode);
|
||||
return accessor;
|
||||
}
|
||||
|
||||
~Lattice() {
|
||||
if ( this->_odata_size ) {
|
||||
dealloc();
|
||||
}
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Expression Template closure support
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename Op, typename T1> inline Lattice<vobj> & operator=(const LatticeUnaryExpression<Op,T1> &expr)
|
||||
{
|
||||
GRID_TRACE("ExpressionTemplateEval");
|
||||
GridBase *egrid(nullptr);
|
||||
GridFromExpression(egrid,expr);
|
||||
assert(egrid!=nullptr);
|
||||
conformable(this->_grid,egrid);
|
||||
|
||||
int cb=-1;
|
||||
CBFromExpression(cb,expr);
|
||||
assert( (cb==Odd) || (cb==Even));
|
||||
this->checkerboard=cb;
|
||||
|
||||
auto exprCopy = expr;
|
||||
ExpressionViewOpen(exprCopy);
|
||||
auto me = View(AcceleratorWriteDiscard);
|
||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
||||
auto tmp = eval(ss,exprCopy);
|
||||
coalescedWrite(me[ss],tmp);
|
||||
});
|
||||
me.ViewClose();
|
||||
ExpressionViewClose(exprCopy);
|
||||
return *this;
|
||||
}
|
||||
template <typename Op, typename T1,typename T2> inline Lattice<vobj> & operator=(const LatticeBinaryExpression<Op,T1,T2> &expr)
|
||||
{
|
||||
GRID_TRACE("ExpressionTemplateEval");
|
||||
GridBase *egrid(nullptr);
|
||||
GridFromExpression(egrid,expr);
|
||||
assert(egrid!=nullptr);
|
||||
conformable(this->_grid,egrid);
|
||||
|
||||
int cb=-1;
|
||||
CBFromExpression(cb,expr);
|
||||
assert( (cb==Odd) || (cb==Even));
|
||||
this->checkerboard=cb;
|
||||
|
||||
auto exprCopy = expr;
|
||||
ExpressionViewOpen(exprCopy);
|
||||
auto me = View(AcceleratorWriteDiscard);
|
||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
||||
auto tmp = eval(ss,exprCopy);
|
||||
coalescedWrite(me[ss],tmp);
|
||||
});
|
||||
me.ViewClose();
|
||||
ExpressionViewClose(exprCopy);
|
||||
return *this;
|
||||
}
|
||||
template <typename Op, typename T1,typename T2,typename T3> inline Lattice<vobj> & operator=(const LatticeTrinaryExpression<Op,T1,T2,T3> &expr)
|
||||
{
|
||||
GRID_TRACE("ExpressionTemplateEval");
|
||||
GridBase *egrid(nullptr);
|
||||
GridFromExpression(egrid,expr);
|
||||
assert(egrid!=nullptr);
|
||||
conformable(this->_grid,egrid);
|
||||
|
||||
int cb=-1;
|
||||
CBFromExpression(cb,expr);
|
||||
assert( (cb==Odd) || (cb==Even));
|
||||
this->checkerboard=cb;
|
||||
auto exprCopy = expr;
|
||||
ExpressionViewOpen(exprCopy);
|
||||
auto me = View(AcceleratorWriteDiscard);
|
||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
||||
auto tmp = eval(ss,exprCopy);
|
||||
coalescedWrite(me[ss],tmp);
|
||||
});
|
||||
me.ViewClose();
|
||||
ExpressionViewClose(exprCopy);
|
||||
return *this;
|
||||
}
|
||||
//GridFromExpression is tricky to do
|
||||
template<class Op,class T1>
|
||||
Lattice(const LatticeUnaryExpression<Op,T1> & expr) {
|
||||
this->_grid = nullptr;
|
||||
GridFromExpression(this->_grid,expr);
|
||||
assert(this->_grid!=nullptr);
|
||||
|
||||
int cb=-1;
|
||||
CBFromExpression(cb,expr);
|
||||
assert( (cb==Odd) || (cb==Even));
|
||||
this->checkerboard=cb;
|
||||
|
||||
resize(this->_grid->oSites());
|
||||
|
||||
*this = expr;
|
||||
}
|
||||
template<class Op,class T1, class T2>
|
||||
Lattice(const LatticeBinaryExpression<Op,T1,T2> & expr) {
|
||||
this->_grid = nullptr;
|
||||
GridFromExpression(this->_grid,expr);
|
||||
assert(this->_grid!=nullptr);
|
||||
|
||||
int cb=-1;
|
||||
CBFromExpression(cb,expr);
|
||||
assert( (cb==Odd) || (cb==Even));
|
||||
this->checkerboard=cb;
|
||||
|
||||
resize(this->_grid->oSites());
|
||||
|
||||
*this = expr;
|
||||
}
|
||||
template<class Op,class T1, class T2, class T3>
|
||||
Lattice(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr) {
|
||||
this->_grid = nullptr;
|
||||
GridFromExpression(this->_grid,expr);
|
||||
assert(this->_grid!=nullptr);
|
||||
|
||||
int cb=-1;
|
||||
CBFromExpression(cb,expr);
|
||||
assert( (cb==Odd) || (cb==Even));
|
||||
this->checkerboard=cb;
|
||||
|
||||
resize(this->_grid->oSites());
|
||||
|
||||
*this = expr;
|
||||
}
|
||||
|
||||
template<class sobj> inline Lattice<vobj> & operator = (const sobj & r){
|
||||
auto me = View(CpuWrite);
|
||||
thread_for(ss,me.size(),{
|
||||
me[ss]= r;
|
||||
});
|
||||
me.ViewClose();
|
||||
return *this;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// Follow rule of five, with Constructor requires "grid" passed
|
||||
// to user defined constructor
|
||||
///////////////////////////////////////////
|
||||
// user defined constructor
|
||||
///////////////////////////////////////////
|
||||
Lattice(GridBase *grid,ViewMode mode=AcceleratorWriteDiscard) {
|
||||
this->_grid = grid;
|
||||
resize(this->_grid->oSites());
|
||||
assert((((uint64_t)&this->_odata[0])&0xF) ==0);
|
||||
this->checkerboard=0;
|
||||
SetViewMode(mode);
|
||||
}
|
||||
|
||||
// virtual ~Lattice(void) = default;
|
||||
|
||||
void reset(GridBase* grid) {
|
||||
if (this->_grid != grid) {
|
||||
this->_grid = grid;
|
||||
this->resize(grid->oSites());
|
||||
this->checkerboard = 0;
|
||||
}
|
||||
}
|
||||
///////////////////////////////////////////
|
||||
// copy constructor
|
||||
///////////////////////////////////////////
|
||||
Lattice(const Lattice& r){
|
||||
this->_grid = r.Grid();
|
||||
resize(this->_grid->oSites());
|
||||
*this = r;
|
||||
}
|
||||
///////////////////////////////////////////
|
||||
// move constructor
|
||||
///////////////////////////////////////////
|
||||
Lattice(Lattice && r){
|
||||
this->_grid = r.Grid();
|
||||
this->_odata = r._odata;
|
||||
this->_odata_size = r._odata_size;
|
||||
this->checkerboard= r.Checkerboard();
|
||||
r._odata = nullptr;
|
||||
r._odata_size = 0;
|
||||
}
|
||||
///////////////////////////////////////////
|
||||
// assignment template
|
||||
///////////////////////////////////////////
|
||||
template<class robj> inline Lattice<vobj> & operator = (const Lattice<robj> & r){
|
||||
typename std::enable_if<!std::is_same<robj,vobj>::value,int>::type i=0;
|
||||
conformable(*this,r);
|
||||
this->checkerboard = r.Checkerboard();
|
||||
auto him= r.View(AcceleratorRead);
|
||||
auto me = View(AcceleratorWriteDiscard);
|
||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
||||
coalescedWrite(me[ss],him(ss));
|
||||
});
|
||||
me.ViewClose(); him.ViewClose();
|
||||
return *this;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////
|
||||
// Copy assignment
|
||||
///////////////////////////////////////////
|
||||
inline Lattice<vobj> & operator = (const Lattice<vobj> & r){
|
||||
this->checkerboard = r.Checkerboard();
|
||||
conformable(*this,r);
|
||||
auto him= r.View(AcceleratorRead);
|
||||
auto me = View(AcceleratorWriteDiscard);
|
||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
||||
coalescedWrite(me[ss],him(ss));
|
||||
});
|
||||
me.ViewClose(); him.ViewClose();
|
||||
return *this;
|
||||
}
|
||||
///////////////////////////////////////////
|
||||
// Move assignment possible if same type
|
||||
///////////////////////////////////////////
|
||||
inline Lattice<vobj> & operator = (Lattice<vobj> && r){
|
||||
|
||||
resize(0); // deletes if appropriate
|
||||
this->_grid = r.Grid();
|
||||
this->_odata = r._odata;
|
||||
this->_odata_size = r._odata_size;
|
||||
this->checkerboard= r.Checkerboard();
|
||||
|
||||
r._odata = nullptr;
|
||||
r._odata_size = 0;
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
// *=,+=,-= operators inherit behvour from correspond */+/- operation
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
template<class T> inline Lattice<vobj> &operator *=(const T &r) {
|
||||
*this = (*this)*r;
|
||||
return *this;
|
||||
}
|
||||
|
||||
template<class T> inline Lattice<vobj> &operator -=(const T &r) {
|
||||
*this = (*this)-r;
|
||||
return *this;
|
||||
}
|
||||
template<class T> inline Lattice<vobj> &operator +=(const T &r) {
|
||||
*this = (*this)+r;
|
||||
return *this;
|
||||
}
|
||||
|
||||
friend inline void swap(Lattice &l, Lattice &r) {
|
||||
conformable(l,r);
|
||||
LatticeAccelerator<vobj> tmp;
|
||||
LatticeAccelerator<vobj> *lp = (LatticeAccelerator<vobj> *)&l;
|
||||
LatticeAccelerator<vobj> *rp = (LatticeAccelerator<vobj> *)&r;
|
||||
tmp = *lp; *lp=*rp; *rp=tmp;
|
||||
}
|
||||
|
||||
}; // class Lattice
|
||||
|
||||
template<class vobj> std::ostream& operator<< (std::ostream& stream, const Lattice<vobj> &o){
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
for(int g=0;g<o.Grid()->_gsites;g++){
|
||||
|
||||
Coordinate gcoor;
|
||||
o.Grid()->GlobalIndexToGlobalCoor(g,gcoor);
|
||||
|
||||
sobj ss;
|
||||
peekSite(ss,o,gcoor);
|
||||
stream<<"[";
|
||||
for(int d=0;d<gcoor.size();d++){
|
||||
stream<<gcoor[d];
|
||||
if(d!=gcoor.size()-1) stream<<",";
|
||||
}
|
||||
stream<<"]\t";
|
||||
stream<<ss<<std::endl;
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/lattice/Lattice_basis.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christoph Lehner <christoph@lhnr.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class Field>
|
||||
void basisOrthogonalize(std::vector<Field> &basis,Field &w,int k)
|
||||
{
|
||||
// If assume basis[j] are already orthonormal,
|
||||
// can take all inner products in parallel saving 2x bandwidth
|
||||
// Save 3x bandwidth on the second line of loop.
|
||||
// perhaps 2.5x speed up.
|
||||
// 2x overall in Multigrid Lanczos
|
||||
for(int j=0; j<k; ++j){
|
||||
auto ip = innerProduct(basis[j],w);
|
||||
w = w - ip*basis[j];
|
||||
}
|
||||
}
|
||||
|
||||
template<class VField, class Matrix>
|
||||
void basisRotate(VField &basis,Matrix& Qt,int j0, int j1, int k0,int k1,int Nm)
|
||||
{
|
||||
typedef decltype(basis[0]) Field;
|
||||
typedef decltype(basis[0].View(AcceleratorRead)) View;
|
||||
|
||||
Vector<View> basis_v; basis_v.reserve(basis.size());
|
||||
typedef typename std::remove_reference<decltype(basis_v[0][0])>::type vobj;
|
||||
typedef typename std::remove_reference<decltype(Qt(0,0))>::type Coeff_t;
|
||||
GridBase* grid = basis[0].Grid();
|
||||
|
||||
for(int k=0;k<basis.size();k++){
|
||||
basis_v.push_back(basis[k].View(AcceleratorWrite));
|
||||
}
|
||||
|
||||
#if ( (!defined(GRID_CUDA)) )
|
||||
int max_threads = thread_max();
|
||||
Vector < vobj > Bt(Nm * max_threads);
|
||||
thread_region
|
||||
{
|
||||
vobj* B = &Bt[Nm * thread_num()];
|
||||
thread_for_in_region(ss, grid->oSites(),{
|
||||
for(int j=j0; j<j1; ++j) B[j]=0.;
|
||||
|
||||
for(int j=j0; j<j1; ++j){
|
||||
for(int k=k0; k<k1; ++k){
|
||||
B[j] +=Qt(j,k) * basis_v[k][ss];
|
||||
}
|
||||
}
|
||||
for(int j=j0; j<j1; ++j){
|
||||
basis_v[j][ss] = B[j];
|
||||
}
|
||||
});
|
||||
}
|
||||
#else
|
||||
View *basis_vp = &basis_v[0];
|
||||
|
||||
int nrot = j1-j0;
|
||||
if (!nrot) // edge case not handled gracefully by Cuda
|
||||
return;
|
||||
|
||||
uint64_t oSites =grid->oSites();
|
||||
uint64_t siteBlock=(grid->oSites()+nrot-1)/nrot; // Maximum 1 additional vector overhead
|
||||
|
||||
Vector <vobj> Bt(siteBlock * nrot);
|
||||
auto Bp=&Bt[0];
|
||||
|
||||
// GPU readable copy of matrix
|
||||
Vector<Coeff_t> Qt_jv(Nm*Nm);
|
||||
Coeff_t *Qt_p = & Qt_jv[0];
|
||||
thread_for(i,Nm*Nm,{
|
||||
int j = i/Nm;
|
||||
int k = i%Nm;
|
||||
Qt_p[i]=Qt(j,k);
|
||||
});
|
||||
|
||||
// Block the loop to keep storage footprint down
|
||||
for(uint64_t s=0;s<oSites;s+=siteBlock){
|
||||
|
||||
// remaining work in this block
|
||||
int ssites=MIN(siteBlock,oSites-s);
|
||||
|
||||
// zero out the accumulators
|
||||
accelerator_for(ss,siteBlock*nrot,vobj::Nsimd(),{
|
||||
decltype(coalescedRead(Bp[ss])) z;
|
||||
z=Zero();
|
||||
coalescedWrite(Bp[ss],z);
|
||||
});
|
||||
|
||||
accelerator_for(sj,ssites*nrot,vobj::Nsimd(),{
|
||||
|
||||
int j =sj%nrot;
|
||||
int jj =j0+j;
|
||||
int ss =sj/nrot;
|
||||
int sss=ss+s;
|
||||
|
||||
for(int k=k0; k<k1; ++k){
|
||||
auto tmp = coalescedRead(Bp[ss*nrot+j]);
|
||||
coalescedWrite(Bp[ss*nrot+j],tmp+ Qt_p[jj*Nm+k] * coalescedRead(basis_vp[k][sss]));
|
||||
}
|
||||
});
|
||||
|
||||
accelerator_for(sj,ssites*nrot,vobj::Nsimd(),{
|
||||
int j =sj%nrot;
|
||||
int jj =j0+j;
|
||||
int ss =sj/nrot;
|
||||
int sss=ss+s;
|
||||
coalescedWrite(basis_vp[jj][sss],coalescedRead(Bp[ss*nrot+j]));
|
||||
});
|
||||
}
|
||||
#endif
|
||||
|
||||
for(int k=0;k<basis.size();k++) basis_v[k].ViewClose();
|
||||
}
|
||||
|
||||
// Extract a single rotated vector
|
||||
template<class Field>
|
||||
void basisRotateJ(Field &result,std::vector<Field> &basis,Eigen::MatrixXd& Qt,int j, int k0,int k1,int Nm)
|
||||
{
|
||||
typedef decltype(basis[0].View(AcceleratorRead)) View;
|
||||
typedef typename Field::vector_object vobj;
|
||||
GridBase* grid = basis[0].Grid();
|
||||
|
||||
result.Checkerboard() = basis[0].Checkerboard();
|
||||
|
||||
Vector<View> basis_v; basis_v.reserve(basis.size());
|
||||
for(int k=0;k<basis.size();k++){
|
||||
basis_v.push_back(basis[k].View(AcceleratorRead));
|
||||
}
|
||||
vobj zz=Zero();
|
||||
Vector<double> Qt_jv(Nm);
|
||||
double * Qt_j = & Qt_jv[0];
|
||||
for(int k=0;k<Nm;++k) Qt_j[k]=Qt(j,k);
|
||||
|
||||
auto basis_vp=& basis_v[0];
|
||||
autoView(result_v,result,AcceleratorWrite);
|
||||
accelerator_for(ss, grid->oSites(),vobj::Nsimd(),{
|
||||
vobj zzz=Zero();
|
||||
auto B=coalescedRead(zzz);
|
||||
for(int k=k0; k<k1; ++k){
|
||||
B +=Qt_j[k] * coalescedRead(basis_vp[k][ss]);
|
||||
}
|
||||
coalescedWrite(result_v[ss], B);
|
||||
});
|
||||
for(int k=0;k<basis.size();k++) basis_v[k].ViewClose();
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
void basisReorderInPlace(std::vector<Field> &_v,std::vector<RealD>& sort_vals, std::vector<int>& idx)
|
||||
{
|
||||
int vlen = idx.size();
|
||||
|
||||
assert(vlen>=1);
|
||||
assert(vlen<=sort_vals.size());
|
||||
assert(vlen<=_v.size());
|
||||
|
||||
for (size_t i=0;i<vlen;i++) {
|
||||
|
||||
if (idx[i] != i) {
|
||||
|
||||
//////////////////////////////////////
|
||||
// idx[i] is a table of desired sources giving a permutation.
|
||||
// Swap v[i] with v[idx[i]].
|
||||
// Find j>i for which _vnew[j] = _vold[i],
|
||||
// track the move idx[j] => idx[i]
|
||||
// track the move idx[i] => i
|
||||
//////////////////////////////////////
|
||||
size_t j;
|
||||
for (j=i;j<idx.size();j++)
|
||||
if (idx[j]==i)
|
||||
break;
|
||||
|
||||
assert(idx[i] > i); assert(j!=idx.size()); assert(idx[j]==i);
|
||||
|
||||
swap(_v[i],_v[idx[i]]); // should use vector move constructor, no data copy
|
||||
std::swap(sort_vals[i],sort_vals[idx[i]]);
|
||||
|
||||
idx[j] = idx[i];
|
||||
idx[i] = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline std::vector<int> basisSortGetIndex(std::vector<RealD>& sort_vals)
|
||||
{
|
||||
std::vector<int> idx(sort_vals.size());
|
||||
std::iota(idx.begin(), idx.end(), 0);
|
||||
|
||||
// sort indexes based on comparing values in v
|
||||
std::sort(idx.begin(), idx.end(), [&sort_vals](int i1, int i2) {
|
||||
return ::fabs(sort_vals[i1]) < ::fabs(sort_vals[i2]);
|
||||
});
|
||||
return idx;
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
void basisSortInPlace(std::vector<Field> & _v,std::vector<RealD>& sort_vals, bool reverse)
|
||||
{
|
||||
std::vector<int> idx = basisSortGetIndex(sort_vals);
|
||||
if (reverse)
|
||||
std::reverse(idx.begin(), idx.end());
|
||||
|
||||
basisReorderInPlace(_v,sort_vals,idx);
|
||||
}
|
||||
|
||||
// PAB: faster to compute the inner products first then fuse loops.
|
||||
// If performance critical can improve.
|
||||
template<class Field>
|
||||
void basisDeflate(const std::vector<Field> &_v,const std::vector<RealD>& eval,const Field& src_orig,Field& result) {
|
||||
result = Zero();
|
||||
assert(_v.size()==eval.size());
|
||||
int N = (int)_v.size();
|
||||
for (int i=0;i<N;i++) {
|
||||
Field& tmp = _v[i];
|
||||
axpy(result,TensorRemove(innerProduct(tmp,src_orig)) / eval[i],tmp,result);
|
||||
}
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,179 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/lattice/Lattice_comparison.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_LATTICE_COMPARISON_H
|
||||
#define GRID_LATTICE_COMPARISON_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// relational operators
|
||||
//
|
||||
// Support <,>,<=,>=,==,!=
|
||||
//
|
||||
//Query supporting bitwise &, |, ^, !
|
||||
//Query supporting logical &&, ||,
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
typedef iScalar<vInteger> vPredicate ;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// compare lattice to lattice
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template<class vfunctor,class lobj,class robj>
|
||||
inline Lattice<vPredicate> LLComparison(vfunctor op,const Lattice<lobj> &lhs,const Lattice<robj> &rhs)
|
||||
{
|
||||
Lattice<vPredicate> ret(rhs.Grid());
|
||||
autoView( lhs_v, lhs, CpuRead);
|
||||
autoView( rhs_v, rhs, CpuRead);
|
||||
autoView( ret_v, ret, CpuWrite);
|
||||
thread_for( ss, rhs_v.size(), {
|
||||
ret_v[ss]=op(lhs_v[ss],rhs_v[ss]);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// compare lattice to scalar
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template<class vfunctor,class lobj,class robj>
|
||||
inline Lattice<vPredicate> LSComparison(vfunctor op,const Lattice<lobj> &lhs,const robj &rhs)
|
||||
{
|
||||
Lattice<vPredicate> ret(lhs.Grid());
|
||||
autoView( lhs_v, lhs, CpuRead);
|
||||
autoView( ret_v, ret, CpuWrite);
|
||||
thread_for( ss, lhs_v.size(), {
|
||||
ret_v[ss]=op(lhs_v[ss],rhs);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// compare scalar to lattice
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template<class vfunctor,class lobj,class robj>
|
||||
inline Lattice<vPredicate> SLComparison(vfunctor op,const lobj &lhs,const Lattice<robj> &rhs)
|
||||
{
|
||||
Lattice<vPredicate> ret(rhs.Grid());
|
||||
autoView( rhs_v, rhs, CpuRead);
|
||||
autoView( ret_v, ret, CpuWrite);
|
||||
thread_for( ss, rhs_v.size(), {
|
||||
ret_v[ss]=op(lhs,rhs_v[ss]);
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Map to functors
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Less than
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator < (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
||||
return LLComparison(vlt<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator < (const Lattice<lobj> & lhs, const robj & rhs) {
|
||||
return LSComparison(vlt<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator < (const lobj & lhs, const Lattice<robj> & rhs) {
|
||||
return SLComparison(vlt<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
|
||||
// Less than equal
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator <= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
||||
return LLComparison(vle<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator <= (const Lattice<lobj> & lhs, const robj & rhs) {
|
||||
return LSComparison(vle<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator <= (const lobj & lhs, const Lattice<robj> & rhs) {
|
||||
return SLComparison(vle<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
|
||||
// Greater than
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator > (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
||||
return LLComparison(vgt<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator > (const Lattice<lobj> & lhs, const robj & rhs) {
|
||||
return LSComparison(vgt<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator > (const lobj & lhs, const Lattice<robj> & rhs) {
|
||||
return SLComparison(vgt<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
|
||||
|
||||
// Greater than equal
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator >= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
||||
return LLComparison(vge<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator >= (const Lattice<lobj> & lhs, const robj & rhs) {
|
||||
return LSComparison(vge<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator >= (const lobj & lhs, const Lattice<robj> & rhs) {
|
||||
return SLComparison(vge<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
|
||||
// equal
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator == (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
||||
return LLComparison(veq<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator == (const Lattice<lobj> & lhs, const robj & rhs) {
|
||||
return LSComparison(veq<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator == (const lobj & lhs, const Lattice<robj> & rhs) {
|
||||
return SLComparison(veq<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
|
||||
|
||||
// not equal
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator != (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
||||
return LLComparison(vne<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator != (const Lattice<lobj> & lhs, const robj & rhs) {
|
||||
return LSComparison(vne<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
template<class lobj,class robj>
|
||||
inline Lattice<vPredicate> operator != (const lobj & lhs, const Lattice<robj> & rhs) {
|
||||
return SLComparison(vne<lobj,robj>(),lhs,rhs);
|
||||
}
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,232 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/lattice/Lattice_comparison_utils.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/////////////////////////////////////////
|
||||
// This implementation is a bit poor.
|
||||
//
|
||||
// Only support relational logical operations (<, > etc)
|
||||
// on scalar objects. Therefore can strip any tensor structures.
|
||||
//
|
||||
// Should guard this with isGridTensor<> enable if?
|
||||
/////////////////////////////////////////
|
||||
//
|
||||
// Generic list of functors
|
||||
//
|
||||
template<class lobj,class robj> class veq {
|
||||
public:
|
||||
accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) == (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class vne {
|
||||
public:
|
||||
accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) != (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class vlt {
|
||||
public:
|
||||
accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) < (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class vle {
|
||||
public:
|
||||
accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) <= (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class vgt {
|
||||
public:
|
||||
accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) > (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class vge {
|
||||
public:
|
||||
accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) >= (rhs);
|
||||
}
|
||||
};
|
||||
|
||||
// Generic list of functors
|
||||
template<class lobj,class robj> class seq {
|
||||
public:
|
||||
accelerator Integer operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) == (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class sne {
|
||||
public:
|
||||
accelerator Integer operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) != (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class slt {
|
||||
public:
|
||||
accelerator Integer operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) < (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class sle {
|
||||
public:
|
||||
accelerator Integer operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) <= (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class sgt {
|
||||
public:
|
||||
accelerator Integer operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) > (rhs);
|
||||
}
|
||||
};
|
||||
template<class lobj,class robj> class sge {
|
||||
public:
|
||||
accelerator Integer operator()(const lobj &lhs, const robj &rhs)
|
||||
{
|
||||
return (lhs) >= (rhs);
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Integer and real get extra relational functions.
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class sfunctor, class vsimd,IfNotComplex<vsimd> = 0>
|
||||
accelerator_inline vInteger Comparison(sfunctor sop,const vsimd & lhs, const vsimd & rhs)
|
||||
{
|
||||
typedef typename vsimd::scalar_type scalar;
|
||||
ExtractBuffer<scalar> vlhs(vsimd::Nsimd()); // Use functors to reduce this to single implementation
|
||||
ExtractBuffer<scalar> vrhs(vsimd::Nsimd());
|
||||
ExtractBuffer<Integer> vpred(vsimd::Nsimd());
|
||||
vInteger ret;
|
||||
extract<vsimd,scalar>(lhs,vlhs);
|
||||
extract<vsimd,scalar>(rhs,vrhs);
|
||||
for(int s=0;s<vsimd::Nsimd();s++){
|
||||
vpred[s] = sop(vlhs[s],vrhs[s]);
|
||||
}
|
||||
merge<vInteger,Integer>(ret,vpred);
|
||||
return ret;
|
||||
}
|
||||
|
||||
template<class sfunctor, class vsimd,IfNotComplex<vsimd> = 0>
|
||||
accelerator_inline vInteger Comparison(sfunctor sop,const vsimd & lhs, const typename vsimd::scalar_type & rhs)
|
||||
{
|
||||
typedef typename vsimd::scalar_type scalar;
|
||||
ExtractBuffer<scalar> vlhs(vsimd::Nsimd()); // Use functors to reduce this to single implementation
|
||||
ExtractBuffer<Integer> vpred(vsimd::Nsimd());
|
||||
vInteger ret;
|
||||
extract<vsimd,scalar>(lhs,vlhs);
|
||||
for(int s=0;s<vsimd::Nsimd();s++){
|
||||
vpred[s] = sop(vlhs[s],rhs);
|
||||
}
|
||||
merge<vInteger,Integer>(ret,vpred);
|
||||
return ret;
|
||||
}
|
||||
|
||||
template<class sfunctor, class vsimd,IfNotComplex<vsimd> = 0>
|
||||
accelerator_inline vInteger Comparison(sfunctor sop,const typename vsimd::scalar_type & lhs, const vsimd & rhs)
|
||||
{
|
||||
typedef typename vsimd::scalar_type scalar;
|
||||
ExtractBuffer<scalar> vrhs(vsimd::Nsimd()); // Use functors to reduce this to single implementation
|
||||
ExtractBuffer<Integer> vpred(vsimd::Nsimd());
|
||||
vInteger ret;
|
||||
extract<vsimd,scalar>(rhs,vrhs);
|
||||
for(int s=0;s<vsimd::Nsimd();s++){
|
||||
vpred[s] = sop(lhs,vrhs[s]);
|
||||
}
|
||||
merge<vInteger,Integer>(ret,vpred);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define DECLARE_RELATIONAL_EQ(op,functor) \
|
||||
template<class vsimd,IfSimd<vsimd> = 0>\
|
||||
accelerator_inline vInteger operator op (const vsimd & lhs, const vsimd & rhs)\
|
||||
{\
|
||||
typedef typename vsimd::scalar_type scalar;\
|
||||
return Comparison(functor<scalar,scalar>(),lhs,rhs);\
|
||||
}\
|
||||
template<class vsimd,IfSimd<vsimd> = 0>\
|
||||
accelerator_inline vInteger operator op (const vsimd & lhs, const typename vsimd::scalar_type & rhs) \
|
||||
{\
|
||||
typedef typename vsimd::scalar_type scalar;\
|
||||
return Comparison(functor<scalar,scalar>(),lhs,rhs);\
|
||||
}\
|
||||
template<class vsimd,IfSimd<vsimd> = 0>\
|
||||
accelerator_inline vInteger operator op (const typename vsimd::scalar_type & lhs, const vsimd & rhs) \
|
||||
{\
|
||||
typedef typename vsimd::scalar_type scalar;\
|
||||
return Comparison(functor<scalar,scalar>(),lhs,rhs);\
|
||||
}\
|
||||
template<class vsimd>\
|
||||
accelerator_inline vInteger operator op(const iScalar<vsimd> &lhs,const typename vsimd::scalar_type &rhs) \
|
||||
{ \
|
||||
return lhs._internal op rhs; \
|
||||
} \
|
||||
template<class vsimd>\
|
||||
accelerator_inline vInteger operator op(const typename vsimd::scalar_type &lhs,const iScalar<vsimd> &rhs) \
|
||||
{ \
|
||||
return lhs op rhs._internal; \
|
||||
} \
|
||||
|
||||
#define DECLARE_RELATIONAL(op,functor) \
|
||||
DECLARE_RELATIONAL_EQ(op,functor) \
|
||||
template<class vsimd>\
|
||||
accelerator_inline vInteger operator op(const iScalar<vsimd> &lhs,const iScalar<vsimd> &rhs)\
|
||||
{ \
|
||||
return lhs._internal op rhs._internal; \
|
||||
}
|
||||
|
||||
DECLARE_RELATIONAL(<,slt);
|
||||
DECLARE_RELATIONAL(<=,sle);
|
||||
DECLARE_RELATIONAL(>,sgt);
|
||||
DECLARE_RELATIONAL(>=,sge);
|
||||
DECLARE_RELATIONAL_EQ(==,seq);
|
||||
DECLARE_RELATIONAL(!=,sne);
|
||||
|
||||
#undef DECLARE_RELATIONAL
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user