mirror of
https://github.com/paboyle/Grid.git
synced 2025-06-15 14:27:06 +01:00
Compare commits
304 Commits
feature/co
...
feature/ca
Author | SHA1 | Date | |
---|---|---|---|
59282f25ec | |||
b0bd173899 | |||
135808dcfa | |||
7f7d06d963 | |||
2bf3b4d576 | |||
f34d34bd17 | |||
e32d5141b4 | |||
6d5277f2d7 | |||
14d82777e0 | |||
2a4e739513 | |||
8079dc2a14 | |||
6ceb556684 | |||
76cde73705 | |||
cc094366a9 | |||
41a575ff9b | |||
12ef413065 | |||
829a328451 | |||
402523c62e | |||
d7bef70b5c | |||
2ad1811642 | |||
a65a497bae | |||
b27b12828e | |||
fe9edf8526 | |||
44204c7e06 | |||
33b3789598 | |||
195ab2888d | |||
85f750d753 | |||
a4ce6e42c7 | |||
5398b7e7e3 | |||
fd13a3f2be | |||
c144b32368 | |||
ba7e371b90 | |||
99e7a5d18a | |||
f824d99059 | |||
749b8022a4 | |||
7e0057d2c4 | |||
cfe9e870d3 | |||
e9c4f06cbf | |||
1f9688417a | |||
16c2a99965 | |||
cda915a345 | |||
7c16189e16 | |||
ecbfccea43 | |||
a8eda8f6da | |||
9b1a0653cf | |||
7cb1ff7395 | |||
ab6ea29913 | |||
b5c81a02b6 | |||
d899ee80fc | |||
4016e705fc | |||
2f4e85e5d6 | |||
8ed0b57b09 | |||
a976fa6746 | |||
6c66b8d997 | |||
9523ad3d73 | |||
73a95fa96f | |||
7e130076d6 | |||
6efdad6f21 | |||
a822c48565 | |||
014fb76e88 | |||
30e5311b43 | |||
67e08aa952 | |||
ed1f20f3a1 | |||
cffc736bb3 | |||
c0d56a1c04 | |||
3206f69478 | |||
b2ccaad761 | |||
8eb1232683 | |||
c6ce3ad03b | |||
b3b033d343 | |||
ca9816bfbb | |||
814d5abc7e | |||
a29122e2bf | |||
e188c0512e | |||
1fb6aaf150 | |||
894654f7ef | |||
109507888b | |||
68650b61fe | |||
7ee66bf453 | |||
8bd70ad8b5 | |||
af98525766 | |||
1c2f218519 | |||
c9aa1f507c | |||
ea7126496d | |||
f660dc67e4 | |||
ede8faea74 | |||
1b750761c2 | |||
145acf2919 | |||
cc4a27b9e6 | |||
b4690e6091 | |||
4b24800132 | |||
9d2238148c | |||
c15493218d | |||
001a556a34 | |||
3d0f88e702 | |||
dd091d0960 | |||
e2abbf9520 | |||
c7baeb5bae | |||
402d80e197 | |||
86e33c8ab2 | |||
5dae6a6dac | |||
361bb8a101 | |||
7efdb3cd2b | |||
65ef4ec29f | |||
d5835c0222 | |||
a7b943b33e | |||
7440cde92f | |||
0fc662bb24 | |||
8195890640 | |||
4c88104a73 | |||
73b944c152 | |||
d1b0b7f5c6 | |||
381d8797d0 | |||
11ee8a1061 | |||
b06526bc1e | |||
3044419111 | |||
bcfa9cf068 | |||
114920b8de | |||
0d588b95f4 | |||
5b3c530aa7 | |||
c6a5499c8b | |||
ec9c3fe77a | |||
6135ad530e | |||
40098424c7 | |||
7163b31a26 | |||
ffbdd91e0e | |||
5d29e175d8 | |||
417dbfa257 | |||
1eda4d8e0b | |||
50181f16e5 | |||
75030637cc | |||
fe5aaf7677 | |||
80ac2a73ca | |||
770680669d | |||
0cdfc5cf22 | |||
d75a66a3e6 | |||
fcc4374d7b | |||
67c3c16fe5 | |||
25e9be50b5 | |||
428b8ba907 | |||
323cf6c038 | |||
29a22ae603 | |||
403bff1a47 | |||
c50f27e68b | |||
80afacec5b | |||
6cd9224dd7 | |||
4bf8196ff1 | |||
4c5440fb06 | |||
a269a3d919 | |||
0c4f585496 | |||
33d2df46a0 | |||
2df308f649 | |||
92def28bd3 | |||
ca10bfa1c7 | |||
298a6ec51e | |||
e5dbe488a6 | |||
0e27e3847d | |||
393727b93b | |||
2b1fcd78c3 | |||
0a4e0b49a0 | |||
76af169f05 | |||
7b89232251 | |||
b5aeae526f | |||
ef0ddd5d04 | |||
9b73dacf50 | |||
244b4aa07f | |||
8cfc7342cd | |||
15ae317858 | |||
834f536b5f | |||
c332d9f08b | |||
cf2923d5dd | |||
0e4413ddde | |||
009ccd581e | |||
8cd4263974 | |||
d45c868656 | |||
955a8113de | |||
dbe210dd53 | |||
54c6b1376d | |||
86e11743ca | |||
f3f11b586f | |||
8083e3f7e8 | |||
980e721f6e | |||
364793154b | |||
3e2ae1e9af | |||
d38ae2fd18 | |||
030e7754e4 | |||
e2a0142d87 | |||
895244ecc3 | |||
addeb621a7 | |||
3b7fce1e76 | |||
4d15417f93 | |||
ab3c855f65 | |||
92e2c517d8 | |||
a7fb25adf6 | |||
e947992957 | |||
bb89a82a07 | |||
2bb374daea | |||
8bdadbadac | |||
15c50a7442 | |||
49b0af2c95 | |||
9c2b37218a | |||
3c67d626ba | |||
51f506553c | |||
226be84937 | |||
001814b442 | |||
db3ac67506 | |||
da91a884ef | |||
a71e6755e3 | |||
cd5891eecd | |||
5bb7336f27 | |||
ce1fc1f48a | |||
82402c6a7c | |||
d9c4afe5b7 | |||
f786ff8d69 | |||
a651caed5f | |||
0e21adb3f6 | |||
58bf9b9e6d | |||
2146eebb65 | |||
6a429ee6d3 | |||
4d1ea15c79 | |||
a76cb005e0 | |||
49ecbc81d4 | |||
9e5fb52eb9 | |||
a9604367c1 | |||
d7065023cc | |||
89d299ceec | |||
e34eda66df | |||
b24181aa4f | |||
aa173e2998 | |||
7a19432e0b | |||
9b15704290 | |||
017f955b2d | |||
f252d69eef | |||
3b06e4655e | |||
d4b4de8f42 | |||
c90beee774 | |||
1eea9d73b9 | |||
679d1d22f7 | |||
b2b5e0b98c | |||
03e54722c1 | |||
442336bd96 | |||
9c9566b9c9 | |||
1059a81a3c | |||
2e61556389 | |||
f9b1f240f6 | |||
69f41469dd | |||
d620b303ff | |||
157fd1428d | |||
c791cb2214 | |||
d5ab571a89 | |||
0ed800f6e4 | |||
0a32183825 | |||
2cacfbde2a | |||
c073e62e0b | |||
e3d019bc2f | |||
7ae030f585 | |||
86b58d5aff | |||
26e8b9f4a5 | |||
35114c9e62 | |||
dfd28a85c9 | |||
a503332924 | |||
1ac13ec3a7 | |||
55de69a569 | |||
eda9ab487b | |||
cd99edcc5f | |||
4705aa541d | |||
3215d88a91 | |||
9b9a53f870 | |||
019ffe17d4 | |||
bc496dd844 | |||
a673b6a54d | |||
1bf2e4d187 | |||
96dd7a8fbd | |||
7905afa9f5 | |||
712bb40650 | |||
81d88d9f4d | |||
77063418da | |||
2983b6fdf6 | |||
69f1f04f74 | |||
11a5fd09d6 | |||
ff1fa98808 | |||
df16202865 | |||
3ff7c2c02a | |||
fc6d07897f | |||
f9c8e5c8ef | |||
8bfa0e74f8 | |||
9b73a937e7 | |||
b0339bc5a4 | |||
fa12b9a329 | |||
45fc7ded3a | |||
74de2d9742 | |||
e759367d42 | |||
299d0de066 | |||
b4c1317ab4 | |||
f36d6f3923 | |||
808f1e0e8c | |||
c438118fd7 | |||
17ec9c5545 | |||
3594ce877b | |||
9bae6b889a | |||
4014dfd5b9 | |||
67023c334b | |||
a3de7026c8 | |||
ee11678b1f |
1
.gitignore
vendored
1
.gitignore
vendored
@ -88,6 +88,7 @@ Thumbs.db
|
|||||||
# build directory #
|
# build directory #
|
||||||
###################
|
###################
|
||||||
build*/*
|
build*/*
|
||||||
|
Documentation/_build
|
||||||
|
|
||||||
# IDE related files #
|
# IDE related files #
|
||||||
#####################
|
#####################
|
||||||
|
56
.travis.yml
56
.travis.yml
@ -1,56 +0,0 @@
|
|||||||
language: cpp
|
|
||||||
|
|
||||||
cache:
|
|
||||||
directories:
|
|
||||||
- clang
|
|
||||||
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- os: osx
|
|
||||||
osx_image: xcode8.3
|
|
||||||
compiler: clang
|
|
||||||
|
|
||||||
before_install:
|
|
||||||
- export GRIDDIR=`pwd`
|
|
||||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]] && [ ! -e clang/bin ]; then wget $CLANG_LINK; tar -xf `basename $CLANG_LINK`; mkdir clang; mv clang+*/* clang/; fi
|
|
||||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export PATH="${GRIDDIR}/clang/bin:${PATH}"; fi
|
|
||||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
|
|
||||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
|
|
||||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc openssl; fi
|
|
||||||
|
|
||||||
install:
|
|
||||||
- export CWD=`pwd`
|
|
||||||
- echo $CWD
|
|
||||||
- export CC=$CC$VERSION
|
|
||||||
- export CXX=$CXX$VERSION
|
|
||||||
- echo $PATH
|
|
||||||
- which autoconf
|
|
||||||
- autoconf --version
|
|
||||||
- which automake
|
|
||||||
- automake --version
|
|
||||||
- which $CC
|
|
||||||
- $CC --version
|
|
||||||
- which $CXX
|
|
||||||
- $CXX --version
|
|
||||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export LDFLAGS='-L/usr/local/lib'; fi
|
|
||||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export EXTRACONF='--with-openssl=/usr/local/opt/openssl'; fi
|
|
||||||
|
|
||||||
script:
|
|
||||||
- ./bootstrap.sh
|
|
||||||
- mkdir build
|
|
||||||
- cd build
|
|
||||||
- mkdir lime
|
|
||||||
- cd lime
|
|
||||||
- mkdir build
|
|
||||||
- cd build
|
|
||||||
- wget http://usqcd-software.github.io/downloads/c-lime/lime-1.3.2.tar.gz
|
|
||||||
- tar xf lime-1.3.2.tar.gz
|
|
||||||
- cd lime-1.3.2
|
|
||||||
- ./configure --prefix=$CWD/build/lime/install
|
|
||||||
- make -j4
|
|
||||||
- make install
|
|
||||||
- cd $CWD/build
|
|
||||||
- ../configure --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install ${EXTRACONF}
|
|
||||||
- make -j4
|
|
||||||
- ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
|
|
||||||
- make check
|
|
@ -54,9 +54,11 @@ Version.h: version-cache
|
|||||||
include Make.inc
|
include Make.inc
|
||||||
include Eigen.inc
|
include Eigen.inc
|
||||||
|
|
||||||
extra_sources+=$(ZWILS_FERMION_FILES)
|
|
||||||
extra_sources+=$(WILS_FERMION_FILES)
|
extra_sources+=$(WILS_FERMION_FILES)
|
||||||
extra_sources+=$(STAG_FERMION_FILES)
|
extra_sources+=$(STAG_FERMION_FILES)
|
||||||
|
if BUILD_ZMOBIUS
|
||||||
|
extra_sources+=$(ZWILS_FERMION_FILES)
|
||||||
|
endif
|
||||||
if BUILD_GPARITY
|
if BUILD_GPARITY
|
||||||
extra_sources+=$(GP_FERMION_FILES)
|
extra_sources+=$(GP_FERMION_FILES)
|
||||||
endif
|
endif
|
||||||
|
@ -358,7 +358,7 @@ public:
|
|||||||
autoView( in_v , in, AcceleratorRead);
|
autoView( in_v , in, AcceleratorRead);
|
||||||
autoView( out_v , out, AcceleratorWrite);
|
autoView( out_v , out, AcceleratorWrite);
|
||||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
autoView( Stencil_v , Stencil, AcceleratorRead);
|
||||||
auto& geom_v = geom;
|
int npoint = geom.npoint;
|
||||||
typedef LatticeView<Cobj> Aview;
|
typedef LatticeView<Cobj> Aview;
|
||||||
|
|
||||||
Vector<Aview> AcceleratorViewContainer;
|
Vector<Aview> AcceleratorViewContainer;
|
||||||
@ -380,7 +380,7 @@ public:
|
|||||||
int ptype;
|
int ptype;
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
|
|
||||||
for(int point=0;point<geom_v.npoint;point++){
|
for(int point=0;point<npoint;point++){
|
||||||
|
|
||||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
SE=Stencil_v.GetEntry(ptype,point,ss);
|
||||||
|
|
||||||
@ -424,7 +424,7 @@ public:
|
|||||||
autoView( in_v , in, AcceleratorRead);
|
autoView( in_v , in, AcceleratorRead);
|
||||||
autoView( out_v , out, AcceleratorWrite);
|
autoView( out_v , out, AcceleratorWrite);
|
||||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
autoView( Stencil_v , Stencil, AcceleratorRead);
|
||||||
auto& geom_v = geom;
|
int npoint = geom.npoint;
|
||||||
typedef LatticeView<Cobj> Aview;
|
typedef LatticeView<Cobj> Aview;
|
||||||
|
|
||||||
Vector<Aview> AcceleratorViewContainer;
|
Vector<Aview> AcceleratorViewContainer;
|
||||||
@ -442,6 +442,8 @@ public:
|
|||||||
for(int p=0; p<geom.npoint; p++)
|
for(int p=0; p<geom.npoint; p++)
|
||||||
points[p] = geom.points_dagger[p];
|
points[p] = geom.points_dagger[p];
|
||||||
|
|
||||||
|
auto points_p = &points[0];
|
||||||
|
|
||||||
RealD* dag_factor_p = &dag_factor[0];
|
RealD* dag_factor_p = &dag_factor[0];
|
||||||
|
|
||||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
||||||
@ -452,8 +454,8 @@ public:
|
|||||||
int ptype;
|
int ptype;
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
|
|
||||||
for(int p=0;p<geom_v.npoint;p++){
|
for(int p=0;p<npoint;p++){
|
||||||
int point = points[p];
|
int point = points_p[p];
|
||||||
|
|
||||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
SE=Stencil_v.GetEntry(ptype,point,ss);
|
||||||
|
|
||||||
@ -708,6 +710,8 @@ public:
|
|||||||
for(int p=0; p<npoint; p++)
|
for(int p=0; p<npoint; p++)
|
||||||
points[p] = (dag && !hermitian) ? geom.points_dagger[p] : p;
|
points[p] = (dag && !hermitian) ? geom.points_dagger[p] : p;
|
||||||
|
|
||||||
|
auto points_p = &points[0];
|
||||||
|
|
||||||
Vector<Aview> AcceleratorViewContainer;
|
Vector<Aview> AcceleratorViewContainer;
|
||||||
for(int p=0;p<npoint;p++) AcceleratorViewContainer.push_back(a[p].View(AcceleratorRead));
|
for(int p=0;p<npoint;p++) AcceleratorViewContainer.push_back(a[p].View(AcceleratorRead));
|
||||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
Aview *Aview_p = & AcceleratorViewContainer[0];
|
||||||
@ -728,7 +732,7 @@ public:
|
|||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
|
|
||||||
for(int p=0;p<npoint;p++){
|
for(int p=0;p<npoint;p++){
|
||||||
int point = points[p];
|
int point = points_p[p];
|
||||||
SE=st_v.GetEntry(ptype,point,ss);
|
SE=st_v.GetEntry(ptype,point,ss);
|
||||||
|
|
||||||
if(SE->_is_local) {
|
if(SE->_is_local) {
|
||||||
@ -754,7 +758,7 @@ public:
|
|||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
|
|
||||||
for(int p=0;p<npoint;p++){
|
for(int p=0;p<npoint;p++){
|
||||||
int point = points[p];
|
int point = points_p[p];
|
||||||
SE=st_v.GetEntry(ptype,point,ss);
|
SE=st_v.GetEntry(ptype,point,ss);
|
||||||
|
|
||||||
if(SE->_is_local) {
|
if(SE->_is_local) {
|
||||||
|
@ -136,7 +136,7 @@ public:
|
|||||||
flops=0;
|
flops=0;
|
||||||
usec =0;
|
usec =0;
|
||||||
Coordinate layout(Nd,1);
|
Coordinate layout(Nd,1);
|
||||||
sgrid = new GridCartesian(dimensions,layout,processors);
|
sgrid = new GridCartesian(dimensions,layout,processors,*grid);
|
||||||
};
|
};
|
||||||
|
|
||||||
~FFT ( void) {
|
~FFT ( void) {
|
||||||
@ -182,7 +182,7 @@ public:
|
|||||||
pencil_gd[dim] = G*processors[dim];
|
pencil_gd[dim] = G*processors[dim];
|
||||||
|
|
||||||
// Pencil global vol LxLxGxLxL per node
|
// Pencil global vol LxLxGxLxL per node
|
||||||
GridCartesian pencil_g(pencil_gd,layout,processors);
|
GridCartesian pencil_g(pencil_gd,layout,processors,*vgrid);
|
||||||
|
|
||||||
// Construct pencils
|
// Construct pencils
|
||||||
typedef typename vobj::scalar_object sobj;
|
typedef typename vobj::scalar_object sobj;
|
||||||
|
@ -52,6 +52,7 @@ public:
|
|||||||
virtual void AdjOp (const Field &in, Field &out) = 0; // Abstract base
|
virtual void AdjOp (const Field &in, Field &out) = 0; // Abstract base
|
||||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)=0;
|
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)=0;
|
||||||
virtual void HermOp(const Field &in, Field &out)=0;
|
virtual void HermOp(const Field &in, Field &out)=0;
|
||||||
|
virtual ~LinearOperatorBase(){};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
@ -507,7 +508,7 @@ class SchurStaggeredOperator : public SchurOperatorBase<Field> {
|
|||||||
virtual void MpcDag (const Field &in, Field &out){
|
virtual void MpcDag (const Field &in, Field &out){
|
||||||
Mpc(in,out);
|
Mpc(in,out);
|
||||||
}
|
}
|
||||||
virtual void MpcDagMpc(const Field &in, Field &out,RealD &ni,RealD &no) {
|
virtual void MpcDagMpc(const Field &in, Field &out) {
|
||||||
assert(0);// Never need with staggered
|
assert(0);// Never need with staggered
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -530,6 +531,16 @@ public:
|
|||||||
template<class Field> class LinearFunction {
|
template<class Field> class LinearFunction {
|
||||||
public:
|
public:
|
||||||
virtual void operator() (const Field &in, Field &out) = 0;
|
virtual void operator() (const Field &in, Field &out) = 0;
|
||||||
|
|
||||||
|
virtual void operator() (const std::vector<Field> &in, std::vector<Field> &out)
|
||||||
|
{
|
||||||
|
assert(in.size() == out.size());
|
||||||
|
|
||||||
|
for (unsigned int i = 0; i < in.size(); ++i)
|
||||||
|
{
|
||||||
|
(*this)(in[i], out[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
template<class Field> class IdentityLinearFunction : public LinearFunction<Field> {
|
template<class Field> class IdentityLinearFunction : public LinearFunction<Field> {
|
||||||
@ -575,6 +586,7 @@ class HermOpOperatorFunction : public OperatorFunction<Field> {
|
|||||||
template<typename Field>
|
template<typename Field>
|
||||||
class PlainHermOp : public LinearFunction<Field> {
|
class PlainHermOp : public LinearFunction<Field> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
LinearOperatorBase<Field> &_Linop;
|
LinearOperatorBase<Field> &_Linop;
|
||||||
|
|
||||||
PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop)
|
PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop)
|
||||||
@ -588,6 +600,7 @@ public:
|
|||||||
template<typename Field>
|
template<typename Field>
|
||||||
class FunctionHermOp : public LinearFunction<Field> {
|
class FunctionHermOp : public LinearFunction<Field> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
OperatorFunction<Field> & _poly;
|
OperatorFunction<Field> & _poly;
|
||||||
LinearOperatorBase<Field> &_Linop;
|
LinearOperatorBase<Field> &_Linop;
|
||||||
|
|
||||||
|
@ -30,13 +30,19 @@ Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
NAMESPACE_BEGIN(Grid);
|
||||||
|
|
||||||
|
template<class Field> using Preconditioner = LinearFunction<Field> ;
|
||||||
|
|
||||||
|
/*
|
||||||
template<class Field> class Preconditioner : public LinearFunction<Field> {
|
template<class Field> class Preconditioner : public LinearFunction<Field> {
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
virtual void operator()(const Field &src, Field & psi)=0;
|
virtual void operator()(const Field &src, Field & psi)=0;
|
||||||
};
|
};
|
||||||
|
*/
|
||||||
|
|
||||||
template<class Field> class TrivialPrecon : public Preconditioner<Field> {
|
template<class Field> class TrivialPrecon : public Preconditioner<Field> {
|
||||||
public:
|
public:
|
||||||
void operator()(const Field &src, Field & psi){
|
using Preconditioner<Field>::operator();
|
||||||
|
virtual void operator()(const Field &src, Field & psi){
|
||||||
psi = src;
|
psi = src;
|
||||||
}
|
}
|
||||||
TrivialPrecon(void){};
|
TrivialPrecon(void){};
|
||||||
|
@ -48,6 +48,7 @@ public:
|
|||||||
virtual void Mdiag (const Field &in, Field &out)=0;
|
virtual void Mdiag (const Field &in, Field &out)=0;
|
||||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp)=0;
|
virtual void Mdir (const Field &in, Field &out,int dir, int disp)=0;
|
||||||
virtual void MdirAll (const Field &in, std::vector<Field> &out)=0;
|
virtual void MdirAll (const Field &in, std::vector<Field> &out)=0;
|
||||||
|
virtual ~SparseMatrixBase() {};
|
||||||
};
|
};
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
@ -72,7 +73,7 @@ public:
|
|||||||
virtual void MeooeDag (const Field &in, Field &out)=0;
|
virtual void MeooeDag (const Field &in, Field &out)=0;
|
||||||
virtual void MooeeDag (const Field &in, Field &out)=0;
|
virtual void MooeeDag (const Field &in, Field &out)=0;
|
||||||
virtual void MooeeInvDag (const Field &in, Field &out)=0;
|
virtual void MooeeInvDag (const Field &in, Field &out)=0;
|
||||||
|
virtual ~CheckerBoardedSparseMatrixBase() {};
|
||||||
};
|
};
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
NAMESPACE_END(Grid);
|
||||||
|
@ -37,6 +37,7 @@ template<class FieldD, class FieldF, typename std::enable_if< getPrecision<Field
|
|||||||
class MixedPrecisionBiCGSTAB : public LinearFunction<FieldD>
|
class MixedPrecisionBiCGSTAB : public LinearFunction<FieldD>
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<FieldD>::operator();
|
||||||
RealD Tolerance;
|
RealD Tolerance;
|
||||||
RealD InnerTolerance; // Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
RealD InnerTolerance; // Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||||
Integer MaxInnerIterations;
|
Integer MaxInnerIterations;
|
||||||
|
@ -36,6 +36,7 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||||
class MixedPrecisionConjugateGradient : public LinearFunction<FieldD> {
|
class MixedPrecisionConjugateGradient : public LinearFunction<FieldD> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<FieldD>::operator();
|
||||||
RealD Tolerance;
|
RealD Tolerance;
|
||||||
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||||
Integer MaxInnerIterations;
|
Integer MaxInnerIterations;
|
||||||
|
@ -33,16 +33,19 @@ namespace Grid {
|
|||||||
template<class Field>
|
template<class Field>
|
||||||
class ZeroGuesser: public LinearFunction<Field> {
|
class ZeroGuesser: public LinearFunction<Field> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
virtual void operator()(const Field &src, Field &guess) { guess = Zero(); };
|
virtual void operator()(const Field &src, Field &guess) { guess = Zero(); };
|
||||||
};
|
};
|
||||||
template<class Field>
|
template<class Field>
|
||||||
class DoNothingGuesser: public LinearFunction<Field> {
|
class DoNothingGuesser: public LinearFunction<Field> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
virtual void operator()(const Field &src, Field &guess) { };
|
virtual void operator()(const Field &src, Field &guess) { };
|
||||||
};
|
};
|
||||||
template<class Field>
|
template<class Field>
|
||||||
class SourceGuesser: public LinearFunction<Field> {
|
class SourceGuesser: public LinearFunction<Field> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
virtual void operator()(const Field &src, Field &guess) { guess = src; };
|
virtual void operator()(const Field &src, Field &guess) { guess = src; };
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -54,15 +57,24 @@ class DeflatedGuesser: public LinearFunction<Field> {
|
|||||||
private:
|
private:
|
||||||
const std::vector<Field> &evec;
|
const std::vector<Field> &evec;
|
||||||
const std::vector<RealD> &eval;
|
const std::vector<RealD> &eval;
|
||||||
|
const unsigned int N;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
|
|
||||||
DeflatedGuesser(const std::vector<Field> & _evec,const std::vector<RealD> & _eval) : evec(_evec), eval(_eval) {};
|
DeflatedGuesser(const std::vector<Field> & _evec,const std::vector<RealD> & _eval)
|
||||||
|
: DeflatedGuesser(_evec, _eval, _evec.size())
|
||||||
|
{}
|
||||||
|
|
||||||
|
DeflatedGuesser(const std::vector<Field> & _evec, const std::vector<RealD> & _eval, const unsigned int _N)
|
||||||
|
: evec(_evec), eval(_eval), N(_N)
|
||||||
|
{
|
||||||
|
assert(evec.size()==eval.size());
|
||||||
|
assert(N <= evec.size());
|
||||||
|
}
|
||||||
|
|
||||||
virtual void operator()(const Field &src,Field &guess) {
|
virtual void operator()(const Field &src,Field &guess) {
|
||||||
guess = Zero();
|
guess = Zero();
|
||||||
assert(evec.size()==eval.size());
|
|
||||||
auto N = evec.size();
|
|
||||||
for (int i=0;i<N;i++) {
|
for (int i=0;i<N;i++) {
|
||||||
const Field& tmp = evec[i];
|
const Field& tmp = evec[i];
|
||||||
axpy(guess,TensorRemove(innerProduct(tmp,src)) / eval[i],tmp,guess);
|
axpy(guess,TensorRemove(innerProduct(tmp,src)) / eval[i],tmp,guess);
|
||||||
@ -79,6 +91,7 @@ private:
|
|||||||
const std::vector<RealD> &eval_coarse;
|
const std::vector<RealD> &eval_coarse;
|
||||||
public:
|
public:
|
||||||
|
|
||||||
|
using LinearFunction<FineField>::operator();
|
||||||
LocalCoherenceDeflatedGuesser(const std::vector<FineField> &_subspace,
|
LocalCoherenceDeflatedGuesser(const std::vector<FineField> &_subspace,
|
||||||
const std::vector<CoarseField> &_evec_coarse,
|
const std::vector<CoarseField> &_evec_coarse,
|
||||||
const std::vector<RealD> &_eval_coarse)
|
const std::vector<RealD> &_eval_coarse)
|
||||||
|
@ -67,6 +67,7 @@ public:
|
|||||||
template<class Fobj,class CComplex,int nbasis>
|
template<class Fobj,class CComplex,int nbasis>
|
||||||
class ProjectedHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
class ProjectedHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
|
||||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||||
@ -97,6 +98,7 @@ public:
|
|||||||
template<class Fobj,class CComplex,int nbasis>
|
template<class Fobj,class CComplex,int nbasis>
|
||||||
class ProjectedFunctionHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
class ProjectedFunctionHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
|
||||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||||
|
@ -43,7 +43,7 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
template<class Field>
|
template<class Field>
|
||||||
class PrecGeneralisedConjugateResidual : public LinearFunction<Field> {
|
class PrecGeneralisedConjugateResidual : public LinearFunction<Field> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
RealD Tolerance;
|
RealD Tolerance;
|
||||||
Integer MaxIterations;
|
Integer MaxIterations;
|
||||||
int verbose;
|
int verbose;
|
||||||
|
@ -43,7 +43,7 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
template<class Field>
|
template<class Field>
|
||||||
class PrecGeneralisedConjugateResidualNonHermitian : public LinearFunction<Field> {
|
class PrecGeneralisedConjugateResidualNonHermitian : public LinearFunction<Field> {
|
||||||
public:
|
public:
|
||||||
|
using LinearFunction<Field>::operator();
|
||||||
RealD Tolerance;
|
RealD Tolerance;
|
||||||
Integer MaxIterations;
|
Integer MaxIterations;
|
||||||
int verbose;
|
int verbose;
|
||||||
@ -119,7 +119,8 @@ public:
|
|||||||
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
||||||
|
|
||||||
RealD cp;
|
RealD cp;
|
||||||
ComplexD a, b, zAz;
|
ComplexD a, b;
|
||||||
|
// ComplexD zAz;
|
||||||
RealD zAAz;
|
RealD zAAz;
|
||||||
ComplexD rq;
|
ComplexD rq;
|
||||||
|
|
||||||
@ -146,7 +147,7 @@ public:
|
|||||||
//////////////////////////////////
|
//////////////////////////////////
|
||||||
MatTimer.Start();
|
MatTimer.Start();
|
||||||
Linop.Op(psi,Az);
|
Linop.Op(psi,Az);
|
||||||
zAz = innerProduct(Az,psi);
|
// zAz = innerProduct(Az,psi);
|
||||||
zAAz= norm2(Az);
|
zAAz= norm2(Az);
|
||||||
MatTimer.Stop();
|
MatTimer.Stop();
|
||||||
|
|
||||||
@ -170,7 +171,7 @@ public:
|
|||||||
|
|
||||||
LinalgTimer.Start();
|
LinalgTimer.Start();
|
||||||
|
|
||||||
zAz = innerProduct(Az,psi);
|
// zAz = innerProduct(Az,psi);
|
||||||
zAAz= norm2(Az);
|
zAAz= norm2(Az);
|
||||||
|
|
||||||
//p[0],q[0],qq[0]
|
//p[0],q[0],qq[0]
|
||||||
@ -212,7 +213,7 @@ public:
|
|||||||
MatTimer.Start();
|
MatTimer.Start();
|
||||||
Linop.Op(z,Az);
|
Linop.Op(z,Az);
|
||||||
MatTimer.Stop();
|
MatTimer.Stop();
|
||||||
zAz = innerProduct(Az,psi);
|
// zAz = innerProduct(Az,psi);
|
||||||
zAAz= norm2(Az);
|
zAAz= norm2(Az);
|
||||||
|
|
||||||
LinalgTimer.Start();
|
LinalgTimer.Start();
|
||||||
|
@ -132,6 +132,31 @@ namespace Grid {
|
|||||||
(*this)(_Matrix,in,out,guess);
|
(*this)(_Matrix,in,out,guess);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RedBlackSource(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &src_o)
|
||||||
|
{
|
||||||
|
GridBase *grid = _Matrix.RedBlackGrid();
|
||||||
|
Field tmp(grid);
|
||||||
|
int nblock = in.size();
|
||||||
|
for(int b=0;b<nblock;b++){
|
||||||
|
RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// James can write his own deflated guesser
|
||||||
|
// with optimised code for the inner products
|
||||||
|
// RedBlackSolveSplitGrid();
|
||||||
|
// RedBlackSolve(_Matrix,src_o,sol_o);
|
||||||
|
|
||||||
|
void RedBlackSolution(Matrix &_Matrix, const std::vector<Field> &in, const std::vector<Field> &sol_o, std::vector<Field> &out)
|
||||||
|
{
|
||||||
|
GridBase *grid = _Matrix.RedBlackGrid();
|
||||||
|
Field tmp(grid);
|
||||||
|
int nblock = in.size();
|
||||||
|
for(int b=0;b<nblock;b++) {
|
||||||
|
pickCheckerboard(Even,tmp,in[b]);
|
||||||
|
RedBlackSolution(_Matrix,sol_o[b],tmp,out[b]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
template<class Guesser>
|
template<class Guesser>
|
||||||
void operator()(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &out,Guesser &guess)
|
void operator()(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &out,Guesser &guess)
|
||||||
{
|
{
|
||||||
@ -150,22 +175,27 @@ namespace Grid {
|
|||||||
////////////////////////////////////////////////
|
////////////////////////////////////////////////
|
||||||
// Prepare RedBlack source
|
// Prepare RedBlack source
|
||||||
////////////////////////////////////////////////
|
////////////////////////////////////////////////
|
||||||
for(int b=0;b<nblock;b++){
|
RedBlackSource(_Matrix,in,src_o);
|
||||||
RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
// for(int b=0;b<nblock;b++){
|
||||||
}
|
// RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
||||||
|
// }
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
////////////////////////////////////////////////
|
||||||
// Make the guesses
|
// Make the guesses
|
||||||
////////////////////////////////////////////////
|
////////////////////////////////////////////////
|
||||||
if ( subGuess ) guess_save.resize(nblock,grid);
|
if ( subGuess ) guess_save.resize(nblock,grid);
|
||||||
|
|
||||||
for(int b=0;b<nblock;b++){
|
|
||||||
if(useSolnAsInitGuess) {
|
if(useSolnAsInitGuess) {
|
||||||
|
for(int b=0;b<nblock;b++){
|
||||||
pickCheckerboard(Odd, sol_o[b], out[b]);
|
pickCheckerboard(Odd, sol_o[b], out[b]);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
guess(src_o[b],sol_o[b]);
|
guess(src_o, sol_o);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( subGuess ) {
|
if ( subGuess ) {
|
||||||
|
for(int b=0;b<nblock;b++){
|
||||||
guess_save[b] = sol_o[b];
|
guess_save[b] = sol_o[b];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -9,14 +9,30 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
#define AccSmall (3)
|
#define AccSmall (3)
|
||||||
#define Shared (4)
|
#define Shared (4)
|
||||||
#define SharedSmall (5)
|
#define SharedSmall (5)
|
||||||
|
#undef GRID_MM_VERBOSE
|
||||||
uint64_t total_shared;
|
uint64_t total_shared;
|
||||||
uint64_t total_device;
|
uint64_t total_device;
|
||||||
uint64_t total_host;;
|
uint64_t total_host;;
|
||||||
void MemoryManager::PrintBytes(void)
|
void MemoryManager::PrintBytes(void)
|
||||||
{
|
{
|
||||||
std::cout << " MemoryManager : "<<total_shared<<" shared bytes "<<std::endl;
|
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
||||||
std::cout << " MemoryManager : "<<total_device<<" accelerator bytes "<<std::endl;
|
std::cout << " MemoryManager : PrintBytes "<<std::endl;
|
||||||
std::cout << " MemoryManager : "<<total_host <<" cpu bytes "<<std::endl;
|
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
||||||
|
std::cout << " MemoryManager : "<<(total_shared>>20)<<" shared Mbytes "<<std::endl;
|
||||||
|
std::cout << " MemoryManager : "<<(total_device>>20)<<" accelerator Mbytes "<<std::endl;
|
||||||
|
std::cout << " MemoryManager : "<<(total_host>>20) <<" cpu Mbytes "<<std::endl;
|
||||||
|
uint64_t cacheBytes;
|
||||||
|
cacheBytes = CacheBytes[Cpu];
|
||||||
|
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" cpu cache Mbytes "<<std::endl;
|
||||||
|
cacheBytes = CacheBytes[Acc];
|
||||||
|
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" acc cache Mbytes "<<std::endl;
|
||||||
|
cacheBytes = CacheBytes[Shared];
|
||||||
|
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" shared cache Mbytes "<<std::endl;
|
||||||
|
|
||||||
|
#ifdef GRID_CUDA
|
||||||
|
cuda_mem();
|
||||||
|
#endif
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
@ -24,86 +40,114 @@ void MemoryManager::PrintBytes(void)
|
|||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
MemoryManager::AllocationCacheEntry MemoryManager::Entries[MemoryManager::NallocType][MemoryManager::NallocCacheMax];
|
MemoryManager::AllocationCacheEntry MemoryManager::Entries[MemoryManager::NallocType][MemoryManager::NallocCacheMax];
|
||||||
int MemoryManager::Victim[MemoryManager::NallocType];
|
int MemoryManager::Victim[MemoryManager::NallocType];
|
||||||
int MemoryManager::Ncache[MemoryManager::NallocType] = { 8, 32, 8, 32, 8, 32 };
|
int MemoryManager::Ncache[MemoryManager::NallocType] = { 2, 8, 2, 8, 2, 8 };
|
||||||
|
uint64_t MemoryManager::CacheBytes[MemoryManager::NallocType];
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
// Actual allocation and deallocation utils
|
// Actual allocation and deallocation utils
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
void *MemoryManager::AcceleratorAllocate(size_t bytes)
|
void *MemoryManager::AcceleratorAllocate(size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_device+=bytes;
|
||||||
void *ptr = (void *) Lookup(bytes,Acc);
|
void *ptr = (void *) Lookup(bytes,Acc);
|
||||||
if ( ptr == (void *) NULL ) {
|
if ( ptr == (void *) NULL ) {
|
||||||
ptr = (void *) acceleratorAllocDevice(bytes);
|
ptr = (void *) acceleratorAllocDevice(bytes);
|
||||||
total_device+=bytes;
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"AcceleratorAllocate "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
return ptr;
|
return ptr;
|
||||||
}
|
}
|
||||||
void MemoryManager::AcceleratorFree (void *ptr,size_t bytes)
|
void MemoryManager::AcceleratorFree (void *ptr,size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_device-=bytes;
|
||||||
void *__freeme = Insert(ptr,bytes,Acc);
|
void *__freeme = Insert(ptr,bytes,Acc);
|
||||||
if ( __freeme ) {
|
if ( __freeme ) {
|
||||||
acceleratorFreeDevice(__freeme);
|
acceleratorFreeDevice(__freeme);
|
||||||
total_device-=bytes;
|
|
||||||
// PrintBytes();
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"AcceleratorFree "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
void *MemoryManager::SharedAllocate(size_t bytes)
|
void *MemoryManager::SharedAllocate(size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_shared+=bytes;
|
||||||
void *ptr = (void *) Lookup(bytes,Shared);
|
void *ptr = (void *) Lookup(bytes,Shared);
|
||||||
if ( ptr == (void *) NULL ) {
|
if ( ptr == (void *) NULL ) {
|
||||||
ptr = (void *) acceleratorAllocShared(bytes);
|
ptr = (void *) acceleratorAllocShared(bytes);
|
||||||
total_shared+=bytes;
|
|
||||||
// std::cout <<"AcceleratorAllocate: allocated Shared pointer "<<std::hex<<ptr<<std::dec<<std::endl;
|
|
||||||
// PrintBytes();
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"SharedAllocate "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
return ptr;
|
return ptr;
|
||||||
}
|
}
|
||||||
void MemoryManager::SharedFree (void *ptr,size_t bytes)
|
void MemoryManager::SharedFree (void *ptr,size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_shared-=bytes;
|
||||||
void *__freeme = Insert(ptr,bytes,Shared);
|
void *__freeme = Insert(ptr,bytes,Shared);
|
||||||
if ( __freeme ) {
|
if ( __freeme ) {
|
||||||
acceleratorFreeShared(__freeme);
|
acceleratorFreeShared(__freeme);
|
||||||
total_shared-=bytes;
|
|
||||||
// PrintBytes();
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"SharedFree "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
#ifdef GRID_UVM
|
#ifdef GRID_UVM
|
||||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
void *MemoryManager::CpuAllocate(size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_host+=bytes;
|
||||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
void *ptr = (void *) Lookup(bytes,Cpu);
|
||||||
if ( ptr == (void *) NULL ) {
|
if ( ptr == (void *) NULL ) {
|
||||||
ptr = (void *) acceleratorAllocShared(bytes);
|
ptr = (void *) acceleratorAllocShared(bytes);
|
||||||
total_host+=bytes;
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"CpuAllocate "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
return ptr;
|
return ptr;
|
||||||
}
|
}
|
||||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_host-=bytes;
|
||||||
NotifyDeletion(_ptr);
|
NotifyDeletion(_ptr);
|
||||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
void *__freeme = Insert(_ptr,bytes,Cpu);
|
||||||
if ( __freeme ) {
|
if ( __freeme ) {
|
||||||
acceleratorFreeShared(__freeme);
|
acceleratorFreeShared(__freeme);
|
||||||
total_host-=bytes;
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"CpuFree "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
void *MemoryManager::CpuAllocate(size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_host+=bytes;
|
||||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
void *ptr = (void *) Lookup(bytes,Cpu);
|
||||||
if ( ptr == (void *) NULL ) {
|
if ( ptr == (void *) NULL ) {
|
||||||
ptr = (void *) acceleratorAllocCpu(bytes);
|
ptr = (void *) acceleratorAllocCpu(bytes);
|
||||||
total_host+=bytes;
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"CpuAllocate "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
return ptr;
|
return ptr;
|
||||||
}
|
}
|
||||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
||||||
{
|
{
|
||||||
|
total_host-=bytes;
|
||||||
NotifyDeletion(_ptr);
|
NotifyDeletion(_ptr);
|
||||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
void *__freeme = Insert(_ptr,bytes,Cpu);
|
||||||
if ( __freeme ) {
|
if ( __freeme ) {
|
||||||
acceleratorFreeCpu(__freeme);
|
acceleratorFreeCpu(__freeme);
|
||||||
total_host-=bytes;
|
|
||||||
}
|
}
|
||||||
|
#ifdef GRID_MM_VERBOSE
|
||||||
|
std::cout <<"CpuFree "<<std::endl;
|
||||||
|
PrintBytes();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -115,7 +159,6 @@ void MemoryManager::Init(void)
|
|||||||
|
|
||||||
char * str;
|
char * str;
|
||||||
int Nc;
|
int Nc;
|
||||||
int NcS;
|
|
||||||
|
|
||||||
str= getenv("GRID_ALLOC_NCACHE_LARGE");
|
str= getenv("GRID_ALLOC_NCACHE_LARGE");
|
||||||
if ( str ) {
|
if ( str ) {
|
||||||
@ -181,13 +224,13 @@ void *MemoryManager::Insert(void *ptr,size_t bytes,int type)
|
|||||||
#ifdef ALLOCATION_CACHE
|
#ifdef ALLOCATION_CACHE
|
||||||
bool small = (bytes < GRID_ALLOC_SMALL_LIMIT);
|
bool small = (bytes < GRID_ALLOC_SMALL_LIMIT);
|
||||||
int cache = type + small;
|
int cache = type + small;
|
||||||
return Insert(ptr,bytes,Entries[cache],Ncache[cache],Victim[cache]);
|
return Insert(ptr,bytes,Entries[cache],Ncache[cache],Victim[cache],CacheBytes[cache]);
|
||||||
#else
|
#else
|
||||||
return ptr;
|
return ptr;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim)
|
void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim, uint64_t &cacheBytes)
|
||||||
{
|
{
|
||||||
assert(ncache>0);
|
assert(ncache>0);
|
||||||
#ifdef GRID_OMP
|
#ifdef GRID_OMP
|
||||||
@ -211,6 +254,7 @@ void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries
|
|||||||
|
|
||||||
if ( entries[v].valid ) {
|
if ( entries[v].valid ) {
|
||||||
ret = entries[v].address;
|
ret = entries[v].address;
|
||||||
|
cacheBytes -= entries[v].bytes;
|
||||||
entries[v].valid = 0;
|
entries[v].valid = 0;
|
||||||
entries[v].address = NULL;
|
entries[v].address = NULL;
|
||||||
entries[v].bytes = 0;
|
entries[v].bytes = 0;
|
||||||
@ -219,6 +263,7 @@ void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries
|
|||||||
entries[v].address=ptr;
|
entries[v].address=ptr;
|
||||||
entries[v].bytes =bytes;
|
entries[v].bytes =bytes;
|
||||||
entries[v].valid =1;
|
entries[v].valid =1;
|
||||||
|
cacheBytes += bytes;
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
@ -228,13 +273,13 @@ void *MemoryManager::Lookup(size_t bytes,int type)
|
|||||||
#ifdef ALLOCATION_CACHE
|
#ifdef ALLOCATION_CACHE
|
||||||
bool small = (bytes < GRID_ALLOC_SMALL_LIMIT);
|
bool small = (bytes < GRID_ALLOC_SMALL_LIMIT);
|
||||||
int cache = type+small;
|
int cache = type+small;
|
||||||
return Lookup(bytes,Entries[cache],Ncache[cache]);
|
return Lookup(bytes,Entries[cache],Ncache[cache],CacheBytes[cache]);
|
||||||
#else
|
#else
|
||||||
return NULL;
|
return NULL;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void *MemoryManager::Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache)
|
void *MemoryManager::Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t & cacheBytes)
|
||||||
{
|
{
|
||||||
assert(ncache>0);
|
assert(ncache>0);
|
||||||
#ifdef GRID_OMP
|
#ifdef GRID_OMP
|
||||||
@ -243,6 +288,7 @@ void *MemoryManager::Lookup(size_t bytes,AllocationCacheEntry *entries,int ncach
|
|||||||
for(int e=0;e<ncache;e++){
|
for(int e=0;e<ncache;e++){
|
||||||
if ( entries[e].valid && ( entries[e].bytes == bytes ) ) {
|
if ( entries[e].valid && ( entries[e].bytes == bytes ) ) {
|
||||||
entries[e].valid = 0;
|
entries[e].valid = 0;
|
||||||
|
cacheBytes -= entries[e].bytes;
|
||||||
return entries[e].address;
|
return entries[e].address;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -82,14 +82,15 @@ private:
|
|||||||
static AllocationCacheEntry Entries[NallocType][NallocCacheMax];
|
static AllocationCacheEntry Entries[NallocType][NallocCacheMax];
|
||||||
static int Victim[NallocType];
|
static int Victim[NallocType];
|
||||||
static int Ncache[NallocType];
|
static int Ncache[NallocType];
|
||||||
|
static uint64_t CacheBytes[NallocType];
|
||||||
|
|
||||||
/////////////////////////////////////////////////
|
/////////////////////////////////////////////////
|
||||||
// Free pool
|
// Free pool
|
||||||
/////////////////////////////////////////////////
|
/////////////////////////////////////////////////
|
||||||
static void *Insert(void *ptr,size_t bytes,int type) ;
|
static void *Insert(void *ptr,size_t bytes,int type) ;
|
||||||
static void *Lookup(size_t bytes,int type) ;
|
static void *Lookup(size_t bytes,int type) ;
|
||||||
static void *Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim) ;
|
static void *Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim,uint64_t &cbytes) ;
|
||||||
static void *Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache) ;
|
static void *Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t &cbytes) ;
|
||||||
|
|
||||||
static void PrintBytes(void);
|
static void PrintBytes(void);
|
||||||
public:
|
public:
|
||||||
@ -113,6 +114,11 @@ private:
|
|||||||
static uint64_t HostToDeviceXfer;
|
static uint64_t HostToDeviceXfer;
|
||||||
static uint64_t DeviceToHostXfer;
|
static uint64_t DeviceToHostXfer;
|
||||||
|
|
||||||
|
static uint64_t DeviceAccesses;
|
||||||
|
static uint64_t HostAccesses;
|
||||||
|
static uint64_t DeviceAccessBytes;
|
||||||
|
static uint64_t HostAccessBytes;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
#ifndef GRID_UVM
|
#ifndef GRID_UVM
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
@ -151,6 +157,7 @@ private:
|
|||||||
|
|
||||||
// static void LRUupdate(AcceleratorViewEntry &AccCache);
|
// static void LRUupdate(AcceleratorViewEntry &AccCache);
|
||||||
static void LRUinsert(AcceleratorViewEntry &AccCache);
|
static void LRUinsert(AcceleratorViewEntry &AccCache);
|
||||||
|
static void LRUinsertback(AcceleratorViewEntry &AccCache);
|
||||||
static void LRUremove(AcceleratorViewEntry &AccCache);
|
static void LRUremove(AcceleratorViewEntry &AccCache);
|
||||||
|
|
||||||
// manage entries in the table
|
// manage entries in the table
|
||||||
@ -169,6 +176,7 @@ private:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
static void Print(void);
|
static void Print(void);
|
||||||
|
static void PrintState( void* CpuPtr);
|
||||||
static int isOpen (void* CpuPtr);
|
static int isOpen (void* CpuPtr);
|
||||||
static void ViewClose(void* CpuPtr,ViewMode mode);
|
static void ViewClose(void* CpuPtr,ViewMode mode);
|
||||||
static void *ViewOpen (void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
static void *ViewOpen (void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
#warning "Using explicit device memory copies"
|
#warning "Using explicit device memory copies"
|
||||||
NAMESPACE_BEGIN(Grid);
|
NAMESPACE_BEGIN(Grid);
|
||||||
//define dprintf(...) printf ( __VA_ARGS__ ); fflush(stdout);
|
//#define dprintf(...) printf ( __VA_ARGS__ ); fflush(stdout);
|
||||||
#define dprintf(...)
|
#define dprintf(...)
|
||||||
|
|
||||||
|
|
||||||
@ -23,6 +23,11 @@ uint64_t MemoryManager::HostToDeviceBytes;
|
|||||||
uint64_t MemoryManager::DeviceToHostBytes;
|
uint64_t MemoryManager::DeviceToHostBytes;
|
||||||
uint64_t MemoryManager::HostToDeviceXfer;
|
uint64_t MemoryManager::HostToDeviceXfer;
|
||||||
uint64_t MemoryManager::DeviceToHostXfer;
|
uint64_t MemoryManager::DeviceToHostXfer;
|
||||||
|
uint64_t MemoryManager::DeviceAccesses;
|
||||||
|
uint64_t MemoryManager::HostAccesses;
|
||||||
|
uint64_t MemoryManager::DeviceAccessBytes;
|
||||||
|
uint64_t MemoryManager::HostAccessBytes;
|
||||||
|
|
||||||
|
|
||||||
////////////////////////////////////
|
////////////////////////////////////
|
||||||
// Priority ordering for unlocked entries
|
// Priority ordering for unlocked entries
|
||||||
@ -86,6 +91,14 @@ void MemoryManager::LRUinsert(AcceleratorViewEntry &AccCache)
|
|||||||
AccCache.LRU_valid = 1;
|
AccCache.LRU_valid = 1;
|
||||||
DeviceLRUBytes+=AccCache.bytes;
|
DeviceLRUBytes+=AccCache.bytes;
|
||||||
}
|
}
|
||||||
|
void MemoryManager::LRUinsertback(AcceleratorViewEntry &AccCache)
|
||||||
|
{
|
||||||
|
assert(AccCache.LRU_valid==0);
|
||||||
|
LRU.push_back(AccCache.CpuPtr);
|
||||||
|
AccCache.LRU_entry = --LRU.end();
|
||||||
|
AccCache.LRU_valid = 1;
|
||||||
|
DeviceLRUBytes+=AccCache.bytes;
|
||||||
|
}
|
||||||
void MemoryManager::LRUremove(AcceleratorViewEntry &AccCache)
|
void MemoryManager::LRUremove(AcceleratorViewEntry &AccCache)
|
||||||
{
|
{
|
||||||
assert(AccCache.LRU_valid==1);
|
assert(AccCache.LRU_valid==1);
|
||||||
@ -129,6 +142,7 @@ void MemoryManager::Evict(AcceleratorViewEntry &AccCache)
|
|||||||
dprintf("MemoryManager: Evict(%llx) %llx\n",(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr);
|
dprintf("MemoryManager: Evict(%llx) %llx\n",(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr);
|
||||||
assert(AccCache.accLock==0);
|
assert(AccCache.accLock==0);
|
||||||
assert(AccCache.cpuLock==0);
|
assert(AccCache.cpuLock==0);
|
||||||
|
|
||||||
if(AccCache.state==AccDirty) {
|
if(AccCache.state==AccDirty) {
|
||||||
Flush(AccCache);
|
Flush(AccCache);
|
||||||
}
|
}
|
||||||
@ -231,6 +245,9 @@ uint64_t MemoryManager::AcceleratorViewOpen(uint64_t CpuPtr,size_t bytes,ViewMod
|
|||||||
EntryCreate(CpuPtr,bytes,mode,hint);
|
EntryCreate(CpuPtr,bytes,mode,hint);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DeviceAccesses++;
|
||||||
|
DeviceAccessBytes+=bytes;
|
||||||
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||||
auto & AccCache = AccCacheIterator->second;
|
auto & AccCache = AccCacheIterator->second;
|
||||||
if (!AccCache.AccPtr) {
|
if (!AccCache.AccPtr) {
|
||||||
@ -349,6 +366,10 @@ void MemoryManager::CpuViewClose(uint64_t CpuPtr)
|
|||||||
assert(AccCache.accLock==0);
|
assert(AccCache.accLock==0);
|
||||||
|
|
||||||
AccCache.cpuLock--;
|
AccCache.cpuLock--;
|
||||||
|
|
||||||
|
if(AccCache.cpuLock==0) {
|
||||||
|
LRUinsertback(AccCache);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
/*
|
/*
|
||||||
* Action State StateNext Flush Clone
|
* Action State StateNext Flush Clone
|
||||||
@ -371,6 +392,9 @@ uint64_t MemoryManager::CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,V
|
|||||||
EntryCreate(CpuPtr,bytes,mode,transient);
|
EntryCreate(CpuPtr,bytes,mode,transient);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
HostAccesses++;
|
||||||
|
HostAccessBytes+=bytes;
|
||||||
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||||
auto & AccCache = AccCacheIterator->second;
|
auto & AccCache = AccCacheIterator->second;
|
||||||
|
|
||||||
@ -416,6 +440,12 @@ uint64_t MemoryManager::CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,V
|
|||||||
|
|
||||||
AccCache.transient= transient? EvictNext : 0;
|
AccCache.transient= transient? EvictNext : 0;
|
||||||
|
|
||||||
|
// If view is opened on host remove from LRU
|
||||||
|
// Host close says evict next from device
|
||||||
|
if(AccCache.LRU_valid==1){
|
||||||
|
LRUremove(AccCache);
|
||||||
|
}
|
||||||
|
|
||||||
return AccCache.CpuPtr;
|
return AccCache.CpuPtr;
|
||||||
}
|
}
|
||||||
void MemoryManager::NotifyDeletion(void *_ptr)
|
void MemoryManager::NotifyDeletion(void *_ptr)
|
||||||
@ -429,6 +459,7 @@ void MemoryManager::NotifyDeletion(void *_ptr)
|
|||||||
}
|
}
|
||||||
void MemoryManager::Print(void)
|
void MemoryManager::Print(void)
|
||||||
{
|
{
|
||||||
|
PrintBytes();
|
||||||
std::cout << GridLogDebug << "--------------------------------------------" << std::endl;
|
std::cout << GridLogDebug << "--------------------------------------------" << std::endl;
|
||||||
std::cout << GridLogDebug << "Memory Manager " << std::endl;
|
std::cout << GridLogDebug << "Memory Manager " << std::endl;
|
||||||
std::cout << GridLogDebug << "--------------------------------------------" << std::endl;
|
std::cout << GridLogDebug << "--------------------------------------------" << std::endl;
|
||||||
@ -473,6 +504,32 @@ int MemoryManager::isOpen (void* _CpuPtr)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void MemoryManager::PrintState(void* _CpuPtr)
|
||||||
|
{
|
||||||
|
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
||||||
|
|
||||||
|
if ( EntryPresent(CpuPtr) ){
|
||||||
|
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||||
|
auto & AccCache = AccCacheIterator->second;
|
||||||
|
std::string str;
|
||||||
|
if ( AccCache.state==Empty ) str = std::string("Empty");
|
||||||
|
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
||||||
|
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
||||||
|
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
||||||
|
if ( AccCache.state==EvictNext) str = std::string("EvictNext");
|
||||||
|
|
||||||
|
std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
|
||||||
|
std::cout << GridLogMessage << "0x"<<std::hex<<AccCache.CpuPtr<<std::dec
|
||||||
|
<< "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
||||||
|
<< "\t" << AccCache.cpuLock
|
||||||
|
<< "\t" << AccCache.accLock
|
||||||
|
<< "\t" << AccCache.LRU_valid<<std::endl;
|
||||||
|
|
||||||
|
} else {
|
||||||
|
std::cout << GridLogMessage << "No Entry in AccCache table." << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
NAMESPACE_END(Grid);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
@ -12,10 +12,18 @@ uint64_t MemoryManager::HostToDeviceBytes;
|
|||||||
uint64_t MemoryManager::DeviceToHostBytes;
|
uint64_t MemoryManager::DeviceToHostBytes;
|
||||||
uint64_t MemoryManager::HostToDeviceXfer;
|
uint64_t MemoryManager::HostToDeviceXfer;
|
||||||
uint64_t MemoryManager::DeviceToHostXfer;
|
uint64_t MemoryManager::DeviceToHostXfer;
|
||||||
|
uint64_t MemoryManager::DeviceAccesses;
|
||||||
|
uint64_t MemoryManager::HostAccesses;
|
||||||
|
uint64_t MemoryManager::DeviceAccessBytes;
|
||||||
|
uint64_t MemoryManager::HostAccessBytes;
|
||||||
|
|
||||||
void MemoryManager::ViewClose(void* AccPtr,ViewMode mode){};
|
void MemoryManager::ViewClose(void* AccPtr,ViewMode mode){};
|
||||||
void *MemoryManager::ViewOpen(void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint){ return CpuPtr; };
|
void *MemoryManager::ViewOpen(void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint){ return CpuPtr; };
|
||||||
int MemoryManager::isOpen (void* CpuPtr) { return 0;}
|
int MemoryManager::isOpen (void* CpuPtr) { return 0;}
|
||||||
|
void MemoryManager::PrintState(void* CpuPtr)
|
||||||
|
{
|
||||||
|
std::cout << GridLogMessage << "Host<->Device memory movement not currently managed by Grid." << std::endl;
|
||||||
|
};
|
||||||
void MemoryManager::Print(void){};
|
void MemoryManager::Print(void){};
|
||||||
void MemoryManager::NotifyDeletion(void *ptr){};
|
void MemoryManager::NotifyDeletion(void *ptr){};
|
||||||
|
|
||||||
|
@ -36,7 +36,7 @@ static const int CbBlack=1;
|
|||||||
static const int Even =CbRed;
|
static const int Even =CbRed;
|
||||||
static const int Odd =CbBlack;
|
static const int Odd =CbBlack;
|
||||||
|
|
||||||
accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex, Coordinate &rdim, Coordinate &chk_dim_msk)
|
accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex,const Coordinate &rdim,const Coordinate &chk_dim_msk)
|
||||||
{
|
{
|
||||||
int nd=rdim.size();
|
int nd=rdim.size();
|
||||||
Coordinate coor(nd);
|
Coordinate coor(nd);
|
||||||
|
@ -33,6 +33,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
NAMESPACE_BEGIN(Grid);
|
||||||
|
|
||||||
|
bool Stencil_force_mpi = true;
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////
|
||||||
// Info that is setup once and indept of cartesian layout
|
// Info that is setup once and indept of cartesian layout
|
||||||
///////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
|
|
||||||
/*************************************************************************************
|
/*************************************************************************************
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
Grid physics library, www.github.com/paboyle/Grid
|
||||||
@ -36,6 +35,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
NAMESPACE_BEGIN(Grid);
|
||||||
|
|
||||||
|
extern bool Stencil_force_mpi ;
|
||||||
|
|
||||||
class CartesianCommunicator : public SharedMemory {
|
class CartesianCommunicator : public SharedMemory {
|
||||||
|
|
||||||
public:
|
public:
|
||||||
@ -108,6 +109,8 @@ public:
|
|||||||
////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////
|
||||||
// Reduction
|
// Reduction
|
||||||
////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////
|
||||||
|
void GlobalMax(RealD &);
|
||||||
|
void GlobalMax(RealF &);
|
||||||
void GlobalSum(RealF &);
|
void GlobalSum(RealF &);
|
||||||
void GlobalSumVector(RealF *,int N);
|
void GlobalSumVector(RealF *,int N);
|
||||||
void GlobalSum(RealD &);
|
void GlobalSum(RealD &);
|
||||||
|
@ -275,6 +275,16 @@ void CartesianCommunicator::GlobalXOR(uint64_t &u){
|
|||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
|
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
|
||||||
assert(ierr==0);
|
assert(ierr==0);
|
||||||
}
|
}
|
||||||
|
void CartesianCommunicator::GlobalMax(float &f)
|
||||||
|
{
|
||||||
|
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_MAX,communicator);
|
||||||
|
assert(ierr==0);
|
||||||
|
}
|
||||||
|
void CartesianCommunicator::GlobalMax(double &d)
|
||||||
|
{
|
||||||
|
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_MAX,communicator);
|
||||||
|
assert(ierr==0);
|
||||||
|
}
|
||||||
void CartesianCommunicator::GlobalSum(float &f){
|
void CartesianCommunicator::GlobalSum(float &f){
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
|
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
|
||||||
assert(ierr==0);
|
assert(ierr==0);
|
||||||
@ -360,7 +370,7 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
|
|||||||
double off_node_bytes=0.0;
|
double off_node_bytes=0.0;
|
||||||
int tag;
|
int tag;
|
||||||
|
|
||||||
if ( gfrom ==MPI_UNDEFINED) {
|
if ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||||
tag= dir+from*32;
|
tag= dir+from*32;
|
||||||
ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,tag,communicator_halo[commdir],&rrq);
|
ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,tag,communicator_halo[commdir],&rrq);
|
||||||
assert(ierr==0);
|
assert(ierr==0);
|
||||||
@ -368,12 +378,18 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
|
|||||||
off_node_bytes+=bytes;
|
off_node_bytes+=bytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( gdest == MPI_UNDEFINED ) {
|
if ( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||||
tag= dir+_processor*32;
|
tag= dir+_processor*32;
|
||||||
ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,tag,communicator_halo[commdir],&xrq);
|
ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,tag,communicator_halo[commdir],&xrq);
|
||||||
assert(ierr==0);
|
assert(ierr==0);
|
||||||
list.push_back(xrq);
|
list.push_back(xrq);
|
||||||
off_node_bytes+=bytes;
|
off_node_bytes+=bytes;
|
||||||
|
} else {
|
||||||
|
// TODO : make a OMP loop on CPU, call threaded bcopy
|
||||||
|
void *shm = (void *) this->ShmBufferTranslate(dest,recv);
|
||||||
|
assert(shm!=NULL);
|
||||||
|
// std::cout <<"acceleratorCopyDeviceToDeviceAsynch"<< std::endl;
|
||||||
|
acceleratorCopyDeviceToDeviceAsynch(xmit,shm,bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( CommunicatorPolicy == CommunicatorPolicySequential ) {
|
if ( CommunicatorPolicy == CommunicatorPolicySequential ) {
|
||||||
@ -384,6 +400,9 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
|
|||||||
}
|
}
|
||||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
|
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
|
||||||
{
|
{
|
||||||
|
// std::cout << "Copy Synchronised\n"<<std::endl;
|
||||||
|
acceleratorCopySynchronise();
|
||||||
|
|
||||||
int nreq=list.size();
|
int nreq=list.size();
|
||||||
|
|
||||||
if (nreq==0) return;
|
if (nreq==0) return;
|
||||||
|
@ -67,6 +67,8 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
|||||||
|
|
||||||
CartesianCommunicator::~CartesianCommunicator(){}
|
CartesianCommunicator::~CartesianCommunicator(){}
|
||||||
|
|
||||||
|
void CartesianCommunicator::GlobalMax(float &){}
|
||||||
|
void CartesianCommunicator::GlobalMax(double &){}
|
||||||
void CartesianCommunicator::GlobalSum(float &){}
|
void CartesianCommunicator::GlobalSum(float &){}
|
||||||
void CartesianCommunicator::GlobalSumVector(float *,int N){}
|
void CartesianCommunicator::GlobalSumVector(float *,int N){}
|
||||||
void CartesianCommunicator::GlobalSum(double &){}
|
void CartesianCommunicator::GlobalSum(double &){}
|
||||||
|
@ -7,6 +7,7 @@
|
|||||||
Copyright (C) 2015
|
Copyright (C) 2015
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||||
|
Author: Christoph Lehner <christoph@lhnr.de>
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
This program is free software; you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
@ -34,6 +35,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|||||||
#endif
|
#endif
|
||||||
#ifdef GRID_HIP
|
#ifdef GRID_HIP
|
||||||
#include <hip/hip_runtime_api.h>
|
#include <hip/hip_runtime_api.h>
|
||||||
|
#endif
|
||||||
|
#ifdef GRID_SYCl
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
NAMESPACE_BEGIN(Grid);
|
||||||
@ -69,6 +73,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
|||||||
WorldNodes = WorldSize/WorldShmSize;
|
WorldNodes = WorldSize/WorldShmSize;
|
||||||
assert( (WorldNodes * WorldShmSize) == WorldSize );
|
assert( (WorldNodes * WorldShmSize) == WorldSize );
|
||||||
|
|
||||||
|
|
||||||
// FIXME: Check all WorldShmSize are the same ?
|
// FIXME: Check all WorldShmSize are the same ?
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////
|
||||||
@ -169,6 +174,23 @@ static inline int divides(int a,int b)
|
|||||||
}
|
}
|
||||||
void GlobalSharedMemory::GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims)
|
void GlobalSharedMemory::GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims)
|
||||||
{
|
{
|
||||||
|
////////////////////////////////////////////////////////////////
|
||||||
|
// Allow user to configure through environment variable
|
||||||
|
////////////////////////////////////////////////////////////////
|
||||||
|
char* str = getenv(("GRID_SHM_DIMS_" + std::to_string(ShmDims.size())).c_str());
|
||||||
|
if ( str ) {
|
||||||
|
std::vector<int> IntShmDims;
|
||||||
|
GridCmdOptionIntVector(std::string(str),IntShmDims);
|
||||||
|
assert(IntShmDims.size() == WorldDims.size());
|
||||||
|
long ShmSize = 1;
|
||||||
|
for (int dim=0;dim<WorldDims.size();dim++) {
|
||||||
|
ShmSize *= (ShmDims[dim] = IntShmDims[dim]);
|
||||||
|
assert(divides(ShmDims[dim],WorldDims[dim]));
|
||||||
|
}
|
||||||
|
assert(ShmSize == WorldShmSize);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////
|
||||||
// Powers of 2,3,5 only in prime decomposition for now
|
// Powers of 2,3,5 only in prime decomposition for now
|
||||||
////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////
|
||||||
@ -428,7 +450,47 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// Hugetlbfs mapping intended
|
// Hugetlbfs mapping intended
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
#if defined(GRID_CUDA) ||defined(GRID_HIP)
|
#if defined(GRID_CUDA) ||defined(GRID_HIP) || defined(GRID_SYCL)
|
||||||
|
|
||||||
|
//if defined(GRID_SYCL)
|
||||||
|
#if 0
|
||||||
|
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||||
|
{
|
||||||
|
void * ShmCommBuf ;
|
||||||
|
assert(_ShmSetup==1);
|
||||||
|
assert(_ShmAlloc==0);
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// allocate the pointer array for shared windows for our group
|
||||||
|
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
MPI_Barrier(WorldShmComm);
|
||||||
|
WorldShmCommBufs.resize(WorldShmSize);
|
||||||
|
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// Each MPI rank should allocate our own buffer
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||||
|
|
||||||
|
if (ShmCommBuf == (void *)NULL ) {
|
||||||
|
std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes
|
||||||
|
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||||
|
|
||||||
|
SharedMemoryZero(ShmCommBuf,bytes);
|
||||||
|
|
||||||
|
assert(WorldShmSize == 1);
|
||||||
|
for(int r=0;r<WorldShmSize;r++){
|
||||||
|
WorldShmCommBufs[r] = ShmCommBuf;
|
||||||
|
}
|
||||||
|
_ShmAllocBytes=bytes;
|
||||||
|
_ShmAlloc=1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if defined(GRID_CUDA) ||defined(GRID_HIP) ||defined(GRID_SYCL)
|
||||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||||
{
|
{
|
||||||
void * ShmCommBuf ;
|
void * ShmCommBuf ;
|
||||||
@ -452,18 +514,16 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
// Each MPI rank should allocate our own buffer
|
// Each MPI rank should allocate our own buffer
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||||
|
|
||||||
if (ShmCommBuf == (void *)NULL ) {
|
if (ShmCommBuf == (void *)NULL ) {
|
||||||
std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
// if ( WorldRank == 0 ){
|
if ( WorldRank == 0 ){
|
||||||
if ( 1 ){
|
|
||||||
std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes
|
std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes
|
||||||
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||||
}
|
}
|
||||||
SharedMemoryZero(ShmCommBuf,bytes);
|
SharedMemoryZero(ShmCommBuf,bytes);
|
||||||
|
std::cout<< "Setting up IPC"<<std::endl;
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// Loop over ranks/gpu's on our node
|
// Loop over ranks/gpu's on our node
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
@ -473,6 +533,29 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
//////////////////////////////////////////////////
|
//////////////////////////////////////////////////
|
||||||
// If it is me, pass around the IPC access key
|
// If it is me, pass around the IPC access key
|
||||||
//////////////////////////////////////////////////
|
//////////////////////////////////////////////////
|
||||||
|
void * thisBuf = ShmCommBuf;
|
||||||
|
if(!Stencil_force_mpi) {
|
||||||
|
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||||
|
typedef struct { int fd; pid_t pid ; } clone_mem_t;
|
||||||
|
|
||||||
|
auto zeDevice = cl::sycl::get_native<cl::sycl::backend::level_zero>(theGridAccelerator->get_device());
|
||||||
|
auto zeContext = cl::sycl::get_native<cl::sycl::backend::level_zero>(theGridAccelerator->get_context());
|
||||||
|
|
||||||
|
ze_ipc_mem_handle_t ihandle;
|
||||||
|
clone_mem_t handle;
|
||||||
|
|
||||||
|
if ( r==WorldShmRank ) {
|
||||||
|
auto err = zeMemGetIpcHandle(zeContext,ShmCommBuf,&ihandle);
|
||||||
|
if ( err != ZE_RESULT_SUCCESS ) {
|
||||||
|
std::cout << "SharedMemoryMPI.cc zeMemGetIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
} else {
|
||||||
|
std::cout << "SharedMemoryMPI.cc zeMemGetIpcHandle succeeded for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||||
|
}
|
||||||
|
memcpy((void *)&handle.fd,(void *)&ihandle,sizeof(int));
|
||||||
|
handle.pid = getpid();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#ifdef GRID_CUDA
|
#ifdef GRID_CUDA
|
||||||
cudaIpcMemHandle_t handle;
|
cudaIpcMemHandle_t handle;
|
||||||
if ( r==WorldShmRank ) {
|
if ( r==WorldShmRank ) {
|
||||||
@ -493,6 +576,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
//////////////////////////////////////////////////
|
//////////////////////////////////////////////////
|
||||||
// Share this IPC handle across the Shm Comm
|
// Share this IPC handle across the Shm Comm
|
||||||
//////////////////////////////////////////////////
|
//////////////////////////////////////////////////
|
||||||
@ -508,7 +592,35 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
///////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////
|
||||||
// If I am not the source, overwrite thisBuf with remote buffer
|
// If I am not the source, overwrite thisBuf with remote buffer
|
||||||
///////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////
|
||||||
void * thisBuf = ShmCommBuf;
|
|
||||||
|
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||||
|
if ( r!=WorldShmRank ) {
|
||||||
|
thisBuf = nullptr;
|
||||||
|
std::cout<<"mapping seeking remote pid/fd "
|
||||||
|
<<handle.pid<<"/"
|
||||||
|
<<handle.fd<<std::endl;
|
||||||
|
|
||||||
|
int pidfd = syscall(SYS_pidfd_open,handle.pid,0);
|
||||||
|
std::cout<<"Using IpcHandle pidfd "<<pidfd<<"\n";
|
||||||
|
// int myfd = syscall(SYS_pidfd_getfd,pidfd,handle.fd,0);
|
||||||
|
int myfd = syscall(438,pidfd,handle.fd,0);
|
||||||
|
|
||||||
|
std::cout<<"Using IpcHandle myfd "<<myfd<<"\n";
|
||||||
|
|
||||||
|
memcpy((void *)&ihandle,(void *)&myfd,sizeof(int));
|
||||||
|
|
||||||
|
auto err = zeMemOpenIpcHandle(zeContext,zeDevice,ihandle,0,&thisBuf);
|
||||||
|
if ( err != ZE_RESULT_SUCCESS ) {
|
||||||
|
std::cout << "SharedMemoryMPI.cc "<<zeContext<<" "<<zeDevice<<std::endl;
|
||||||
|
std::cout << "SharedMemoryMPI.cc zeMemOpenIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
} else {
|
||||||
|
std::cout << "SharedMemoryMPI.cc zeMemOpenIpcHandle succeeded for rank "<<r<<std::endl;
|
||||||
|
std::cout << "SharedMemoryMPI.cc zeMemOpenIpcHandle pointer is "<<std::hex<<thisBuf<<std::dec<<std::endl;
|
||||||
|
}
|
||||||
|
assert(thisBuf!=nullptr);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#ifdef GRID_CUDA
|
#ifdef GRID_CUDA
|
||||||
if ( r!=WorldShmRank ) {
|
if ( r!=WorldShmRank ) {
|
||||||
auto err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
|
auto err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
|
||||||
@ -530,6 +642,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
///////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////
|
||||||
// Save a copy of the device buffers
|
// Save a copy of the device buffers
|
||||||
///////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////
|
||||||
|
}
|
||||||
WorldShmCommBufs[r] = thisBuf;
|
WorldShmCommBufs[r] = thisBuf;
|
||||||
#else
|
#else
|
||||||
WorldShmCommBufs[r] = ShmCommBuf;
|
WorldShmCommBufs[r] = ShmCommBuf;
|
||||||
@ -539,6 +652,8 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
_ShmAllocBytes=bytes;
|
_ShmAllocBytes=bytes;
|
||||||
_ShmAlloc=1;
|
_ShmAlloc=1;
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
#else
|
#else
|
||||||
#ifdef GRID_MPI3_SHMMMAP
|
#ifdef GRID_MPI3_SHMMMAP
|
||||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||||
@ -709,16 +824,16 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
|||||||
/////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////
|
||||||
void GlobalSharedMemory::SharedMemoryZero(void *dest,size_t bytes)
|
void GlobalSharedMemory::SharedMemoryZero(void *dest,size_t bytes)
|
||||||
{
|
{
|
||||||
#ifdef GRID_CUDA
|
#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||||
cudaMemset(dest,0,bytes);
|
acceleratorMemSet(dest,0,bytes);
|
||||||
#else
|
#else
|
||||||
bzero(dest,bytes);
|
bzero(dest,bytes);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
void GlobalSharedMemory::SharedMemoryCopy(void *dest,void *src,size_t bytes)
|
void GlobalSharedMemory::SharedMemoryCopy(void *dest,void *src,size_t bytes)
|
||||||
{
|
{
|
||||||
#ifdef GRID_CUDA
|
#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||||
cudaMemcpy(dest,src,bytes,cudaMemcpyDefault);
|
acceleratorCopyToDevice(src,dest,bytes);
|
||||||
#else
|
#else
|
||||||
bcopy(src,dest,bytes);
|
bcopy(src,dest,bytes);
|
||||||
#endif
|
#endif
|
||||||
@ -782,7 +897,7 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
SharedMemoryTest();
|
//SharedMemoryTest();
|
||||||
}
|
}
|
||||||
//////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////
|
||||||
// On node barrier
|
// On node barrier
|
||||||
|
@ -112,7 +112,9 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
|
|||||||
if ( cbmask ==0x3){
|
if ( cbmask ==0x3){
|
||||||
#ifdef ACCELERATOR_CSHIFT
|
#ifdef ACCELERATOR_CSHIFT
|
||||||
autoView(rhs_v , rhs, AcceleratorRead);
|
autoView(rhs_v , rhs, AcceleratorRead);
|
||||||
accelerator_for2d(n,e1,b,e2,1,{
|
accelerator_for(nn,e1*e2,1,{
|
||||||
|
int n = nn%e1;
|
||||||
|
int b = nn/e1;
|
||||||
int o = n*n1;
|
int o = n*n1;
|
||||||
int offset = b+n*e2;
|
int offset = b+n*e2;
|
||||||
|
|
||||||
@ -135,7 +137,9 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
|
|||||||
std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
|
std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
|
||||||
#ifdef ACCELERATOR_CSHIFT
|
#ifdef ACCELERATOR_CSHIFT
|
||||||
autoView(rhs_v , rhs, AcceleratorRead);
|
autoView(rhs_v , rhs, AcceleratorRead);
|
||||||
accelerator_for2d(n,e1,b,e2,1,{
|
accelerator_for(nn,e1*e2,1,{
|
||||||
|
int n = nn%e1;
|
||||||
|
int b = nn/e1;
|
||||||
|
|
||||||
Coordinate coor;
|
Coordinate coor;
|
||||||
|
|
||||||
@ -257,7 +261,9 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
|
|||||||
int _slice_block = rhs.Grid()->_slice_block[dimension];
|
int _slice_block = rhs.Grid()->_slice_block[dimension];
|
||||||
#ifdef ACCELERATOR_CSHIFT
|
#ifdef ACCELERATOR_CSHIFT
|
||||||
autoView( rhs_v , rhs, AcceleratorWrite);
|
autoView( rhs_v , rhs, AcceleratorWrite);
|
||||||
accelerator_for2d(n,e1,b,e2,1,{
|
accelerator_for(nn,e1*e2,1,{
|
||||||
|
int n = nn%e1;
|
||||||
|
int b = nn/e1;
|
||||||
int o = n*_slice_stride;
|
int o = n*_slice_stride;
|
||||||
int offset = b+n*_slice_block;
|
int offset = b+n*_slice_block;
|
||||||
merge(rhs_v[so+o+b],pointers,offset);
|
merge(rhs_v[so+o+b],pointers,offset);
|
||||||
@ -274,7 +280,7 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
|
|||||||
|
|
||||||
// Case of SIMD split AND checker dim cannot currently be hit, except in
|
// Case of SIMD split AND checker dim cannot currently be hit, except in
|
||||||
// Test_cshift_red_black code.
|
// Test_cshift_red_black code.
|
||||||
// std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
||||||
std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
|
std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
|
||||||
assert(0); // This will fail if hit on GPU
|
assert(0); // This will fail if hit on GPU
|
||||||
autoView( rhs_v, rhs, CpuWrite);
|
autoView( rhs_v, rhs, CpuWrite);
|
||||||
|
@ -122,8 +122,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
|
|||||||
assert(shift<fd);
|
assert(shift<fd);
|
||||||
|
|
||||||
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
||||||
cshiftVector<vobj> send_buf(buffer_size);
|
static cshiftVector<vobj> send_buf; send_buf.resize(buffer_size);
|
||||||
cshiftVector<vobj> recv_buf(buffer_size);
|
static cshiftVector<vobj> recv_buf; recv_buf.resize(buffer_size);
|
||||||
|
|
||||||
int cb= (cbmask==0x2)? Odd : Even;
|
int cb= (cbmask==0x2)? Odd : Even;
|
||||||
int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
||||||
@ -198,8 +198,8 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
|
|||||||
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
||||||
// int words = sizeof(vobj)/sizeof(vector_type);
|
// int words = sizeof(vobj)/sizeof(vector_type);
|
||||||
|
|
||||||
std::vector<cshiftVector<scalar_object> > send_buf_extract(Nsimd);
|
static std::vector<cshiftVector<scalar_object> > send_buf_extract; send_buf_extract.resize(Nsimd);
|
||||||
std::vector<cshiftVector<scalar_object> > recv_buf_extract(Nsimd);
|
static std::vector<cshiftVector<scalar_object> > recv_buf_extract; recv_buf_extract.resize(Nsimd);
|
||||||
scalar_object * recv_buf_extract_mpi;
|
scalar_object * recv_buf_extract_mpi;
|
||||||
scalar_object * send_buf_extract_mpi;
|
scalar_object * send_buf_extract_mpi;
|
||||||
|
|
||||||
@ -294,8 +294,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
|
|||||||
assert(shift<fd);
|
assert(shift<fd);
|
||||||
|
|
||||||
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
||||||
cshiftVector<vobj> send_buf_v(buffer_size);
|
static cshiftVector<vobj> send_buf_v; send_buf_v.resize(buffer_size);
|
||||||
cshiftVector<vobj> recv_buf_v(buffer_size);
|
static cshiftVector<vobj> recv_buf_v; recv_buf_v.resize(buffer_size);
|
||||||
vobj *send_buf;
|
vobj *send_buf;
|
||||||
vobj *recv_buf;
|
vobj *recv_buf;
|
||||||
{
|
{
|
||||||
@ -381,8 +381,8 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
|
|||||||
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
||||||
// int words = sizeof(vobj)/sizeof(vector_type);
|
// int words = sizeof(vobj)/sizeof(vector_type);
|
||||||
|
|
||||||
std::vector<cshiftVector<scalar_object> > send_buf_extract(Nsimd);
|
static std::vector<cshiftVector<scalar_object> > send_buf_extract; send_buf_extract.resize(Nsimd);
|
||||||
std::vector<cshiftVector<scalar_object> > recv_buf_extract(Nsimd);
|
static std::vector<cshiftVector<scalar_object> > recv_buf_extract; recv_buf_extract.resize(Nsimd);
|
||||||
scalar_object * recv_buf_extract_mpi;
|
scalar_object * recv_buf_extract_mpi;
|
||||||
scalar_object * send_buf_extract_mpi;
|
scalar_object * send_buf_extract_mpi;
|
||||||
{
|
{
|
||||||
|
@ -225,7 +225,7 @@ void axpy(Lattice<vobj> &ret,sobj a,const Lattice<vobj> &x,const Lattice<vobj> &
|
|||||||
autoView( x_v , x, AcceleratorRead);
|
autoView( x_v , x, AcceleratorRead);
|
||||||
autoView( y_v , y, AcceleratorRead);
|
autoView( y_v , y, AcceleratorRead);
|
||||||
accelerator_for(ss,x_v.size(),vobj::Nsimd(),{
|
accelerator_for(ss,x_v.size(),vobj::Nsimd(),{
|
||||||
auto tmp = a*x_v(ss)+y_v(ss);
|
auto tmp = a*coalescedRead(x_v[ss])+coalescedRead(y_v[ss]);
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
coalescedWrite(ret_v[ss],tmp);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -88,6 +88,13 @@ public:
|
|||||||
LatticeView<vobj> accessor(*( (LatticeAccelerator<vobj> *) this),mode);
|
LatticeView<vobj> accessor(*( (LatticeAccelerator<vobj> *) this),mode);
|
||||||
accessor.ViewClose();
|
accessor.ViewClose();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper function to print the state of this object in the AccCache
|
||||||
|
void PrintCacheState(void)
|
||||||
|
{
|
||||||
|
MemoryManager::PrintState(this->_odata);
|
||||||
|
}
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////////
|
||||||
// Return a view object that may be dereferenced in site loops.
|
// Return a view object that may be dereferenced in site loops.
|
||||||
// The view is trivially copy constructible and may be copied to an accelerator device
|
// The view is trivially copy constructible and may be copied to an accelerator device
|
||||||
|
@ -125,7 +125,7 @@ void basisRotate(VField &basis,Matrix& Qt,int j0, int j1, int k0,int k1,int Nm)
|
|||||||
|
|
||||||
for(int k=k0; k<k1; ++k){
|
for(int k=k0; k<k1; ++k){
|
||||||
auto tmp = coalescedRead(Bp[ss*nrot+j]);
|
auto tmp = coalescedRead(Bp[ss*nrot+j]);
|
||||||
coalescedWrite(Bp[ss*nrot+j],tmp+ Qt_p[jj*Nm+k] * coalescedRead(basis_v[k][sss]));
|
coalescedWrite(Bp[ss*nrot+j],tmp+ Qt_p[jj*Nm+k] * coalescedRead(basis_vp[k][sss]));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -134,7 +134,7 @@ void basisRotate(VField &basis,Matrix& Qt,int j0, int j1, int k0,int k1,int Nm)
|
|||||||
int jj =j0+j;
|
int jj =j0+j;
|
||||||
int ss =sj/nrot;
|
int ss =sj/nrot;
|
||||||
int sss=ss+s;
|
int sss=ss+s;
|
||||||
coalescedWrite(basis_v[jj][sss],coalescedRead(Bp[ss*nrot+j]));
|
coalescedWrite(basis_vp[jj][sss],coalescedRead(Bp[ss*nrot+j]));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -96,8 +96,34 @@ inline typename vobj::scalar_objectD sumD_cpu(const vobj *arg, Integer osites)
|
|||||||
ssobj ret = ssum;
|
ssobj ret = ssum;
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
/*
|
||||||
|
Threaded max, don't use for now
|
||||||
|
template<class Double>
|
||||||
|
inline Double max(const Double *arg, Integer osites)
|
||||||
|
{
|
||||||
|
// const int Nsimd = vobj::Nsimd();
|
||||||
|
const int nthread = GridThread::GetThreads();
|
||||||
|
|
||||||
|
std::vector<Double> maxarray(nthread);
|
||||||
|
|
||||||
|
thread_for(thr,nthread, {
|
||||||
|
int nwork, mywork, myoff;
|
||||||
|
nwork = osites;
|
||||||
|
GridThread::GetWork(nwork,thr,mywork,myoff);
|
||||||
|
Double max=arg[0];
|
||||||
|
for(int ss=myoff;ss<mywork+myoff; ss++){
|
||||||
|
if( arg[ss] > max ) max = arg[ss];
|
||||||
|
}
|
||||||
|
maxarray[thr]=max;
|
||||||
|
});
|
||||||
|
|
||||||
|
Double tmax=maxarray[0];
|
||||||
|
for(int i=0;i<nthread;i++){
|
||||||
|
if (maxarray[i]>tmax) tmax = maxarray[i];
|
||||||
|
}
|
||||||
|
return tmax;
|
||||||
|
}
|
||||||
|
*/
|
||||||
template<class vobj>
|
template<class vobj>
|
||||||
inline typename vobj::scalar_object sum(const vobj *arg, Integer osites)
|
inline typename vobj::scalar_object sum(const vobj *arg, Integer osites)
|
||||||
{
|
{
|
||||||
@ -141,6 +167,32 @@ template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
|
|||||||
return real(nrm);
|
return real(nrm);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//The global maximum of the site norm2
|
||||||
|
template<class vobj> inline RealD maxLocalNorm2(const Lattice<vobj> &arg)
|
||||||
|
{
|
||||||
|
typedef typename vobj::tensor_reduced vscalar; //iScalar<iScalar<.... <vPODtype> > >
|
||||||
|
typedef typename vscalar::scalar_object scalar; //iScalar<iScalar<.... <PODtype> > >
|
||||||
|
|
||||||
|
Lattice<vscalar> inner = localNorm2(arg);
|
||||||
|
|
||||||
|
auto grid = arg.Grid();
|
||||||
|
|
||||||
|
RealD max;
|
||||||
|
for(int l=0;l<grid->lSites();l++){
|
||||||
|
Coordinate coor;
|
||||||
|
scalar val;
|
||||||
|
RealD r;
|
||||||
|
grid->LocalIndexToLocalCoor(l,coor);
|
||||||
|
peekLocalSite(val,inner,coor);
|
||||||
|
r=real(TensorRemove(val));
|
||||||
|
if( (l==0) || (r>max)){
|
||||||
|
max=r;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
grid->GlobalMax(max);
|
||||||
|
return max;
|
||||||
|
}
|
||||||
|
|
||||||
// Double inner product
|
// Double inner product
|
||||||
template<class vobj>
|
template<class vobj>
|
||||||
inline ComplexD rankInnerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right)
|
inline ComplexD rankInnerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right)
|
||||||
@ -309,6 +361,7 @@ template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<
|
|||||||
// But easily avoided by using double precision fields
|
// But easily avoided by using double precision fields
|
||||||
///////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////
|
||||||
typedef typename vobj::scalar_object sobj;
|
typedef typename vobj::scalar_object sobj;
|
||||||
|
typedef typename vobj::scalar_object::scalar_type scalar_type;
|
||||||
GridBase *grid = Data.Grid();
|
GridBase *grid = Data.Grid();
|
||||||
assert(grid!=NULL);
|
assert(grid!=NULL);
|
||||||
|
|
||||||
@ -367,20 +420,19 @@ template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sum over nodes.
|
// sum over nodes.
|
||||||
sobj gsum;
|
|
||||||
for(int t=0;t<fd;t++){
|
for(int t=0;t<fd;t++){
|
||||||
int pt = t/ld; // processor plane
|
int pt = t/ld; // processor plane
|
||||||
int lt = t%ld;
|
int lt = t%ld;
|
||||||
if ( pt == grid->_processor_coor[orthogdim] ) {
|
if ( pt == grid->_processor_coor[orthogdim] ) {
|
||||||
gsum=lsSum[lt];
|
result[t]=lsSum[lt];
|
||||||
} else {
|
} else {
|
||||||
gsum=Zero();
|
result[t]=Zero();
|
||||||
}
|
}
|
||||||
|
|
||||||
grid->GlobalSum(gsum);
|
|
||||||
|
|
||||||
result[t]=gsum;
|
|
||||||
}
|
}
|
||||||
|
scalar_type * ptr = (scalar_type *) &result[0];
|
||||||
|
int words = fd*sizeof(sobj)/sizeof(scalar_type);
|
||||||
|
grid->GlobalSumVector(ptr, words);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<class vobj>
|
template<class vobj>
|
||||||
|
@ -42,7 +42,6 @@ void getNumBlocksAndThreads(const Iterator n, const size_t sizeofsobj, Iterator
|
|||||||
std::cout << GridLogDebug << "\twarpSize = " << warpSize << std::endl;
|
std::cout << GridLogDebug << "\twarpSize = " << warpSize << std::endl;
|
||||||
std::cout << GridLogDebug << "\tsharedMemPerBlock = " << sharedMemPerBlock << std::endl;
|
std::cout << GridLogDebug << "\tsharedMemPerBlock = " << sharedMemPerBlock << std::endl;
|
||||||
std::cout << GridLogDebug << "\tmaxThreadsPerBlock = " << maxThreadsPerBlock << std::endl;
|
std::cout << GridLogDebug << "\tmaxThreadsPerBlock = " << maxThreadsPerBlock << std::endl;
|
||||||
std::cout << GridLogDebug << "\tmaxThreadsPerBlock = " << warpSize << std::endl;
|
|
||||||
std::cout << GridLogDebug << "\tmultiProcessorCount = " << multiProcessorCount << std::endl;
|
std::cout << GridLogDebug << "\tmultiProcessorCount = " << multiProcessorCount << std::endl;
|
||||||
|
|
||||||
if (warpSize != WARP_SIZE) {
|
if (warpSize != WARP_SIZE) {
|
||||||
@ -52,6 +51,10 @@ void getNumBlocksAndThreads(const Iterator n, const size_t sizeofsobj, Iterator
|
|||||||
|
|
||||||
// let the number of threads in a block be a multiple of 2, starting from warpSize
|
// let the number of threads in a block be a multiple of 2, starting from warpSize
|
||||||
threads = warpSize;
|
threads = warpSize;
|
||||||
|
if ( threads*sizeofsobj > sharedMemPerBlock ) {
|
||||||
|
std::cout << GridLogError << "The object is too large for the shared memory." << std::endl;
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
while( 2*threads*sizeofsobj < sharedMemPerBlock && 2*threads <= maxThreadsPerBlock ) threads *= 2;
|
while( 2*threads*sizeofsobj < sharedMemPerBlock && 2*threads <= maxThreadsPerBlock ) threads *= 2;
|
||||||
// keep all the streaming multiprocessors busy
|
// keep all the streaming multiprocessors busy
|
||||||
blocks = nextPow2(multiProcessorCount);
|
blocks = nextPow2(multiProcessorCount);
|
||||||
|
@ -85,6 +85,76 @@ template<class vobj> inline void setCheckerboard(Lattice<vobj> &full,const Latti
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template<class vobj> inline void acceleratorPickCheckerboard(int cb,Lattice<vobj> &half,const Lattice<vobj> &full, int checker_dim_half=0)
|
||||||
|
{
|
||||||
|
half.Checkerboard() = cb;
|
||||||
|
autoView(half_v, half, AcceleratorWrite);
|
||||||
|
autoView(full_v, full, AcceleratorRead);
|
||||||
|
Coordinate rdim_full = full.Grid()->_rdimensions;
|
||||||
|
Coordinate rdim_half = half.Grid()->_rdimensions;
|
||||||
|
unsigned long ndim_half = half.Grid()->_ndimension;
|
||||||
|
Coordinate checker_dim_mask_half = half.Grid()->_checker_dim_mask;
|
||||||
|
Coordinate ostride_half = half.Grid()->_ostride;
|
||||||
|
accelerator_for(ss, full.Grid()->oSites(),full.Grid()->Nsimd(),{
|
||||||
|
|
||||||
|
Coordinate coor;
|
||||||
|
int cbos;
|
||||||
|
int linear=0;
|
||||||
|
|
||||||
|
Lexicographic::CoorFromIndex(coor,ss,rdim_full);
|
||||||
|
assert(coor.size()==ndim_half);
|
||||||
|
|
||||||
|
for(int d=0;d<ndim_half;d++){
|
||||||
|
if(checker_dim_mask_half[d]) linear += coor[d];
|
||||||
|
}
|
||||||
|
cbos = (linear&0x1);
|
||||||
|
|
||||||
|
if (cbos==cb) {
|
||||||
|
int ssh=0;
|
||||||
|
for(int d=0;d<ndim_half;d++) {
|
||||||
|
if (d == checker_dim_half) ssh += ostride_half[d] * ((coor[d] / 2) % rdim_half[d]);
|
||||||
|
else ssh += ostride_half[d] * (coor[d] % rdim_half[d]);
|
||||||
|
}
|
||||||
|
coalescedWrite(half_v[ssh],full_v(ss));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
template<class vobj> inline void acceleratorSetCheckerboard(Lattice<vobj> &full,const Lattice<vobj> &half, int checker_dim_half=0)
|
||||||
|
{
|
||||||
|
int cb = half.Checkerboard();
|
||||||
|
autoView(half_v , half, AcceleratorRead);
|
||||||
|
autoView(full_v , full, AcceleratorWrite);
|
||||||
|
Coordinate rdim_full = full.Grid()->_rdimensions;
|
||||||
|
Coordinate rdim_half = half.Grid()->_rdimensions;
|
||||||
|
unsigned long ndim_half = half.Grid()->_ndimension;
|
||||||
|
Coordinate checker_dim_mask_half = half.Grid()->_checker_dim_mask;
|
||||||
|
Coordinate ostride_half = half.Grid()->_ostride;
|
||||||
|
accelerator_for(ss,full.Grid()->oSites(),full.Grid()->Nsimd(),{
|
||||||
|
|
||||||
|
Coordinate coor;
|
||||||
|
int cbos;
|
||||||
|
int linear=0;
|
||||||
|
|
||||||
|
Lexicographic::CoorFromIndex(coor,ss,rdim_full);
|
||||||
|
assert(coor.size()==ndim_half);
|
||||||
|
|
||||||
|
for(int d=0;d<ndim_half;d++){
|
||||||
|
if(checker_dim_mask_half[d]) linear += coor[d];
|
||||||
|
}
|
||||||
|
cbos = (linear&0x1);
|
||||||
|
|
||||||
|
if (cbos==cb) {
|
||||||
|
int ssh=0;
|
||||||
|
for(int d=0;d<ndim_half;d++){
|
||||||
|
if (d == checker_dim_half) ssh += ostride_half[d] * ((coor[d] / 2) % rdim_half[d]);
|
||||||
|
else ssh += ostride_half[d] * (coor[d] % rdim_half[d]);
|
||||||
|
}
|
||||||
|
coalescedWrite(full_v[ss],half_v(ssh));
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// Flexible Type Conversion for internal promotion to double as well as graceful
|
// Flexible Type Conversion for internal promotion to double as well as graceful
|
||||||
// treatment of scalar-compatible types
|
// treatment of scalar-compatible types
|
||||||
@ -97,6 +167,20 @@ accelerator_inline void convertType(ComplexF & out, const std::complex<float> &
|
|||||||
out = in;
|
out = in;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template<typename T>
|
||||||
|
accelerator_inline EnableIf<isGridFundamental<T>> convertType(T & out, const T & in) {
|
||||||
|
out = in;
|
||||||
|
}
|
||||||
|
|
||||||
|
// This would allow for conversions between GridFundamental types, but is not strictly needed as yet
|
||||||
|
/*template<typename T1, typename T2>
|
||||||
|
accelerator_inline typename std::enable_if<isGridFundamental<T1>::value && isGridFundamental<T2>::value>::type
|
||||||
|
// Or to make this very broad, conversions between anything that's not a GridTensor could be allowed
|
||||||
|
//accelerator_inline typename std::enable_if<!isGridTensor<T1>::value && !isGridTensor<T2>::value>::type
|
||||||
|
convertType(T1 & out, const T2 & in) {
|
||||||
|
out = in;
|
||||||
|
}*/
|
||||||
|
|
||||||
#ifdef GRID_SIMT
|
#ifdef GRID_SIMT
|
||||||
accelerator_inline void convertType(vComplexF & out, const ComplexF & in) {
|
accelerator_inline void convertType(vComplexF & out, const ComplexF & in) {
|
||||||
((ComplexF*)&out)[acceleratorSIMTlane(vComplexF::Nsimd())] = in;
|
((ComplexF*)&out)[acceleratorSIMTlane(vComplexF::Nsimd())] = in;
|
||||||
@ -117,23 +201,18 @@ accelerator_inline void convertType(vComplexD2 & out, const vComplexF & in) {
|
|||||||
Optimization::PrecisionChange::StoD(in.v,out._internal[0].v,out._internal[1].v);
|
Optimization::PrecisionChange::StoD(in.v,out._internal[0].v,out._internal[1].v);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename T1,typename T2,int N>
|
template<typename T1,typename T2>
|
||||||
accelerator_inline void convertType(iMatrix<T1,N> & out, const iMatrix<T2,N> & in);
|
accelerator_inline void convertType(iScalar<T1> & out, const iScalar<T2> & in) {
|
||||||
template<typename T1,typename T2,int N>
|
convertType(out._internal,in._internal);
|
||||||
accelerator_inline void convertType(iVector<T1,N> & out, const iVector<T2,N> & in);
|
|
||||||
|
|
||||||
template<typename T1,typename T2, typename std::enable_if<!isGridScalar<T1>::value, T1>::type* = nullptr>
|
|
||||||
accelerator_inline void convertType(T1 & out, const iScalar<T2> & in) {
|
|
||||||
convertType(out,in._internal);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename T1, typename std::enable_if<!isGridScalar<T1>::value, T1>::type* = nullptr>
|
template<typename T1,typename T2>
|
||||||
accelerator_inline void convertType(T1 & out, const iScalar<T1> & in) {
|
accelerator_inline NotEnableIf<isGridScalar<T1>> convertType(T1 & out, const iScalar<T2> & in) {
|
||||||
convertType(out,in._internal);
|
convertType(out,in._internal);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename T1,typename T2>
|
template<typename T1,typename T2>
|
||||||
accelerator_inline void convertType(iScalar<T1> & out, const T2 & in) {
|
accelerator_inline NotEnableIf<isGridScalar<T2>> convertType(iScalar<T1> & out, const T2 & in) {
|
||||||
convertType(out._internal,in);
|
convertType(out._internal,in);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -150,11 +229,6 @@ accelerator_inline void convertType(iVector<T1,N> & out, const iVector<T2,N> & i
|
|||||||
convertType(out._internal[i],in._internal[i]);
|
convertType(out._internal[i],in._internal[i]);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename T, typename std::enable_if<isGridFundamental<T>::value, T>::type* = nullptr>
|
|
||||||
accelerator_inline void convertType(T & out, const T & in) {
|
|
||||||
out = in;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<typename T1,typename T2>
|
template<typename T1,typename T2>
|
||||||
accelerator_inline void convertType(Lattice<T1> & out, const Lattice<T2> & in) {
|
accelerator_inline void convertType(Lattice<T1> & out, const Lattice<T2> & in) {
|
||||||
autoView( out_v , out,AcceleratorWrite);
|
autoView( out_v , out,AcceleratorWrite);
|
||||||
@ -360,15 +434,21 @@ inline void blockSum(Lattice<vobj> &coarseData,const Lattice<vobj> &fineData)
|
|||||||
autoView( coarseData_ , coarseData, AcceleratorWrite);
|
autoView( coarseData_ , coarseData, AcceleratorWrite);
|
||||||
autoView( fineData_ , fineData, AcceleratorRead);
|
autoView( fineData_ , fineData, AcceleratorRead);
|
||||||
|
|
||||||
|
auto coarseData_p = &coarseData_[0];
|
||||||
|
auto fineData_p = &fineData_[0];
|
||||||
|
|
||||||
Coordinate fine_rdimensions = fine->_rdimensions;
|
Coordinate fine_rdimensions = fine->_rdimensions;
|
||||||
Coordinate coarse_rdimensions = coarse->_rdimensions;
|
Coordinate coarse_rdimensions = coarse->_rdimensions;
|
||||||
|
|
||||||
|
vobj zz = Zero();
|
||||||
|
|
||||||
accelerator_for(sc,coarse->oSites(),1,{
|
accelerator_for(sc,coarse->oSites(),1,{
|
||||||
|
|
||||||
// One thread per sub block
|
// One thread per sub block
|
||||||
Coordinate coor_c(_ndimension);
|
Coordinate coor_c(_ndimension);
|
||||||
Lexicographic::CoorFromIndex(coor_c,sc,coarse_rdimensions); // Block coordinate
|
Lexicographic::CoorFromIndex(coor_c,sc,coarse_rdimensions); // Block coordinate
|
||||||
coarseData_[sc]=Zero();
|
|
||||||
|
vobj cd = zz;
|
||||||
|
|
||||||
for(int sb=0;sb<blockVol;sb++){
|
for(int sb=0;sb<blockVol;sb++){
|
||||||
|
|
||||||
@ -379,9 +459,11 @@ inline void blockSum(Lattice<vobj> &coarseData,const Lattice<vobj> &fineData)
|
|||||||
for(int d=0;d<_ndimension;d++) coor_f[d]=coor_c[d]*block_r[d] + coor_b[d];
|
for(int d=0;d<_ndimension;d++) coor_f[d]=coor_c[d]*block_r[d] + coor_b[d];
|
||||||
Lexicographic::IndexFromCoor(coor_f,sf,fine_rdimensions);
|
Lexicographic::IndexFromCoor(coor_f,sf,fine_rdimensions);
|
||||||
|
|
||||||
coarseData_[sc]=coarseData_[sc]+fineData_[sf];
|
cd=cd+fineData_p[sf];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
coarseData_p[sc] = cd;
|
||||||
|
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -67,8 +67,13 @@ public:
|
|||||||
accelerator_inline const vobj & operator()(size_t i) const { return this->_odata[i]; }
|
accelerator_inline const vobj & operator()(size_t i) const { return this->_odata[i]; }
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if 1
|
||||||
|
// accelerator_inline const vobj & operator[](size_t i) const { return this->_odata[i]; };
|
||||||
|
accelerator_inline vobj & operator[](size_t i) const { return this->_odata[i]; };
|
||||||
|
#else
|
||||||
accelerator_inline const vobj & operator[](size_t i) const { return this->_odata[i]; };
|
accelerator_inline const vobj & operator[](size_t i) const { return this->_odata[i]; };
|
||||||
accelerator_inline vobj & operator[](size_t i) { return this->_odata[i]; };
|
accelerator_inline vobj & operator[](size_t i) { return this->_odata[i]; };
|
||||||
|
#endif
|
||||||
|
|
||||||
accelerator_inline uint64_t begin(void) const { return 0;};
|
accelerator_inline uint64_t begin(void) const { return 0;};
|
||||||
accelerator_inline uint64_t end(void) const { return this->_odata_size; };
|
accelerator_inline uint64_t end(void) const { return this->_odata_size; };
|
||||||
|
@ -43,7 +43,7 @@ inline void whereWolf(Lattice<vobj> &ret,const Lattice<iobj> &predicate,Lattice<
|
|||||||
conformable(iftrue,predicate);
|
conformable(iftrue,predicate);
|
||||||
conformable(iftrue,ret);
|
conformable(iftrue,ret);
|
||||||
|
|
||||||
GridBase *grid=iftrue._grid;
|
GridBase *grid=iftrue.Grid();
|
||||||
|
|
||||||
typedef typename vobj::scalar_object scalar_object;
|
typedef typename vobj::scalar_object scalar_object;
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
typedef typename vobj::scalar_type scalar_type;
|
||||||
@ -52,22 +52,23 @@ inline void whereWolf(Lattice<vobj> &ret,const Lattice<iobj> &predicate,Lattice<
|
|||||||
|
|
||||||
const int Nsimd = grid->Nsimd();
|
const int Nsimd = grid->Nsimd();
|
||||||
|
|
||||||
std::vector<Integer> mask(Nsimd);
|
autoView(iftrue_v,iftrue,CpuRead);
|
||||||
std::vector<scalar_object> truevals (Nsimd);
|
autoView(iffalse_v,iffalse,CpuRead);
|
||||||
std::vector<scalar_object> falsevals(Nsimd);
|
autoView(predicate_v,predicate,CpuRead);
|
||||||
|
autoView(ret_v,ret,CpuWrite);
|
||||||
parallel_for(int ss=0;ss<iftrue._grid->oSites(); ss++){
|
Integer NN= grid->oSites();
|
||||||
|
thread_for(ss,NN,{
|
||||||
extract(iftrue._odata[ss] ,truevals);
|
Integer mask;
|
||||||
extract(iffalse._odata[ss] ,falsevals);
|
scalar_object trueval;
|
||||||
extract<vInteger,Integer>(TensorRemove(predicate._odata[ss]),mask);
|
scalar_object falseval;
|
||||||
|
for(int l=0;l<Nsimd;l++){
|
||||||
for(int s=0;s<Nsimd;s++){
|
trueval =extractLane(l,iftrue_v[ss]);
|
||||||
if (mask[s]) falsevals[s]=truevals[s];
|
falseval=extractLane(l,iffalse_v[ss]);
|
||||||
}
|
mask =extractLane(l,predicate_v[ss]);
|
||||||
|
if (mask) falseval=trueval;
|
||||||
merge(ret._odata[ss],falsevals);
|
insertLane(l,ret_v[ss],falseval);
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
template<class vobj,class iobj>
|
template<class vobj,class iobj>
|
||||||
@ -76,9 +77,9 @@ inline Lattice<vobj> whereWolf(const Lattice<iobj> &predicate,Lattice<vobj> &ift
|
|||||||
conformable(iftrue,iffalse);
|
conformable(iftrue,iffalse);
|
||||||
conformable(iftrue,predicate);
|
conformable(iftrue,predicate);
|
||||||
|
|
||||||
Lattice<vobj> ret(iftrue._grid);
|
Lattice<vobj> ret(iftrue.Grid());
|
||||||
|
|
||||||
where(ret,predicate,iftrue,iffalse);
|
whereWolf(ret,predicate,iftrue,iffalse);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
@ -576,6 +576,8 @@ class ScidacReader : public GridLimeReader {
|
|||||||
std::string rec_name(ILDG_BINARY_DATA);
|
std::string rec_name(ILDG_BINARY_DATA);
|
||||||
while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) {
|
while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) {
|
||||||
if ( !strncmp(limeReaderType(LimeR), rec_name.c_str(),strlen(rec_name.c_str()) ) ) {
|
if ( !strncmp(limeReaderType(LimeR), rec_name.c_str(),strlen(rec_name.c_str()) ) ) {
|
||||||
|
// in principle should do the line below, but that breaks backard compatibility with old data
|
||||||
|
// skipPastObjectRecord(std::string(GRID_FIELD_NORM));
|
||||||
skipPastObjectRecord(std::string(SCIDAC_CHECKSUM));
|
skipPastObjectRecord(std::string(SCIDAC_CHECKSUM));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -128,7 +128,7 @@ inline void MachineCharacteristics(FieldMetaData &header)
|
|||||||
std::time_t t = std::time(nullptr);
|
std::time_t t = std::time(nullptr);
|
||||||
std::tm tm_ = *std::localtime(&t);
|
std::tm tm_ = *std::localtime(&t);
|
||||||
std::ostringstream oss;
|
std::ostringstream oss;
|
||||||
// oss << std::put_time(&tm_, "%c %Z");
|
oss << std::put_time(&tm_, "%c %Z");
|
||||||
header.creation_date = oss.str();
|
header.creation_date = oss.str();
|
||||||
header.archive_date = header.creation_date;
|
header.archive_date = header.creation_date;
|
||||||
|
|
||||||
|
@ -205,11 +205,20 @@ public:
|
|||||||
std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
|
std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Preferred interface
|
||||||
|
template<class GaugeStats=PeriodicGaugeStatistics>
|
||||||
|
static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
|
||||||
|
std::string file,
|
||||||
|
std::string ens_label = std::string("DWF"))
|
||||||
|
{
|
||||||
|
writeConfiguration(Umu,file,0,1,ens_label);
|
||||||
|
}
|
||||||
template<class GaugeStats=PeriodicGaugeStatistics>
|
template<class GaugeStats=PeriodicGaugeStatistics>
|
||||||
static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
|
static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
|
||||||
std::string file,
|
std::string file,
|
||||||
int two_row,
|
int two_row,
|
||||||
int bits32)
|
int bits32,
|
||||||
|
std::string ens_label = std::string("DWF"))
|
||||||
{
|
{
|
||||||
typedef vLorentzColourMatrixD vobj;
|
typedef vLorentzColourMatrixD vobj;
|
||||||
typedef typename vobj::scalar_object sobj;
|
typedef typename vobj::scalar_object sobj;
|
||||||
@ -219,8 +228,8 @@ public:
|
|||||||
// Following should become arguments
|
// Following should become arguments
|
||||||
///////////////////////////////////////////
|
///////////////////////////////////////////
|
||||||
header.sequence_number = 1;
|
header.sequence_number = 1;
|
||||||
header.ensemble_id = "UKQCD";
|
header.ensemble_id = std::string("UKQCD");
|
||||||
header.ensemble_label = "DWF";
|
header.ensemble_label = ens_label;
|
||||||
|
|
||||||
typedef LorentzColourMatrixD fobj3D;
|
typedef LorentzColourMatrixD fobj3D;
|
||||||
typedef LorentzColour2x3D fobj2D;
|
typedef LorentzColour2x3D fobj2D;
|
||||||
|
@ -41,7 +41,7 @@ class Action
|
|||||||
public:
|
public:
|
||||||
bool is_smeared = false;
|
bool is_smeared = false;
|
||||||
// Heatbath?
|
// Heatbath?
|
||||||
virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
|
virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
|
||||||
virtual RealD S(const GaugeField& U) = 0; // evaluate the action
|
virtual RealD S(const GaugeField& U) = 0; // evaluate the action
|
||||||
virtual void deriv(const GaugeField& U, GaugeField& dSdU) = 0; // evaluate the action derivative
|
virtual void deriv(const GaugeField& U, GaugeField& dSdU) = 0; // evaluate the action derivative
|
||||||
virtual std::string action_name() = 0; // return the action name
|
virtual std::string action_name() = 0; // return the action name
|
||||||
|
@ -115,9 +115,9 @@ typedef WilsonFermion<WilsonImplR> WilsonFermionR;
|
|||||||
typedef WilsonFermion<WilsonImplF> WilsonFermionF;
|
typedef WilsonFermion<WilsonImplF> WilsonFermionF;
|
||||||
typedef WilsonFermion<WilsonImplD> WilsonFermionD;
|
typedef WilsonFermion<WilsonImplD> WilsonFermionD;
|
||||||
|
|
||||||
typedef WilsonFermion<WilsonImplRL> WilsonFermionRL;
|
//typedef WilsonFermion<WilsonImplRL> WilsonFermionRL;
|
||||||
typedef WilsonFermion<WilsonImplFH> WilsonFermionFH;
|
//typedef WilsonFermion<WilsonImplFH> WilsonFermionFH;
|
||||||
typedef WilsonFermion<WilsonImplDF> WilsonFermionDF;
|
//typedef WilsonFermion<WilsonImplDF> WilsonFermionDF;
|
||||||
|
|
||||||
typedef WilsonFermion<WilsonAdjImplR> WilsonAdjFermionR;
|
typedef WilsonFermion<WilsonAdjImplR> WilsonAdjFermionR;
|
||||||
typedef WilsonFermion<WilsonAdjImplF> WilsonAdjFermionF;
|
typedef WilsonFermion<WilsonAdjImplF> WilsonAdjFermionF;
|
||||||
@ -158,41 +158,41 @@ typedef DomainWallFermion<WilsonImplR> DomainWallFermionR;
|
|||||||
typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;
|
typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;
|
||||||
typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
|
typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
|
||||||
|
|
||||||
typedef DomainWallFermion<WilsonImplRL> DomainWallFermionRL;
|
//typedef DomainWallFermion<WilsonImplRL> DomainWallFermionRL;
|
||||||
typedef DomainWallFermion<WilsonImplFH> DomainWallFermionFH;
|
//typedef DomainWallFermion<WilsonImplFH> DomainWallFermionFH;
|
||||||
typedef DomainWallFermion<WilsonImplDF> DomainWallFermionDF;
|
//typedef DomainWallFermion<WilsonImplDF> DomainWallFermionDF;
|
||||||
|
|
||||||
typedef DomainWallEOFAFermion<WilsonImplR> DomainWallEOFAFermionR;
|
typedef DomainWallEOFAFermion<WilsonImplR> DomainWallEOFAFermionR;
|
||||||
typedef DomainWallEOFAFermion<WilsonImplF> DomainWallEOFAFermionF;
|
typedef DomainWallEOFAFermion<WilsonImplF> DomainWallEOFAFermionF;
|
||||||
typedef DomainWallEOFAFermion<WilsonImplD> DomainWallEOFAFermionD;
|
typedef DomainWallEOFAFermion<WilsonImplD> DomainWallEOFAFermionD;
|
||||||
|
|
||||||
typedef DomainWallEOFAFermion<WilsonImplRL> DomainWallEOFAFermionRL;
|
//typedef DomainWallEOFAFermion<WilsonImplRL> DomainWallEOFAFermionRL;
|
||||||
typedef DomainWallEOFAFermion<WilsonImplFH> DomainWallEOFAFermionFH;
|
//typedef DomainWallEOFAFermion<WilsonImplFH> DomainWallEOFAFermionFH;
|
||||||
typedef DomainWallEOFAFermion<WilsonImplDF> DomainWallEOFAFermionDF;
|
//typedef DomainWallEOFAFermion<WilsonImplDF> DomainWallEOFAFermionDF;
|
||||||
|
|
||||||
typedef MobiusFermion<WilsonImplR> MobiusFermionR;
|
typedef MobiusFermion<WilsonImplR> MobiusFermionR;
|
||||||
typedef MobiusFermion<WilsonImplF> MobiusFermionF;
|
typedef MobiusFermion<WilsonImplF> MobiusFermionF;
|
||||||
typedef MobiusFermion<WilsonImplD> MobiusFermionD;
|
typedef MobiusFermion<WilsonImplD> MobiusFermionD;
|
||||||
|
|
||||||
typedef MobiusFermion<WilsonImplRL> MobiusFermionRL;
|
//typedef MobiusFermion<WilsonImplRL> MobiusFermionRL;
|
||||||
typedef MobiusFermion<WilsonImplFH> MobiusFermionFH;
|
//typedef MobiusFermion<WilsonImplFH> MobiusFermionFH;
|
||||||
typedef MobiusFermion<WilsonImplDF> MobiusFermionDF;
|
//typedef MobiusFermion<WilsonImplDF> MobiusFermionDF;
|
||||||
|
|
||||||
typedef MobiusEOFAFermion<WilsonImplR> MobiusEOFAFermionR;
|
typedef MobiusEOFAFermion<WilsonImplR> MobiusEOFAFermionR;
|
||||||
typedef MobiusEOFAFermion<WilsonImplF> MobiusEOFAFermionF;
|
typedef MobiusEOFAFermion<WilsonImplF> MobiusEOFAFermionF;
|
||||||
typedef MobiusEOFAFermion<WilsonImplD> MobiusEOFAFermionD;
|
typedef MobiusEOFAFermion<WilsonImplD> MobiusEOFAFermionD;
|
||||||
|
|
||||||
typedef MobiusEOFAFermion<WilsonImplRL> MobiusEOFAFermionRL;
|
//typedef MobiusEOFAFermion<WilsonImplRL> MobiusEOFAFermionRL;
|
||||||
typedef MobiusEOFAFermion<WilsonImplFH> MobiusEOFAFermionFH;
|
//typedef MobiusEOFAFermion<WilsonImplFH> MobiusEOFAFermionFH;
|
||||||
typedef MobiusEOFAFermion<WilsonImplDF> MobiusEOFAFermionDF;
|
//typedef MobiusEOFAFermion<WilsonImplDF> MobiusEOFAFermionDF;
|
||||||
|
|
||||||
typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
|
typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
|
||||||
typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
|
typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
|
||||||
typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
|
typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
|
||||||
|
|
||||||
typedef ZMobiusFermion<ZWilsonImplRL> ZMobiusFermionRL;
|
//typedef ZMobiusFermion<ZWilsonImplRL> ZMobiusFermionRL;
|
||||||
typedef ZMobiusFermion<ZWilsonImplFH> ZMobiusFermionFH;
|
//typedef ZMobiusFermion<ZWilsonImplFH> ZMobiusFermionFH;
|
||||||
typedef ZMobiusFermion<ZWilsonImplDF> ZMobiusFermionDF;
|
//typedef ZMobiusFermion<ZWilsonImplDF> ZMobiusFermionDF;
|
||||||
|
|
||||||
// Ls vectorised
|
// Ls vectorised
|
||||||
typedef ScaledShamirFermion<WilsonImplR> ScaledShamirFermionR;
|
typedef ScaledShamirFermion<WilsonImplR> ScaledShamirFermionR;
|
||||||
@ -235,49 +235,49 @@ typedef WilsonFermion<GparityWilsonImplR> GparityWilsonFermionR;
|
|||||||
typedef WilsonFermion<GparityWilsonImplF> GparityWilsonFermionF;
|
typedef WilsonFermion<GparityWilsonImplF> GparityWilsonFermionF;
|
||||||
typedef WilsonFermion<GparityWilsonImplD> GparityWilsonFermionD;
|
typedef WilsonFermion<GparityWilsonImplD> GparityWilsonFermionD;
|
||||||
|
|
||||||
typedef WilsonFermion<GparityWilsonImplRL> GparityWilsonFermionRL;
|
//typedef WilsonFermion<GparityWilsonImplRL> GparityWilsonFermionRL;
|
||||||
typedef WilsonFermion<GparityWilsonImplFH> GparityWilsonFermionFH;
|
//typedef WilsonFermion<GparityWilsonImplFH> GparityWilsonFermionFH;
|
||||||
typedef WilsonFermion<GparityWilsonImplDF> GparityWilsonFermionDF;
|
//typedef WilsonFermion<GparityWilsonImplDF> GparityWilsonFermionDF;
|
||||||
|
|
||||||
typedef DomainWallFermion<GparityWilsonImplR> GparityDomainWallFermionR;
|
typedef DomainWallFermion<GparityWilsonImplR> GparityDomainWallFermionR;
|
||||||
typedef DomainWallFermion<GparityWilsonImplF> GparityDomainWallFermionF;
|
typedef DomainWallFermion<GparityWilsonImplF> GparityDomainWallFermionF;
|
||||||
typedef DomainWallFermion<GparityWilsonImplD> GparityDomainWallFermionD;
|
typedef DomainWallFermion<GparityWilsonImplD> GparityDomainWallFermionD;
|
||||||
|
|
||||||
typedef DomainWallFermion<GparityWilsonImplRL> GparityDomainWallFermionRL;
|
//typedef DomainWallFermion<GparityWilsonImplRL> GparityDomainWallFermionRL;
|
||||||
typedef DomainWallFermion<GparityWilsonImplFH> GparityDomainWallFermionFH;
|
//typedef DomainWallFermion<GparityWilsonImplFH> GparityDomainWallFermionFH;
|
||||||
typedef DomainWallFermion<GparityWilsonImplDF> GparityDomainWallFermionDF;
|
//typedef DomainWallFermion<GparityWilsonImplDF> GparityDomainWallFermionDF;
|
||||||
|
|
||||||
typedef DomainWallEOFAFermion<GparityWilsonImplR> GparityDomainWallEOFAFermionR;
|
typedef DomainWallEOFAFermion<GparityWilsonImplR> GparityDomainWallEOFAFermionR;
|
||||||
typedef DomainWallEOFAFermion<GparityWilsonImplF> GparityDomainWallEOFAFermionF;
|
typedef DomainWallEOFAFermion<GparityWilsonImplF> GparityDomainWallEOFAFermionF;
|
||||||
typedef DomainWallEOFAFermion<GparityWilsonImplD> GparityDomainWallEOFAFermionD;
|
typedef DomainWallEOFAFermion<GparityWilsonImplD> GparityDomainWallEOFAFermionD;
|
||||||
|
|
||||||
typedef DomainWallEOFAFermion<GparityWilsonImplRL> GparityDomainWallEOFAFermionRL;
|
//typedef DomainWallEOFAFermion<GparityWilsonImplRL> GparityDomainWallEOFAFermionRL;
|
||||||
typedef DomainWallEOFAFermion<GparityWilsonImplFH> GparityDomainWallEOFAFermionFH;
|
//typedef DomainWallEOFAFermion<GparityWilsonImplFH> GparityDomainWallEOFAFermionFH;
|
||||||
typedef DomainWallEOFAFermion<GparityWilsonImplDF> GparityDomainWallEOFAFermionDF;
|
//typedef DomainWallEOFAFermion<GparityWilsonImplDF> GparityDomainWallEOFAFermionDF;
|
||||||
|
|
||||||
typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionR;
|
typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionR;
|
||||||
typedef WilsonTMFermion<GparityWilsonImplF> GparityWilsonTMFermionF;
|
typedef WilsonTMFermion<GparityWilsonImplF> GparityWilsonTMFermionF;
|
||||||
typedef WilsonTMFermion<GparityWilsonImplD> GparityWilsonTMFermionD;
|
typedef WilsonTMFermion<GparityWilsonImplD> GparityWilsonTMFermionD;
|
||||||
|
|
||||||
typedef WilsonTMFermion<GparityWilsonImplRL> GparityWilsonTMFermionRL;
|
//typedef WilsonTMFermion<GparityWilsonImplRL> GparityWilsonTMFermionRL;
|
||||||
typedef WilsonTMFermion<GparityWilsonImplFH> GparityWilsonTMFermionFH;
|
//typedef WilsonTMFermion<GparityWilsonImplFH> GparityWilsonTMFermionFH;
|
||||||
typedef WilsonTMFermion<GparityWilsonImplDF> GparityWilsonTMFermionDF;
|
//typedef WilsonTMFermion<GparityWilsonImplDF> GparityWilsonTMFermionDF;
|
||||||
|
|
||||||
typedef MobiusFermion<GparityWilsonImplR> GparityMobiusFermionR;
|
typedef MobiusFermion<GparityWilsonImplR> GparityMobiusFermionR;
|
||||||
typedef MobiusFermion<GparityWilsonImplF> GparityMobiusFermionF;
|
typedef MobiusFermion<GparityWilsonImplF> GparityMobiusFermionF;
|
||||||
typedef MobiusFermion<GparityWilsonImplD> GparityMobiusFermionD;
|
typedef MobiusFermion<GparityWilsonImplD> GparityMobiusFermionD;
|
||||||
|
|
||||||
typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
|
//typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
|
||||||
typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
|
//typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
|
||||||
typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
|
//typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
|
||||||
|
|
||||||
typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusEOFAFermionR;
|
typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusEOFAFermionR;
|
||||||
typedef MobiusEOFAFermion<GparityWilsonImplF> GparityMobiusEOFAFermionF;
|
typedef MobiusEOFAFermion<GparityWilsonImplF> GparityMobiusEOFAFermionF;
|
||||||
typedef MobiusEOFAFermion<GparityWilsonImplD> GparityMobiusEOFAFermionD;
|
typedef MobiusEOFAFermion<GparityWilsonImplD> GparityMobiusEOFAFermionD;
|
||||||
|
|
||||||
typedef MobiusEOFAFermion<GparityWilsonImplRL> GparityMobiusEOFAFermionRL;
|
//typedef MobiusEOFAFermion<GparityWilsonImplRL> GparityMobiusEOFAFermionRL;
|
||||||
typedef MobiusEOFAFermion<GparityWilsonImplFH> GparityMobiusEOFAFermionFH;
|
//typedef MobiusEOFAFermion<GparityWilsonImplFH> GparityMobiusEOFAFermionFH;
|
||||||
typedef MobiusEOFAFermion<GparityWilsonImplDF> GparityMobiusEOFAFermionDF;
|
//typedef MobiusEOFAFermion<GparityWilsonImplDF> GparityMobiusEOFAFermionDF;
|
||||||
|
|
||||||
typedef ImprovedStaggeredFermion<StaggeredImplR> ImprovedStaggeredFermionR;
|
typedef ImprovedStaggeredFermion<StaggeredImplR> ImprovedStaggeredFermionR;
|
||||||
typedef ImprovedStaggeredFermion<StaggeredImplF> ImprovedStaggeredFermionF;
|
typedef ImprovedStaggeredFermion<StaggeredImplF> ImprovedStaggeredFermionF;
|
||||||
@ -291,12 +291,6 @@ typedef ImprovedStaggeredFermion5D<StaggeredImplR> ImprovedStaggeredFermion5DR;
|
|||||||
typedef ImprovedStaggeredFermion5D<StaggeredImplF> ImprovedStaggeredFermion5DF;
|
typedef ImprovedStaggeredFermion5D<StaggeredImplF> ImprovedStaggeredFermion5DF;
|
||||||
typedef ImprovedStaggeredFermion5D<StaggeredImplD> ImprovedStaggeredFermion5DD;
|
typedef ImprovedStaggeredFermion5D<StaggeredImplD> ImprovedStaggeredFermion5DD;
|
||||||
|
|
||||||
#ifndef GRID_CUDA
|
|
||||||
typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplR> ImprovedStaggeredFermionVec5dR;
|
|
||||||
typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplF> ImprovedStaggeredFermionVec5dF;
|
|
||||||
typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplD> ImprovedStaggeredFermionVec5dD;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
NAMESPACE_END(Grid);
|
||||||
|
|
||||||
////////////////////
|
////////////////////
|
||||||
|
@ -153,8 +153,8 @@ public:
|
|||||||
typedef typename Impl::StencilImpl StencilImpl; \
|
typedef typename Impl::StencilImpl StencilImpl; \
|
||||||
typedef typename Impl::ImplParams ImplParams; \
|
typedef typename Impl::ImplParams ImplParams; \
|
||||||
typedef typename Impl::StencilImpl::View_type StencilView; \
|
typedef typename Impl::StencilImpl::View_type StencilView; \
|
||||||
typedef typename ViewMap<FermionField>::Type FermionFieldView; \
|
typedef const typename ViewMap<FermionField>::Type FermionFieldView; \
|
||||||
typedef typename ViewMap<DoubledGaugeField>::Type DoubledGaugeFieldView;
|
typedef const typename ViewMap<DoubledGaugeField>::Type DoubledGaugeFieldView;
|
||||||
|
|
||||||
#define INHERIT_IMPL_TYPES(Base) \
|
#define INHERIT_IMPL_TYPES(Base) \
|
||||||
INHERIT_GIMPL_TYPES(Base) \
|
INHERIT_GIMPL_TYPES(Base) \
|
||||||
@ -183,7 +183,8 @@ NAMESPACE_CHECK(ImplStaggered);
|
|||||||
/////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////
|
||||||
// Single flavour one component spinors with colour index. 5d vec
|
// Single flavour one component spinors with colour index. 5d vec
|
||||||
/////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////
|
||||||
#include <Grid/qcd/action/fermion/StaggeredVec5dImpl.h>
|
// Deprecate Vec5d
|
||||||
NAMESPACE_CHECK(ImplStaggered5dVec);
|
//#include <Grid/qcd/action/fermion/StaggeredVec5dImpl.h>
|
||||||
|
//NAMESPACE_CHECK(ImplStaggered5dVec);
|
||||||
|
|
||||||
|
|
||||||
|
@ -327,8 +327,8 @@ typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffReal> Gparit
|
|||||||
typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffReal> GparityWilsonImplF; // Float
|
typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffReal> GparityWilsonImplF; // Float
|
||||||
typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffReal> GparityWilsonImplD; // Double
|
typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffReal> GparityWilsonImplD; // Double
|
||||||
|
|
||||||
typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplRL; // Real.. whichever prec
|
//typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplRL; // Real.. whichever prec
|
||||||
typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplFH; // Float
|
//typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplFH; // Float
|
||||||
typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplDF; // Double
|
//typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplDF; // Double
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
NAMESPACE_END(Grid);
|
||||||
|
@ -85,7 +85,7 @@ class MADWF
|
|||||||
maxiter =_maxiter;
|
maxiter =_maxiter;
|
||||||
};
|
};
|
||||||
|
|
||||||
void operator() (const FermionFieldo &src4,FermionFieldo &sol5)
|
void operator() (const FermionFieldo &src,FermionFieldo &sol5)
|
||||||
{
|
{
|
||||||
std::cout << GridLogMessage<< " ************************************************" << std::endl;
|
std::cout << GridLogMessage<< " ************************************************" << std::endl;
|
||||||
std::cout << GridLogMessage<< " MADWF-like algorithm " << std::endl;
|
std::cout << GridLogMessage<< " MADWF-like algorithm " << std::endl;
|
||||||
@ -114,8 +114,16 @@ class MADWF
|
|||||||
///////////////////////////////////////
|
///////////////////////////////////////
|
||||||
//Import source, include Dminus factors
|
//Import source, include Dminus factors
|
||||||
///////////////////////////////////////
|
///////////////////////////////////////
|
||||||
Mato.ImportPhysicalFermionSource(src4,b);
|
GridBase *src_grid = src.Grid();
|
||||||
std::cout << GridLogMessage << " src4 " <<norm2(src4)<<std::endl;
|
|
||||||
|
assert( (src_grid == Mato.GaugeGrid()) || (src_grid == Mato.FermionGrid()));
|
||||||
|
|
||||||
|
if ( src_grid == Mato.GaugeGrid() ) {
|
||||||
|
Mato.ImportPhysicalFermionSource(src,b);
|
||||||
|
} else {
|
||||||
|
b=src;
|
||||||
|
}
|
||||||
|
std::cout << GridLogMessage << " src " <<norm2(src)<<std::endl;
|
||||||
std::cout << GridLogMessage << " b " <<norm2(b)<<std::endl;
|
std::cout << GridLogMessage << " b " <<norm2(b)<<std::endl;
|
||||||
|
|
||||||
defect = b;
|
defect = b;
|
||||||
|
@ -72,19 +72,23 @@ public:
|
|||||||
|
|
||||||
StaggeredImpl(const ImplParams &p = ImplParams()) : Params(p){};
|
StaggeredImpl(const ImplParams &p = ImplParams()) : Params(p){};
|
||||||
|
|
||||||
static accelerator_inline void multLink(SiteSpinor &phi,
|
template<class _Spinor>
|
||||||
|
static accelerator_inline void multLink(_Spinor &phi,
|
||||||
const SiteDoubledGaugeField &U,
|
const SiteDoubledGaugeField &U,
|
||||||
const SiteSpinor &chi,
|
const _Spinor &chi,
|
||||||
int mu)
|
int mu)
|
||||||
{
|
{
|
||||||
mult(&phi(), &U(mu), &chi());
|
auto UU = coalescedRead(U(mu));
|
||||||
|
mult(&phi(), &UU, &chi());
|
||||||
}
|
}
|
||||||
static accelerator_inline void multLinkAdd(SiteSpinor &phi,
|
template<class _Spinor>
|
||||||
|
static accelerator_inline void multLinkAdd(_Spinor &phi,
|
||||||
const SiteDoubledGaugeField &U,
|
const SiteDoubledGaugeField &U,
|
||||||
const SiteSpinor &chi,
|
const _Spinor &chi,
|
||||||
int mu)
|
int mu)
|
||||||
{
|
{
|
||||||
mac(&phi(), &U(mu), &chi());
|
auto UU = coalescedRead(U(mu));
|
||||||
|
mac(&phi(), &UU, &chi());
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class ref>
|
template <class ref>
|
||||||
|
@ -245,7 +245,7 @@ public:
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
protected:
|
||||||
// here fixing the 4 dimensions, make it more general?
|
// here fixing the 4 dimensions, make it more general?
|
||||||
|
|
||||||
RealD csw_r; // Clover coefficient - spatial
|
RealD csw_r; // Clover coefficient - spatial
|
||||||
|
@ -61,18 +61,19 @@ public:
|
|||||||
typedef typename SiteHalfSpinor::vector_type vComplexHigh;
|
typedef typename SiteHalfSpinor::vector_type vComplexHigh;
|
||||||
constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
|
constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
|
||||||
|
|
||||||
accelerator_inline int CommDatumSize(void) {
|
accelerator_inline int CommDatumSize(void) const {
|
||||||
return sizeof(SiteHalfCommSpinor);
|
return sizeof(SiteHalfCommSpinor);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
/* Compress includes precision change if mpi data is not same */
|
/* Compress includes precision change if mpi data is not same */
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
template<class _SiteHalfSpinor, class _SiteSpinor>
|
accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const {
|
||||||
accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) {
|
typedef decltype(coalescedRead(buf)) sobj;
|
||||||
_SiteHalfSpinor tmp;
|
sobj sp;
|
||||||
projector::Proj(tmp,in,mu,dag);
|
auto sin = coalescedRead(in);
|
||||||
vstream(buf[o],tmp);
|
projector::Proj(sp,sin,mu,dag);
|
||||||
|
coalescedWrite(buf,sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
@ -81,19 +82,24 @@ public:
|
|||||||
accelerator_inline void Exchange(SiteHalfSpinor *mp,
|
accelerator_inline void Exchange(SiteHalfSpinor *mp,
|
||||||
const SiteHalfSpinor * __restrict__ vp0,
|
const SiteHalfSpinor * __restrict__ vp0,
|
||||||
const SiteHalfSpinor * __restrict__ vp1,
|
const SiteHalfSpinor * __restrict__ vp1,
|
||||||
Integer type,Integer o){
|
Integer type,Integer o) const {
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
exchangeSIMT(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type);
|
||||||
|
#else
|
||||||
SiteHalfSpinor tmp1;
|
SiteHalfSpinor tmp1;
|
||||||
SiteHalfSpinor tmp2;
|
SiteHalfSpinor tmp2;
|
||||||
exchange(tmp1,tmp2,vp0[o],vp1[o],type);
|
exchange(tmp1,tmp2,vp0[o],vp1[o],type);
|
||||||
vstream(mp[2*o ],tmp1);
|
vstream(mp[2*o ],tmp1);
|
||||||
vstream(mp[2*o+1],tmp2);
|
vstream(mp[2*o+1],tmp2);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
/* Have a decompression step if mpi data is not same */
|
/* Have a decompression step if mpi data is not same */
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
accelerator_inline void Decompress(SiteHalfSpinor * __restrict__ out,
|
accelerator_inline void Decompress(SiteHalfSpinor * __restrict__ out,
|
||||||
SiteHalfSpinor * __restrict__ in, Integer o) {
|
SiteHalfSpinor * __restrict__ in, Integer o) const {
|
||||||
assert(0);
|
assert(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -103,8 +109,30 @@ public:
|
|||||||
accelerator_inline void CompressExchange(SiteHalfSpinor * __restrict__ out0,
|
accelerator_inline void CompressExchange(SiteHalfSpinor * __restrict__ out0,
|
||||||
SiteHalfSpinor * __restrict__ out1,
|
SiteHalfSpinor * __restrict__ out1,
|
||||||
const SiteSpinor * __restrict__ in,
|
const SiteSpinor * __restrict__ in,
|
||||||
Integer j,Integer k, Integer m,Integer type)
|
Integer j,Integer k, Integer m,Integer type) const
|
||||||
{
|
{
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
typedef SiteSpinor vobj;
|
||||||
|
typedef SiteHalfSpinor hvobj;
|
||||||
|
typedef decltype(coalescedRead(*in)) sobj;
|
||||||
|
typedef decltype(coalescedRead(*out0)) hsobj;
|
||||||
|
|
||||||
|
unsigned int Nsimd = vobj::Nsimd();
|
||||||
|
unsigned int mask = Nsimd >> (type + 1);
|
||||||
|
int lane = acceleratorSIMTlane(Nsimd);
|
||||||
|
int j0 = lane &(~mask); // inner coor zero
|
||||||
|
int j1 = lane |(mask) ; // inner coor one
|
||||||
|
const vobj *vp0 = &in[k];
|
||||||
|
const vobj *vp1 = &in[m];
|
||||||
|
const vobj *vp = (lane&mask) ? vp1:vp0;
|
||||||
|
auto sa = coalescedRead(*vp,j0);
|
||||||
|
auto sb = coalescedRead(*vp,j1);
|
||||||
|
hsobj psa, psb;
|
||||||
|
projector::Proj(psa,sa,mu,dag);
|
||||||
|
projector::Proj(psb,sb,mu,dag);
|
||||||
|
coalescedWrite(out0[j],psa);
|
||||||
|
coalescedWrite(out1[j],psb);
|
||||||
|
#else
|
||||||
SiteHalfSpinor temp1, temp2;
|
SiteHalfSpinor temp1, temp2;
|
||||||
SiteHalfSpinor temp3, temp4;
|
SiteHalfSpinor temp3, temp4;
|
||||||
projector::Proj(temp1,in[k],mu,dag);
|
projector::Proj(temp1,in[k],mu,dag);
|
||||||
@ -112,15 +140,17 @@ public:
|
|||||||
exchange(temp3,temp4,temp1,temp2,type);
|
exchange(temp3,temp4,temp1,temp2,type);
|
||||||
vstream(out0[j],temp3);
|
vstream(out0[j],temp3);
|
||||||
vstream(out1[j],temp4);
|
vstream(out1[j],temp4);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
/* Pass the info to the stencil */
|
/* Pass the info to the stencil */
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
accelerator_inline bool DecompressionStep(void) { return false; }
|
accelerator_inline bool DecompressionStep(void) const { return false; }
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#if 0
|
||||||
template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
|
template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
|
||||||
class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector,
|
class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector,
|
||||||
typename std::enable_if<!std::is_same<_HCspinor,_Hspinor>::value>::type >
|
typename std::enable_if<!std::is_same<_HCspinor,_Hspinor>::value>::type >
|
||||||
@ -142,20 +172,30 @@ public:
|
|||||||
typedef typename SiteHalfSpinor::vector_type vComplexHigh;
|
typedef typename SiteHalfSpinor::vector_type vComplexHigh;
|
||||||
constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
|
constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
|
||||||
|
|
||||||
accelerator_inline int CommDatumSize(void) {
|
accelerator_inline int CommDatumSize(void) const {
|
||||||
return sizeof(SiteHalfCommSpinor);
|
return sizeof(SiteHalfCommSpinor);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
/* Compress includes precision change if mpi data is not same */
|
/* Compress includes precision change if mpi data is not same */
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
template<class _SiteHalfSpinor, class _SiteSpinor>
|
accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const {
|
||||||
accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) {
|
SiteHalfSpinor hsp;
|
||||||
_SiteHalfSpinor hsp;
|
|
||||||
SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf;
|
SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf;
|
||||||
projector::Proj(hsp,in,mu,dag);
|
projector::Proj(hsp,in,mu,dag);
|
||||||
precisionChange((vComplexLow *)&hbuf[o],(vComplexHigh *)&hsp,Nw);
|
precisionChange((vComplexLow *)&hbuf[o],(vComplexHigh *)&hsp,Nw);
|
||||||
}
|
}
|
||||||
|
accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const {
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
typedef decltype(coalescedRead(buf)) sobj;
|
||||||
|
sobj sp;
|
||||||
|
auto sin = coalescedRead(in);
|
||||||
|
projector::Proj(sp,sin,mu,dag);
|
||||||
|
coalescedWrite(buf,sp);
|
||||||
|
#else
|
||||||
|
projector::Proj(buf,in,mu,dag);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
/* Exchange includes precision change if mpi data is not same */
|
/* Exchange includes precision change if mpi data is not same */
|
||||||
@ -163,7 +203,7 @@ public:
|
|||||||
accelerator_inline void Exchange(SiteHalfSpinor *mp,
|
accelerator_inline void Exchange(SiteHalfSpinor *mp,
|
||||||
SiteHalfSpinor *vp0,
|
SiteHalfSpinor *vp0,
|
||||||
SiteHalfSpinor *vp1,
|
SiteHalfSpinor *vp1,
|
||||||
Integer type,Integer o){
|
Integer type,Integer o) const {
|
||||||
SiteHalfSpinor vt0,vt1;
|
SiteHalfSpinor vt0,vt1;
|
||||||
SiteHalfCommSpinor *vpp0 = (SiteHalfCommSpinor *)vp0;
|
SiteHalfCommSpinor *vpp0 = (SiteHalfCommSpinor *)vp0;
|
||||||
SiteHalfCommSpinor *vpp1 = (SiteHalfCommSpinor *)vp1;
|
SiteHalfCommSpinor *vpp1 = (SiteHalfCommSpinor *)vp1;
|
||||||
@ -175,7 +215,7 @@ public:
|
|||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
/* Have a decompression step if mpi data is not same */
|
/* Have a decompression step if mpi data is not same */
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o){
|
accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o) const {
|
||||||
SiteHalfCommSpinor *hin=(SiteHalfCommSpinor *)in;
|
SiteHalfCommSpinor *hin=(SiteHalfCommSpinor *)in;
|
||||||
precisionChange((vComplexHigh *)&out[o],(vComplexLow *)&hin[o],Nw);
|
precisionChange((vComplexHigh *)&out[o],(vComplexLow *)&hin[o],Nw);
|
||||||
}
|
}
|
||||||
@ -186,7 +226,7 @@ public:
|
|||||||
accelerator_inline void CompressExchange(SiteHalfSpinor *out0,
|
accelerator_inline void CompressExchange(SiteHalfSpinor *out0,
|
||||||
SiteHalfSpinor *out1,
|
SiteHalfSpinor *out1,
|
||||||
const SiteSpinor *in,
|
const SiteSpinor *in,
|
||||||
Integer j,Integer k, Integer m,Integer type){
|
Integer j,Integer k, Integer m,Integer type) const {
|
||||||
SiteHalfSpinor temp1, temp2,temp3,temp4;
|
SiteHalfSpinor temp1, temp2,temp3,temp4;
|
||||||
SiteHalfCommSpinor *hout0 = (SiteHalfCommSpinor *)out0;
|
SiteHalfCommSpinor *hout0 = (SiteHalfCommSpinor *)out0;
|
||||||
SiteHalfCommSpinor *hout1 = (SiteHalfCommSpinor *)out1;
|
SiteHalfCommSpinor *hout1 = (SiteHalfCommSpinor *)out1;
|
||||||
@ -200,9 +240,10 @@ public:
|
|||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
/* Pass the info to the stencil */
|
/* Pass the info to the stencil */
|
||||||
/*****************************************************/
|
/*****************************************************/
|
||||||
accelerator_inline bool DecompressionStep(void) { return true; }
|
accelerator_inline bool DecompressionStep(void) const { return true; }
|
||||||
|
|
||||||
};
|
};
|
||||||
|
#endif
|
||||||
|
|
||||||
#define DECLARE_PROJ(Projector,Compressor,spProj) \
|
#define DECLARE_PROJ(Projector,Compressor,spProj) \
|
||||||
class Projector { \
|
class Projector { \
|
||||||
@ -253,33 +294,8 @@ public:
|
|||||||
typedef typename Base::View_type View_type;
|
typedef typename Base::View_type View_type;
|
||||||
typedef typename Base::StencilVector StencilVector;
|
typedef typename Base::StencilVector StencilVector;
|
||||||
|
|
||||||
double timer0;
|
void ZeroCountersi(void) { }
|
||||||
double timer1;
|
void Reporti(int calls) { }
|
||||||
double timer2;
|
|
||||||
double timer3;
|
|
||||||
double timer4;
|
|
||||||
double timer5;
|
|
||||||
double timer6;
|
|
||||||
uint64_t callsi;
|
|
||||||
void ZeroCountersi(void)
|
|
||||||
{
|
|
||||||
timer0=0;
|
|
||||||
timer1=0;
|
|
||||||
timer2=0;
|
|
||||||
timer3=0;
|
|
||||||
timer4=0;
|
|
||||||
timer5=0;
|
|
||||||
timer6=0;
|
|
||||||
callsi=0;
|
|
||||||
}
|
|
||||||
void Reporti(int calls)
|
|
||||||
{
|
|
||||||
if ( timer0 ) std::cout << GridLogMessage << " timer0 (HaloGatherOpt) " <<timer0/calls <<std::endl;
|
|
||||||
if ( timer1 ) std::cout << GridLogMessage << " timer1 (Communicate) " <<timer1/calls <<std::endl;
|
|
||||||
if ( timer2 ) std::cout << GridLogMessage << " timer2 (CommsMerge ) " <<timer2/calls <<std::endl;
|
|
||||||
if ( timer3 ) std::cout << GridLogMessage << " timer3 (commsMergeShm) " <<timer3/calls <<std::endl;
|
|
||||||
if ( timer4 ) std::cout << GridLogMessage << " timer4 " <<timer4 <<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<int> surface_list;
|
std::vector<int> surface_list;
|
||||||
|
|
||||||
@ -321,26 +337,18 @@ public:
|
|||||||
{
|
{
|
||||||
std::vector<std::vector<CommsRequest_t> > reqs;
|
std::vector<std::vector<CommsRequest_t> > reqs;
|
||||||
this->HaloExchangeOptGather(source,compress);
|
this->HaloExchangeOptGather(source,compress);
|
||||||
double t1=usecond();
|
|
||||||
// Asynchronous MPI calls multidirectional, Isend etc...
|
// Asynchronous MPI calls multidirectional, Isend etc...
|
||||||
// Non-overlapped directions within a thread. Asynchronous calls except MPI3, threaded up to comm threads ways.
|
// Non-overlapped directions within a thread. Asynchronous calls except MPI3, threaded up to comm threads ways.
|
||||||
this->Communicate();
|
this->Communicate();
|
||||||
double t2=usecond(); timer1 += t2-t1;
|
|
||||||
this->CommsMerge(compress);
|
this->CommsMerge(compress);
|
||||||
double t3=usecond(); timer2 += t3-t2;
|
|
||||||
this->CommsMergeSHM(compress);
|
this->CommsMergeSHM(compress);
|
||||||
double t4=usecond(); timer3 += t4-t3;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class compressor>
|
template <class compressor>
|
||||||
void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress)
|
void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress)
|
||||||
{
|
{
|
||||||
this->Prepare();
|
this->Prepare();
|
||||||
double t0=usecond();
|
|
||||||
this->HaloGatherOpt(source,compress);
|
this->HaloGatherOpt(source,compress);
|
||||||
double t1=usecond();
|
|
||||||
timer0 += t1-t0;
|
|
||||||
callsi++;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class compressor>
|
template <class compressor>
|
||||||
@ -352,12 +360,9 @@ public:
|
|||||||
typedef typename compressor::SiteHalfSpinor SiteHalfSpinor;
|
typedef typename compressor::SiteHalfSpinor SiteHalfSpinor;
|
||||||
typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor;
|
typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor;
|
||||||
|
|
||||||
this->mpi3synctime_g-=usecond();
|
|
||||||
this->_grid->StencilBarrier();
|
this->_grid->StencilBarrier();
|
||||||
this->mpi3synctime_g+=usecond();
|
|
||||||
|
|
||||||
assert(source.Grid()==this->_grid);
|
assert(source.Grid()==this->_grid);
|
||||||
this->halogtime-=usecond();
|
|
||||||
|
|
||||||
this->u_comm_offset=0;
|
this->u_comm_offset=0;
|
||||||
|
|
||||||
@ -393,7 +398,6 @@ public:
|
|||||||
}
|
}
|
||||||
this->face_table_computed=1;
|
this->face_table_computed=1;
|
||||||
assert(this->u_comm_offset==this->_unified_buffer_size);
|
assert(this->u_comm_offset==this->_unified_buffer_size);
|
||||||
this->halogtime+=usecond();
|
|
||||||
accelerator_barrier();
|
accelerator_barrier();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -72,7 +72,7 @@ public:
|
|||||||
typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
|
typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
|
||||||
typedef WilsonImplParams ImplParams;
|
typedef WilsonImplParams ImplParams;
|
||||||
typedef WilsonStencil<SiteSpinor, SiteHalfSpinor,ImplParams> StencilImpl;
|
typedef WilsonStencil<SiteSpinor, SiteHalfSpinor,ImplParams> StencilImpl;
|
||||||
typedef typename StencilImpl::View_type StencilView;
|
typedef const typename StencilImpl::View_type StencilView;
|
||||||
|
|
||||||
ImplParams Params;
|
ImplParams Params;
|
||||||
|
|
||||||
@ -106,11 +106,15 @@ public:
|
|||||||
const _SpinorField & phi,
|
const _SpinorField & phi,
|
||||||
int mu)
|
int mu)
|
||||||
{
|
{
|
||||||
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
autoView( out_v, out, AcceleratorWrite);
|
autoView( out_v, out, AcceleratorWrite);
|
||||||
autoView( phi_v, phi, AcceleratorRead);
|
autoView( phi_v, phi, AcceleratorRead);
|
||||||
autoView( Umu_v, Umu, AcceleratorRead);
|
autoView( Umu_v, Umu, AcceleratorRead);
|
||||||
accelerator_for(sss,out.Grid()->oSites(),1,{
|
typedef decltype(coalescedRead(out_v[0])) calcSpinor;
|
||||||
multLink(out_v[sss],Umu_v[sss],phi_v[sss],mu);
|
accelerator_for(sss,out.Grid()->oSites(),Nsimd,{
|
||||||
|
calcSpinor tmp;
|
||||||
|
multLink(tmp,Umu_v[sss],phi_v(sss),mu);
|
||||||
|
coalescedWrite(out_v[sss],tmp);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -180,18 +184,22 @@ public:
|
|||||||
mat = TraceIndex<SpinIndex>(P);
|
mat = TraceIndex<SpinIndex>(P);
|
||||||
}
|
}
|
||||||
|
|
||||||
inline void extractLinkField(std::vector<GaugeLinkField> &mat, DoubledGaugeField &Uds){
|
inline void extractLinkField(std::vector<GaugeLinkField> &mat, DoubledGaugeField &Uds)
|
||||||
|
{
|
||||||
for (int mu = 0; mu < Nd; mu++)
|
for (int mu = 0; mu < Nd; mu++)
|
||||||
mat[mu] = PeekIndex<LorentzIndex>(Uds, mu);
|
mat[mu] = PeekIndex<LorentzIndex>(Uds, mu);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu)
|
||||||
inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu){
|
{
|
||||||
|
#undef USE_OLD_INSERT_FORCE
|
||||||
int Ls=Btilde.Grid()->_fdimensions[0];
|
int Ls=Btilde.Grid()->_fdimensions[0];
|
||||||
|
autoView( mat_v , mat, AcceleratorWrite);
|
||||||
|
#ifdef USE_OLD_INSERT_FORCE
|
||||||
GaugeLinkField tmp(mat.Grid());
|
GaugeLinkField tmp(mat.Grid());
|
||||||
tmp = Zero();
|
tmp = Zero();
|
||||||
{
|
{
|
||||||
|
const int Nsimd = SiteSpinor::Nsimd();
|
||||||
autoView( tmp_v , tmp, AcceleratorWrite);
|
autoView( tmp_v , tmp, AcceleratorWrite);
|
||||||
autoView( Btilde_v , Btilde, AcceleratorRead);
|
autoView( Btilde_v , Btilde, AcceleratorRead);
|
||||||
autoView( Atilde_v , Atilde, AcceleratorRead);
|
autoView( Atilde_v , Atilde, AcceleratorRead);
|
||||||
@ -204,6 +212,29 @@ public:
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
PokeIndex<LorentzIndex>(mat,tmp,mu);
|
PokeIndex<LorentzIndex>(mat,tmp,mu);
|
||||||
|
#else
|
||||||
|
{
|
||||||
|
const int Nsimd = SiteSpinor::Nsimd();
|
||||||
|
autoView( Btilde_v , Btilde, AcceleratorRead);
|
||||||
|
autoView( Atilde_v , Atilde, AcceleratorRead);
|
||||||
|
accelerator_for(sss,mat.Grid()->oSites(),Nsimd,{
|
||||||
|
int sU=sss;
|
||||||
|
typedef decltype(coalescedRead(mat_v[sU](mu)() )) ColorMatrixType;
|
||||||
|
ColorMatrixType sum;
|
||||||
|
zeroit(sum);
|
||||||
|
for(int s=0;s<Ls;s++){
|
||||||
|
int sF = s+Ls*sU;
|
||||||
|
for(int spn=0;spn<Ns;spn++){ //sum over spin
|
||||||
|
auto bb = coalescedRead(Btilde_v[sF]()(spn) ); //color vector
|
||||||
|
auto aa = coalescedRead(Atilde_v[sF]()(spn) );
|
||||||
|
auto op = outerProduct(bb,aa);
|
||||||
|
sum = sum + op;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
coalescedWrite(mat_v[sU](mu)(), sum);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -212,17 +243,17 @@ typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffReal > WilsonImplR
|
|||||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffReal > WilsonImplF; // Float
|
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffReal > WilsonImplF; // Float
|
||||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffReal > WilsonImplD; // Double
|
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffReal > WilsonImplD; // Double
|
||||||
|
|
||||||
typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplRL; // Real.. whichever prec
|
//typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplRL; // Real.. whichever prec
|
||||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplFH; // Float
|
//typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplFH; // Float
|
||||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplDF; // Double
|
//typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplDF; // Double
|
||||||
|
|
||||||
typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffComplex > ZWilsonImplR; // Real.. whichever prec
|
typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffComplex > ZWilsonImplR; // Real.. whichever prec
|
||||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplex > ZWilsonImplF; // Float
|
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplex > ZWilsonImplF; // Float
|
||||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplex > ZWilsonImplD; // Double
|
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplex > ZWilsonImplD; // Double
|
||||||
|
|
||||||
typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplRL; // Real.. whichever prec
|
//typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplRL; // Real.. whichever prec
|
||||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplFH; // Float
|
//typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplFH; // Float
|
||||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplDF; // Double
|
//typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplDF; // Double
|
||||||
|
|
||||||
typedef WilsonImpl<vComplex, AdjointRepresentation, CoeffReal > WilsonAdjImplR; // Real.. whichever prec
|
typedef WilsonImpl<vComplex, AdjointRepresentation, CoeffReal > WilsonAdjImplR; // Real.. whichever prec
|
||||||
typedef WilsonImpl<vComplexF, AdjointRepresentation, CoeffReal > WilsonAdjImplF; // Float
|
typedef WilsonImpl<vComplexF, AdjointRepresentation, CoeffReal > WilsonAdjImplF; // Float
|
||||||
|
@ -49,9 +49,17 @@ public:
|
|||||||
|
|
||||||
INHERIT_IMPL_TYPES(Impl);
|
INHERIT_IMPL_TYPES(Impl);
|
||||||
typedef FermionOperator<Impl> Base;
|
typedef FermionOperator<Impl> Base;
|
||||||
|
typedef AcceleratorVector<int,STENCIL_MAX> StencilVector;
|
||||||
public:
|
public:
|
||||||
|
|
||||||
|
#ifdef GRID_SYCL
|
||||||
|
#define SYCL_HACK
|
||||||
|
#endif
|
||||||
|
#ifdef SYCL_HACK
|
||||||
|
static void HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor *buf,
|
||||||
|
int ss,int sU,const SiteSpinor *in, SiteSpinor *out);
|
||||||
|
#endif
|
||||||
|
|
||||||
static void DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField &U, SiteHalfSpinor * buf,
|
static void DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField &U, SiteHalfSpinor * buf,
|
||||||
int Ls, int Nsite, const FermionField &in, FermionField &out,
|
int Ls, int Nsite, const FermionField &in, FermionField &out,
|
||||||
int interior=1,int exterior=1) ;
|
int interior=1,int exterior=1) ;
|
||||||
|
@ -828,6 +828,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
|
|||||||
|
|
||||||
#if (!defined(GRID_HIP))
|
#if (!defined(GRID_HIP))
|
||||||
int tshift = (mu == Nd-1) ? 1 : 0;
|
int tshift = (mu == Nd-1) ? 1 : 0;
|
||||||
|
unsigned int LLt = GridDefaultLatt()[Tp];
|
||||||
////////////////////////////////////////////////
|
////////////////////////////////////////////////
|
||||||
// GENERAL CAYLEY CASE
|
// GENERAL CAYLEY CASE
|
||||||
////////////////////////////////////////////////
|
////////////////////////////////////////////////
|
||||||
@ -880,17 +881,29 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
|
|||||||
}
|
}
|
||||||
|
|
||||||
std::vector<RealD> G_s(Ls,1.0);
|
std::vector<RealD> G_s(Ls,1.0);
|
||||||
|
RealD sign = 1.0; // sign flip for vector/tadpole
|
||||||
if ( curr_type == Current::Axial ) {
|
if ( curr_type == Current::Axial ) {
|
||||||
for(int s=0;s<Ls/2;s++){
|
for(int s=0;s<Ls/2;s++){
|
||||||
G_s[s] = -1.0;
|
G_s[s] = -1.0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
else if ( curr_type == Current::Tadpole ) {
|
||||||
|
auto b=this->_b;
|
||||||
|
auto c=this->_c;
|
||||||
|
if ( b == 1 && c == 0 ) {
|
||||||
|
sign = -1.0;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
std::cerr << "Error: Tadpole implementation currently unavailable for non-Shamir actions." << std::endl;
|
||||||
|
assert(b==1 && c==0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for(int s=0;s<Ls;s++){
|
for(int s=0;s<Ls;s++){
|
||||||
|
|
||||||
int sp = (s+1)%Ls;
|
int sp = (s+1)%Ls;
|
||||||
int sr = Ls-1-s;
|
// int sr = Ls-1-s;
|
||||||
int srp= (sr+1)%Ls;
|
// int srp= (sr+1)%Ls;
|
||||||
|
|
||||||
// Mobius parameters
|
// Mobius parameters
|
||||||
auto b=this->bs[s];
|
auto b=this->bs[s];
|
||||||
@ -907,7 +920,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
|
|||||||
|
|
||||||
tmp = Cshift(tmp,mu,1);
|
tmp = Cshift(tmp,mu,1);
|
||||||
Impl::multLinkField(Utmp,this->Umu,tmp,mu);
|
Impl::multLinkField(Utmp,this->Umu,tmp,mu);
|
||||||
tmp = G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop
|
tmp = sign*G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop
|
||||||
tmp = where((lcoor>=tmin),tmp,zz); // Mask the time
|
tmp = where((lcoor>=tmin),tmp,zz); // Mask the time
|
||||||
L_Q = where((lcoor<=tmax),tmp,zz); // Position of current complicated
|
L_Q = where((lcoor<=tmax),tmp,zz); // Position of current complicated
|
||||||
|
|
||||||
@ -922,7 +935,13 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
|
|||||||
tmp = Cshift(tmp,mu,-1);
|
tmp = Cshift(tmp,mu,-1);
|
||||||
Impl::multLinkField(Utmp,this->Umu,tmp,mu+Nd); // Adjoint link
|
Impl::multLinkField(Utmp,this->Umu,tmp,mu+Nd); // Adjoint link
|
||||||
tmp = -G_s[s]*( Utmp + gmu*Utmp );
|
tmp = -G_s[s]*( Utmp + gmu*Utmp );
|
||||||
tmp = where((lcoor>=tmin+tshift),tmp,zz); // Mask the time
|
// Mask the time
|
||||||
|
if (tmax == LLt - 1 && tshift == 1){ // quick fix to include timeslice 0 if tmax + tshift is over the last timeslice
|
||||||
|
unsigned int t0 = 0;
|
||||||
|
tmp = where(((lcoor==t0) || (lcoor>=tmin+tshift)),tmp,zz);
|
||||||
|
} else {
|
||||||
|
tmp = where((lcoor>=tmin+tshift),tmp,zz);
|
||||||
|
}
|
||||||
L_Q += where((lcoor<=tmax+tshift),tmp,zz); // Position of current complicated
|
L_Q += where((lcoor<=tmax+tshift),tmp,zz); // Position of current complicated
|
||||||
|
|
||||||
InsertSlice(L_Q, q_out, s , 0);
|
InsertSlice(L_Q, q_out, s , 0);
|
||||||
|
@ -680,7 +680,8 @@ void StaggeredKernels<Impl>::DhopSiteAsm(StencilView &st,
|
|||||||
gauge2 =(uint64_t)&UU[sU]( Z ); \
|
gauge2 =(uint64_t)&UU[sU]( Z ); \
|
||||||
gauge3 =(uint64_t)&UU[sU]( T );
|
gauge3 =(uint64_t)&UU[sU]( T );
|
||||||
|
|
||||||
|
#undef STAG_VEC5D
|
||||||
|
#ifdef STAG_VEC5D
|
||||||
// This is the single precision 5th direction vectorised kernel
|
// This is the single precision 5th direction vectorised kernel
|
||||||
#include <Grid/simd/Intel512single.h>
|
#include <Grid/simd/Intel512single.h>
|
||||||
template <> void StaggeredKernels<StaggeredVec5dImplF>::DhopSiteAsm(StencilView &st,
|
template <> void StaggeredKernels<StaggeredVec5dImplF>::DhopSiteAsm(StencilView &st,
|
||||||
@ -790,7 +791,7 @@ template <> void StaggeredKernels<StaggeredVec5dImplD>::DhopSiteAsm(StencilView
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
#define PERMUTE_DIR3 __asm__ ( \
|
#define PERMUTE_DIR3 __asm__ ( \
|
||||||
|
@ -32,25 +32,50 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
NAMESPACE_BEGIN(Grid);
|
||||||
|
|
||||||
#define LOAD_CHI(b) \
|
#ifdef GRID_SIMT
|
||||||
|
|
||||||
|
#define LOAD_CHI(ptype,b) \
|
||||||
const SiteSpinor & ref (b[offset]); \
|
const SiteSpinor & ref (b[offset]); \
|
||||||
Chi_0=ref()()(0);\
|
Chi_0=coalescedReadPermute<ptype>(ref()()(0),perm,lane); \
|
||||||
Chi_1=ref()()(1);\
|
Chi_1=coalescedReadPermute<ptype>(ref()()(1),perm,lane); \
|
||||||
|
Chi_2=coalescedReadPermute<ptype>(ref()()(2),perm,lane);
|
||||||
|
|
||||||
|
#define LOAD_CHI_COMMS(b) \
|
||||||
|
const SiteSpinor & ref (b[offset]); \
|
||||||
|
Chi_0=coalescedRead(ref()()(0),lane); \
|
||||||
|
Chi_1=coalescedRead(ref()()(1),lane); \
|
||||||
|
Chi_2=coalescedRead(ref()()(2),lane);
|
||||||
|
|
||||||
|
#define PERMUTE_DIR(dir) ;
|
||||||
|
#else
|
||||||
|
#define LOAD_CHI(ptype,b) LOAD_CHI_COMMS(b)
|
||||||
|
|
||||||
|
#define LOAD_CHI_COMMS(b) \
|
||||||
|
const SiteSpinor & ref (b[offset]); \
|
||||||
|
Chi_0=ref()()(0); \
|
||||||
|
Chi_1=ref()()(1); \
|
||||||
Chi_2=ref()()(2);
|
Chi_2=ref()()(2);
|
||||||
|
|
||||||
|
#define PERMUTE_DIR(dir) \
|
||||||
|
permute##dir(Chi_0,Chi_0); \
|
||||||
|
permute##dir(Chi_1,Chi_1); \
|
||||||
|
permute##dir(Chi_2,Chi_2);
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
// To splat or not to splat depends on the implementation
|
// To splat or not to splat depends on the implementation
|
||||||
#define MULT(A,UChi) \
|
#define MULT(A,UChi) \
|
||||||
auto & ref(U[sU](A)); \
|
auto & ref(U[sU](A)); \
|
||||||
Impl::loadLinkElement(U_00,ref()(0,0)); \
|
U_00=coalescedRead(ref()(0,0),lane); \
|
||||||
Impl::loadLinkElement(U_10,ref()(1,0)); \
|
U_10=coalescedRead(ref()(1,0),lane); \
|
||||||
Impl::loadLinkElement(U_20,ref()(2,0)); \
|
U_20=coalescedRead(ref()(2,0),lane); \
|
||||||
Impl::loadLinkElement(U_01,ref()(0,1)); \
|
U_01=coalescedRead(ref()(0,1),lane); \
|
||||||
Impl::loadLinkElement(U_11,ref()(1,1)); \
|
U_11=coalescedRead(ref()(1,1),lane); \
|
||||||
Impl::loadLinkElement(U_21,ref()(2,1)); \
|
U_21=coalescedRead(ref()(2,1),lane); \
|
||||||
Impl::loadLinkElement(U_02,ref()(0,2)); \
|
U_02=coalescedRead(ref()(0,2),lane); \
|
||||||
Impl::loadLinkElement(U_12,ref()(1,2)); \
|
U_12=coalescedRead(ref()(1,2),lane); \
|
||||||
Impl::loadLinkElement(U_22,ref()(2,2)); \
|
U_22=coalescedRead(ref()(2,2),lane); \
|
||||||
UChi ## _0 = U_00*Chi_0; \
|
UChi ## _0 = U_00*Chi_0; \
|
||||||
UChi ## _1 = U_10*Chi_0;\
|
UChi ## _1 = U_10*Chi_0;\
|
||||||
UChi ## _2 = U_20*Chi_0;\
|
UChi ## _2 = U_20*Chi_0;\
|
||||||
@ -63,15 +88,15 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
|
|
||||||
#define MULT_ADD(U,A,UChi) \
|
#define MULT_ADD(U,A,UChi) \
|
||||||
auto & ref(U[sU](A)); \
|
auto & ref(U[sU](A)); \
|
||||||
Impl::loadLinkElement(U_00,ref()(0,0)); \
|
U_00=coalescedRead(ref()(0,0),lane); \
|
||||||
Impl::loadLinkElement(U_10,ref()(1,0)); \
|
U_10=coalescedRead(ref()(1,0),lane); \
|
||||||
Impl::loadLinkElement(U_20,ref()(2,0)); \
|
U_20=coalescedRead(ref()(2,0),lane); \
|
||||||
Impl::loadLinkElement(U_01,ref()(0,1)); \
|
U_01=coalescedRead(ref()(0,1),lane); \
|
||||||
Impl::loadLinkElement(U_11,ref()(1,1)); \
|
U_11=coalescedRead(ref()(1,1),lane); \
|
||||||
Impl::loadLinkElement(U_21,ref()(2,1)); \
|
U_21=coalescedRead(ref()(2,1),lane); \
|
||||||
Impl::loadLinkElement(U_02,ref()(0,2)); \
|
U_02=coalescedRead(ref()(0,2),lane); \
|
||||||
Impl::loadLinkElement(U_12,ref()(1,2)); \
|
U_12=coalescedRead(ref()(1,2),lane); \
|
||||||
Impl::loadLinkElement(U_22,ref()(2,2)); \
|
U_22=coalescedRead(ref()(2,2),lane); \
|
||||||
UChi ## _0 += U_00*Chi_0; \
|
UChi ## _0 += U_00*Chi_0; \
|
||||||
UChi ## _1 += U_10*Chi_0;\
|
UChi ## _1 += U_10*Chi_0;\
|
||||||
UChi ## _2 += U_20*Chi_0;\
|
UChi ## _2 += U_20*Chi_0;\
|
||||||
@ -83,24 +108,18 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
UChi ## _2 += U_22*Chi_2;
|
UChi ## _2 += U_22*Chi_2;
|
||||||
|
|
||||||
|
|
||||||
#define PERMUTE_DIR(dir) \
|
|
||||||
permute##dir(Chi_0,Chi_0); \
|
|
||||||
permute##dir(Chi_1,Chi_1); \
|
|
||||||
permute##dir(Chi_2,Chi_2);
|
|
||||||
|
|
||||||
|
|
||||||
#define HAND_STENCIL_LEG_BASE(Dir,Perm,skew) \
|
#define HAND_STENCIL_LEG_BASE(Dir,Perm,skew) \
|
||||||
SE=st.GetEntry(ptype,Dir+skew,sF); \
|
SE=st.GetEntry(ptype,Dir+skew,sF); \
|
||||||
offset = SE->_offset; \
|
offset = SE->_offset; \
|
||||||
local = SE->_is_local; \
|
local = SE->_is_local; \
|
||||||
perm = SE->_permute; \
|
perm = SE->_permute; \
|
||||||
if ( local ) { \
|
if ( local ) { \
|
||||||
LOAD_CHI(in); \
|
LOAD_CHI(Perm,in); \
|
||||||
if ( perm) { \
|
if ( perm) { \
|
||||||
PERMUTE_DIR(Perm); \
|
PERMUTE_DIR(Perm); \
|
||||||
} \
|
} \
|
||||||
} else { \
|
} else { \
|
||||||
LOAD_CHI(buf); \
|
LOAD_CHI_COMMS(buf); \
|
||||||
}
|
}
|
||||||
|
|
||||||
#define HAND_STENCIL_LEG_BEGIN(Dir,Perm,skew,even) \
|
#define HAND_STENCIL_LEG_BEGIN(Dir,Perm,skew,even) \
|
||||||
@ -116,19 +135,18 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#define HAND_STENCIL_LEG_INT(U,Dir,Perm,skew,even) \
|
#define HAND_STENCIL_LEG_INT(U,Dir,Perm,skew,even) \
|
||||||
SE=st.GetEntry(ptype,Dir+skew,sF); \
|
SE=st.GetEntry(ptype,Dir+skew,sF); \
|
||||||
offset = SE->_offset; \
|
offset = SE->_offset; \
|
||||||
local = SE->_is_local; \
|
local = SE->_is_local; \
|
||||||
perm = SE->_permute; \
|
perm = SE->_permute; \
|
||||||
if ( local ) { \
|
if ( local ) { \
|
||||||
LOAD_CHI(in); \
|
LOAD_CHI(Perm,in); \
|
||||||
if ( perm) { \
|
if ( perm) { \
|
||||||
PERMUTE_DIR(Perm); \
|
PERMUTE_DIR(Perm); \
|
||||||
} \
|
} \
|
||||||
} else if ( st.same_node[Dir] ) { \
|
} else if ( st.same_node[Dir] ) { \
|
||||||
LOAD_CHI(buf); \
|
LOAD_CHI_COMMS(buf); \
|
||||||
} \
|
} \
|
||||||
if (local || st.same_node[Dir] ) { \
|
if (local || st.same_node[Dir] ) { \
|
||||||
MULT_ADD(U,Dir,even); \
|
MULT_ADD(U,Dir,even); \
|
||||||
@ -140,10 +158,32 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
local = SE->_is_local; \
|
local = SE->_is_local; \
|
||||||
if ((!local) && (!st.same_node[Dir]) ) { \
|
if ((!local) && (!st.same_node[Dir]) ) { \
|
||||||
nmu++; \
|
nmu++; \
|
||||||
{ LOAD_CHI(buf); } \
|
{ LOAD_CHI_COMMS(buf); } \
|
||||||
{ MULT_ADD(U,Dir,even); } \
|
{ MULT_ADD(U,Dir,even); } \
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define HAND_DECLARATIONS(Simd) \
|
||||||
|
Simd even_0; \
|
||||||
|
Simd even_1; \
|
||||||
|
Simd even_2; \
|
||||||
|
Simd odd_0; \
|
||||||
|
Simd odd_1; \
|
||||||
|
Simd odd_2; \
|
||||||
|
\
|
||||||
|
Simd Chi_0; \
|
||||||
|
Simd Chi_1; \
|
||||||
|
Simd Chi_2; \
|
||||||
|
\
|
||||||
|
Simd U_00; \
|
||||||
|
Simd U_10; \
|
||||||
|
Simd U_20; \
|
||||||
|
Simd U_01; \
|
||||||
|
Simd U_11; \
|
||||||
|
Simd U_21; \
|
||||||
|
Simd U_02; \
|
||||||
|
Simd U_12; \
|
||||||
|
Simd U_22;
|
||||||
|
|
||||||
|
|
||||||
template <class Impl>
|
template <class Impl>
|
||||||
template <int Naik> accelerator_inline
|
template <int Naik> accelerator_inline
|
||||||
@ -155,28 +195,14 @@ void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
|
|||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
|
||||||
Simd even_0; // 12 regs on knc
|
|
||||||
Simd even_1;
|
|
||||||
Simd even_2;
|
|
||||||
Simd odd_0; // 12 regs on knc
|
|
||||||
Simd odd_1;
|
|
||||||
Simd odd_2;
|
|
||||||
|
|
||||||
Simd Chi_0; // two spinor; 6 regs
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
Simd Chi_1;
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
Simd Chi_2;
|
typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
|
||||||
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
Simd U_00; // two rows of U matrix
|
typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
|
||||||
Simd U_10;
|
calcSiteSpinor result;
|
||||||
Simd U_20;
|
|
||||||
Simd U_01;
|
|
||||||
Simd U_11;
|
|
||||||
Simd U_21; // 2 reg left.
|
|
||||||
Simd U_02;
|
|
||||||
Simd U_12;
|
|
||||||
Simd U_22;
|
|
||||||
|
|
||||||
SiteSpinor result;
|
|
||||||
int offset,local,perm, ptype;
|
int offset,local,perm, ptype;
|
||||||
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
@ -215,7 +241,7 @@ void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
|
|||||||
result()()(1) = even_1 + odd_1;
|
result()()(1) = even_1 + odd_1;
|
||||||
result()()(2) = even_2 + odd_2;
|
result()()(2) = even_2 + odd_2;
|
||||||
}
|
}
|
||||||
vstream(out[sF],result);
|
coalescedWrite(out[sF],result);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -230,28 +256,13 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
|
|||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
|
||||||
Simd even_0; // 12 regs on knc
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
Simd even_1;
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
Simd even_2;
|
typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
|
||||||
Simd odd_0; // 12 regs on knc
|
HAND_DECLARATIONS(Simt);
|
||||||
Simd odd_1;
|
|
||||||
Simd odd_2;
|
|
||||||
|
|
||||||
Simd Chi_0; // two spinor; 6 regs
|
typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
|
||||||
Simd Chi_1;
|
calcSiteSpinor result;
|
||||||
Simd Chi_2;
|
|
||||||
|
|
||||||
Simd U_00; // two rows of U matrix
|
|
||||||
Simd U_10;
|
|
||||||
Simd U_20;
|
|
||||||
Simd U_01;
|
|
||||||
Simd U_11;
|
|
||||||
Simd U_21; // 2 reg left.
|
|
||||||
Simd U_02;
|
|
||||||
Simd U_12;
|
|
||||||
Simd U_22;
|
|
||||||
|
|
||||||
SiteSpinor result;
|
|
||||||
int offset, ptype, local, perm;
|
int offset, ptype, local, perm;
|
||||||
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
@ -261,8 +272,8 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
|
|||||||
// int sF=s+LLs*sU;
|
// int sF=s+LLs*sU;
|
||||||
{
|
{
|
||||||
|
|
||||||
even_0 = Zero(); even_1 = Zero(); even_2 = Zero();
|
zeroit(even_0); zeroit(even_1); zeroit(even_2);
|
||||||
odd_0 = Zero(); odd_1 = Zero(); odd_2 = Zero();
|
zeroit(odd_0); zeroit(odd_1); zeroit(odd_2);
|
||||||
|
|
||||||
skew = 0;
|
skew = 0;
|
||||||
HAND_STENCIL_LEG_INT(U,Xp,3,skew,even);
|
HAND_STENCIL_LEG_INT(U,Xp,3,skew,even);
|
||||||
@ -294,7 +305,7 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
|
|||||||
result()()(1) = even_1 + odd_1;
|
result()()(1) = even_1 + odd_1;
|
||||||
result()()(2) = even_2 + odd_2;
|
result()()(2) = even_2 + odd_2;
|
||||||
}
|
}
|
||||||
vstream(out[sF],result);
|
coalescedWrite(out[sF],result);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -309,28 +320,13 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
|||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
|
||||||
Simd even_0; // 12 regs on knc
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
Simd even_1;
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
Simd even_2;
|
typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
|
||||||
Simd odd_0; // 12 regs on knc
|
HAND_DECLARATIONS(Simt);
|
||||||
Simd odd_1;
|
|
||||||
Simd odd_2;
|
|
||||||
|
|
||||||
Simd Chi_0; // two spinor; 6 regs
|
typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
|
||||||
Simd Chi_1;
|
calcSiteSpinor result;
|
||||||
Simd Chi_2;
|
|
||||||
|
|
||||||
Simd U_00; // two rows of U matrix
|
|
||||||
Simd U_10;
|
|
||||||
Simd U_20;
|
|
||||||
Simd U_01;
|
|
||||||
Simd U_11;
|
|
||||||
Simd U_21; // 2 reg left.
|
|
||||||
Simd U_02;
|
|
||||||
Simd U_12;
|
|
||||||
Simd U_22;
|
|
||||||
|
|
||||||
SiteSpinor result;
|
|
||||||
int offset, ptype, local;
|
int offset, ptype, local;
|
||||||
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
@ -340,8 +336,8 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
|||||||
// int sF=s+LLs*sU;
|
// int sF=s+LLs*sU;
|
||||||
{
|
{
|
||||||
|
|
||||||
even_0 = Zero(); even_1 = Zero(); even_2 = Zero();
|
zeroit(even_0); zeroit(even_1); zeroit(even_2);
|
||||||
odd_0 = Zero(); odd_1 = Zero(); odd_2 = Zero();
|
zeroit(odd_0); zeroit(odd_1); zeroit(odd_2);
|
||||||
int nmu=0;
|
int nmu=0;
|
||||||
skew = 0;
|
skew = 0;
|
||||||
HAND_STENCIL_LEG_EXT(U,Xp,3,skew,even);
|
HAND_STENCIL_LEG_EXT(U,Xp,3,skew,even);
|
||||||
@ -374,7 +370,7 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
|||||||
result()()(1) = even_1 + odd_1;
|
result()()(1) = even_1 + odd_1;
|
||||||
result()()(2) = even_2 + odd_2;
|
result()()(2) = even_2 + odd_2;
|
||||||
}
|
}
|
||||||
out[sF] = out[sF] + result;
|
coalescedWrite(out[sF] , out(sF)+ result);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -397,6 +393,7 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
|||||||
const FermionFieldView &in, FermionFieldView &out, int dag); \
|
const FermionFieldView &in, FermionFieldView &out, int dag); \
|
||||||
*/
|
*/
|
||||||
#undef LOAD_CHI
|
#undef LOAD_CHI
|
||||||
|
#undef HAND_DECLARATIONS
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
NAMESPACE_END(Grid);
|
||||||
|
|
||||||
|
@ -35,39 +35,32 @@ NAMESPACE_BEGIN(Grid);
|
|||||||
#define GENERIC_STENCIL_LEG(U,Dir,skew,multLink) \
|
#define GENERIC_STENCIL_LEG(U,Dir,skew,multLink) \
|
||||||
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
||||||
if (SE->_is_local ) { \
|
if (SE->_is_local ) { \
|
||||||
if (SE->_permute) { \
|
int perm= SE->_permute; \
|
||||||
chi_p = χ \
|
chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\
|
||||||
permute(chi, in[SE->_offset], ptype); \
|
|
||||||
} else { \
|
} else { \
|
||||||
chi_p = &in[SE->_offset]; \
|
chi = coalescedRead(buf[SE->_offset],lane); \
|
||||||
} \
|
} \
|
||||||
} else { \
|
acceleratorSynchronise(); \
|
||||||
chi_p = &buf[SE->_offset]; \
|
multLink(Uchi, U[sU], chi, Dir);
|
||||||
} \
|
|
||||||
multLink(Uchi, U[sU], *chi_p, Dir);
|
|
||||||
|
|
||||||
#define GENERIC_STENCIL_LEG_INT(U,Dir,skew,multLink) \
|
#define GENERIC_STENCIL_LEG_INT(U,Dir,skew,multLink) \
|
||||||
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
||||||
if (SE->_is_local ) { \
|
if (SE->_is_local ) { \
|
||||||
if (SE->_permute) { \
|
int perm= SE->_permute; \
|
||||||
chi_p = χ \
|
chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\
|
||||||
permute(chi, in[SE->_offset], ptype); \
|
|
||||||
} else { \
|
|
||||||
chi_p = &in[SE->_offset]; \
|
|
||||||
} \
|
|
||||||
} else if ( st.same_node[Dir] ) { \
|
} else if ( st.same_node[Dir] ) { \
|
||||||
chi_p = &buf[SE->_offset]; \
|
chi = coalescedRead(buf[SE->_offset],lane); \
|
||||||
} \
|
} \
|
||||||
if (SE->_is_local || st.same_node[Dir] ) { \
|
if (SE->_is_local || st.same_node[Dir] ) { \
|
||||||
multLink(Uchi, U[sU], *chi_p, Dir); \
|
multLink(Uchi, U[sU], chi, Dir); \
|
||||||
}
|
}
|
||||||
|
|
||||||
#define GENERIC_STENCIL_LEG_EXT(U,Dir,skew,multLink) \
|
#define GENERIC_STENCIL_LEG_EXT(U,Dir,skew,multLink) \
|
||||||
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
||||||
if ((!SE->_is_local) && (!st.same_node[Dir]) ) { \
|
if ((!SE->_is_local) && (!st.same_node[Dir]) ) { \
|
||||||
nmu++; \
|
nmu++; \
|
||||||
chi_p = &buf[SE->_offset]; \
|
chi = coalescedRead(buf[SE->_offset],lane); \
|
||||||
multLink(Uchi, U[sU], *chi_p, Dir); \
|
multLink(Uchi, U[sU], chi, Dir); \
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class Impl>
|
template <class Impl>
|
||||||
@ -84,12 +77,14 @@ void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st,
|
|||||||
SiteSpinor *buf, int sF, int sU,
|
SiteSpinor *buf, int sF, int sU,
|
||||||
const FermionFieldView &in, FermionFieldView &out, int dag)
|
const FermionFieldView &in, FermionFieldView &out, int dag)
|
||||||
{
|
{
|
||||||
const SiteSpinor *chi_p;
|
typedef decltype(coalescedRead(in[0])) calcSpinor;
|
||||||
SiteSpinor chi;
|
calcSpinor chi;
|
||||||
SiteSpinor Uchi;
|
calcSpinor Uchi;
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
int ptype;
|
int ptype;
|
||||||
int skew;
|
int skew;
|
||||||
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
// for(int s=0;s<LLs;s++){
|
// for(int s=0;s<LLs;s++){
|
||||||
//
|
//
|
||||||
@ -118,7 +113,7 @@ void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st,
|
|||||||
if ( dag ) {
|
if ( dag ) {
|
||||||
Uchi = - Uchi;
|
Uchi = - Uchi;
|
||||||
}
|
}
|
||||||
vstream(out[sF], Uchi);
|
coalescedWrite(out[sF], Uchi,lane);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -130,13 +125,16 @@ template <int Naik> accelerator_inline
|
|||||||
void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st,
|
void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st,
|
||||||
DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
|
DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
|
||||||
SiteSpinor *buf, int sF, int sU,
|
SiteSpinor *buf, int sF, int sU,
|
||||||
const FermionFieldView &in, FermionFieldView &out,int dag) {
|
const FermionFieldView &in, FermionFieldView &out,int dag)
|
||||||
const SiteSpinor *chi_p;
|
{
|
||||||
SiteSpinor chi;
|
typedef decltype(coalescedRead(in[0])) calcSpinor;
|
||||||
SiteSpinor Uchi;
|
calcSpinor chi;
|
||||||
|
calcSpinor Uchi;
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
int ptype;
|
int ptype;
|
||||||
int skew ;
|
int skew ;
|
||||||
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
// for(int s=0;s<LLs;s++){
|
// for(int s=0;s<LLs;s++){
|
||||||
// int sF=LLs*sU+s;
|
// int sF=LLs*sU+s;
|
||||||
@ -165,7 +163,7 @@ void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st,
|
|||||||
if ( dag ) {
|
if ( dag ) {
|
||||||
Uchi = - Uchi;
|
Uchi = - Uchi;
|
||||||
}
|
}
|
||||||
vstream(out[sF], Uchi);
|
coalescedWrite(out[sF], Uchi,lane);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -178,14 +176,17 @@ template <int Naik> accelerator_inline
|
|||||||
void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st,
|
void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st,
|
||||||
DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
|
DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
|
||||||
SiteSpinor *buf, int sF, int sU,
|
SiteSpinor *buf, int sF, int sU,
|
||||||
const FermionFieldView &in, FermionFieldView &out,int dag) {
|
const FermionFieldView &in, FermionFieldView &out,int dag)
|
||||||
const SiteSpinor *chi_p;
|
{
|
||||||
// SiteSpinor chi;
|
typedef decltype(coalescedRead(in[0])) calcSpinor;
|
||||||
SiteSpinor Uchi;
|
calcSpinor chi;
|
||||||
|
calcSpinor Uchi;
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
int ptype;
|
int ptype;
|
||||||
int nmu=0;
|
int nmu=0;
|
||||||
int skew ;
|
int skew ;
|
||||||
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
// for(int s=0;s<LLs;s++){
|
// for(int s=0;s<LLs;s++){
|
||||||
// int sF=LLs*sU+s;
|
// int sF=LLs*sU+s;
|
||||||
@ -212,10 +213,11 @@ void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st,
|
|||||||
GENERIC_STENCIL_LEG_EXT(UUU,Tm,skew,Impl::multLinkAdd);
|
GENERIC_STENCIL_LEG_EXT(UUU,Tm,skew,Impl::multLinkAdd);
|
||||||
}
|
}
|
||||||
if ( nmu ) {
|
if ( nmu ) {
|
||||||
|
auto _out = coalescedRead(out[sF],lane);
|
||||||
if ( dag ) {
|
if ( dag ) {
|
||||||
out[sF] = out[sF] - Uchi;
|
coalescedWrite(out[sF], _out-Uchi,lane);
|
||||||
} else {
|
} else {
|
||||||
out[sF] = out[sF] + Uchi;
|
coalescedWrite(out[sF], _out+Uchi,lane);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -261,6 +263,8 @@ void StaggeredKernels<Impl>::DhopImproved(StencilImpl &st, LebesgueOrder &lo,
|
|||||||
GridBase *FGrid=in.Grid();
|
GridBase *FGrid=in.Grid();
|
||||||
GridBase *UGrid=U.Grid();
|
GridBase *UGrid=U.Grid();
|
||||||
typedef StaggeredKernels<Impl> ThisKernel;
|
typedef StaggeredKernels<Impl> ThisKernel;
|
||||||
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
autoView( UUU_v , UUU, AcceleratorRead);
|
autoView( UUU_v , UUU, AcceleratorRead);
|
||||||
autoView( U_v , U, AcceleratorRead);
|
autoView( U_v , U, AcceleratorRead);
|
||||||
autoView( in_v , in, AcceleratorRead);
|
autoView( in_v , in, AcceleratorRead);
|
||||||
@ -301,6 +305,8 @@ void StaggeredKernels<Impl>::DhopNaive(StencilImpl &st, LebesgueOrder &lo,
|
|||||||
GridBase *FGrid=in.Grid();
|
GridBase *FGrid=in.Grid();
|
||||||
GridBase *UGrid=U.Grid();
|
GridBase *UGrid=U.Grid();
|
||||||
typedef StaggeredKernels<Impl> ThisKernel;
|
typedef StaggeredKernels<Impl> ThisKernel;
|
||||||
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
autoView( UUU_v , U, AcceleratorRead);
|
autoView( UUU_v , U, AcceleratorRead);
|
||||||
autoView( U_v , U, AcceleratorRead);
|
autoView( U_v , U, AcceleratorRead);
|
||||||
autoView( in_v , in, AcceleratorRead);
|
autoView( in_v , in, AcceleratorRead);
|
||||||
|
@ -397,6 +397,7 @@ void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U, co
|
|||||||
template <class Impl>
|
template <class Impl>
|
||||||
void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int dag)
|
void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int dag)
|
||||||
{
|
{
|
||||||
|
DhopCalls+=2;
|
||||||
conformable(in.Grid(), _grid); // verifies full grid
|
conformable(in.Grid(), _grid); // verifies full grid
|
||||||
conformable(in.Grid(), out.Grid());
|
conformable(in.Grid(), out.Grid());
|
||||||
|
|
||||||
@ -408,6 +409,7 @@ void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int da
|
|||||||
template <class Impl>
|
template <class Impl>
|
||||||
void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int dag)
|
void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int dag)
|
||||||
{
|
{
|
||||||
|
DhopCalls++;
|
||||||
conformable(in.Grid(), _cbgrid); // verifies half grid
|
conformable(in.Grid(), _cbgrid); // verifies half grid
|
||||||
conformable(in.Grid(), out.Grid()); // drops the cb check
|
conformable(in.Grid(), out.Grid()); // drops the cb check
|
||||||
|
|
||||||
@ -420,6 +422,7 @@ void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int
|
|||||||
template <class Impl>
|
template <class Impl>
|
||||||
void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
|
void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
|
||||||
{
|
{
|
||||||
|
DhopCalls++;
|
||||||
conformable(in.Grid(), _cbgrid); // verifies half grid
|
conformable(in.Grid(), _cbgrid); // verifies half grid
|
||||||
conformable(in.Grid(), out.Grid()); // drops the cb check
|
conformable(in.Grid(), out.Grid()); // drops the cb check
|
||||||
|
|
||||||
|
@ -73,17 +73,17 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -102,17 +102,17 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -131,17 +131,17 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -165,17 +165,17 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -194,17 +194,17 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -223,17 +223,17 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
//#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -280,17 +280,17 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<WilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<ZWilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -309,17 +309,17 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<WilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<ZWilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -338,17 +338,17 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<WilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<ZWilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////
|
||||||
@ -371,17 +371,17 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<WilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -400,17 +400,17 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<WilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -429,17 +429,17 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<WilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
#pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
// #pragma GCC optimize ("-O3", "-fno-schedule-insns", "-fno-schedule-insns2")
|
||||||
template<> void
|
// template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
// WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
// #include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -74,15 +74,15 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
//
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -97,15 +97,15 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
//
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
@ -121,15 +121,15 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
//
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////
|
||||||
// XYZT vectorised, dag Kernel, single
|
// XYZT vectorised, dag Kernel, single
|
||||||
@ -148,15 +148,15 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
//
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -171,15 +171,15 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
//
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#undef INTERIOR
|
#undef INTERIOR
|
||||||
@ -194,15 +194,15 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
//
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef MAYBEPERM
|
#undef MAYBEPERM
|
||||||
#undef MULT_2SPIN
|
#undef MULT_2SPIN
|
||||||
@ -228,14 +228,14 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSite(StencilView &st, DoubledGaugeF
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -249,14 +249,14 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteInt(StencilView &st, DoubledGau
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#undef INTERIOR
|
#undef INTERIOR
|
||||||
@ -273,15 +273,15 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteExt(StencilView &st, DoubledGau
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
//
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////
|
||||||
// Ls vectorised, dag Kernel, single
|
// Ls vectorised, dag Kernel, single
|
||||||
@ -299,14 +299,14 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteDag(StencilView &st, DoubledGau
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -320,14 +320,14 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteDagInt(StencilView &st, Doubled
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#undef INTERIOR
|
#undef INTERIOR
|
||||||
@ -341,14 +341,14 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteDagExt(StencilView &st, Doubled
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#endif // VEC 5D
|
#endif // VEC 5D
|
||||||
|
|
||||||
@ -392,14 +392,14 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -413,14 +413,14 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#undef INTERIOR
|
#undef INTERIOR
|
||||||
@ -434,14 +434,14 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////
|
||||||
// XYZT vectorised, dag Kernel, single
|
// XYZT vectorised, dag Kernel, single
|
||||||
@ -459,14 +459,14 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldVi
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -480,14 +480,14 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#undef INTERIOR
|
#undef INTERIOR
|
||||||
@ -501,14 +501,14 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFiel
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<WilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<WilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef MAYBEPERM
|
#undef MAYBEPERM
|
||||||
#undef MULT_2SPIN
|
#undef MULT_2SPIN
|
||||||
@ -533,14 +533,14 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSite(StencilView &st, DoubledGaugeF
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -554,14 +554,14 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteInt(StencilView &st, DoubledGau
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#undef INTERIOR
|
#undef INTERIOR
|
||||||
@ -577,14 +577,14 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteExt(StencilView &st, DoubledGau
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////
|
||||||
// Ls vectorised, dag Kernel, single
|
// Ls vectorised, dag Kernel, single
|
||||||
@ -602,14 +602,14 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteDag(StencilView &st, DoubledGau
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#define INTERIOR
|
#define INTERIOR
|
||||||
@ -623,14 +623,14 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteDagInt(StencilView &st, Doubled
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#undef INTERIOR_AND_EXTERIOR
|
#undef INTERIOR_AND_EXTERIOR
|
||||||
#undef INTERIOR
|
#undef INTERIOR
|
||||||
@ -645,14 +645,14 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteDagExt(StencilView &st, Doubled
|
|||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
template<> void
|
//template<> void
|
||||||
WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
//WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
// int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
//#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||||
|
|
||||||
#endif // VEC 5D
|
#endif // VEC 5D
|
||||||
|
|
||||||
|
@ -76,7 +76,24 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
|
|
||||||
#define REGISTER
|
#define REGISTER
|
||||||
|
|
||||||
#define LOAD_CHIMU \
|
#ifdef GRID_SIMT
|
||||||
|
#define LOAD_CHIMU(Ptype) \
|
||||||
|
{const SiteSpinor & ref (in[offset]); \
|
||||||
|
Chimu_00=coalescedReadPermute<Ptype>(ref()(0)(0),perm,lane); \
|
||||||
|
Chimu_01=coalescedReadPermute<Ptype>(ref()(0)(1),perm,lane); \
|
||||||
|
Chimu_02=coalescedReadPermute<Ptype>(ref()(0)(2),perm,lane); \
|
||||||
|
Chimu_10=coalescedReadPermute<Ptype>(ref()(1)(0),perm,lane); \
|
||||||
|
Chimu_11=coalescedReadPermute<Ptype>(ref()(1)(1),perm,lane); \
|
||||||
|
Chimu_12=coalescedReadPermute<Ptype>(ref()(1)(2),perm,lane); \
|
||||||
|
Chimu_20=coalescedReadPermute<Ptype>(ref()(2)(0),perm,lane); \
|
||||||
|
Chimu_21=coalescedReadPermute<Ptype>(ref()(2)(1),perm,lane); \
|
||||||
|
Chimu_22=coalescedReadPermute<Ptype>(ref()(2)(2),perm,lane); \
|
||||||
|
Chimu_30=coalescedReadPermute<Ptype>(ref()(3)(0),perm,lane); \
|
||||||
|
Chimu_31=coalescedReadPermute<Ptype>(ref()(3)(1),perm,lane); \
|
||||||
|
Chimu_32=coalescedReadPermute<Ptype>(ref()(3)(2),perm,lane); }
|
||||||
|
#define PERMUTE_DIR(dir) ;
|
||||||
|
#else
|
||||||
|
#define LOAD_CHIMU(Ptype) \
|
||||||
{const SiteSpinor & ref (in[offset]); \
|
{const SiteSpinor & ref (in[offset]); \
|
||||||
Chimu_00=ref()(0)(0);\
|
Chimu_00=ref()(0)(0);\
|
||||||
Chimu_01=ref()(0)(1);\
|
Chimu_01=ref()(0)(1);\
|
||||||
@ -91,54 +108,54 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
Chimu_31=ref()(3)(1);\
|
Chimu_31=ref()(3)(1);\
|
||||||
Chimu_32=ref()(3)(2);}
|
Chimu_32=ref()(3)(2);}
|
||||||
|
|
||||||
#define LOAD_CHI\
|
#define PERMUTE_DIR(dir) \
|
||||||
{const SiteHalfSpinor &ref(buf[offset]); \
|
permute##dir(Chi_00,Chi_00); \
|
||||||
Chi_00 = ref()(0)(0);\
|
permute##dir(Chi_01,Chi_01); \
|
||||||
Chi_01 = ref()(0)(1);\
|
permute##dir(Chi_02,Chi_02); \
|
||||||
Chi_02 = ref()(0)(2);\
|
permute##dir(Chi_10,Chi_10); \
|
||||||
Chi_10 = ref()(1)(0);\
|
permute##dir(Chi_11,Chi_11); \
|
||||||
Chi_11 = ref()(1)(1);\
|
permute##dir(Chi_12,Chi_12);
|
||||||
Chi_12 = ref()(1)(2);}
|
|
||||||
|
#endif
|
||||||
|
|
||||||
// To splat or not to splat depends on the implementation
|
|
||||||
#define MULT_2SPIN(A)\
|
#define MULT_2SPIN(A)\
|
||||||
{auto & ref(U[sU](A)); \
|
{auto & ref(U[sU](A)); \
|
||||||
Impl::loadLinkElement(U_00,ref()(0,0)); \
|
U_00=coalescedRead(ref()(0,0),lane); \
|
||||||
Impl::loadLinkElement(U_10,ref()(1,0)); \
|
U_10=coalescedRead(ref()(1,0),lane); \
|
||||||
Impl::loadLinkElement(U_20,ref()(2,0)); \
|
U_20=coalescedRead(ref()(2,0),lane); \
|
||||||
Impl::loadLinkElement(U_01,ref()(0,1)); \
|
U_01=coalescedRead(ref()(0,1),lane); \
|
||||||
Impl::loadLinkElement(U_11,ref()(1,1)); \
|
U_11=coalescedRead(ref()(1,1),lane); \
|
||||||
Impl::loadLinkElement(U_21,ref()(2,1)); \
|
U_21=coalescedRead(ref()(2,1),lane); \
|
||||||
UChi_00 = U_00*Chi_00;\
|
UChi_00 = U_00*Chi_00; \
|
||||||
UChi_10 = U_00*Chi_10;\
|
UChi_10 = U_00*Chi_10; \
|
||||||
UChi_01 = U_10*Chi_00;\
|
UChi_01 = U_10*Chi_00; \
|
||||||
UChi_11 = U_10*Chi_10;\
|
UChi_11 = U_10*Chi_10; \
|
||||||
UChi_02 = U_20*Chi_00;\
|
UChi_02 = U_20*Chi_00; \
|
||||||
UChi_12 = U_20*Chi_10;\
|
UChi_12 = U_20*Chi_10; \
|
||||||
UChi_00+= U_01*Chi_01;\
|
UChi_00+= U_01*Chi_01; \
|
||||||
UChi_10+= U_01*Chi_11;\
|
UChi_10+= U_01*Chi_11; \
|
||||||
UChi_01+= U_11*Chi_01;\
|
UChi_01+= U_11*Chi_01; \
|
||||||
UChi_11+= U_11*Chi_11;\
|
UChi_11+= U_11*Chi_11; \
|
||||||
UChi_02+= U_21*Chi_01;\
|
UChi_02+= U_21*Chi_01; \
|
||||||
UChi_12+= U_21*Chi_11;\
|
UChi_12+= U_21*Chi_11; \
|
||||||
Impl::loadLinkElement(U_00,ref()(0,2)); \
|
U_00=coalescedRead(ref()(0,2),lane); \
|
||||||
Impl::loadLinkElement(U_10,ref()(1,2)); \
|
U_10=coalescedRead(ref()(1,2),lane); \
|
||||||
Impl::loadLinkElement(U_20,ref()(2,2)); \
|
U_20=coalescedRead(ref()(2,2),lane); \
|
||||||
UChi_00+= U_00*Chi_02;\
|
UChi_00+= U_00*Chi_02; \
|
||||||
UChi_10+= U_00*Chi_12;\
|
UChi_10+= U_00*Chi_12; \
|
||||||
UChi_01+= U_10*Chi_02;\
|
UChi_01+= U_10*Chi_02; \
|
||||||
UChi_11+= U_10*Chi_12;\
|
UChi_11+= U_10*Chi_12; \
|
||||||
UChi_02+= U_20*Chi_02;\
|
UChi_02+= U_20*Chi_02; \
|
||||||
UChi_12+= U_20*Chi_12;}
|
UChi_12+= U_20*Chi_12;}
|
||||||
|
|
||||||
|
#define LOAD_CHI \
|
||||||
#define PERMUTE_DIR(dir) \
|
{const SiteHalfSpinor &ref(buf[offset]); \
|
||||||
permute##dir(Chi_00,Chi_00);\
|
Chi_00 = coalescedRead(ref()(0)(0),lane); \
|
||||||
permute##dir(Chi_01,Chi_01);\
|
Chi_01 = coalescedRead(ref()(0)(1),lane); \
|
||||||
permute##dir(Chi_02,Chi_02);\
|
Chi_02 = coalescedRead(ref()(0)(2),lane); \
|
||||||
permute##dir(Chi_10,Chi_10);\
|
Chi_10 = coalescedRead(ref()(1)(0),lane); \
|
||||||
permute##dir(Chi_11,Chi_11);\
|
Chi_11 = coalescedRead(ref()(1)(1),lane); \
|
||||||
permute##dir(Chi_12,Chi_12);
|
Chi_12 = coalescedRead(ref()(1)(2),lane);}
|
||||||
|
|
||||||
// hspin(0)=fspin(0)+timesI(fspin(3));
|
// hspin(0)=fspin(0)+timesI(fspin(3));
|
||||||
// hspin(1)=fspin(1)+timesI(fspin(2));
|
// hspin(1)=fspin(1)+timesI(fspin(2));
|
||||||
@ -353,13 +370,14 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
result_31-= UChi_11; \
|
result_31-= UChi_11; \
|
||||||
result_32-= UChi_12;
|
result_32-= UChi_12;
|
||||||
|
|
||||||
#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \
|
#define HAND_STENCIL_LEGB(PROJ,PERM,DIR,RECON) \
|
||||||
|
{int ptype; \
|
||||||
SE=st.GetEntry(ptype,DIR,ss); \
|
SE=st.GetEntry(ptype,DIR,ss); \
|
||||||
offset = SE->_offset; \
|
auto offset = SE->_offset; \
|
||||||
local = SE->_is_local; \
|
auto local = SE->_is_local; \
|
||||||
perm = SE->_permute; \
|
auto perm = SE->_permute; \
|
||||||
if ( local ) { \
|
if ( local ) { \
|
||||||
LOAD_CHIMU; \
|
LOAD_CHIMU(PERM); \
|
||||||
PROJ; \
|
PROJ; \
|
||||||
if ( perm) { \
|
if ( perm) { \
|
||||||
PERMUTE_DIR(PERM); \
|
PERMUTE_DIR(PERM); \
|
||||||
@ -367,16 +385,48 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
} else { \
|
} else { \
|
||||||
LOAD_CHI; \
|
LOAD_CHI; \
|
||||||
} \
|
} \
|
||||||
|
acceleratorSynchronise(); \
|
||||||
MULT_2SPIN(DIR); \
|
MULT_2SPIN(DIR); \
|
||||||
RECON;
|
RECON; }
|
||||||
|
|
||||||
|
#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \
|
||||||
|
{ SE=&st_p[DIR+8*ss]; \
|
||||||
|
auto ptype=st_perm[DIR]; \
|
||||||
|
auto offset = SE->_offset; \
|
||||||
|
auto local = SE->_is_local; \
|
||||||
|
auto perm = SE->_permute; \
|
||||||
|
if ( local ) { \
|
||||||
|
LOAD_CHIMU(PERM); \
|
||||||
|
PROJ; \
|
||||||
|
if ( perm) { \
|
||||||
|
PERMUTE_DIR(PERM); \
|
||||||
|
} \
|
||||||
|
} else { \
|
||||||
|
LOAD_CHI; \
|
||||||
|
} \
|
||||||
|
acceleratorSynchronise(); \
|
||||||
|
MULT_2SPIN(DIR); \
|
||||||
|
RECON; }
|
||||||
|
|
||||||
|
#define HAND_STENCIL_LEGA(PROJ,PERM,DIR,RECON) \
|
||||||
|
{ SE=&st_p[DIR+8*ss]; \
|
||||||
|
auto ptype=st_perm[DIR]; \
|
||||||
|
/*SE=st.GetEntry(ptype,DIR,ss);*/ \
|
||||||
|
auto offset = SE->_offset; \
|
||||||
|
auto perm = SE->_permute; \
|
||||||
|
LOAD_CHIMU(PERM); \
|
||||||
|
PROJ; \
|
||||||
|
MULT_2SPIN(DIR); \
|
||||||
|
RECON; }
|
||||||
|
|
||||||
#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON) \
|
#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON) \
|
||||||
|
{ int ptype; \
|
||||||
SE=st.GetEntry(ptype,DIR,ss); \
|
SE=st.GetEntry(ptype,DIR,ss); \
|
||||||
offset = SE->_offset; \
|
auto offset = SE->_offset; \
|
||||||
local = SE->_is_local; \
|
auto local = SE->_is_local; \
|
||||||
perm = SE->_permute; \
|
auto perm = SE->_permute; \
|
||||||
if ( local ) { \
|
if ( local ) { \
|
||||||
LOAD_CHIMU; \
|
LOAD_CHIMU(PERM); \
|
||||||
PROJ; \
|
PROJ; \
|
||||||
if ( perm) { \
|
if ( perm) { \
|
||||||
PERMUTE_DIR(PERM); \
|
PERMUTE_DIR(PERM); \
|
||||||
@ -384,57 +434,60 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
} else if ( st.same_node[DIR] ) { \
|
} else if ( st.same_node[DIR] ) { \
|
||||||
LOAD_CHI; \
|
LOAD_CHI; \
|
||||||
} \
|
} \
|
||||||
|
acceleratorSynchronise(); \
|
||||||
if (local || st.same_node[DIR] ) { \
|
if (local || st.same_node[DIR] ) { \
|
||||||
MULT_2SPIN(DIR); \
|
MULT_2SPIN(DIR); \
|
||||||
RECON; \
|
RECON; \
|
||||||
}
|
} \
|
||||||
|
acceleratorSynchronise(); }
|
||||||
|
|
||||||
#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON) \
|
#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON) \
|
||||||
|
{ int ptype; \
|
||||||
SE=st.GetEntry(ptype,DIR,ss); \
|
SE=st.GetEntry(ptype,DIR,ss); \
|
||||||
offset = SE->_offset; \
|
auto offset = SE->_offset; \
|
||||||
if((!SE->_is_local)&&(!st.same_node[DIR]) ) { \
|
if((!SE->_is_local)&&(!st.same_node[DIR]) ) { \
|
||||||
LOAD_CHI; \
|
LOAD_CHI; \
|
||||||
MULT_2SPIN(DIR); \
|
MULT_2SPIN(DIR); \
|
||||||
RECON; \
|
RECON; \
|
||||||
nmu++; \
|
nmu++; \
|
||||||
}
|
} \
|
||||||
|
acceleratorSynchronise(); }
|
||||||
|
|
||||||
#define HAND_RESULT(ss) \
|
#define HAND_RESULT(ss) \
|
||||||
{ \
|
{ \
|
||||||
SiteSpinor & ref (out[ss]); \
|
SiteSpinor & ref (out[ss]); \
|
||||||
vstream(ref()(0)(0),result_00); \
|
coalescedWrite(ref()(0)(0),result_00,lane); \
|
||||||
vstream(ref()(0)(1),result_01); \
|
coalescedWrite(ref()(0)(1),result_01,lane); \
|
||||||
vstream(ref()(0)(2),result_02); \
|
coalescedWrite(ref()(0)(2),result_02,lane); \
|
||||||
vstream(ref()(1)(0),result_10); \
|
coalescedWrite(ref()(1)(0),result_10,lane); \
|
||||||
vstream(ref()(1)(1),result_11); \
|
coalescedWrite(ref()(1)(1),result_11,lane); \
|
||||||
vstream(ref()(1)(2),result_12); \
|
coalescedWrite(ref()(1)(2),result_12,lane); \
|
||||||
vstream(ref()(2)(0),result_20); \
|
coalescedWrite(ref()(2)(0),result_20,lane); \
|
||||||
vstream(ref()(2)(1),result_21); \
|
coalescedWrite(ref()(2)(1),result_21,lane); \
|
||||||
vstream(ref()(2)(2),result_22); \
|
coalescedWrite(ref()(2)(2),result_22,lane); \
|
||||||
vstream(ref()(3)(0),result_30); \
|
coalescedWrite(ref()(3)(0),result_30,lane); \
|
||||||
vstream(ref()(3)(1),result_31); \
|
coalescedWrite(ref()(3)(1),result_31,lane); \
|
||||||
vstream(ref()(3)(2),result_32); \
|
coalescedWrite(ref()(3)(2),result_32,lane); \
|
||||||
}
|
}
|
||||||
|
|
||||||
#define HAND_RESULT_EXT(ss) \
|
#define HAND_RESULT_EXT(ss) \
|
||||||
if (nmu){ \
|
{ \
|
||||||
SiteSpinor & ref (out[ss]); \
|
SiteSpinor & ref (out[ss]); \
|
||||||
ref()(0)(0)+=result_00; \
|
coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00,lane); \
|
||||||
ref()(0)(1)+=result_01; \
|
coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01,lane); \
|
||||||
ref()(0)(2)+=result_02; \
|
coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02,lane); \
|
||||||
ref()(1)(0)+=result_10; \
|
coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10,lane); \
|
||||||
ref()(1)(1)+=result_11; \
|
coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11,lane); \
|
||||||
ref()(1)(2)+=result_12; \
|
coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12,lane); \
|
||||||
ref()(2)(0)+=result_20; \
|
coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20,lane); \
|
||||||
ref()(2)(1)+=result_21; \
|
coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21,lane); \
|
||||||
ref()(2)(2)+=result_22; \
|
coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22,lane); \
|
||||||
ref()(3)(0)+=result_30; \
|
coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30,lane); \
|
||||||
ref()(3)(1)+=result_31; \
|
coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31,lane); \
|
||||||
ref()(3)(2)+=result_32; \
|
coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32,lane); \
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define HAND_DECLARATIONS(Simd) \
|
||||||
#define HAND_DECLARATIONS(a) \
|
|
||||||
Simd result_00; \
|
Simd result_00; \
|
||||||
Simd result_01; \
|
Simd result_01; \
|
||||||
Simd result_02; \
|
Simd result_02; \
|
||||||
@ -467,18 +520,18 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
Simd U_21;
|
Simd U_21;
|
||||||
|
|
||||||
#define ZERO_RESULT \
|
#define ZERO_RESULT \
|
||||||
result_00=Zero(); \
|
zeroit(result_00); \
|
||||||
result_01=Zero(); \
|
zeroit(result_01); \
|
||||||
result_02=Zero(); \
|
zeroit(result_02); \
|
||||||
result_10=Zero(); \
|
zeroit(result_10); \
|
||||||
result_11=Zero(); \
|
zeroit(result_11); \
|
||||||
result_12=Zero(); \
|
zeroit(result_12); \
|
||||||
result_20=Zero(); \
|
zeroit(result_20); \
|
||||||
result_21=Zero(); \
|
zeroit(result_21); \
|
||||||
result_22=Zero(); \
|
zeroit(result_22); \
|
||||||
result_30=Zero(); \
|
zeroit(result_30); \
|
||||||
result_31=Zero(); \
|
zeroit(result_31); \
|
||||||
result_32=Zero();
|
zeroit(result_32);
|
||||||
|
|
||||||
#define Chimu_00 Chi_00
|
#define Chimu_00 Chi_00
|
||||||
#define Chimu_01 Chi_01
|
#define Chimu_01 Chi_01
|
||||||
@ -495,19 +548,54 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
|||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
NAMESPACE_BEGIN(Grid);
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef SYCL_HACK
|
||||||
template<class Impl> accelerator_inline void
|
template<class Impl> accelerator_inline void
|
||||||
WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
WilsonKernels<Impl>::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor *buf,
|
||||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int sU,const SiteSpinor *in, SiteSpinor *out)
|
||||||
{
|
{
|
||||||
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
typedef iSinglet<Simd> vCplx;
|
||||||
|
// typedef decltype( coalescedRead( vCplx()()() )) Simt;
|
||||||
|
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||||
|
|
||||||
HAND_DECLARATIONS(ignore);
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
int offset,local,perm, ptype;
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
|
HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
|
||||||
|
HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
|
||||||
|
HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
|
||||||
|
HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
|
||||||
|
HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
|
||||||
|
HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
|
||||||
|
HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
|
||||||
|
HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
|
||||||
|
HAND_RESULT(ss);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
template<class Impl> accelerator_inline void
|
||||||
|
WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
|
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||||
|
{
|
||||||
|
auto st_p = st._entries_p;
|
||||||
|
auto st_perm = st._permute_type;
|
||||||
|
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||||
|
typedef typename Simd::scalar_type S;
|
||||||
|
typedef typename Simd::vector_type V;
|
||||||
|
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||||
|
|
||||||
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
|
StencilEntry *SE;
|
||||||
HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
|
HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
|
||||||
HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
|
HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
|
||||||
HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
|
HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
|
||||||
@ -523,14 +611,18 @@ template<class Impl> accelerator_inline
|
|||||||
void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||||
{
|
{
|
||||||
|
auto st_p = st._entries_p;
|
||||||
|
auto st_perm = st._permute_type;
|
||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||||
|
|
||||||
HAND_DECLARATIONS(ignore);
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
int offset,local,perm, ptype;
|
|
||||||
|
|
||||||
HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON);
|
HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON);
|
||||||
HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM);
|
HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM);
|
||||||
HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
|
HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
|
||||||
@ -546,13 +638,18 @@ template<class Impl> accelerator_inline void
|
|||||||
WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||||
{
|
{
|
||||||
|
// auto st_p = st._entries_p;
|
||||||
|
// auto st_perm = st._permute_type;
|
||||||
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||||
|
|
||||||
HAND_DECLARATIONS(ignore);
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
int offset,local,perm, ptype;
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
ZERO_RESULT;
|
ZERO_RESULT;
|
||||||
HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
|
HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
|
||||||
@ -570,13 +667,18 @@ template<class Impl> accelerator_inline
|
|||||||
void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||||
{
|
{
|
||||||
|
// auto st_p = st._entries_p;
|
||||||
|
// auto st_perm = st._permute_type;
|
||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||||
|
|
||||||
HAND_DECLARATIONS(ignore);
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
int offset,local,perm, ptype;
|
|
||||||
ZERO_RESULT;
|
ZERO_RESULT;
|
||||||
HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
|
HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
|
||||||
HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
|
HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
|
||||||
@ -593,13 +695,19 @@ template<class Impl> accelerator_inline void
|
|||||||
WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||||
{
|
{
|
||||||
|
// auto st_p = st._entries_p;
|
||||||
|
// auto st_perm = st._permute_type;
|
||||||
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||||
|
|
||||||
HAND_DECLARATIONS(ignore);
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
int offset, ptype;
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
|
// int offset, ptype;
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
int nmu=0;
|
int nmu=0;
|
||||||
ZERO_RESULT;
|
ZERO_RESULT;
|
||||||
@ -618,13 +726,19 @@ template<class Impl> accelerator_inline
|
|||||||
void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||||
{
|
{
|
||||||
|
// auto st_p = st._entries_p;
|
||||||
|
// auto st_perm = st._permute_type;
|
||||||
typedef typename Simd::scalar_type S;
|
typedef typename Simd::scalar_type S;
|
||||||
typedef typename Simd::vector_type V;
|
typedef typename Simd::vector_type V;
|
||||||
|
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||||
|
|
||||||
HAND_DECLARATIONS(ignore);
|
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||||
|
const int lane=acceleratorSIMTlane(Nsimd);
|
||||||
|
|
||||||
|
HAND_DECLARATIONS(Simt);
|
||||||
|
|
||||||
StencilEntry *SE;
|
StencilEntry *SE;
|
||||||
int offset, ptype;
|
// int offset, ptype;
|
||||||
int nmu=0;
|
int nmu=0;
|
||||||
ZERO_RESULT;
|
ZERO_RESULT;
|
||||||
HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
|
HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
|
||||||
@ -682,3 +796,4 @@ NAMESPACE_END(Grid);
|
|||||||
#undef HAND_RESULT
|
#undef HAND_RESULT
|
||||||
#undef HAND_RESULT_INT
|
#undef HAND_RESULT_INT
|
||||||
#undef HAND_RESULT_EXT
|
#undef HAND_RESULT_EXT
|
||||||
|
#undef HAND_DECLARATIONS
|
||||||
|
@ -416,6 +416,20 @@ void WilsonKernels<Impl>::DhopDirKernel( StencilImpl &st, DoubledGaugeField &U,S
|
|||||||
#undef LoopBody
|
#undef LoopBody
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define KERNEL_CALL_TMP(A) \
|
||||||
|
const uint64_t NN = Nsite*Ls; \
|
||||||
|
auto U_p = & U_v[0]; \
|
||||||
|
auto in_p = & in_v[0]; \
|
||||||
|
auto out_p = & out_v[0]; \
|
||||||
|
auto st_p = st_v._entries_p; \
|
||||||
|
auto st_perm = st_v._permute_type; \
|
||||||
|
accelerator_forNB( ss, NN, Simd::Nsimd(), { \
|
||||||
|
int sF = ss; \
|
||||||
|
int sU = ss/Ls; \
|
||||||
|
WilsonKernels<Impl>::A(st_perm,st_p,U_p,buf,sF,sU,in_p,out_p); \
|
||||||
|
}); \
|
||||||
|
accelerator_barrier();
|
||||||
|
|
||||||
#define KERNEL_CALLNB(A) \
|
#define KERNEL_CALLNB(A) \
|
||||||
const uint64_t NN = Nsite*Ls; \
|
const uint64_t NN = Nsite*Ls; \
|
||||||
accelerator_forNB( ss, NN, Simd::Nsimd(), { \
|
accelerator_forNB( ss, NN, Simd::Nsimd(), { \
|
||||||
@ -445,20 +459,24 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField
|
|||||||
|
|
||||||
if( interior && exterior ) {
|
if( interior && exterior ) {
|
||||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSite); return;}
|
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSite); return;}
|
||||||
#ifndef GRID_CUDA
|
#ifdef SYCL_HACK
|
||||||
|
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL_TMP(HandDhopSiteSycl); return; }
|
||||||
|
#else
|
||||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSite); return;}
|
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSite); return;}
|
||||||
|
#endif
|
||||||
|
#ifndef GRID_CUDA
|
||||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSite); return;}
|
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSite); return;}
|
||||||
#endif
|
#endif
|
||||||
} else if( interior ) {
|
} else if( interior ) {
|
||||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALLNB(GenericDhopSiteInt); return;}
|
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALLNB(GenericDhopSiteInt); return;}
|
||||||
#ifndef GRID_CUDA
|
|
||||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALLNB(HandDhopSiteInt); return;}
|
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALLNB(HandDhopSiteInt); return;}
|
||||||
|
#ifndef GRID_CUDA
|
||||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteInt); return;}
|
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteInt); return;}
|
||||||
#endif
|
#endif
|
||||||
} else if( exterior ) {
|
} else if( exterior ) {
|
||||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteExt); return;}
|
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteExt); return;}
|
||||||
#ifndef GRID_CUDA
|
|
||||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteExt); return;}
|
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteExt); return;}
|
||||||
|
#ifndef GRID_CUDA
|
||||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteExt); return;}
|
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteExt); return;}
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
@ -476,20 +494,20 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField
|
|||||||
|
|
||||||
if( interior && exterior ) {
|
if( interior && exterior ) {
|
||||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDag); return;}
|
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDag); return;}
|
||||||
#ifndef GRID_CUDA
|
|
||||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDag); return;}
|
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDag); return;}
|
||||||
|
#ifndef GRID_CUDA
|
||||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDag); return;}
|
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDag); return;}
|
||||||
#endif
|
#endif
|
||||||
} else if( interior ) {
|
} else if( interior ) {
|
||||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagInt); return;}
|
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagInt); return;}
|
||||||
#ifndef GRID_CUDA
|
|
||||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagInt); return;}
|
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagInt); return;}
|
||||||
|
#ifndef GRID_CUDA
|
||||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagInt); return;}
|
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagInt); return;}
|
||||||
#endif
|
#endif
|
||||||
} else if( exterior ) {
|
} else if( exterior ) {
|
||||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagExt); return;}
|
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagExt); return;}
|
||||||
#ifndef GRID_CUDA
|
|
||||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagExt); return;}
|
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagExt); return;}
|
||||||
|
#ifndef GRID_CUDA
|
||||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagExt); return;}
|
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagExt); return;}
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
@ -1 +0,0 @@
|
|||||||
../CayleyFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../DomainWallEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../MobiusEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../PartialFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonCloverFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonKernelsInstantiationGparity.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonTMFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
#define IMPLEMENTATION GparityWilsonImplDF
|
|
@ -1 +0,0 @@
|
|||||||
../CayleyFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../DomainWallEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../MobiusEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../PartialFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonCloverFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonKernelsInstantiationGparity.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonTMFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
#define IMPLEMENTATION GparityWilsonImplFH
|
|
@ -1 +0,0 @@
|
|||||||
../CayleyFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../DomainWallEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../MobiusEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../PartialFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonCloverFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermionInstantiation.cc.master
|
|
@ -1,51 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
|
|
||||||
|
|
||||||
Copyright (C) 2015, 2020
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#include <Grid/qcd/action/fermion/FermionCore.h>
|
|
||||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
|
|
||||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
|
|
||||||
|
|
||||||
#ifndef AVX512
|
|
||||||
#ifndef QPX
|
|
||||||
#ifndef A64FX
|
|
||||||
#ifndef A64FXFIXEDSIZE
|
|
||||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
#include "impl.h"
|
|
||||||
template class WilsonKernels<IMPLEMENTATION>;
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonTMFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
#define IMPLEMENTATION WilsonImplDF
|
|
@ -1 +0,0 @@
|
|||||||
../CayleyFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../DomainWallEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../MobiusEOFAFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../PartialFractionFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonCloverFermionInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermion5DInstantiation.cc.master
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonFermionInstantiation.cc.master
|
|
@ -1,51 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
|
|
||||||
|
|
||||||
Copyright (C) 2015, 2020
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#include <Grid/qcd/action/fermion/FermionCore.h>
|
|
||||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
|
|
||||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
|
|
||||||
|
|
||||||
#ifndef AVX512
|
|
||||||
#ifndef QPX
|
|
||||||
#ifndef A64FX
|
|
||||||
#ifndef A64FXFIXEDSIZE
|
|
||||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
#include "impl.h"
|
|
||||||
template class WilsonKernels<IMPLEMENTATION>;
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
@ -1 +0,0 @@
|
|||||||
../WilsonTMFermionInstantiation.cc.master
|
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user