mirror of
https://github.com/paboyle/Grid.git
synced 2025-11-04 05:54:32 +00:00
Hand unrolled version of dslash in a separate class.
Useful to compare; raises Intel compiler from 9GFlop/s to 17.5 Gflops.
on ivybridge core. Raises Clang form 14.5 to 17.5
This commit is contained in:
Peter Boyle
@@ -1,4 +1,3 @@
|
||||
|
||||
#include <Grid.h>
|
||||
|
||||
namespace Grid {
|
||||
@@ -7,15 +6,7 @@ namespace QCD {
|
||||
const std::vector<int> WilsonMatrix::directions ({0,1,2,3, 0, 1, 2, 3});
|
||||
const std::vector<int> WilsonMatrix::displacements({1,1,1,1,-1,-1,-1,-1});
|
||||
|
||||
// Should be in header?
|
||||
const int WilsonMatrix::Xp = 0;
|
||||
const int WilsonMatrix::Yp = 1;
|
||||
const int WilsonMatrix::Zp = 2;
|
||||
const int WilsonMatrix::Tp = 3;
|
||||
const int WilsonMatrix::Xm = 4;
|
||||
const int WilsonMatrix::Ym = 5;
|
||||
const int WilsonMatrix::Zm = 6;
|
||||
const int WilsonMatrix::Tm = 7;
|
||||
int WilsonMatrix::HandOptDslash;
|
||||
|
||||
class WilsonCompressor {
|
||||
public:
|
||||
@@ -39,28 +30,28 @@ const int WilsonMatrix::Tm = 7;
|
||||
mudag=(mu+Nd)%(2*Nd);
|
||||
}
|
||||
switch(mudag) {
|
||||
case WilsonMatrix::Xp:
|
||||
case Xp:
|
||||
spProjXp(ret,in);
|
||||
break;
|
||||
case WilsonMatrix::Yp:
|
||||
case Yp:
|
||||
spProjYp(ret,in);
|
||||
break;
|
||||
case WilsonMatrix::Zp:
|
||||
case Zp:
|
||||
spProjZp(ret,in);
|
||||
break;
|
||||
case WilsonMatrix::Tp:
|
||||
case Tp:
|
||||
spProjTp(ret,in);
|
||||
break;
|
||||
case WilsonMatrix::Xm:
|
||||
case Xm:
|
||||
spProjXm(ret,in);
|
||||
break;
|
||||
case WilsonMatrix::Ym:
|
||||
case Ym:
|
||||
spProjYm(ret,in);
|
||||
break;
|
||||
case WilsonMatrix::Zm:
|
||||
case Zm:
|
||||
spProjZm(ret,in);
|
||||
break;
|
||||
case WilsonMatrix::Tm:
|
||||
case Tm:
|
||||
spProjTm(ret,in);
|
||||
break;
|
||||
default:
|
||||
@@ -157,316 +148,36 @@ void WilsonMatrix::MooeeInvDag(const LatticeFermion &in, LatticeFermion &out)
|
||||
MooeeInv(in,out);
|
||||
}
|
||||
|
||||
void WilsonMatrix::DhopSite(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out)
|
||||
{
|
||||
vHalfSpinColourVector tmp;
|
||||
vHalfSpinColourVector chi;
|
||||
vSpinColourVector result;
|
||||
vHalfSpinColourVector Uchi;
|
||||
int offset,local,perm, ptype;
|
||||
|
||||
//#define VERBOSE( A) if ( ss<10 ) { std::cout << "site " <<ss << " " #A " neigh " << offset << " perm "<< perm <<std::endl;}
|
||||
|
||||
// Xp
|
||||
offset = st._offsets [Xp][ss];
|
||||
local = st._is_local[Xp][ss];
|
||||
perm = st._permute[Xp][ss];
|
||||
|
||||
ptype = st._permute_type[Xp];
|
||||
if ( local && perm ) {
|
||||
spProjXp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjXp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Xp),&chi());
|
||||
spReconXp(result,Uchi);
|
||||
|
||||
// Yp
|
||||
offset = st._offsets [Yp][ss];
|
||||
local = st._is_local[Yp][ss];
|
||||
perm = st._permute[Yp][ss];
|
||||
ptype = st._permute_type[Yp];
|
||||
if ( local && perm ) {
|
||||
spProjYp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjYp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Yp),&chi());
|
||||
accumReconYp(result,Uchi);
|
||||
|
||||
// Zp
|
||||
offset = st._offsets [Zp][ss];
|
||||
local = st._is_local[Zp][ss];
|
||||
perm = st._permute[Zp][ss];
|
||||
ptype = st._permute_type[Zp];
|
||||
if ( local && perm ) {
|
||||
spProjZp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjZp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Zp),&chi());
|
||||
accumReconZp(result,Uchi);
|
||||
|
||||
// Tp
|
||||
offset = st._offsets [Tp][ss];
|
||||
local = st._is_local[Tp][ss];
|
||||
perm = st._permute[Tp][ss];
|
||||
ptype = st._permute_type[Tp];
|
||||
if ( local && perm ) {
|
||||
spProjTp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjTp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Tp),&chi());
|
||||
accumReconTp(result,Uchi);
|
||||
|
||||
// Xm
|
||||
offset = st._offsets [Xm][ss];
|
||||
local = st._is_local[Xm][ss];
|
||||
perm = st._permute[Xm][ss];
|
||||
ptype = st._permute_type[Xm];
|
||||
|
||||
if ( local && perm )
|
||||
{
|
||||
spProjXm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjXm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Xm),&chi());
|
||||
accumReconXm(result,Uchi);
|
||||
|
||||
|
||||
// Ym
|
||||
offset = st._offsets [Ym][ss];
|
||||
local = st._is_local[Ym][ss];
|
||||
perm = st._permute[Ym][ss];
|
||||
ptype = st._permute_type[Ym];
|
||||
|
||||
if ( local && perm ) {
|
||||
spProjYm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjYm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Ym),&chi());
|
||||
accumReconYm(result,Uchi);
|
||||
|
||||
// Zm
|
||||
offset = st._offsets [Zm][ss];
|
||||
local = st._is_local[Zm][ss];
|
||||
perm = st._permute[Zm][ss];
|
||||
ptype = st._permute_type[Zm];
|
||||
if ( local && perm ) {
|
||||
spProjZm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjZm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Zm),&chi());
|
||||
accumReconZm(result,Uchi);
|
||||
|
||||
// Tm
|
||||
offset = st._offsets [Tm][ss];
|
||||
local = st._is_local[Tm][ss];
|
||||
perm = st._permute[Tm][ss];
|
||||
ptype = st._permute_type[Tm];
|
||||
if ( local && perm ) {
|
||||
spProjTm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjTm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Tm),&chi());
|
||||
accumReconTm(result,Uchi);
|
||||
|
||||
vstream(out._odata[ss],result);
|
||||
}
|
||||
|
||||
void WilsonMatrix::DhopSiteDag(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out)
|
||||
{
|
||||
vHalfSpinColourVector tmp;
|
||||
vHalfSpinColourVector chi;
|
||||
vSpinColourVector result;
|
||||
vHalfSpinColourVector Uchi;
|
||||
int offset,local,perm, ptype;
|
||||
|
||||
// Xp
|
||||
offset = st._offsets [Xm][ss];
|
||||
local = st._is_local[Xm][ss];
|
||||
perm = st._permute[Xm][ss];
|
||||
|
||||
ptype = st._permute_type[Xm];
|
||||
if ( local && perm ) {
|
||||
spProjXp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjXp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Xm),&chi());
|
||||
spReconXp(result,Uchi);
|
||||
|
||||
// Yp
|
||||
offset = st._offsets [Ym][ss];
|
||||
local = st._is_local[Ym][ss];
|
||||
perm = st._permute[Ym][ss];
|
||||
ptype = st._permute_type[Ym];
|
||||
if ( local && perm ) {
|
||||
spProjYp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjYp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Ym),&chi());
|
||||
accumReconYp(result,Uchi);
|
||||
|
||||
// Zp
|
||||
offset = st._offsets [Zm][ss];
|
||||
local = st._is_local[Zm][ss];
|
||||
perm = st._permute[Zm][ss];
|
||||
ptype = st._permute_type[Zm];
|
||||
if ( local && perm ) {
|
||||
spProjZp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjZp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Zm),&chi());
|
||||
accumReconZp(result,Uchi);
|
||||
|
||||
// Tp
|
||||
offset = st._offsets [Tm][ss];
|
||||
local = st._is_local[Tm][ss];
|
||||
perm = st._permute[Tm][ss];
|
||||
ptype = st._permute_type[Tm];
|
||||
if ( local && perm ) {
|
||||
spProjTp(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjTp(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Tm),&chi());
|
||||
accumReconTp(result,Uchi);
|
||||
|
||||
// Xm
|
||||
offset = st._offsets [Xp][ss];
|
||||
local = st._is_local[Xp][ss];
|
||||
perm = st._permute[Xp][ss];
|
||||
ptype = st._permute_type[Xp];
|
||||
|
||||
if ( local && perm )
|
||||
{
|
||||
spProjXm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjXm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Xp),&chi());
|
||||
accumReconXm(result,Uchi);
|
||||
|
||||
// Ym
|
||||
offset = st._offsets [Yp][ss];
|
||||
local = st._is_local[Yp][ss];
|
||||
perm = st._permute[Yp][ss];
|
||||
ptype = st._permute_type[Yp];
|
||||
|
||||
if ( local && perm ) {
|
||||
spProjYm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjYm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Yp),&chi());
|
||||
accumReconYm(result,Uchi);
|
||||
|
||||
// Zm
|
||||
offset = st._offsets [Zp][ss];
|
||||
local = st._is_local[Zp][ss];
|
||||
perm = st._permute[Zp][ss];
|
||||
ptype = st._permute_type[Zp];
|
||||
if ( local && perm ) {
|
||||
spProjZm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjZm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Zp),&chi());
|
||||
accumReconZm(result,Uchi);
|
||||
|
||||
// Tm
|
||||
offset = st._offsets [Tp][ss];
|
||||
local = st._is_local[Tp][ss];
|
||||
perm = st._permute[Tp][ss];
|
||||
ptype = st._permute_type[Tp];
|
||||
if ( local && perm ) {
|
||||
spProjTm(tmp,in._odata[offset]);
|
||||
permute(chi,tmp,ptype);
|
||||
} else if ( local ) {
|
||||
spProjTm(chi,in._odata[offset]);
|
||||
} else {
|
||||
chi=buf[offset];
|
||||
}
|
||||
mult(&Uchi(),&U._odata[ss](Tp),&chi());
|
||||
accumReconTm(result,Uchi);
|
||||
|
||||
vstream(out._odata[ss],result);
|
||||
}
|
||||
|
||||
void WilsonMatrix::DhopInternal(CartesianStencil & st,LatticeDoubledGaugeField & U,
|
||||
const LatticeFermion &in, LatticeFermion &out,int dag)
|
||||
{
|
||||
assert((dag==DaggerNo) ||(dag==DaggerYes));
|
||||
WilsonCompressor compressor(dag);
|
||||
|
||||
st.HaloExchange<vSpinColourVector,vHalfSpinColourVector,WilsonCompressor>(in,comm_buf,compressor);
|
||||
|
||||
if ( dag == DaggerYes ) {
|
||||
if( HandOptDslash ) {
|
||||
PARALLEL_FOR_LOOP
|
||||
for(int sss=0;sss<in._grid->oSites();sss++){
|
||||
DhopSiteDag(st,U,comm_buf,sss,in,out);
|
||||
for(int sss=0;sss<in._grid->oSites();sss++){
|
||||
DiracOptHand::DhopSiteDag(st,U,comm_buf,sss,in,out);
|
||||
}
|
||||
} else {
|
||||
PARALLEL_FOR_LOOP
|
||||
for(int sss=0;sss<in._grid->oSites();sss++){
|
||||
DiracOpt::DhopSiteDag(st,U,comm_buf,sss,in,out);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if( HandOptDslash ) {
|
||||
PARALLEL_FOR_LOOP
|
||||
for(int sss=0;sss<in._grid->oSites();sss++){
|
||||
DhopSite(st,U,comm_buf,sss,in,out);
|
||||
for(int sss=0;sss<in._grid->oSites();sss++){
|
||||
DiracOptHand::DhopSite(st,U,comm_buf,sss,in,out);
|
||||
}
|
||||
} else {
|
||||
PARALLEL_FOR_LOOP
|
||||
for(int sss=0;sss<in._grid->oSites();sss++){
|
||||
DiracOpt::DhopSite(st,U,comm_buf,sss,in,out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,10 +6,22 @@ namespace Grid {
|
||||
|
||||
namespace QCD {
|
||||
|
||||
// Should be in header?
|
||||
const int Xp = 0;
|
||||
const int Yp = 1;
|
||||
const int Zp = 2;
|
||||
const int Tp = 3;
|
||||
const int Xm = 4;
|
||||
const int Ym = 5;
|
||||
const int Zm = 6;
|
||||
const int Tm = 7;
|
||||
|
||||
class WilsonMatrix : public CheckerBoardedSparseMatrixBase<LatticeFermion>
|
||||
{
|
||||
//NB r=1;
|
||||
public:
|
||||
static int HandOptDslash;
|
||||
|
||||
double mass;
|
||||
// GridBase * grid; // Inherited
|
||||
// GridBase * cbgrid;
|
||||
@@ -56,14 +68,6 @@ namespace Grid {
|
||||
void DhopEO(const LatticeFermion &in, LatticeFermion &out,int dag);
|
||||
void DhopInternal(CartesianStencil & st,LatticeDoubledGaugeField &U,
|
||||
const LatticeFermion &in, LatticeFermion &out,int dag);
|
||||
// These ones will need to be package intelligently. WilsonType base class
|
||||
// for use by DWF etc..
|
||||
void DhopSite(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out);
|
||||
void DhopSiteDag(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out);
|
||||
|
||||
typedef iScalar<iMatrix<vComplex, Nc> > matrix;
|
||||
|
||||
@@ -71,6 +75,31 @@ namespace Grid {
|
||||
};
|
||||
|
||||
|
||||
class DiracOpt {
|
||||
public:
|
||||
// These ones will need to be package intelligently. WilsonType base class
|
||||
// for use by DWF etc..
|
||||
static void DhopSite(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out);
|
||||
static void DhopSiteDag(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out);
|
||||
|
||||
};
|
||||
class DiracOptHand {
|
||||
public:
|
||||
// These ones will need to be package intelligently. WilsonType base class
|
||||
// for use by DWF etc..
|
||||
static void DhopSite(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out);
|
||||
static void DhopSiteDag(CartesianStencil &st,LatticeDoubledGaugeField &U,
|
||||
std::vector<vHalfSpinColourVector,alignedAllocator<vHalfSpinColourVector> > &buf,
|
||||
int ss,const LatticeFermion &in, LatticeFermion &out);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user