diff --git a/lib/qcd/action/fermion/WilsonKernels.h b/lib/qcd/action/fermion/WilsonKernels.h index 8f9b62f0..f08fb554 100644 --- a/lib/qcd/action/fermion/WilsonKernels.h +++ b/lib/qcd/action/fermion/WilsonKernels.h @@ -56,27 +56,27 @@ public: public: - template accelerator + template static accelerator typename std::enable_if::type - DhopSite(int Opt,typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + DhopSite(int Opt,typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in, FermionFieldView &out,int interior=1,int exterior=1) { bgq_l1p_optimisation(1); switch(Opt) { #if defined(AVX512) || defined (QPX) case WilsonKernelsStatic::OptInlineAsm: - if(interior&&exterior) WilsonKernels::AsmDhopSite (st,lo,U,buf,sF,sU,Ls,Nsite,in,out); - else if (interior) WilsonKernels::AsmDhopSiteInt(st,lo,U,buf,sF,sU,Ls,Nsite,in,out); - else if (exterior) WilsonKernels::AsmDhopSiteExt(st,lo,U,buf,sF,sU,Ls,Nsite,in,out); + if(interior&&exterior) WilsonKernels::AsmDhopSite (st,U,buf,sF,sU,Ls,Nsite,in,out); + else if (interior) WilsonKernels::AsmDhopSiteInt(st,U,buf,sF,sU,Ls,Nsite,in,out); + else if (exterior) WilsonKernels::AsmDhopSiteExt(st,U,buf,sF,sU,Ls,Nsite,in,out); else assert(0); break; #endif case WilsonKernelsStatic::OptHandUnroll: for (int site = 0; site < Nsite; site++) { for (int s = 0; s < Ls; s++) { - if(interior&&exterior) WilsonKernels::HandDhopSite(st,lo,U,buf,sF,sU,in,out); - else if (interior) WilsonKernels::HandDhopSiteInt(st,lo,U,buf,sF,sU,in,out); - else if (exterior) WilsonKernels::HandDhopSiteExt(st,lo,U,buf,sF,sU,in,out); + if(interior&&exterior) WilsonKernels::HandDhopSite(st,U,buf,sF,sU,in,out); + else if (interior) WilsonKernels::HandDhopSiteInt(st,U,buf,sF,sU,in,out); + else if (exterior) WilsonKernels::HandDhopSiteExt(st,U,buf,sF,sU,in,out); sF++; } sU++; @@ -85,9 +85,9 @@ public: case WilsonKernelsStatic::OptGeneric: for (int site = 0; site < Nsite; site++) { for (int s = 0; s < Ls; s++) { - if(interior&&exterior) WilsonKernels::GenericDhopSite(st,lo,U,buf,sF,sU,in,out); - else if (interior) WilsonKernels::GenericDhopSiteInt(st,lo,U,buf,sF,sU,in,out); - else if (exterior) WilsonKernels::GenericDhopSiteExt(st,lo,U,buf,sF,sU,in,out); + if(interior&&exterior) WilsonKernels::GenericDhopSite(st,U,buf,sF,sU,in,out); + else if (interior) WilsonKernels::GenericDhopSiteInt(st,U,buf,sF,sU,in,out); + else if (exterior) WilsonKernels::GenericDhopSiteExt(st,U,buf,sF,sU,in,out); else assert(0); sF++; } @@ -100,16 +100,16 @@ public: bgq_l1p_optimisation(0); } - template accelerator + template static accelerator typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type - DhopSite(int Opt, typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + DhopSite(int Opt, typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in, FermionFieldView &out,int interior=1,int exterior=1 ) { // no kernel choice for (int site = 0; site < Nsite; site++) { for (int s = 0; s < Ls; s++) { - if(interior&&exterior) WilsonKernels::GenericDhopSite(st,lo,U,buf,sF,sU,in,out); - else if (interior) WilsonKernels::GenericDhopSiteInt(st,lo,U,buf,sF,sU,in,out); - else if (exterior) WilsonKernels::GenericDhopSiteExt(st,lo,U,buf,sF,sU,in,out); + if(interior&&exterior) WilsonKernels::GenericDhopSite(st,U,buf,sF,sU,in,out); + else if (interior) WilsonKernels::GenericDhopSiteInt(st,U,buf,sF,sU,in,out); + else if (exterior) WilsonKernels::GenericDhopSiteExt(st,U,buf,sF,sU,in,out); else assert(0); sF++; } @@ -117,27 +117,27 @@ public: } } - template accelerator + template static accelerator typename std::enable_if::type - DhopSiteDag(int Opt, typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + DhopSiteDag(int Opt, typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in, FermionFieldView &out,int interior=1,int exterior=1) { bgq_l1p_optimisation(1); switch(Opt) { #if defined(AVX512) || defined (QPX) case WilsonKernelsStatic::OptInlineAsm: - if(interior&&exterior) WilsonKernels::AsmDhopSiteDag (st,lo,U,buf,sF,sU,Ls,Nsite,in,out); - else if (interior) WilsonKernels::AsmDhopSiteDagInt(st,lo,U,buf,sF,sU,Ls,Nsite,in,out); - else if (exterior) WilsonKernels::AsmDhopSiteDagExt(st,lo,U,buf,sF,sU,Ls,Nsite,in,out); + if(interior&&exterior) WilsonKernels::AsmDhopSiteDag (st,U,buf,sF,sU,Ls,Nsite,in,out); + else if (interior) WilsonKernels::AsmDhopSiteDagInt(st,U,buf,sF,sU,Ls,Nsite,in,out); + else if (exterior) WilsonKernels::AsmDhopSiteDagExt(st,U,buf,sF,sU,Ls,Nsite,in,out); else assert(0); break; #endif case WilsonKernelsStatic::OptHandUnroll: for (int site = 0; site < Nsite; site++) { for (int s = 0; s < Ls; s++) { - if(interior&&exterior) WilsonKernels::HandDhopSiteDag(st,lo,U,buf,sF,sU,in,out); - else if (interior) WilsonKernels::HandDhopSiteDagInt(st,lo,U,buf,sF,sU,in,out); - else if (exterior) WilsonKernels::HandDhopSiteDagExt(st,lo,U,buf,sF,sU,in,out); + if(interior&&exterior) WilsonKernels::HandDhopSiteDag(st,U,buf,sF,sU,in,out); + else if (interior) WilsonKernels::HandDhopSiteDagInt(st,U,buf,sF,sU,in,out); + else if (exterior) WilsonKernels::HandDhopSiteDagExt(st,U,buf,sF,sU,in,out); else assert(0); sF++; } @@ -147,9 +147,9 @@ public: case WilsonKernelsStatic::OptGeneric: for (int site = 0; site < Nsite; site++) { for (int s = 0; s < Ls; s++) { - if(interior&&exterior) WilsonKernels::GenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out); - else if (interior) WilsonKernels::GenericDhopSiteDagInt(st,lo,U,buf,sF,sU,in,out); - else if (exterior) WilsonKernels::GenericDhopSiteDagExt(st,lo,U,buf,sF,sU,in,out); + if(interior&&exterior) WilsonKernels::GenericDhopSiteDag(st,U,buf,sF,sU,in,out); + else if (interior) WilsonKernels::GenericDhopSiteDagInt(st,U,buf,sF,sU,in,out); + else if (exterior) WilsonKernels::GenericDhopSiteDagExt(st,U,buf,sF,sU,in,out); else assert(0); sF++; } @@ -162,16 +162,16 @@ public: bgq_l1p_optimisation(0); } - template accelerator + template static accelerator typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool,void>::type - DhopSiteDag(int Opt,typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U,SiteHalfSpinor * buf, + DhopSiteDag(int Opt,typename StencilImpl::View_type &st, DoubledGaugeFieldView &U,SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in, FermionFieldView &out,int interior=1,int exterior=1) { for (int site = 0; site < Nsite; site++) { for (int s = 0; s < Ls; s++) { - if(interior&&exterior) WilsonKernels::GenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out); - else if (interior) WilsonKernels::GenericDhopSiteDagInt(st,lo,U,buf,sF,sU,in,out); - else if (exterior) WilsonKernels::GenericDhopSiteDagExt(st,lo,U,buf,sF,sU,in,out); + if(interior&&exterior) WilsonKernels::GenericDhopSiteDag(st,U,buf,sF,sU,in,out); + else if (interior) WilsonKernels::GenericDhopSiteDagInt(st,U,buf,sF,sU,in,out); + else if (exterior) WilsonKernels::GenericDhopSiteDagExt(st,U,buf,sF,sU,in,out); else assert(0); sF++; } @@ -179,34 +179,34 @@ public: } } - accelerator void DhopDirK(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U,SiteHalfSpinor * buf, - int sF, int sU, const FermionFieldView &in, FermionFieldView &out, int dirdisp, int gamma); + static accelerator void DhopDirK(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U,SiteHalfSpinor * buf, + int sF, int sU, const FermionFieldView &in, FermionFieldView &out, int dirdisp, int gamma); ////////////////////////////////////////////////////////////////////////////// // Utilities for inserting Wilson conserved current. ////////////////////////////////////////////////////////////////////////////// - void ContractConservedCurrentSiteFwd(const SitePropagator &q_in_1, + static void ContractConservedCurrentSiteFwd(const SitePropagator &q_in_1, const SitePropagator &q_in_2, SitePropagator &q_out, DoubledGaugeFieldView &U, unsigned int sU, unsigned int mu, bool switch_sign = false); - void ContractConservedCurrentSiteBwd(const SitePropagator &q_in_1, + static void ContractConservedCurrentSiteBwd(const SitePropagator &q_in_1, const SitePropagator &q_in_2, SitePropagator &q_out, DoubledGaugeFieldView &U, unsigned int sU, unsigned int mu, bool switch_sign = false); - void SeqConservedCurrentSiteFwd(const SitePropagator &q_in, + static void SeqConservedCurrentSiteFwd(const SitePropagator &q_in, SitePropagator &q_out, DoubledGaugeFieldView &U, unsigned int sU, unsigned int mu, vInteger t_mask, bool switch_sign = false); - void SeqConservedCurrentSiteBwd(const SitePropagator &q_in, + static void SeqConservedCurrentSiteBwd(const SitePropagator &q_in, SitePropagator &q_out, DoubledGaugeFieldView &U, unsigned int sU, @@ -216,59 +216,59 @@ public: private: // Specialised variants - accelerator void GenericDhopSite(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void GenericDhopSite(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void GenericDhopSiteDag(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void GenericDhopSiteDag(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void GenericDhopSiteInt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void GenericDhopSiteInt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void GenericDhopSiteDagInt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void GenericDhopSiteDagInt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void GenericDhopSiteExt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void GenericDhopSiteExt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void GenericDhopSiteDagExt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void GenericDhopSiteDagExt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void AsmDhopSite(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void AsmDhopSite(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in,FermionFieldView &out); - accelerator void AsmDhopSiteDag(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void AsmDhopSiteDag(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in, FermionFieldView &out); - accelerator void AsmDhopSiteInt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void AsmDhopSiteInt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in,FermionFieldView &out); - accelerator void AsmDhopSiteDagInt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void AsmDhopSiteDagInt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in, FermionFieldView &out); - accelerator void AsmDhopSiteExt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void AsmDhopSiteExt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in,FermionFieldView &out); - accelerator void AsmDhopSiteDagExt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void AsmDhopSiteDagExt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, int Ls, int Nsite, const FermionFieldView &in, FermionFieldView &out); - accelerator void HandDhopSite(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void HandDhopSite(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void HandDhopSiteDag(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void HandDhopSiteDag(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void HandDhopSiteInt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void HandDhopSiteInt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void HandDhopSiteDagInt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void HandDhopSiteDagInt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void HandDhopSiteExt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void HandDhopSiteExt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); - accelerator void HandDhopSiteDagExt(typename StencilImpl::View_type &st, LebesgueOrder &lo, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, + static accelerator void HandDhopSiteDagExt(typename StencilImpl::View_type &st, DoubledGaugeFieldView &U, SiteHalfSpinor * buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out); public: