diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc index aa6b5f6b..50816495 100644 --- a/lib/qcd/action/fermion/WilsonKernelsHand.cc +++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc @@ -30,181 +30,60 @@ Author: paboyle #define REGISTER -#define LOAD_CHIMU_BODY(F) \ - Chimu_00=ref(F)(0)(0); \ - Chimu_01=ref(F)(0)(1); \ - Chimu_02=ref(F)(0)(2); \ - Chimu_10=ref(F)(1)(0); \ - Chimu_11=ref(F)(1)(1); \ - Chimu_12=ref(F)(1)(2); \ - Chimu_20=ref(F)(2)(0); \ - Chimu_21=ref(F)(2)(1); \ - Chimu_22=ref(F)(2)(2); \ - Chimu_30=ref(F)(3)(0); \ - Chimu_31=ref(F)(3)(1); \ - Chimu_32=ref(F)(3)(2) +#define LOAD_CHIMU \ + {const SiteSpinor & ref (in._odata[offset]); \ + Chimu_00=ref()(0)(0);\ + Chimu_01=ref()(0)(1);\ + Chimu_02=ref()(0)(2);\ + Chimu_10=ref()(1)(0);\ + Chimu_11=ref()(1)(1);\ + Chimu_12=ref()(1)(2);\ + Chimu_20=ref()(2)(0);\ + Chimu_21=ref()(2)(1);\ + Chimu_22=ref()(2)(2);\ + Chimu_30=ref()(3)(0);\ + Chimu_31=ref()(3)(1);\ + Chimu_32=ref()(3)(2);} -#define LOAD_CHIMU(DIR,F,PERM) \ - { const SiteSpinor & ref (in._odata[offset]); LOAD_CHIMU_BODY(F); } - -#define LOAD_CHI_BODY(F) \ - Chi_00 = ref(F)(0)(0);\ - Chi_01 = ref(F)(0)(1);\ - Chi_02 = ref(F)(0)(2);\ - Chi_10 = ref(F)(1)(0);\ - Chi_11 = ref(F)(1)(1);\ - Chi_12 = ref(F)(1)(2) - -#define LOAD_CHI(DIR,F,PERM) \ - {const SiteHalfSpinor &ref(buf[offset]); LOAD_CHI_BODY(F); } - - -//G-parity implementations using in-place intrinsic ops - -//1l 1h -> 1h 1l -//0l 0h , 1h 1l -> 0l 1h 0h,1l -//0h,1l -> 1l,0h -//if( (distance == 1 && !perm_will_occur) || (distance == -1 && perm_will_occur) ) -//Pulled fermion through forwards face, GPBC on upper component -//Need 0= 0l 1h 1= 1l 0h -//else if( (distance == -1 && !perm) || (distance == 1 && perm) ) -//Pulled fermion through backwards face, GPBC on lower component -//Need 0= 1l 0h 1= 0l 1h - -//1l 1h -> 1h 1l -//0l 0h , 1h 1l -> 0l 1h 0h,1l -#define DO_TWIST_0L_1H(INTO,S,C,F, PERM, tmp1, tmp2, tmp3) \ - permute##PERM(tmp1, ref(1)(S)(C)); \ - exchange##PERM(tmp2,tmp3, ref(0)(S)(C), tmp1); \ - INTO = tmp2; - -//0l 0h -> 0h 0l -//1l 1h, 0h 0l -> 1l 0h, 1h 0l -#define DO_TWIST_1L_0H(INTO,S,C,F, PERM, tmp1, tmp2, tmp3) \ - permute##PERM(tmp1, ref(0)(S)(C)); \ - exchange##PERM(tmp2,tmp3, ref(1)(S)(C), tmp1); \ - INTO = tmp2; - - - - -#define LOAD_CHI_SETUP(DIR,F) \ - g = F; \ - direction = st._directions[DIR]; \ - distance = st._distances[DIR]; \ - sl = st._grid->_simd_layout[direction]; \ - inplace_twist = 0; \ - if(SE->_around_the_world && this->Params.twists[DIR % 4]){ \ - if(sl == 1){ \ - g = (F+1) % 2; \ - }else{ \ - inplace_twist = 1; \ - } \ - } - -#define LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F,PERM) \ - { const SiteSpinor &ref(in._odata[offset]); \ - LOAD_CHI_SETUP(DIR,F); \ - if(!inplace_twist){ \ - LOAD_CHIMU_BODY(g); \ - }else{ \ - if( ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \ - ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \ - DO_TWIST_0L_1H(Chimu_00,0,0,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chimu_01,0,1,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_0L_1H(Chimu_02,0,2,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chimu_10,1,0,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_0L_1H(Chimu_11,1,1,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chimu_12,1,2,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_0L_1H(Chimu_20,2,0,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chimu_21,2,1,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_0L_1H(Chimu_22,2,2,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chimu_30,3,0,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_0L_1H(Chimu_31,3,1,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chimu_32,3,2,F,PERM, U_11,U_20,U_21); \ - }else{ \ - DO_TWIST_1L_0H(Chimu_00,0,0,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chimu_01,0,1,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_1L_0H(Chimu_02,0,2,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chimu_10,1,0,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_1L_0H(Chimu_11,1,1,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chimu_12,1,2,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_1L_0H(Chimu_20,2,0,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chimu_21,2,1,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_1L_0H(Chimu_22,2,2,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chimu_30,3,0,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_1L_0H(Chimu_31,3,1,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chimu_32,3,2,F,PERM, U_11,U_20,U_21); \ - } \ - } \ - } - - -#define LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F,PERM) \ - { const SiteHalfSpinor &ref(buf[offset]); \ - LOAD_CHI_SETUP(DIR,F); \ - if(!inplace_twist){ \ - LOAD_CHI_BODY(g); \ - }else{ \ - if( ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \ - ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \ - DO_TWIST_0L_1H(Chi_00,0,0,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chi_01,0,1,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_0L_1H(Chi_02,0,2,F,PERM, UChi_00,UChi_01,UChi_02); \ - DO_TWIST_0L_1H(Chi_10,1,0,F,PERM, UChi_10,UChi_11,UChi_12); \ - DO_TWIST_0L_1H(Chi_11,1,1,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_0L_1H(Chi_12,1,2,F,PERM, U_11,U_20,U_21); \ - }else{ \ - DO_TWIST_1L_0H(Chi_00,0,0,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chi_01,0,1,F,PERM, U_11,U_20,U_21); \ - DO_TWIST_1L_0H(Chi_02,0,2,F,PERM, UChi_00,UChi_01,UChi_02); \ - DO_TWIST_1L_0H(Chi_10,1,0,F,PERM, UChi_10,UChi_11,UChi_12); \ - DO_TWIST_1L_0H(Chi_11,1,1,F,PERM, U_00,U_01,U_10); \ - DO_TWIST_1L_0H(Chi_12,1,2,F,PERM, U_11,U_20,U_21); \ - } \ - } \ - } - - -#define LOAD_CHI_GPARITY(DIR,F,PERM) LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F,PERM) -#define LOAD_CHIMU_GPARITY(DIR,F,PERM) LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F,PERM) +#define LOAD_CHI\ + {const SiteHalfSpinor &ref(buf[offset]); \ + Chi_00 = ref()(0)(0);\ + Chi_01 = ref()(0)(1);\ + Chi_02 = ref()(0)(2);\ + Chi_10 = ref()(1)(0);\ + Chi_11 = ref()(1)(1);\ + Chi_12 = ref()(1)(2);} // To splat or not to splat depends on the implementation -#define MULT_2SPIN_BODY \ - Impl::loadLinkElement(U_00,ref()(0,0)); \ - Impl::loadLinkElement(U_10,ref()(1,0)); \ - Impl::loadLinkElement(U_20,ref()(2,0)); \ - Impl::loadLinkElement(U_01,ref()(0,1)); \ - Impl::loadLinkElement(U_11,ref()(1,1)); \ - Impl::loadLinkElement(U_21,ref()(2,1)); \ - UChi_00 = U_00*Chi_00; \ - UChi_10 = U_00*Chi_10; \ - UChi_01 = U_10*Chi_00; \ - UChi_11 = U_10*Chi_10; \ - UChi_02 = U_20*Chi_00; \ - UChi_12 = U_20*Chi_10; \ - UChi_00+= U_01*Chi_01; \ - UChi_10+= U_01*Chi_11; \ - UChi_01+= U_11*Chi_01; \ - UChi_11+= U_11*Chi_11; \ - UChi_02+= U_21*Chi_01; \ - UChi_12+= U_21*Chi_11; \ - Impl::loadLinkElement(U_00,ref()(0,2)); \ - Impl::loadLinkElement(U_10,ref()(1,2)); \ - Impl::loadLinkElement(U_20,ref()(2,2)); \ - UChi_00+= U_00*Chi_02; \ - UChi_10+= U_00*Chi_12; \ - UChi_01+= U_10*Chi_02; \ - UChi_11+= U_10*Chi_12; \ - UChi_02+= U_20*Chi_02; \ - UChi_12+= U_20*Chi_12 - - -#define MULT_2SPIN(A,F) \ - {auto & ref(U._odata[sU](A)); MULT_2SPIN_BODY; } - -#define MULT_2SPIN_GPARITY(A,F) \ - {auto & ref(U._odata[sU](F)(A)); MULT_2SPIN_BODY; } +#define MULT_2SPIN(A)\ + {auto & ref(U._odata[sU](A)); \ + Impl::loadLinkElement(U_00,ref()(0,0)); \ + Impl::loadLinkElement(U_10,ref()(1,0)); \ + Impl::loadLinkElement(U_20,ref()(2,0)); \ + Impl::loadLinkElement(U_01,ref()(0,1)); \ + Impl::loadLinkElement(U_11,ref()(1,1)); \ + Impl::loadLinkElement(U_21,ref()(2,1)); \ + UChi_00 = U_00*Chi_00;\ + UChi_10 = U_00*Chi_10;\ + UChi_01 = U_10*Chi_00;\ + UChi_11 = U_10*Chi_10;\ + UChi_02 = U_20*Chi_00;\ + UChi_12 = U_20*Chi_10;\ + UChi_00+= U_01*Chi_01;\ + UChi_10+= U_01*Chi_11;\ + UChi_01+= U_11*Chi_01;\ + UChi_11+= U_11*Chi_11;\ + UChi_02+= U_21*Chi_01;\ + UChi_12+= U_21*Chi_11;\ + Impl::loadLinkElement(U_00,ref()(0,2)); \ + Impl::loadLinkElement(U_10,ref()(1,2)); \ + Impl::loadLinkElement(U_20,ref()(2,2)); \ + UChi_00+= U_00*Chi_02;\ + UChi_10+= U_00*Chi_12;\ + UChi_01+= U_10*Chi_02;\ + UChi_11+= U_10*Chi_12;\ + UChi_02+= U_20*Chi_02;\ + UChi_12+= U_20*Chi_12;} #define PERMUTE_DIR(dir) \ @@ -428,87 +307,84 @@ Author: paboyle result_31-= UChi_11; \ result_32-= UChi_12; -#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ +#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \ SE=st.GetEntry(ptype,DIR,ss); \ offset = SE->_offset; \ local = SE->_is_local; \ perm = SE->_permute; \ if ( local ) { \ - LOAD_CHIMU_IMPL(DIR,F,PERM); \ + LOAD_CHIMU; \ PROJ; \ if ( perm) { \ PERMUTE_DIR(PERM); \ } \ } else { \ - LOAD_CHI_IMPL(DIR,F,PERM); \ + LOAD_CHI; \ } \ - MULT_2SPIN_IMPL(DIR,F); \ + MULT_2SPIN(DIR); \ RECON; - -#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ +#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON) \ SE=st.GetEntry(ptype,DIR,ss); \ offset = SE->_offset; \ local = SE->_is_local; \ perm = SE->_permute; \ if ( local ) { \ - LOAD_CHIMU_IMPL(DIR,F,PERM); \ + LOAD_CHIMU; \ PROJ; \ if ( perm) { \ PERMUTE_DIR(PERM); \ } \ } else if ( st.same_node[DIR] ) { \ - LOAD_CHI_IMPL(DIR,F,PERM); \ + LOAD_CHI; \ } \ if (local || st.same_node[DIR] ) { \ - MULT_2SPIN_IMPL(DIR,F); \ + MULT_2SPIN(DIR); \ RECON; \ } -#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ +#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON) \ SE=st.GetEntry(ptype,DIR,ss); \ offset = SE->_offset; \ - local = SE->_is_local; \ - perm = SE->_permute; \ if((!SE->_is_local)&&(!st.same_node[DIR]) ) { \ - LOAD_CHI_IMPL(DIR,F,PERM); \ - MULT_2SPIN_IMPL(DIR,F); \ + LOAD_CHI; \ + MULT_2SPIN(DIR); \ RECON; \ nmu++; \ } -#define HAND_RESULT(ss,F) \ +#define HAND_RESULT(ss) \ { \ SiteSpinor & ref (out._odata[ss]); \ - vstream(ref(F)(0)(0),result_00); \ - vstream(ref(F)(0)(1),result_01); \ - vstream(ref(F)(0)(2),result_02); \ - vstream(ref(F)(1)(0),result_10); \ - vstream(ref(F)(1)(1),result_11); \ - vstream(ref(F)(1)(2),result_12); \ - vstream(ref(F)(2)(0),result_20); \ - vstream(ref(F)(2)(1),result_21); \ - vstream(ref(F)(2)(2),result_22); \ - vstream(ref(F)(3)(0),result_30); \ - vstream(ref(F)(3)(1),result_31); \ - vstream(ref(F)(3)(2),result_32); \ + vstream(ref()(0)(0),result_00); \ + vstream(ref()(0)(1),result_01); \ + vstream(ref()(0)(2),result_02); \ + vstream(ref()(1)(0),result_10); \ + vstream(ref()(1)(1),result_11); \ + vstream(ref()(1)(2),result_12); \ + vstream(ref()(2)(0),result_20); \ + vstream(ref()(2)(1),result_21); \ + vstream(ref()(2)(2),result_22); \ + vstream(ref()(3)(0),result_30); \ + vstream(ref()(3)(1),result_31); \ + vstream(ref()(3)(2),result_32); \ } -#define HAND_RESULT_EXT(ss,F) \ +#define HAND_RESULT_EXT(ss) \ if (nmu){ \ SiteSpinor & ref (out._odata[ss]); \ - ref(F)(0)(0)+=result_00; \ - ref(F)(0)(1)+=result_01; \ - ref(F)(0)(2)+=result_02; \ - ref(F)(1)(0)+=result_10; \ - ref(F)(1)(1)+=result_11; \ - ref(F)(1)(2)+=result_12; \ - ref(F)(2)(0)+=result_20; \ - ref(F)(2)(1)+=result_21; \ - ref(F)(2)(2)+=result_22; \ - ref(F)(3)(0)+=result_30; \ - ref(F)(3)(1)+=result_31; \ - ref(F)(3)(2)+=result_32; \ + ref()(0)(0)+=result_00; \ + ref()(0)(1)+=result_01; \ + ref()(0)(2)+=result_02; \ + ref()(1)(0)+=result_10; \ + ref()(1)(1)+=result_11; \ + ref()(1)(2)+=result_12; \ + ref()(2)(0)+=result_20; \ + ref()(2)(1)+=result_21; \ + ref()(2)(2)+=result_22; \ + ref()(3)(0)+=result_30; \ + ref()(3)(1)+=result_31; \ + ref()(3)(2)+=result_32; \ } @@ -587,18 +463,15 @@ WilsonKernels::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGauge int offset,local,perm, ptype; StencilEntry *SE; -#define HAND_DOP_SITE(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ - HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_RESULT(ss,F) - - HAND_DOP_SITE(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN); + HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON); + HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM); + HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM); + HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM); + HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM); + HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM); + HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM); + HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM); + HAND_RESULT(ss); } template @@ -612,19 +485,16 @@ void WilsonKernels::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,Doub StencilEntry *SE; int offset,local,perm, ptype; - -#define HAND_DOP_SITE_DAG(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ - HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(TP_PROJ,0,Tp,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(XM_PROJ,3,Xm,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(YM_PROJ,2,Ym,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(ZM_PROJ,1,Zm,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG(TM_PROJ,0,Tm,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_RESULT(ss,F) - - HAND_DOP_SITE_DAG(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN); + + HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON); + HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM); + HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM); + HAND_STENCIL_LEG(TP_PROJ,0,Tp,TP_RECON_ACCUM); + HAND_STENCIL_LEG(XM_PROJ,3,Xm,XM_RECON_ACCUM); + HAND_STENCIL_LEG(YM_PROJ,2,Ym,YM_RECON_ACCUM); + HAND_STENCIL_LEG(ZM_PROJ,1,Zm,ZM_RECON_ACCUM); + HAND_STENCIL_LEG(TM_PROJ,0,Tm,TM_RECON_ACCUM); + HAND_RESULT(ss); } template void @@ -639,20 +509,16 @@ WilsonKernels::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGa int offset,local,perm, ptype; StencilEntry *SE; - -#define HAND_DOP_SITE_INT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ - ZERO_RESULT; \ - HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(YM_PROJ,2,Yp,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(TM_PROJ,0,Tp,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(XP_PROJ,3,Xm,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(YP_PROJ,2,Ym,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(TP_PROJ,0,Tm,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_RESULT(ss,F) - - HAND_DOP_SITE_INT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN); + ZERO_RESULT; + HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM); + HAND_STENCIL_LEG_INT(YM_PROJ,2,Yp,YM_RECON_ACCUM); + HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM); + HAND_STENCIL_LEG_INT(TM_PROJ,0,Tp,TM_RECON_ACCUM); + HAND_STENCIL_LEG_INT(XP_PROJ,3,Xm,XP_RECON_ACCUM); + HAND_STENCIL_LEG_INT(YP_PROJ,2,Ym,YP_RECON_ACCUM); + HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM); + HAND_STENCIL_LEG_INT(TP_PROJ,0,Tm,TP_RECON_ACCUM); + HAND_RESULT(ss); } template @@ -666,20 +532,16 @@ void WilsonKernels::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,D StencilEntry *SE; int offset,local,perm, ptype; - -#define HAND_DOP_SITE_DAG_INT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ - ZERO_RESULT; \ - HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(TP_PROJ,0,Tp,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(XM_PROJ,3,Xm,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(YM_PROJ,2,Ym,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_INT(TM_PROJ,0,Tm,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_RESULT(ss,F) - - HAND_DOP_SITE_DAG_INT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN); + ZERO_RESULT; + HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM); + HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM); + HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM); + HAND_STENCIL_LEG_INT(TP_PROJ,0,Tp,TP_RECON_ACCUM); + HAND_STENCIL_LEG_INT(XM_PROJ,3,Xm,XM_RECON_ACCUM); + HAND_STENCIL_LEG_INT(YM_PROJ,2,Ym,YM_RECON_ACCUM); + HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM); + HAND_STENCIL_LEG_INT(TM_PROJ,0,Tm,TM_RECON_ACCUM); + HAND_RESULT(ss); } template void @@ -695,20 +557,16 @@ WilsonKernels::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGa int offset,local,perm, ptype; StencilEntry *SE; int nmu=0; - -#define HAND_DOP_SITE_EXT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ - ZERO_RESULT; \ - HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xp,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(YM_PROJ,2,Yp,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tp,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xm,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(YP_PROJ,2,Ym,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tm,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_RESULT_EXT(ss,F) - - HAND_DOP_SITE_EXT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN); + ZERO_RESULT; + HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xp,XM_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(YM_PROJ,2,Yp,YM_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tp,TM_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xm,XP_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(YP_PROJ,2,Ym,YP_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tm,TP_RECON_ACCUM); + HAND_RESULT_EXT(ss); } template @@ -723,193 +581,18 @@ void WilsonKernels::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D StencilEntry *SE; int offset,local,perm, ptype; int nmu=0; - -#define HAND_DOP_SITE_DAG_EXT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \ - ZERO_RESULT; \ - HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(YP_PROJ,2,Yp,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tp,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xm,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(YM_PROJ,2,Ym,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tm,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \ - HAND_RESULT_EXT(ss,F) - - HAND_DOP_SITE_DAG_EXT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN); + ZERO_RESULT; + HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(YP_PROJ,2,Yp,YP_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tp,TP_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xm,XM_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(YM_PROJ,2,Ym,YM_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM); + HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tm,TM_RECON_ACCUM); + HAND_RESULT_EXT(ss); } - //////////////////////////////////////////////// - // Specialise Gparity to simple implementation - //////////////////////////////////////////////// -#define HAND_SPECIALISE_EMPTY(IMPL) \ - template<> void \ - WilsonKernels::HandDhopSite(StencilImpl &st, \ - LebesgueOrder &lo, \ - DoubledGaugeField &U, \ - SiteHalfSpinor *buf, \ - int sF,int sU, \ - const FermionField &in, \ - FermionField &out){ assert(0); } \ - template<> void \ - WilsonKernels::HandDhopSiteDag(StencilImpl &st, \ - LebesgueOrder &lo, \ - DoubledGaugeField &U, \ - SiteHalfSpinor *buf, \ - int sF,int sU, \ - const FermionField &in, \ - FermionField &out){ assert(0); } \ - template<> void \ - WilsonKernels::HandDhopSiteInt(StencilImpl &st, \ - LebesgueOrder &lo, \ - DoubledGaugeField &U, \ - SiteHalfSpinor *buf, \ - int sF,int sU, \ - const FermionField &in, \ - FermionField &out){ assert(0); } \ - template<> void \ - WilsonKernels::HandDhopSiteExt(StencilImpl &st, \ - LebesgueOrder &lo, \ - DoubledGaugeField &U, \ - SiteHalfSpinor *buf, \ - int sF,int sU, \ - const FermionField &in, \ - FermionField &out){ assert(0); } \ - template<> void \ - WilsonKernels::HandDhopSiteDagInt(StencilImpl &st, \ - LebesgueOrder &lo, \ - DoubledGaugeField &U, \ - SiteHalfSpinor *buf, \ - int sF,int sU, \ - const FermionField &in, \ - FermionField &out){ assert(0); } \ - template<> void \ - WilsonKernels::HandDhopSiteDagExt(StencilImpl &st, \ - LebesgueOrder &lo, \ - DoubledGaugeField &U, \ - SiteHalfSpinor *buf, \ - int sF,int sU, \ - const FermionField &in, \ - FermionField &out){ assert(0); } \ - - - -#define HAND_SPECIALISE_GPARITY(IMPL) \ - template<> void \ - WilsonKernels::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \ - int ss,int sU,const FermionField &in, FermionField &out) \ - { \ - typedef IMPL Impl; \ - typedef typename Simd::scalar_type S; \ - typedef typename Simd::vector_type V; \ - \ - HAND_DECLARATIONS(ignore); \ - \ - int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \ - StencilEntry *SE; \ - HAND_DOP_SITE(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - HAND_DOP_SITE(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - } \ - \ - template<> \ - void WilsonKernels::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \ - int ss,int sU,const FermionField &in, FermionField &out) \ - { \ - typedef IMPL Impl; \ - typedef typename Simd::scalar_type S; \ - typedef typename Simd::vector_type V; \ - \ - HAND_DECLARATIONS(ignore); \ - \ - StencilEntry *SE; \ - int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \ - HAND_DOP_SITE_DAG(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - HAND_DOP_SITE_DAG(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - } \ - \ - template<> void \ - WilsonKernels::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \ - int ss,int sU,const FermionField &in, FermionField &out) \ - { \ - typedef IMPL Impl; \ - typedef typename Simd::scalar_type S; \ - typedef typename Simd::vector_type V; \ - \ - HAND_DECLARATIONS(ignore); \ - \ - int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \ - StencilEntry *SE; \ - HAND_DOP_SITE_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - HAND_DOP_SITE_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - } \ - \ - template<> \ - void WilsonKernels::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \ - int ss,int sU,const FermionField &in, FermionField &out) \ - { \ - typedef IMPL Impl; \ - typedef typename Simd::scalar_type S; \ - typedef typename Simd::vector_type V; \ - \ - HAND_DECLARATIONS(ignore); \ - \ - StencilEntry *SE; \ - int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \ - HAND_DOP_SITE_DAG_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - HAND_DOP_SITE_DAG_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - } \ - \ - template<> void \ - WilsonKernels::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \ - int ss,int sU,const FermionField &in, FermionField &out) \ - { \ - typedef IMPL Impl; \ - typedef typename Simd::scalar_type S; \ - typedef typename Simd::vector_type V; \ - \ - HAND_DECLARATIONS(ignore); \ - \ - int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \ - StencilEntry *SE; \ - int nmu=0; \ - HAND_DOP_SITE_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - nmu = 0; \ - HAND_DOP_SITE_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - } \ - template<> \ - void WilsonKernels::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \ - int ss,int sU,const FermionField &in, FermionField &out) \ - { \ - typedef IMPL Impl; \ - typedef typename Simd::scalar_type S; \ - typedef typename Simd::vector_type V; \ - \ - HAND_DECLARATIONS(ignore); \ - \ - StencilEntry *SE; \ - int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \ - int nmu=0; \ - HAND_DOP_SITE_DAG_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - nmu = 0; \ - HAND_DOP_SITE_DAG_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \ - } - - -HAND_SPECIALISE_GPARITY(GparityWilsonImplF); -HAND_SPECIALISE_GPARITY(GparityWilsonImplD); -HAND_SPECIALISE_GPARITY(GparityWilsonImplFH); -HAND_SPECIALISE_GPARITY(GparityWilsonImplDF); - - - - - - - - - - - ////////////// Wilson ; uses this implementation ///////////////////// #define INSTANTIATE_THEM(A) \ @@ -930,8 +613,6 @@ INSTANTIATE_THEM(WilsonImplF); INSTANTIATE_THEM(WilsonImplD); INSTANTIATE_THEM(ZWilsonImplF); INSTANTIATE_THEM(ZWilsonImplD); -INSTANTIATE_THEM(GparityWilsonImplF); -INSTANTIATE_THEM(GparityWilsonImplD); INSTANTIATE_THEM(DomainWallVec5dImplF); INSTANTIATE_THEM(DomainWallVec5dImplD); INSTANTIATE_THEM(ZDomainWallVec5dImplF); @@ -940,12 +621,11 @@ INSTANTIATE_THEM(WilsonImplFH); INSTANTIATE_THEM(WilsonImplDF); INSTANTIATE_THEM(ZWilsonImplFH); INSTANTIATE_THEM(ZWilsonImplDF); -INSTANTIATE_THEM(GparityWilsonImplFH); -INSTANTIATE_THEM(GparityWilsonImplDF); INSTANTIATE_THEM(DomainWallVec5dImplFH); INSTANTIATE_THEM(DomainWallVec5dImplDF); INSTANTIATE_THEM(ZDomainWallVec5dImplFH); INSTANTIATE_THEM(ZDomainWallVec5dImplDF); INSTANTIATE_THEM(WilsonTwoIndexAntiSymmetricImplF); INSTANTIATE_THEM(WilsonTwoIndexAntiSymmetricImplD); + }}