mirror of
https://github.com/paboyle/Grid.git
synced 2024-09-20 09:15:38 +01:00
Move to implementation subdir
This commit is contained in:
Peter Boyle
parent
567aa5f366
commit
778450e0c8
198
Grid/qcd/action/fermion/implementation/WilsonKernelsAsmBody.h
Normal file
198
Grid/qcd/action/fermion/implementation/WilsonKernelsAsmBody.h
Normal file
@ -0,0 +1,198 @@
|
||||
#ifdef KERNEL_DAG
|
||||
#define DIR0_PROJMEM(base) XP_PROJMEM(base);
|
||||
#define DIR1_PROJMEM(base) YP_PROJMEM(base);
|
||||
#define DIR2_PROJMEM(base) ZP_PROJMEM(base);
|
||||
#define DIR3_PROJMEM(base) TP_PROJMEM(base);
|
||||
#define DIR4_PROJMEM(base) XM_PROJMEM(base);
|
||||
#define DIR5_PROJMEM(base) YM_PROJMEM(base);
|
||||
#define DIR6_PROJMEM(base) ZM_PROJMEM(base);
|
||||
#define DIR7_PROJMEM(base) TM_PROJMEM(base);
|
||||
#define DIR0_RECON XP_RECON
|
||||
#define DIR1_RECON YP_RECON_ACCUM
|
||||
#define DIR2_RECON ZP_RECON_ACCUM
|
||||
#define DIR3_RECON TP_RECON_ACCUM
|
||||
#define DIR4_RECON XM_RECON_ACCUM
|
||||
#define DIR5_RECON YM_RECON_ACCUM
|
||||
#define DIR6_RECON ZM_RECON_ACCUM
|
||||
#define DIR7_RECON TM_RECON_ACCUM
|
||||
#else
|
||||
#define DIR0_PROJMEM(base) XM_PROJMEM(base);
|
||||
#define DIR1_PROJMEM(base) YM_PROJMEM(base);
|
||||
#define DIR2_PROJMEM(base) ZM_PROJMEM(base);
|
||||
#define DIR3_PROJMEM(base) TM_PROJMEM(base);
|
||||
#define DIR4_PROJMEM(base) XP_PROJMEM(base);
|
||||
#define DIR5_PROJMEM(base) YP_PROJMEM(base);
|
||||
#define DIR6_PROJMEM(base) ZP_PROJMEM(base);
|
||||
#define DIR7_PROJMEM(base) TP_PROJMEM(base);
|
||||
#define DIR0_RECON XM_RECON
|
||||
#define DIR1_RECON YM_RECON_ACCUM
|
||||
#define DIR2_RECON ZM_RECON_ACCUM
|
||||
#define DIR3_RECON TM_RECON_ACCUM
|
||||
#define DIR4_RECON XP_RECON_ACCUM
|
||||
#define DIR5_RECON YP_RECON_ACCUM
|
||||
#define DIR6_RECON ZP_RECON_ACCUM
|
||||
#define DIR7_RECON TP_RECON_ACCUM
|
||||
#endif
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Comms then compute kernel
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef INTERIOR_AND_EXTERIOR
|
||||
|
||||
#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) \
|
||||
basep = st.GetPFInfo(nent,plocal); nent++; \
|
||||
if ( local ) { \
|
||||
LOAD64(%r10,isigns); \
|
||||
PROJ(base); \
|
||||
MAYBEPERM(PERMUTE_DIR,perm); \
|
||||
} else { \
|
||||
LOAD_CHI(base); \
|
||||
} \
|
||||
base = st.GetInfo(ptype,local,perm,NxtDir,ent,plocal); ent++; \
|
||||
PREFETCH_CHIMU(base); \
|
||||
MULT_2SPIN_DIR_PF(Dir,basep); \
|
||||
LOAD64(%r10,isigns); \
|
||||
RECON; \
|
||||
|
||||
#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) \
|
||||
base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++; \
|
||||
PF_GAUGE(Xp); \
|
||||
PREFETCH1_CHIMU(base); \
|
||||
ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)
|
||||
|
||||
#define RESULT(base,basep) SAVE_RESULT(base,basep);
|
||||
|
||||
#endif
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Pre comms kernel -- prefetch like normal because it is mostly right
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef INTERIOR
|
||||
|
||||
#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) \
|
||||
basep = st.GetPFInfo(nent,plocal); nent++; \
|
||||
if ( local ) { \
|
||||
LOAD64(%r10,isigns); \
|
||||
PROJ(base); \
|
||||
MAYBEPERM(PERMUTE_DIR,perm); \
|
||||
}else if ( st.same_node[Dir] ) {LOAD_CHI(base);} \
|
||||
if ( local || st.same_node[Dir] ) { \
|
||||
MULT_2SPIN_DIR_PF(Dir,basep); \
|
||||
LOAD64(%r10,isigns); \
|
||||
RECON; \
|
||||
} \
|
||||
base = st.GetInfo(ptype,local,perm,NxtDir,ent,plocal); ent++; \
|
||||
PREFETCH_CHIMU(base); \
|
||||
|
||||
#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) \
|
||||
base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++; \
|
||||
PF_GAUGE(Xp); \
|
||||
PREFETCH1_CHIMU(base); \
|
||||
{ ZERO_PSI; } \
|
||||
ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)
|
||||
|
||||
#define RESULT(base,basep) SAVE_RESULT(base,basep);
|
||||
|
||||
#endif
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Post comms kernel
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef EXTERIOR
|
||||
|
||||
|
||||
#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) \
|
||||
base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++; \
|
||||
if((!local)&&(!st.same_node[Dir]) ) { \
|
||||
LOAD_CHI(base); \
|
||||
MULT_2SPIN_DIR_PF(Dir,base); \
|
||||
LOAD64(%r10,isigns); \
|
||||
RECON; \
|
||||
nmu++; \
|
||||
}
|
||||
|
||||
#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) \
|
||||
nmu=0; \
|
||||
{ ZERO_PSI;} \
|
||||
base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++; \
|
||||
if((!local)&&(!st.same_node[Dir]) ) { \
|
||||
LOAD_CHI(base); \
|
||||
MULT_2SPIN_DIR_PF(Dir,base); \
|
||||
LOAD64(%r10,isigns); \
|
||||
RECON; \
|
||||
nmu++; \
|
||||
}
|
||||
|
||||
#define RESULT(base,basep) if (nmu){ ADD_RESULT(base,base);}
|
||||
|
||||
#endif
|
||||
{
|
||||
int nmu;
|
||||
int local,perm, ptype;
|
||||
uint64_t base;
|
||||
uint64_t basep;
|
||||
const uint64_t plocal =(uint64_t) & in[0];
|
||||
|
||||
COMPLEX_SIGNS(isigns);
|
||||
MASK_REGS;
|
||||
int nmax=U.oSites();
|
||||
for(int site=0;site<Ns;site++) {
|
||||
#ifndef EXTERIOR
|
||||
// int sU =lo.Reorder(ssU);
|
||||
int sU =ssU;
|
||||
int ssn=ssU+1; if(ssn>=nmax) ssn=0;
|
||||
// int sUn=lo.Reorder(ssn);
|
||||
int sUn=ssn;
|
||||
LOCK_GAUGE(0);
|
||||
#else
|
||||
int sU =ssU;
|
||||
int ssn=ssU+1; if(ssn>=nmax) ssn=0;
|
||||
int sUn=ssn;
|
||||
#endif
|
||||
for(int s=0;s<Ls;s++) {
|
||||
ss =sU*Ls+s;
|
||||
ssn=sUn*Ls+s;
|
||||
int ent=ss*8;// 2*Ndim
|
||||
int nent=ssn*8;
|
||||
|
||||
ASM_LEG_XP(Xp,Yp,PERMUTE_DIR3,DIR0_PROJMEM,DIR0_RECON);
|
||||
ASM_LEG(Yp,Zp,PERMUTE_DIR2,DIR1_PROJMEM,DIR1_RECON);
|
||||
ASM_LEG(Zp,Tp,PERMUTE_DIR1,DIR2_PROJMEM,DIR2_RECON);
|
||||
ASM_LEG(Tp,Xm,PERMUTE_DIR0,DIR3_PROJMEM,DIR3_RECON);
|
||||
|
||||
ASM_LEG(Xm,Ym,PERMUTE_DIR3,DIR4_PROJMEM,DIR4_RECON);
|
||||
ASM_LEG(Ym,Zm,PERMUTE_DIR2,DIR5_PROJMEM,DIR5_RECON);
|
||||
ASM_LEG(Zm,Tm,PERMUTE_DIR1,DIR6_PROJMEM,DIR6_RECON);
|
||||
ASM_LEG(Tm,Xp,PERMUTE_DIR0,DIR7_PROJMEM,DIR7_RECON);
|
||||
|
||||
#ifdef EXTERIOR
|
||||
if (nmu==0) break;
|
||||
// if (nmu!=0) std::cout << "EXT "<<sU<<std::endl;
|
||||
#endif
|
||||
base = (uint64_t) &out[ss];
|
||||
basep= st.GetPFInfo(nent,plocal); nent++;
|
||||
RESULT(base,basep);
|
||||
}
|
||||
ssU++;
|
||||
UNLOCK_GAUGE(0);
|
||||
}
|
||||
}
|
||||
|
||||
#undef DIR0_PROJMEM
|
||||
#undef DIR1_PROJMEM
|
||||
#undef DIR2_PROJMEM
|
||||
#undef DIR3_PROJMEM
|
||||
#undef DIR4_PROJMEM
|
||||
#undef DIR5_PROJMEM
|
||||
#undef DIR6_PROJMEM
|
||||
#undef DIR7_PROJMEM
|
||||
#undef DIR0_RECON
|
||||
#undef DIR1_RECON
|
||||
#undef DIR2_RECON
|
||||
#undef DIR3_RECON
|
||||
#undef DIR4_RECON
|
||||
#undef DIR5_RECON
|
||||
#undef DIR6_RECON
|
||||
#undef DIR7_RECON
|
||||
#undef ASM_LEG
|
||||
#undef ASM_LEG_XP
|
||||
#undef RESULT
|
||||
161
Grid/qcd/action/fermion/implementation/WilsonKernelsAsmBody.h.ab
Normal file
161
Grid/qcd/action/fermion/implementation/WilsonKernelsAsmBody.h.ab
Normal file
@ -0,0 +1,161 @@
|
||||
{
|
||||
int locala,perma, ptypea;
|
||||
int localb,permb, ptypeb;
|
||||
uint64_t basea, baseb;
|
||||
const uint64_t plocal =(uint64_t) & in._odata[0];
|
||||
|
||||
// vComplexF isigns[2] = { signs[0], signs[1] };
|
||||
vComplexF *isigns = &signs[0];
|
||||
|
||||
MASK_REGS;
|
||||
|
||||
for(int site=0;site<Ns;site++) {
|
||||
int sU=lo.Reorder(ssU);
|
||||
for(int s=0;s<Ls;s++) {
|
||||
ss=sU*Ls+s;
|
||||
////////////////////////////////
|
||||
// Xp
|
||||
////////////////////////////////
|
||||
int ent=ss*8;// 2*Ndim
|
||||
basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++;
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
|
||||
|
||||
if ( locala ) {
|
||||
LOAD64(%r10,isigns);
|
||||
XM_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR3,perma);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFXP(Xp,baseb);
|
||||
}
|
||||
LOAD64(%r10,isigns);
|
||||
XM_RECON;
|
||||
|
||||
////////////////////////////////
|
||||
// Yp
|
||||
////////////////////////////////
|
||||
basea = st.GetInfo(ptypea,locala,perma,Zp,ent,plocal); ent++;
|
||||
if ( localb ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YM_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR2,permb);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFYP(Yp,basea);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YM_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Zp
|
||||
////////////////////////////////
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Tp,ent,plocal); ent++;
|
||||
if ( locala ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZM_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR1,perma);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFZP(Zp,baseb);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZM_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Tp
|
||||
////////////////////////////////
|
||||
basea = st.GetInfo(ptypea,locala,perma,Xm,ent,plocal); ent++;
|
||||
if ( localb ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TM_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR0,permb);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFTP(Tp,basea);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TM_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Xm
|
||||
////////////////////////////////
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Ym,ent,plocal); ent++;
|
||||
if ( locala ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
XP_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR3,perma);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFXM(Xm,baseb);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
XP_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Ym
|
||||
////////////////////////////////
|
||||
basea = st.GetInfo(ptypea,locala,perma,Zm,ent,plocal); ent++;
|
||||
if ( localb ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YP_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR2,permb);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFYM(Ym,basea);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YP_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Zm
|
||||
////////////////////////////////
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Tm,ent,plocal); ent++;
|
||||
if ( locala ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZP_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR1,perma);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFZM(Zm,baseb);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZP_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Tm
|
||||
////////////////////////////////
|
||||
basea = (uint64_t)&out._odata[ss];
|
||||
if ( localb ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TP_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR0,permb);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
}
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Xp,ent,plocal);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFTM(Tm,basea);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TP_RECON_ACCUM;
|
||||
|
||||
SAVE_RESULT(&out._odata[ss],baseb);
|
||||
|
||||
}
|
||||
ssU++;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,187 @@
|
||||
{
|
||||
int locala,perma, ptypea;
|
||||
int localb,permb, ptypeb;
|
||||
int localc,permc, ptypec;
|
||||
uint64_t basea, baseb, basec;
|
||||
uint64_t basex;
|
||||
const uint64_t plocal =(uint64_t) & in._odata[0];
|
||||
|
||||
// vComplexF isigns[2] = { signs[0], signs[1] };
|
||||
vComplexF *isigns = &signs[0];
|
||||
|
||||
MASK_REGS;
|
||||
|
||||
for(int site=0;site<Ns;site++) {
|
||||
int sU=lo.Reorder(ssU);
|
||||
|
||||
for(int s=0;s<Ls;s++) {
|
||||
ss =sU*Ls+s;
|
||||
|
||||
////////////////////////////////
|
||||
// Xp
|
||||
////////////////////////////////
|
||||
int ent=ss*8;// 2*Ndim
|
||||
basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basea);
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(baseb);
|
||||
basec = st.GetInfo(ptypec,localc,permc,Zp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basec);
|
||||
|
||||
basex = basea;
|
||||
|
||||
label(FX(XP) );
|
||||
if ( locala ) {
|
||||
LOAD64(%r10,isigns);
|
||||
XM_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR3,perma);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFXP(Xp,baseb);
|
||||
}
|
||||
LOAD64(%r10,isigns);
|
||||
XM_RECON;
|
||||
|
||||
////////////////////////////////
|
||||
// Yp
|
||||
////////////////////////////////
|
||||
basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basea);
|
||||
label(FX(YP) );
|
||||
if ( localb ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YM_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR2,permb);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFYP(Yp,basec);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YM_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Zp
|
||||
////////////////////////////////
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(baseb);
|
||||
label(FX(ZP) );
|
||||
if ( localc ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZM_PROJMEM(basec);
|
||||
MAYBEPERM(PERMUTE_DIR1,permc);
|
||||
} else {
|
||||
LOAD_CHI(basec);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFZP(Zp,basea);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZM_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Tp
|
||||
////////////////////////////////
|
||||
basec = st.GetInfo(ptypec,localc,permc,Xp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basec);
|
||||
label(FX(TP) );
|
||||
if ( locala ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TM_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR0,perma);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFTP(Tp,baseb);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TM_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Xm
|
||||
////////////////////////////////
|
||||
basea = st.GetInfo(ptypea,locala,perma,Yp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basea);
|
||||
label(FX(XM) );
|
||||
if ( localb ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
XP_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR3,permb);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFXM(Xm,basec);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
XP_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Ym
|
||||
////////////////////////////////
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Xp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(baseb);
|
||||
label(FX(YM) );
|
||||
if ( localc ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YP_PROJMEM(basec);
|
||||
MAYBEPERM(PERMUTE_DIR2,permc);
|
||||
} else {
|
||||
LOAD_CHI(basec);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFYM(Ym,basea);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YP_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Zm
|
||||
////////////////////////////////
|
||||
basec = st.GetInfo(ptypec,localc,permc,Yp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basec);
|
||||
label(FX(ZM) );
|
||||
if ( locala ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZP_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR1,perma);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFZM(Zm,baseb);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZP_RECON_ACCUM;
|
||||
|
||||
////////////////////////////////
|
||||
// Tm
|
||||
////////////////////////////////
|
||||
basea = (uint64_t)&out._odata[ss];
|
||||
PREFETCH_CHIMU(basea);
|
||||
label(FX(TM) );
|
||||
if ( localb ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TP_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR0,permb);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
}
|
||||
{
|
||||
MULT_2SPIN_DIR_PFTM(Tm,basec);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TP_RECON_ACCUM;
|
||||
|
||||
// PREFETCH_CHIMU(basex);
|
||||
label(FX(SAV) );
|
||||
SAVE_RESULT(&out._odata[ss]);
|
||||
|
||||
}
|
||||
ssU++;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,127 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
|
||||
|
||||
Source file: ./lib/qcd/action/fermion/WilsonKernelsAsm.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Guido Cossu <guido.cossu@ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <Grid/qcd/action/fermion/FermionCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
// Default to no assembler implementation
|
||||
///////////////////////////////////////////////////////////
|
||||
template<class Impl> void
|
||||
WilsonKernels<Impl >::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
template<class Impl> void
|
||||
WilsonKernels<Impl >::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
template<class Impl> void
|
||||
WilsonKernels<Impl >::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
template<class Impl> void
|
||||
WilsonKernels<Impl >::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
template<class Impl> void
|
||||
WilsonKernels<Impl >::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
template<class Impl> void
|
||||
WilsonKernels<Impl >::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmAvx512.h>
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmQPX.h>
|
||||
|
||||
#define INSTANTIATE_ASM(A) \
|
||||
template void WilsonKernels<A>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,\
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out);\
|
||||
\
|
||||
template void WilsonKernels<A>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,\
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out);\
|
||||
template void WilsonKernels<A>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,\
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out);\
|
||||
\
|
||||
template void WilsonKernels<A>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,\
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out);\
|
||||
template void WilsonKernels<A>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,\
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out);\
|
||||
\
|
||||
template void WilsonKernels<A>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,\
|
||||
int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out);\
|
||||
|
||||
//INSTANTIATE_ASM(WilsonImplF);
|
||||
//INSTANTIATE_ASM(WilsonImplD);
|
||||
INSTANTIATE_ASM(GparityWilsonImplF);
|
||||
INSTANTIATE_ASM(GparityWilsonImplD);
|
||||
//INSTANTIATE_ASM(ZWilsonImplF);
|
||||
//INSTANTIATE_ASM(ZWilsonImplD);
|
||||
//INSTANTIATE_ASM(DomainWallVec5dImplF);
|
||||
//INSTANTIATE_ASM(DomainWallVec5dImplD);
|
||||
//INSTANTIATE_ASM(ZDomainWallVec5dImplF);
|
||||
//INSTANTIATE_ASM(ZDomainWallVec5dImplD);
|
||||
|
||||
//INSTANTIATE_ASM(WilsonImplFH);
|
||||
//INSTANTIATE_ASM(WilsonImplDF);
|
||||
//INSTANTIATE_ASM(ZWilsonImplFH);
|
||||
//INSTANTIATE_ASM(ZWilsonImplDF);
|
||||
INSTANTIATE_ASM(GparityWilsonImplFH);
|
||||
INSTANTIATE_ASM(GparityWilsonImplDF);
|
||||
//INSTANTIATE_ASM(DomainWallVec5dImplFH);
|
||||
//INSTANTIATE_ASM(DomainWallVec5dImplDF);
|
||||
//INSTANTIATE_ASM(ZDomainWallVec5dImplFH);
|
||||
//INSTANTIATE_ASM(ZDomainWallVec5dImplDF);
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
150
Grid/qcd/action/fermion/implementation/WilsonKernelsAsmQPX.h
Normal file
150
Grid/qcd/action/fermion/implementation/WilsonKernelsAsmQPX.h
Normal file
@ -0,0 +1,150 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
|
||||
|
||||
Source file: ./lib/qcd/action/fermion/WilsonKernelsAsmQPX.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
#if defined(QPX)
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
// If we are QPX specialise the single precision routine
|
||||
///////////////////////////////////////////////////////////
|
||||
|
||||
#include <simd/IBM_qpx.h>
|
||||
#include <simd/IBM_qpx_single.h>
|
||||
|
||||
#define MAYBEPERM(A,perm) if (perm) { A ; }
|
||||
#define MULT_2SPIN(ptr,pf) MULT_2SPIN_QPX(ptr,pf)
|
||||
#define COMPLEX_SIGNS(isigns)
|
||||
|
||||
#define INTERIOR_AND_EXTERIOR
|
||||
#undef INTERIOR
|
||||
#undef EXTERIOR
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// XYZT vectorised, undag Kernel, single
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#undef KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<WilsonImplF>::AsmDhopSite(StencilView &st, DoubledGaugeField &U, SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// XYZT vectorised, dag Kernel, single
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#define KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<WilsonImplF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeField &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
|
||||
#undef MAYBEPERM
|
||||
#undef MULT_2SPIN
|
||||
#define MAYBEPERM(A,B)
|
||||
#define MULT_2SPIN(ptr,pf) MULT_2SPIN_QPX_LS(ptr,pf)
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Ls vectorised, undag Kernel, single
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#undef KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<DomainWallVec5dImplF>::AsmDhopSite(StencilView &st, DoubledGaugeField &U, SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Ls vectorised, dag Kernel, single
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#define KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<DomainWallVec5dImplF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeField &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
#undef MAYBEPERM
|
||||
#undef MULT_2SPIN
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
// DP routines
|
||||
///////////////////////////////////////////////////////////
|
||||
|
||||
#include <simd/IBM_qpx_double.h>
|
||||
|
||||
#define MAYBEPERM(A,perm) if (perm) { A ; }
|
||||
#define MULT_2SPIN(ptr,pf) MULT_2SPIN_QPX(ptr,pf)
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// XYZT Vectorised, undag Kernel, double
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#undef KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<WilsonImplD>::AsmDhopSite(StencilView &st, DoubledGaugeField &U, SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
/////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// XYZT Vectorised, dag Kernel, double
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#define KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<WilsonImplD>::AsmDhopSiteDag(StencilView &st, DoubledGaugeField &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
/////////////////////////////////////////////////////////////////
|
||||
|
||||
#undef MAYBEPERM
|
||||
#undef MULT_2SPIN
|
||||
#define MAYBEPERM(A,B)
|
||||
#define MULT_2SPIN(ptr,pf) MULT_2SPIN_QPX_LS(ptr,pf)
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Ls vectorised, undag Kernel, double
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#undef KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<DomainWallVec5dImplD>::AsmDhopSite(StencilView &st, DoubledGaugeField &U, SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
/////////////////////////////////////////////////////////////////
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Ls vectorised, dag Kernel, double
|
||||
/////////////////////////////////////////////////////////////////
|
||||
#define KERNEL_DAG
|
||||
template<> void
|
||||
WilsonKernels<DomainWallVec5dImplD>::AsmDhopSiteDag(StencilView &st, DoubledGaugeField &U,SiteHalfSpinor *buf,
|
||||
int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
|
||||
#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
|
||||
/////////////////////////////////////////////////////////////////
|
||||
|
||||
#undef MAYBEPERM
|
||||
#undef MULT_2SPIN
|
||||
|
||||
#endif
|
||||
Loading…
Reference in New Issue
Block a user