mirror of
https://github.com/paboyle/Grid.git
synced 2025-06-11 03:46:55 +01:00
Updated to have perfect prefetching for the s-vectorised kernel with any cache blocking.
This commit is contained in:
paboyle
@ -1,43 +1,44 @@
|
||||
{
|
||||
int locala,perma, ptypea;
|
||||
int localb,permb, ptypeb;
|
||||
int localc,permc, ptypec;
|
||||
uint64_t basea, baseb, basec;
|
||||
|
||||
int local,perm, ptype;
|
||||
uint64_t base;
|
||||
uint64_t basep;
|
||||
const uint64_t plocal =(uint64_t) & in._odata[0];
|
||||
|
||||
// vComplexF isigns[2] = { signs[0], signs[1] };
|
||||
vComplexF *isigns = &signs[0];
|
||||
|
||||
MASK_REGS;
|
||||
|
||||
int nmax=U._grid->oSites();
|
||||
for(int site=0;site<Ns;site++) {
|
||||
int sU =lo.Reorder(ssU);
|
||||
int ssn=ssU+1;
|
||||
if(ssn>=nmax) ssn=0;
|
||||
int sUn=lo.Reorder(ssn);
|
||||
for(int s=0;s<Ls;s++) {
|
||||
ss =sU*Ls+s;
|
||||
|
||||
ss =sU*Ls+s;
|
||||
ssn=sUn*Ls+s;
|
||||
////////////////////////////////
|
||||
// Xp
|
||||
////////////////////////////////
|
||||
int ent=ss*8;// 2*Ndim
|
||||
basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++;
|
||||
PREFETCH1_CHIMU(basea);
|
||||
int ent=ss*8;// 2*Ndim
|
||||
int nent=ssn*8;
|
||||
|
||||
PF_GAUGE(Xp);
|
||||
base = st.GetInfo(ptype,local,perm,Xp,ent,plocal); ent++;
|
||||
PREFETCH1_CHIMU(base);
|
||||
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(baseb);
|
||||
basec = st.GetInfo(ptypec,localc,permc,Zp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basec);
|
||||
|
||||
if ( locala ) {
|
||||
basep = st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns);
|
||||
XM_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR3,perma);
|
||||
XM_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR3,perm);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base = st.GetInfo(ptype,local,perm,Yp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFXP(Xp,baseb);
|
||||
MULT_2SPIN_DIR_PFXP(Xp,basep);
|
||||
}
|
||||
LOAD64(%r10,isigns);
|
||||
XM_RECON;
|
||||
@ -45,17 +46,18 @@
|
||||
////////////////////////////////
|
||||
// Yp
|
||||
////////////////////////////////
|
||||
basea = st.GetInfo(ptypea,locala,perma,Tp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basea);
|
||||
if ( localb ) {
|
||||
basep = st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YM_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR2,permb);
|
||||
YM_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR2,perm);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base = st.GetInfo(ptype,local,perm,Zp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFYP(Yp,basec);
|
||||
MULT_2SPIN_DIR_PFYP(Yp,basep);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YM_RECON_ACCUM;
|
||||
@ -63,17 +65,18 @@
|
||||
////////////////////////////////
|
||||
// Zp
|
||||
////////////////////////////////
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Xm,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(baseb);
|
||||
if ( localc ) {
|
||||
basep = st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZM_PROJMEM(basec);
|
||||
MAYBEPERM(PERMUTE_DIR1,permc);
|
||||
ZM_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR1,perm);
|
||||
} else {
|
||||
LOAD_CHI(basec);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base = st.GetInfo(ptype,local,perm,Tp,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFZP(Zp,basea);
|
||||
MULT_2SPIN_DIR_PFZP(Zp,basep);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZM_RECON_ACCUM;
|
||||
@ -81,17 +84,18 @@
|
||||
////////////////////////////////
|
||||
// Tp
|
||||
////////////////////////////////
|
||||
basec = st.GetInfo(ptypec,localc,permc,Ym,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basec);
|
||||
if ( locala ) {
|
||||
basep = st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TM_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR0,perma);
|
||||
TM_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR0,perm);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base = st.GetInfo(ptype,local,perm,Xm,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFTP(Tp,baseb);
|
||||
MULT_2SPIN_DIR_PFTP(Tp,basep);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TM_RECON_ACCUM;
|
||||
@ -99,17 +103,19 @@
|
||||
////////////////////////////////
|
||||
// Xm
|
||||
////////////////////////////////
|
||||
basea = st.GetInfo(ptypea,locala,perma,Zm,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(basea);
|
||||
if ( localb ) {
|
||||
basep= (uint64_t) &out._odata[ss];
|
||||
// basep= st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
XP_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR3,permb);
|
||||
XP_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR3,perm);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base = st.GetInfo(ptype,local,perm,Ym,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFXM(Xm,basec);
|
||||
MULT_2SPIN_DIR_PFXM(Xm,basep);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
XP_RECON_ACCUM;
|
||||
@ -117,17 +123,18 @@
|
||||
////////////////////////////////
|
||||
// Ym
|
||||
////////////////////////////////
|
||||
baseb = st.GetInfo(ptypeb,localb,permb,Tm,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(baseb);
|
||||
if ( localc ) {
|
||||
basep= st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YP_PROJMEM(basec);
|
||||
MAYBEPERM(PERMUTE_DIR2,permc);
|
||||
YP_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR2,perm);
|
||||
} else {
|
||||
LOAD_CHI(basec);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base = st.GetInfo(ptype,local,perm,Zm,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFYM(Ym,basea);
|
||||
MULT_2SPIN_DIR_PFYM(Ym,basep);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
YP_RECON_ACCUM;
|
||||
@ -135,17 +142,18 @@
|
||||
////////////////////////////////
|
||||
// Zm
|
||||
////////////////////////////////
|
||||
basec = (uint64_t)&out._odata[ss];
|
||||
PREFETCH_CHIMU(basec);
|
||||
if ( locala ) {
|
||||
basep= st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZP_PROJMEM(basea);
|
||||
MAYBEPERM(PERMUTE_DIR1,perma);
|
||||
ZP_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR1,perm);
|
||||
} else {
|
||||
LOAD_CHI(basea);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base = st.GetInfo(ptype,local,perm,Tm,ent,plocal); ent++;
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFZM(Zm,baseb);
|
||||
MULT_2SPIN_DIR_PFZM(Zm,basep);
|
||||
}
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
ZP_RECON_ACCUM;
|
||||
@ -153,23 +161,24 @@
|
||||
////////////////////////////////
|
||||
// Tm
|
||||
////////////////////////////////
|
||||
// basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++;
|
||||
// PREFETCH_CHIMU(basea);
|
||||
if ( localb ) {
|
||||
basep= st.GetPFInfo(nent,plocal); nent++;
|
||||
if ( local ) {
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TP_PROJMEM(baseb);
|
||||
MAYBEPERM(PERMUTE_DIR0,permb);
|
||||
TP_PROJMEM(base);
|
||||
MAYBEPERM(PERMUTE_DIR0,perm);
|
||||
} else {
|
||||
LOAD_CHI(baseb);
|
||||
LOAD_CHI(base);
|
||||
}
|
||||
base= (uint64_t) &out._odata[ss];
|
||||
PREFETCH_CHIMU(base);
|
||||
{
|
||||
MULT_2SPIN_DIR_PFTM(Tm,basec);
|
||||
MULT_2SPIN_DIR_PFTM(Tm,basep);
|
||||
}
|
||||
// baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
|
||||
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
|
||||
TP_RECON_ACCUM;
|
||||
|
||||
SAVE_RESULT(&out._odata[ss],basec);
|
||||
basep= st.GetPFInfo(nent,plocal); nent++;
|
||||
SAVE_RESULT(base,basep);
|
||||
|
||||
}
|
||||
ssU++;
|
||||
|
||||
Reference in New Issue
Block a user