portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2025-10-31 03:54:33 +00:00
Code Releases Activity

Updated to have perfect prefetching for the s-vectorised kernel with any cache blocking.

Browse Source
This commit is contained in:
paboyle
2016-06-30 13:07:42 -07:00
parent 8fcefc021a
commit bdaa5b1767
3 changed files with 119 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/Stencil.h
View File

@@ -272,6 +272,11 @@
if (local) return base + _entries[ent]._byte_offset;
else return _entries[ent]._byte_offset;
}
inline uint64_t GetPFInfo(int ent,uint64_t base) {
int local = _entries[ent]._is_local;
if (local) return base + _entries[ent]._byte_offset;
else return _entries[ent]._byte_offset;
}
// Comms buffers
std::vector<Vector<scalar_object> > u_simd_send_buf;

149
lib/qcd/action/fermion/WilsonKernelsAsmBody.h
View File

@@ -1,43 +1,44 @@
{
int locala,perma, ptypea;
int localb,permb, ptypeb;
int localc,permc, ptypec;
uint64_t basea, baseb, basec;
int local,perm, ptype;
uint64_t base;
uint64_t basep;
const uint64_t plocal =(uint64_t) & in._odata[0];
// vComplexF isigns[2] = { signs[0], signs[1] };
vComplexF *isigns = &signs[0];
MASK_REGS;
int nmax=U._grid->oSites();
for(int site=0;site<Ns;site++) {
int sU =lo.Reorder(ssU);
int ssn=ssU+1;
if(ssn>=nmax) ssn=0;
int sUn=lo.Reorder(ssn);
for(int s=0;s<Ls;s++) {
ss =sU*Ls+s;
ssn=sUn*Ls+s;
////////////////////////////////
// Xp
////////////////////////////////
int ent=ss*8;// 2*Ndim
basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++;
PREFETCH1_CHIMU(basea);
int nent=ssn*8;
PF_GAUGE(Xp);
base = st.GetInfo(ptype,local,perm,Xp,ent,plocal); ent++;
PREFETCH1_CHIMU(base);
baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
PREFETCH_CHIMU(baseb);
basec = st.GetInfo(ptypec,localc,permc,Zp,ent,plocal); ent++;
PREFETCH_CHIMU(basec);
if ( locala ) {
basep = st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns);
XM_PROJMEM(basea);
MAYBEPERM(PERMUTE_DIR3,perma);
XM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR3,perm);
} else {
LOAD_CHI(basea);
LOAD_CHI(base);
}
base = st.GetInfo(ptype,local,perm,Yp,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFXP(Xp,baseb);
MULT_2SPIN_DIR_PFXP(Xp,basep);
}
LOAD64(%r10,isigns);
XM_RECON;
@@ -45,17 +46,18 @@
////////////////////////////////
// Yp
////////////////////////////////
basea = st.GetInfo(ptypea,locala,perma,Tp,ent,plocal); ent++;
PREFETCH_CHIMU(basea);
if ( localb ) {
basep = st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YM_PROJMEM(baseb);
MAYBEPERM(PERMUTE_DIR2,permb);
YM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR2,perm);
} else {
LOAD_CHI(baseb);
LOAD_CHI(base);
}
base = st.GetInfo(ptype,local,perm,Zp,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFYP(Yp,basec);
MULT_2SPIN_DIR_PFYP(Yp,basep);
}
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YM_RECON_ACCUM;
@@ -63,17 +65,18 @@
////////////////////////////////
// Zp
////////////////////////////////
baseb = st.GetInfo(ptypeb,localb,permb,Xm,ent,plocal); ent++;
PREFETCH_CHIMU(baseb);
if ( localc ) {
basep = st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZM_PROJMEM(basec);
MAYBEPERM(PERMUTE_DIR1,permc);
ZM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR1,perm);
} else {
LOAD_CHI(basec);
LOAD_CHI(base);
}
base = st.GetInfo(ptype,local,perm,Tp,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFZP(Zp,basea);
MULT_2SPIN_DIR_PFZP(Zp,basep);
}
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZM_RECON_ACCUM;
@@ -81,17 +84,18 @@
////////////////////////////////
// Tp
////////////////////////////////
basec = st.GetInfo(ptypec,localc,permc,Ym,ent,plocal); ent++;
PREFETCH_CHIMU(basec);
if ( locala ) {
basep = st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TM_PROJMEM(basea);
MAYBEPERM(PERMUTE_DIR0,perma);
TM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR0,perm);
} else {
LOAD_CHI(basea);
LOAD_CHI(base);
}
base = st.GetInfo(ptype,local,perm,Xm,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFTP(Tp,baseb);
MULT_2SPIN_DIR_PFTP(Tp,basep);
}
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TM_RECON_ACCUM;
@@ -99,17 +103,19 @@
////////////////////////////////
// Xm
////////////////////////////////
basea = st.GetInfo(ptypea,locala,perma,Zm,ent,plocal); ent++;
PREFETCH_CHIMU(basea);
if ( localb ) {
basep= (uint64_t) &out._odata[ss];
// basep= st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
XP_PROJMEM(baseb);
MAYBEPERM(PERMUTE_DIR3,permb);
XP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR3,perm);
} else {
LOAD_CHI(baseb);
LOAD_CHI(base);
}
base = st.GetInfo(ptype,local,perm,Ym,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFXM(Xm,basec);
MULT_2SPIN_DIR_PFXM(Xm,basep);
}
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
XP_RECON_ACCUM;
@@ -117,17 +123,18 @@
////////////////////////////////
// Ym
////////////////////////////////
baseb = st.GetInfo(ptypeb,localb,permb,Tm,ent,plocal); ent++;
PREFETCH_CHIMU(baseb);
if ( localc ) {
basep= st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YP_PROJMEM(basec);
MAYBEPERM(PERMUTE_DIR2,permc);
YP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR2,perm);
} else {
LOAD_CHI(basec);
LOAD_CHI(base);
}
base = st.GetInfo(ptype,local,perm,Zm,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFYM(Ym,basea);
MULT_2SPIN_DIR_PFYM(Ym,basep);
}
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YP_RECON_ACCUM;
@@ -135,17 +142,18 @@
////////////////////////////////
// Zm
////////////////////////////////
basec = (uint64_t)&out._odata[ss];
PREFETCH_CHIMU(basec);
if ( locala ) {
basep= st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZP_PROJMEM(basea);
MAYBEPERM(PERMUTE_DIR1,perma);
ZP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR1,perm);
} else {
LOAD_CHI(basea);
LOAD_CHI(base);
}
base = st.GetInfo(ptype,local,perm,Tm,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFZM(Zm,baseb);
MULT_2SPIN_DIR_PFZM(Zm,basep);
}
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZP_RECON_ACCUM;
@@ -153,23 +161,24 @@
////////////////////////////////
// Tm
////////////////////////////////
// basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++;
// PREFETCH_CHIMU(basea);
if ( localb ) {
basep= st.GetPFInfo(nent,plocal); nent++;
if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TP_PROJMEM(baseb);
MAYBEPERM(PERMUTE_DIR0,permb);
TP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR0,perm);
} else {
LOAD_CHI(baseb);
LOAD_CHI(base);
}
base= (uint64_t) &out._odata[ss];
PREFETCH_CHIMU(base);
{
MULT_2SPIN_DIR_PFTM(Tm,basec);
MULT_2SPIN_DIR_PFTM(Tm,basep);
}
// baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TP_RECON_ACCUM;
SAVE_RESULT(&out._odata[ss],basec);
basep= st.GetPFInfo(nent,plocal); nent++;
SAVE_RESULT(base,basep);
}
ssU++;

56
lib/simd/Intel512wilson.h
View File

@@ -261,8 +261,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
#define XM_PROJMEM(PTR) \
LOAD64(%r8,PTR)\
__asm__ ( \
SHUF_CHIMU23i \
LOAD_CHIi \
SHUF_CHIMU23i \
VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_30)\
VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_31)\
VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_32)\
@@ -290,8 +290,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
#define ZM_PROJMEM(PTR) \
LOAD64(%r8,PTR) \
__asm__ ( \
SHUF_CHIMU23i \
LOAD_CHIi \
SHUF_CHIMU23i \
VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_20)\
VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_21)\
VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_22)\
@@ -549,23 +549,24 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
#undef AVX512_PF_L2_LINEAR
#ifdef AVX512_PF_L2_TABLE
// P1 Fetches the base pointer for next link into L1 with P1
// M1 Fetches the next site pointer into L2
#define VPREFETCH_P1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_P2(A,B) VPREFETCH1(A,B)
#else
#define VPREFETCH_P2(A,B)
#define VPREFETCH_M1(A,B) VPREFETCH2(A,B)
#define VPREFETCH_M2(A,B)
#endif
#ifdef AVX512_PF_L2_LINEAR
#define VPREFETCH_M1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_M2(A,B) VPREFETCH2(A,B)
#define VPREFETCH_P1(A,B)
#define VPREFETCH_P2(A,B)
#endif
#ifdef AVX512_PF_L2_LINEAR
#define VPREFETCH_M1(A,B)
#define VPREFETCH_M2(A,B)
#else
#define VPREFETCH_M1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_M2(A,B) VPREFETCH2(A,B)
#endif
#ifdef AVX512_PF_L2_GAUGE
#define VPREFETCH_G1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_G2(A,B) VPREFETCH2(A,B)
#else
#endif
#define PF_GAUGE(A) \
@@ -593,21 +594,26 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
VSTORE(11,%r8,result_32) VPREFETCH_M1(11,%r9) \
);
#ifdef AVX512_PF_L2_TABLE
#define PREFETCH_CHIMU(A) \
LOAD64(%r9,A) \
__asm__ ( \
VPREFETCH_P2(0,%r9) \
VPREFETCH_P2(1,%r9) \
VPREFETCH_P2(2,%r9) \
VPREFETCH_P2(3,%r9) \
VPREFETCH_P2(4,%r9) \
VPREFETCH_P2(5,%r9) \
VPREFETCH_P2(6,%r9) \
VPREFETCH_P2(7,%r9) \
VPREFETCH_P2(8,%r9) \
VPREFETCH_P2(9,%r9) \
VPREFETCH_P2(10,%r9) \
VPREFETCH_P2(11,%r9));
VPREFETCH_P1(0,%r9) \
VPREFETCH_P1(1,%r9) \
VPREFETCH_P1(2,%r9) \
VPREFETCH_P1(3,%r9) \
VPREFETCH_P1(4,%r9) \
VPREFETCH_P1(5,%r9) \
VPREFETCH_P1(6,%r9) \
VPREFETCH_P1(7,%r9) \
VPREFETCH_P1(8,%r9) \
VPREFETCH_P1(9,%r9) \
VPREFETCH_P1(10,%r9) \
VPREFETCH_P1(11,%r9));
#else
#define PREFETCH_CHIMU(A)
#endif
#define PREFETCH1_CHIMU(A) \
LOAD64(%r9,A) \
@@ -811,6 +817,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
VPREFETCH_G1(3,%r8) \
VPREFETCH_G2(4,%r8) \
VPREFETCH_G2(5,%r8) \
VPREFETCH_G2(6,%r8) \
VPREFETCH_G2(7,%r8) \
/*42 insns*/ );
#define MULT_ADDSUB_2SPIN_LSNOPF(ptr,pf) \