portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2024-11-10 07:55:35 +00:00
Code Releases Activity

Updated to have perfect prefetching for the s-vectorised kernel with any cache blocking.

Browse Source
This commit is contained in:
paboyle 2016-06-30 13:07:42 -07:00
parent 8fcefc021a
commit bdaa5b1767
3 changed files with 119 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/Stencil.h
View File

@ -272,6 +272,11 @@
if (local) return base + _entries[ent]._byte_offset; if (local) return base + _entries[ent]._byte_offset;
else return _entries[ent]._byte_offset; else return _entries[ent]._byte_offset;
} }
inline uint64_t GetPFInfo(int ent,uint64_t base) {
int local = _entries[ent]._is_local;
if (local) return base + _entries[ent]._byte_offset;
else return _entries[ent]._byte_offset;
}
// Comms buffers // Comms buffers
std::vector<Vector<scalar_object> > u_simd_send_buf; std::vector<Vector<scalar_object> > u_simd_send_buf;

149
lib/qcd/action/fermion/WilsonKernelsAsmBody.h
View File

@ -1,43 +1,44 @@
{ {
int locala,perma, ptypea; int local,perm, ptype;
int localb,permb, ptypeb; uint64_t base;
int localc,permc, ptypec; uint64_t basep;
uint64_t basea, baseb, basec;
const uint64_t plocal =(uint64_t) & in._odata[0]; const uint64_t plocal =(uint64_t) & in._odata[0];
// vComplexF isigns[2] = { signs[0], signs[1] }; // vComplexF isigns[2] = { signs[0], signs[1] };
vComplexF *isigns = &signs[0]; vComplexF *isigns = &signs[0];
MASK_REGS; MASK_REGS;
int nmax=U._grid->oSites();
for(int site=0;site<Ns;site++) { for(int site=0;site<Ns;site++) {
int sU =lo.Reorder(ssU); int sU =lo.Reorder(ssU);
int ssn=ssU+1;
if(ssn>=nmax) ssn=0;
int sUn=lo.Reorder(ssn);
for(int s=0;s<Ls;s++) { for(int s=0;s<Ls;s++) {
ss =sU*Ls+s; ss =sU*Ls+s;
ssn=sUn*Ls+s;
//////////////////////////////// ////////////////////////////////
// Xp // Xp
//////////////////////////////// ////////////////////////////////
int ent=ss*8;// 2*Ndim int ent=ss*8;// 2*Ndim
basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++; int nent=ssn*8;
PREFETCH1_CHIMU(basea);
PF_GAUGE(Xp); PF_GAUGE(Xp);
base = st.GetInfo(ptype,local,perm,Xp,ent,plocal); ent++;
PREFETCH1_CHIMU(base);
baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++; basep = st.GetPFInfo(nent,plocal); nent++;
PREFETCH_CHIMU(baseb); if ( local ) {
basec = st.GetInfo(ptypec,localc,permc,Zp,ent,plocal); ent++;
PREFETCH_CHIMU(basec);
if ( locala ) {
LOAD64(%r10,isigns); LOAD64(%r10,isigns);
XM_PROJMEM(basea); XM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR3,perma); MAYBEPERM(PERMUTE_DIR3,perm);
} else { } else {
LOAD_CHI(basea); LOAD_CHI(base);
} }
base = st.GetInfo(ptype,local,perm,Yp,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFXP(Xp,baseb); MULT_2SPIN_DIR_PFXP(Xp,basep);
} }
LOAD64(%r10,isigns); LOAD64(%r10,isigns);
XM_RECON; XM_RECON;
@ -45,17 +46,18 @@
//////////////////////////////// ////////////////////////////////
// Yp // Yp
//////////////////////////////// ////////////////////////////////
basea = st.GetInfo(ptypea,locala,perma,Tp,ent,plocal); ent++; basep = st.GetPFInfo(nent,plocal); nent++;
PREFETCH_CHIMU(basea); if ( local ) {
if ( localb ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YM_PROJMEM(baseb); YM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR2,permb); MAYBEPERM(PERMUTE_DIR2,perm);
} else { } else {
LOAD_CHI(baseb); LOAD_CHI(base);
} }
base = st.GetInfo(ptype,local,perm,Zp,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFYP(Yp,basec); MULT_2SPIN_DIR_PFYP(Yp,basep);
} }
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YM_RECON_ACCUM; YM_RECON_ACCUM;
@ -63,17 +65,18 @@
//////////////////////////////// ////////////////////////////////
// Zp // Zp
//////////////////////////////// ////////////////////////////////
baseb = st.GetInfo(ptypeb,localb,permb,Xm,ent,plocal); ent++; basep = st.GetPFInfo(nent,plocal); nent++;
PREFETCH_CHIMU(baseb); if ( local ) {
if ( localc ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZM_PROJMEM(basec); ZM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR1,permc); MAYBEPERM(PERMUTE_DIR1,perm);
} else { } else {
LOAD_CHI(basec); LOAD_CHI(base);
} }
base = st.GetInfo(ptype,local,perm,Tp,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFZP(Zp,basea); MULT_2SPIN_DIR_PFZP(Zp,basep);
} }
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZM_RECON_ACCUM; ZM_RECON_ACCUM;
@ -81,17 +84,18 @@
//////////////////////////////// ////////////////////////////////
// Tp // Tp
//////////////////////////////// ////////////////////////////////
basec = st.GetInfo(ptypec,localc,permc,Ym,ent,plocal); ent++; basep = st.GetPFInfo(nent,plocal); nent++;
PREFETCH_CHIMU(basec); if ( local ) {
if ( locala ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TM_PROJMEM(basea); TM_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR0,perma); MAYBEPERM(PERMUTE_DIR0,perm);
} else { } else {
LOAD_CHI(basea); LOAD_CHI(base);
} }
base = st.GetInfo(ptype,local,perm,Xm,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFTP(Tp,baseb); MULT_2SPIN_DIR_PFTP(Tp,basep);
} }
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TM_RECON_ACCUM; TM_RECON_ACCUM;
@ -99,17 +103,19 @@
//////////////////////////////// ////////////////////////////////
// Xm // Xm
//////////////////////////////// ////////////////////////////////
basea = st.GetInfo(ptypea,locala,perma,Zm,ent,plocal); ent++; basep= (uint64_t) &out._odata[ss];
PREFETCH_CHIMU(basea); // basep= st.GetPFInfo(nent,plocal); nent++;
if ( localb ) { if ( local ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
XP_PROJMEM(baseb); XP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR3,permb); MAYBEPERM(PERMUTE_DIR3,perm);
} else { } else {
LOAD_CHI(baseb); LOAD_CHI(base);
} }
base = st.GetInfo(ptype,local,perm,Ym,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFXM(Xm,basec); MULT_2SPIN_DIR_PFXM(Xm,basep);
} }
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
XP_RECON_ACCUM; XP_RECON_ACCUM;
@ -117,17 +123,18 @@
//////////////////////////////// ////////////////////////////////
// Ym // Ym
//////////////////////////////// ////////////////////////////////
baseb = st.GetInfo(ptypeb,localb,permb,Tm,ent,plocal); ent++; basep= st.GetPFInfo(nent,plocal); nent++;
PREFETCH_CHIMU(baseb); if ( local ) {
if ( localc ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YP_PROJMEM(basec); YP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR2,permc); MAYBEPERM(PERMUTE_DIR2,perm);
} else { } else {
LOAD_CHI(basec); LOAD_CHI(base);
} }
base = st.GetInfo(ptype,local,perm,Zm,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFYM(Ym,basea); MULT_2SPIN_DIR_PFYM(Ym,basep);
} }
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
YP_RECON_ACCUM; YP_RECON_ACCUM;
@ -135,17 +142,18 @@
//////////////////////////////// ////////////////////////////////
// Zm // Zm
//////////////////////////////// ////////////////////////////////
basec = (uint64_t)&out._odata[ss]; basep= st.GetPFInfo(nent,plocal); nent++;
PREFETCH_CHIMU(basec); if ( local ) {
if ( locala ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZP_PROJMEM(basea); ZP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR1,perma); MAYBEPERM(PERMUTE_DIR1,perm);
} else { } else {
LOAD_CHI(basea); LOAD_CHI(base);
} }
base = st.GetInfo(ptype,local,perm,Tm,ent,plocal); ent++;
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFZM(Zm,baseb); MULT_2SPIN_DIR_PFZM(Zm,basep);
} }
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
ZP_RECON_ACCUM; ZP_RECON_ACCUM;
@ -153,23 +161,24 @@
//////////////////////////////// ////////////////////////////////
// Tm // Tm
//////////////////////////////// ////////////////////////////////
// basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++; basep= st.GetPFInfo(nent,plocal); nent++;
// PREFETCH_CHIMU(basea); if ( local ) {
if ( localb ) {
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TP_PROJMEM(baseb); TP_PROJMEM(base);
MAYBEPERM(PERMUTE_DIR0,permb); MAYBEPERM(PERMUTE_DIR0,perm);
} else { } else {
LOAD_CHI(baseb); LOAD_CHI(base);
} }
base= (uint64_t) &out._odata[ss];
PREFETCH_CHIMU(base);
{ {
MULT_2SPIN_DIR_PFTM(Tm,basec); MULT_2SPIN_DIR_PFTM(Tm,basep);
} }
// baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++;
LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit
TP_RECON_ACCUM; TP_RECON_ACCUM;
SAVE_RESULT(&out._odata[ss],basec); basep= st.GetPFInfo(nent,plocal); nent++;
SAVE_RESULT(base,basep);
} }
ssU++; ssU++;

56
lib/simd/Intel512wilson.h
View File

@ -261,8 +261,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
#define XM_PROJMEM(PTR) \ #define XM_PROJMEM(PTR) \
LOAD64(%r8,PTR)\ LOAD64(%r8,PTR)\
__asm__ ( \ __asm__ ( \
SHUF_CHIMU23i \
LOAD_CHIi \ LOAD_CHIi \
SHUF_CHIMU23i \
VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_30)\ VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_30)\
VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_31)\ VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_31)\
VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_32)\ VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_32)\
@ -290,8 +290,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
#define ZM_PROJMEM(PTR) \ #define ZM_PROJMEM(PTR) \
LOAD64(%r8,PTR) \ LOAD64(%r8,PTR) \
__asm__ ( \ __asm__ ( \
SHUF_CHIMU23i \
LOAD_CHIi \ LOAD_CHIi \
SHUF_CHIMU23i \
VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_20)\ VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_20)\
VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_21)\ VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_21)\
VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_22)\ VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_22)\
@ -549,23 +549,24 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
#undef AVX512_PF_L2_LINEAR #undef AVX512_PF_L2_LINEAR
#ifdef AVX512_PF_L2_TABLE #ifdef AVX512_PF_L2_TABLE
// P1 Fetches the base pointer for next link into L1 with P1
// M1 Fetches the next site pointer into L2
#define VPREFETCH_P1(A,B) VPREFETCH1(A,B) #define VPREFETCH_P1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_P2(A,B) VPREFETCH1(A,B) #define VPREFETCH_P2(A,B)
#else #define VPREFETCH_M1(A,B) VPREFETCH2(A,B)
#define VPREFETCH_M2(A,B)
#endif
#ifdef AVX512_PF_L2_LINEAR
#define VPREFETCH_M1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_M2(A,B) VPREFETCH2(A,B)
#define VPREFETCH_P1(A,B) #define VPREFETCH_P1(A,B)
#define VPREFETCH_P2(A,B) #define VPREFETCH_P2(A,B)
#endif #endif
#ifdef AVX512_PF_L2_LINEAR
#define VPREFETCH_M1(A,B)
#define VPREFETCH_M2(A,B)
#else
#define VPREFETCH_M1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_M2(A,B) VPREFETCH2(A,B)
#endif
#ifdef AVX512_PF_L2_GAUGE #ifdef AVX512_PF_L2_GAUGE
#define VPREFETCH_G1(A,B) VPREFETCH1(A,B) #define VPREFETCH_G1(A,B) VPREFETCH1(A,B)
#define VPREFETCH_G2(A,B) VPREFETCH2(A,B) #define VPREFETCH_G2(A,B) VPREFETCH2(A,B)
#else
#endif #endif
#define PF_GAUGE(A) \ #define PF_GAUGE(A) \
@ -593,21 +594,26 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
VSTORE(11,%r8,result_32) VPREFETCH_M1(11,%r9) \ VSTORE(11,%r8,result_32) VPREFETCH_M1(11,%r9) \
); );
#ifdef AVX512_PF_L2_TABLE
#define PREFETCH_CHIMU(A) \ #define PREFETCH_CHIMU(A) \
LOAD64(%r9,A) \ LOAD64(%r9,A) \
__asm__ ( \ __asm__ ( \
VPREFETCH_P2(0,%r9) \ VPREFETCH_P1(0,%r9) \
VPREFETCH_P2(1,%r9) \ VPREFETCH_P1(1,%r9) \
VPREFETCH_P2(2,%r9) \ VPREFETCH_P1(2,%r9) \
VPREFETCH_P2(3,%r9) \ VPREFETCH_P1(3,%r9) \
VPREFETCH_P2(4,%r9) \ VPREFETCH_P1(4,%r9) \
VPREFETCH_P2(5,%r9) \ VPREFETCH_P1(5,%r9) \
VPREFETCH_P2(6,%r9) \ VPREFETCH_P1(6,%r9) \
VPREFETCH_P2(7,%r9) \ VPREFETCH_P1(7,%r9) \
VPREFETCH_P2(8,%r9) \ VPREFETCH_P1(8,%r9) \
VPREFETCH_P2(9,%r9) \ VPREFETCH_P1(9,%r9) \
VPREFETCH_P2(10,%r9) \ VPREFETCH_P1(10,%r9) \
VPREFETCH_P2(11,%r9)); VPREFETCH_P1(11,%r9));
#else
#define PREFETCH_CHIMU(A)
#endif
#define PREFETCH1_CHIMU(A) \ #define PREFETCH1_CHIMU(A) \
LOAD64(%r9,A) \ LOAD64(%r9,A) \
@ -811,6 +817,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
VPREFETCH_G1(3,%r8) \ VPREFETCH_G1(3,%r8) \
VPREFETCH_G2(4,%r8) \ VPREFETCH_G2(4,%r8) \
VPREFETCH_G2(5,%r8) \ VPREFETCH_G2(5,%r8) \
VPREFETCH_G2(6,%r8) \
VPREFETCH_G2(7,%r8) \
/*42 insns*/ ); /*42 insns*/ );
#define MULT_ADDSUB_2SPIN_LSNOPF(ptr,pf) \ #define MULT_ADDSUB_2SPIN_LSNOPF(ptr,pf) \