From bdaa5b17670b728cbd65e7ea0a802dc1d9e1bc65 Mon Sep 17 00:00:00 2001 From: paboyle Date: Thu, 30 Jun 2016 13:07:42 -0700 Subject: [PATCH] Updated to have perfect prefetching for the s-vectorised kernel with any cache blocking. --- lib/Stencil.h | 5 + lib/qcd/action/fermion/WilsonKernelsAsmBody.h | 153 +++++++++--------- lib/simd/Intel512wilson.h | 58 ++++--- 3 files changed, 119 insertions(+), 97 deletions(-) diff --git a/lib/Stencil.h b/lib/Stencil.h index bc015370..f5b6c288 100644 --- a/lib/Stencil.h +++ b/lib/Stencil.h @@ -272,6 +272,11 @@ if (local) return base + _entries[ent]._byte_offset; else return _entries[ent]._byte_offset; } + inline uint64_t GetPFInfo(int ent,uint64_t base) { + int local = _entries[ent]._is_local; + if (local) return base + _entries[ent]._byte_offset; + else return _entries[ent]._byte_offset; + } // Comms buffers std::vector > u_simd_send_buf; diff --git a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h index 7373d2eb..4f3ef861 100644 --- a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h +++ b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h @@ -1,43 +1,44 @@ { - int locala,perma, ptypea; - int localb,permb, ptypeb; - int localc,permc, ptypec; - uint64_t basea, baseb, basec; - + int local,perm, ptype; + uint64_t base; + uint64_t basep; const uint64_t plocal =(uint64_t) & in._odata[0]; // vComplexF isigns[2] = { signs[0], signs[1] }; vComplexF *isigns = &signs[0]; MASK_REGS; - + int nmax=U._grid->oSites(); for(int site=0;site=nmax) ssn=0; + int sUn=lo.Reorder(ssn); for(int s=0;s shuffle and xor the real part sign bit - YM_PROJMEM(baseb); - MAYBEPERM(PERMUTE_DIR2,permb); + YM_PROJMEM(base); + MAYBEPERM(PERMUTE_DIR2,perm); } else { - LOAD_CHI(baseb); + LOAD_CHI(base); } + base = st.GetInfo(ptype,local,perm,Zp,ent,plocal); ent++; + PREFETCH_CHIMU(base); { - MULT_2SPIN_DIR_PFYP(Yp,basec); + MULT_2SPIN_DIR_PFYP(Yp,basep); } LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit YM_RECON_ACCUM; @@ -63,17 +65,18 @@ //////////////////////////////// // Zp //////////////////////////////// - baseb = st.GetInfo(ptypeb,localb,permb,Xm,ent,plocal); ent++; - PREFETCH_CHIMU(baseb); - if ( localc ) { + basep = st.GetPFInfo(nent,plocal); nent++; + if ( local ) { LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit - ZM_PROJMEM(basec); - MAYBEPERM(PERMUTE_DIR1,permc); + ZM_PROJMEM(base); + MAYBEPERM(PERMUTE_DIR1,perm); } else { - LOAD_CHI(basec); + LOAD_CHI(base); } + base = st.GetInfo(ptype,local,perm,Tp,ent,plocal); ent++; + PREFETCH_CHIMU(base); { - MULT_2SPIN_DIR_PFZP(Zp,basea); + MULT_2SPIN_DIR_PFZP(Zp,basep); } LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit ZM_RECON_ACCUM; @@ -81,17 +84,18 @@ //////////////////////////////// // Tp //////////////////////////////// - basec = st.GetInfo(ptypec,localc,permc,Ym,ent,plocal); ent++; - PREFETCH_CHIMU(basec); - if ( locala ) { + basep = st.GetPFInfo(nent,plocal); nent++; + if ( local ) { LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit - TM_PROJMEM(basea); - MAYBEPERM(PERMUTE_DIR0,perma); + TM_PROJMEM(base); + MAYBEPERM(PERMUTE_DIR0,perm); } else { - LOAD_CHI(basea); + LOAD_CHI(base); } + base = st.GetInfo(ptype,local,perm,Xm,ent,plocal); ent++; + PREFETCH_CHIMU(base); { - MULT_2SPIN_DIR_PFTP(Tp,baseb); + MULT_2SPIN_DIR_PFTP(Tp,basep); } LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit TM_RECON_ACCUM; @@ -99,17 +103,19 @@ //////////////////////////////// // Xm //////////////////////////////// - basea = st.GetInfo(ptypea,locala,perma,Zm,ent,plocal); ent++; - PREFETCH_CHIMU(basea); - if ( localb ) { + basep= (uint64_t) &out._odata[ss]; + // basep= st.GetPFInfo(nent,plocal); nent++; + if ( local ) { LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit - XP_PROJMEM(baseb); - MAYBEPERM(PERMUTE_DIR3,permb); + XP_PROJMEM(base); + MAYBEPERM(PERMUTE_DIR3,perm); } else { - LOAD_CHI(baseb); + LOAD_CHI(base); } + base = st.GetInfo(ptype,local,perm,Ym,ent,plocal); ent++; + PREFETCH_CHIMU(base); { - MULT_2SPIN_DIR_PFXM(Xm,basec); + MULT_2SPIN_DIR_PFXM(Xm,basep); } LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit XP_RECON_ACCUM; @@ -117,17 +123,18 @@ //////////////////////////////// // Ym //////////////////////////////// - baseb = st.GetInfo(ptypeb,localb,permb,Tm,ent,plocal); ent++; - PREFETCH_CHIMU(baseb); - if ( localc ) { + basep= st.GetPFInfo(nent,plocal); nent++; + if ( local ) { LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit - YP_PROJMEM(basec); - MAYBEPERM(PERMUTE_DIR2,permc); + YP_PROJMEM(base); + MAYBEPERM(PERMUTE_DIR2,perm); } else { - LOAD_CHI(basec); + LOAD_CHI(base); } + base = st.GetInfo(ptype,local,perm,Zm,ent,plocal); ent++; + PREFETCH_CHIMU(base); { - MULT_2SPIN_DIR_PFYM(Ym,basea); + MULT_2SPIN_DIR_PFYM(Ym,basep); } LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit YP_RECON_ACCUM; @@ -135,17 +142,18 @@ //////////////////////////////// // Zm //////////////////////////////// - basec = (uint64_t)&out._odata[ss]; - PREFETCH_CHIMU(basec); - if ( locala ) { + basep= st.GetPFInfo(nent,plocal); nent++; + if ( local ) { LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit - ZP_PROJMEM(basea); - MAYBEPERM(PERMUTE_DIR1,perma); + ZP_PROJMEM(base); + MAYBEPERM(PERMUTE_DIR1,perm); } else { - LOAD_CHI(basea); + LOAD_CHI(base); } + base = st.GetInfo(ptype,local,perm,Tm,ent,plocal); ent++; + PREFETCH_CHIMU(base); { - MULT_2SPIN_DIR_PFZM(Zm,baseb); + MULT_2SPIN_DIR_PFZM(Zm,basep); } LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit ZP_RECON_ACCUM; @@ -153,23 +161,24 @@ //////////////////////////////// // Tm //////////////////////////////// - // basea = st.GetInfo(ptypea,locala,perma,Xp,ent,plocal); ent++; - // PREFETCH_CHIMU(basea); - if ( localb ) { + basep= st.GetPFInfo(nent,plocal); nent++; + if ( local ) { LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit - TP_PROJMEM(baseb); - MAYBEPERM(PERMUTE_DIR0,permb); + TP_PROJMEM(base); + MAYBEPERM(PERMUTE_DIR0,perm); } else { - LOAD_CHI(baseb); + LOAD_CHI(base); } + base= (uint64_t) &out._odata[ss]; + PREFETCH_CHIMU(base); { - MULT_2SPIN_DIR_PFTM(Tm,basec); + MULT_2SPIN_DIR_PFTM(Tm,basep); } - // baseb = st.GetInfo(ptypeb,localb,permb,Yp,ent,plocal); ent++; LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit TP_RECON_ACCUM; - SAVE_RESULT(&out._odata[ss],basec); + basep= st.GetPFInfo(nent,plocal); nent++; + SAVE_RESULT(base,basep); } ssU++; diff --git a/lib/simd/Intel512wilson.h b/lib/simd/Intel512wilson.h index 9deffd80..660d07d6 100644 --- a/lib/simd/Intel512wilson.h +++ b/lib/simd/Intel512wilson.h @@ -261,8 +261,8 @@ Author: paboyle #define XM_PROJMEM(PTR) \ LOAD64(%r8,PTR)\ __asm__ ( \ - SHUF_CHIMU23i \ LOAD_CHIi \ + SHUF_CHIMU23i \ VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_30)\ VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_31)\ VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_32)\ @@ -290,8 +290,8 @@ Author: paboyle #define ZM_PROJMEM(PTR) \ LOAD64(%r8,PTR) \ __asm__ ( \ - SHUF_CHIMU23i \ LOAD_CHIi \ + SHUF_CHIMU23i \ VACCTIMESMINUSI1(Chi_00,Chi_00,Chimu_20)\ VACCTIMESMINUSI1(Chi_01,Chi_01,Chimu_21)\ VACCTIMESMINUSI1(Chi_02,Chi_02,Chimu_22)\ @@ -548,24 +548,25 @@ Author: paboyle #define AVX512_PF_L2_TABLE #undef AVX512_PF_L2_LINEAR -#ifdef AVX512_PF_L2_TABLE -#define VPREFETCH_P1(A,B) VPREFETCH1(A,B) -#define VPREFETCH_P2(A,B) VPREFETCH1(A,B) -#else -#define VPREFETCH_P1(A,B) -#define VPREFETCH_P2(A,B) -#endif -#ifdef AVX512_PF_L2_LINEAR -#define VPREFETCH_M1(A,B) +#ifdef AVX512_PF_L2_TABLE +// P1 Fetches the base pointer for next link into L1 with P1 +// M1 Fetches the next site pointer into L2 +#define VPREFETCH_P1(A,B) VPREFETCH1(A,B) +#define VPREFETCH_P2(A,B) +#define VPREFETCH_M1(A,B) VPREFETCH2(A,B) #define VPREFETCH_M2(A,B) -#else +#endif + +#ifdef AVX512_PF_L2_LINEAR #define VPREFETCH_M1(A,B) VPREFETCH1(A,B) #define VPREFETCH_M2(A,B) VPREFETCH2(A,B) +#define VPREFETCH_P1(A,B) +#define VPREFETCH_P2(A,B) #endif + #ifdef AVX512_PF_L2_GAUGE #define VPREFETCH_G1(A,B) VPREFETCH1(A,B) #define VPREFETCH_G2(A,B) VPREFETCH2(A,B) -#else #endif #define PF_GAUGE(A) \ @@ -593,21 +594,26 @@ Author: paboyle VSTORE(11,%r8,result_32) VPREFETCH_M1(11,%r9) \ ); +#ifdef AVX512_PF_L2_TABLE #define PREFETCH_CHIMU(A) \ LOAD64(%r9,A) \ __asm__ ( \ - VPREFETCH_P2(0,%r9) \ - VPREFETCH_P2(1,%r9) \ - VPREFETCH_P2(2,%r9) \ - VPREFETCH_P2(3,%r9) \ - VPREFETCH_P2(4,%r9) \ - VPREFETCH_P2(5,%r9) \ - VPREFETCH_P2(6,%r9) \ - VPREFETCH_P2(7,%r9) \ - VPREFETCH_P2(8,%r9) \ - VPREFETCH_P2(9,%r9) \ - VPREFETCH_P2(10,%r9) \ - VPREFETCH_P2(11,%r9)); + VPREFETCH_P1(0,%r9) \ + VPREFETCH_P1(1,%r9) \ + VPREFETCH_P1(2,%r9) \ + VPREFETCH_P1(3,%r9) \ + VPREFETCH_P1(4,%r9) \ + VPREFETCH_P1(5,%r9) \ + VPREFETCH_P1(6,%r9) \ + VPREFETCH_P1(7,%r9) \ + VPREFETCH_P1(8,%r9) \ + VPREFETCH_P1(9,%r9) \ + VPREFETCH_P1(10,%r9) \ + VPREFETCH_P1(11,%r9)); + +#else +#define PREFETCH_CHIMU(A) +#endif #define PREFETCH1_CHIMU(A) \ LOAD64(%r9,A) \ @@ -811,6 +817,8 @@ Author: paboyle VPREFETCH_G1(3,%r8) \ VPREFETCH_G2(4,%r8) \ VPREFETCH_G2(5,%r8) \ + VPREFETCH_G2(6,%r8) \ + VPREFETCH_G2(7,%r8) \ /*42 insns*/ ); #define MULT_ADDSUB_2SPIN_LSNOPF(ptr,pf) \