portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2025-04-27 14:15:55 +01:00
Code Releases Activity

NAMESPACE & format

Browse Source
This commit is contained in:
paboyle 2018-01-12 18:05:36 +00:00
parent 08682c5461
commit fbc2380cb8
1 changed files with 530 additions and 539 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
lib/simd/Grid_avx512.h
View File

@ -1,4 +1,4 @@
/************************************************************************************* /*************************************************************************************
Grid physics library, www.github.com/paboyle/Grid Grid physics library, www.github.com/paboyle/Grid
@ -25,25 +25,24 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
See the full license in the file "LICENSE" in the top level distribution directory See the full license in the file "LICENSE" in the top level distribution directory
*************************************************************************************/ *************************************************************************************/
/* END LEGAL */ /* END LEGAL */
#include <immintrin.h> #include <immintrin.h>
NAMESPACE_BEGIN(Grid);
NAMESPACE_BEGIN(Optimization);
namespace Grid{ union u512f {
namespace Optimization {
union u512f {
__m512 v; __m512 v;
float f[16]; float f[16];
}; };
union u512d { union u512d {
__m512d v; __m512d v;
double f[8]; double f[8];
}; };
struct Vsplat{ struct Vsplat{
//Complex float //Complex float
inline __m512 operator()(float a, float b){ inline __m512 operator()(float a, float b){
return _mm512_set_ps(b,a,b,a,b,a,b,a,b,a,b,a,b,a,b,a); return _mm512_set_ps(b,a,b,a,b,a,b,a,b,a,b,a,b,a,b,a);
@ -64,9 +63,9 @@ namespace Optimization {
inline __m512i operator()(Integer a){ inline __m512i operator()(Integer a){
return _mm512_set1_epi32(a); return _mm512_set1_epi32(a);
} }
}; };
struct Vstore{ struct Vstore{
//Float //Float
inline void operator()(__m512 a, float* F){ inline void operator()(__m512 a, float* F){
_mm512_store_ps(F,a); _mm512_store_ps(F,a);
@ -80,10 +79,9 @@ namespace Optimization {
_mm512_store_si512((__m512i *)I,a); _mm512_store_si512((__m512i *)I,a);
} }
}; };
struct Vstream{
struct Vstream{
//Float //Float
inline void operator()(float * a, __m512 b){ inline void operator()(float * a, __m512 b){
_mm512_stream_ps(a,b); _mm512_stream_ps(a,b);
@ -95,11 +93,9 @@ namespace Optimization {
// _mm512_store_pd(a,b); // _mm512_store_pd(a,b);
} }
}; };
struct Vset{
struct Vset{
// Complex float // Complex float
inline __m512 operator()(Grid::ComplexF *a){ inline __m512 operator()(Grid::ComplexF *a){
return _mm512_set_ps(a[7].imag(),a[7].real(),a[6].imag(),a[6].real(), return _mm512_set_ps(a[7].imag(),a[7].real(),a[6].imag(),a[6].real(),
@ -127,10 +123,9 @@ namespace Optimization {
a[7],a[6],a[5],a[4],a[3],a[2],a[1],a[0]); a[7],a[6],a[5],a[4],a[3],a[2],a[1],a[0]);
} }
};
}; template <typename Out_type, typename In_type>
template <typename Out_type, typename In_type>
struct Reduce{ struct Reduce{
//Need templated class to overload output type //Need templated class to overload output type
//General form must generate error if compiled //General form must generate error if compiled
@ -142,12 +137,10 @@ namespace Optimization {
}; };
/////////////////////////////////////////////////////
// Arithmetic operations
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
// Arithmetic operations struct Sum{
/////////////////////////////////////////////////////
struct Sum{
//Complex/Real float //Complex/Real float
inline __m512 operator()(__m512 a, __m512 b){ inline __m512 operator()(__m512 a, __m512 b){
return _mm512_add_ps(a,b); return _mm512_add_ps(a,b);
@ -160,9 +153,9 @@ namespace Optimization {
inline __m512i operator()(__m512i a, __m512i b){ inline __m512i operator()(__m512i a, __m512i b){
return _mm512_add_epi32(a,b); return _mm512_add_epi32(a,b);
} }
}; };
struct Sub{ struct Sub{
//Complex/Real float //Complex/Real float
inline __m512 operator()(__m512 a, __m512 b){ inline __m512 operator()(__m512 a, __m512 b){
return _mm512_sub_ps(a,b); return _mm512_sub_ps(a,b);
@ -175,21 +168,21 @@ namespace Optimization {
inline __m512i operator()(__m512i a, __m512i b){ inline __m512i operator()(__m512i a, __m512i b){
return _mm512_sub_epi32(a,b); return _mm512_sub_epi32(a,b);
} }
}; };
// Note, we can beat the shuf overhead in chain with two temporaries // Note, we can beat the shuf overhead in chain with two temporaries
// Ar Ai , Br Bi, Ai Ar // one shuf // Ar Ai , Br Bi, Ai Ar // one shuf
//tmpr Ar Br, Ai Bi // Mul/Mac/Mac //tmpr Ar Br, Ai Bi // Mul/Mac/Mac
//tmpi Br Ai, Bi Ar // Mul/Mac/Mac //tmpi Br Ai, Bi Ar // Mul/Mac/Mac
// add tmpi,shuf(tmpi) // add tmpi,shuf(tmpi)
// sub tmpr,shuf(tmpi) // sub tmpr,shuf(tmpi)
// shuf(tmpr,tmpi). // Could drop/trade for write mask // shuf(tmpr,tmpi). // Could drop/trade for write mask
// Gives // Gives
// 2mul,4 mac +add+sub = 8 flop type insns // 2mul,4 mac +add+sub = 8 flop type insns
// 3shuf + 2 (+shuf) = 5/6 simd perm and 1/2 the load. // 3shuf + 2 (+shuf) = 5/6 simd perm and 1/2 the load.
struct MultRealPart{ struct MultRealPart{
inline __m512 operator()(__m512 a, __m512 b){ inline __m512 operator()(__m512 a, __m512 b){
__m512 ymm0; __m512 ymm0;
ymm0 = _mm512_moveldup_ps(a); // ymm0 <- ar ar, ymm0 = _mm512_moveldup_ps(a); // ymm0 <- ar ar,
@ -200,8 +193,8 @@ namespace Optimization {
ymm0 = _mm512_shuffle_pd(a,a,0x00); // ymm0 <- ar ar, ar,ar b'00,00 ymm0 = _mm512_shuffle_pd(a,a,0x00); // ymm0 <- ar ar, ar,ar b'00,00
return _mm512_mul_pd(ymm0,b); // ymm0 <- ar bi, ar br return _mm512_mul_pd(ymm0,b); // ymm0 <- ar bi, ar br
} }
}; };
struct MaddRealPart{ struct MaddRealPart{
inline __m512 operator()(__m512 a, __m512 b, __m512 c){ inline __m512 operator()(__m512 a, __m512 b, __m512 c){
__m512 ymm0 = _mm512_moveldup_ps(a); // ymm0 <- ar ar, __m512 ymm0 = _mm512_moveldup_ps(a); // ymm0 <- ar ar,
return _mm512_fmadd_ps( ymm0, b, c); return _mm512_fmadd_ps( ymm0, b, c);
@ -210,9 +203,9 @@ namespace Optimization {
__m512d ymm0 = _mm512_shuffle_pd( a, a, 0x00 ); __m512d ymm0 = _mm512_shuffle_pd( a, a, 0x00 );
return _mm512_fmadd_pd( ymm0, b, c); return _mm512_fmadd_pd( ymm0, b, c);
} }
}; };
struct MultComplex{ struct MultComplex{
// Complex float // Complex float
inline __m512 operator()(__m512 a, __m512 b){ inline __m512 operator()(__m512 a, __m512 b){
// dup, dup, perm, mul, madd // dup, dup, perm, mul, madd
@ -228,9 +221,9 @@ namespace Optimization {
a_imag = _mm512_mul_pd( a_imag, _mm512_permute_pd( b, 0x55 ) ); a_imag = _mm512_mul_pd( a_imag, _mm512_permute_pd( b, 0x55 ) );
return _mm512_fmaddsub_pd( a_real, b, a_imag ); return _mm512_fmaddsub_pd( a_real, b, a_imag );
} }
}; };
struct Mult{ struct Mult{
inline void mac(__m512 &a, __m512 b, __m512 c){ inline void mac(__m512 &a, __m512 b, __m512 c){
a= _mm512_fmadd_ps( b, c, a); a= _mm512_fmadd_ps( b, c, a);
@ -250,9 +243,9 @@ namespace Optimization {
inline __m512i operator()(__m512i a, __m512i b){ inline __m512i operator()(__m512i a, __m512i b){
return _mm512_mullo_epi32(a,b); return _mm512_mullo_epi32(a,b);
} }
}; };
struct Div{ struct Div{
// Real float // Real float
inline __m512 operator()(__m512 a, __m512 b){ inline __m512 operator()(__m512 a, __m512 b){
return _mm512_div_ps(a,b); return _mm512_div_ps(a,b);
@ -261,10 +254,10 @@ namespace Optimization {
inline __m512d operator()(__m512d a, __m512d b){ inline __m512d operator()(__m512d a, __m512d b){
return _mm512_div_pd(a,b); return _mm512_div_pd(a,b);
} }
}; };
struct Conj{ struct Conj{
// Complex single // Complex single
inline __m512 operator()(__m512 in){ inline __m512 operator()(__m512 in){
return _mm512_mask_sub_ps(in,0xaaaa,_mm512_setzero_ps(),in); // Zero out 0+real 0-imag return _mm512_mask_sub_ps(in,0xaaaa,_mm512_setzero_ps(),in); // Zero out 0+real 0-imag
@ -274,9 +267,9 @@ namespace Optimization {
return _mm512_mask_sub_pd(in, 0xaa,_mm512_setzero_pd(), in); return _mm512_mask_sub_pd(in, 0xaa,_mm512_setzero_pd(), in);
} }
// do not define for integer input // do not define for integer input
}; };
struct TimesMinusI{ struct TimesMinusI{
//Complex single //Complex single
inline __m512 operator()(__m512 in, __m512 ret){ inline __m512 operator()(__m512 in, __m512 ret){
//__m512 tmp = _mm512_mask_sub_ps(in,0xaaaa,_mm512_setzero_ps(),in); // real -imag //__m512 tmp = _mm512_mask_sub_ps(in,0xaaaa,_mm512_setzero_ps(),in); // real -imag
@ -291,9 +284,9 @@ namespace Optimization {
__m512d tmp = _mm512_shuffle_pd(in,in,0x55); __m512d tmp = _mm512_shuffle_pd(in,in,0x55);
return _mm512_mask_sub_pd(tmp,0xaa,_mm512_setzero_pd(),tmp); return _mm512_mask_sub_pd(tmp,0xaa,_mm512_setzero_pd(),tmp);
} }
}; };
struct TimesI{ struct TimesI{
//Complex single //Complex single
inline __m512 operator()(__m512 in, __m512 ret){ inline __m512 operator()(__m512 in, __m512 ret){
__m512 tmp = _mm512_shuffle_ps(in,in,_MM_SELECT_FOUR_FOUR(2,3,0,1)); __m512 tmp = _mm512_shuffle_ps(in,in,_MM_SELECT_FOUR_FOUR(2,3,0,1));
@ -306,12 +299,10 @@ namespace Optimization {
} }
}; };
// Gpermute utilities consider coalescing into 1 Gpermute
struct Permute{
// Gpermute utilities consider coalescing into 1 Gpermute
struct Permute{
static inline __m512 Permute0(__m512 in){ static inline __m512 Permute0(__m512 in){
return _mm512_shuffle_f32x4(in,in,_MM_SELECT_FOUR_FOUR(1,0,3,2)); return _mm512_shuffle_f32x4(in,in,_MM_SELECT_FOUR_FOUR(1,0,3,2));
@ -339,9 +330,9 @@ namespace Optimization {
return in; return in;
}; };
}; };
#define USE_FP16 #define USE_FP16
struct PrecisionChange { struct PrecisionChange {
static inline __m512i StoH (__m512 a,__m512 b) { static inline __m512i StoH (__m512 a,__m512 b) {
__m512i h; __m512i h;
#ifdef USE_FP16 #ifdef USE_FP16
@ -354,6 +345,7 @@ namespace Optimization {
#endif #endif
return h; return h;
} }
static inline void HtoS (__m512i h,__m512 &sa,__m512 &sb) { static inline void HtoS (__m512i h,__m512 &sa,__m512 &sb) {
#ifdef USE_FP16 #ifdef USE_FP16
sa = _mm512_cvtph_ps((__m256i)_mm512_extractf64x4_pd((__m512d)h,0)); sa = _mm512_cvtph_ps((__m256i)_mm512_extractf64x4_pd((__m512d)h,0));
@ -362,6 +354,7 @@ namespace Optimization {
assert(0); assert(0);
#endif #endif
} }
static inline __m512 DtoS (__m512d a,__m512d b) { static inline __m512 DtoS (__m512d a,__m512d b) {
__m256 sa = _mm512_cvtpd_ps(a); __m256 sa = _mm512_cvtpd_ps(a);
__m256 sb = _mm512_cvtpd_ps(b); __m256 sb = _mm512_cvtpd_ps(b);
@ -369,27 +362,30 @@ namespace Optimization {
s =(__m512) _mm512_insertf64x4((__m512d)s,(__m256d)sb,1); s =(__m512) _mm512_insertf64x4((__m512d)s,(__m256d)sb,1);
return s; return s;
} }
static inline void StoD (__m512 s,__m512d &a,__m512d &b) { static inline void StoD (__m512 s,__m512d &a,__m512d &b) {
a = _mm512_cvtps_pd((__m256)_mm512_extractf64x4_pd((__m512d)s,0)); a = _mm512_cvtps_pd((__m256)_mm512_extractf64x4_pd((__m512d)s,0));
b = _mm512_cvtps_pd((__m256)_mm512_extractf64x4_pd((__m512d)s,1)); b = _mm512_cvtps_pd((__m256)_mm512_extractf64x4_pd((__m512d)s,1));
} }
static inline __m512i DtoH (__m512d a,__m512d b,__m512d c,__m512d d) { static inline __m512i DtoH (__m512d a,__m512d b,__m512d c,__m512d d) {
__m512 sa,sb; __m512 sa,sb;
sa = DtoS(a,b); sa = DtoS(a,b);
sb = DtoS(c,d); sb = DtoS(c,d);
return StoH(sa,sb); return StoH(sa,sb);
} }
static inline void HtoD (__m512i h,__m512d &a,__m512d &b,__m512d &c,__m512d &d) { static inline void HtoD (__m512i h,__m512d &a,__m512d &b,__m512d &c,__m512d &d) {
__m512 sa,sb; __m512 sa,sb;
HtoS(h,sa,sb); HtoS(h,sa,sb);
StoD(sa,a,b); StoD(sa,a,b);
StoD(sb,c,d); StoD(sb,c,d);
} }
}; };
// On extracting face: Ah Al , Bh Bl -> Ah Bh, Al Bl // On extracting face: Ah Al , Bh Bl -> Ah Bh, Al Bl
// On merging buffers: Ah,Bh , Al Bl -> Ah Al, Bh, Bl // On merging buffers: Ah,Bh , Al Bl -> Ah Al, Bh, Bl
// The operation is its own inverse // The operation is its own inverse
struct Exchange{ struct Exchange{
// 3210 ordering // 3210 ordering
static inline void Exchange0(__m512 &out1,__m512 &out2,__m512 in1,__m512 in2){ static inline void Exchange0(__m512 &out1,__m512 &out2,__m512 in1,__m512 in2){
out1= _mm512_shuffle_f32x4(in1,in2,_MM_SELECT_FOUR_FOUR(1,0,1,0)); out1= _mm512_shuffle_f32x4(in1,in2,_MM_SELECT_FOUR_FOUR(1,0,1,0));
@ -430,10 +426,10 @@ namespace Optimization {
assert(0); assert(0);
return; return;
}; };
}; };
struct Rotate{ struct Rotate{
static inline __m512 rotate(__m512 in,int n){ static inline __m512 rotate(__m512 in,int n){
switch(n){ switch(n){
@ -479,17 +475,17 @@ namespace Optimization {
return (__m512d)_mm512_alignr_epi64((__m512i)in,(__m512i)in,n); return (__m512d)_mm512_alignr_epi64((__m512i)in,(__m512i)in,n);
}; };
}; };
////////////////////////////////////////////// //////////////////////////////////////////////
// Some Template specialization // Some Template specialization
// Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases
#ifndef __INTEL_COMPILER #ifndef __INTEL_COMPILER
#warning "Slow reduction due to incomplete reduce intrinsics" #warning "Slow reduction due to incomplete reduce intrinsics"
//Complex float Reduce //Complex float Reduce
template<> template<>
inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){ inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){
__m512 v1,v2; __m512 v1,v2;
v1=Optimization::Permute::Permute0(in); // avx 512; quad complex single v1=Optimization::Permute::Permute0(in); // avx 512; quad complex single
v1= _mm512_add_ps(v1,in); v1= _mm512_add_ps(v1,in);
@ -499,11 +495,11 @@ namespace Optimization {
v1 = _mm512_add_ps(v1,v2); v1 = _mm512_add_ps(v1,v2);
u512f conv; conv.v = v1; u512f conv; conv.v = v1;
return Grid::ComplexF(conv.f[0],conv.f[1]); return Grid::ComplexF(conv.f[0],conv.f[1]);
} }
//Real float Reduce //Real float Reduce
template<> template<>
inline Grid::RealF Reduce<Grid::RealF, __m512>::operator()(__m512 in){ inline Grid::RealF Reduce<Grid::RealF, __m512>::operator()(__m512 in){
__m512 v1,v2; __m512 v1,v2;
v1 = Optimization::Permute::Permute0(in); // avx 512; octo-double v1 = Optimization::Permute::Permute0(in); // avx 512; octo-double
v1 = _mm512_add_ps(v1,in); v1 = _mm512_add_ps(v1,in);
@ -515,12 +511,11 @@ namespace Optimization {
v1 = _mm512_add_ps(v1,v2); v1 = _mm512_add_ps(v1,v2);
u512f conv; conv.v=v1; u512f conv; conv.v=v1;
return conv.f[0]; return conv.f[0];
} }
//Complex double Reduce
//Complex double Reduce template<>
template<> inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){
inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){
__m512d v1; __m512d v1;
v1 = Optimization::Permute::Permute0(in); // sse 128; paired complex single v1 = Optimization::Permute::Permute0(in); // sse 128; paired complex single
v1 = _mm512_add_pd(v1,in); v1 = _mm512_add_pd(v1,in);
@ -528,11 +523,11 @@ namespace Optimization {
v1 = _mm512_add_pd(v1,in); v1 = _mm512_add_pd(v1,in);
u512d conv; conv.v = v1; u512d conv; conv.v = v1;
return Grid::ComplexD(conv.f[0],conv.f[1]); return Grid::ComplexD(conv.f[0],conv.f[1]);
} }
//Real double Reduce //Real double Reduce
template<> template<>
inline Grid::RealD Reduce<Grid::RealD, __m512d>::operator()(__m512d in){ inline Grid::RealD Reduce<Grid::RealD, __m512d>::operator()(__m512d in){
__m512d v1,v2; __m512d v1,v2;
v1 = Optimization::Permute::Permute0(in); // avx 512; quad double v1 = Optimization::Permute::Permute0(in); // avx 512; quad double
v1 = _mm512_add_pd(v1,in); v1 = _mm512_add_pd(v1,in);
@ -542,11 +537,11 @@ namespace Optimization {
v1 = _mm512_add_pd(v1,v2); v1 = _mm512_add_pd(v1,v2);
u512d conv; conv.v = v1; u512d conv; conv.v = v1;
return conv.f[0]; return conv.f[0];
} }
//Integer Reduce //Integer Reduce
template<> template<>
inline Integer Reduce<Integer, __m512i>::operator()(__m512i in){ inline Integer Reduce<Integer, __m512i>::operator()(__m512i in){
// No full vector reduce, use AVX to add upper and lower halves of register // No full vector reduce, use AVX to add upper and lower halves of register
// and perform AVX reduction. // and perform AVX reduction.
__m256i v1, v2, v3; __m256i v1, v2, v3;
@ -560,81 +555,77 @@ namespace Optimization {
u2 = _mm256_extracti128_si256(v2, 1); // lower half u2 = _mm256_extracti128_si256(v2, 1); // lower half
ret = _mm_add_epi32(u1, u2); ret = _mm_add_epi32(u1, u2);
return _mm_cvtsi128_si32(ret); return _mm_cvtsi128_si32(ret);
} }
#else #else
//Complex float Reduce //Complex float Reduce
template<> template<>
inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){ inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){
return Grid::ComplexF(_mm512_mask_reduce_add_ps(0x5555, in),_mm512_mask_reduce_add_ps(0xAAAA, in)); return Grid::ComplexF(_mm512_mask_reduce_add_ps(0x5555, in),_mm512_mask_reduce_add_ps(0xAAAA, in));
} }
//Real float Reduce //Real float Reduce
template<> template<>
inline Grid::RealF Reduce<Grid::RealF, __m512>::operator()(__m512 in){ inline Grid::RealF Reduce<Grid::RealF, __m512>::operator()(__m512 in){
return _mm512_reduce_add_ps(in); return _mm512_reduce_add_ps(in);
} }
//Complex double Reduce //Complex double Reduce
template<> template<>
inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){ inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){
return Grid::ComplexD(_mm512_mask_reduce_add_pd(0x55, in),_mm512_mask_reduce_add_pd(0xAA, in)); return Grid::ComplexD(_mm512_mask_reduce_add_pd(0x55, in),_mm512_mask_reduce_add_pd(0xAA, in));
} }
//Real double Reduce //Real double Reduce
template<> template<>
inline Grid::RealD Reduce<Grid::RealD, __m512d>::operator()(__m512d in){ inline Grid::RealD Reduce<Grid::RealD, __m512d>::operator()(__m512d in){
return _mm512_reduce_add_pd(in); return _mm512_reduce_add_pd(in);
} }
//Integer Reduce //Integer Reduce
template<> template<>
inline Integer Reduce<Integer, __m512i>::operator()(__m512i in){ inline Integer Reduce<Integer, __m512i>::operator()(__m512i in){
return _mm512_reduce_add_epi32(in); return _mm512_reduce_add_epi32(in);
} }
#endif #endif
} NAMESPACE_END(Optimization);
////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////
// Here assign types // Here assign types
typedef __m512i SIMD_Htype; // Single precision type
typedef __m512 SIMD_Ftype; // Single precision type
typedef __m512d SIMD_Dtype; // Double precision type
typedef __m512i SIMD_Itype; // Integer type
typedef __m512i SIMD_Htype; // Single precision type // prefecth
typedef __m512 SIMD_Ftype; // Single precision type inline void v_prefetch0(int size, const char *ptr){
typedef __m512d SIMD_Dtype; // Double precision type
typedef __m512i SIMD_Itype; // Integer type
// prefecth
inline void v_prefetch0(int size, const char *ptr){
for(int i=0;i<size;i+=64){ // Define L1 linesize above for(int i=0;i<size;i+=64){ // Define L1 linesize above
_mm_prefetch(ptr+i+4096,_MM_HINT_T1); _mm_prefetch(ptr+i+4096,_MM_HINT_T1);
_mm_prefetch(ptr+i+512,_MM_HINT_T0); _mm_prefetch(ptr+i+512,_MM_HINT_T0);
} }
}
inline void prefetch_HINT_T0(const char *ptr){
_mm_prefetch(ptr,_MM_HINT_T0);
}
// Function name aliases
typedef Optimization::Vsplat VsplatSIMD;
typedef Optimization::Vstore VstoreSIMD;
typedef Optimization::Vset VsetSIMD;
typedef Optimization::Vstream VstreamSIMD;
template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
// Arithmetic operations
typedef Optimization::Sum SumSIMD;
typedef Optimization::Sub SubSIMD;
typedef Optimization::Mult MultSIMD;
typedef Optimization::Div DivSIMD;
typedef Optimization::MultComplex MultComplexSIMD;
typedef Optimization::MultRealPart MultRealPartSIMD;
typedef Optimization::MaddRealPart MaddRealPartSIMD;
typedef Optimization::Conj ConjSIMD;
typedef Optimization::TimesMinusI TimesMinusISIMD;
typedef Optimization::TimesI TimesISIMD;
} }
inline void prefetch_HINT_T0(const char *ptr){
_mm_prefetch(ptr,_MM_HINT_T0);
}
// Function name aliases
typedef Optimization::Vsplat VsplatSIMD;
typedef Optimization::Vstore VstoreSIMD;
typedef Optimization::Vset VsetSIMD;
typedef Optimization::Vstream VstreamSIMD;
template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
// Arithmetic operations
typedef Optimization::Sum SumSIMD;
typedef Optimization::Sub SubSIMD;
typedef Optimization::Mult MultSIMD;
typedef Optimization::Div DivSIMD;
typedef Optimization::MultComplex MultComplexSIMD;
typedef Optimization::MultRealPart MultRealPartSIMD;
typedef Optimization::MaddRealPart MaddRealPartSIMD;
typedef Optimization::Conj ConjSIMD;
typedef Optimization::TimesMinusI TimesMinusISIMD;
typedef Optimization::TimesI TimesISIMD;
NAMESPACE_END(Grid);