mirror of
https://github.com/paboyle/Grid.git
synced 2025-04-11 06:30:45 +01:00
Format, NAMESPACE
This commit is contained in:
paboyle
parent
0a6168eef0
commit
62fcee72c5
@ -1,4 +1,4 @@
|
||||
/*************************************************************************************
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
@ -24,16 +24,16 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <zmmintrin.h>
|
||||
|
||||
namespace Grid{
|
||||
namespace Optimization {
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
NAMESPACE_BEGIN(Optimization);
|
||||
|
||||
struct Vsplat{
|
||||
struct Vsplat{
|
||||
//Complex float
|
||||
inline __m512 operator()(float a, float b){
|
||||
return _mm512_set_ps(b,a,b,a,b,a,b,a,b,a,b,a,b,a,b,a);
|
||||
@ -54,9 +54,9 @@ namespace Optimization {
|
||||
inline __m512i operator()(Integer a){
|
||||
return _mm512_set1_epi32(a);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
struct Vstore{
|
||||
struct Vstore{
|
||||
//Float
|
||||
inline void operator()(__m512 a, float* F){
|
||||
_mm512_store_ps(F,a);
|
||||
@ -70,10 +70,9 @@ namespace Optimization {
|
||||
_mm512_store_si512((__m512i *)I,a);
|
||||
}
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
struct Vstream{
|
||||
struct Vstream{
|
||||
//Float
|
||||
inline void operator()(float * a, __m512 b){
|
||||
_mm512_storenrngo_ps(a,b);
|
||||
@ -84,11 +83,9 @@ namespace Optimization {
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
|
||||
struct Vset{
|
||||
struct Vset{
|
||||
// Complex float
|
||||
inline __m512 operator()(Grid::ComplexF *a){
|
||||
return _mm512_set_ps(a[7].imag(),a[7].real(),a[6].imag(),a[6].real(),
|
||||
@ -117,10 +114,10 @@ namespace Optimization {
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
template <typename Out_type, typename In_type>
|
||||
struct Reduce{
|
||||
template <typename Out_type, typename In_type>
|
||||
struct Reduce{
|
||||
//Need templated class to overload output type
|
||||
//General form must generate error if compiled
|
||||
inline Out_type operator()(In_type in){
|
||||
@ -128,15 +125,12 @@ namespace Optimization {
|
||||
exit(1);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Arithmetic operations
|
||||
/////////////////////////////////////////////////////
|
||||
struct Sum{
|
||||
/////////////////////////////////////////////////////
|
||||
// Arithmetic operations
|
||||
/////////////////////////////////////////////////////
|
||||
struct Sum{
|
||||
//Complex/Real float
|
||||
inline __m512 operator()(__m512 a, __m512 b){
|
||||
return _mm512_add_ps(a,b);
|
||||
@ -149,9 +143,9 @@ namespace Optimization {
|
||||
inline __m512i operator()(__m512i a, __m512i b){
|
||||
return _mm512_add_epi32(a,b);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
struct Sub{
|
||||
struct Sub{
|
||||
//Complex/Real float
|
||||
inline __m512 operator()(__m512 a, __m512 b){
|
||||
return _mm512_sub_ps(a,b);
|
||||
@ -164,10 +158,9 @@ namespace Optimization {
|
||||
inline __m512i operator()(__m512i a, __m512i b){
|
||||
return _mm512_sub_epi32(a,b);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
struct MultComplex{
|
||||
struct MultComplex{
|
||||
// Complex float
|
||||
inline __m512 operator()(__m512 a, __m512 b){
|
||||
__m512 vzero,ymm0,ymm1,real, imag;
|
||||
@ -210,9 +203,9 @@ namespace Optimization {
|
||||
ymm0 = _mm512_swizzle_pd(b, _MM_SWIZ_REG_CDAB); // OK
|
||||
return _mm512_fmadd_pd(ymm0,imag,ymm1);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
struct Mult{
|
||||
struct Mult{
|
||||
|
||||
inline void mac(__m512 &a, __m512 b, __m512 c){
|
||||
a= _mm512_fmadd_ps( b, c, a);
|
||||
@ -234,9 +227,9 @@ namespace Optimization {
|
||||
inline __m512i operator()(__m512i a, __m512i b){
|
||||
return _mm512_mullo_epi32(a,b);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
struct Div{
|
||||
struct Div{
|
||||
// Real float
|
||||
inline __m512 operator()(__m512 a, __m512 b){
|
||||
return _mm512_div_ps(a,b);
|
||||
@ -245,10 +238,9 @@ namespace Optimization {
|
||||
inline __m512d operator()(__m512d a, __m512d b){
|
||||
return _mm512_div_pd(a,b);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
struct Conj{
|
||||
struct Conj{
|
||||
// Complex single
|
||||
inline __m512 operator()(__m512 in){
|
||||
return _mm512_mask_sub_ps(in,0xaaaa,_mm512_setzero_ps(),in); // Zero out 0+real 0-imag
|
||||
@ -258,9 +250,9 @@ namespace Optimization {
|
||||
return _mm512_mask_sub_pd(in, 0xaa,_mm512_setzero_pd(), in);
|
||||
}
|
||||
// do not define for integer input
|
||||
};
|
||||
};
|
||||
|
||||
struct TimesMinusI{
|
||||
struct TimesMinusI{
|
||||
//Complex single
|
||||
inline __m512 operator()(__m512 in, __m512 ret){
|
||||
__m512 tmp = _mm512_mask_sub_ps(in,0xaaaa,_mm512_setzero_ps(),in); // real -imag
|
||||
@ -271,11 +263,9 @@ namespace Optimization {
|
||||
__m512d tmp = _mm512_mask_sub_pd(in,0xaa,_mm512_setzero_pd(),in); // real -imag
|
||||
return _mm512_swizzle_pd(tmp, _MM_SWIZ_REG_CDAB);// OK
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
};
|
||||
|
||||
struct TimesI{
|
||||
struct TimesI{
|
||||
//Complex single
|
||||
inline __m512 operator()(__m512 in, __m512 ret){
|
||||
__m512 tmp = _mm512_swizzle_ps(in, _MM_SWIZ_REG_CDAB);// OK
|
||||
@ -288,10 +278,9 @@ namespace Optimization {
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
struct Permute{
|
||||
struct Permute{
|
||||
|
||||
static inline __m512 Permute0(__m512 in){
|
||||
return _mm512_permute4f128_ps(in,(_MM_PERM_ENUM)_MM_SELECT_FOUR_FOUR(1,0,3,2));
|
||||
@ -319,9 +308,9 @@ namespace Optimization {
|
||||
return in;
|
||||
};
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
struct Rotate{
|
||||
struct Rotate{
|
||||
|
||||
static inline __m512 rotate(__m512 in,int n){
|
||||
switch(n){
|
||||
@ -367,82 +356,76 @@ namespace Optimization {
|
||||
return (__m512d)_mm512_alignr_epi32((__m512i)in,(__m512i)in,2*n);
|
||||
};
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////
|
||||
// Some Template specialization
|
||||
|
||||
|
||||
//////////////////////////////////////////////
|
||||
// Some Template specialization
|
||||
|
||||
//Complex float Reduce
|
||||
template<>
|
||||
inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){
|
||||
//Complex float Reduce
|
||||
template<>
|
||||
inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){
|
||||
return Grid::ComplexF(_mm512_mask_reduce_add_ps(0x5555, in),_mm512_mask_reduce_add_ps(0xAAAA, in));
|
||||
}
|
||||
//Real float Reduce
|
||||
template<>
|
||||
inline Grid::RealF Reduce<Grid::RealF, __m512>::operator()(__m512 in){
|
||||
return _mm512_reduce_add_ps(in);
|
||||
}
|
||||
|
||||
|
||||
//Complex double Reduce
|
||||
template<>
|
||||
inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){
|
||||
return Grid::ComplexD(_mm512_mask_reduce_add_pd(0x55, in),_mm512_mask_reduce_add_pd(0xAA, in));
|
||||
}
|
||||
|
||||
//Real double Reduce
|
||||
template<>
|
||||
inline Grid::RealD Reduce<Grid::RealD, __m512d>::operator()(__m512d in){
|
||||
return _mm512_reduce_add_pd(in);
|
||||
}
|
||||
|
||||
//Integer Reduce
|
||||
template<>
|
||||
inline Integer Reduce<Integer, __m512i>::operator()(__m512i in){
|
||||
return _mm512_reduce_add_epi32(in);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
//Real float Reduce
|
||||
template<>
|
||||
inline Grid::RealF Reduce<Grid::RealF, __m512>::operator()(__m512 in){
|
||||
return _mm512_reduce_add_ps(in);
|
||||
}
|
||||
|
||||
|
||||
//Complex double Reduce
|
||||
template<>
|
||||
inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){
|
||||
return Grid::ComplexD(_mm512_mask_reduce_add_pd(0x55, in),_mm512_mask_reduce_add_pd(0xAA, in));
|
||||
}
|
||||
|
||||
//Real double Reduce
|
||||
template<>
|
||||
inline Grid::RealD Reduce<Grid::RealD, __m512d>::operator()(__m512d in){
|
||||
return _mm512_reduce_add_pd(in);
|
||||
}
|
||||
|
||||
//Integer Reduce
|
||||
template<>
|
||||
inline Integer Reduce<Integer, __m512i>::operator()(__m512i in){
|
||||
return _mm512_reduce_add_epi32(in);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Here assign types
|
||||
|
||||
typedef __m512 SIMD_Ftype; // Single precision type
|
||||
typedef __m512d SIMD_Dtype; // Double precision type
|
||||
typedef __m512i SIMD_Itype; // Integer type
|
||||
typedef __m512 SIMD_Ftype; // Single precision type
|
||||
typedef __m512d SIMD_Dtype; // Double precision type
|
||||
typedef __m512i SIMD_Itype; // Integer type
|
||||
|
||||
// prefecth
|
||||
inline void v_prefetch0(int size, const char *ptr){
|
||||
// prefecth
|
||||
inline void v_prefetch0(int size, const char *ptr){
|
||||
for(int i=0;i<size;i+=64){ // Define L1 linesize above
|
||||
_mm_prefetch(ptr+i+4096,_MM_HINT_T1);
|
||||
_mm_prefetch(ptr+i+512,_MM_HINT_T0);
|
||||
}
|
||||
}
|
||||
inline void prefetch_HINT_T0(const char *ptr){
|
||||
_mm_prefetch(ptr,_MM_HINT_T0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Function name aliases
|
||||
typedef Optimization::Vsplat VsplatSIMD;
|
||||
typedef Optimization::Vstore VstoreSIMD;
|
||||
typedef Optimization::Vset VsetSIMD;
|
||||
typedef Optimization::Vstream VstreamSIMD;
|
||||
template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
|
||||
|
||||
|
||||
// Arithmetic operations
|
||||
typedef Optimization::Sum SumSIMD;
|
||||
typedef Optimization::Sub SubSIMD;
|
||||
typedef Optimization::Div DivSIMD;
|
||||
typedef Optimization::Mult MultSIMD;
|
||||
typedef Optimization::MultComplex MultComplexSIMD;
|
||||
typedef Optimization::Conj ConjSIMD;
|
||||
typedef Optimization::TimesMinusI TimesMinusISIMD;
|
||||
typedef Optimization::TimesI TimesISIMD;
|
||||
|
||||
}
|
||||
inline void prefetch_HINT_T0(const char *ptr){
|
||||
_mm_prefetch(ptr,_MM_HINT_T0);
|
||||
}
|
||||
|
||||
// Function name aliases
|
||||
typedef Optimization::Vsplat VsplatSIMD;
|
||||
typedef Optimization::Vstore VstoreSIMD;
|
||||
typedef Optimization::Vset VsetSIMD;
|
||||
typedef Optimization::Vstream VstreamSIMD;
|
||||
template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
|
||||
|
||||
// Arithmetic operations
|
||||
typedef Optimization::Sum SumSIMD;
|
||||
typedef Optimization::Sub SubSIMD;
|
||||
typedef Optimization::Div DivSIMD;
|
||||
typedef Optimization::Mult MultSIMD;
|
||||
typedef Optimization::MultComplex MultComplexSIMD;
|
||||
typedef Optimization::Conj ConjSIMD;
|
||||
typedef Optimization::TimesMinusI TimesMinusISIMD;
|
||||
typedef Optimization::TimesI TimesISIMD;
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user