mirror of
https://github.com/paboyle/Grid.git
synced 2025-04-09 13:40:46 +01:00
clean up, reduction in acle
This commit is contained in:
nmeyer-ur
parent
28d49a3b60
commit
5f8a76d490
@ -2,17 +2,11 @@
|
|||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
Grid physics library, www.github.com/paboyle/Grid
|
||||||
|
|
||||||
Source file: ./lib/simd/Grid_a64fx-1.h
|
Source file: Grid_a64fx-2.h
|
||||||
|
|
||||||
Copyright (C) 2020
|
Copyright (C) 2020
|
||||||
|
|
||||||
Author: Nils Meyer <nils.meyer@ur.de>
|
Author: Nils Meyer <nils.meyer@ur.de>
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
Copyright (C) 2017
|
|
||||||
|
|
||||||
Author: Antonin Portelli <antonin.portelli@me.com>
|
|
||||||
Andrew Lawson <andrew.lawson1991@gmail.com>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
This program is free software; you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
@ -47,8 +41,8 @@ static_assert(GEN_SIMD_WIDTH % 64u == 0, "A64FX SIMD vector size is 64 bytes");
|
|||||||
#pragma error "Missing SVE feature"
|
#pragma error "Missing SVE feature"
|
||||||
#endif /* __ARM_FEATURE_SVE */
|
#endif /* __ARM_FEATURE_SVE */
|
||||||
|
|
||||||
namespace Grid {
|
NAMESPACE_BEGIN(Grid);
|
||||||
namespace Optimization {
|
NAMESPACE_BEGIN(Optimization);
|
||||||
|
|
||||||
// type traits giving the number of elements for each vector type
|
// type traits giving the number of elements for each vector type
|
||||||
template <typename T> struct W;
|
template <typename T> struct W;
|
||||||
@ -83,12 +77,12 @@ namespace Optimization {
|
|||||||
typedef vec<uint16_t> vech; // half precision comms
|
typedef vec<uint16_t> vech; // half precision comms
|
||||||
typedef vec<Integer> veci;
|
typedef vec<Integer> veci;
|
||||||
|
|
||||||
}} // Grid::Optimization
|
NAMESPACE_END(Optimization)
|
||||||
|
NAMESPACE_END(Grid)
|
||||||
|
|
||||||
// low-level API
|
// low-level API
|
||||||
namespace Grid {
|
NAMESPACE_BEGIN(Grid);
|
||||||
namespace Optimization {
|
NAMESPACE_BEGIN(Optimization);
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
struct acle{};
|
struct acle{};
|
||||||
@ -242,21 +236,16 @@ struct Vsplat{
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Vstore{
|
struct Vstore{
|
||||||
// Real
|
// Real
|
||||||
template <typename T>
|
template <typename T>
|
||||||
inline void operator()(vec<T> a, T *D){
|
inline void operator()(vec<T> a, T *D){
|
||||||
|
|
||||||
svbool_t pg1 = acle<T>::pg1();
|
svbool_t pg1 = acle<T>::pg1();
|
||||||
typename acle<T>::vt a_v = svld1(pg1, (typename acle<T>::pt*)&a.v);
|
typename acle<T>::vt a_v = svld1(pg1, (typename acle<T>::pt*)&a.v);
|
||||||
// NOTE illegal '&' here causes SIGBUS at runtime, related to CAS-35230-H2H6T1
|
svst1(pg1, D, a_v);
|
||||||
// svst1(pg1, (typename acle<T>::pt*)&D, a_v);
|
}
|
||||||
svst1(pg1, D, a_v);
|
};
|
||||||
|
|
||||||
// non temporal version
|
|
||||||
//svstnt1(pg1, D, a_v);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
struct Vstream{
|
struct Vstream{
|
||||||
// Real
|
// Real
|
||||||
@ -265,7 +254,6 @@ struct Vsplat{
|
|||||||
|
|
||||||
svbool_t pg1 = acle<T>::pg1();
|
svbool_t pg1 = acle<T>::pg1();
|
||||||
typename acle<T>::vt b_v = svld1(pg1, b.v);
|
typename acle<T>::vt b_v = svld1(pg1, b.v);
|
||||||
// FIXME non-temporal store causes compiler crash CAS-35230-H2H6T1
|
|
||||||
svstnt1(pg1, a, b_v);
|
svstnt1(pg1, a, b_v);
|
||||||
//svst1(pg1, a, b_v);
|
//svst1(pg1, a, b_v);
|
||||||
}
|
}
|
||||||
@ -297,40 +285,40 @@ struct Vsplat{
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////
|
||||||
// Arithmetic operations
|
// Arithmetic operations
|
||||||
/////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
|
||||||
struct Sum{
|
struct Sum{
|
||||||
template <typename T>
|
template <typename T>
|
||||||
inline vec<T> operator()(vec<T> a, vec<T> b){
|
inline vec<T> operator()(vec<T> a, vec<T> b){
|
||||||
|
|
||||||
vec<T> out;
|
vec<T> out;
|
||||||
svbool_t pg1 = acle<T>::pg1();
|
svbool_t pg1 = acle<T>::pg1();
|
||||||
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
||||||
typename acle<T>::vt b_v = svld1(pg1, b.v);
|
typename acle<T>::vt b_v = svld1(pg1, b.v);
|
||||||
typename acle<T>::vt r_v = svadd_x(pg1, a_v, b_v);
|
typename acle<T>::vt r_v = svadd_x(pg1, a_v, b_v);
|
||||||
svst1(pg1, out.v, r_v);
|
svst1(pg1, out.v, r_v);
|
||||||
|
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Sub{
|
struct Sub{
|
||||||
template <typename T>
|
template <typename T>
|
||||||
inline vec<T> operator()(vec<T> a, vec<T> b){
|
inline vec<T> operator()(vec<T> a, vec<T> b){
|
||||||
|
|
||||||
vec<T> out;
|
vec<T> out;
|
||||||
svbool_t pg1 = acle<T>::pg1();
|
svbool_t pg1 = acle<T>::pg1();
|
||||||
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
||||||
typename acle<T>::vt b_v = svld1(pg1, b.v);
|
typename acle<T>::vt b_v = svld1(pg1, b.v);
|
||||||
typename acle<T>::vt r_v = svsub_x(pg1, a_v, b_v);
|
typename acle<T>::vt r_v = svsub_x(pg1, a_v, b_v);
|
||||||
svst1(pg1, out.v, r_v);
|
svst1(pg1, out.v, r_v);
|
||||||
|
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
struct Mult{
|
struct Mult{
|
||||||
@ -440,45 +428,45 @@ struct Conj{
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
struct TimesMinusI{
|
struct TimesMinusI{
|
||||||
// Complex
|
// Complex
|
||||||
template <typename T>
|
template <typename T>
|
||||||
inline vec<T> operator()(vec<T> a, vec<T> b){
|
inline vec<T> operator()(vec<T> a, vec<T> b){
|
||||||
|
|
||||||
vec<T> out;
|
vec<T> out;
|
||||||
const vec<typename acle<T>::uint> tbl_swap = acle<T>::tbl_swap();
|
const vec<typename acle<T>::uint> tbl_swap = acle<T>::tbl_swap();
|
||||||
svbool_t pg1 = acle<T>::pg1();
|
svbool_t pg1 = acle<T>::pg1();
|
||||||
svbool_t pg_odd = acle<T>::pg_odd();
|
svbool_t pg_odd = acle<T>::pg_odd();
|
||||||
|
|
||||||
typename acle<T>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
|
typename acle<T>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
|
||||||
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
||||||
a_v = svtbl(a_v, tbl_swap_v);
|
a_v = svtbl(a_v, tbl_swap_v);
|
||||||
typename acle<T>::vt r_v = svneg_x(pg_odd, a_v);
|
typename acle<T>::vt r_v = svneg_x(pg_odd, a_v);
|
||||||
svst1(pg1, out.v, r_v);
|
svst1(pg1, out.v, r_v);
|
||||||
|
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
struct TimesI{
|
struct TimesI{
|
||||||
// Complex
|
// Complex
|
||||||
template <typename T>
|
template <typename T>
|
||||||
inline vec<T> operator()(vec<T> a, vec<T> b){
|
inline vec<T> operator()(vec<T> a, vec<T> b){
|
||||||
|
|
||||||
vec<T> out;
|
vec<T> out;
|
||||||
const vec<typename acle<T>::uint> tbl_swap = acle<T>::tbl_swap();
|
const vec<typename acle<T>::uint> tbl_swap = acle<T>::tbl_swap();
|
||||||
svbool_t pg1 = acle<T>::pg1();
|
svbool_t pg1 = acle<T>::pg1();
|
||||||
svbool_t pg_even = acle<T>::pg_even();
|
svbool_t pg_even = acle<T>::pg_even();
|
||||||
|
|
||||||
typename acle<T>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
|
typename acle<T>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
|
||||||
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
typename acle<T>::vt a_v = svld1(pg1, a.v);
|
||||||
a_v = svtbl(a_v, tbl_swap_v);
|
a_v = svtbl(a_v, tbl_swap_v);
|
||||||
typename acle<T>::vt r_v = svneg_x(pg_even, a_v);
|
typename acle<T>::vt r_v = svneg_x(pg_even, a_v);
|
||||||
svst1(pg1, out.v, r_v);
|
svst1(pg1, out.v, r_v);
|
||||||
|
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
struct PrecisionChange {
|
struct PrecisionChange {
|
||||||
@ -587,71 +575,71 @@ struct PrecisionChange {
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
struct Exchange{
|
struct Exchange{
|
||||||
|
|
||||||
// Exchange0 is valid for arbitrary SVE vector length
|
// Exchange0 is valid for arbitrary SVE vector length
|
||||||
template <typename T>
|
template <typename T>
|
||||||
static inline void Exchange0(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
|
static inline void Exchange0(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
|
||||||
|
|
||||||
svbool_t pg1 = acle<T>::pg1();
|
svbool_t pg1 = acle<T>::pg1();
|
||||||
typename acle<T>::vt a1_v = svld1(pg1, in1.v);
|
typename acle<T>::vt a1_v = svld1(pg1, in1.v);
|
||||||
typename acle<T>::vt a2_v = svld1(pg1, in2.v);
|
typename acle<T>::vt a2_v = svld1(pg1, in2.v);
|
||||||
typename acle<T>::vt r1_v = svext(a1_v, a1_v, (uint64_t)W<T>::c);
|
typename acle<T>::vt r1_v = svext(a1_v, a1_v, (uint64_t)W<T>::c);
|
||||||
r1_v = svext(r1_v, a2_v, (uint64_t)W<T>::c);
|
r1_v = svext(r1_v, a2_v, (uint64_t)W<T>::c);
|
||||||
typename acle<T>::vt r2_v = svext(a2_v, a2_v, (uint64_t)W<T>::c);
|
typename acle<T>::vt r2_v = svext(a2_v, a2_v, (uint64_t)W<T>::c);
|
||||||
r2_v = svext(a1_v, r2_v, (uint64_t)W<T>::c);
|
r2_v = svext(a1_v, r2_v, (uint64_t)W<T>::c);
|
||||||
svst1(pg1, out1.v, r1_v);
|
svst1(pg1, out1.v, r1_v);
|
||||||
svst1(pg1, out2.v, r2_v);
|
svst1(pg1, out2.v, r2_v);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
static inline void Exchange1(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
|
static inline void Exchange1(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
|
||||||
|
|
||||||
svbool_t pg4 = acle<double>::pg4();
|
svbool_t pg4 = acle<double>::pg4();
|
||||||
typename acle<double>::vt4 in1_v4 = svld4(pg4, (typename acle<double>::pt*)in1.v);
|
typename acle<double>::vt4 in1_v4 = svld4(pg4, (typename acle<double>::pt*)in1.v);
|
||||||
typename acle<double>::vt4 in2_v4 = svld4(pg4, (typename acle<double>::pt*)in2.v);
|
typename acle<double>::vt4 in2_v4 = svld4(pg4, (typename acle<double>::pt*)in2.v);
|
||||||
typename acle<double>::vt4 out1_v4;
|
typename acle<double>::vt4 out1_v4;
|
||||||
typename acle<double>::vt4 out2_v4;
|
typename acle<double>::vt4 out2_v4;
|
||||||
out1_v4.v0 = in1_v4.v0;
|
out1_v4.v0 = in1_v4.v0;
|
||||||
out1_v4.v1 = in1_v4.v1;
|
out1_v4.v1 = in1_v4.v1;
|
||||||
out1_v4.v2 = in2_v4.v0;
|
out1_v4.v2 = in2_v4.v0;
|
||||||
out1_v4.v3 = in2_v4.v1;
|
out1_v4.v3 = in2_v4.v1;
|
||||||
out2_v4.v0 = in1_v4.v2;
|
out2_v4.v0 = in1_v4.v2;
|
||||||
out2_v4.v1 = in1_v4.v3;
|
out2_v4.v1 = in1_v4.v3;
|
||||||
out2_v4.v2 = in2_v4.v2;
|
out2_v4.v2 = in2_v4.v2;
|
||||||
out2_v4.v3 = in2_v4.v3;
|
out2_v4.v3 = in2_v4.v3;
|
||||||
svst4(pg4, (typename acle<double>::pt*)out1.v, out1_v4);
|
svst4(pg4, (typename acle<double>::pt*)out1.v, out1_v4);
|
||||||
svst4(pg4, (typename acle<double>::pt*)out2.v, out2_v4);
|
svst4(pg4, (typename acle<double>::pt*)out2.v, out2_v4);
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
static inline void Exchange2(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
|
static inline void Exchange2(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
|
||||||
|
|
||||||
svbool_t pg1 = acle<double>::pg1();
|
svbool_t pg1 = acle<double>::pg1();
|
||||||
typename acle<double>::vt a1_v = svld1(pg1, (typename acle<double>::pt*)in1.v);
|
typename acle<double>::vt a1_v = svld1(pg1, (typename acle<double>::pt*)in1.v);
|
||||||
typename acle<double>::vt a2_v = svld1(pg1, (typename acle<double>::pt*)in2.v);
|
typename acle<double>::vt a2_v = svld1(pg1, (typename acle<double>::pt*)in2.v);
|
||||||
typename acle<double>::vt r1_v = svtrn1(a1_v, a2_v);
|
typename acle<double>::vt r1_v = svtrn1(a1_v, a2_v);
|
||||||
typename acle<double>::vt r2_v = svtrn2(a1_v, a2_v);
|
typename acle<double>::vt r2_v = svtrn2(a1_v, a2_v);
|
||||||
svst1(pg1, (typename acle<double>::pt*)out1.v, r1_v);
|
svst1(pg1, (typename acle<double>::pt*)out1.v, r1_v);
|
||||||
svst1(pg1, (typename acle<double>::pt*)out2.v, r2_v);
|
svst1(pg1, (typename acle<double>::pt*)out2.v, r2_v);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void Exchange3(vecf &out1, vecf &out2, const vecf &in1, const vecf &in2){
|
static inline void Exchange3(vecf &out1, vecf &out2, const vecf &in1, const vecf &in2){
|
||||||
|
|
||||||
svbool_t pg1 = acle<float>::pg1();
|
svbool_t pg1 = acle<float>::pg1();
|
||||||
typename acle<float>::vt a1_v = svld1(pg1, in1.v);
|
typename acle<float>::vt a1_v = svld1(pg1, in1.v);
|
||||||
typename acle<float>::vt a2_v = svld1(pg1, in2.v);
|
typename acle<float>::vt a2_v = svld1(pg1, in2.v);
|
||||||
typename acle<float>::vt r1_v = svtrn1(a1_v, a2_v);
|
typename acle<float>::vt r1_v = svtrn1(a1_v, a2_v);
|
||||||
typename acle<float>::vt r2_v = svtrn2(a1_v, a2_v);
|
typename acle<float>::vt r2_v = svtrn2(a1_v, a2_v);
|
||||||
svst1(pg1, out1.v, r1_v);
|
svst1(pg1, out1.v, r1_v);
|
||||||
svst1(pg1, out2.v, r2_v);
|
svst1(pg1, out2.v, r2_v);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void Exchange3(vecd &out1, vecd &out2, const vecd &in1, const vecd &in2){
|
static inline void Exchange3(vecd &out1, vecd &out2, const vecd &in1, const vecd &in2){
|
||||||
assert(0);
|
assert(0);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
@ -780,7 +768,7 @@ struct Rotate{
|
|||||||
};
|
};
|
||||||
|
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
/* SVE ACLE reducedoes not compile, check later
|
// SVE ACLE reduce does not compile, check later
|
||||||
|
|
||||||
// tree-based reduction
|
// tree-based reduction
|
||||||
#define svred(pg, v)\
|
#define svred(pg, v)\
|
||||||
@ -864,11 +852,11 @@ inline Integer Reduce<Integer, veci>::operator()(veci in){
|
|||||||
}
|
}
|
||||||
|
|
||||||
#undef svred
|
#undef svred
|
||||||
*/
|
// */
|
||||||
|
|
||||||
// =======================================================================
|
// =======================================================================
|
||||||
|
|
||||||
|
/*
|
||||||
#define acc(v, a, off, step, n)\
|
#define acc(v, a, off, step, n)\
|
||||||
for (unsigned int i = off; i < n; i += step)\
|
for (unsigned int i = off; i < n; i += step)\
|
||||||
{\
|
{\
|
||||||
@ -939,39 +927,39 @@ inline Integer Reduce<Integer, veci>::operator()(veci in){
|
|||||||
}
|
}
|
||||||
|
|
||||||
#undef acc // EIGEN compatibility
|
#undef acc // EIGEN compatibility
|
||||||
|
*/
|
||||||
|
|
||||||
|
NAMESPACE_END(Optimization)
|
||||||
} // Optimization
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////////////////////
|
||||||
// Here assign types
|
// Here assign types
|
||||||
|
|
||||||
typedef Optimization::vech SIMD_Htype; // Reduced precision type
|
typedef Optimization::vech SIMD_Htype; // Reduced precision type
|
||||||
typedef Optimization::vecf SIMD_Ftype; // Single precision type
|
typedef Optimization::vecf SIMD_Ftype; // Single precision type
|
||||||
typedef Optimization::vecd SIMD_Dtype; // Double precision type
|
typedef Optimization::vecd SIMD_Dtype; // Double precision type
|
||||||
typedef Optimization::veci SIMD_Itype; // Integer type
|
typedef Optimization::veci SIMD_Itype; // Integer type
|
||||||
|
|
||||||
// prefetch utilities
|
// prefetch utilities
|
||||||
inline void v_prefetch0(int size, const char *ptr){};
|
inline void v_prefetch0(int size, const char *ptr){};
|
||||||
inline void prefetch_HINT_T0(const char *ptr){};
|
inline void prefetch_HINT_T0(const char *ptr){};
|
||||||
|
|
||||||
// Function name aliases
|
// Function name aliases
|
||||||
typedef Optimization::Vsplat VsplatSIMD;
|
typedef Optimization::Vsplat VsplatSIMD;
|
||||||
typedef Optimization::Vstore VstoreSIMD;
|
typedef Optimization::Vstore VstoreSIMD;
|
||||||
typedef Optimization::Vset VsetSIMD;
|
typedef Optimization::Vset VsetSIMD;
|
||||||
typedef Optimization::Vstream VstreamSIMD;
|
typedef Optimization::Vstream VstreamSIMD;
|
||||||
template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
|
template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
|
||||||
|
|
||||||
// Arithmetic operations
|
// Arithmetic operations
|
||||||
typedef Optimization::Sum SumSIMD;
|
typedef Optimization::Sum SumSIMD;
|
||||||
typedef Optimization::Sub SubSIMD;
|
typedef Optimization::Sub SubSIMD;
|
||||||
typedef Optimization::Div DivSIMD;
|
typedef Optimization::Div DivSIMD;
|
||||||
typedef Optimization::Mult MultSIMD;
|
typedef Optimization::Mult MultSIMD;
|
||||||
typedef Optimization::MultComplex MultComplexSIMD;
|
typedef Optimization::MultComplex MultComplexSIMD;
|
||||||
typedef Optimization::MultRealPart MultRealPartSIMD;
|
typedef Optimization::MultRealPart MultRealPartSIMD;
|
||||||
typedef Optimization::MaddRealPart MaddRealPartSIMD;
|
typedef Optimization::MaddRealPart MaddRealPartSIMD;
|
||||||
typedef Optimization::Conj ConjSIMD;
|
typedef Optimization::Conj ConjSIMD;
|
||||||
typedef Optimization::TimesMinusI TimesMinusISIMD;
|
typedef Optimization::TimesMinusI TimesMinusISIMD;
|
||||||
typedef Optimization::TimesI TimesISIMD;
|
typedef Optimization::TimesI TimesISIMD;
|
||||||
|
|
||||||
}
|
NAMESPACE_END(Grid)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user