mirror of
https://github.com/paboyle/Grid.git
synced 2024-11-09 23:45:36 +00:00
Half precision conversion
This commit is contained in:
paboyle
parent
b12dc89d26
commit
cb6b81ae82
10
configure.ac
10
configure.ac
@ -176,19 +176,19 @@ case ${ax_cv_cxx_compiler_vendor} in
|
||||
case ${ac_SIMD} in
|
||||
SSE4)
|
||||
AC_DEFINE([SSE4],[1],[SSE4 intrinsics])
|
||||
SIMD_FLAGS='-msse4.2';;
|
||||
SIMD_FLAGS='-msse4.2 -mf16c';;
|
||||
AVX)
|
||||
AC_DEFINE([AVX1],[1],[AVX intrinsics])
|
||||
SIMD_FLAGS='-mavx';;
|
||||
SIMD_FLAGS='-mavx -mf16c';;
|
||||
AVXFMA4)
|
||||
AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
|
||||
SIMD_FLAGS='-mavx -mfma4';;
|
||||
SIMD_FLAGS='-mavx -mfma4 -mf16c';;
|
||||
AVXFMA)
|
||||
AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA3])
|
||||
SIMD_FLAGS='-mavx -mfma';;
|
||||
SIMD_FLAGS='-mavx -mfma -mf16c';;
|
||||
AVX2)
|
||||
AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
|
||||
SIMD_FLAGS='-mavx2 -mfma';;
|
||||
SIMD_FLAGS='-mavx2 -mfma -mf16c';;
|
||||
AVX512)
|
||||
AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
|
||||
SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
|
||||
|
||||
@ -471,6 +471,42 @@ namespace Optimization {
|
||||
};
|
||||
};
|
||||
|
||||
struct PrecisionChange {
|
||||
static inline __m256i StoH (__m256 a,__m256 b) {
|
||||
__m128i ha = _mm256_cvtps_ph(a,0);
|
||||
__m128i hb = _mm256_cvtps_ph(b,0);
|
||||
__m256 h = _mm256_castps128_ps256(ha);
|
||||
h = _mm256_insertf128_ps(h,hb,1);
|
||||
return h;
|
||||
}
|
||||
static inline void HtoS (__m256i h,__m256 &sa,__m256 &sb) {
|
||||
sa = _mm256_cvtph_ps(_mm256_extractf128_ps(h,0));
|
||||
sb = _mm256_cvtph_ps(_mm256_extractf128_ps(h,1));
|
||||
}
|
||||
static inline __m256 DtoS (__m256d a,__m256d b) {
|
||||
__m128 sa = _mm256_cvtpd_ps(a);
|
||||
__m128 sb = _mm256_cvtpd_ps(b);
|
||||
__m256 s = _mm256_castps128_ps256(sa);
|
||||
s = _mm256_insertf128_ps(s,sb,1);
|
||||
return s;
|
||||
}
|
||||
static inline void StoD (__m256 s,__m256d &a,__m256d &b) {
|
||||
a = _mm256_cvtps_pd(_mm256_extractf128_ps(s,0));
|
||||
b = _mm256_cvtps_pd(_mm256_extractf128_ps(s,1));
|
||||
}
|
||||
static inline __m256 DtoH (__m256i a,__m256 b,__m256 c,__m256 d) {
|
||||
__m256 sa,sb;
|
||||
sa = DtoS(a,b);
|
||||
sb = DtoS(c,d);
|
||||
return StoH(sa,sb);
|
||||
}
|
||||
static inline void HtoD (__m256i h,__m256d &a,__m256d &b,__m256d &c,__m256d &d) {
|
||||
__m256 sa,sb;
|
||||
HtoS(h,sa,sb);
|
||||
StoD(sa,a,b);
|
||||
StoD(sb,c,d);
|
||||
}
|
||||
};
|
||||
struct Exchange{
|
||||
// 3210 ordering
|
||||
static inline void Exchange0(__m256 &out1,__m256 &out2,__m256 in1,__m256 in2){
|
||||
@ -675,6 +711,7 @@ namespace Optimization {
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Here assign types
|
||||
|
||||
typedef __m256i SIMD_Htype; // Single precision type
|
||||
typedef __m256 SIMD_Ftype; // Single precision type
|
||||
typedef __m256d SIMD_Dtype; // Double precision type
|
||||
typedef __m256i SIMD_Itype; // Integer type
|
||||
|
||||
@ -343,6 +343,42 @@ namespace Optimization {
|
||||
|
||||
};
|
||||
|
||||
struct PrecisionChange {
|
||||
static inline __m512i StoH (__m512 a,__m512 b) {
|
||||
__m256i ha = _mm512_cvtps_ph(a,0);
|
||||
__m256i hb = _mm512_cvtps_ph(b,0);
|
||||
__m512 h = _mm512_castps256_ps512(ha);
|
||||
h = _mm512_insertf256_ps(h,hb,1);
|
||||
return h;
|
||||
}
|
||||
static inline void HtoS (__m512i h,__m512 &sa,__m512 &sb) {
|
||||
sa = _mm512_cvtph_ps(_mm512_extractf256_ps(h,0));
|
||||
sb = _mm512_cvtph_ps(_mm512_extractf256_ps(h,1));
|
||||
}
|
||||
static inline __m512 DtoS (__m512d a,__m512d b) {
|
||||
__m256 sa = _mm512_cvtpd_ps(a);
|
||||
__m256 sb = _mm512_cvtpd_ps(b);
|
||||
__m512 s = _mm512_castps256_ps512(sa);
|
||||
s = _mm512_insertf256_ps(s,sb,1);
|
||||
return s;
|
||||
}
|
||||
static inline void StoD (__m512 s,__m512d &a,__m512d &b) {
|
||||
a = _mm512_cvtps_pd(_mm512_extractf256_ps(s,0));
|
||||
b = _mm512_cvtps_pd(_mm512_extractf256_ps(s,1));
|
||||
}
|
||||
static inline __m512 DtoH (__m512i a,__m512 b,__m512 c,__m512 d) {
|
||||
__m512 sa,sb;
|
||||
sa = DtoS(a,b);
|
||||
sb = DtoS(c,d);
|
||||
return StoH(sa,sb);
|
||||
}
|
||||
static inline void HtoD (__m512i h,__m512d &a,__m512d &b,__m512d &c,__m512d &d) {
|
||||
__m512 sa,sb;
|
||||
HtoS(h,sa,sb);
|
||||
StoD(sa,a,b);
|
||||
StoD(sb,c,d);
|
||||
}
|
||||
};
|
||||
// On extracting face: Ah Al , Bh Bl -> Ah Bh, Al Bl
|
||||
// On merging buffers: Ah,Bh , Al Bl -> Ah Al, Bh, Bl
|
||||
// The operation is its own inverse
|
||||
|
||||
@ -38,6 +38,7 @@ Author: neo <cossu@post.kek.jp>
|
||||
|
||||
#include <pmmintrin.h>
|
||||
|
||||
|
||||
namespace Grid {
|
||||
namespace Optimization {
|
||||
|
||||
@ -328,6 +329,48 @@ namespace Optimization {
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
#ifndef _mm_alignr_epi64
|
||||
#define _mm_alignr_epi32(a,b,n) _mm_alignr_epi8(a,b,(n*4)%16)
|
||||
#define _mm_alignr_epi64(a,b,n) _mm_alignr_epi8(a,b,(n*8)%16)
|
||||
#endif
|
||||
struct PrecisionChange {
|
||||
static inline __m128i StoH (__m128 a,__m128 b) {
|
||||
__m128i ha = _mm_cvtps_ph(a,0);
|
||||
__m128i hb = _mm_cvtps_ph(b,0);
|
||||
__m128i h =(__m128i) _mm_shuffle_ps((__m128)ha,(__m128)hb,_MM_SELECT_FOUR_FOUR(1,0,1,0));
|
||||
return h;
|
||||
}
|
||||
static inline void HtoS (__m128i h,__m128 &sa,__m128 &sb) {
|
||||
sa = _mm_cvtph_ps(h);
|
||||
h = (__m128)_mm_alignr_epi32((__m128i)h,(__m128i)h,2);
|
||||
sb = _mm_cvtph_ps(h);
|
||||
}
|
||||
static inline __m128 DtoS (__m128d a,__m128d b) {
|
||||
__m128 sa = _mm_cvtpd_ps(a);
|
||||
__m128 sb = _mm_cvtpd_ps(b);
|
||||
__m128 s = _mm_shuffle_ps(sa,sb,_MM_SELECT_FOUR_FOUR(1,0,1,0));
|
||||
return s;
|
||||
}
|
||||
static inline void StoD (__m128 s,__m128d &a,__m128d &b) {
|
||||
a = _mm_cvtps_pd(s);
|
||||
s = (__m128)_mm_alignr_epi32((__m128i)s,(__m128i)s,2);
|
||||
b = _mm_cvtps_pd(s);
|
||||
}
|
||||
static inline __m128 DtoH (__m128i a,__m128 b,__m128 c,__m128 d) {
|
||||
__m128 sa,sb;
|
||||
sa = DtoS(a,b);
|
||||
sb = DtoS(c,d);
|
||||
return StoH(sa,sb);
|
||||
}
|
||||
static inline void HtoD (__m128i h,__m128d &a,__m128d &b,__m128d &c,__m128d &d) {
|
||||
__m128 sa,sb;
|
||||
HtoS(h,sa,sb);
|
||||
StoD(sa,a,b);
|
||||
StoD(sb,c,d);
|
||||
}
|
||||
};
|
||||
|
||||
struct Exchange{
|
||||
// 3210 ordering
|
||||
static inline void Exchange0(__m128 &out1,__m128 &out2,__m128 in1,__m128 in2){
|
||||
@ -335,8 +378,10 @@ namespace Optimization {
|
||||
out2= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(3,2,3,2));
|
||||
};
|
||||
static inline void Exchange1(__m128 &out1,__m128 &out2,__m128 in1,__m128 in2){
|
||||
out1= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(2,0,2,0));
|
||||
out2= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(3,1,3,1));
|
||||
out1= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(2,0,2,0)); /*ACEG*/
|
||||
out2= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(3,1,3,1)); /*BDFH*/
|
||||
out1= _mm_shuffle_ps(out1,out1,_MM_SELECT_FOUR_FOUR(3,1,2,0)); /*AECG*/
|
||||
out2= _mm_shuffle_ps(out2,out2,_MM_SELECT_FOUR_FOUR(3,1,2,0)); /*AECG*/
|
||||
};
|
||||
static inline void Exchange2(__m128 &out1,__m128 &out2,__m128 in1,__m128 in2){
|
||||
assert(0);
|
||||
@ -383,11 +428,6 @@ namespace Optimization {
|
||||
default: assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef _mm_alignr_epi64
|
||||
#define _mm_alignr_epi32(a,b,n) _mm_alignr_epi8(a,b,(n*4)%16)
|
||||
#define _mm_alignr_epi64(a,b,n) _mm_alignr_epi8(a,b,(n*8)%16)
|
||||
#endif
|
||||
|
||||
template<int n> static inline __m128 tRotate(__m128 in){ return (__m128)_mm_alignr_epi32((__m128i)in,(__m128i)in,n); };
|
||||
template<int n> static inline __m128d tRotate(__m128d in){ return (__m128d)_mm_alignr_epi64((__m128i)in,(__m128i)in,n); };
|
||||
@ -450,7 +490,8 @@ namespace Optimization {
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Here assign types
|
||||
|
||||
typedef __m128 SIMD_Ftype; // Single precision type
|
||||
typedef __m128i SIMD_Htype; // Single precision type
|
||||
typedef __m128 SIMD_Ftype; // Single precision type
|
||||
typedef __m128d SIMD_Dtype; // Double precision type
|
||||
typedef __m128i SIMD_Itype; // Integer type
|
||||
|
||||
|
||||
@ -358,16 +358,12 @@ class Grid_simd {
|
||||
{
|
||||
if (n==3) {
|
||||
Optimization::Exchange::Exchange3(out1.v,out2.v,in1.v,in2.v);
|
||||
// std::cout << " Exchange3 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
|
||||
} else if(n==2) {
|
||||
Optimization::Exchange::Exchange2(out1.v,out2.v,in1.v,in2.v);
|
||||
// std::cout << " Exchange2 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
|
||||
} else if(n==1) {
|
||||
Optimization::Exchange::Exchange1(out1.v,out2.v,in1.v,in2.v);
|
||||
// std::cout << " Exchange1 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
|
||||
} else if(n==0) {
|
||||
Optimization::Exchange::Exchange0(out1.v,out2.v,in1.v,in2.v);
|
||||
// std::cout << " Exchange0 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
@ -758,6 +754,67 @@ typedef Grid_simd<std::complex<float>, SIMD_Ftype> vComplexF;
|
||||
typedef Grid_simd<std::complex<double>, SIMD_Dtype> vComplexD;
|
||||
typedef Grid_simd<Integer, SIMD_Itype> vInteger;
|
||||
|
||||
// Half precision; no arithmetic support
|
||||
typedef Grid_simd<uint16_t, SIMD_Htype> vRealH;
|
||||
typedef Grid_simd<std::complex<uint16_t>, SIMD_Htype> vComplexH;
|
||||
|
||||
inline void precisionChange(vRealF *out,vRealD *in,int nvec)
|
||||
{
|
||||
assert((nvec&0x1)==0);
|
||||
for(int m=0;m*2<nvec;m++){
|
||||
int n=m*2;
|
||||
out[m].v=Optimization::PrecisionChange::DtoS(in[n].v,in[n+1].v);
|
||||
}
|
||||
}
|
||||
inline void precisionChange(vRealH *out,vRealD *in,int nvec)
|
||||
{
|
||||
assert((nvec&0x3)==0);
|
||||
for(int m=0;m*4<nvec;m++){
|
||||
int n=m*4;
|
||||
out[m].v=Optimization::PrecisionChange::DtoH(in[n].v,in[n+1].v,in[n+2].v,in[n+3].v);
|
||||
}
|
||||
}
|
||||
inline void precisionChange(vRealH *out,vRealF *in,int nvec)
|
||||
{
|
||||
assert((nvec&0x1)==0);
|
||||
for(int m=0;m*2<nvec;m++){
|
||||
int n=m*2;
|
||||
out[m].v=Optimization::PrecisionChange::StoH(in[n].v,in[n+1].v);
|
||||
}
|
||||
}
|
||||
inline void precisionChange(vRealD *out,vRealF *in,int nvec)
|
||||
{
|
||||
assert((nvec&0x1)==0);
|
||||
for(int m=0;m*2<nvec;m++){
|
||||
int n=m*2;
|
||||
Optimization::PrecisionChange::StoD(in[m].v,out[n].v,out[n+1].v);
|
||||
}
|
||||
}
|
||||
inline void precisionChange(vRealD *out,vRealH *in,int nvec)
|
||||
{
|
||||
assert((nvec&0x3)==0);
|
||||
for(int m=0;m*4<nvec;m++){
|
||||
int n=m*4;
|
||||
Optimization::PrecisionChange::HtoD(in[m].v,out[n].v,out[n+1].v,out[n+2].v,out[n+3].v);
|
||||
}
|
||||
}
|
||||
inline void precisionChange(vRealF *out,vRealH *in,int nvec)
|
||||
{
|
||||
assert((nvec&0x1)==0);
|
||||
for(int m=0;m*2<nvec;m++){
|
||||
int n=m*2;
|
||||
Optimization::PrecisionChange::HtoS(in[m].v,out[n].v,out[n+1].v);
|
||||
}
|
||||
}
|
||||
inline void precisionChange(vComplexF *out,vComplexD *in,int nvec){ precisionChange((vRealF *)out,(vRealD *)in,nvec);}
|
||||
inline void precisionChange(vComplexH *out,vComplexD *in,int nvec){ precisionChange((vRealH *)out,(vRealD *)in,nvec);}
|
||||
inline void precisionChange(vComplexH *out,vComplexF *in,int nvec){ precisionChange((vRealH *)out,(vRealF *)in,nvec);}
|
||||
inline void precisionChange(vComplexD *out,vComplexF *in,int nvec){ precisionChange((vRealD *)out,(vRealF *)in,nvec);}
|
||||
inline void precisionChange(vComplexD *out,vComplexH *in,int nvec){ precisionChange((vRealD *)out,(vRealH *)in,nvec);}
|
||||
inline void precisionChange(vComplexF *out,vComplexH *in,int nvec){ precisionChange((vRealF *)out,(vRealH *)in,nvec);}
|
||||
|
||||
|
||||
|
||||
// Check our vector types are of an appropriate size.
|
||||
#if defined QPX
|
||||
static_assert(2*sizeof(SIMD_Ftype) == sizeof(SIMD_Dtype), "SIMD vector lengths incorrect");
|
||||
|
||||
@ -478,12 +478,19 @@ void ExchangeTester(const functor &func)
|
||||
assert(found==1);
|
||||
}
|
||||
|
||||
/*
|
||||
for(int i=0;i<Nsimd;i++){
|
||||
std::cout << " i "<< i
|
||||
<<" test1 "<<test1[i]
|
||||
<<" test2 "<<test2[i]
|
||||
<<" input1 "<<input1[i]
|
||||
<<" input2 "<<input2[i]<<std::endl;
|
||||
}
|
||||
*/
|
||||
for(int i=0;i<Nsimd;i++){
|
||||
assert(test1[i]==input1[i]);
|
||||
assert(test2[i]==input2[i]);
|
||||
}// std::cout << " i "<< i<<" test1"<<test1[i]<<" "<<input1[i]<<std::endl;
|
||||
// std::cout << " i "<< i<<" test2"<<test2[i]<<" "<<input2[i]<<std::endl;
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -678,5 +685,68 @@ int main (int argc, char ** argv)
|
||||
IntTester(funcMinus());
|
||||
IntTester(funcTimes());
|
||||
|
||||
std::cout<<GridLogMessage << "==================================="<< std::endl;
|
||||
std::cout<<GridLogMessage << "Testing precisionChange "<< std::endl;
|
||||
std::cout<<GridLogMessage << "==================================="<< std::endl;
|
||||
{
|
||||
GridSerialRNG sRNG;
|
||||
sRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
|
||||
const int Ndp = 16;
|
||||
const int Nsp = Ndp/2;
|
||||
const int Nhp = Ndp/4;
|
||||
std::vector<vRealH,alignedAllocator<vRealH> > H (Nhp);
|
||||
std::vector<vRealF,alignedAllocator<vRealF> > F (Nsp);
|
||||
std::vector<vRealF,alignedAllocator<vRealF> > FF(Nsp);
|
||||
std::vector<vRealD,alignedAllocator<vRealD> > D (Ndp);
|
||||
std::vector<vRealD,alignedAllocator<vRealD> > DD(Ndp);
|
||||
for(int i=0;i<16;i++){
|
||||
random(sRNG,D[i]);
|
||||
}
|
||||
// Double to Single
|
||||
precisionChange(&F[0],&D[0],Ndp);
|
||||
precisionChange(&DD[0],&F[0],Ndp);
|
||||
std::cout << GridLogMessage<<"Double to single";
|
||||
for(int i=0;i<Ndp;i++){
|
||||
// std::cout << "DD["<<i<<"] = "<< DD[i]<<" "<<D[i]<<" "<<DD[i]-D[i] <<std::endl;
|
||||
DD[i] = DD[i] - D[i];
|
||||
decltype(innerProduct(DD[0],DD[0])) nrm;
|
||||
nrm = innerProduct(DD[i],DD[i]);
|
||||
auto tmp = Reduce(nrm);
|
||||
// std::cout << tmp << std::endl;
|
||||
assert( tmp < 1.0e-14 );
|
||||
}
|
||||
std::cout <<" OK ! "<<std::endl;
|
||||
|
||||
// Double to Half
|
||||
std::cout << GridLogMessage<< "Double to half" ;
|
||||
precisionChange(&H[0],&D[0],Ndp);
|
||||
precisionChange(&DD[0],&H[0],Ndp);
|
||||
for(int i=0;i<Ndp;i++){
|
||||
// std::cout << "DD["<<i<<"] = "<< DD[i]<<" "<<D[i]<<" "<<DD[i]-D[i]<<std::endl;
|
||||
DD[i] = DD[i] - D[i];
|
||||
decltype(innerProduct(DD[0],DD[0])) nrm;
|
||||
nrm = innerProduct(DD[i],DD[i]);
|
||||
auto tmp = Reduce(nrm);
|
||||
// std::cout << tmp << std::endl;
|
||||
assert( tmp < 1.0e-6 );
|
||||
}
|
||||
std::cout <<" OK ! "<<std::endl;
|
||||
|
||||
std::cout << GridLogMessage<< "Single to half";
|
||||
// Single to Half
|
||||
precisionChange(&H[0] ,&F[0],Nsp);
|
||||
precisionChange(&FF[0],&H[0],Nsp);
|
||||
for(int i=0;i<Nsp;i++){
|
||||
// std::cout << "FF["<<i<<"] = "<< FF[i]<<" "<<F[i]<<" "<<FF[i]-F[i]<<std::endl;
|
||||
FF[i] = FF[i] - F[i];
|
||||
decltype(innerProduct(FF[0],FF[0])) nrm;
|
||||
nrm = innerProduct(FF[i],FF[i]);
|
||||
auto tmp = Reduce(nrm);
|
||||
// std::cout << tmp << std::endl;
|
||||
assert( tmp < 1.0e-6 );
|
||||
}
|
||||
std::cout <<" OK ! "<<std::endl;
|
||||
|
||||
}
|
||||
Grid_finalize();
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user