portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-19 08:59:36 +01:00
Code Releases Activity

definetely the right merge upstream/develop

Browse Source
This commit is contained in:
Alessandro Lupo
2023-06-16 14:19:46 +01:00
parent 4895ff260e ffd7301649
commit e09dfbf1c2
446 changed files with 46860 additions and 16333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Grid/tensors/Tensor_SIMT.h
+21
View File
@@ -31,6 +31,27 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
NAMESPACE_BEGIN(Grid);
////////////////////////////////////////////////
// Inside a GPU thread
////////////////////////////////////////////////
template<class vobj>
accelerator_inline void exchangeSIMT(vobj &mp0,vobj &mp1,const vobj &vp0,const vobj &vp1,Integer type)
{
typedef decltype(coalescedRead(mp0)) sobj;
unsigned int Nsimd = vobj::Nsimd();
unsigned int mask = Nsimd >> (type + 1);
int lane = acceleratorSIMTlane(Nsimd);
int j0 = lane &(~mask); // inner coor zero
int j1 = lane |(mask) ; // inner coor one
const vobj *vpa = &vp0;
const vobj *vpb = &vp1;
const vobj *vp = (lane&mask) ? (vpb) : (vpa);
auto sa = coalescedRead(vp[0],j0);
auto sb = coalescedRead(vp[0],j1);
coalescedWrite(mp0,sa);
coalescedWrite(mp1,sb);
}
#ifndef GRID_SIMT
//////////////////////////////////////////
Grid/tensors/Tensor_class.h
+3
View File
@@ -178,6 +178,7 @@ public:
stream << "S {" << o._internal << "}";
return stream;
};
// FIXME These will break with change of data layout
strong_inline const scalar_type * begin() const { return reinterpret_cast<const scalar_type *>(&_internal); }
strong_inline scalar_type * begin() { return reinterpret_cast< scalar_type *>(&_internal); }
strong_inline const scalar_type * end() const { return begin() + Traits::count; }
@@ -288,6 +289,7 @@ public:
// return _internal[i];
// }
// FIXME These will break with change of data layout
strong_inline const scalar_type * begin() const { return reinterpret_cast<const scalar_type *>(_internal); }
strong_inline scalar_type * begin() { return reinterpret_cast< scalar_type *>(_internal); }
strong_inline const scalar_type * end() const { return begin() + Traits::count; }
@@ -430,6 +432,7 @@ public:
// return _internal[i][j];
// }
// FIXME These will break with change of data layout
strong_inline const scalar_type * begin() const { return reinterpret_cast<const scalar_type *>(_internal[0]); }
strong_inline scalar_type * begin() { return reinterpret_cast< scalar_type *>(_internal[0]); }
strong_inline const scalar_type * end() const { return begin() + Traits::count; }
Grid/tensors/Tensor_exp.h
+1 -1
View File
@@ -55,7 +55,7 @@ template<class vtype, int N> accelerator_inline iVector<vtype, N> Exponentiate(c
// Specialisation: Cayley-Hamilton exponential for SU(3)
#ifndef GRID_CUDA
#ifndef GRID_ACCELERATED
template<class vtype, typename std::enable_if< GridTypeMapper<vtype>::TensorLevel == 0>::type * =nullptr>
accelerator_inline iMatrix<vtype,3> Exponentiate(const iMatrix<vtype,3> &arg, RealD alpha , Integer Nexp = DEFAULT_MAT_EXP )
{
Grid/tensors/Tensor_extract_merge.h
+64 -22
View File
@@ -1,5 +1,5 @@
/*************************************************************************************
n
Grid physics library, www.github.com/paboyle/Grid
Source file: ./lib/tensors/Tensor_extract_merge.h
@@ -62,8 +62,18 @@ void extract(const vobj &vec,ExtractBuffer<sobj> &extracted)
const int words=sizeof(vobj)/sizeof(vector_type);
const int Nsimd=vector_type::Nsimd();
const int Nextr=extracted.size();
vector_type * vp = (vector_type *)&vec;
const int s=Nsimd/Nextr;
sobj_scalar_type *sp = (sobj_scalar_type *) &extracted[0];
sobj_scalar_type stmp;
for(int w=0;w<words;w++){
for(int i=0;i<Nextr;i++){
stmp = vp[w].getlane(i*s);
sp[i*words+w] =stmp;
// memcpy((char *)&sp[i*words+w],(char *)&stmp,sizeof(stmp));
}
}
/*
scalar_type *vp = (scalar_type *)&vec;
scalar_type vtmp;
sobj_scalar_type stmp;
@@ -74,6 +84,8 @@ void extract(const vobj &vec,ExtractBuffer<sobj> &extracted)
memcpy((char *)&sp[i*words+w],(char *)&stmp,sizeof(stmp));
}
}
*/
return;
}
@@ -93,7 +105,7 @@ void merge(vobj &vec,ExtractBuffer<sobj> &extracted)
const int s=Nsimd/Nextr;
sobj_scalar_type *sp = (sobj_scalar_type *)&extracted[0];
scalar_type *vp = (scalar_type *)&vec;
vector_type *vp = (vector_type *)&vec;
scalar_type vtmp;
sobj_scalar_type stmp;
for(int w=0;w<words;w++){
@@ -101,7 +113,8 @@ void merge(vobj &vec,ExtractBuffer<sobj> &extracted)
for(int ii=0;ii<s;ii++){
memcpy((char *)&stmp,(char *)&sp[i*words+w],sizeof(stmp));
vtmp = stmp;
memcpy((char *)&vp[w*Nsimd+i*s+ii],(char *)&vtmp,sizeof(vtmp));
vp[w].putlane(vtmp,i*s+ii);
// memcpy((char *)&vp[w*Nsimd+i*s+ii],(char *)&vtmp,sizeof(vtmp));
}
}
}
@@ -117,16 +130,15 @@ typename vobj::scalar_object extractLane(int lane, const vobj & __restrict__ vec
typedef typename vobj::scalar_object scalar_object;
typedef typename vobj::vector_type vector_type;
typedef typename ExtractTypeMap<scalar_type>::extract_type extract_type;
typedef extract_type * pointer;
typedef scalar_type * pointer;
constexpr int words=sizeof(vobj)/sizeof(vector_type);
constexpr int Nsimd=vector_type::Nsimd();
scalar_object extracted;
pointer __restrict__ sp = (pointer)&extracted; // Type pun
pointer __restrict__ vp = (pointer)&vec;
vector_type *vp = (vector_type *)&vec;
for(int w=0;w<words;w++){
sp[w]=vp[w*Nsimd+lane];
sp[w]=vp[w].getlane(lane);
}
return extracted;
}
@@ -137,15 +149,14 @@ void insertLane(int lane, vobj & __restrict__ vec,const typename vobj::scalar_ob
typedef typename vobj::vector_type vector_type;
typedef typename vector_type::scalar_type scalar_type;
typedef typename ExtractTypeMap<scalar_type>::extract_type extract_type;
typedef extract_type * pointer;
typedef scalar_type * pointer;
constexpr int words=sizeof(vobj)/sizeof(vector_type);
constexpr int Nsimd=vector_type::Nsimd();
pointer __restrict__ sp = (pointer)&extracted;
pointer __restrict__ vp = (pointer)&vec;
vector_type *vp = (vector_type *)&vec;
for(int w=0;w<words;w++){
vp[w*Nsimd+lane]=sp[w];
vp[w].putlane(sp[w],lane);
}
}
@@ -164,15 +175,11 @@ void extract(const vobj &vec,const ExtractPointerArray<sobj> &extracted, int off
const int Nextr=extracted.size();
const int s = Nsimd/Nextr;
scalar_type * vp = (scalar_type *)&vec;
scalar_type vtmp;
sobj_scalar_type stmp;
vector_type * vp = (vector_type *)&vec;
for(int w=0;w<words;w++){
for(int i=0;i<Nextr;i++){
sobj_scalar_type * pointer = (sobj_scalar_type *)& extracted[i][offset];
memcpy((char *)&vtmp,(char *)&vp[w*Nsimd+i*s],sizeof(vtmp));
stmp = vtmp;
memcpy((char *)&pointer[w],(char *)&stmp,sizeof(stmp)); // may do a precision conversion
pointer[w] = vp[w].getlane(i*s);
}
}
}
@@ -192,21 +199,56 @@ void merge(vobj &vec,const ExtractPointerArray<sobj> &extracted, int offset)
const int Nextr=extracted.size();
const int s = Nsimd/Nextr;
scalar_type * vp = (scalar_type *)&vec;
vector_type * vp = (vector_type *)&vec;
scalar_type vtmp;
sobj_scalar_type stmp;
for(int w=0;w<words;w++){
for(int i=0;i<Nextr;i++){
sobj_scalar_type * pointer = (sobj_scalar_type *)& extracted[i][offset];
for(int ii=0;ii<s;ii++){
memcpy((char *)&stmp,(char *)&pointer[w],sizeof(stmp));
vtmp=stmp;
memcpy((char *)&vp[w*Nsimd+i*s+ii],(char *)&vtmp,sizeof(vtmp));
vtmp=pointer[w];
vp[w].putlane(vtmp,i*s+ii);
}
}
}
}
//////////////////////////////////////////////////////////////////////////////////
//Copy a single lane of a SIMD tensor type from one object to another
//Output object must be of the same tensor type but may be of a different precision (i.e. it can have a different root data type)
///////////////////////////////////////////////////////////////////////////////////
template<class vobjOut, class vobjIn>
accelerator_inline
void copyLane(vobjOut & __restrict__ vecOut, int lane_out, const vobjIn & __restrict__ vecIn, int lane_in)
{
static_assert( std::is_same<typename vobjOut::scalar_typeD, typename vobjIn::scalar_typeD>::value == 1, "copyLane: tensor types must be the same" ); //if tensor types are same the DoublePrecision type must be the same
typedef typename vobjOut::vector_type ovector_type;
typedef typename vobjIn::vector_type ivector_type;
constexpr int owords=sizeof(vobjOut)/sizeof(ovector_type);
constexpr int iwords=sizeof(vobjIn)/sizeof(ivector_type);
static_assert( owords == iwords, "copyLane: Expected number of vector words in input and output objects to be equal" );
typedef typename vobjOut::scalar_type oscalar_type;
typedef typename vobjIn::scalar_type iscalar_type;
typedef typename ExtractTypeMap<oscalar_type>::extract_type oextract_type;
typedef typename ExtractTypeMap<iscalar_type>::extract_type iextract_type;
typedef oextract_type * opointer;
typedef iextract_type * ipointer;
iscalar_type itmp;
oscalar_type otmp;
ovector_type * __restrict__ op = (ovector_type *)&vecOut;
ivector_type * __restrict__ ip = (ivector_type *)&vecIn;
for(int w=0;w<owords;w++){
itmp = ip[w].getlane(lane_in);
otmp = itmp; //potential precision change
op[w].putlane(otmp,lane_out);
}
}
NAMESPACE_END(Grid);
Grid/tensors/Tensor_inner.h
+8 -12
View File
@@ -214,24 +214,20 @@ accelerator_inline vRealD innerProductD2(const vRealD &l,const vRealD &
accelerator_inline vComplexD2 innerProductD2(const vComplexF &l,const vComplexF &r)
{
vComplexD la,lb;
vComplexD ra,rb;
Optimization::PrecisionChange::StoD(l.v,la.v,lb.v);
Optimization::PrecisionChange::StoD(r.v,ra.v,rb.v);
vComplexD2 dl,dr;
vComplexD2 ret;
ret._internal[0] = innerProduct(la,ra);
ret._internal[1] = innerProduct(lb,rb);
precisionChange(dl,l);
precisionChange(dr,r);
ret = innerProduct(dl,dr);
return ret;
}
accelerator_inline vRealD2 innerProductD2(const vRealF &l,const vRealF &r)
{
vRealD la,lb;
vRealD ra,rb;
Optimization::PrecisionChange::StoD(l.v,la.v,lb.v);
Optimization::PrecisionChange::StoD(r.v,ra.v,rb.v);
vRealD2 dl,dr;
vRealD2 ret;
ret._internal[0]=innerProduct(la,ra);
ret._internal[1]=innerProduct(lb,rb);
precisionChange(dl,l);
precisionChange(dr,r);
ret=innerProduct(dl,dr);
return ret;
}
Grid/tensors/Tensor_traits.h
+11 -72
View File
@@ -42,39 +42,6 @@ NAMESPACE_BEGIN(Grid);
template<typename T> struct isGridScalar : public std::false_type { static constexpr bool notvalue = true; };
template<class T> struct isGridScalar<iScalar<T>> : public std::true_type { static constexpr bool notvalue = false; };
// Store double-precision data in single-precision grids for precision promoted localInnerProductD
template<typename T>
class TypePair {
public:
T _internal[2];
TypePair<T>& operator=(const Grid::Zero& o) {
_internal[0] = Zero();
_internal[1] = Zero();
return *this;
}
TypePair<T> operator+(const TypePair<T>& o) const {
TypePair<T> r;
r._internal[0] = _internal[0] + o._internal[0];
r._internal[1] = _internal[1] + o._internal[1];
return r;
}
TypePair<T>& operator+=(const TypePair<T>& o) {
_internal[0] += o._internal[0];
_internal[1] += o._internal[1];
return *this;
}
friend accelerator_inline void add(TypePair<T>* ret, const TypePair<T>* a, const TypePair<T>* b) {
add(&ret->_internal[0],&a->_internal[0],&b->_internal[0]);
add(&ret->_internal[1],&a->_internal[1],&b->_internal[1]);
}
};
typedef TypePair<ComplexD> ComplexD2;
typedef TypePair<RealD> RealD2;
typedef TypePair<vComplexD> vComplexD2;
typedef TypePair<vRealD> vRealD2;
// Traits to identify fundamental data types
template<typename T> struct isGridFundamental : public std::false_type { static constexpr bool notvalue = true; };
@@ -88,8 +55,6 @@ NAMESPACE_BEGIN(Grid);
template<> struct isGridFundamental<RealD> : public std::true_type { static constexpr bool notvalue = false; };
template<> struct isGridFundamental<vComplexD2> : public std::true_type { static constexpr bool notvalue = false; };
template<> struct isGridFundamental<vRealD2> : public std::true_type { static constexpr bool notvalue = false; };
template<> struct isGridFundamental<ComplexD2> : public std::true_type { static constexpr bool notvalue = false; };
template<> struct isGridFundamental<RealD2> : public std::true_type { static constexpr bool notvalue = false; };
//////////////////////////////////////////////////////////////////////////////////
@@ -136,7 +101,7 @@ NAMESPACE_BEGIN(Grid);
typedef ComplexF Complexified;
typedef RealF Realified;
typedef RealD DoublePrecision;
typedef RealD2 DoublePrecision2;
typedef RealD DoublePrecision2;
};
template<> struct GridTypeMapper<RealD> : public GridTypeMapper_Base {
typedef RealD scalar_type;
@@ -151,19 +116,6 @@ NAMESPACE_BEGIN(Grid);
typedef RealD DoublePrecision;
typedef RealD DoublePrecision2;
};
template<> struct GridTypeMapper<RealD2> : public GridTypeMapper_Base {
typedef RealD2 scalar_type;
typedef RealD2 scalar_typeD;
typedef RealD2 vector_type;
typedef RealD2 vector_typeD;
typedef RealD2 tensor_reduced;
typedef RealD2 scalar_object;
typedef RealD2 scalar_objectD;
typedef ComplexD2 Complexified;
typedef RealD2 Realified;
typedef RealD2 DoublePrecision;
typedef RealD2 DoublePrecision2;
};
template<> struct GridTypeMapper<ComplexF> : public GridTypeMapper_Base {
typedef ComplexF scalar_type;
typedef ComplexD scalar_typeD;
@@ -175,7 +127,7 @@ NAMESPACE_BEGIN(Grid);
typedef ComplexF Complexified;
typedef RealF Realified;
typedef ComplexD DoublePrecision;
typedef ComplexD2 DoublePrecision2;
typedef ComplexD DoublePrecision2;
};
template<> struct GridTypeMapper<ComplexD> : public GridTypeMapper_Base {
typedef ComplexD scalar_type;
@@ -191,7 +143,7 @@ NAMESPACE_BEGIN(Grid);
typedef ComplexD DoublePrecision2;
};
#ifdef GRID_CUDA
#if defined(GRID_CUDA) || defined(GRID_HIP)
template<> struct GridTypeMapper<std::complex<float> > : public GridTypeMapper_Base {
typedef std::complex<float> scalar_type;
typedef std::complex<double> scalar_typeD;
@@ -220,19 +172,6 @@ NAMESPACE_BEGIN(Grid);
};
#endif
template<> struct GridTypeMapper<ComplexD2> : public GridTypeMapper_Base {
typedef ComplexD2 scalar_type;
typedef ComplexD2 scalar_typeD;
typedef ComplexD2 vector_type;
typedef ComplexD2 vector_typeD;
typedef ComplexD2 tensor_reduced;
typedef ComplexD2 scalar_object;
typedef ComplexD2 scalar_objectD;
typedef ComplexD2 Complexified;
typedef RealD2 Realified;
typedef ComplexD2 DoublePrecision;
typedef ComplexD2 DoublePrecision2;
};
template<> struct GridTypeMapper<Integer> : public GridTypeMapper_Base {
typedef Integer scalar_type;
typedef Integer scalar_typeD;
@@ -274,13 +213,13 @@ NAMESPACE_BEGIN(Grid);
typedef vRealD DoublePrecision2;
};
template<> struct GridTypeMapper<vRealD2> : public GridTypeMapper_Base {
typedef RealD2 scalar_type;
typedef RealD2 scalar_typeD;
typedef RealD scalar_type;
typedef RealD scalar_typeD;
typedef vRealD2 vector_type;
typedef vRealD2 vector_typeD;
typedef vRealD2 tensor_reduced;
typedef RealD2 scalar_object;
typedef RealD2 scalar_objectD;
typedef RealD scalar_object;
typedef RealD scalar_objectD;
typedef vComplexD2 Complexified;
typedef vRealD2 Realified;
typedef vRealD2 DoublePrecision;
@@ -341,13 +280,13 @@ NAMESPACE_BEGIN(Grid);
typedef vComplexD DoublePrecision2;
};
template<> struct GridTypeMapper<vComplexD2> : public GridTypeMapper_Base {
typedef ComplexD2 scalar_type;
typedef ComplexD2 scalar_typeD;
typedef ComplexD scalar_type;
typedef ComplexD scalar_typeD;
typedef vComplexD2 vector_type;
typedef vComplexD2 vector_typeD;
typedef vComplexD2 tensor_reduced;
typedef ComplexD2 scalar_object;
typedef ComplexD2 scalar_objectD;
typedef ComplexD scalar_object;
typedef ComplexD scalar_objectD;
typedef vComplexD2 Complexified;
typedef vRealD2 Realified;
typedef vComplexD2 DoublePrecision;