mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-22 18:39:36 +01:00
Deprecate doubled vector as much as possible.
Arm complex instructions on M3/M4 NEON v8.3 and simplify A64FX code paths/broaden. Grid_vector_types and Simd.h mainly reorg and prep for sComplexD and sComplexF alternat Nsimd=1 types
This commit is contained in:
Peter Boyle
+139
-192
@@ -28,216 +28,163 @@ See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_SIMD_H
|
||||
#define GRID_SIMD_H
|
||||
#pragma once
|
||||
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP)
|
||||
#include <thrust/complex.h>
|
||||
#endif
|
||||
#include <type_traits>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Define scalar and vector floating point types
|
||||
//
|
||||
// Scalar: RealF, RealD, ComplexF, ComplexD
|
||||
//
|
||||
// Vector: vRealF, vRealD, vComplexF, vComplexD
|
||||
//
|
||||
// Vector types are arch dependent
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#define _MM_SELECT_FOUR_FOUR(A,B,C,D) ((A<<6)|(B<<4)|(C<<2)|(D))
|
||||
#define _MM_SELECT_FOUR_FOUR_STRING(A,B,C,D) "((" #A "<<6)|(" #B "<<4)|(" #C "<<2)|(" #D "))"
|
||||
#define _MM_SELECT_EIGHT_TWO(A,B,C,D,E,F,G,H) ((A<<7)|(B<<6)|(C<<5)|(D<<4)|(E<<3)|(F<<2)|(G<<4)|(H))
|
||||
#define _MM_SELECT_FOUR_TWO (A,B,C,D) _MM_SELECT_EIGHT_TWO(0,0,0,0,A,B,C,D)
|
||||
#define _MM_SELECT_TWO_TWO (A,B) _MM_SELECT_FOUR_TWO(0,0,A,B)
|
||||
|
||||
#define RotateBit (0x100)
|
||||
#include <Grid/simd/Grid_scalar_support.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
typedef uint32_t Integer;
|
||||
|
||||
typedef float RealF;
|
||||
typedef double RealD;
|
||||
#ifdef GRID_DEFAULT_PRECISION_DOUBLE
|
||||
typedef RealD Real;
|
||||
#else
|
||||
typedef RealF Real;
|
||||
#endif
|
||||
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP)
|
||||
typedef thrust::complex<RealF> ComplexF;
|
||||
typedef thrust::complex<RealD> ComplexD;
|
||||
typedef thrust::complex<Real> Complex;
|
||||
typedef thrust::complex<uint16_t> ComplexH;
|
||||
template<class T> using complex = thrust::complex<T>;
|
||||
|
||||
accelerator_inline ComplexD pow(const ComplexD& r,RealD y){ return(thrust::pow(r,(double)y)); }
|
||||
accelerator_inline ComplexF pow(const ComplexF& r,RealF y){ return(thrust::pow(r,(float)y)); }
|
||||
#else
|
||||
typedef std::complex<RealF> ComplexF;
|
||||
typedef std::complex<RealD> ComplexD;
|
||||
typedef std::complex<Real> Complex;
|
||||
typedef std::complex<uint16_t> ComplexH; // Hack
|
||||
template<class T> using complex = std::complex<T>;
|
||||
|
||||
accelerator_inline ComplexD pow(const ComplexD& r,RealD y){ return(std::pow(r,y)); }
|
||||
accelerator_inline ComplexF pow(const ComplexF& r,RealF y){ return(std::pow(r,y)); }
|
||||
#endif
|
||||
|
||||
//accelerator_inline RealD pow(const RealD& r,RealD y){ return(std::pow(r,y)); }
|
||||
//accelerator_inline RealD sqrt(const RealD & r){ return std::sqrt(r); }
|
||||
|
||||
// This comes from ::pow already from math.h and CUDA
|
||||
// Calls either Grid::pow for complex, or std::pow for real
|
||||
// Problem is CUDA math_functions is exposing ::pow, and I can't define
|
||||
|
||||
using std::abs;
|
||||
using std::pow;
|
||||
using std::sqrt;
|
||||
using std::log;
|
||||
using std::exp;
|
||||
using std::sin;
|
||||
using std::cos;
|
||||
using std::asin;
|
||||
using std::acos;
|
||||
//////////////////////////////////////
|
||||
// To take the floating point type of real/complex type
|
||||
//////////////////////////////////////
|
||||
template <typename T>
|
||||
struct RealPart {
|
||||
typedef T type;
|
||||
};
|
||||
template <typename T>
|
||||
struct RealPart<std::complex<T> > {
|
||||
typedef T type;
|
||||
};
|
||||
|
||||
|
||||
accelerator_inline RealF conjugate(const RealF & r){ return r; }
|
||||
accelerator_inline RealD conjugate(const RealD & r){ return r; }
|
||||
accelerator_inline ComplexD conjugate(const ComplexD& r){ return(conj(r)); }
|
||||
accelerator_inline ComplexF conjugate(const ComplexF& r ){ return(conj(r)); }
|
||||
// type alias used to simplify the syntax of std::enable_if
|
||||
template <typename T> using Invoke = typename T::type;
|
||||
template <typename Condition, typename ReturnType = void> using EnableIf = Invoke<std::enable_if<Condition::value, ReturnType> >;
|
||||
template <typename Condition, typename ReturnType = void> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
|
||||
|
||||
accelerator_inline RealF adj(const RealF & r){ return r; }
|
||||
accelerator_inline RealD adj(const RealD & r){ return r; }
|
||||
accelerator_inline ComplexD adj(const ComplexD& r){ return(conjugate(r)); }
|
||||
accelerator_inline ComplexF adj(const ComplexF& r ){ return(conjugate(r)); }
|
||||
////////////////////////////////////////////////////////
|
||||
// Check for complexity with type traits
|
||||
template <typename T> struct is_complex : public std::false_type {};
|
||||
template <> struct is_complex<ComplexD> : public std::true_type {};
|
||||
template <> struct is_complex<ComplexF> : public std::true_type {};
|
||||
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP)
|
||||
//Provide for convenience
|
||||
inline std::complex<double> conjugate(const std::complex<double>& r){ return(conj(r)); }
|
||||
inline std::complex<float> conjugate(const std::complex<float>& r) { return(conj(r)); }
|
||||
inline std::complex<double> adj(const std::complex<double>& r) { return(conj(r)); }
|
||||
inline std::complex<float> adj(const std::complex<float>& r) { return(conj(r)); }
|
||||
#endif
|
||||
template <typename T> struct is_ComplexD : public std::false_type {};
|
||||
template <> struct is_ComplexD<ComplexD> : public std::true_type {};
|
||||
|
||||
accelerator_inline RealF real(const RealF & r){ return r; }
|
||||
accelerator_inline RealD real(const RealD & r){ return r; }
|
||||
accelerator_inline RealF real(const ComplexF & r){ return r.real(); }
|
||||
accelerator_inline RealD real(const ComplexD & r){ return r.real(); }
|
||||
template <typename T> struct is_ComplexF : public std::false_type {};
|
||||
template <> struct is_ComplexF<ComplexF> : public std::true_type {};
|
||||
|
||||
accelerator_inline RealF imag(const ComplexF & r){ return r.imag(); }
|
||||
accelerator_inline RealD imag(const ComplexD & r){ return r.imag(); }
|
||||
template<typename T, typename V=void> struct is_real : public std::false_type {};
|
||||
template<typename T> struct is_real<T, typename std::enable_if<std::is_floating_point<T>::value,
|
||||
void>::type> : public std::true_type {};
|
||||
|
||||
accelerator_inline ComplexD innerProduct(const ComplexD & l, const ComplexD & r) { return conjugate(l)*r; }
|
||||
accelerator_inline ComplexF innerProduct(const ComplexF & l, const ComplexF & r) { return conjugate(l)*r; }
|
||||
accelerator_inline RealD innerProduct(const RealD & l, const RealD & r) { return l*r; }
|
||||
accelerator_inline RealF innerProduct(const RealF & l, const RealF & r) { return l*r; }
|
||||
template<typename T, typename V=void> struct is_integer : public std::false_type {};
|
||||
template<typename T> struct is_integer<T, typename std::enable_if<std::is_integral<T>::value,
|
||||
void>::type> : public std::true_type {};
|
||||
|
||||
accelerator_inline ComplexD Reduce(const ComplexD& r){ return r; }
|
||||
accelerator_inline ComplexF Reduce(const ComplexF& r){ return r; }
|
||||
accelerator_inline RealD Reduce(const RealD& r){ return r; }
|
||||
accelerator_inline RealF Reduce(const RealF& r){ return r; }
|
||||
template <typename T> using IfReal = Invoke<std::enable_if<is_real<T>::value, int> >;
|
||||
template <typename T> using IfComplex = Invoke<std::enable_if<is_complex<T>::value, int> >;
|
||||
template <typename T> using IfInteger = Invoke<std::enable_if<is_integer<T>::value, int> >;
|
||||
template <typename T1,typename T2> using IfSame = Invoke<std::enable_if<std::is_same<T1,T2>::value, int> >;
|
||||
|
||||
accelerator_inline RealD toReal(const ComplexD& r){ return r.real(); }
|
||||
accelerator_inline RealF toReal(const ComplexF& r){ return r.real(); }
|
||||
accelerator_inline RealD toReal(const RealD& r){ return r; }
|
||||
accelerator_inline RealF toReal(const RealF& r){ return r; }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//Provide support functions for basic real and complex data types required by Grid
|
||||
//Single and double precision versions. Should be able to template this once only.
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
accelerator_inline void mac (ComplexD * __restrict__ y,const ComplexD * __restrict__ a,const ComplexD *__restrict__ x){ *y = (*a) * (*x)+(*y); };
|
||||
accelerator_inline void mult(ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) * (*r);}
|
||||
accelerator_inline void sub (ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) - (*r);}
|
||||
accelerator_inline void add (ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) + (*r);}
|
||||
// conjugate already supported for complex
|
||||
|
||||
accelerator_inline void mac (ComplexF * __restrict__ y,const ComplexF * __restrict__ a,const ComplexF *__restrict__ x){ *y = (*a) * (*x)+(*y); }
|
||||
accelerator_inline void mult(ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) * (*r); }
|
||||
accelerator_inline void sub (ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) - (*r); }
|
||||
accelerator_inline void add (ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) + (*r); }
|
||||
|
||||
//conjugate already supported for complex
|
||||
accelerator_inline ComplexF timesI(const ComplexF &r) { return(ComplexF(-r.imag(),r.real()));}
|
||||
accelerator_inline ComplexD timesI(const ComplexD &r) { return(ComplexD(-r.imag(),r.real()));}
|
||||
accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(ComplexF(r.imag(),-r.real()));}
|
||||
accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(ComplexD(r.imag(),-r.real()));}
|
||||
//accelerator_inline ComplexF timesI(const ComplexF &r) { return(r*ComplexF(0.0,1.0));}
|
||||
//accelerator_inline ComplexD timesI(const ComplexD &r) { return(r*ComplexD(0.0,1.0));}
|
||||
//accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(r*ComplexF(0.0,-1.0));}
|
||||
//accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(r*ComplexD(0.0,-1.0));}
|
||||
|
||||
// define projections to real and imaginay parts
|
||||
accelerator_inline ComplexF projReal(const ComplexF &r){return( ComplexF(r.real(), 0.0));}
|
||||
accelerator_inline ComplexD projReal(const ComplexD &r){return( ComplexD(r.real(), 0.0));}
|
||||
accelerator_inline ComplexF projImag(const ComplexF &r){return (ComplexF(r.imag(), 0.0 ));}
|
||||
accelerator_inline ComplexD projImag(const ComplexD &r){return (ComplexD(r.imag(), 0.0));}
|
||||
|
||||
// define auxiliary functions for complex computations
|
||||
accelerator_inline void timesI(ComplexF &ret,const ComplexF &r) { ret = timesI(r);}
|
||||
accelerator_inline void timesI(ComplexD &ret,const ComplexD &r) { ret = timesI(r);}
|
||||
accelerator_inline void timesMinusI(ComplexF &ret,const ComplexF &r){ ret = timesMinusI(r);}
|
||||
accelerator_inline void timesMinusI(ComplexD &ret,const ComplexD &r){ ret = timesMinusI(r);}
|
||||
|
||||
accelerator_inline void mac (RealD * __restrict__ y,const RealD * __restrict__ a,const RealD *__restrict__ x){ *y = (*a) * (*x)+(*y);}
|
||||
accelerator_inline void mult(RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) * (*r);}
|
||||
accelerator_inline void sub (RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) - (*r);}
|
||||
accelerator_inline void add (RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) + (*r);}
|
||||
|
||||
accelerator_inline void mac (RealF * __restrict__ y,const RealF * __restrict__ a,const RealF *__restrict__ x){ *y = (*a) * (*x)+(*y); }
|
||||
accelerator_inline void mult(RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) * (*r); }
|
||||
accelerator_inline void sub (RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) - (*r); }
|
||||
accelerator_inline void add (RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) + (*r); }
|
||||
|
||||
accelerator_inline void vstream(ComplexF &l, const ComplexF &r){ l=r;}
|
||||
accelerator_inline void vstream(ComplexD &l, const ComplexD &r){ l=r;}
|
||||
accelerator_inline void vstream(RealF &l, const RealF &r){ l=r;}
|
||||
accelerator_inline void vstream(RealD &l, const RealD &r){ l=r;}
|
||||
|
||||
accelerator_inline ComplexD toComplex(const RealD &in) { return ComplexD(in);}
|
||||
accelerator_inline ComplexF toComplex(const RealF &in) { return ComplexF(in);}
|
||||
|
||||
class Zero{};
|
||||
//static Zero Zero();
|
||||
template<class itype> accelerator_inline void zeroit(itype &arg) { arg=Zero();};
|
||||
template<> accelerator_inline void zeroit(ComplexF &arg){ arg=0; };
|
||||
template<> accelerator_inline void zeroit(ComplexD &arg){ arg=0; };
|
||||
template<> accelerator_inline void zeroit(RealF &arg) { arg=0; };
|
||||
template<> accelerator_inline void zeroit(RealD &arg) { arg=0; };
|
||||
|
||||
// More limited Integer support
|
||||
accelerator_inline Integer Reduce(const Integer& r){ return r; }
|
||||
accelerator_inline void mac (Integer * __restrict__ y,const Integer * __restrict__ a,const Integer *__restrict__ x){ *y = (*a) * (*x)+(*y); }
|
||||
accelerator_inline void mult(Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) * (*r); }
|
||||
accelerator_inline void sub (Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) - (*r); }
|
||||
accelerator_inline void add (Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) + (*r); }
|
||||
accelerator_inline void vstream(Integer &l, const RealD &r){ l=r;}
|
||||
template<> accelerator_inline void zeroit(Integer &arg) { arg=0; };
|
||||
|
||||
accelerator_inline Integer mod (Integer a,Integer y) { return a%y;}
|
||||
accelerator_inline Integer div (Integer a,Integer y) { return a/y;}
|
||||
//accelerator_inline Integer abs (Integer &a) { return a%y;}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// Permute
|
||||
// Permute 0 every ABCDEFGH -> BA DC FE HG
|
||||
// Permute 1 every ABCDEFGH -> CD AB GH EF
|
||||
// Permute 2 every ABCDEFGH -> EFGH ABCD
|
||||
// Permute 3 possible on longer iVector lengths (512bit = 8 double = 16 single)
|
||||
// Permute 4 possible on half precision @512bit vectors.
|
||||
//
|
||||
// Defined inside SIMD specialization files
|
||||
//////////////////////////////////////////////////////////
|
||||
template<class VectorSIMD>
|
||||
accelerator_inline void Gpermute(VectorSIMD &y,const VectorSIMD &b,int perm);
|
||||
template <typename T> using IfNotReal = Invoke<std::enable_if<!is_real<T>::value, int> >;
|
||||
template <typename T> using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
|
||||
template <typename T> using IfNotInteger = Invoke<std::enable_if<!is_integer<T>::value, int> >;
|
||||
template <typename T1,typename T2> using IfNotSame = Invoke<std::enable_if<!std::is_same<T1,T2>::value, int> >;
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// Define the operation templates functors
|
||||
// general forms to allow for vsplat syntax
|
||||
// need explicit declaration of types when used since
|
||||
// clang cannot automatically determine the output type sometimes
|
||||
template <class Out, class Input1, class Input2, class Input3, class Operation>
|
||||
Out accelerator_inline trinary(Input1 src_1, Input2 src_2, Input3 src_3, Operation op) {
|
||||
return op(src_1, src_2, src_3);
|
||||
}
|
||||
template <class Out, class Input1, class Input2, class Operation>
|
||||
Out accelerator_inline binary(Input1 src_1, Input2 src_2, Operation op) {
|
||||
return op(src_1, src_2);
|
||||
}
|
||||
template <class Out, class Input, class Operation>
|
||||
Out accelerator_inline unary(Input src, Operation op) {
|
||||
return op(src);
|
||||
}
|
||||
NAMESPACE_END(Grid);
|
||||
///////////////////////////////////////////////
|
||||
|
||||
#include <Grid/simd/Grid_vector_types.h>
|
||||
#include <Grid/simd/Grid_doubled_vector.h>
|
||||
#include <Grid/simd/Grid_scalar_types.h>
|
||||
|
||||
#ifdef GRID_SYCL
|
||||
template<> struct sycl::is_device_copyable<Grid::vComplexF> : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::vComplexD> : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::vRealF > : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::vRealD > : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::vInteger > : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::sComplexF> : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::sComplexD> : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::sRealF > : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::sRealD > : public std::true_type {};
|
||||
template<> struct sycl::is_device_copyable<Grid::sInteger > : public std::true_type {};
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////
|
||||
// Detect vector types
|
||||
/////////////////////////////////////////
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
template <typename T>
|
||||
struct is_simd : public std::false_type {};
|
||||
template <> struct is_simd<vRealF> : public std::true_type {};
|
||||
template <> struct is_simd<vRealD> : public std::true_type {};
|
||||
template <> struct is_simd<vRealH> : public std::true_type {};
|
||||
template <> struct is_simd<vComplexF> : public std::true_type {};
|
||||
template <> struct is_simd<vComplexD> : public std::true_type {};
|
||||
template <> struct is_simd<vComplexH> : public std::true_type {};
|
||||
template <> struct is_simd<vInteger> : public std::true_type {};
|
||||
template <> struct is_simd<sRealF> : public std::true_type {};
|
||||
template <> struct is_simd<sRealD> : public std::true_type {};
|
||||
template <> struct is_simd<sComplexF> : public std::true_type {};
|
||||
template <> struct is_simd<sComplexD> : public std::true_type {};
|
||||
template <> struct is_simd<sInteger> : public std::true_type {};
|
||||
|
||||
template <typename T> using IfSimd = Invoke<std::enable_if<is_simd<T>::value, int> >;
|
||||
template <typename T> using IfNotSimd = Invoke<std::enable_if<!is_simd<T>::value, unsigned> >;
|
||||
|
||||
|
||||
///////////////////////////////////////////////
|
||||
// insert / extract with complex support
|
||||
///////////////////////////////////////////////
|
||||
template <class S, class V>
|
||||
accelerator_inline S getlane(const Grid_simd<S, V> &in,int lane) {
|
||||
return in.getlane(lane);
|
||||
}
|
||||
template <class S, class V>
|
||||
accelerator_inline void putlane(Grid_simd<S, V> &vec,const S &_S, int lane){
|
||||
vec.putlane(_S,lane);
|
||||
}
|
||||
template <class S,IfNotSimd<S> = 0 >
|
||||
accelerator_inline S getlane(const S &in,int lane) {
|
||||
return in;
|
||||
}
|
||||
template <class S,IfNotSimd<S> = 0 >
|
||||
accelerator_inline void putlane(S &vec,const S &_S, int lane){
|
||||
vec = _S;
|
||||
}
|
||||
template <class S, class V>
|
||||
accelerator_inline S getlane(const Grid_simd2<S, V> &in,int lane) {
|
||||
return in.getlane(lane);
|
||||
}
|
||||
template <class S, class V>
|
||||
accelerator_inline void putlane(Grid_simd2<S, V> &vec,const S &_S, int lane){
|
||||
vec.putlane(_S,lane);
|
||||
}
|
||||
template <class S>
|
||||
accelerator_inline S getlane(const Grid_simd1<S> &in,int lane) {
|
||||
return in.getlane(lane);
|
||||
}
|
||||
template <class S>
|
||||
accelerator_inline void putlane(Grid_simd1<S> &vec,const S &_S, int lane){
|
||||
vec.putlane(_S,lane);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
|
||||
#include <Grid/simd/Grid_vector_unops.h>
|
||||
#include <Grid/simd/Grid_scalar_unops.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
@@ -317,4 +264,4 @@ inline std::ostream& operator<< (std::ostream& stream, const vInteger &o){
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid)
|
||||
#endif
|
||||
|
||||
|
||||
Reference in New Issue
Block a user