mirror of
https://github.com/paboyle/Grid.git
synced 2025-04-03 18:55:56 +01:00
Christoph Lehner
GitHub
commit
e2abbf9520
56
.travis.yml
56
.travis.yml
@ -1,56 +0,0 @@
|
||||
language: cpp
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- clang
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- os: osx
|
||||
osx_image: xcode8.3
|
||||
compiler: clang
|
||||
|
||||
before_install:
|
||||
- export GRIDDIR=`pwd`
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]] && [ ! -e clang/bin ]; then wget $CLANG_LINK; tar -xf `basename $CLANG_LINK`; mkdir clang; mv clang+*/* clang/; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export PATH="${GRIDDIR}/clang/bin:${PATH}"; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc openssl; fi
|
||||
|
||||
install:
|
||||
- export CWD=`pwd`
|
||||
- echo $CWD
|
||||
- export CC=$CC$VERSION
|
||||
- export CXX=$CXX$VERSION
|
||||
- echo $PATH
|
||||
- which autoconf
|
||||
- autoconf --version
|
||||
- which automake
|
||||
- automake --version
|
||||
- which $CC
|
||||
- $CC --version
|
||||
- which $CXX
|
||||
- $CXX --version
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export LDFLAGS='-L/usr/local/lib'; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export EXTRACONF='--with-openssl=/usr/local/opt/openssl'; fi
|
||||
|
||||
script:
|
||||
- ./bootstrap.sh
|
||||
- mkdir build
|
||||
- cd build
|
||||
- mkdir lime
|
||||
- cd lime
|
||||
- mkdir build
|
||||
- cd build
|
||||
- wget http://usqcd-software.github.io/downloads/c-lime/lime-1.3.2.tar.gz
|
||||
- tar xf lime-1.3.2.tar.gz
|
||||
- cd lime-1.3.2
|
||||
- ./configure --prefix=$CWD/build/lime/install
|
||||
- make -j4
|
||||
- make install
|
||||
- cd $CWD/build
|
||||
- ../configure --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install ${EXTRACONF}
|
||||
- make -j4
|
||||
- ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
|
||||
- make check
|
||||
@ -54,9 +54,11 @@ Version.h: version-cache
|
||||
include Make.inc
|
||||
include Eigen.inc
|
||||
|
||||
extra_sources+=$(ZWILS_FERMION_FILES)
|
||||
extra_sources+=$(WILS_FERMION_FILES)
|
||||
extra_sources+=$(STAG_FERMION_FILES)
|
||||
if BUILD_ZMOBIUS
|
||||
extra_sources+=$(ZWILS_FERMION_FILES)
|
||||
endif
|
||||
if BUILD_GPARITY
|
||||
extra_sources+=$(GP_FERMION_FILES)
|
||||
endif
|
||||
|
||||
@ -36,7 +36,7 @@ static const int CbBlack=1;
|
||||
static const int Even =CbRed;
|
||||
static const int Odd =CbBlack;
|
||||
|
||||
accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex, Coordinate &rdim, Coordinate &chk_dim_msk)
|
||||
accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex,const Coordinate &rdim,const Coordinate &chk_dim_msk)
|
||||
{
|
||||
int nd=rdim.size();
|
||||
Coordinate coor(nd);
|
||||
|
||||
@ -35,6 +35,12 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#ifdef GRID_MPI3_SHM_NVLINK
|
||||
const bool Stencil_force_mpi = true;
|
||||
#else
|
||||
const bool Stencil_force_mpi = false;
|
||||
#endif
|
||||
|
||||
class CartesianCommunicator : public SharedMemory {
|
||||
|
||||
public:
|
||||
|
||||
@ -370,7 +370,7 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
|
||||
double off_node_bytes=0.0;
|
||||
int tag;
|
||||
|
||||
if ( gfrom ==MPI_UNDEFINED) {
|
||||
if ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
tag= dir+from*32;
|
||||
ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,tag,communicator_halo[commdir],&rrq);
|
||||
assert(ierr==0);
|
||||
@ -378,7 +378,7 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
|
||||
off_node_bytes+=bytes;
|
||||
}
|
||||
|
||||
if ( gdest == MPI_UNDEFINED ) {
|
||||
if ( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
tag= dir+_processor*32;
|
||||
ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,tag,communicator_halo[commdir],&xrq);
|
||||
assert(ierr==0);
|
||||
|
||||
@ -35,6 +35,9 @@ Author: Christoph Lehner <christoph@lhnr.de>
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
#include <hip/hip_runtime_api.h>
|
||||
#endif
|
||||
#ifdef GRID_SYCl
|
||||
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
@ -70,6 +73,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
WorldNodes = WorldSize/WorldShmSize;
|
||||
assert( (WorldNodes * WorldShmSize) == WorldSize );
|
||||
|
||||
|
||||
// FIXME: Check all WorldShmSize are the same ?
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
@ -446,7 +450,47 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hugetlbfs mapping intended
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if defined(GRID_CUDA) ||defined(GRID_HIP)
|
||||
#if defined(GRID_CUDA) ||defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
|
||||
//if defined(GRID_SYCL)
|
||||
#if 0
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
void * ShmCommBuf ;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the pointer array for shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Each MPI rank should allocate our own buffer
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||
|
||||
if (ShmCommBuf == (void *)NULL ) {
|
||||
std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||
|
||||
SharedMemoryZero(ShmCommBuf,bytes);
|
||||
|
||||
assert(WorldShmSize == 1);
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
WorldShmCommBufs[r] = ShmCommBuf;
|
||||
}
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(GRID_CUDA) ||defined(GRID_HIP) ||defined(GRID_SYCL)
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
void * ShmCommBuf ;
|
||||
@ -469,8 +513,16 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Each MPI rank should allocate our own buffer
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||
auto zeDevice = cl::sycl::get_native<cl::sycl::backend::level_zero>(theGridAccelerator->get_device());
|
||||
auto zeContext= cl::sycl::get_native<cl::sycl::backend::level_zero>(theGridAccelerator->get_context());
|
||||
ze_device_mem_alloc_desc_t zeDesc = {};
|
||||
zeMemAllocDevice(zeContext,&zeDesc,bytes,2*1024*1024,zeDevice,&ShmCommBuf);
|
||||
std::cout << WorldRank << header " SharedMemoryMPI.cc zeMemAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||
#else
|
||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||
|
||||
#endif
|
||||
if (ShmCommBuf == (void *)NULL ) {
|
||||
std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
@ -480,8 +532,8 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||
}
|
||||
SharedMemoryZero(ShmCommBuf,bytes);
|
||||
|
||||
// SharedMemoryZero(ShmCommBuf,bytes);
|
||||
std::cout<< "Setting up IPC"<<std::endl;
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Loop over ranks/gpu's on our node
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@ -491,6 +543,23 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
//////////////////////////////////////////////////
|
||||
// If it is me, pass around the IPC access key
|
||||
//////////////////////////////////////////////////
|
||||
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||
ze_ipc_mem_handle_t handle;
|
||||
if ( r==WorldShmRank ) {
|
||||
auto err = zeMemGetIpcHandle(zeContext,ShmCommBuf,&handle);
|
||||
if ( err != ZE_RESULT_SUCCESS ) {
|
||||
std::cerr << "SharedMemoryMPI.cc zeMemGetIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
} else {
|
||||
std::cerr << "SharedMemoryMPI.cc zeMemGetIpcHandle succeeded for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
}
|
||||
std::cerr<<"Allocated IpcHandle rank "<<r<<" (hex) ";
|
||||
for(int c=0;c<ZE_MAX_IPC_HANDLE_SIZE;c++){
|
||||
std::cerr<<std::hex<<(uint32_t)((uint8_t)handle.data[c])<<std::dec;
|
||||
}
|
||||
std::cerr<<std::endl;
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_CUDA
|
||||
cudaIpcMemHandle_t handle;
|
||||
if ( r==WorldShmRank ) {
|
||||
@ -527,6 +596,25 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
// If I am not the source, overwrite thisBuf with remote buffer
|
||||
///////////////////////////////////////////////////////////////
|
||||
void * thisBuf = ShmCommBuf;
|
||||
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||
if ( r!=WorldShmRank ) {
|
||||
thisBuf = nullptr;
|
||||
std::cerr<<"Using IpcHandle rank "<<r<<" ";
|
||||
for(int c=0;c<ZE_MAX_IPC_HANDLE_SIZE;c++){
|
||||
std::cerr<<std::hex<<(uint32_t)((uint8_t)handle.data[c])<<std::dec;
|
||||
}
|
||||
std::cerr<<std::endl;
|
||||
auto err = zeMemOpenIpcHandle(zeContext,zeDevice,handle,0,&thisBuf);
|
||||
if ( err != ZE_RESULT_SUCCESS ) {
|
||||
std::cerr << "SharedMemoryMPI.cc "<<zeContext<<" "<<zeDevice<<std::endl;
|
||||
std::cerr << "SharedMemoryMPI.cc zeMemOpenIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
} else {
|
||||
std::cerr << "SharedMemoryMPI.cc zeMemOpenIpcHandle succeeded for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
}
|
||||
assert(thisBuf!=nullptr);
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_CUDA
|
||||
if ( r!=WorldShmRank ) {
|
||||
auto err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
|
||||
@ -557,6 +645,8 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
}
|
||||
#endif
|
||||
|
||||
#else
|
||||
#ifdef GRID_MPI3_SHMMMAP
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
@ -727,16 +817,16 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
void GlobalSharedMemory::SharedMemoryZero(void *dest,size_t bytes)
|
||||
{
|
||||
#ifdef GRID_CUDA
|
||||
cudaMemset(dest,0,bytes);
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
acceleratorMemSet(dest,0,bytes);
|
||||
#else
|
||||
bzero(dest,bytes);
|
||||
#endif
|
||||
}
|
||||
void GlobalSharedMemory::SharedMemoryCopy(void *dest,void *src,size_t bytes)
|
||||
{
|
||||
#ifdef GRID_CUDA
|
||||
cudaMemcpy(dest,src,bytes,cudaMemcpyDefault);
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
acceleratorCopyToDevice(src,dest,bytes);
|
||||
#else
|
||||
bcopy(src,dest,bytes);
|
||||
#endif
|
||||
@ -800,7 +890,7 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
}
|
||||
#endif
|
||||
|
||||
SharedMemoryTest();
|
||||
//SharedMemoryTest();
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// On node barrier
|
||||
|
||||
@ -110,9 +110,11 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
|
||||
int n1=rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if ( cbmask ==0x3){
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for2d(n,e1,b,e2,1,{
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
int o = n*n1;
|
||||
int offset = b+n*e2;
|
||||
|
||||
@ -135,7 +137,9 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
|
||||
std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for2d(n,e1,b,e2,1,{
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
|
||||
Coordinate coor;
|
||||
|
||||
@ -257,7 +261,9 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
|
||||
int _slice_block = rhs.Grid()->_slice_block[dimension];
|
||||
#ifdef ACCELERATOR_CSHIFT
|
||||
autoView( rhs_v , rhs, AcceleratorWrite);
|
||||
accelerator_for2d(n,e1,b,e2,1,{
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
int o = n*_slice_stride;
|
||||
int offset = b+n*_slice_block;
|
||||
merge(rhs_v[so+o+b],pointers,offset);
|
||||
@ -274,7 +280,7 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
|
||||
|
||||
// Case of SIMD split AND checker dim cannot currently be hit, except in
|
||||
// Test_cshift_red_black code.
|
||||
// std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
||||
std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
||||
std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
|
||||
assert(0); // This will fail if hit on GPU
|
||||
autoView( rhs_v, rhs, CpuWrite);
|
||||
|
||||
@ -97,6 +97,20 @@ accelerator_inline void convertType(ComplexF & out, const std::complex<float> &
|
||||
out = in;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
accelerator_inline EnableIf<isGridFundamental<T>> convertType(T & out, const T & in) {
|
||||
out = in;
|
||||
}
|
||||
|
||||
// This would allow for conversions between GridFundamental types, but is not strictly needed as yet
|
||||
/*template<typename T1, typename T2>
|
||||
accelerator_inline typename std::enable_if<isGridFundamental<T1>::value && isGridFundamental<T2>::value>::type
|
||||
// Or to make this very broad, conversions between anything that's not a GridTensor could be allowed
|
||||
//accelerator_inline typename std::enable_if<!isGridTensor<T1>::value && !isGridTensor<T2>::value>::type
|
||||
convertType(T1 & out, const T2 & in) {
|
||||
out = in;
|
||||
}*/
|
||||
|
||||
#ifdef GRID_SIMT
|
||||
accelerator_inline void convertType(vComplexF & out, const ComplexF & in) {
|
||||
((ComplexF*)&out)[acceleratorSIMTlane(vComplexF::Nsimd())] = in;
|
||||
@ -117,23 +131,18 @@ accelerator_inline void convertType(vComplexD2 & out, const vComplexF & in) {
|
||||
Optimization::PrecisionChange::StoD(in.v,out._internal[0].v,out._internal[1].v);
|
||||
}
|
||||
|
||||
template<typename T1,typename T2,int N>
|
||||
accelerator_inline void convertType(iMatrix<T1,N> & out, const iMatrix<T2,N> & in);
|
||||
template<typename T1,typename T2,int N>
|
||||
accelerator_inline void convertType(iVector<T1,N> & out, const iVector<T2,N> & in);
|
||||
|
||||
template<typename T1,typename T2, typename std::enable_if<!isGridScalar<T1>::value, T1>::type* = nullptr>
|
||||
accelerator_inline void convertType(T1 & out, const iScalar<T2> & in) {
|
||||
convertType(out,in._internal);
|
||||
template<typename T1,typename T2>
|
||||
accelerator_inline void convertType(iScalar<T1> & out, const iScalar<T2> & in) {
|
||||
convertType(out._internal,in._internal);
|
||||
}
|
||||
|
||||
template<typename T1, typename std::enable_if<!isGridScalar<T1>::value, T1>::type* = nullptr>
|
||||
accelerator_inline void convertType(T1 & out, const iScalar<T1> & in) {
|
||||
template<typename T1,typename T2>
|
||||
accelerator_inline NotEnableIf<isGridScalar<T1>> convertType(T1 & out, const iScalar<T2> & in) {
|
||||
convertType(out,in._internal);
|
||||
}
|
||||
|
||||
template<typename T1,typename T2>
|
||||
accelerator_inline void convertType(iScalar<T1> & out, const T2 & in) {
|
||||
accelerator_inline NotEnableIf<isGridScalar<T2>> convertType(iScalar<T1> & out, const T2 & in) {
|
||||
convertType(out._internal,in);
|
||||
}
|
||||
|
||||
@ -150,11 +159,6 @@ accelerator_inline void convertType(iVector<T1,N> & out, const iVector<T2,N> & i
|
||||
convertType(out._internal[i],in._internal[i]);
|
||||
}
|
||||
|
||||
template<typename T, typename std::enable_if<isGridFundamental<T>::value, T>::type* = nullptr>
|
||||
accelerator_inline void convertType(T & out, const T & in) {
|
||||
out = in;
|
||||
}
|
||||
|
||||
template<typename T1,typename T2>
|
||||
accelerator_inline void convertType(Lattice<T1> & out, const Lattice<T2> & in) {
|
||||
autoView( out_v , out,AcceleratorWrite);
|
||||
|
||||
@ -43,7 +43,7 @@ inline void whereWolf(Lattice<vobj> &ret,const Lattice<iobj> &predicate,Lattice<
|
||||
conformable(iftrue,predicate);
|
||||
conformable(iftrue,ret);
|
||||
|
||||
GridBase *grid=iftrue._grid;
|
||||
GridBase *grid=iftrue.Grid();
|
||||
|
||||
typedef typename vobj::scalar_object scalar_object;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
@ -52,22 +52,23 @@ inline void whereWolf(Lattice<vobj> &ret,const Lattice<iobj> &predicate,Lattice<
|
||||
|
||||
const int Nsimd = grid->Nsimd();
|
||||
|
||||
std::vector<Integer> mask(Nsimd);
|
||||
std::vector<scalar_object> truevals (Nsimd);
|
||||
std::vector<scalar_object> falsevals(Nsimd);
|
||||
|
||||
parallel_for(int ss=0;ss<iftrue._grid->oSites(); ss++){
|
||||
|
||||
extract(iftrue._odata[ss] ,truevals);
|
||||
extract(iffalse._odata[ss] ,falsevals);
|
||||
extract<vInteger,Integer>(TensorRemove(predicate._odata[ss]),mask);
|
||||
|
||||
for(int s=0;s<Nsimd;s++){
|
||||
if (mask[s]) falsevals[s]=truevals[s];
|
||||
autoView(iftrue_v,iftrue,CpuRead);
|
||||
autoView(iffalse_v,iffalse,CpuRead);
|
||||
autoView(predicate_v,predicate,CpuRead);
|
||||
autoView(ret_v,ret,CpuWrite);
|
||||
Integer NN= grid->oSites();
|
||||
thread_for(ss,NN,{
|
||||
Integer mask;
|
||||
scalar_object trueval;
|
||||
scalar_object falseval;
|
||||
for(int l=0;l<Nsimd;l++){
|
||||
trueval =extractLane(l,iftrue_v[ss]);
|
||||
falseval=extractLane(l,iffalse_v[ss]);
|
||||
mask =extractLane(l,predicate_v[ss]);
|
||||
if (mask) falseval=trueval;
|
||||
insertLane(l,ret_v[ss],falseval);
|
||||
}
|
||||
|
||||
merge(ret._odata[ss],falsevals);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
template<class vobj,class iobj>
|
||||
@ -76,9 +77,9 @@ inline Lattice<vobj> whereWolf(const Lattice<iobj> &predicate,Lattice<vobj> &ift
|
||||
conformable(iftrue,iffalse);
|
||||
conformable(iftrue,predicate);
|
||||
|
||||
Lattice<vobj> ret(iftrue._grid);
|
||||
Lattice<vobj> ret(iftrue.Grid());
|
||||
|
||||
where(ret,predicate,iftrue,iffalse);
|
||||
whereWolf(ret,predicate,iftrue,iffalse);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -128,7 +128,7 @@ inline void MachineCharacteristics(FieldMetaData &header)
|
||||
std::time_t t = std::time(nullptr);
|
||||
std::tm tm_ = *std::localtime(&t);
|
||||
std::ostringstream oss;
|
||||
// oss << std::put_time(&tm_, "%c %Z");
|
||||
oss << std::put_time(&tm_, "%c %Z");
|
||||
header.creation_date = oss.str();
|
||||
header.archive_date = header.creation_date;
|
||||
|
||||
|
||||
@ -205,11 +205,20 @@ public:
|
||||
std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
|
||||
}
|
||||
|
||||
// Preferred interface
|
||||
template<class GaugeStats=PeriodicGaugeStatistics>
|
||||
static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
|
||||
std::string file,
|
||||
std::string ens_label = std::string("DWF"))
|
||||
{
|
||||
writeConfiguration(Umu,file,0,1,ens_label);
|
||||
}
|
||||
template<class GaugeStats=PeriodicGaugeStatistics>
|
||||
static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
|
||||
std::string file,
|
||||
int two_row,
|
||||
int bits32)
|
||||
int bits32,
|
||||
std::string ens_label = std::string("DWF"))
|
||||
{
|
||||
typedef vLorentzColourMatrixD vobj;
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
@ -219,8 +228,8 @@ public:
|
||||
// Following should become arguments
|
||||
///////////////////////////////////////////
|
||||
header.sequence_number = 1;
|
||||
header.ensemble_id = "UKQCD";
|
||||
header.ensemble_label = "DWF";
|
||||
header.ensemble_id = std::string("UKQCD");
|
||||
header.ensemble_label = ens_label;
|
||||
|
||||
typedef LorentzColourMatrixD fobj3D;
|
||||
typedef LorentzColour2x3D fobj2D;
|
||||
@ -232,7 +241,7 @@ public:
|
||||
GaugeStats Stats; Stats(Umu,header);
|
||||
MachineCharacteristics(header);
|
||||
|
||||
uint64_t offset;
|
||||
uint64_t offset;
|
||||
|
||||
// Sod it -- always write 3x3 double
|
||||
header.floating_point = std::string("IEEE64BIG");
|
||||
|
||||
@ -41,7 +41,7 @@ class Action
|
||||
public:
|
||||
bool is_smeared = false;
|
||||
// Heatbath?
|
||||
virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
|
||||
virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
|
||||
virtual RealD S(const GaugeField& U) = 0; // evaluate the action
|
||||
virtual void deriv(const GaugeField& U, GaugeField& dSdU) = 0; // evaluate the action derivative
|
||||
virtual std::string action_name() = 0; // return the action name
|
||||
|
||||
@ -115,9 +115,9 @@ typedef WilsonFermion<WilsonImplR> WilsonFermionR;
|
||||
typedef WilsonFermion<WilsonImplF> WilsonFermionF;
|
||||
typedef WilsonFermion<WilsonImplD> WilsonFermionD;
|
||||
|
||||
typedef WilsonFermion<WilsonImplRL> WilsonFermionRL;
|
||||
typedef WilsonFermion<WilsonImplFH> WilsonFermionFH;
|
||||
typedef WilsonFermion<WilsonImplDF> WilsonFermionDF;
|
||||
//typedef WilsonFermion<WilsonImplRL> WilsonFermionRL;
|
||||
//typedef WilsonFermion<WilsonImplFH> WilsonFermionFH;
|
||||
//typedef WilsonFermion<WilsonImplDF> WilsonFermionDF;
|
||||
|
||||
typedef WilsonFermion<WilsonAdjImplR> WilsonAdjFermionR;
|
||||
typedef WilsonFermion<WilsonAdjImplF> WilsonAdjFermionF;
|
||||
@ -158,41 +158,41 @@ typedef DomainWallFermion<WilsonImplR> DomainWallFermionR;
|
||||
typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;
|
||||
typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
|
||||
|
||||
typedef DomainWallFermion<WilsonImplRL> DomainWallFermionRL;
|
||||
typedef DomainWallFermion<WilsonImplFH> DomainWallFermionFH;
|
||||
typedef DomainWallFermion<WilsonImplDF> DomainWallFermionDF;
|
||||
//typedef DomainWallFermion<WilsonImplRL> DomainWallFermionRL;
|
||||
//typedef DomainWallFermion<WilsonImplFH> DomainWallFermionFH;
|
||||
//typedef DomainWallFermion<WilsonImplDF> DomainWallFermionDF;
|
||||
|
||||
typedef DomainWallEOFAFermion<WilsonImplR> DomainWallEOFAFermionR;
|
||||
typedef DomainWallEOFAFermion<WilsonImplF> DomainWallEOFAFermionF;
|
||||
typedef DomainWallEOFAFermion<WilsonImplD> DomainWallEOFAFermionD;
|
||||
|
||||
typedef DomainWallEOFAFermion<WilsonImplRL> DomainWallEOFAFermionRL;
|
||||
typedef DomainWallEOFAFermion<WilsonImplFH> DomainWallEOFAFermionFH;
|
||||
typedef DomainWallEOFAFermion<WilsonImplDF> DomainWallEOFAFermionDF;
|
||||
//typedef DomainWallEOFAFermion<WilsonImplRL> DomainWallEOFAFermionRL;
|
||||
//typedef DomainWallEOFAFermion<WilsonImplFH> DomainWallEOFAFermionFH;
|
||||
//typedef DomainWallEOFAFermion<WilsonImplDF> DomainWallEOFAFermionDF;
|
||||
|
||||
typedef MobiusFermion<WilsonImplR> MobiusFermionR;
|
||||
typedef MobiusFermion<WilsonImplF> MobiusFermionF;
|
||||
typedef MobiusFermion<WilsonImplD> MobiusFermionD;
|
||||
|
||||
typedef MobiusFermion<WilsonImplRL> MobiusFermionRL;
|
||||
typedef MobiusFermion<WilsonImplFH> MobiusFermionFH;
|
||||
typedef MobiusFermion<WilsonImplDF> MobiusFermionDF;
|
||||
//typedef MobiusFermion<WilsonImplRL> MobiusFermionRL;
|
||||
//typedef MobiusFermion<WilsonImplFH> MobiusFermionFH;
|
||||
//typedef MobiusFermion<WilsonImplDF> MobiusFermionDF;
|
||||
|
||||
typedef MobiusEOFAFermion<WilsonImplR> MobiusEOFAFermionR;
|
||||
typedef MobiusEOFAFermion<WilsonImplF> MobiusEOFAFermionF;
|
||||
typedef MobiusEOFAFermion<WilsonImplD> MobiusEOFAFermionD;
|
||||
|
||||
typedef MobiusEOFAFermion<WilsonImplRL> MobiusEOFAFermionRL;
|
||||
typedef MobiusEOFAFermion<WilsonImplFH> MobiusEOFAFermionFH;
|
||||
typedef MobiusEOFAFermion<WilsonImplDF> MobiusEOFAFermionDF;
|
||||
//typedef MobiusEOFAFermion<WilsonImplRL> MobiusEOFAFermionRL;
|
||||
//typedef MobiusEOFAFermion<WilsonImplFH> MobiusEOFAFermionFH;
|
||||
//typedef MobiusEOFAFermion<WilsonImplDF> MobiusEOFAFermionDF;
|
||||
|
||||
typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
|
||||
typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
|
||||
typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
|
||||
|
||||
typedef ZMobiusFermion<ZWilsonImplRL> ZMobiusFermionRL;
|
||||
typedef ZMobiusFermion<ZWilsonImplFH> ZMobiusFermionFH;
|
||||
typedef ZMobiusFermion<ZWilsonImplDF> ZMobiusFermionDF;
|
||||
//typedef ZMobiusFermion<ZWilsonImplRL> ZMobiusFermionRL;
|
||||
//typedef ZMobiusFermion<ZWilsonImplFH> ZMobiusFermionFH;
|
||||
//typedef ZMobiusFermion<ZWilsonImplDF> ZMobiusFermionDF;
|
||||
|
||||
// Ls vectorised
|
||||
typedef ScaledShamirFermion<WilsonImplR> ScaledShamirFermionR;
|
||||
@ -235,49 +235,49 @@ typedef WilsonFermion<GparityWilsonImplR> GparityWilsonFermionR;
|
||||
typedef WilsonFermion<GparityWilsonImplF> GparityWilsonFermionF;
|
||||
typedef WilsonFermion<GparityWilsonImplD> GparityWilsonFermionD;
|
||||
|
||||
typedef WilsonFermion<GparityWilsonImplRL> GparityWilsonFermionRL;
|
||||
typedef WilsonFermion<GparityWilsonImplFH> GparityWilsonFermionFH;
|
||||
typedef WilsonFermion<GparityWilsonImplDF> GparityWilsonFermionDF;
|
||||
//typedef WilsonFermion<GparityWilsonImplRL> GparityWilsonFermionRL;
|
||||
//typedef WilsonFermion<GparityWilsonImplFH> GparityWilsonFermionFH;
|
||||
//typedef WilsonFermion<GparityWilsonImplDF> GparityWilsonFermionDF;
|
||||
|
||||
typedef DomainWallFermion<GparityWilsonImplR> GparityDomainWallFermionR;
|
||||
typedef DomainWallFermion<GparityWilsonImplF> GparityDomainWallFermionF;
|
||||
typedef DomainWallFermion<GparityWilsonImplD> GparityDomainWallFermionD;
|
||||
|
||||
typedef DomainWallFermion<GparityWilsonImplRL> GparityDomainWallFermionRL;
|
||||
typedef DomainWallFermion<GparityWilsonImplFH> GparityDomainWallFermionFH;
|
||||
typedef DomainWallFermion<GparityWilsonImplDF> GparityDomainWallFermionDF;
|
||||
//typedef DomainWallFermion<GparityWilsonImplRL> GparityDomainWallFermionRL;
|
||||
//typedef DomainWallFermion<GparityWilsonImplFH> GparityDomainWallFermionFH;
|
||||
//typedef DomainWallFermion<GparityWilsonImplDF> GparityDomainWallFermionDF;
|
||||
|
||||
typedef DomainWallEOFAFermion<GparityWilsonImplR> GparityDomainWallEOFAFermionR;
|
||||
typedef DomainWallEOFAFermion<GparityWilsonImplF> GparityDomainWallEOFAFermionF;
|
||||
typedef DomainWallEOFAFermion<GparityWilsonImplD> GparityDomainWallEOFAFermionD;
|
||||
|
||||
typedef DomainWallEOFAFermion<GparityWilsonImplRL> GparityDomainWallEOFAFermionRL;
|
||||
typedef DomainWallEOFAFermion<GparityWilsonImplFH> GparityDomainWallEOFAFermionFH;
|
||||
typedef DomainWallEOFAFermion<GparityWilsonImplDF> GparityDomainWallEOFAFermionDF;
|
||||
//typedef DomainWallEOFAFermion<GparityWilsonImplRL> GparityDomainWallEOFAFermionRL;
|
||||
//typedef DomainWallEOFAFermion<GparityWilsonImplFH> GparityDomainWallEOFAFermionFH;
|
||||
//typedef DomainWallEOFAFermion<GparityWilsonImplDF> GparityDomainWallEOFAFermionDF;
|
||||
|
||||
typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionR;
|
||||
typedef WilsonTMFermion<GparityWilsonImplF> GparityWilsonTMFermionF;
|
||||
typedef WilsonTMFermion<GparityWilsonImplD> GparityWilsonTMFermionD;
|
||||
|
||||
typedef WilsonTMFermion<GparityWilsonImplRL> GparityWilsonTMFermionRL;
|
||||
typedef WilsonTMFermion<GparityWilsonImplFH> GparityWilsonTMFermionFH;
|
||||
typedef WilsonTMFermion<GparityWilsonImplDF> GparityWilsonTMFermionDF;
|
||||
//typedef WilsonTMFermion<GparityWilsonImplRL> GparityWilsonTMFermionRL;
|
||||
//typedef WilsonTMFermion<GparityWilsonImplFH> GparityWilsonTMFermionFH;
|
||||
//typedef WilsonTMFermion<GparityWilsonImplDF> GparityWilsonTMFermionDF;
|
||||
|
||||
typedef MobiusFermion<GparityWilsonImplR> GparityMobiusFermionR;
|
||||
typedef MobiusFermion<GparityWilsonImplF> GparityMobiusFermionF;
|
||||
typedef MobiusFermion<GparityWilsonImplD> GparityMobiusFermionD;
|
||||
|
||||
typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
|
||||
typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
|
||||
typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
|
||||
//typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
|
||||
//typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
|
||||
//typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
|
||||
|
||||
typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusEOFAFermionR;
|
||||
typedef MobiusEOFAFermion<GparityWilsonImplF> GparityMobiusEOFAFermionF;
|
||||
typedef MobiusEOFAFermion<GparityWilsonImplD> GparityMobiusEOFAFermionD;
|
||||
|
||||
typedef MobiusEOFAFermion<GparityWilsonImplRL> GparityMobiusEOFAFermionRL;
|
||||
typedef MobiusEOFAFermion<GparityWilsonImplFH> GparityMobiusEOFAFermionFH;
|
||||
typedef MobiusEOFAFermion<GparityWilsonImplDF> GparityMobiusEOFAFermionDF;
|
||||
//typedef MobiusEOFAFermion<GparityWilsonImplRL> GparityMobiusEOFAFermionRL;
|
||||
//typedef MobiusEOFAFermion<GparityWilsonImplFH> GparityMobiusEOFAFermionFH;
|
||||
//typedef MobiusEOFAFermion<GparityWilsonImplDF> GparityMobiusEOFAFermionDF;
|
||||
|
||||
typedef ImprovedStaggeredFermion<StaggeredImplR> ImprovedStaggeredFermionR;
|
||||
typedef ImprovedStaggeredFermion<StaggeredImplF> ImprovedStaggeredFermionF;
|
||||
@ -291,12 +291,6 @@ typedef ImprovedStaggeredFermion5D<StaggeredImplR> ImprovedStaggeredFermion5DR;
|
||||
typedef ImprovedStaggeredFermion5D<StaggeredImplF> ImprovedStaggeredFermion5DF;
|
||||
typedef ImprovedStaggeredFermion5D<StaggeredImplD> ImprovedStaggeredFermion5DD;
|
||||
|
||||
#ifndef GRID_CUDA
|
||||
typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplR> ImprovedStaggeredFermionVec5dR;
|
||||
typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplF> ImprovedStaggeredFermionVec5dF;
|
||||
typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplD> ImprovedStaggeredFermionVec5dD;
|
||||
#endif
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
////////////////////
|
||||
|
||||
@ -153,8 +153,8 @@ public:
|
||||
typedef typename Impl::StencilImpl StencilImpl; \
|
||||
typedef typename Impl::ImplParams ImplParams; \
|
||||
typedef typename Impl::StencilImpl::View_type StencilView; \
|
||||
typedef typename ViewMap<FermionField>::Type FermionFieldView; \
|
||||
typedef typename ViewMap<DoubledGaugeField>::Type DoubledGaugeFieldView;
|
||||
typedef const typename ViewMap<FermionField>::Type FermionFieldView; \
|
||||
typedef const typename ViewMap<DoubledGaugeField>::Type DoubledGaugeFieldView;
|
||||
|
||||
#define INHERIT_IMPL_TYPES(Base) \
|
||||
INHERIT_GIMPL_TYPES(Base) \
|
||||
@ -183,7 +183,8 @@ NAMESPACE_CHECK(ImplStaggered);
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
// Single flavour one component spinors with colour index. 5d vec
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
#include <Grid/qcd/action/fermion/StaggeredVec5dImpl.h>
|
||||
NAMESPACE_CHECK(ImplStaggered5dVec);
|
||||
// Deprecate Vec5d
|
||||
//#include <Grid/qcd/action/fermion/StaggeredVec5dImpl.h>
|
||||
//NAMESPACE_CHECK(ImplStaggered5dVec);
|
||||
|
||||
|
||||
|
||||
@ -327,8 +327,8 @@ typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffReal> Gparit
|
||||
typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffReal> GparityWilsonImplF; // Float
|
||||
typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffReal> GparityWilsonImplD; // Double
|
||||
|
||||
typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplRL; // Real.. whichever prec
|
||||
typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplFH; // Float
|
||||
typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplDF; // Double
|
||||
//typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplRL; // Real.. whichever prec
|
||||
//typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplFH; // Float
|
||||
//typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplDF; // Double
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@ -72,19 +72,23 @@ public:
|
||||
|
||||
StaggeredImpl(const ImplParams &p = ImplParams()) : Params(p){};
|
||||
|
||||
static accelerator_inline void multLink(SiteSpinor &phi,
|
||||
template<class _Spinor>
|
||||
static accelerator_inline void multLink(_Spinor &phi,
|
||||
const SiteDoubledGaugeField &U,
|
||||
const SiteSpinor &chi,
|
||||
const _Spinor &chi,
|
||||
int mu)
|
||||
{
|
||||
mult(&phi(), &U(mu), &chi());
|
||||
auto UU = coalescedRead(U(mu));
|
||||
mult(&phi(), &UU, &chi());
|
||||
}
|
||||
static accelerator_inline void multLinkAdd(SiteSpinor &phi,
|
||||
template<class _Spinor>
|
||||
static accelerator_inline void multLinkAdd(_Spinor &phi,
|
||||
const SiteDoubledGaugeField &U,
|
||||
const SiteSpinor &chi,
|
||||
const _Spinor &chi,
|
||||
int mu)
|
||||
{
|
||||
mac(&phi(), &U(mu), &chi());
|
||||
auto UU = coalescedRead(U(mu));
|
||||
mac(&phi(), &UU, &chi());
|
||||
}
|
||||
|
||||
template <class ref>
|
||||
|
||||
@ -61,18 +61,19 @@ public:
|
||||
typedef typename SiteHalfSpinor::vector_type vComplexHigh;
|
||||
constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
|
||||
|
||||
accelerator_inline int CommDatumSize(void) {
|
||||
accelerator_inline int CommDatumSize(void) const {
|
||||
return sizeof(SiteHalfCommSpinor);
|
||||
}
|
||||
|
||||
/*****************************************************/
|
||||
/* Compress includes precision change if mpi data is not same */
|
||||
/*****************************************************/
|
||||
template<class _SiteHalfSpinor, class _SiteSpinor>
|
||||
accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) {
|
||||
_SiteHalfSpinor tmp;
|
||||
projector::Proj(tmp,in,mu,dag);
|
||||
vstream(buf[o],tmp);
|
||||
accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const {
|
||||
typedef decltype(coalescedRead(buf)) sobj;
|
||||
sobj sp;
|
||||
auto sin = coalescedRead(in);
|
||||
projector::Proj(sp,sin,mu,dag);
|
||||
coalescedWrite(buf,sp);
|
||||
}
|
||||
|
||||
/*****************************************************/
|
||||
@ -81,19 +82,24 @@ public:
|
||||
accelerator_inline void Exchange(SiteHalfSpinor *mp,
|
||||
const SiteHalfSpinor * __restrict__ vp0,
|
||||
const SiteHalfSpinor * __restrict__ vp1,
|
||||
Integer type,Integer o){
|
||||
Integer type,Integer o) const {
|
||||
#ifdef GRID_SIMT
|
||||
exchangeSIMT(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type);
|
||||
#else
|
||||
SiteHalfSpinor tmp1;
|
||||
SiteHalfSpinor tmp2;
|
||||
exchange(tmp1,tmp2,vp0[o],vp1[o],type);
|
||||
vstream(mp[2*o ],tmp1);
|
||||
vstream(mp[2*o+1],tmp2);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/*****************************************************/
|
||||
/* Have a decompression step if mpi data is not same */
|
||||
/*****************************************************/
|
||||
accelerator_inline void Decompress(SiteHalfSpinor * __restrict__ out,
|
||||
SiteHalfSpinor * __restrict__ in, Integer o) {
|
||||
SiteHalfSpinor * __restrict__ in, Integer o) const {
|
||||
assert(0);
|
||||
}
|
||||
|
||||
@ -103,8 +109,30 @@ public:
|
||||
accelerator_inline void CompressExchange(SiteHalfSpinor * __restrict__ out0,
|
||||
SiteHalfSpinor * __restrict__ out1,
|
||||
const SiteSpinor * __restrict__ in,
|
||||
Integer j,Integer k, Integer m,Integer type)
|
||||
Integer j,Integer k, Integer m,Integer type) const
|
||||
{
|
||||
#ifdef GRID_SIMT
|
||||
typedef SiteSpinor vobj;
|
||||
typedef SiteHalfSpinor hvobj;
|
||||
typedef decltype(coalescedRead(*in)) sobj;
|
||||
typedef decltype(coalescedRead(*out0)) hsobj;
|
||||
|
||||
unsigned int Nsimd = vobj::Nsimd();
|
||||
unsigned int mask = Nsimd >> (type + 1);
|
||||
int lane = acceleratorSIMTlane(Nsimd);
|
||||
int j0 = lane &(~mask); // inner coor zero
|
||||
int j1 = lane |(mask) ; // inner coor one
|
||||
const vobj *vp0 = &in[k];
|
||||
const vobj *vp1 = &in[m];
|
||||
const vobj *vp = (lane&mask) ? vp1:vp0;
|
||||
auto sa = coalescedRead(*vp,j0);
|
||||
auto sb = coalescedRead(*vp,j1);
|
||||
hsobj psa, psb;
|
||||
projector::Proj(psa,sa,mu,dag);
|
||||
projector::Proj(psb,sb,mu,dag);
|
||||
coalescedWrite(out0[j],psa);
|
||||
coalescedWrite(out1[j],psb);
|
||||
#else
|
||||
SiteHalfSpinor temp1, temp2;
|
||||
SiteHalfSpinor temp3, temp4;
|
||||
projector::Proj(temp1,in[k],mu,dag);
|
||||
@ -112,15 +140,17 @@ public:
|
||||
exchange(temp3,temp4,temp1,temp2,type);
|
||||
vstream(out0[j],temp3);
|
||||
vstream(out1[j],temp4);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*****************************************************/
|
||||
/* Pass the info to the stencil */
|
||||
/*****************************************************/
|
||||
accelerator_inline bool DecompressionStep(void) { return false; }
|
||||
accelerator_inline bool DecompressionStep(void) const { return false; }
|
||||
|
||||
};
|
||||
|
||||
#if 0
|
||||
template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
|
||||
class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector,
|
||||
typename std::enable_if<!std::is_same<_HCspinor,_Hspinor>::value>::type >
|
||||
@ -142,20 +172,30 @@ public:
|
||||
typedef typename SiteHalfSpinor::vector_type vComplexHigh;
|
||||
constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
|
||||
|
||||
accelerator_inline int CommDatumSize(void) {
|
||||
accelerator_inline int CommDatumSize(void) const {
|
||||
return sizeof(SiteHalfCommSpinor);
|
||||
}
|
||||
|
||||
/*****************************************************/
|
||||
/* Compress includes precision change if mpi data is not same */
|
||||
/*****************************************************/
|
||||
template<class _SiteHalfSpinor, class _SiteSpinor>
|
||||
accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) {
|
||||
_SiteHalfSpinor hsp;
|
||||
accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const {
|
||||
SiteHalfSpinor hsp;
|
||||
SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf;
|
||||
projector::Proj(hsp,in,mu,dag);
|
||||
precisionChange((vComplexLow *)&hbuf[o],(vComplexHigh *)&hsp,Nw);
|
||||
}
|
||||
accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const {
|
||||
#ifdef GRID_SIMT
|
||||
typedef decltype(coalescedRead(buf)) sobj;
|
||||
sobj sp;
|
||||
auto sin = coalescedRead(in);
|
||||
projector::Proj(sp,sin,mu,dag);
|
||||
coalescedWrite(buf,sp);
|
||||
#else
|
||||
projector::Proj(buf,in,mu,dag);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*****************************************************/
|
||||
/* Exchange includes precision change if mpi data is not same */
|
||||
@ -163,7 +203,7 @@ public:
|
||||
accelerator_inline void Exchange(SiteHalfSpinor *mp,
|
||||
SiteHalfSpinor *vp0,
|
||||
SiteHalfSpinor *vp1,
|
||||
Integer type,Integer o){
|
||||
Integer type,Integer o) const {
|
||||
SiteHalfSpinor vt0,vt1;
|
||||
SiteHalfCommSpinor *vpp0 = (SiteHalfCommSpinor *)vp0;
|
||||
SiteHalfCommSpinor *vpp1 = (SiteHalfCommSpinor *)vp1;
|
||||
@ -175,7 +215,7 @@ public:
|
||||
/*****************************************************/
|
||||
/* Have a decompression step if mpi data is not same */
|
||||
/*****************************************************/
|
||||
accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o){
|
||||
accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o) const {
|
||||
SiteHalfCommSpinor *hin=(SiteHalfCommSpinor *)in;
|
||||
precisionChange((vComplexHigh *)&out[o],(vComplexLow *)&hin[o],Nw);
|
||||
}
|
||||
@ -186,7 +226,7 @@ public:
|
||||
accelerator_inline void CompressExchange(SiteHalfSpinor *out0,
|
||||
SiteHalfSpinor *out1,
|
||||
const SiteSpinor *in,
|
||||
Integer j,Integer k, Integer m,Integer type){
|
||||
Integer j,Integer k, Integer m,Integer type) const {
|
||||
SiteHalfSpinor temp1, temp2,temp3,temp4;
|
||||
SiteHalfCommSpinor *hout0 = (SiteHalfCommSpinor *)out0;
|
||||
SiteHalfCommSpinor *hout1 = (SiteHalfCommSpinor *)out1;
|
||||
@ -200,9 +240,10 @@ public:
|
||||
/*****************************************************/
|
||||
/* Pass the info to the stencil */
|
||||
/*****************************************************/
|
||||
accelerator_inline bool DecompressionStep(void) { return true; }
|
||||
accelerator_inline bool DecompressionStep(void) const { return true; }
|
||||
|
||||
};
|
||||
#endif
|
||||
|
||||
#define DECLARE_PROJ(Projector,Compressor,spProj) \
|
||||
class Projector { \
|
||||
@ -253,33 +294,8 @@ public:
|
||||
typedef typename Base::View_type View_type;
|
||||
typedef typename Base::StencilVector StencilVector;
|
||||
|
||||
double timer0;
|
||||
double timer1;
|
||||
double timer2;
|
||||
double timer3;
|
||||
double timer4;
|
||||
double timer5;
|
||||
double timer6;
|
||||
uint64_t callsi;
|
||||
void ZeroCountersi(void)
|
||||
{
|
||||
timer0=0;
|
||||
timer1=0;
|
||||
timer2=0;
|
||||
timer3=0;
|
||||
timer4=0;
|
||||
timer5=0;
|
||||
timer6=0;
|
||||
callsi=0;
|
||||
}
|
||||
void Reporti(int calls)
|
||||
{
|
||||
if ( timer0 ) std::cout << GridLogMessage << " timer0 (HaloGatherOpt) " <<timer0/calls <<std::endl;
|
||||
if ( timer1 ) std::cout << GridLogMessage << " timer1 (Communicate) " <<timer1/calls <<std::endl;
|
||||
if ( timer2 ) std::cout << GridLogMessage << " timer2 (CommsMerge ) " <<timer2/calls <<std::endl;
|
||||
if ( timer3 ) std::cout << GridLogMessage << " timer3 (commsMergeShm) " <<timer3/calls <<std::endl;
|
||||
if ( timer4 ) std::cout << GridLogMessage << " timer4 " <<timer4 <<std::endl;
|
||||
}
|
||||
void ZeroCountersi(void) { }
|
||||
void Reporti(int calls) { }
|
||||
|
||||
std::vector<int> surface_list;
|
||||
|
||||
@ -321,26 +337,18 @@ public:
|
||||
{
|
||||
std::vector<std::vector<CommsRequest_t> > reqs;
|
||||
this->HaloExchangeOptGather(source,compress);
|
||||
double t1=usecond();
|
||||
// Asynchronous MPI calls multidirectional, Isend etc...
|
||||
// Non-overlapped directions within a thread. Asynchronous calls except MPI3, threaded up to comm threads ways.
|
||||
this->Communicate();
|
||||
double t2=usecond(); timer1 += t2-t1;
|
||||
this->CommsMerge(compress);
|
||||
double t3=usecond(); timer2 += t3-t2;
|
||||
this->CommsMergeSHM(compress);
|
||||
double t4=usecond(); timer3 += t4-t3;
|
||||
}
|
||||
|
||||
template <class compressor>
|
||||
void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress)
|
||||
{
|
||||
this->Prepare();
|
||||
double t0=usecond();
|
||||
this->HaloGatherOpt(source,compress);
|
||||
double t1=usecond();
|
||||
timer0 += t1-t0;
|
||||
callsi++;
|
||||
}
|
||||
|
||||
template <class compressor>
|
||||
@ -352,12 +360,9 @@ public:
|
||||
typedef typename compressor::SiteHalfSpinor SiteHalfSpinor;
|
||||
typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor;
|
||||
|
||||
this->mpi3synctime_g-=usecond();
|
||||
this->_grid->StencilBarrier();
|
||||
this->mpi3synctime_g+=usecond();
|
||||
|
||||
assert(source.Grid()==this->_grid);
|
||||
this->halogtime-=usecond();
|
||||
|
||||
this->u_comm_offset=0;
|
||||
|
||||
@ -393,7 +398,6 @@ public:
|
||||
}
|
||||
this->face_table_computed=1;
|
||||
assert(this->u_comm_offset==this->_unified_buffer_size);
|
||||
this->halogtime+=usecond();
|
||||
accelerator_barrier();
|
||||
}
|
||||
|
||||
|
||||
@ -72,7 +72,7 @@ public:
|
||||
typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
|
||||
typedef WilsonImplParams ImplParams;
|
||||
typedef WilsonStencil<SiteSpinor, SiteHalfSpinor,ImplParams> StencilImpl;
|
||||
typedef typename StencilImpl::View_type StencilView;
|
||||
typedef const typename StencilImpl::View_type StencilView;
|
||||
|
||||
ImplParams Params;
|
||||
|
||||
@ -184,18 +184,22 @@ public:
|
||||
mat = TraceIndex<SpinIndex>(P);
|
||||
}
|
||||
|
||||
inline void extractLinkField(std::vector<GaugeLinkField> &mat, DoubledGaugeField &Uds){
|
||||
inline void extractLinkField(std::vector<GaugeLinkField> &mat, DoubledGaugeField &Uds)
|
||||
{
|
||||
for (int mu = 0; mu < Nd; mu++)
|
||||
mat[mu] = PeekIndex<LorentzIndex>(Uds, mu);
|
||||
}
|
||||
|
||||
|
||||
inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu){
|
||||
|
||||
inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu)
|
||||
{
|
||||
#undef USE_OLD_INSERT_FORCE
|
||||
int Ls=Btilde.Grid()->_fdimensions[0];
|
||||
autoView( mat_v , mat, AcceleratorWrite);
|
||||
#ifdef USE_OLD_INSERT_FORCE
|
||||
GaugeLinkField tmp(mat.Grid());
|
||||
tmp = Zero();
|
||||
{
|
||||
const int Nsimd = SiteSpinor::Nsimd();
|
||||
autoView( tmp_v , tmp, AcceleratorWrite);
|
||||
autoView( Btilde_v , Btilde, AcceleratorRead);
|
||||
autoView( Atilde_v , Atilde, AcceleratorRead);
|
||||
@ -208,6 +212,29 @@ public:
|
||||
});
|
||||
}
|
||||
PokeIndex<LorentzIndex>(mat,tmp,mu);
|
||||
#else
|
||||
{
|
||||
const int Nsimd = SiteSpinor::Nsimd();
|
||||
autoView( Btilde_v , Btilde, AcceleratorRead);
|
||||
autoView( Atilde_v , Atilde, AcceleratorRead);
|
||||
accelerator_for(sss,mat.Grid()->oSites(),Nsimd,{
|
||||
int sU=sss;
|
||||
typedef decltype(coalescedRead(mat_v[sU](mu)() )) ColorMatrixType;
|
||||
ColorMatrixType sum;
|
||||
zeroit(sum);
|
||||
for(int s=0;s<Ls;s++){
|
||||
int sF = s+Ls*sU;
|
||||
for(int spn=0;spn<Ns;spn++){ //sum over spin
|
||||
auto bb = coalescedRead(Btilde_v[sF]()(spn) ); //color vector
|
||||
auto aa = coalescedRead(Atilde_v[sF]()(spn) );
|
||||
auto op = outerProduct(bb,aa);
|
||||
sum = sum + op;
|
||||
}
|
||||
}
|
||||
coalescedWrite(mat_v[sU](mu)(), sum);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@ -216,17 +243,17 @@ typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffReal > WilsonImplR
|
||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffReal > WilsonImplF; // Float
|
||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffReal > WilsonImplD; // Double
|
||||
|
||||
typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplRL; // Real.. whichever prec
|
||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplFH; // Float
|
||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplDF; // Double
|
||||
//typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplRL; // Real.. whichever prec
|
||||
//typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplFH; // Float
|
||||
//typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplDF; // Double
|
||||
|
||||
typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffComplex > ZWilsonImplR; // Real.. whichever prec
|
||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplex > ZWilsonImplF; // Float
|
||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplex > ZWilsonImplD; // Double
|
||||
|
||||
typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplRL; // Real.. whichever prec
|
||||
typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplFH; // Float
|
||||
typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplDF; // Double
|
||||
//typedef WilsonImpl<vComplex, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplRL; // Real.. whichever prec
|
||||
//typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplFH; // Float
|
||||
//typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplDF; // Double
|
||||
|
||||
typedef WilsonImpl<vComplex, AdjointRepresentation, CoeffReal > WilsonAdjImplR; // Real.. whichever prec
|
||||
typedef WilsonImpl<vComplexF, AdjointRepresentation, CoeffReal > WilsonAdjImplF; // Float
|
||||
|
||||
@ -49,9 +49,17 @@ public:
|
||||
|
||||
INHERIT_IMPL_TYPES(Impl);
|
||||
typedef FermionOperator<Impl> Base;
|
||||
|
||||
typedef AcceleratorVector<int,STENCIL_MAX> StencilVector;
|
||||
public:
|
||||
|
||||
#ifdef GRID_SYCL
|
||||
#define SYCL_HACK
|
||||
#endif
|
||||
#ifdef SYCL_HACK
|
||||
static void HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const SiteSpinor *in, SiteSpinor *out);
|
||||
#endif
|
||||
|
||||
static void DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField &U, SiteHalfSpinor * buf,
|
||||
int Ls, int Nsite, const FermionField &in, FermionField &out,
|
||||
int interior=1,int exterior=1) ;
|
||||
|
||||
@ -880,17 +880,29 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
|
||||
}
|
||||
|
||||
std::vector<RealD> G_s(Ls,1.0);
|
||||
RealD sign = 1; // sign flip for vector/tadpole
|
||||
if ( curr_type == Current::Axial ) {
|
||||
for(int s=0;s<Ls/2;s++){
|
||||
G_s[s] = -1.0;
|
||||
}
|
||||
}
|
||||
else if ( curr_type == Current::Tadpole ) {
|
||||
auto b=this->_b;
|
||||
auto c=this->_c;
|
||||
if ( b == 1 && c == 0 ) {
|
||||
sign = -1;
|
||||
}
|
||||
else {
|
||||
std::cerr << "Error: Tadpole implementation currently unavailable for non-Shamir actions." << std::endl;
|
||||
assert(b==1 && c==0);
|
||||
}
|
||||
}
|
||||
|
||||
for(int s=0;s<Ls;s++){
|
||||
|
||||
int sp = (s+1)%Ls;
|
||||
int sr = Ls-1-s;
|
||||
int srp= (sr+1)%Ls;
|
||||
// int sr = Ls-1-s;
|
||||
// int srp= (sr+1)%Ls;
|
||||
|
||||
// Mobius parameters
|
||||
auto b=this->bs[s];
|
||||
@ -907,7 +919,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
|
||||
|
||||
tmp = Cshift(tmp,mu,1);
|
||||
Impl::multLinkField(Utmp,this->Umu,tmp,mu);
|
||||
tmp = G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop
|
||||
tmp = sign*G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop
|
||||
tmp = where((lcoor>=tmin),tmp,zz); // Mask the time
|
||||
L_Q = where((lcoor<=tmax),tmp,zz); // Position of current complicated
|
||||
|
||||
|
||||
@ -680,7 +680,8 @@ void StaggeredKernels<Impl>::DhopSiteAsm(StencilView &st,
|
||||
gauge2 =(uint64_t)&UU[sU]( Z ); \
|
||||
gauge3 =(uint64_t)&UU[sU]( T );
|
||||
|
||||
|
||||
#undef STAG_VEC5D
|
||||
#ifdef STAG_VEC5D
|
||||
// This is the single precision 5th direction vectorised kernel
|
||||
#include <Grid/simd/Intel512single.h>
|
||||
template <> void StaggeredKernels<StaggeredVec5dImplF>::DhopSiteAsm(StencilView &st,
|
||||
@ -790,7 +791,7 @@ template <> void StaggeredKernels<StaggeredVec5dImplD>::DhopSiteAsm(StencilView
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
#define PERMUTE_DIR3 __asm__ ( \
|
||||
|
||||
@ -32,25 +32,50 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#define LOAD_CHI(b) \
|
||||
#ifdef GRID_SIMT
|
||||
|
||||
#define LOAD_CHI(ptype,b) \
|
||||
const SiteSpinor & ref (b[offset]); \
|
||||
Chi_0=coalescedReadPermute<ptype>(ref()()(0),perm,lane); \
|
||||
Chi_1=coalescedReadPermute<ptype>(ref()()(1),perm,lane); \
|
||||
Chi_2=coalescedReadPermute<ptype>(ref()()(2),perm,lane);
|
||||
|
||||
#define LOAD_CHI_COMMS(b) \
|
||||
const SiteSpinor & ref (b[offset]); \
|
||||
Chi_0=ref()()(0);\
|
||||
Chi_1=ref()()(1);\
|
||||
Chi_2=ref()()(2);
|
||||
Chi_0=coalescedRead(ref()()(0),lane); \
|
||||
Chi_1=coalescedRead(ref()()(1),lane); \
|
||||
Chi_2=coalescedRead(ref()()(2),lane);
|
||||
|
||||
#define PERMUTE_DIR(dir) ;
|
||||
#else
|
||||
#define LOAD_CHI(ptype,b) LOAD_CHI_COMMS(b)
|
||||
|
||||
#define LOAD_CHI_COMMS(b) \
|
||||
const SiteSpinor & ref (b[offset]); \
|
||||
Chi_0=ref()()(0); \
|
||||
Chi_1=ref()()(1); \
|
||||
Chi_2=ref()()(2);
|
||||
|
||||
#define PERMUTE_DIR(dir) \
|
||||
permute##dir(Chi_0,Chi_0); \
|
||||
permute##dir(Chi_1,Chi_1); \
|
||||
permute##dir(Chi_2,Chi_2);
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// To splat or not to splat depends on the implementation
|
||||
#define MULT(A,UChi) \
|
||||
auto & ref(U[sU](A)); \
|
||||
Impl::loadLinkElement(U_00,ref()(0,0)); \
|
||||
Impl::loadLinkElement(U_10,ref()(1,0)); \
|
||||
Impl::loadLinkElement(U_20,ref()(2,0)); \
|
||||
Impl::loadLinkElement(U_01,ref()(0,1)); \
|
||||
Impl::loadLinkElement(U_11,ref()(1,1)); \
|
||||
Impl::loadLinkElement(U_21,ref()(2,1)); \
|
||||
Impl::loadLinkElement(U_02,ref()(0,2)); \
|
||||
Impl::loadLinkElement(U_12,ref()(1,2)); \
|
||||
Impl::loadLinkElement(U_22,ref()(2,2)); \
|
||||
U_00=coalescedRead(ref()(0,0),lane); \
|
||||
U_10=coalescedRead(ref()(1,0),lane); \
|
||||
U_20=coalescedRead(ref()(2,0),lane); \
|
||||
U_01=coalescedRead(ref()(0,1),lane); \
|
||||
U_11=coalescedRead(ref()(1,1),lane); \
|
||||
U_21=coalescedRead(ref()(2,1),lane); \
|
||||
U_02=coalescedRead(ref()(0,2),lane); \
|
||||
U_12=coalescedRead(ref()(1,2),lane); \
|
||||
U_22=coalescedRead(ref()(2,2),lane); \
|
||||
UChi ## _0 = U_00*Chi_0; \
|
||||
UChi ## _1 = U_10*Chi_0;\
|
||||
UChi ## _2 = U_20*Chi_0;\
|
||||
@ -63,15 +88,15 @@ NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#define MULT_ADD(U,A,UChi) \
|
||||
auto & ref(U[sU](A)); \
|
||||
Impl::loadLinkElement(U_00,ref()(0,0)); \
|
||||
Impl::loadLinkElement(U_10,ref()(1,0)); \
|
||||
Impl::loadLinkElement(U_20,ref()(2,0)); \
|
||||
Impl::loadLinkElement(U_01,ref()(0,1)); \
|
||||
Impl::loadLinkElement(U_11,ref()(1,1)); \
|
||||
Impl::loadLinkElement(U_21,ref()(2,1)); \
|
||||
Impl::loadLinkElement(U_02,ref()(0,2)); \
|
||||
Impl::loadLinkElement(U_12,ref()(1,2)); \
|
||||
Impl::loadLinkElement(U_22,ref()(2,2)); \
|
||||
U_00=coalescedRead(ref()(0,0),lane); \
|
||||
U_10=coalescedRead(ref()(1,0),lane); \
|
||||
U_20=coalescedRead(ref()(2,0),lane); \
|
||||
U_01=coalescedRead(ref()(0,1),lane); \
|
||||
U_11=coalescedRead(ref()(1,1),lane); \
|
||||
U_21=coalescedRead(ref()(2,1),lane); \
|
||||
U_02=coalescedRead(ref()(0,2),lane); \
|
||||
U_12=coalescedRead(ref()(1,2),lane); \
|
||||
U_22=coalescedRead(ref()(2,2),lane); \
|
||||
UChi ## _0 += U_00*Chi_0; \
|
||||
UChi ## _1 += U_10*Chi_0;\
|
||||
UChi ## _2 += U_20*Chi_0;\
|
||||
@ -83,24 +108,18 @@ NAMESPACE_BEGIN(Grid);
|
||||
UChi ## _2 += U_22*Chi_2;
|
||||
|
||||
|
||||
#define PERMUTE_DIR(dir) \
|
||||
permute##dir(Chi_0,Chi_0); \
|
||||
permute##dir(Chi_1,Chi_1); \
|
||||
permute##dir(Chi_2,Chi_2);
|
||||
|
||||
|
||||
#define HAND_STENCIL_LEG_BASE(Dir,Perm,skew) \
|
||||
SE=st.GetEntry(ptype,Dir+skew,sF); \
|
||||
offset = SE->_offset; \
|
||||
local = SE->_is_local; \
|
||||
perm = SE->_permute; \
|
||||
if ( local ) { \
|
||||
LOAD_CHI(in); \
|
||||
LOAD_CHI(Perm,in); \
|
||||
if ( perm) { \
|
||||
PERMUTE_DIR(Perm); \
|
||||
} \
|
||||
} else { \
|
||||
LOAD_CHI(buf); \
|
||||
LOAD_CHI_COMMS(buf); \
|
||||
}
|
||||
|
||||
#define HAND_STENCIL_LEG_BEGIN(Dir,Perm,skew,even) \
|
||||
@ -116,19 +135,18 @@ NAMESPACE_BEGIN(Grid);
|
||||
}
|
||||
|
||||
|
||||
|
||||
#define HAND_STENCIL_LEG_INT(U,Dir,Perm,skew,even) \
|
||||
SE=st.GetEntry(ptype,Dir+skew,sF); \
|
||||
offset = SE->_offset; \
|
||||
local = SE->_is_local; \
|
||||
perm = SE->_permute; \
|
||||
if ( local ) { \
|
||||
LOAD_CHI(in); \
|
||||
LOAD_CHI(Perm,in); \
|
||||
if ( perm) { \
|
||||
PERMUTE_DIR(Perm); \
|
||||
} \
|
||||
} else if ( st.same_node[Dir] ) { \
|
||||
LOAD_CHI(buf); \
|
||||
LOAD_CHI_COMMS(buf); \
|
||||
} \
|
||||
if (local || st.same_node[Dir] ) { \
|
||||
MULT_ADD(U,Dir,even); \
|
||||
@ -140,10 +158,32 @@ NAMESPACE_BEGIN(Grid);
|
||||
local = SE->_is_local; \
|
||||
if ((!local) && (!st.same_node[Dir]) ) { \
|
||||
nmu++; \
|
||||
{ LOAD_CHI(buf); } \
|
||||
{ LOAD_CHI_COMMS(buf); } \
|
||||
{ MULT_ADD(U,Dir,even); } \
|
||||
}
|
||||
|
||||
#define HAND_DECLARATIONS(Simd) \
|
||||
Simd even_0; \
|
||||
Simd even_1; \
|
||||
Simd even_2; \
|
||||
Simd odd_0; \
|
||||
Simd odd_1; \
|
||||
Simd odd_2; \
|
||||
\
|
||||
Simd Chi_0; \
|
||||
Simd Chi_1; \
|
||||
Simd Chi_2; \
|
||||
\
|
||||
Simd U_00; \
|
||||
Simd U_10; \
|
||||
Simd U_20; \
|
||||
Simd U_01; \
|
||||
Simd U_11; \
|
||||
Simd U_21; \
|
||||
Simd U_02; \
|
||||
Simd U_12; \
|
||||
Simd U_22;
|
||||
|
||||
|
||||
template <class Impl>
|
||||
template <int Naik> accelerator_inline
|
||||
@ -155,28 +195,14 @@ void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
|
||||
Simd even_0; // 12 regs on knc
|
||||
Simd even_1;
|
||||
Simd even_2;
|
||||
Simd odd_0; // 12 regs on knc
|
||||
Simd odd_1;
|
||||
Simd odd_2;
|
||||
|
||||
Simd Chi_0; // two spinor; 6 regs
|
||||
Simd Chi_1;
|
||||
Simd Chi_2;
|
||||
|
||||
Simd U_00; // two rows of U matrix
|
||||
Simd U_10;
|
||||
Simd U_20;
|
||||
Simd U_01;
|
||||
Simd U_11;
|
||||
Simd U_21; // 2 reg left.
|
||||
Simd U_02;
|
||||
Simd U_12;
|
||||
Simd U_22;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
SiteSpinor result;
|
||||
typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
|
||||
calcSiteSpinor result;
|
||||
int offset,local,perm, ptype;
|
||||
|
||||
StencilEntry *SE;
|
||||
@ -215,7 +241,7 @@ void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
|
||||
result()()(1) = even_1 + odd_1;
|
||||
result()()(2) = even_2 + odd_2;
|
||||
}
|
||||
vstream(out[sF],result);
|
||||
coalescedWrite(out[sF],result);
|
||||
}
|
||||
}
|
||||
|
||||
@ -230,28 +256,13 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
|
||||
Simd even_0; // 12 regs on knc
|
||||
Simd even_1;
|
||||
Simd even_2;
|
||||
Simd odd_0; // 12 regs on knc
|
||||
Simd odd_1;
|
||||
Simd odd_2;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
Simd Chi_0; // two spinor; 6 regs
|
||||
Simd Chi_1;
|
||||
Simd Chi_2;
|
||||
|
||||
Simd U_00; // two rows of U matrix
|
||||
Simd U_10;
|
||||
Simd U_20;
|
||||
Simd U_01;
|
||||
Simd U_11;
|
||||
Simd U_21; // 2 reg left.
|
||||
Simd U_02;
|
||||
Simd U_12;
|
||||
Simd U_22;
|
||||
|
||||
SiteSpinor result;
|
||||
typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
|
||||
calcSiteSpinor result;
|
||||
int offset, ptype, local, perm;
|
||||
|
||||
StencilEntry *SE;
|
||||
@ -261,8 +272,8 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
|
||||
// int sF=s+LLs*sU;
|
||||
{
|
||||
|
||||
even_0 = Zero(); even_1 = Zero(); even_2 = Zero();
|
||||
odd_0 = Zero(); odd_1 = Zero(); odd_2 = Zero();
|
||||
zeroit(even_0); zeroit(even_1); zeroit(even_2);
|
||||
zeroit(odd_0); zeroit(odd_1); zeroit(odd_2);
|
||||
|
||||
skew = 0;
|
||||
HAND_STENCIL_LEG_INT(U,Xp,3,skew,even);
|
||||
@ -294,7 +305,7 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
|
||||
result()()(1) = even_1 + odd_1;
|
||||
result()()(2) = even_2 + odd_2;
|
||||
}
|
||||
vstream(out[sF],result);
|
||||
coalescedWrite(out[sF],result);
|
||||
}
|
||||
}
|
||||
|
||||
@ -309,28 +320,13 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
|
||||
Simd even_0; // 12 regs on knc
|
||||
Simd even_1;
|
||||
Simd even_2;
|
||||
Simd odd_0; // 12 regs on knc
|
||||
Simd odd_1;
|
||||
Simd odd_2;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
Simd Chi_0; // two spinor; 6 regs
|
||||
Simd Chi_1;
|
||||
Simd Chi_2;
|
||||
|
||||
Simd U_00; // two rows of U matrix
|
||||
Simd U_10;
|
||||
Simd U_20;
|
||||
Simd U_01;
|
||||
Simd U_11;
|
||||
Simd U_21; // 2 reg left.
|
||||
Simd U_02;
|
||||
Simd U_12;
|
||||
Simd U_22;
|
||||
|
||||
SiteSpinor result;
|
||||
typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
|
||||
calcSiteSpinor result;
|
||||
int offset, ptype, local;
|
||||
|
||||
StencilEntry *SE;
|
||||
@ -340,8 +336,8 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
||||
// int sF=s+LLs*sU;
|
||||
{
|
||||
|
||||
even_0 = Zero(); even_1 = Zero(); even_2 = Zero();
|
||||
odd_0 = Zero(); odd_1 = Zero(); odd_2 = Zero();
|
||||
zeroit(even_0); zeroit(even_1); zeroit(even_2);
|
||||
zeroit(odd_0); zeroit(odd_1); zeroit(odd_2);
|
||||
int nmu=0;
|
||||
skew = 0;
|
||||
HAND_STENCIL_LEG_EXT(U,Xp,3,skew,even);
|
||||
@ -374,7 +370,7 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
||||
result()()(1) = even_1 + odd_1;
|
||||
result()()(2) = even_2 + odd_2;
|
||||
}
|
||||
out[sF] = out[sF] + result;
|
||||
coalescedWrite(out[sF] , out(sF)+ result);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -397,6 +393,7 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
|
||||
const FermionFieldView &in, FermionFieldView &out, int dag); \
|
||||
*/
|
||||
#undef LOAD_CHI
|
||||
#undef HAND_DECLARATIONS
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@ -35,39 +35,32 @@ NAMESPACE_BEGIN(Grid);
|
||||
#define GENERIC_STENCIL_LEG(U,Dir,skew,multLink) \
|
||||
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
||||
if (SE->_is_local ) { \
|
||||
if (SE->_permute) { \
|
||||
chi_p = χ \
|
||||
permute(chi, in[SE->_offset], ptype); \
|
||||
} else { \
|
||||
chi_p = &in[SE->_offset]; \
|
||||
} \
|
||||
int perm= SE->_permute; \
|
||||
chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\
|
||||
} else { \
|
||||
chi_p = &buf[SE->_offset]; \
|
||||
chi = coalescedRead(buf[SE->_offset],lane); \
|
||||
} \
|
||||
multLink(Uchi, U[sU], *chi_p, Dir);
|
||||
acceleratorSynchronise(); \
|
||||
multLink(Uchi, U[sU], chi, Dir);
|
||||
|
||||
#define GENERIC_STENCIL_LEG_INT(U,Dir,skew,multLink) \
|
||||
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
||||
if (SE->_is_local ) { \
|
||||
if (SE->_permute) { \
|
||||
chi_p = χ \
|
||||
permute(chi, in[SE->_offset], ptype); \
|
||||
} else { \
|
||||
chi_p = &in[SE->_offset]; \
|
||||
} \
|
||||
int perm= SE->_permute; \
|
||||
chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\
|
||||
} else if ( st.same_node[Dir] ) { \
|
||||
chi_p = &buf[SE->_offset]; \
|
||||
chi = coalescedRead(buf[SE->_offset],lane); \
|
||||
} \
|
||||
if (SE->_is_local || st.same_node[Dir] ) { \
|
||||
multLink(Uchi, U[sU], *chi_p, Dir); \
|
||||
multLink(Uchi, U[sU], chi, Dir); \
|
||||
}
|
||||
|
||||
#define GENERIC_STENCIL_LEG_EXT(U,Dir,skew,multLink) \
|
||||
SE = st.GetEntry(ptype, Dir+skew, sF); \
|
||||
if ((!SE->_is_local) && (!st.same_node[Dir]) ) { \
|
||||
nmu++; \
|
||||
chi_p = &buf[SE->_offset]; \
|
||||
multLink(Uchi, U[sU], *chi_p, Dir); \
|
||||
chi = coalescedRead(buf[SE->_offset],lane); \
|
||||
multLink(Uchi, U[sU], chi, Dir); \
|
||||
}
|
||||
|
||||
template <class Impl>
|
||||
@ -84,12 +77,14 @@ void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st,
|
||||
SiteSpinor *buf, int sF, int sU,
|
||||
const FermionFieldView &in, FermionFieldView &out, int dag)
|
||||
{
|
||||
const SiteSpinor *chi_p;
|
||||
SiteSpinor chi;
|
||||
SiteSpinor Uchi;
|
||||
typedef decltype(coalescedRead(in[0])) calcSpinor;
|
||||
calcSpinor chi;
|
||||
calcSpinor Uchi;
|
||||
StencilEntry *SE;
|
||||
int ptype;
|
||||
int skew;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
// for(int s=0;s<LLs;s++){
|
||||
//
|
||||
@ -118,7 +113,7 @@ void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st,
|
||||
if ( dag ) {
|
||||
Uchi = - Uchi;
|
||||
}
|
||||
vstream(out[sF], Uchi);
|
||||
coalescedWrite(out[sF], Uchi,lane);
|
||||
}
|
||||
};
|
||||
|
||||
@ -130,13 +125,16 @@ template <int Naik> accelerator_inline
|
||||
void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st,
|
||||
DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
|
||||
SiteSpinor *buf, int sF, int sU,
|
||||
const FermionFieldView &in, FermionFieldView &out,int dag) {
|
||||
const SiteSpinor *chi_p;
|
||||
SiteSpinor chi;
|
||||
SiteSpinor Uchi;
|
||||
const FermionFieldView &in, FermionFieldView &out,int dag)
|
||||
{
|
||||
typedef decltype(coalescedRead(in[0])) calcSpinor;
|
||||
calcSpinor chi;
|
||||
calcSpinor Uchi;
|
||||
StencilEntry *SE;
|
||||
int ptype;
|
||||
int skew ;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
// for(int s=0;s<LLs;s++){
|
||||
// int sF=LLs*sU+s;
|
||||
@ -165,7 +163,7 @@ void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st,
|
||||
if ( dag ) {
|
||||
Uchi = - Uchi;
|
||||
}
|
||||
vstream(out[sF], Uchi);
|
||||
coalescedWrite(out[sF], Uchi,lane);
|
||||
}
|
||||
};
|
||||
|
||||
@ -178,14 +176,17 @@ template <int Naik> accelerator_inline
|
||||
void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st,
|
||||
DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
|
||||
SiteSpinor *buf, int sF, int sU,
|
||||
const FermionFieldView &in, FermionFieldView &out,int dag) {
|
||||
const SiteSpinor *chi_p;
|
||||
// SiteSpinor chi;
|
||||
SiteSpinor Uchi;
|
||||
const FermionFieldView &in, FermionFieldView &out,int dag)
|
||||
{
|
||||
typedef decltype(coalescedRead(in[0])) calcSpinor;
|
||||
calcSpinor chi;
|
||||
calcSpinor Uchi;
|
||||
StencilEntry *SE;
|
||||
int ptype;
|
||||
int nmu=0;
|
||||
int skew ;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
// for(int s=0;s<LLs;s++){
|
||||
// int sF=LLs*sU+s;
|
||||
@ -211,11 +212,12 @@ void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st,
|
||||
GENERIC_STENCIL_LEG_EXT(UUU,Zm,skew,Impl::multLinkAdd);
|
||||
GENERIC_STENCIL_LEG_EXT(UUU,Tm,skew,Impl::multLinkAdd);
|
||||
}
|
||||
if ( nmu ) {
|
||||
if ( dag ) {
|
||||
out[sF] = out[sF] - Uchi;
|
||||
if ( nmu ) {
|
||||
auto _out = coalescedRead(out[sF],lane);
|
||||
if ( dag ) {
|
||||
coalescedWrite(out[sF], _out-Uchi,lane);
|
||||
} else {
|
||||
out[sF] = out[sF] + Uchi;
|
||||
coalescedWrite(out[sF], _out+Uchi,lane);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -261,6 +263,8 @@ void StaggeredKernels<Impl>::DhopImproved(StencilImpl &st, LebesgueOrder &lo,
|
||||
GridBase *FGrid=in.Grid();
|
||||
GridBase *UGrid=U.Grid();
|
||||
typedef StaggeredKernels<Impl> ThisKernel;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
autoView( UUU_v , UUU, AcceleratorRead);
|
||||
autoView( U_v , U, AcceleratorRead);
|
||||
autoView( in_v , in, AcceleratorRead);
|
||||
@ -301,6 +305,8 @@ void StaggeredKernels<Impl>::DhopNaive(StencilImpl &st, LebesgueOrder &lo,
|
||||
GridBase *FGrid=in.Grid();
|
||||
GridBase *UGrid=U.Grid();
|
||||
typedef StaggeredKernels<Impl> ThisKernel;
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
autoView( UUU_v , U, AcceleratorRead);
|
||||
autoView( U_v , U, AcceleratorRead);
|
||||
autoView( in_v , in, AcceleratorRead);
|
||||
|
||||
@ -76,7 +76,24 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
#define REGISTER
|
||||
|
||||
#define LOAD_CHIMU \
|
||||
#ifdef GRID_SIMT
|
||||
#define LOAD_CHIMU(ptype) \
|
||||
{const SiteSpinor & ref (in[offset]); \
|
||||
Chimu_00=coalescedReadPermute<ptype>(ref()(0)(0),perm,lane); \
|
||||
Chimu_01=coalescedReadPermute<ptype>(ref()(0)(1),perm,lane); \
|
||||
Chimu_02=coalescedReadPermute<ptype>(ref()(0)(2),perm,lane); \
|
||||
Chimu_10=coalescedReadPermute<ptype>(ref()(1)(0),perm,lane); \
|
||||
Chimu_11=coalescedReadPermute<ptype>(ref()(1)(1),perm,lane); \
|
||||
Chimu_12=coalescedReadPermute<ptype>(ref()(1)(2),perm,lane); \
|
||||
Chimu_20=coalescedReadPermute<ptype>(ref()(2)(0),perm,lane); \
|
||||
Chimu_21=coalescedReadPermute<ptype>(ref()(2)(1),perm,lane); \
|
||||
Chimu_22=coalescedReadPermute<ptype>(ref()(2)(2),perm,lane); \
|
||||
Chimu_30=coalescedReadPermute<ptype>(ref()(3)(0),perm,lane); \
|
||||
Chimu_31=coalescedReadPermute<ptype>(ref()(3)(1),perm,lane); \
|
||||
Chimu_32=coalescedReadPermute<ptype>(ref()(3)(2),perm,lane); }
|
||||
#define PERMUTE_DIR(dir) ;
|
||||
#else
|
||||
#define LOAD_CHIMU(ptype) \
|
||||
{const SiteSpinor & ref (in[offset]); \
|
||||
Chimu_00=ref()(0)(0);\
|
||||
Chimu_01=ref()(0)(1);\
|
||||
@ -91,55 +108,55 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Chimu_31=ref()(3)(1);\
|
||||
Chimu_32=ref()(3)(2);}
|
||||
|
||||
#define LOAD_CHI\
|
||||
{const SiteHalfSpinor &ref(buf[offset]); \
|
||||
Chi_00 = ref()(0)(0);\
|
||||
Chi_01 = ref()(0)(1);\
|
||||
Chi_02 = ref()(0)(2);\
|
||||
Chi_10 = ref()(1)(0);\
|
||||
Chi_11 = ref()(1)(1);\
|
||||
Chi_12 = ref()(1)(2);}
|
||||
|
||||
// To splat or not to splat depends on the implementation
|
||||
#define MULT_2SPIN(A)\
|
||||
{auto & ref(U[sU](A)); \
|
||||
Impl::loadLinkElement(U_00,ref()(0,0)); \
|
||||
Impl::loadLinkElement(U_10,ref()(1,0)); \
|
||||
Impl::loadLinkElement(U_20,ref()(2,0)); \
|
||||
Impl::loadLinkElement(U_01,ref()(0,1)); \
|
||||
Impl::loadLinkElement(U_11,ref()(1,1)); \
|
||||
Impl::loadLinkElement(U_21,ref()(2,1)); \
|
||||
UChi_00 = U_00*Chi_00;\
|
||||
UChi_10 = U_00*Chi_10;\
|
||||
UChi_01 = U_10*Chi_00;\
|
||||
UChi_11 = U_10*Chi_10;\
|
||||
UChi_02 = U_20*Chi_00;\
|
||||
UChi_12 = U_20*Chi_10;\
|
||||
UChi_00+= U_01*Chi_01;\
|
||||
UChi_10+= U_01*Chi_11;\
|
||||
UChi_01+= U_11*Chi_01;\
|
||||
UChi_11+= U_11*Chi_11;\
|
||||
UChi_02+= U_21*Chi_01;\
|
||||
UChi_12+= U_21*Chi_11;\
|
||||
Impl::loadLinkElement(U_00,ref()(0,2)); \
|
||||
Impl::loadLinkElement(U_10,ref()(1,2)); \
|
||||
Impl::loadLinkElement(U_20,ref()(2,2)); \
|
||||
UChi_00+= U_00*Chi_02;\
|
||||
UChi_10+= U_00*Chi_12;\
|
||||
UChi_01+= U_10*Chi_02;\
|
||||
UChi_11+= U_10*Chi_12;\
|
||||
UChi_02+= U_20*Chi_02;\
|
||||
UChi_12+= U_20*Chi_12;}
|
||||
|
||||
|
||||
#define PERMUTE_DIR(dir) \
|
||||
permute##dir(Chi_00,Chi_00);\
|
||||
permute##dir(Chi_00,Chi_00); \
|
||||
permute##dir(Chi_01,Chi_01);\
|
||||
permute##dir(Chi_02,Chi_02);\
|
||||
permute##dir(Chi_10,Chi_10);\
|
||||
permute##dir(Chi_10,Chi_10); \
|
||||
permute##dir(Chi_11,Chi_11);\
|
||||
permute##dir(Chi_12,Chi_12);
|
||||
|
||||
#endif
|
||||
|
||||
#define MULT_2SPIN(A)\
|
||||
{auto & ref(U[sU](A)); \
|
||||
U_00=coalescedRead(ref()(0,0),lane); \
|
||||
U_10=coalescedRead(ref()(1,0),lane); \
|
||||
U_20=coalescedRead(ref()(2,0),lane); \
|
||||
U_01=coalescedRead(ref()(0,1),lane); \
|
||||
U_11=coalescedRead(ref()(1,1),lane); \
|
||||
U_21=coalescedRead(ref()(2,1),lane); \
|
||||
UChi_00 = U_00*Chi_00; \
|
||||
UChi_10 = U_00*Chi_10; \
|
||||
UChi_01 = U_10*Chi_00; \
|
||||
UChi_11 = U_10*Chi_10; \
|
||||
UChi_02 = U_20*Chi_00; \
|
||||
UChi_12 = U_20*Chi_10; \
|
||||
UChi_00+= U_01*Chi_01; \
|
||||
UChi_10+= U_01*Chi_11; \
|
||||
UChi_01+= U_11*Chi_01; \
|
||||
UChi_11+= U_11*Chi_11; \
|
||||
UChi_02+= U_21*Chi_01; \
|
||||
UChi_12+= U_21*Chi_11; \
|
||||
U_00=coalescedRead(ref()(0,2),lane); \
|
||||
U_10=coalescedRead(ref()(1,2),lane); \
|
||||
U_20=coalescedRead(ref()(2,2),lane); \
|
||||
UChi_00+= U_00*Chi_02; \
|
||||
UChi_10+= U_00*Chi_12; \
|
||||
UChi_01+= U_10*Chi_02; \
|
||||
UChi_11+= U_10*Chi_12; \
|
||||
UChi_02+= U_20*Chi_02; \
|
||||
UChi_12+= U_20*Chi_12;}
|
||||
|
||||
#define LOAD_CHI \
|
||||
{const SiteHalfSpinor &ref(buf[offset]); \
|
||||
Chi_00 = coalescedRead(ref()(0)(0),lane); \
|
||||
Chi_01 = coalescedRead(ref()(0)(1),lane); \
|
||||
Chi_02 = coalescedRead(ref()(0)(2),lane); \
|
||||
Chi_10 = coalescedRead(ref()(1)(0),lane); \
|
||||
Chi_11 = coalescedRead(ref()(1)(1),lane); \
|
||||
Chi_12 = coalescedRead(ref()(1)(2),lane);}
|
||||
|
||||
// hspin(0)=fspin(0)+timesI(fspin(3));
|
||||
// hspin(1)=fspin(1)+timesI(fspin(2));
|
||||
#define XP_PROJ \
|
||||
@ -353,13 +370,13 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
result_31-= UChi_11; \
|
||||
result_32-= UChi_12;
|
||||
|
||||
#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \
|
||||
#define HAND_STENCIL_LEGB(PROJ,PERM,DIR,RECON) \
|
||||
SE=st.GetEntry(ptype,DIR,ss); \
|
||||
offset = SE->_offset; \
|
||||
local = SE->_is_local; \
|
||||
perm = SE->_permute; \
|
||||
if ( local ) { \
|
||||
LOAD_CHIMU; \
|
||||
LOAD_CHIMU(PERM); \
|
||||
PROJ; \
|
||||
if ( perm) { \
|
||||
PERMUTE_DIR(PERM); \
|
||||
@ -367,6 +384,37 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
} else { \
|
||||
LOAD_CHI; \
|
||||
} \
|
||||
acceleratorSynchronise(); \
|
||||
MULT_2SPIN(DIR); \
|
||||
RECON;
|
||||
|
||||
#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \
|
||||
SE=&st_p[DIR+8*ss]; \
|
||||
ptype=st_perm[DIR]; \
|
||||
offset = SE->_offset; \
|
||||
local = SE->_is_local; \
|
||||
perm = SE->_permute; \
|
||||
if ( local ) { \
|
||||
LOAD_CHIMU(PERM); \
|
||||
PROJ; \
|
||||
if ( perm) { \
|
||||
PERMUTE_DIR(PERM); \
|
||||
} \
|
||||
} else { \
|
||||
LOAD_CHI; \
|
||||
} \
|
||||
acceleratorSynchronise(); \
|
||||
MULT_2SPIN(DIR); \
|
||||
RECON;
|
||||
|
||||
#define HAND_STENCIL_LEGA(PROJ,PERM,DIR,RECON) \
|
||||
SE=&st_p[DIR+8*ss]; \
|
||||
ptype=st_perm[DIR]; \
|
||||
/*SE=st.GetEntry(ptype,DIR,ss);*/ \
|
||||
offset = SE->_offset; \
|
||||
perm = SE->_permute; \
|
||||
LOAD_CHIMU(PERM); \
|
||||
PROJ; \
|
||||
MULT_2SPIN(DIR); \
|
||||
RECON;
|
||||
|
||||
@ -376,7 +424,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
local = SE->_is_local; \
|
||||
perm = SE->_permute; \
|
||||
if ( local ) { \
|
||||
LOAD_CHIMU; \
|
||||
LOAD_CHIMU(PERM); \
|
||||
PROJ; \
|
||||
if ( perm) { \
|
||||
PERMUTE_DIR(PERM); \
|
||||
@ -384,10 +432,12 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
} else if ( st.same_node[DIR] ) { \
|
||||
LOAD_CHI; \
|
||||
} \
|
||||
acceleratorSynchronise(); \
|
||||
if (local || st.same_node[DIR] ) { \
|
||||
MULT_2SPIN(DIR); \
|
||||
RECON; \
|
||||
}
|
||||
} \
|
||||
acceleratorSynchronise();
|
||||
|
||||
#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON) \
|
||||
SE=st.GetEntry(ptype,DIR,ss); \
|
||||
@ -397,44 +447,44 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
MULT_2SPIN(DIR); \
|
||||
RECON; \
|
||||
nmu++; \
|
||||
}
|
||||
} \
|
||||
acceleratorSynchronise();
|
||||
|
||||
#define HAND_RESULT(ss) \
|
||||
{ \
|
||||
SiteSpinor & ref (out[ss]); \
|
||||
vstream(ref()(0)(0),result_00); \
|
||||
vstream(ref()(0)(1),result_01); \
|
||||
vstream(ref()(0)(2),result_02); \
|
||||
vstream(ref()(1)(0),result_10); \
|
||||
vstream(ref()(1)(1),result_11); \
|
||||
vstream(ref()(1)(2),result_12); \
|
||||
vstream(ref()(2)(0),result_20); \
|
||||
vstream(ref()(2)(1),result_21); \
|
||||
vstream(ref()(2)(2),result_22); \
|
||||
vstream(ref()(3)(0),result_30); \
|
||||
vstream(ref()(3)(1),result_31); \
|
||||
vstream(ref()(3)(2),result_32); \
|
||||
SiteSpinor & ref (out[ss]); \
|
||||
coalescedWrite(ref()(0)(0),result_00,lane); \
|
||||
coalescedWrite(ref()(0)(1),result_01,lane); \
|
||||
coalescedWrite(ref()(0)(2),result_02,lane); \
|
||||
coalescedWrite(ref()(1)(0),result_10,lane); \
|
||||
coalescedWrite(ref()(1)(1),result_11,lane); \
|
||||
coalescedWrite(ref()(1)(2),result_12,lane); \
|
||||
coalescedWrite(ref()(2)(0),result_20,lane); \
|
||||
coalescedWrite(ref()(2)(1),result_21,lane); \
|
||||
coalescedWrite(ref()(2)(2),result_22,lane); \
|
||||
coalescedWrite(ref()(3)(0),result_30,lane); \
|
||||
coalescedWrite(ref()(3)(1),result_31,lane); \
|
||||
coalescedWrite(ref()(3)(2),result_32,lane); \
|
||||
}
|
||||
|
||||
#define HAND_RESULT_EXT(ss) \
|
||||
if (nmu){ \
|
||||
SiteSpinor & ref (out[ss]); \
|
||||
ref()(0)(0)+=result_00; \
|
||||
ref()(0)(1)+=result_01; \
|
||||
ref()(0)(2)+=result_02; \
|
||||
ref()(1)(0)+=result_10; \
|
||||
ref()(1)(1)+=result_11; \
|
||||
ref()(1)(2)+=result_12; \
|
||||
ref()(2)(0)+=result_20; \
|
||||
ref()(2)(1)+=result_21; \
|
||||
ref()(2)(2)+=result_22; \
|
||||
ref()(3)(0)+=result_30; \
|
||||
ref()(3)(1)+=result_31; \
|
||||
ref()(3)(2)+=result_32; \
|
||||
#define HAND_RESULT_EXT(ss) \
|
||||
{ \
|
||||
SiteSpinor & ref (out[ss]); \
|
||||
coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00,lane); \
|
||||
coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01,lane); \
|
||||
coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02,lane); \
|
||||
coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10,lane); \
|
||||
coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11,lane); \
|
||||
coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12,lane); \
|
||||
coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20,lane); \
|
||||
coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21,lane); \
|
||||
coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22,lane); \
|
||||
coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30,lane); \
|
||||
coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31,lane); \
|
||||
coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32,lane); \
|
||||
}
|
||||
|
||||
|
||||
#define HAND_DECLARATIONS(a) \
|
||||
#define HAND_DECLARATIONS(Simd) \
|
||||
Simd result_00; \
|
||||
Simd result_01; \
|
||||
Simd result_02; \
|
||||
@ -466,19 +516,19 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Simd U_11; \
|
||||
Simd U_21;
|
||||
|
||||
#define ZERO_RESULT \
|
||||
result_00=Zero(); \
|
||||
result_01=Zero(); \
|
||||
result_02=Zero(); \
|
||||
result_10=Zero(); \
|
||||
result_11=Zero(); \
|
||||
result_12=Zero(); \
|
||||
result_20=Zero(); \
|
||||
result_21=Zero(); \
|
||||
result_22=Zero(); \
|
||||
result_30=Zero(); \
|
||||
result_31=Zero(); \
|
||||
result_32=Zero();
|
||||
#define ZERO_RESULT \
|
||||
zeroit(result_00); \
|
||||
zeroit(result_01); \
|
||||
zeroit(result_02); \
|
||||
zeroit(result_10); \
|
||||
zeroit(result_11); \
|
||||
zeroit(result_12); \
|
||||
zeroit(result_20); \
|
||||
zeroit(result_21); \
|
||||
zeroit(result_22); \
|
||||
zeroit(result_30); \
|
||||
zeroit(result_31); \
|
||||
zeroit(result_32);
|
||||
|
||||
#define Chimu_00 Chi_00
|
||||
#define Chimu_01 Chi_01
|
||||
@ -495,15 +545,53 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
#ifdef SYCL_HACK
|
||||
template<class Impl> accelerator_inline void
|
||||
WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||
WilsonKernels<Impl>::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const SiteSpinor *in, SiteSpinor *out)
|
||||
{
|
||||
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
typedef iSinglet<Simd> vCplx;
|
||||
// typedef decltype( coalescedRead( vCplx()()() )) Simt;
|
||||
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||
|
||||
HAND_DECLARATIONS(ignore);
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
int offset,local,perm, ptype;
|
||||
StencilEntry *SE;
|
||||
HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
|
||||
HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
|
||||
HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
|
||||
HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
|
||||
HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
|
||||
HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
|
||||
HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
|
||||
HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
|
||||
HAND_RESULT(ss);
|
||||
}
|
||||
#endif
|
||||
|
||||
template<class Impl> accelerator_inline void
|
||||
WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
auto st_p = st._entries_p;
|
||||
auto st_perm = st._permute_type;
|
||||
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
int offset,local,perm, ptype;
|
||||
StencilEntry *SE;
|
||||
@ -523,10 +611,16 @@ template<class Impl> accelerator_inline
|
||||
void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
auto st_p = st._entries_p;
|
||||
auto st_perm = st._permute_type;
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||
|
||||
HAND_DECLARATIONS(ignore);
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
StencilEntry *SE;
|
||||
int offset,local,perm, ptype;
|
||||
@ -546,11 +640,17 @@ template<class Impl> accelerator_inline void
|
||||
WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
auto st_p = st._entries_p;
|
||||
auto st_perm = st._permute_type;
|
||||
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||
|
||||
HAND_DECLARATIONS(ignore);
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
int offset,local,perm, ptype;
|
||||
StencilEntry *SE;
|
||||
@ -570,10 +670,16 @@ template<class Impl> accelerator_inline
|
||||
void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
auto st_p = st._entries_p;
|
||||
auto st_perm = st._permute_type;
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||
|
||||
HAND_DECLARATIONS(ignore);
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
StencilEntry *SE;
|
||||
int offset,local,perm, ptype;
|
||||
@ -593,11 +699,17 @@ template<class Impl> accelerator_inline void
|
||||
WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
auto st_p = st._entries_p;
|
||||
auto st_perm = st._permute_type;
|
||||
// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||
|
||||
HAND_DECLARATIONS(ignore);
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
int offset, ptype;
|
||||
StencilEntry *SE;
|
||||
@ -618,10 +730,16 @@ template<class Impl> accelerator_inline
|
||||
void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
|
||||
int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
|
||||
{
|
||||
auto st_p = st._entries_p;
|
||||
auto st_perm = st._permute_type;
|
||||
typedef typename Simd::scalar_type S;
|
||||
typedef typename Simd::vector_type V;
|
||||
typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
|
||||
|
||||
HAND_DECLARATIONS(ignore);
|
||||
const int Nsimd = SiteHalfSpinor::Nsimd();
|
||||
const int lane=acceleratorSIMTlane(Nsimd);
|
||||
|
||||
HAND_DECLARATIONS(Simt);
|
||||
|
||||
StencilEntry *SE;
|
||||
int offset, ptype;
|
||||
@ -682,3 +800,4 @@ NAMESPACE_END(Grid);
|
||||
#undef HAND_RESULT
|
||||
#undef HAND_RESULT_INT
|
||||
#undef HAND_RESULT_EXT
|
||||
#undef HAND_DECLARATIONS
|
||||
|
||||
@ -416,7 +416,21 @@ void WilsonKernels<Impl>::DhopDirKernel( StencilImpl &st, DoubledGaugeField &U,S
|
||||
#undef LoopBody
|
||||
}
|
||||
|
||||
#define KERNEL_CALLNB(A) \
|
||||
#define KERNEL_CALL_TMP(A) \
|
||||
const uint64_t NN = Nsite*Ls; \
|
||||
auto U_p = & U_v[0]; \
|
||||
auto in_p = & in_v[0]; \
|
||||
auto out_p = & out_v[0]; \
|
||||
auto st_p = st_v._entries_p; \
|
||||
auto st_perm = st_v._permute_type; \
|
||||
accelerator_forNB( ss, NN, Simd::Nsimd(), { \
|
||||
int sF = ss; \
|
||||
int sU = ss/Ls; \
|
||||
WilsonKernels<Impl>::A(st_perm,st_p,U_p,buf,sF,sU,in_p,out_p); \
|
||||
}); \
|
||||
accelerator_barrier();
|
||||
|
||||
#define KERNEL_CALLNB(A) \
|
||||
const uint64_t NN = Nsite*Ls; \
|
||||
accelerator_forNB( ss, NN, Simd::Nsimd(), { \
|
||||
int sF = ss; \
|
||||
@ -445,20 +459,24 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField
|
||||
|
||||
if( interior && exterior ) {
|
||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSite); return;}
|
||||
#ifndef GRID_CUDA
|
||||
#ifdef SYCL_HACK
|
||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL_TMP(HandDhopSiteSycl); return; }
|
||||
#else
|
||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSite); return;}
|
||||
#endif
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSite); return;}
|
||||
#endif
|
||||
} else if( interior ) {
|
||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALLNB(GenericDhopSiteInt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALLNB(HandDhopSiteInt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteInt); return;}
|
||||
#endif
|
||||
} else if( exterior ) {
|
||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteExt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteExt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteExt); return;}
|
||||
#endif
|
||||
}
|
||||
@ -476,20 +494,20 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField
|
||||
|
||||
if( interior && exterior ) {
|
||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDag); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDag); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDag); return;}
|
||||
#endif
|
||||
} else if( interior ) {
|
||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagInt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagInt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagInt); return;}
|
||||
#endif
|
||||
} else if( exterior ) {
|
||||
if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagExt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagExt); return;}
|
||||
#ifndef GRID_CUDA
|
||||
if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagExt); return;}
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -1 +0,0 @@
|
||||
../CayleyFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../DomainWallEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../MobiusEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../PartialFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonCloverFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonKernelsInstantiationGparity.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonTMFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
#define IMPLEMENTATION GparityWilsonImplDF
|
||||
@ -1 +0,0 @@
|
||||
../CayleyFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../DomainWallEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../MobiusEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../PartialFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonCloverFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonKernelsInstantiationGparity.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonTMFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
#define IMPLEMENTATION GparityWilsonImplFH
|
||||
@ -1 +0,0 @@
|
||||
../CayleyFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../DomainWallEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../MobiusEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../PartialFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonCloverFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermionInstantiation.cc.master
|
||||
@ -1,51 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
|
||||
|
||||
Copyright (C) 2015, 2020
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/qcd/action/fermion/FermionCore.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
|
||||
|
||||
#ifndef AVX512
|
||||
#ifndef QPX
|
||||
#ifndef A64FX
|
||||
#ifndef A64FXFIXEDSIZE
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#include "impl.h"
|
||||
template class WilsonKernels<IMPLEMENTATION>;
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@ -1 +0,0 @@
|
||||
../WilsonTMFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
#define IMPLEMENTATION WilsonImplDF
|
||||
@ -1 +0,0 @@
|
||||
../CayleyFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../DomainWallEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../MobiusEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../PartialFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonCloverFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermionInstantiation.cc.master
|
||||
@ -1,51 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
|
||||
|
||||
Copyright (C) 2015, 2020
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/qcd/action/fermion/FermionCore.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
|
||||
|
||||
#ifndef AVX512
|
||||
#ifndef QPX
|
||||
#ifndef A64FX
|
||||
#ifndef A64FXFIXEDSIZE
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#include "impl.h"
|
||||
template class WilsonKernels<IMPLEMENTATION>;
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@ -1 +0,0 @@
|
||||
../WilsonTMFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
#define IMPLEMENTATION WilsonImplFH
|
||||
@ -1 +0,0 @@
|
||||
../CayleyFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../DomainWallEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../MobiusEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../PartialFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermion5DInstantiation.cc.master
|
||||
@ -1,51 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
|
||||
|
||||
Copyright (C) 2015, 2020
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/qcd/action/fermion/FermionCore.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
|
||||
|
||||
#ifndef AVX512
|
||||
#ifndef QPX
|
||||
#ifndef A64FX
|
||||
#ifndef A64FXFIXEDSIZE
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#include "impl.h"
|
||||
template class WilsonKernels<IMPLEMENTATION>;
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@ -1 +0,0 @@
|
||||
#define IMPLEMENTATION ZWilsonImplDF
|
||||
@ -1 +0,0 @@
|
||||
../CayleyFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../ContinuedFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../DomainWallEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../MobiusEOFAFermionInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../PartialFractionFermion5DInstantiation.cc.master
|
||||
@ -1 +0,0 @@
|
||||
../WilsonFermion5DInstantiation.cc.master
|
||||
@ -1,51 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
|
||||
|
||||
Copyright (C) 2015, 2020
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution
|
||||
directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/qcd/action/fermion/FermionCore.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
|
||||
|
||||
#ifndef AVX512
|
||||
#ifndef QPX
|
||||
#ifndef A64FX
|
||||
#ifndef A64FXFIXEDSIZE
|
||||
#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#include "impl.h"
|
||||
template class WilsonKernels<IMPLEMENTATION>;
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@ -1 +0,0 @@
|
||||
#define IMPLEMENTATION ZWilsonImplFH
|
||||
@ -9,8 +9,6 @@ STAG5_IMPL_LIST=""
|
||||
WILSON_IMPL_LIST=" \
|
||||
WilsonImplF \
|
||||
WilsonImplD \
|
||||
WilsonImplFH \
|
||||
WilsonImplDF \
|
||||
WilsonAdjImplF \
|
||||
WilsonAdjImplD \
|
||||
WilsonTwoIndexSymmetricImplF \
|
||||
@ -18,26 +16,17 @@ WILSON_IMPL_LIST=" \
|
||||
WilsonTwoIndexAntiSymmetricImplF \
|
||||
WilsonTwoIndexAntiSymmetricImplD \
|
||||
GparityWilsonImplF \
|
||||
GparityWilsonImplD \
|
||||
GparityWilsonImplFH \
|
||||
GparityWilsonImplDF"
|
||||
GparityWilsonImplD "
|
||||
|
||||
DWF_IMPL_LIST=" \
|
||||
WilsonImplF \
|
||||
WilsonImplD \
|
||||
WilsonImplFH \
|
||||
WilsonImplDF \
|
||||
ZWilsonImplF \
|
||||
ZWilsonImplD \
|
||||
ZWilsonImplFH \
|
||||
ZWilsonImplDF "
|
||||
ZWilsonImplD "
|
||||
|
||||
GDWF_IMPL_LIST=" \
|
||||
GparityWilsonImplF \
|
||||
GparityWilsonImplD \
|
||||
GparityWilsonImplFH \
|
||||
GparityWilsonImplDF"
|
||||
|
||||
GparityWilsonImplD "
|
||||
|
||||
IMPL_LIST="$STAG_IMPL_LIST $WILSON_IMPL_LIST $DWF_IMPL_LIST $GDWF_IMPL_LIST"
|
||||
|
||||
|
||||
@ -96,7 +96,7 @@ public:
|
||||
///////////////////////////////////////////////////////////
|
||||
// Move these to another class
|
||||
// HMC auxiliary functions
|
||||
static inline void generate_momenta(Field &P, GridParallelRNG &pRNG)
|
||||
static inline void generate_momenta(Field &P, GridSerialRNG & sRNG, GridParallelRNG &pRNG)
|
||||
{
|
||||
// Zbigniew Srocinsky thesis:
|
||||
//
|
||||
|
||||
@ -49,7 +49,7 @@ public:
|
||||
|
||||
virtual std::string action_name(){return "PlaqPlusRectangleAction";}
|
||||
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {}; // noop as no pseudoferms
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {}; // noop as no pseudoferms
|
||||
|
||||
virtual std::string LogParameters(){
|
||||
std::stringstream sstream;
|
||||
|
||||
@ -54,8 +54,7 @@ public:
|
||||
return sstream.str();
|
||||
}
|
||||
|
||||
virtual void refresh(const GaugeField &U,
|
||||
GridParallelRNG &pRNG){}; // noop as no pseudoferms
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG){}; // noop as no pseudoferms
|
||||
|
||||
virtual RealD S(const GaugeField &U) {
|
||||
RealD plaq = WilsonLoops<Gimpl>::avgPlaquette(U);
|
||||
|
||||
@ -124,7 +124,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
//
|
||||
// As a check of rational require \Phi^dag M_{EOFA} \Phi == eta^dag M^-1/2^dag M M^-1/2 eta = eta^dag eta
|
||||
//
|
||||
virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG)
|
||||
virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG)
|
||||
{
|
||||
Lop.ImportGauge(U);
|
||||
Rop.ImportGauge(U);
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
@ -43,8 +42,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
//
|
||||
|
||||
template <class Impl>
|
||||
class OneFlavourEvenOddRationalPseudoFermionAction
|
||||
: public Action<typename Impl::GaugeField> {
|
||||
class OneFlavourEvenOddRationalPseudoFermionAction : public Action<typename Impl::GaugeField> {
|
||||
public:
|
||||
INHERIT_IMPL_TYPES(Impl);
|
||||
|
||||
@ -103,7 +101,7 @@ public:
|
||||
return sstream.str();
|
||||
}
|
||||
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {
|
||||
// P(phi) = e^{- phi^dag (MpcdagMpc)^-1/2 phi}
|
||||
// = e^{- phi^dag (MpcdagMpc)^-1/4 (MpcdagMpc)^-1/4 phi}
|
||||
// Phi = MpcdagMpc^{1/4} eta
|
||||
@ -156,7 +154,10 @@ public:
|
||||
|
||||
msCG(Mpc, PhiOdd, Y);
|
||||
|
||||
if ( (rand()%param.BoundsCheckFreq)==0 ) {
|
||||
auto grid = FermOp.FermionGrid();
|
||||
auto r=rand();
|
||||
grid->Broadcast(0,r);
|
||||
if ( (r%param.BoundsCheckFreq)==0 ) {
|
||||
FermionField gauss(FermOp.FermionRedBlackGrid());
|
||||
gauss = PhiOdd;
|
||||
HighBoundCheck(Mpc,gauss,param.hi);
|
||||
|
||||
@ -101,7 +101,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
}
|
||||
|
||||
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
|
||||
|
||||
// S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi
|
||||
//
|
||||
@ -170,7 +170,10 @@ NAMESPACE_BEGIN(Grid);
|
||||
msCG_M(MdagM,X,Y);
|
||||
|
||||
// Randomly apply rational bounds checks.
|
||||
if ( (rand()%param.BoundsCheckFreq)==0 ) {
|
||||
auto grid = NumOp.FermionGrid();
|
||||
auto r=rand();
|
||||
grid->Broadcast(0,r);
|
||||
if ( (r%param.BoundsCheckFreq)==0 ) {
|
||||
FermionField gauss(NumOp.FermionRedBlackGrid());
|
||||
gauss = PhiOdd;
|
||||
HighBoundCheck(MdagM,gauss,param.hi);
|
||||
|
||||
@ -98,7 +98,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
|
||||
|
||||
|
||||
// P(phi) = e^{- phi^dag (MdagM)^-1/2 phi}
|
||||
@ -142,7 +142,10 @@ NAMESPACE_BEGIN(Grid);
|
||||
|
||||
msCG(MdagMOp,Phi,Y);
|
||||
|
||||
if ( (rand()%param.BoundsCheckFreq)==0 ) {
|
||||
auto grid = FermOp.FermionGrid();
|
||||
auto r=rand();
|
||||
grid->Broadcast(0,r);
|
||||
if ( (r%param.BoundsCheckFreq)==0 ) {
|
||||
FermionField gauss(FermOp.FermionGrid());
|
||||
gauss = Phi;
|
||||
HighBoundCheck(MdagMOp,gauss,param.hi);
|
||||
|
||||
@ -95,7 +95,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
}
|
||||
|
||||
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
|
||||
|
||||
// S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi
|
||||
//
|
||||
@ -156,7 +156,10 @@ NAMESPACE_BEGIN(Grid);
|
||||
msCG_M(MdagM,X,Y);
|
||||
|
||||
// Randomly apply rational bounds checks.
|
||||
if ( (rand()%param.BoundsCheckFreq)==0 ) {
|
||||
auto grid = NumOp.FermionGrid();
|
||||
auto r=rand();
|
||||
grid->Broadcast(0,r);
|
||||
if ( (r%param.BoundsCheckFreq)==0 ) {
|
||||
FermionField gauss(NumOp.FermionGrid());
|
||||
gauss = Phi;
|
||||
HighBoundCheck(MdagM,gauss,param.hi);
|
||||
|
||||
@ -73,7 +73,7 @@ public:
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Push the gauge field in to the dops. Assume any BC's and smearing already applied
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {
|
||||
// P(phi) = e^{- phi^dag (MdagM)^-1 phi}
|
||||
// Phi = Mdag eta
|
||||
// P(eta) = e^{- eta^dag eta}
|
||||
|
||||
@ -77,7 +77,7 @@ public:
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Push the gauge field in to the dops. Assume any BC's and smearing already applied
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
|
||||
|
||||
// P(phi) = e^{- phi^dag (MpcdagMpc)^-1 phi}
|
||||
// Phi = McpDag eta
|
||||
|
||||
@ -84,7 +84,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
}
|
||||
|
||||
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
|
||||
|
||||
// P(phi) = e^{- phi^dag Vpc (MpcdagMpc)^-1 Vpcdag phi}
|
||||
//
|
||||
|
||||
@ -64,7 +64,7 @@ public:
|
||||
return sstream.str();
|
||||
}
|
||||
|
||||
virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
|
||||
virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
|
||||
|
||||
// P(phi) = e^{- phi^dag V (MdagM)^-1 Vdag phi}
|
||||
//
|
||||
|
||||
@ -55,7 +55,7 @@ public:
|
||||
}
|
||||
virtual std::string action_name() {return "ScalarAction";}
|
||||
|
||||
virtual void refresh(const Field &U, GridParallelRNG &pRNG) {} // noop as no pseudoferms
|
||||
virtual void refresh(const Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {} // noop as no pseudoferms
|
||||
|
||||
virtual RealD S(const Field &p) {
|
||||
return (mass_square * 0.5 + Nd) * ScalarObs<Impl>::sumphisquared(p) +
|
||||
|
||||
@ -27,7 +27,7 @@ public:
|
||||
typedef Field FermionField;
|
||||
typedef Field PropagatorField;
|
||||
|
||||
static inline void generate_momenta(Field& P, GridParallelRNG& pRNG){
|
||||
static inline void generate_momenta(Field& P, GridSerialRNG &sRNG, GridParallelRNG& pRNG){
|
||||
RealD scale = ::sqrt(HMC_MOMENTUM_DENOMINATOR); // CPS/UKQCD momentum rescaling
|
||||
gaussian(pRNG, P);
|
||||
P *= scale;
|
||||
@ -151,7 +151,7 @@ public:
|
||||
out = one / out;
|
||||
}
|
||||
|
||||
static inline void generate_momenta(Field &P, GridParallelRNG &pRNG)
|
||||
static inline void generate_momenta(Field &P, GridSerialRNG & sRNG, GridParallelRNG &pRNG)
|
||||
{
|
||||
RealD scale = ::sqrt(HMC_MOMENTUM_DENOMINATOR); // CPS/UKQCD momentum rescaling
|
||||
#ifndef USE_FFT_ACCELERATION
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user