Fix and compiles

.travis.yml

@@ -9,6 +9,11 @@ matrix:
     - os:        osx
       osx_image: xcode8.3
       compiler: clang
+      env: PREC=single
+    - os:        osx
+      osx_image: xcode8.3
+      compiler: clang
+      env: PREC=double
       
 before_install:
     - export GRIDDIR=`pwd`
@@ -50,7 +55,7 @@ script:
     - make -j4
     - make install
     - cd $CWD/build
-    - ../configure --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install ${EXTRACONF}
+    - ../configure --enable-precision=$PREC --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install ${EXTRACONF}
     - make -j4 
     - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - make check

Grid/GridStd.h

@@ -28,7 +28,4 @@
 ///////////////////
 #include "Config.h"
 
-#ifdef TOFU
-#undef GRID_COMMS_THREADS
-#endif
 #endif /* GRID_STD_H */

Grid/Grid_Eigen_Dense.h

@@ -34,12 +34,6 @@
 #define __SYCL__REDEFINE__
 #endif
 
-/* HIP save and restore compile environment*/
-#ifdef GRID_HIP
-#pragma push
-#pragma push_macro("__HIP_DEVICE_COMPILE__")
-#endif
-#define EIGEN_NO_HIP
 
 #include <Grid/Eigen/Dense>
 #include <Grid/Eigen/unsupported/CXX11/Tensor>
@@ -48,7 +42,7 @@
 #ifdef __NVCC__REDEFINE__
 #pragma pop_macro("__CUDACC__")
 #pragma pop_macro("__NVCC__")
-#pragma pop_macro("__CUDA_ARCH__")
+#pragma pop_macro("GRID_SIMT")
 #pragma pop
 #endif
 
@@ -58,12 +52,6 @@
 #pragma pop
 #endif
 
-/*HIP restore*/
-#ifdef __HIP__REDEFINE__
-#pragma pop_macro("__HIP_DEVICE_COMPILE__")
-#pragma pop
-#endif
-
 #if defined __GNUC__
 #pragma GCC diagnostic pop
 #endif

Grid/allocator/AlignedAllocator.h

@@ -65,7 +65,8 @@ public:
     MemoryManager::CpuFree((void *)__p,bytes);
   }
 
-  // FIXME: hack for the copy constructor: it must be avoided to avoid single thread loop
+  // FIXME: hack for the copy constructor, eventually it must be avoided
+  //void construct(pointer __p, const _Tp& __val) { new((void *)__p) _Tp(__val); };
   void construct(pointer __p, const _Tp& __val) { assert(0);};
   void construct(pointer __p) { };
   void destroy(pointer __p) { };
@@ -73,9 +74,6 @@ public:
 template<typename _Tp>  inline bool operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
 template<typename _Tp>  inline bool operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
 
-//////////////////////////////////////////////////////////////////////////////////////
-// Unified virtual memory
-//////////////////////////////////////////////////////////////////////////////////////
 template<typename _Tp>
 class uvmAllocator {
 public: 
@@ -111,71 +109,22 @@ public:
     MemoryManager::SharedFree((void *)__p,bytes);
   }
 
+  // FIXME: hack for the copy constructor, eventually it must be avoided
   void construct(pointer __p, const _Tp& __val) { new((void *)__p) _Tp(__val); };
+  //void construct(pointer __p, const _Tp& __val) { };
   void construct(pointer __p) { };
   void destroy(pointer __p) { };
 };
 template<typename _Tp>  inline bool operator==(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return true; }
 template<typename _Tp>  inline bool operator!=(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return false; }
 
-////////////////////////////////////////////////////////////////////////////////
-// Device memory
-////////////////////////////////////////////////////////////////////////////////
-template<typename _Tp>
-class devAllocator {
-public: 
-  typedef std::size_t     size_type;
-  typedef std::ptrdiff_t  difference_type;
-  typedef _Tp*       pointer;
-  typedef const _Tp* const_pointer;
-  typedef _Tp&       reference;
-  typedef const _Tp& const_reference;
-  typedef _Tp        value_type;
-
-  template<typename _Tp1>  struct rebind { typedef devAllocator<_Tp1> other; };
-  devAllocator() throw() { }
-  devAllocator(const devAllocator&) throw() { }
-  template<typename _Tp1> devAllocator(const devAllocator<_Tp1>&) throw() { }
-  ~devAllocator() throw() { }
-  pointer       address(reference __x)       const { return &__x; }
-  size_type  max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
-
-  pointer allocate(size_type __n, const void* _p= 0)
-  { 
-    size_type bytes = __n*sizeof(_Tp);
-    profilerAllocate(bytes);
-    _Tp *ptr = (_Tp*) MemoryManager::AcceleratorAllocate(bytes);
-    assert( ( (_Tp*)ptr != (_Tp *)NULL ) );
-    return ptr;
-  }
-
-  void deallocate(pointer __p, size_type __n) 
-  { 
-    size_type bytes = __n * sizeof(_Tp);
-    profilerFree(bytes);
-    MemoryManager::AcceleratorFree((void *)__p,bytes);
-  }
-  void construct(pointer __p, const _Tp& __val) { };
-  void construct(pointer __p) { };
-  void destroy(pointer __p) { };
-};
-template<typename _Tp>  inline bool operator==(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return true; }
-template<typename _Tp>  inline bool operator!=(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return false; }
-
 ////////////////////////////////////////////////////////////////////////////////
 // Template typedefs
 ////////////////////////////////////////////////////////////////////////////////
-#ifdef ACCELERATOR_CSHIFT
-// Cshift on device
-template<class T> using cshiftAllocator = devAllocator<T>;
-#else
-// Cshift on host
-template<class T> using cshiftAllocator = std::allocator<T>;
-#endif
-
+template<class T> using commAllocator = uvmAllocator<T>;
 template<class T> using Vector     = std::vector<T,uvmAllocator<T> >;           
-template<class T> using commVector = std::vector<T,devAllocator<T> >;
-template<class T> using cshiftVector = std::vector<T,cshiftAllocator<T> >;
+template<class T> using commVector = std::vector<T,uvmAllocator<T> >;
+//template<class T> using Matrix     = std::vector<std::vector<T,alignedAllocator<T> > >;
 
 NAMESPACE_END(Grid);
 

Grid/allocator/MemoryManager.cc

@@ -136,15 +136,6 @@ void MemoryManager::Init(void)
       Ncache[SharedSmall]=Nc;
     }
   }
-
-}
-
-void MemoryManager::InitMessage(void) {
-
-#ifndef GRID_UVM
-  std::cout << GridLogMessage << "MemoryManager Cache "<< MemoryManager::DeviceMaxBytes <<" bytes "<<std::endl;
-#endif
-  
   std::cout << GridLogMessage<< "MemoryManager::Init() setting up"<<std::endl;
 #ifdef ALLOCATION_CACHE
   std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent allocations: SMALL "<<Ncache[CpuSmall]<<" LARGE "<<Ncache[Cpu]<<std::endl;
@@ -173,7 +164,6 @@ void MemoryManager::InitMessage(void) {
   std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_device"<<std::endl;
 #endif
 #endif
-
 }
 
 void *MemoryManager::Insert(void *ptr,size_t bytes,int type) 

Grid/allocator/MemoryManager.h

@@ -93,12 +93,11 @@ private:
   static void *Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim) ;
   static void *Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache) ;
 
+  static void *AcceleratorAllocate(size_t bytes);
+  static void  AcceleratorFree    (void *ptr,size_t bytes);
   static void PrintBytes(void);
  public:
   static void Init(void);
-  static void InitMessage(void);
-  static void *AcceleratorAllocate(size_t bytes);
-  static void  AcceleratorFree    (void *ptr,size_t bytes);
   static void *SharedAllocate(size_t bytes);
   static void  SharedFree    (void *ptr,size_t bytes);
   static void *CpuAllocate(size_t bytes);

Grid/communicator/Communicator_base.h

@@ -138,6 +138,21 @@ public:
 		      int recv_from_rank,
 		      int bytes);
   
+  void SendRecvPacket(void *xmit,
+		      void *recv,
+		      int xmit_to_rank,
+		      int recv_from_rank,
+		      int bytes);
+  
+  void SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+			   void *xmit,
+			   int xmit_to_rank,
+			   void *recv,
+			   int recv_from_rank,
+			   int bytes);
+  
+  void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
+
   double StencilSendToRecvFrom(void *xmit,
 			       int xmit_to_rank,
 			       void *recv,

Grid/communicator/Communicator_mpi3.cc

@@ -43,16 +43,8 @@ void CartesianCommunicator::Init(int *argc, char ***argv)
 
   MPI_Initialized(&flag); // needed to coexist with other libs apparently
   if ( !flag ) {
-
-#ifndef GRID_COMMS_THREADS
-    nCommThreads=1;
-    // wrong results here too
-    // For now: comms-overlap leads to wrong results in Benchmark_wilson even on single node MPI runs
-    // other comms schemes are ok
-    MPI_Init_thread(argc,argv,MPI_THREAD_SERIALIZED,&provided);
-#else
     MPI_Init_thread(argc,argv,MPI_THREAD_MULTIPLE,&provided);
-#endif
+
     //If only 1 comms thread we require any threading mode other than SINGLE, but for multiple comms threads we need MULTIPLE
     if( (nCommThreads == 1) && (provided == MPI_THREAD_SINGLE) ) {
       assert(0);
@@ -302,28 +294,60 @@ void CartesianCommunicator::SendToRecvFrom(void *xmit,
 					   int bytes)
 {
   std::vector<CommsRequest_t> reqs(0);
-  unsigned long  xcrc = crc32(0L, Z_NULL, 0);
-  unsigned long  rcrc = crc32(0L, Z_NULL, 0);
-
+  //    unsigned long  xcrc = crc32(0L, Z_NULL, 0);
+  //    unsigned long  rcrc = crc32(0L, Z_NULL, 0);
+  //    xcrc = crc32(xcrc,(unsigned char *)xmit,bytes);
+  SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes);
+  SendToRecvFromComplete(reqs);
+  //    rcrc = crc32(rcrc,(unsigned char *)recv,bytes);
+  //    printf("proc %d SendToRecvFrom %d bytes %lx %lx\n",_processor,bytes,xcrc,rcrc);
+}
+void CartesianCommunicator::SendRecvPacket(void *xmit,
+					   void *recv,
+					   int sender,
+					   int receiver,
+					   int bytes)
+{
+  MPI_Status stat;
+  assert(sender != receiver);
+  int tag = sender;
+  if ( _processor == sender ) {
+    MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator);
+  }
+  if ( _processor == receiver ) { 
+    MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat);
+  }
+}
+// Basic Halo comms primitive
+void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						void *xmit,
+						int dest,
+						void *recv,
+						int from,
+						int bytes)
+{
   int myrank = _processor;
   int ierr;
 
-  // Enforce no UVM in comms, device or host OK
-  assert(acceleratorIsCommunicable(xmit));
-  assert(acceleratorIsCommunicable(recv));
+  if ( CommunicatorPolicy == CommunicatorPolicyConcurrent ) { 
+    MPI_Request xrq;
+    MPI_Request rrq;
 
+    ierr =MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+    ierr|=MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+    
+    assert(ierr==0);
+    list.push_back(xrq);
+    list.push_back(rrq);
+  } else { 
     // Give the CPU to MPI immediately; can use threads to overlap optionally
-  //  printf("proc %d SendToRecvFrom %d bytes Sendrecv \n",_processor,bytes);
     ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,dest,myrank,
 		      recv,bytes,MPI_CHAR,from, from,
 		      communicator,MPI_STATUS_IGNORE);
     assert(ierr==0);
-
-  //  xcrc = crc32(xcrc,(unsigned char *)xmit,bytes);
-  //  rcrc = crc32(rcrc,(unsigned char *)recv,bytes);
-  //  printf("proc %d SendToRecvFrom %d bytes xcrc %lx rcrc %lx\n",_processor,bytes,xcrc,rcrc); fflush
+  }
 }
-// Basic Halo comms primitive
+
 double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
 						     int dest,
 						     void *recv,
@@ -358,19 +382,16 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
   assert(from != _processor);
   assert(gme  == ShmRank);
   double off_node_bytes=0.0;
-  int tag;
 
   if ( gfrom ==MPI_UNDEFINED) {
-    tag= dir+from*32;
-    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,tag,communicator_halo[commdir],&rrq);
+    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator_halo[commdir],&rrq);
     assert(ierr==0);
     list.push_back(rrq);
     off_node_bytes+=bytes;
   }
 
   if ( gdest == MPI_UNDEFINED ) {
-    tag= dir+_processor*32;
-    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,tag,communicator_halo[commdir],&xrq);
+    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator_halo[commdir],&xrq);
     assert(ierr==0);
     list.push_back(xrq);
     off_node_bytes+=bytes;
@@ -382,7 +403,15 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 
   return off_node_bytes;
 }
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
+{
+  SendToRecvFromComplete(waitall);
+}
+void CartesianCommunicator::StencilBarrier(void)
+{
+  MPI_Barrier  (ShmComm);
+}
+void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
 {
   int nreq=list.size();
 
@@ -393,13 +422,6 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsReque
   assert(ierr==0);
   list.resize(0);
 }
-void CartesianCommunicator::StencilBarrier(void)
-{
-  MPI_Barrier  (ShmComm);
-}
-//void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
-//{
-//}
 void CartesianCommunicator::Barrier(void)
 {
   int ierr = MPI_Barrier(communicator);
@@ -461,3 +483,5 @@ void CartesianCommunicator::AllToAll(void  *in,void *out,uint64_t words,uint64_t
 }
 
 NAMESPACE_END(Grid);
+
+

Grid/communicator/Communicator_none.cc

@@ -77,6 +77,15 @@ void CartesianCommunicator::GlobalSumVector(uint64_t *,int N){}
 void CartesianCommunicator::GlobalXOR(uint32_t &){}
 void CartesianCommunicator::GlobalXOR(uint64_t &){}
 
+void CartesianCommunicator::SendRecvPacket(void *xmit,
+					   void *recv,
+					   int xmit_to_rank,
+					   int recv_from_rank,
+					   int bytes)
+{
+  assert(0);
+}
+
 
 // Basic Halo comms primitive -- should never call in single node
 void CartesianCommunicator::SendToRecvFrom(void *xmit,
@@ -87,6 +96,20 @@ void CartesianCommunicator::SendToRecvFrom(void *xmit,
 {
   assert(0);
 }
+void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						void *xmit,
+						int dest,
+						void *recv,
+						int from,
+						int bytes)
+{
+  assert(0);
+}
+
+void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
+{
+  assert(0);
+}
 void CartesianCommunicator::AllToAll(int dim,void  *in,void *out,uint64_t words,uint64_t bytes)
 {
   bcopy(in,out,bytes*words);
@@ -114,6 +137,10 @@ double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
 						     int recv_from_rank,
 						     int bytes, int dir)
 {
+  std::vector<CommsRequest_t> list;
+  // Discard the "dir"
+  SendToRecvFromBegin   (list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
+  SendToRecvFromComplete(list);
   return 2.0*bytes;
 }
 double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
@@ -123,10 +150,13 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 							 int recv_from_rank,
 							 int bytes, int dir)
 {
+  // Discard the "dir"
+  SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
   return 2.0*bytes;
 }
 void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
 {
+  SendToRecvFromComplete(waitall);
 }
 
 void CartesianCommunicator::StencilBarrier(void){};

Grid/communicator/SharedMemoryMPI.cc

@@ -32,9 +32,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_CUDA
 #include <cuda_runtime_api.h>
 #endif
-#ifdef GRID_HIP
-#include <hip/hip_runtime_api.h>
-#endif
 
 NAMESPACE_BEGIN(Grid); 
 #define header "SharedMemoryMpi: "
@@ -50,12 +47,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
   /////////////////////////////////////////////////////////////////////
   // Split into groups that can share memory
   /////////////////////////////////////////////////////////////////////
-#ifndef GRID_MPI3_SHM_NONE
   MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&WorldShmComm);
-#else
-  MPI_Comm_split(comm, WorldRank, 0, &WorldShmComm);
-#endif
-
   MPI_Comm_rank(WorldShmComm     ,&WorldShmRank);
   MPI_Comm_size(WorldShmComm     ,&WorldShmSize);
 
@@ -428,7 +420,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
 ////////////////////////////////////////////////////////////////////////////////////////////
 // Hugetlbfs mapping intended
 ////////////////////////////////////////////////////////////////////////////////////////////
-#if defined(GRID_CUDA) ||defined(GRID_HIP)
+#ifdef GRID_CUDA
 void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
 {
   void * ShmCommBuf ; 
@@ -451,16 +443,17 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
   // Each MPI rank should allocate our own buffer
   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-  ShmCommBuf = acceleratorAllocDevice(bytes);
-
-  if (ShmCommBuf == (void *)NULL ) {
-    std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
+  auto err =  cudaMalloc(&ShmCommBuf, bytes);
+  if ( err !=  cudaSuccess) {
+    std::cerr << " SharedMemoryMPI.cc cudaMallocManaged failed for " << bytes<<" bytes " <<cudaGetErrorString(err)<< std::endl;
     exit(EXIT_FAILURE);  
   }
-  //  if ( WorldRank == 0 ){
-  if ( 1 ){
-    std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes 
-	      << "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
+  if (ShmCommBuf == (void *)NULL ) {
+    std::cerr << " SharedMemoryMPI.cc cudaMallocManaged failed NULL pointer for " << bytes<<" bytes " << std::endl;
+    exit(EXIT_FAILURE);  
+  }
+  if ( WorldRank == 0 ){
+    std::cout << header " SharedMemoryMPI.cc cudaMalloc "<< bytes << "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
   }
   SharedMemoryZero(ShmCommBuf,bytes);
 
@@ -469,30 +462,18 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
   for(int r=0;r<WorldShmSize;r++){
     
-#ifndef GRID_MPI3_SHM_NONE
     //////////////////////////////////////////////////
     // If it is me, pass around the IPC access key
     //////////////////////////////////////////////////
-#ifdef GRID_CUDA
     cudaIpcMemHandle_t handle;
+    
     if ( r==WorldShmRank ) { 
-      auto err = cudaIpcGetMemHandle(&handle,ShmCommBuf);
+      err = cudaIpcGetMemHandle(&handle,ShmCommBuf);
       if ( err !=  cudaSuccess) {
 	std::cerr << " SharedMemoryMPI.cc cudaIpcGetMemHandle failed for rank" << r <<" "<<cudaGetErrorString(err)<< std::endl;
 	exit(EXIT_FAILURE);
       }
     }
-#endif
-#ifdef GRID_HIP
-    hipIpcMemHandle_t handle;    
-    if ( r==WorldShmRank ) { 
-      auto err = hipIpcGetMemHandle(&handle,ShmCommBuf);
-      if ( err !=  hipSuccess) {
-	std::cerr << " SharedMemoryMPI.cc hipIpcGetMemHandle failed for rank" << r <<" "<<hipGetErrorString(err)<< std::endl;
-	exit(EXIT_FAILURE);
-      }
-    }
-#endif
     //////////////////////////////////////////////////
     // Share this IPC handle across the Shm Comm
     //////////////////////////////////////////////////
@@ -509,31 +490,17 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
     // If I am not the source, overwrite thisBuf with remote buffer
     ///////////////////////////////////////////////////////////////
     void * thisBuf = ShmCommBuf;
-#ifdef GRID_CUDA
     if ( r!=WorldShmRank ) { 
-      auto err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
+      err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
       if ( err !=  cudaSuccess) {
 	std::cerr << " SharedMemoryMPI.cc cudaIpcOpenMemHandle failed for rank" << r <<" "<<cudaGetErrorString(err)<< std::endl;
 	exit(EXIT_FAILURE);
       }
     }
-#endif
-#ifdef GRID_HIP
-    if ( r!=WorldShmRank ) { 
-      auto err = hipIpcOpenMemHandle(&thisBuf,handle,hipIpcMemLazyEnablePeerAccess);
-      if ( err !=  hipSuccess) {
-	std::cerr << " SharedMemoryMPI.cc hipIpcOpenMemHandle failed for rank" << r <<" "<<hipGetErrorString(err)<< std::endl;
-	exit(EXIT_FAILURE);
-      }
-    }
-#endif
     ///////////////////////////////////////////////////////////////
     // Save a copy of the device buffers
     ///////////////////////////////////////////////////////////////
     WorldShmCommBufs[r] = thisBuf;
-#else
-    WorldShmCommBufs[r] = ShmCommBuf;
-#endif
   }
 
   _ShmAllocBytes=bytes;
@@ -738,11 +705,7 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
   /////////////////////////////////////////////////////////////////////
   // Split into groups that can share memory
   /////////////////////////////////////////////////////////////////////
-#ifndef GRID_MPI3_SHM_NONE
   MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm);
-#else
-  MPI_Comm_split(comm, rank, 0, &ShmComm);
-#endif
   MPI_Comm_rank(ShmComm     ,&ShmRank);
   MPI_Comm_size(ShmComm     ,&ShmSize);
   ShmCommBufs.resize(ShmSize);
@@ -772,13 +735,22 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
   std::vector<int> ranks(size);   for(int r=0;r<size;r++) ranks[r]=r;
   MPI_Group_translate_ranks (FullGroup,size,&ranks[0],ShmGroup, &ShmRanks[0]); 
 
-#ifdef GRID_SHM_DISABLE
-  // Hide the shared memory path between ranks
-  {
+#ifdef GRID_IBM_SUMMIT
+  // Hide the shared memory path between sockets 
+  // if even number of nodes
+  if ( (ShmSize & 0x1)==0 ) {
+    int SocketSize = ShmSize/2;
+    int mySocket = ShmRank/SocketSize; 
     for(int r=0;r<size;r++){
+      int hisRank=ShmRanks[r];
+      if ( hisRank!= MPI_UNDEFINED ) {
+	int hisSocket=hisRank/SocketSize;
+	if ( hisSocket != mySocket ) {
 	  ShmRanks[r] = MPI_UNDEFINED;
 	}
       }
+    }
+  }
 #endif
 
   SharedMemoryTest();

Grid/cshift/Cshift.h

@@ -52,8 +52,23 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 NAMESPACE_BEGIN(Grid);
 
-template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> 
-auto Cshift(const Expression &expr,int dim,int shift)  -> decltype(closure(expr)) 
+template<typename Op, typename T1> 
+auto Cshift(const LatticeUnaryExpression<Op,T1> &expr,int dim,int shift)
+    -> Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> 
+{
+  return Cshift(closure(expr),dim,shift);
+}
+template <class Op, class T1, class T2>
+auto Cshift(const LatticeBinaryExpression<Op,T1,T2> &expr,int dim,int shift)
+  -> Lattice<decltype(expr.op.func(eval(0, expr.arg1),eval(0, expr.arg2)))> 
+{
+  return Cshift(closure(expr),dim,shift);
+}
+template <class Op, class T1, class T2, class T3>
+auto Cshift(const LatticeTrinaryExpression<Op,T1,T2,T3> &expr,int dim,int shift)
+  -> Lattice<decltype(expr.op.func(eval(0, expr.arg1),
+				   eval(0, expr.arg2),
+				   eval(0, expr.arg3)))> 
 {
   return Cshift(closure(expr),dim,shift);
 }

Grid/cshift/Cshift_common.h

@@ -35,7 +35,7 @@ extern Vector<std::pair<int,int> > Cshift_table;
 // Gather for when there is no need to SIMD split 
 ///////////////////////////////////////////////////////////////////
 template<class vobj> void 
-Gather_plane_simple (const Lattice<vobj> &rhs,cshiftVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
+Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
 {
   int rd = rhs.Grid()->_rdimensions[dimension];
 
@@ -73,19 +73,12 @@ Gather_plane_simple (const Lattice<vobj> &rhs,cshiftVector<vobj> &buffer,int dim
      }
   }
   {
+    autoView(rhs_v , rhs, AcceleratorRead);
     auto buffer_p = & buffer[0];
     auto table = &Cshift_table[0];
-#ifdef ACCELERATOR_CSHIFT    
-    autoView(rhs_v , rhs, AcceleratorRead);
-    accelerator_for(i,ent,vobj::Nsimd(),{
-	coalescedWrite(buffer_p[table[i].first],coalescedRead(rhs_v[table[i].second]));
-    });
-#else
-    autoView(rhs_v , rhs, CpuRead);
-    thread_for(i,ent,{
+    accelerator_for(i,ent,1,{
       buffer_p[table[i].first]=rhs_v[table[i].second];
     });
-#endif
   }
 }
 
@@ -110,7 +103,6 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
   int n1=rhs.Grid()->_slice_stride[dimension];
 
   if ( cbmask ==0x3){
-#ifdef ACCELERATOR_CSHIFT    
     autoView(rhs_v , rhs, AcceleratorRead);
     accelerator_for2d(n,e1,b,e2,1,{
 	int o      =   n*n1;
@@ -119,22 +111,12 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
 	vobj temp =rhs_v[so+o+b];
 	extract<vobj>(temp,pointers,offset);
       });
-#else
-    autoView(rhs_v , rhs, CpuRead);
-    thread_for2d(n,e1,b,e2,{
-	int o      =   n*n1;
-	int offset = b+n*e2;
-	
-	vobj temp =rhs_v[so+o+b];
-	extract<vobj>(temp,pointers,offset);
-      });
-#endif
   } else { 
+    autoView(rhs_v , rhs, AcceleratorRead);
+
     Coordinate rdim=rhs.Grid()->_rdimensions;
     Coordinate cdm =rhs.Grid()->_checker_dim_mask;
     std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
-#ifdef ACCELERATOR_CSHIFT    
-    autoView(rhs_v , rhs, AcceleratorRead);
     accelerator_for2d(n,e1,b,e2,1,{
 
 	Coordinate coor;
@@ -152,33 +134,13 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
 	  extract<vobj>(temp,pointers,offset);
 	}
       });
-#else
-    autoView(rhs_v , rhs, CpuRead);
-    thread_for2d(n,e1,b,e2,{
-
-	Coordinate coor;
-
-	int o=n*n1;
-	int oindex = o+b;
-
-       	int cb = RedBlackCheckerBoardFromOindex(oindex, rdim, cdm);
-
-	int ocb=1<<cb;
-	int offset = b+n*e2;
-
-	if ( ocb & cbmask ) {
-	  vobj temp =rhs_v[so+o+b];
-	  extract<vobj>(temp,pointers,offset);
-	}
-      });
-#endif
   }
 }
 
 //////////////////////////////////////////////////////
 // Scatter for when there is no need to SIMD split
 //////////////////////////////////////////////////////
-template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,cshiftVector<vobj> &buffer, int dimension,int plane,int cbmask)
+template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
 {
   int rd = rhs.Grid()->_rdimensions[dimension];
 
@@ -220,19 +182,12 @@ template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,cshiftVector<
   }
   
   {
+    autoView( rhs_v, rhs, AcceleratorWrite);
     auto buffer_p = & buffer[0];
     auto table = &Cshift_table[0];
-#ifdef ACCELERATOR_CSHIFT    
-    autoView( rhs_v, rhs, AcceleratorWrite);
-    accelerator_for(i,ent,vobj::Nsimd(),{
-	coalescedWrite(rhs_v[table[i].first],coalescedRead(buffer_p[table[i].second]));
-    });
-#else
-    autoView( rhs_v, rhs, CpuWrite);
-    thread_for(i,ent,{
+    accelerator_for(i,ent,1,{
 	rhs_v[table[i].first]=buffer_p[table[i].second];
     });
-#endif
   }
 }
 
@@ -253,30 +208,18 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
   int e2=rhs.Grid()->_slice_block[dimension];
 
   if(cbmask ==0x3 ) {
-    int _slice_stride = rhs.Grid()->_slice_stride[dimension];
-    int _slice_block = rhs.Grid()->_slice_block[dimension];
-#ifdef ACCELERATOR_CSHIFT    
     autoView( rhs_v , rhs, AcceleratorWrite);
     accelerator_for2d(n,e1,b,e2,1,{
-	int o      = n*_slice_stride;
-	int offset = b+n*_slice_block;
+	int o      = n*rhs.Grid()->_slice_stride[dimension];
+	int offset = b+n*rhs.Grid()->_slice_block[dimension];
 	merge(rhs_v[so+o+b],pointers,offset);
       });
-#else
-    autoView( rhs_v , rhs, CpuWrite);
-    thread_for2d(n,e1,b,e2,{
-	int o      = n*_slice_stride;
-	int offset = b+n*_slice_block;
-	merge(rhs_v[so+o+b],pointers,offset);
-    });
-#endif
   } else { 
 
     // Case of SIMD split AND checker dim cannot currently be hit, except in 
     // Test_cshift_red_black code.
     //    std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
     std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
-    assert(0); // This will fail if hit on GPU
     autoView( rhs_v, rhs, CpuWrite);
     for(int n=0;n<e1;n++){
       for(int b=0;b<e2;b++){
@@ -334,20 +277,12 @@ template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs
   }
 
   {
-    auto table = &Cshift_table[0];
-#ifdef ACCELERATOR_CSHIFT    
     autoView(rhs_v , rhs, AcceleratorRead);
     autoView(lhs_v , lhs, AcceleratorWrite);
-    accelerator_for(i,ent,vobj::Nsimd(),{
-      coalescedWrite(lhs_v[table[i].first],coalescedRead(rhs_v[table[i].second]));
-    });
-#else
-    autoView(rhs_v , rhs, CpuRead);
-    autoView(lhs_v , lhs, CpuWrite);
-    thread_for(i,ent,{
+    auto table = &Cshift_table[0];
+    accelerator_for(i,ent,1,{
       lhs_v[table[i].first]=rhs_v[table[i].second];
     });
-#endif
   }
 }
 
@@ -386,20 +321,12 @@ template<class vobj> void Copy_plane_permute(Lattice<vobj>& lhs,const Lattice<vo
   }
 
   {
-    auto table = &Cshift_table[0];
-#ifdef ACCELERATOR_CSHIFT    
     autoView( rhs_v, rhs, AcceleratorRead);
     autoView( lhs_v, lhs, AcceleratorWrite);
+    auto table = &Cshift_table[0];
     accelerator_for(i,ent,1,{
       permute(lhs_v[table[i].first],rhs_v[table[i].second],permute_type);
     });
-#else
-    autoView( rhs_v, rhs, CpuRead);
-    autoView( lhs_v, lhs, CpuWrite);
-    thread_for(i,ent,{
-      permute(lhs_v[table[i].first],rhs_v[table[i].second],permute_type);
-    });
-#endif
   }
 }
 

Grid/cshift/Cshift_mpi.h

@@ -101,8 +101,7 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj>& ret,const Lattice<vob
     Cshift_comms_simd(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
   }
 }
-#define ACCELERATOR_CSHIFT_NO_COPY
-#ifdef ACCELERATOR_CSHIFT_NO_COPY
+
 template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
 {
   typedef typename vobj::vector_type vector_type;
@@ -122,8 +121,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
   assert(shift<fd);
   
   int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
-  cshiftVector<vobj> send_buf(buffer_size);
-  cshiftVector<vobj> recv_buf(buffer_size);
+  commVector<vobj> send_buf(buffer_size);
+  commVector<vobj> recv_buf(buffer_size);
 
   int cb= (cbmask==0x2)? Odd : Even;
   int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
@@ -139,7 +138,7 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
 
     } else {
 
-      int words = buffer_size;
+      int words = send_buf.size();
       if (cbmask != 0x3) words=words>>1;
 
       int bytes = words * sizeof(vobj);
@@ -151,14 +150,12 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
       int xmit_to_rank;
       grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
 
-      grid->Barrier();
 
       grid->SendToRecvFrom((void *)&send_buf[0],
 			   xmit_to_rank,
 			   (void *)&recv_buf[0],
 			   recv_from_rank,
 			   bytes);
-
       grid->Barrier();
 
       Scatter_plane_simple (ret,recv_buf,dimension,x,cbmask);
@@ -198,15 +195,8 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
   //  int words = sizeof(vobj)/sizeof(vector_type);
 
-  std::vector<cshiftVector<scalar_object> >  send_buf_extract(Nsimd);
-  std::vector<cshiftVector<scalar_object> >  recv_buf_extract(Nsimd);
-  scalar_object *  recv_buf_extract_mpi;
-  scalar_object *  send_buf_extract_mpi;
- 
-  for(int s=0;s<Nsimd;s++){
-    send_buf_extract[s].resize(buffer_size);
-    recv_buf_extract[s].resize(buffer_size);
-  }
+  std::vector<commVector<scalar_object> >   send_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
+  std::vector<commVector<scalar_object> >   recv_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
 
   int bytes = buffer_size*sizeof(scalar_object);
 
@@ -252,204 +242,11 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
       if(nbr_proc){
 	grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
 
-	grid->Barrier();
-
-	send_buf_extract_mpi = &send_buf_extract[nbr_lane][0];
-	recv_buf_extract_mpi = &recv_buf_extract[i][0];
-	grid->SendToRecvFrom((void *)send_buf_extract_mpi,
+	grid->SendToRecvFrom((void *)&send_buf_extract[nbr_lane][0],
 			     xmit_to_rank,
-			     (void *)recv_buf_extract_mpi,
+			     (void *)&recv_buf_extract[i][0],
 			     recv_from_rank,
 			     bytes);
-
-	grid->Barrier();
-
-	rpointers[i] = &recv_buf_extract[i][0];
-      } else { 
-	rpointers[i] = &send_buf_extract[nbr_lane][0];
-      }
-
-    }
-    Scatter_plane_merge(ret,rpointers,dimension,x,cbmask);
-  }
-
-}
-#else 
-template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
-{
-  typedef typename vobj::vector_type vector_type;
-  typedef typename vobj::scalar_type scalar_type;
-
-  GridBase *grid=rhs.Grid();
-  Lattice<vobj> temp(rhs.Grid());
-
-  int fd              = rhs.Grid()->_fdimensions[dimension];
-  int rd              = rhs.Grid()->_rdimensions[dimension];
-  int pd              = rhs.Grid()->_processors[dimension];
-  int simd_layout     = rhs.Grid()->_simd_layout[dimension];
-  int comm_dim        = rhs.Grid()->_processors[dimension] >1 ;
-  assert(simd_layout==1);
-  assert(comm_dim==1);
-  assert(shift>=0);
-  assert(shift<fd);
-  
-  int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
-  cshiftVector<vobj> send_buf_v(buffer_size);
-  cshiftVector<vobj> recv_buf_v(buffer_size);
-  vobj *send_buf;
-  vobj *recv_buf;
-  {
-    grid->ShmBufferFreeAll();
-    size_t bytes = buffer_size*sizeof(vobj);
-    send_buf=(vobj *)grid->ShmBufferMalloc(bytes);
-    recv_buf=(vobj *)grid->ShmBufferMalloc(bytes);
-  }
-    
-  int cb= (cbmask==0x2)? Odd : Even;
-  int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
-
-  for(int x=0;x<rd;x++){       
-
-    int sx        =  (x+sshift)%rd;
-    int comm_proc = ((x+sshift)/rd)%pd;
-    
-    if (comm_proc==0) {
-
-      Copy_plane(ret,rhs,dimension,x,sx,cbmask); 
-
-    } else {
-
-      int words = buffer_size;
-      if (cbmask != 0x3) words=words>>1;
-
-      int bytes = words * sizeof(vobj);
-
-      Gather_plane_simple (rhs,send_buf_v,dimension,sx,cbmask);
-
-      //      int rank           = grid->_processor;
-      int recv_from_rank;
-      int xmit_to_rank;
-      grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
-
-
-      grid->Barrier();
-
-      acceleratorCopyDeviceToDevice((void *)&send_buf_v[0],(void *)&send_buf[0],bytes);
-      grid->SendToRecvFrom((void *)&send_buf[0],
-			   xmit_to_rank,
-			   (void *)&recv_buf[0],
-			   recv_from_rank,
-			   bytes);
-      acceleratorCopyDeviceToDevice((void *)&recv_buf[0],(void *)&recv_buf_v[0],bytes);
-
-      grid->Barrier();
-
-      Scatter_plane_simple (ret,recv_buf_v,dimension,x,cbmask);
-    }
-  }
-}
-
-template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
-{
-  GridBase *grid=rhs.Grid();
-  const int Nsimd = grid->Nsimd();
-  typedef typename vobj::vector_type vector_type;
-  typedef typename vobj::scalar_object scalar_object;
-  typedef typename vobj::scalar_type scalar_type;
-   
-  int fd = grid->_fdimensions[dimension];
-  int rd = grid->_rdimensions[dimension];
-  int ld = grid->_ldimensions[dimension];
-  int pd = grid->_processors[dimension];
-  int simd_layout     = grid->_simd_layout[dimension];
-  int comm_dim        = grid->_processors[dimension] >1 ;
-
-  //std::cout << "Cshift_comms_simd dim "<< dimension << " fd "<<fd<<" rd "<<rd
-  //    << " ld "<<ld<<" pd " << pd<<" simd_layout "<<simd_layout 
-  //    << " comm_dim " << comm_dim << " cbmask " << cbmask <<std::endl;
-
-  assert(comm_dim==1);
-  assert(simd_layout==2);
-  assert(shift>=0);
-  assert(shift<fd);
-
-  int permute_type=grid->PermuteType(dimension);
-
-  ///////////////////////////////////////////////
-  // Simd direction uses an extract/merge pair
-  ///////////////////////////////////////////////
-  int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
-  //  int words = sizeof(vobj)/sizeof(vector_type);
-
-  std::vector<cshiftVector<scalar_object> >  send_buf_extract(Nsimd);
-  std::vector<cshiftVector<scalar_object> >  recv_buf_extract(Nsimd);
-  scalar_object *  recv_buf_extract_mpi;
-  scalar_object *  send_buf_extract_mpi;
-  {
-    size_t bytes = sizeof(scalar_object)*buffer_size;
-    grid->ShmBufferFreeAll();
-    send_buf_extract_mpi = (scalar_object *)grid->ShmBufferMalloc(bytes);
-    recv_buf_extract_mpi = (scalar_object *)grid->ShmBufferMalloc(bytes);
-  }
-  for(int s=0;s<Nsimd;s++){
-    send_buf_extract[s].resize(buffer_size);
-    recv_buf_extract[s].resize(buffer_size);
-  }
-
-  int bytes = buffer_size*sizeof(scalar_object);
-
-  ExtractPointerArray<scalar_object>  pointers(Nsimd); // 
-  ExtractPointerArray<scalar_object> rpointers(Nsimd); // received pointers
-
-  ///////////////////////////////////////////
-  // Work out what to send where
-  ///////////////////////////////////////////
-  int cb    = (cbmask==0x2)? Odd : Even;
-  int sshift= grid->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
-
-  // loop over outer coord planes orthog to dim
-  for(int x=0;x<rd;x++){       
-
-    // FIXME call local permute copy if none are offnode.
-    for(int i=0;i<Nsimd;i++){       
-      pointers[i] = &send_buf_extract[i][0];
-    }
-    int sx   = (x+sshift)%rd;
-    Gather_plane_extract(rhs,pointers,dimension,sx,cbmask);
-
-    for(int i=0;i<Nsimd;i++){
-      
-      int inner_bit = (Nsimd>>(permute_type+1));
-      int ic= (i&inner_bit)? 1:0;
-
-      int my_coor          = rd*ic + x;
-      int nbr_coor         = my_coor+sshift;
-      int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
-
-      int nbr_ic   = (nbr_coor%ld)/rd;    // inner coord of peer
-      int nbr_ox   = (nbr_coor%rd);       // outer coord of peer
-      int nbr_lane = (i&(~inner_bit));
-
-      int recv_from_rank;
-      int xmit_to_rank;
-
-      if (nbr_ic) nbr_lane|=inner_bit;
-
-      assert (sx == nbr_ox);
-
-      if(nbr_proc){
-	grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
-
-	grid->Barrier();
-
-	acceleratorCopyDeviceToDevice((void *)&send_buf_extract[nbr_lane][0],(void *)send_buf_extract_mpi,bytes);
-	grid->SendToRecvFrom((void *)send_buf_extract_mpi,
-			     xmit_to_rank,
-			     (void *)recv_buf_extract_mpi,
-			     recv_from_rank,
-			     bytes);
-	acceleratorCopyDeviceToDevice((void *)recv_buf_extract_mpi,(void *)&recv_buf_extract[i][0],bytes);
-
 	grid->Barrier();
 	rpointers[i] = &recv_buf_extract[i][0];
       } else { 
@@ -461,7 +258,7 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   }
 
 }
-#endif
+
 NAMESPACE_END(Grid); 
 
 #endif

Grid/lattice/Lattice.h

@@ -36,8 +36,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/lattice/Lattice_local.h>
 #include <Grid/lattice/Lattice_reduction.h>
 #include <Grid/lattice/Lattice_peekpoke.h>
-#include <Grid/lattice/Lattice_reality.h>
-#include <Grid/lattice/Lattice_real_imag.h>
+//#include <Grid/lattice/Lattice_reality.h>
 #include <Grid/lattice/Lattice_comparison_utils.h>
 #include <Grid/lattice/Lattice_comparison.h>
 #include <Grid/lattice/Lattice_coordinate.h>

Grid/lattice/Lattice_ET.h

@@ -42,24 +42,9 @@ NAMESPACE_BEGIN(Grid);
 ////////////////////////////////////////////////////
 // Predicated where support
 ////////////////////////////////////////////////////
-#ifdef GRID_SIMT
-// drop to scalar in SIMT; cleaner in fact
 template <class iobj, class vobj, class robj>
-accelerator_inline vobj predicatedWhere(const iobj &predicate, 
-					const vobj &iftrue, 
-					const robj &iffalse) 
-{
-  Integer mask = TensorRemove(predicate);
-  typename std::remove_const<vobj>::type ret= iffalse;
-  if (mask) ret=iftrue;
-  return ret;
-}
-#else
-template <class iobj, class vobj, class robj>
-accelerator_inline vobj predicatedWhere(const iobj &predicate, 
-					const vobj &iftrue, 
-					const robj &iffalse) 
-{
+accelerator_inline vobj predicatedWhere(const iobj &predicate, const vobj &iftrue,
+                            const robj &iffalse) {
   typename std::remove_const<vobj>::type ret;
 
   typedef typename vobj::scalar_object scalar_object;
@@ -83,7 +68,6 @@ accelerator_inline vobj predicatedWhere(const iobj &predicate,
   merge(ret, falsevals);
   return ret;
 }
-#endif
 
 /////////////////////////////////////////////////////
 //Specialization of getVectorType for lattices
@@ -97,62 +81,32 @@ struct getVectorType<Lattice<T> >{
 //--  recursive evaluation of expressions; --
 // handle leaves of syntax tree
 ///////////////////////////////////////////////////
-template<class sobj,
-  typename std::enable_if<!is_lattice<sobj>::value&&!is_lattice_expr<sobj>::value,sobj>::type * = nullptr> 
-accelerator_inline 
+template<class sobj> accelerator_inline 
 sobj eval(const uint64_t ss, const sobj &arg)
 {
   return arg;
 }
-template <class lobj> accelerator_inline 
-auto eval(const uint64_t ss, const LatticeView<lobj> &arg) -> decltype(arg(ss))
-{
-  return arg(ss);
-}
 
-////////////////////////////////////////////
-//--  recursive evaluation of expressions; --
-// whole vector return, used only for expression return type inference
-///////////////////////////////////////////////////
-template<class sobj> accelerator_inline 
-sobj vecEval(const uint64_t ss, const sobj &arg)
-{
-  return arg;
-}
 template <class lobj> accelerator_inline 
-const lobj & vecEval(const uint64_t ss, const LatticeView<lobj> &arg) 
+const lobj & eval(const uint64_t ss, const LatticeView<lobj> &arg) 
 {
   return arg[ss];
 }
 
-///////////////////////////////////////////////////
-// handle nodes in syntax tree- eval one operand
-// vecEval needed (but never called as all expressions offloaded) to infer the return type
-// in SIMT contexts of closure.
-///////////////////////////////////////////////////
-template <typename Op, typename T1> accelerator_inline 
-auto vecEval(const uint64_t ss, const LatticeUnaryExpression<Op, T1> &expr)  
-  -> decltype(expr.op.func( vecEval(ss, expr.arg1)))
+// What needs this?
+// Cannot be legal on accelerator
+// Comparison must convert
+#if 1
+template <class lobj> accelerator_inline 
+const lobj & eval(const uint64_t ss, const Lattice<lobj> &arg) 
 {
-  return expr.op.func( vecEval(ss, expr.arg1) );
-}
-// vecEval two operands
-template <typename Op, typename T1, typename T2> accelerator_inline
-auto vecEval(const uint64_t ss, const LatticeBinaryExpression<Op, T1, T2> &expr)  
-  -> decltype(expr.op.func( vecEval(ss,expr.arg1),vecEval(ss,expr.arg2)))
-{
-  return expr.op.func( vecEval(ss,expr.arg1), vecEval(ss,expr.arg2) );
-}
-// vecEval three operands
-template <typename Op, typename T1, typename T2, typename T3> accelerator_inline
-auto vecEval(const uint64_t ss, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)  
-  -> decltype(expr.op.func(vecEval(ss, expr.arg1), vecEval(ss, expr.arg2), vecEval(ss, expr.arg3)))
-{
-  return expr.op.func(vecEval(ss, expr.arg1), vecEval(ss, expr.arg2), vecEval(ss, expr.arg3));
+  auto view = arg.View(AcceleratorRead);
+  return view[ss];
 }
+#endif
 
 ///////////////////////////////////////////////////
-// handle nodes in syntax tree- eval one operand coalesced
+// handle nodes in syntax tree- eval one operand
 ///////////////////////////////////////////////////
 template <typename Op, typename T1> accelerator_inline 
 auto eval(const uint64_t ss, const LatticeUnaryExpression<Op, T1> &expr)  
@@ -160,41 +114,23 @@ auto eval(const uint64_t ss, const LatticeUnaryExpression<Op, T1> &expr)
 {
   return expr.op.func( eval(ss, expr.arg1) );
 }
+///////////////////////
 // eval two operands
+///////////////////////
 template <typename Op, typename T1, typename T2> accelerator_inline
 auto eval(const uint64_t ss, const LatticeBinaryExpression<Op, T1, T2> &expr)  
   -> decltype(expr.op.func( eval(ss,expr.arg1),eval(ss,expr.arg2)))
 {
   return expr.op.func( eval(ss,expr.arg1), eval(ss,expr.arg2) );
 }
+///////////////////////
 // eval three operands
+///////////////////////
 template <typename Op, typename T1, typename T2, typename T3> accelerator_inline
 auto eval(const uint64_t ss, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)  
-  -> decltype(expr.op.func(eval(ss, expr.arg1), 
-			   eval(ss, expr.arg2), 
-			   eval(ss, expr.arg3)))
+  -> decltype(expr.op.func(eval(ss, expr.arg1), eval(ss, expr.arg2), eval(ss, expr.arg3)))
 {
-#ifdef GRID_SIMT
-  // Handles Nsimd (vInteger) != Nsimd(ComplexD)
-  typedef decltype(vecEval(ss, expr.arg2)) rvobj;
-  typedef typename std::remove_reference<rvobj>::type vobj;
-
-  const int Nsimd = vobj::vector_type::Nsimd();
-
-  auto vpred = vecEval(ss,expr.arg1);
-
-  ExtractBuffer<Integer> mask(Nsimd);
-  extract<vInteger, Integer>(TensorRemove(vpred), mask);
-
-  int s = acceleratorSIMTlane(Nsimd);
-  return expr.op.func(mask[s],
-		      eval(ss, expr.arg2), 
-		      eval(ss, expr.arg3));
-#else
-  return expr.op.func(eval(ss, expr.arg1),
-		      eval(ss, expr.arg2), 
-		      eval(ss, expr.arg3));
-#endif
+  return expr.op.func(eval(ss, expr.arg1), eval(ss, expr.arg2), eval(ss, expr.arg3));
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -292,7 +228,7 @@ template <typename Op, typename T1, typename T2> inline
 void ExpressionViewOpen(LatticeBinaryExpression<Op, T1, T2> &expr) 
 {
   ExpressionViewOpen(expr.arg1);  // recurse AST
-  ExpressionViewOpen(expr.arg2);  // rrecurse AST
+  ExpressionViewOpen(expr.arg2);  // recurse AST
 }
 template <typename Op, typename T1, typename T2, typename T3>
 inline void ExpressionViewOpen(LatticeTrinaryExpression<Op, T1, T2, T3> &expr) 
@@ -336,20 +272,28 @@ inline void ExpressionViewClose(LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
 // Unary operators and funcs
 ////////////////////////////////////////////
 #define GridUnopClass(name, ret)					\
+  template <class arg>							\
   struct name {								\
-    template<class _arg> static auto accelerator_inline func(const _arg a) -> decltype(ret) { return ret; } \
+    static auto accelerator_inline func(const arg a) -> decltype(ret) { return ret; } \
   };
 
 GridUnopClass(UnarySub, -a);
 GridUnopClass(UnaryNot, Not(a));
+GridUnopClass(UnaryAdj, adj(a));
+GridUnopClass(UnaryConj, conjugate(a));
 GridUnopClass(UnaryTrace, trace(a));
 GridUnopClass(UnaryTranspose, transpose(a));
 GridUnopClass(UnaryTa, Ta(a));
 GridUnopClass(UnaryProjectOnGroup, ProjectOnGroup(a));
+GridUnopClass(UnaryReal, real(a));
+GridUnopClass(UnaryImag, imag(a));
+GridUnopClass(UnaryToReal, toReal(a));
+GridUnopClass(UnaryToComplex, toComplex(a));
 GridUnopClass(UnaryTimesI, timesI(a));
 GridUnopClass(UnaryTimesMinusI, timesMinusI(a));
 GridUnopClass(UnaryAbs, abs(a));
 GridUnopClass(UnarySqrt, sqrt(a));
+GridUnopClass(UnaryRsqrt, rsqrt(a));
 GridUnopClass(UnarySin, sin(a));
 GridUnopClass(UnaryCos, cos(a));
 GridUnopClass(UnaryAsin, asin(a));
@@ -361,10 +305,10 @@ GridUnopClass(UnaryExp, exp(a));
 // Binary operators
 ////////////////////////////////////////////
 #define GridBinOpClass(name, combination)			\
+  template <class left, class right>				\
   struct name {							\
-    template <class _left, class _right>			\
     static auto accelerator_inline				\
-    func(const _left &lhs, const _right &rhs)			\
+    func(const left &lhs, const right &rhs)			\
       -> decltype(combination) const				\
     {								\
       return combination;					\
@@ -384,10 +328,10 @@ GridBinOpClass(BinaryOrOr, lhs || rhs);
 // Trinary conditional op
 ////////////////////////////////////////////////////
 #define GridTrinOpClass(name, combination)				\
+  template <class predicate, class left, class right>			\
   struct name {								\
-    template <class _predicate,class _left, class _right>		\
     static auto accelerator_inline					\
-    func(const _predicate &pred, const _left &lhs, const _right &rhs)	\
+    func(const predicate &pred, const left &lhs, const right &rhs)	\
       -> decltype(combination) const					\
     {									\
       return combination;						\
@@ -395,17 +339,17 @@ GridBinOpClass(BinaryOrOr, lhs || rhs);
   };
 
 GridTrinOpClass(TrinaryWhere,
-		(predicatedWhere<
-		 typename std::remove_reference<_predicate>::type, 
-		 typename std::remove_reference<_left>::type,
-		 typename std::remove_reference<_right>::type>(pred, lhs,rhs)));
+		(predicatedWhere<predicate, 
+		 typename std::remove_reference<left>::type,
+		 typename std::remove_reference<right>::type>(pred, lhs,rhs)));
 
 ////////////////////////////////////////////
 // Operator syntactical glue
 ////////////////////////////////////////////
-#define GRID_UNOP(name)   name
-#define GRID_BINOP(name)  name
-#define GRID_TRINOP(name) name
+
+#define GRID_UNOP(name)   name<decltype(eval(0, arg))>
+#define GRID_BINOP(name)  name<decltype(eval(0, lhs)), decltype(eval(0, rhs))>
+#define GRID_TRINOP(name) name<decltype(eval(0, pred)), decltype(eval(0, lhs)), decltype(eval(0, rhs))>
 
 #define GRID_DEF_UNOP(op, name)						\
   template <typename T1, typename std::enable_if<is_lattice<T1>::value||is_lattice_expr<T1>::value,T1>::type * = nullptr> \
@@ -451,17 +395,22 @@ GridTrinOpClass(TrinaryWhere,
 GRID_DEF_UNOP(operator-, UnarySub);
 GRID_DEF_UNOP(Not, UnaryNot);
 GRID_DEF_UNOP(operator!, UnaryNot);
-//GRID_DEF_UNOP(adj, UnaryAdj);
-//GRID_DEF_UNOP(conjugate, UnaryConj);
+GRID_DEF_UNOP(adj, UnaryAdj);
+GRID_DEF_UNOP(conjugate, UnaryConj);
 GRID_DEF_UNOP(trace, UnaryTrace);
 GRID_DEF_UNOP(transpose, UnaryTranspose);
 GRID_DEF_UNOP(Ta, UnaryTa);
 GRID_DEF_UNOP(ProjectOnGroup, UnaryProjectOnGroup);
+GRID_DEF_UNOP(real, UnaryReal);
+GRID_DEF_UNOP(imag, UnaryImag);
+GRID_DEF_UNOP(toReal, UnaryToReal);
+GRID_DEF_UNOP(toComplex, UnaryToComplex);
 GRID_DEF_UNOP(timesI, UnaryTimesI);
 GRID_DEF_UNOP(timesMinusI, UnaryTimesMinusI);
 GRID_DEF_UNOP(abs, UnaryAbs);  // abs overloaded in cmath C++98; DON'T do the
                                // abs-fabs-dabs-labs thing
 GRID_DEF_UNOP(sqrt, UnarySqrt);
+GRID_DEF_UNOP(rsqrt, UnaryRsqrt);
 GRID_DEF_UNOP(sin, UnarySin);
 GRID_DEF_UNOP(cos, UnaryCos);
 GRID_DEF_UNOP(asin, UnaryAsin);
@@ -486,36 +435,29 @@ GRID_DEF_TRINOP(where, TrinaryWhere);
 /////////////////////////////////////////////////////////////
 template <class Op, class T1>
 auto closure(const LatticeUnaryExpression<Op, T1> &expr)
-  -> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1)))>::type > 
+  -> Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> 
 {
-  Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1)))>::type > ret(expr);
+  Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> ret(expr);
   return ret;
 }
 template <class Op, class T1, class T2>
 auto closure(const LatticeBinaryExpression<Op, T1, T2> &expr)
-  -> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),vecEval(0, expr.arg2)))>::type >
+  -> Lattice<decltype(expr.op.func(eval(0, expr.arg1),eval(0, expr.arg2)))> 
 {
-  Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),vecEval(0, expr.arg2)))>::type > ret(expr);
+  Lattice<decltype(expr.op.func(eval(0, expr.arg1),eval(0, expr.arg2)))> ret(expr);
   return ret;
 }
 template <class Op, class T1, class T2, class T3>
 auto closure(const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
-  -> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),
-				   vecEval(0, expr.arg2),
-				   vecEval(0, expr.arg3)))>::type >
+  -> Lattice<decltype(expr.op.func(eval(0, expr.arg1),
+				   eval(0, expr.arg2),
+				   eval(0, expr.arg3)))> 
 {
-  Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),
-				vecEval(0, expr.arg2),
-			        vecEval(0, expr.arg3)))>::type >  ret(expr);
+  Lattice<decltype(expr.op.func(eval(0, expr.arg1),
+				eval(0, expr.arg2),
+				eval(0, expr.arg3)))>  ret(expr);
   return ret;
 }
-#define EXPRESSION_CLOSURE(function)					\
-  template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> \
-    auto function(Expression &expr) -> decltype(function(closure(expr))) \
-  {									\
-    return function(closure(expr));					\
-  }
-
 
 #undef GRID_UNOP
 #undef GRID_BINOP

Grid/lattice/Lattice_arith.h

@@ -60,9 +60,9 @@ void mac(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
   autoView( lhs_v , lhs, AcceleratorRead);
   autoView( rhs_v , rhs, AcceleratorRead);
   accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
+    decltype(coalescedRead(obj1())) tmp;
     auto lhs_t=lhs_v(ss);
     auto rhs_t=rhs_v(ss);
-    auto tmp  =ret_v(ss);
     mac(&tmp,&lhs_t,&rhs_t);
     coalescedWrite(ret_v[ss],tmp);
   });
@@ -124,7 +124,7 @@ void mac(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
   autoView( ret_v , ret, AcceleratorWrite);
   autoView( lhs_v , lhs, AcceleratorRead);
   accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
-    auto tmp  =ret_v(ss);
+    decltype(coalescedRead(obj1())) tmp;
     auto lhs_t=lhs_v(ss);
     mac(&tmp,&lhs_t,&rhs);
     coalescedWrite(ret_v[ss],tmp);
@@ -182,7 +182,7 @@ void mac(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
   autoView( ret_v , ret, AcceleratorWrite);
   autoView( rhs_v , lhs, AcceleratorRead);
   accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
-    auto tmp  =ret_v(ss);
+    decltype(coalescedRead(obj1())) tmp;
     auto rhs_t=rhs_v(ss);
     mac(&tmp,&lhs,&rhs_t);
     coalescedWrite(ret_v[ss],tmp);

Grid/lattice/Lattice_base.h

@@ -123,9 +123,9 @@ public:
     auto exprCopy = expr;
     ExpressionViewOpen(exprCopy);
     auto me  = View(AcceleratorWriteDiscard);
-    accelerator_for(ss,me.size(),vobj::Nsimd(),{
+    accelerator_for(ss,me.size(),1,{
       auto tmp = eval(ss,exprCopy);
-      coalescedWrite(me[ss],tmp);
+      vstream(me[ss],tmp);
     });
     me.ViewClose();
     ExpressionViewClose(exprCopy);
@@ -146,9 +146,9 @@ public:
     auto exprCopy = expr;
     ExpressionViewOpen(exprCopy);
     auto me  = View(AcceleratorWriteDiscard);
-    accelerator_for(ss,me.size(),vobj::Nsimd(),{
+    accelerator_for(ss,me.size(),1,{
       auto tmp = eval(ss,exprCopy);
-      coalescedWrite(me[ss],tmp);
+      vstream(me[ss],tmp);
     });
     me.ViewClose();
     ExpressionViewClose(exprCopy);
@@ -168,9 +168,9 @@ public:
     auto exprCopy = expr;
     ExpressionViewOpen(exprCopy);
     auto me  = View(AcceleratorWriteDiscard);
-    accelerator_for(ss,me.size(),vobj::Nsimd(),{
+    accelerator_for(ss,me.size(),1,{
       auto tmp = eval(ss,exprCopy);
-      coalescedWrite(me[ss],tmp);
+      vstream(me[ss],tmp);
     });
     me.ViewClose();
     ExpressionViewClose(exprCopy);

Grid/lattice/Lattice_basis.h

@@ -54,34 +54,13 @@ void basisRotate(VField &basis,Matrix& Qt,int j0, int j1, int k0,int k1,int Nm)
   typedef decltype(basis[0].View(AcceleratorRead)) View;
 
   Vector<View> basis_v; basis_v.reserve(basis.size());
-  typedef typename std::remove_reference<decltype(basis_v[0][0])>::type vobj;
-  typedef typename std::remove_reference<decltype(Qt(0,0))>::type Coeff_t;
   GridBase* grid = basis[0].Grid();
       
   for(int k=0;k<basis.size();k++){
     basis_v.push_back(basis[k].View(AcceleratorWrite));
   }
 
-#if ( (!defined(GRID_SYCL)) && (!defined(GRID_CUDA)) )
-  int max_threads = thread_max();
-  Vector < vobj > Bt(Nm * max_threads);
-  thread_region
-    {
-      vobj* B = &Bt[Nm * thread_num()];
-      thread_for_in_region(ss, grid->oSites(),{
-	  for(int j=j0; j<j1; ++j) B[j]=0.;
 
-	  for(int j=j0; j<j1; ++j){
-	    for(int k=k0; k<k1; ++k){
-	      B[j] +=Qt(j,k) * basis_v[k][ss];
-	    }
-	  }
-	  for(int j=j0; j<j1; ++j){
-	    basis_v[j][ss] = B[j];
-	  }
-	});
-    }
-#else
   View *basis_vp = &basis_v[0];
 
   int nrot = j1-j0;
@@ -91,12 +70,14 @@ void basisRotate(VField &basis,Matrix& Qt,int j0, int j1, int k0,int k1,int Nm)
   uint64_t oSites   =grid->oSites();
   uint64_t siteBlock=(grid->oSites()+nrot-1)/nrot; // Maximum 1 additional vector overhead
 
+  typedef typename std::remove_reference<decltype(basis_v[0][0])>::type vobj;
+
   Vector <vobj> Bt(siteBlock * nrot); 
   auto Bp=&Bt[0];
 
   // GPU readable copy of matrix
-  Vector<Coeff_t> Qt_jv(Nm*Nm);
-  Coeff_t *Qt_p = & Qt_jv[0];
+  Vector<double> Qt_jv(Nm*Nm);
+  double *Qt_p = & Qt_jv[0];
   thread_for(i,Nm*Nm,{
       int j = i/Nm;
       int k = i%Nm;
@@ -137,7 +118,6 @@ void basisRotate(VField &basis,Matrix& Qt,int j0, int j1, int k0,int k1,int Nm)
 	coalescedWrite(basis_v[jj][sss],coalescedRead(Bp[ss*nrot+j]));
       });
   }
-#endif
 
   for(int k=0;k<basis.size();k++) basis_v[k].ViewClose();
 }
@@ -161,12 +141,11 @@ void basisRotateJ(Field &result,std::vector<Field> &basis,Eigen::MatrixXd& Qt,in
   double * Qt_j = & Qt_jv[0];
   for(int k=0;k<Nm;++k) Qt_j[k]=Qt(j,k);
 
-  auto basis_vp=& basis_v[0];
   autoView(result_v,result,AcceleratorWrite);
   accelerator_for(ss, grid->oSites(),vobj::Nsimd(),{
     auto B=coalescedRead(zz);
     for(int k=k0; k<k1; ++k){
-      B +=Qt_j[k] * coalescedRead(basis_vp[k][ss]);
+      B +=Qt_j[k] * coalescedRead(basis_v[k][ss]);
     }
     coalescedWrite(result_v[ss], B);
   });

Grid/lattice/Lattice_comparison.h

@@ -42,6 +42,34 @@ NAMESPACE_BEGIN(Grid);
 
 typedef iScalar<vInteger> vPredicate ;
 
+/*
+template <class iobj, class vobj, class robj> accelerator_inline 
+vobj predicatedWhere(const iobj &predicate, const vobj &iftrue, const robj &iffalse) 
+{
+  typename std::remove_const<vobj>::type ret;
+
+  typedef typename vobj::scalar_object scalar_object;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+
+  const int Nsimd = vobj::vector_type::Nsimd();
+
+  ExtractBuffer<Integer> mask(Nsimd);
+  ExtractBuffer<scalar_object> truevals(Nsimd);
+  ExtractBuffer<scalar_object> falsevals(Nsimd);
+
+  extract(iftrue, truevals);
+  extract(iffalse, falsevals);
+  extract<vInteger, Integer>(TensorRemove(predicate), mask);
+
+  for (int s = 0; s < Nsimd; s++) {
+    if (mask[s]) falsevals[s] = truevals[s];
+  }
+
+  merge(ret, falsevals);
+  return ret;
+}
+*/
 //////////////////////////////////////////////////////////////////////////
 // compare lattice to lattice
 //////////////////////////////////////////////////////////////////////////

Grid/lattice/Lattice_peekpoke.h

@@ -182,14 +182,6 @@ inline void peekLocalSite(sobj &s,const LatticeView<vobj> &l,Coordinate &site)
       
   return;
 };
-template<class vobj,class sobj>
-inline void peekLocalSite(sobj &s,const Lattice<vobj> &l,Coordinate &site)
-{
-  autoView(lv,l,CpuRead);
-  peekLocalSite(s,lv,site);
-  return;
-};
-
 // Must be CPU write view
 template<class vobj,class sobj>
 inline void pokeLocalSite(const sobj &s,LatticeView<vobj> &l,Coordinate &site)
@@ -218,14 +210,6 @@ inline void pokeLocalSite(const sobj &s,LatticeView<vobj> &l,Coordinate &site)
   return;
 };
 
-template<class vobj,class sobj>
-inline void pokeLocalSite(const sobj &s, Lattice<vobj> &l,Coordinate &site)
-{
-  autoView(lv,l,CpuWrite);
-  pokeLocalSite(s,lv,site);
-  return;
-};
-
 NAMESPACE_END(Grid);
 #endif
 

Grid/lattice/Lattice_real_imag.h

@@ -1,79 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/lattice/Lattice_reality.h
-
-    Copyright (C) 2015
-
-Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: neo <cossu@post.kek.jp>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifndef GRID_LATTICE_REAL_IMAG_H
-#define GRID_LATTICE_REAL_IMAG_H
-
-
-// FIXME .. this is the sector of the code 
-// I am most worried about the directions
-// The choice of burying complex in the SIMD
-// is making the use of "real" and "imag" very cumbersome
-
-NAMESPACE_BEGIN(Grid);
-
-template<class vobj> inline Lattice<vobj> real(const Lattice<vobj> &lhs){
-  Lattice<vobj> ret(lhs.Grid());
-
-  autoView( lhs_v, lhs, AcceleratorRead);
-  autoView( ret_v, ret, AcceleratorWrite);
-
-  ret.Checkerboard()=lhs.Checkerboard();
-  accelerator_for( ss, lhs_v.size(), 1, {
-    ret_v[ss] =real(lhs_v[ss]);
-  });
-  return ret;
-};
-template<class vobj> inline Lattice<vobj> imag(const Lattice<vobj> &lhs){
-  Lattice<vobj> ret(lhs.Grid());
-
-  autoView( lhs_v, lhs, AcceleratorRead);
-  autoView( ret_v, ret, AcceleratorWrite);
-
-  ret.Checkerboard()=lhs.Checkerboard();
-  accelerator_for( ss, lhs_v.size(), 1, {
-    ret_v[ss] =imag(lhs_v[ss]);
-  });
-  return ret;
-};
-
-template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> 
-  auto real(const Expression &expr) -> decltype(real(closure(expr)))		
-{									
-  return real(closure(expr));					
-}
-template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> 
-  auto imag(const Expression &expr) -> decltype(imag(closure(expr)))		
-{									
-  return imag(closure(expr));					
-}
-
-NAMESPACE_END(Grid);
-
-#endif

Grid/lattice/Lattice_reality.h

@@ -45,8 +45,8 @@ template<class vobj> inline Lattice<vobj> adj(const Lattice<vobj> &lhs){
   autoView( ret_v, ret, AcceleratorWrite);
 
   ret.Checkerboard()=lhs.Checkerboard();
-  accelerator_for( ss, lhs_v.size(), 1, {
-     ret_v[ss] = adj(lhs_v[ss]);
+  accelerator_for( ss, lhs_v.size(), vobj::Nsimd(), {
+    coalescedWrite(ret_v[ss], adj(lhs_v(ss)));
   });
   return ret;
 };
@@ -64,53 +64,6 @@ template<class vobj> inline Lattice<vobj> conjugate(const Lattice<vobj> &lhs){
   return ret;
 };
 
-template<class vobj> inline Lattice<typename vobj::Complexified> toComplex(const Lattice<vobj> &lhs){
-  Lattice<typename vobj::Complexified> ret(lhs.Grid());
-
-  autoView( lhs_v, lhs, AcceleratorRead);
-  autoView( ret_v, ret, AcceleratorWrite);
-
-  ret.Checkerboard() = lhs.Checkerboard();
-  accelerator_for( ss, lhs_v.size(), 1, {
-    ret_v[ss] = toComplex(lhs_v[ss]);
-  });
-  return ret;
-};
-template<class vobj> inline Lattice<typename vobj::Realified> toReal(const Lattice<vobj> &lhs){
-  Lattice<typename vobj::Realified> ret(lhs.Grid());
-
-  autoView( lhs_v, lhs, AcceleratorRead);
-  autoView( ret_v, ret, AcceleratorWrite);
-
-  ret.Checkerboard() = lhs.Checkerboard();
-  accelerator_for( ss, lhs_v.size(), 1, {
-    ret_v[ss] = toReal(lhs_v[ss]);
-  });
-  return ret;
-};
-
-
-template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> 
-auto toComplex(const Expression &expr)  -> decltype(closure(expr)) 
-{
-  return toComplex(closure(expr));
-}
-template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> 
-auto toReal(const Expression &expr)  -> decltype(closure(expr)) 
-{
-  return toReal(closure(expr));
-}
-template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> 
-auto adj(const Expression &expr)  -> decltype(closure(expr)) 
-{
-  return adj(closure(expr));
-}
-template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> 
-auto conjugate(const Expression &expr)  -> decltype(closure(expr)) 
-{
-  return conjugate(closure(expr));
-}
-
 NAMESPACE_END(Grid);
 
 #endif

Grid/lattice/Lattice_reduction_gpu.h

@@ -2,13 +2,12 @@ NAMESPACE_BEGIN(Grid);
 
 #ifdef GRID_HIP
 extern hipDeviceProp_t *gpu_props;
-#define WARP_SIZE 64
 #endif
 #ifdef GRID_CUDA
 extern cudaDeviceProp *gpu_props;
-#define WARP_SIZE 32
 #endif
 
+#define WARP_SIZE 32
 __device__ unsigned int retirementCount = 0;
 
 template <class Iterator>
@@ -65,7 +64,7 @@ __device__ void reduceBlock(volatile sobj *sdata, sobj mySum, const Iterator tid
   
   // cannot use overloaded operators for sobj as they are not volatile-qualified
   memcpy((void *)&sdata[tid], (void *)&mySum, sizeof(sobj));
-  acceleratorSynchronise();
+  __syncwarp();
   
   const Iterator VEC = WARP_SIZE;
   const Iterator vid = tid & (VEC-1);
@@ -79,9 +78,9 @@ __device__ void reduceBlock(volatile sobj *sdata, sobj mySum, const Iterator tid
       beta += temp;
       memcpy((void *)&sdata[tid], (void *)&beta, sizeof(sobj));
     }
-    acceleratorSynchronise();
+    __syncwarp();
   }
-  acceleratorSynchroniseAll();
+  __syncthreads();
   
   if (threadIdx.x == 0) {
     beta  = Zero();
@@ -91,7 +90,7 @@ __device__ void reduceBlock(volatile sobj *sdata, sobj mySum, const Iterator tid
     }
     memcpy((void *)&sdata[0], (void *)&beta, sizeof(sobj));
   }
-  acceleratorSynchroniseAll();
+  __syncthreads();
 }
 
 

Grid/lattice/Lattice_transfer.h

@@ -240,8 +240,6 @@ template<class vobj,class vobj2,class CComplex>
   autoView( fineX_  , fineX, AcceleratorRead);
   autoView( fineY_  , fineY, AcceleratorRead);
   autoView( coarseA_, coarseA, AcceleratorRead);
-  Coordinate fine_rdimensions = fine->_rdimensions;
-  Coordinate coarse_rdimensions = coarse->_rdimensions;
 
   accelerator_for(sf, fine->oSites(), CComplex::Nsimd(), {
 
@@ -249,9 +247,9 @@ template<class vobj,class vobj2,class CComplex>
       Coordinate coor_c(_ndimension);
       Coordinate coor_f(_ndimension);
 
-      Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
+      Lexicographic::CoorFromIndex(coor_f,sf,fine->_rdimensions);
       for(int d=0;d<_ndimension;d++) coor_c[d]=coor_f[d]/block_r[d];
-      Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
+      Lexicographic::IndexFromCoor(coor_c,sc,coarse->_rdimensions);
 
       // z = A x + y
 #ifdef GRID_SIMT
@@ -355,14 +353,11 @@ inline void blockSum(Lattice<vobj> &coarseData,const Lattice<vobj> &fineData)
   autoView( coarseData_ , coarseData, AcceleratorWrite);
   autoView( fineData_   , fineData, AcceleratorRead);
 
-  Coordinate fine_rdimensions = fine->_rdimensions;
-  Coordinate coarse_rdimensions = coarse->_rdimensions;
-  
   accelerator_for(sc,coarse->oSites(),1,{
 
       // One thread per sub block
       Coordinate coor_c(_ndimension);
-      Lexicographic::CoorFromIndex(coor_c,sc,coarse_rdimensions);  // Block coordinate
+      Lexicographic::CoorFromIndex(coor_c,sc,coarse->_rdimensions);  // Block coordinate
       coarseData_[sc]=Zero();
 
       for(int sb=0;sb<blockVol;sb++){
@@ -372,7 +367,7 @@ inline void blockSum(Lattice<vobj> &coarseData,const Lattice<vobj> &fineData)
 	Coordinate coor_f(_ndimension);
 	Lexicographic::CoorFromIndex(coor_b,sb,block_r);               // Block sub coordinate
 	for(int d=0;d<_ndimension;d++) coor_f[d]=coor_c[d]*block_r[d] + coor_b[d];
-	Lexicographic::IndexFromCoor(coor_f,sf,fine_rdimensions);
+	Lexicographic::IndexFromCoor(coor_f,sf,fine->_rdimensions);
 
 	coarseData_[sc]=coarseData_[sc]+fineData_[sf];
       }

Grid/log/Log.h

@@ -130,8 +130,6 @@ public:
   friend std::ostream& operator<< (std::ostream& stream, Logger& log){
 
     if ( log.active ) {
-      std::ios_base::fmtflags f(stream.flags());
-
       stream << log.background()<<  std::left;
       if (log.topWidth > 0)
       {
@@ -154,8 +152,6 @@ public:
 	       << now	       << log.background() << " : " ;
       }
       stream << log.colour();
-      stream.flags(f);
-
       return stream;
     } else { 
       return devnull;

Grid/parallelIO/BinaryIO.cc

@@ -1,4 +1,3 @@
 #include <Grid/GridCore.h>
 
 int Grid::BinaryIO::latticeWriteMaxRetry = -1;
-Grid::BinaryIO::IoPerf Grid::BinaryIO::lastPerf;

Grid/parallelIO/BinaryIO.h

@@ -79,13 +79,6 @@ inline void removeWhitespace(std::string &key)
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 class BinaryIO {
  public:
-  struct IoPerf
-  {
-    uint64_t size{0},time{0};
-    double   mbytesPerSecond{0.};
-  };
-
-  static IoPerf lastPerf;
   static int latticeWriteMaxRetry;
 
   /////////////////////////////////////////////////////////////////////////////
@@ -509,15 +502,12 @@ class BinaryIO {
       timer.Stop();
     }
     
-    lastPerf.size            = sizeof(fobj)*iodata.size()*nrank;
-    lastPerf.time            = timer.useconds();
-    lastPerf.mbytesPerSecond = lastPerf.size/1024./1024./(lastPerf.time/1.0e6);
     std::cout<<GridLogMessage<<"IOobject: ";
     if ( control & BINARYIO_READ) std::cout << " read  ";
     else                          std::cout << " write ";
     uint64_t bytes = sizeof(fobj)*iodata.size()*nrank;
-    std::cout<< lastPerf.size <<" bytes in "<< timer.Elapsed() <<" "
-	     << lastPerf.mbytesPerSecond <<" MB/s "<<std::endl;
+    std::cout<< bytes <<" bytes in "<<timer.Elapsed() <<" "
+	     << (double)bytes/ (double)timer.useconds() <<" MB/s "<<std::endl;
 
     std::cout<<GridLogMessage<<"IOobject: endian and checksum overhead "<<bstimer.Elapsed()  <<std::endl;
 
@@ -673,15 +663,10 @@ class BinaryIO {
 	     nersc_csum,scidac_csuma,scidac_csumb);
 
     timer.Start();
-    thread_for(lidx,lsites,{  // FIX ME, suboptimal implementation
+    thread_for(lidx,lsites,{
       std::vector<RngStateType> tmp(RngStateCount);
       std::copy(iodata[lidx].begin(),iodata[lidx].end(),tmp.begin());
-      Coordinate lcoor;
-      grid->LocalIndexToLocalCoor(lidx, lcoor);
-      int o_idx=grid->oIndex(lcoor);
-      int i_idx=grid->iIndex(lcoor);
-      int gidx=parallel_rng.generator_idx(o_idx,i_idx);
-      parallel_rng.SetState(tmp,gidx);
+      parallel_rng.SetState(tmp,lidx);
       });
     timer.Stop();
 
@@ -738,12 +723,7 @@ class BinaryIO {
     std::vector<RNGstate> iodata(lsites);
     thread_for(lidx,lsites,{
       std::vector<RngStateType> tmp(RngStateCount);
-      Coordinate lcoor;
-      grid->LocalIndexToLocalCoor(lidx, lcoor);
-      int o_idx=grid->oIndex(lcoor);
-      int i_idx=grid->iIndex(lcoor);
-      int gidx=parallel_rng.generator_idx(o_idx,i_idx);
-      parallel_rng.GetState(tmp,gidx);
+      parallel_rng.GetState(tmp,lidx);
       std::copy(tmp.begin(),tmp.end(),iodata[lidx].begin());
     });
     timer.Stop();

Grid/qcd/QCD.h

@@ -47,7 +47,7 @@ static constexpr int Ym = 5;
 static constexpr int Zm = 6;
 static constexpr int Tm = 7;
 
-static constexpr int Nc=Config_Nc;
+static constexpr int Nc=3;
 static constexpr int Ns=4;
 static constexpr int Nd=4;
 static constexpr int Nhs=2; // half spinor

Grid/qcd/action/fermion/ImprovedStaggeredFermion5D.h

@@ -208,7 +208,7 @@ public:
   LebesgueOrder LebesgueEvenOdd;
     
   // Comms buffer
-  //  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  comm_buf;
+  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  comm_buf;
     
   ///////////////////////////////////////////////////////////////
   // Conserved current utilities

Grid/qcd/action/fermion/StaggeredKernels.h

@@ -63,20 +63,17 @@ template<class Impl> class StaggeredKernels : public FermionOperator<Impl> , pub
    ///////////////////////////////////////////////////////////////////////////////////////
    // Generic Nc kernels
    ///////////////////////////////////////////////////////////////////////////////////////
-   template<int Naik> 
-   static accelerator_inline
+   template<int Naik> accelerator_inline
    void DhopSiteGeneric(StencilView &st, 
 			DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU, 
 			SiteSpinor * buf, int LLs, int sU, 
 			const FermionFieldView &in, FermionFieldView &out,int dag);
-   
-   template<int Naik> static accelerator_inline
+   template<int Naik> accelerator_inline
    void DhopSiteGenericInt(StencilView &st, 
 			   DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU, 
 			   SiteSpinor * buf, int LLs, int sU, 
 			   const FermionFieldView &in, FermionFieldView &out,int dag);
-   
-   template<int Naik> static accelerator_inline
+   template<int Naik> accelerator_inline
    void DhopSiteGenericExt(StencilView &st, 
 			   DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 			   SiteSpinor * buf, int LLs, int sU, 
@@ -85,20 +82,17 @@ template<class Impl> class StaggeredKernels : public FermionOperator<Impl> , pub
    ///////////////////////////////////////////////////////////////////////////////////////
    // Nc=3 specific kernels
    ///////////////////////////////////////////////////////////////////////////////////////
-   
-   template<int Naik> static accelerator_inline
+   template<int Naik> accelerator_inline
    void DhopSiteHand(StencilView &st, 
 		     DoubledGaugeFieldView &U,DoubledGaugeFieldView &UUU, 
 		     SiteSpinor * buf, int LLs, int sU, 
 		     const FermionFieldView &in, FermionFieldView &out,int dag);
-   
-   template<int Naik> static accelerator_inline
+   template<int Naik> accelerator_inline
    void DhopSiteHandInt(StencilView &st, 
 			DoubledGaugeFieldView &U,DoubledGaugeFieldView &UUU, 
 			SiteSpinor * buf, int LLs, int sU, 
 			const FermionFieldView &in, FermionFieldView &out,int dag);
-   
-   template<int Naik> static accelerator_inline
+   template<int Naik> accelerator_inline
    void DhopSiteHandExt(StencilView &st, 
 			DoubledGaugeFieldView &U,DoubledGaugeFieldView &UUU, 
 			SiteSpinor * buf, int LLs, int sU, 
@@ -107,7 +101,6 @@ template<class Impl> class StaggeredKernels : public FermionOperator<Impl> , pub
    ///////////////////////////////////////////////////////////////////////////////////////
    // Asm Nc=3 specific kernels
    ///////////////////////////////////////////////////////////////////////////////////////
-   
    void DhopSiteAsm(StencilView &st, 
 		    DoubledGaugeFieldView &U,DoubledGaugeFieldView &UUU, 
 		    SiteSpinor * buf, int LLs, int sU, 

Grid/qcd/action/fermion/WilsonFermion.h

@@ -74,20 +74,6 @@ public:
   FermionField _tmp;
   FermionField &tmp(void) { return _tmp; }
 
-  void Report(void);
-  void ZeroCounters(void);
-  double DhopCalls;
-  double DhopCommTime;
-  double DhopComputeTime;
-  double DhopComputeTime2;
-  double DhopFaceTime;
-  double DhopTotalTime;
-
-  double DerivCalls;
-  double DerivCommTime;
-  double DerivComputeTime;
-  double DerivDhopComputeTime;
-
   //////////////////////////////////////////////////////////////////
   // override multiply; cut number routines if pass dagger argument
   // and also make interface more uniformly consistent
@@ -210,3 +196,5 @@ typedef WilsonFermion<WilsonImplF> WilsonFermionF;
 typedef WilsonFermion<WilsonImplD> WilsonFermionD;
 
 NAMESPACE_END(Grid);
+
+

Grid/qcd/action/fermion/WilsonFermion5D.h

@@ -215,7 +215,7 @@ public:
   LebesgueOrder LebesgueEvenOdd;
     
   // Comms buffer
-  //  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  comm_buf;
+  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  comm_buf;
 
 
 };

Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h

@@ -799,7 +799,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
 
   PropagatorField tmp(UGrid);
   PropagatorField Utmp(UGrid);
-  PropagatorField zz (UGrid);   zz=0.0;
+  LatticeInteger zz (UGrid);   zz=0.0;
   LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
   for (int s=0;s<Ls;s++) {
 
@@ -850,7 +850,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
   PropagatorField tmp(UGrid);
   PropagatorField Utmp(UGrid);
 
-  PropagatorField  zz (UGrid);   zz=0.0;
+  LatticeInteger zz (UGrid);   zz=0.0;
   LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
 
   for(int s=0;s<Ls;s++){

Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h

@@ -146,7 +146,7 @@ NAMESPACE_BEGIN(Grid);
 
 
 template <class Impl>
-template <int Naik> accelerator_inline
+template <int Naik>
 void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
 					  DoubledGaugeFieldView &U,DoubledGaugeFieldView &UUU,
 					  SiteSpinor *buf, int sF, int sU, 
@@ -221,7 +221,7 @@ void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
 
 
 template <class Impl>
-template <int Naik> accelerator_inline
+template <int Naik>
 void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st, 
 					     DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 					     SiteSpinor *buf, int sF, int sU, 
@@ -300,7 +300,7 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
 
 
 template <class Impl>
-template <int Naik> accelerator_inline
+template <int Naik>
 void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
 					     DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 					     SiteSpinor *buf, int sF, int sU, 

Grid/qcd/action/fermion/implementation/StaggeredKernelsImplementation.h

@@ -78,7 +78,7 @@ StaggeredKernels<Impl>::StaggeredKernels(const ImplParams &p) : Base(p){};
 // Int, Ext, Int+Ext cases for comms overlap
 ////////////////////////////////////////////////////////////////////////////////////
 template <class Impl>
-template <int Naik> accelerator_inline
+template <int Naik>
 void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st, 
 					     DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 					     SiteSpinor *buf, int sF, int sU, 
@@ -126,7 +126,7 @@ void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st,
   // Only contributions from interior of our node
   ///////////////////////////////////////////////////
 template <class Impl>
-template <int Naik> accelerator_inline
+template <int Naik>
 void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st, 
 						DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 						SiteSpinor *buf, int sF, int sU, 
@@ -174,7 +174,7 @@ void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st,
   // Only contributions from exterior of our node
   ///////////////////////////////////////////////////
 template <class Impl>
-template <int Naik> accelerator_inline
+template <int Naik>
 void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st, 
 						DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 						SiteSpinor *buf, int sF, int sU,

Grid/qcd/action/fermion/implementation/WilsonCloverFermionImplementation.h

@@ -133,14 +133,14 @@ void WilsonCloverFermion<Impl>::ImportGauge(const GaugeField &_Umu)
   pickCheckerboard(Even, CloverTermEven, CloverTerm);
   pickCheckerboard(Odd, CloverTermOdd, CloverTerm);
 
-  pickCheckerboard(Even, CloverTermDagEven, adj(CloverTerm));
-  pickCheckerboard(Odd, CloverTermDagOdd, adj(CloverTerm));
+  pickCheckerboard(Even, CloverTermDagEven, closure(adj(CloverTerm)));
+  pickCheckerboard(Odd, CloverTermDagOdd, closure(adj(CloverTerm)));
 
   pickCheckerboard(Even, CloverTermInvEven, CloverTermInv);
   pickCheckerboard(Odd, CloverTermInvOdd, CloverTermInv);
 
-  pickCheckerboard(Even, CloverTermInvDagEven, adj(CloverTermInv));
-  pickCheckerboard(Odd, CloverTermInvDagOdd, adj(CloverTermInv));
+  pickCheckerboard(Even, CloverTermInvDagEven, closure(adj(CloverTermInv)));
+  pickCheckerboard(Odd, CloverTermInvDagOdd, closure(adj(CloverTermInv)));
 }
 
 template <class Impl>

Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h

@@ -75,91 +75,6 @@ WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
   StencilOdd.BuildSurfaceList(1,vol4);
 }
 
-template<class Impl>
-void WilsonFermion<Impl>::Report(void)
-{
-  RealD NP = _grid->_Nprocessors;
-  RealD NN = _grid->NodeCount();
-  RealD volume = 1;
-  Coordinate latt = _grid->GlobalDimensions();
-  for(int mu=0;mu<Nd;mu++) volume=volume*latt[mu];
-
-  if ( DhopCalls > 0 ) {
-    std::cout << GridLogMessage << "#### Dhop calls report " << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion Number of DhopEO Calls   : " << DhopCalls   << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion TotalTime   /Calls        : " << DhopTotalTime   / DhopCalls << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion CommTime    /Calls        : " << DhopCommTime    / DhopCalls << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion FaceTime    /Calls        : " << DhopFaceTime    / DhopCalls << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion ComputeTime1/Calls        : " << DhopComputeTime / DhopCalls << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion ComputeTime2/Calls        : " << DhopComputeTime2/ DhopCalls << " us" << std::endl;
-
-    // Average the compute time
-    _grid->GlobalSum(DhopComputeTime);
-    DhopComputeTime/=NP;
-    RealD mflops = 1320*volume*DhopCalls/DhopComputeTime/2; // 2 for red black counting
-    std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per rank       : " << mflops/NP << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NN << std::endl;
-
-    RealD Fullmflops = 1320*volume*DhopCalls/(DhopTotalTime)/2; // 2 for red black counting
-    std::cout << GridLogMessage << "Average mflops/s per call (full)         : " << Fullmflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per rank (full): " << Fullmflops/NP << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per node (full): " << Fullmflops/NN << std::endl;
-
-   }
-
-  if ( DerivCalls > 0 ) {
-    std::cout << GridLogMessage << "#### Deriv calls report "<< std::endl;
-    std::cout << GridLogMessage << "WilsonFermion Number of Deriv Calls    : " <<DerivCalls <<std::endl;
-    std::cout << GridLogMessage << "WilsonFermion CommTime/Calls           : " <<DerivCommTime/DerivCalls<<" us" <<std::endl;
-    std::cout << GridLogMessage << "WilsonFermion ComputeTime/Calls        : " <<DerivComputeTime/DerivCalls<<" us" <<std::endl;
-    std::cout << GridLogMessage << "WilsonFermion Dhop ComputeTime/Calls   : " <<DerivDhopComputeTime/DerivCalls<<" us" <<std::endl;
-
-    // how to count flops here?
-    RealD mflops = 144*volume*DerivCalls/DerivDhopComputeTime;
-    std::cout << GridLogMessage << "Average mflops/s per call               ? : " << mflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per node      ? : " << mflops/NP << std::endl;
-
-    // how to count flops here?
-    RealD Fullmflops = 144*volume*DerivCalls/(DerivDhopComputeTime+DerivCommTime)/2; // 2 for red black counting
-    std::cout << GridLogMessage << "Average mflops/s per call (full)        ? : " << Fullmflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per node (full) ? : " << Fullmflops/NP << std::endl;  }
-
-  if (DerivCalls > 0 || DhopCalls > 0){
-    std::cout << GridLogMessage << "WilsonFermion Stencil"    <<std::endl;  Stencil.Report();
-    std::cout << GridLogMessage << "WilsonFermion StencilEven"<<std::endl;  StencilEven.Report();
-    std::cout << GridLogMessage << "WilsonFermion StencilOdd" <<std::endl;  StencilOdd.Report();
-  }
-  if ( DhopCalls > 0){
-    std::cout << GridLogMessage << "WilsonFermion Stencil     Reporti()"    <<std::endl;  Stencil.Reporti(DhopCalls);
-    std::cout << GridLogMessage << "WilsonFermion StencilEven Reporti()"<<std::endl;  StencilEven.Reporti(DhopCalls);
-    std::cout << GridLogMessage << "WilsonFermion StencilOdd  Reporti()" <<std::endl;  StencilOdd.Reporti(DhopCalls);
-  }
-}
-
-template<class Impl>
-void WilsonFermion<Impl>::ZeroCounters(void) {
-  DhopCalls       = 0; // ok
-  DhopCommTime    = 0;
-  DhopComputeTime = 0;
-  DhopComputeTime2= 0;
-  DhopFaceTime    = 0;
-  DhopTotalTime   = 0;
-
-  DerivCalls       = 0; // ok
-  DerivCommTime    = 0;
-  DerivComputeTime = 0;
-  DerivDhopComputeTime = 0;
-
-  Stencil.ZeroCounters();
-  StencilEven.ZeroCounters();
-  StencilOdd.ZeroCounters();
-  Stencil.ZeroCountersi();
-  StencilEven.ZeroCountersi();
-  StencilOdd.ZeroCountersi();
-}
-
-
 template <class Impl>
 void WilsonFermion<Impl>::ImportGauge(const GaugeField &_Umu) 
 {
@@ -319,7 +234,6 @@ template <class Impl>
 void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
                                         GaugeField &mat, const FermionField &A,
                                         const FermionField &B, int dag) {
-  DerivCalls++;
   assert((dag == DaggerNo) || (dag == DaggerYes));
 
   Compressor compressor(dag);
@@ -328,11 +242,8 @@ void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
   FermionField Atilde(B.Grid());
   Atilde = A;
 
-  DerivCommTime-=usecond();
   st.HaloExchange(B, compressor);
-  DerivCommTime+=usecond();
 
-  DerivComputeTime-=usecond();
   for (int mu = 0; mu < Nd; mu++) {
     ////////////////////////////////////////////////////////////////////////
     // Flip gamma (1+g)<->(1-g) if dag
@@ -340,7 +251,6 @@ void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
     int gamma = mu;
     if (!dag) gamma += Nd;
 
-    DerivDhopComputeTime -= usecond();
     int Ls=1;
     Kernels::DhopDirKernel(st, U, st.CommBuf(), Ls, B.Grid()->oSites(), B, Btilde, mu, gamma);
 
@@ -348,9 +258,7 @@ void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
     // spin trace outer product
     //////////////////////////////////////////////////
     Impl::InsertForce4D(mat, Btilde, Atilde, mu);
-    DerivDhopComputeTime += usecond();
   }
-  DerivComputeTime += usecond();
 }
 
 template <class Impl>
@@ -484,14 +392,13 @@ void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
                                        const FermionField &in,
                                        FermionField &out, int dag) 
 {
-  DhopTotalTime-=usecond();
 #ifdef GRID_OMP
   if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute )
     DhopInternalOverlappedComms(st,lo,U,in,out,dag);
   else
 #endif 
     DhopInternalSerial(st,lo,U,in,out,dag);
-  DhopTotalTime+=usecond();
+
 }
 
 template <class Impl>
@@ -510,53 +417,38 @@ void WilsonFermion<Impl>::DhopInternalOverlappedComms(StencilImpl &st, LebesgueO
   /////////////////////////////
   std::vector<std::vector<CommsRequest_t> > requests;
   st.Prepare();
-  DhopFaceTime-=usecond();
   st.HaloGather(in,compressor);
-  DhopFaceTime+=usecond();
-
-  DhopCommTime -=usecond();
   st.CommunicateBegin(requests);
 
   /////////////////////////////
   // Overlap with comms
   /////////////////////////////
-  DhopFaceTime-=usecond();
   st.CommsMergeSHM(compressor);
-  DhopFaceTime+=usecond();
 
   /////////////////////////////
   // do the compute interior
   /////////////////////////////
   int Opt = WilsonKernelsStatic::Opt;
-  DhopComputeTime-=usecond();
   if (dag == DaggerYes) {
     Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
   } else {
     Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
   } 
-  DhopComputeTime+=usecond();
 
   /////////////////////////////
   // Complete comms
   /////////////////////////////
   st.CommunicateComplete(requests);
-  DhopCommTime   +=usecond();
-
-  DhopFaceTime-=usecond();
   st.CommsMerge(compressor);
-  DhopFaceTime+=usecond();
 
   /////////////////////////////
   // do the compute exterior
   /////////////////////////////
-
-  DhopComputeTime2-=usecond();
   if (dag == DaggerYes) {
     Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
   } else {
     Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
   }
-  DhopComputeTime2+=usecond();
 };
 
 
@@ -568,18 +460,14 @@ void WilsonFermion<Impl>::DhopInternalSerial(StencilImpl &st, LebesgueOrder &lo,
 {
   assert((dag == DaggerNo) || (dag == DaggerYes));
   Compressor compressor(dag);
-  DhopCommTime-=usecond();
   st.HaloExchange(in, compressor);
-  DhopCommTime+=usecond();
 
-  DhopComputeTime-=usecond();
   int Opt = WilsonKernelsStatic::Opt;
   if (dag == DaggerYes) {
     Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
   } else {
     Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
   }
-  DhopComputeTime+=usecond();
 };
 /*Change ends */
 

Grid/qcd/action/fermion/implementation/WilsonKernelsAsmA64FX.h

@@ -1,574 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-
-
-    Source file: ./lib/qcd/action/fermion/WilsonKernelsAsmA64FX.h
-
-    Copyright (C) 2020
-
-Author: Nils Meyer  <nils.meyer@ur.de>  Regensburg University
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#pragma once
-
-//#if defined(A64FXASM)
-#if defined(A64FX)
-
-// safety include
-#include <arm_sve.h>
-
-// undefine everything related to kernels
-#include <simd/Fujitsu_A64FX_undef.h>
-
-// enable A64FX body
-#define WILSONKERNELSASMBODYA64FX
-//#pragma message("A64FX Dslash: WilsonKernelsAsmBodyA64FX.h")
-
-    ///////////////////////////////////////////////////////////
-    // If we are A64FX specialise the single precision routine
-    ///////////////////////////////////////////////////////////
-#if defined(DSLASHINTRIN)
-//#pragma message ("A64FX Dslash: intrin")
-#include <simd/Fujitsu_A64FX_intrin_single.h>
-#else
-#pragma message ("A64FX Dslash: asm")
-#include <simd/Fujitsu_A64FX_asm_single.h>
-#endif
-
-/// Switch off the 5d vectorised code optimisations
-#undef DWFVEC5D
-
-/////////////////////////////////////////////////////////////////
-// XYZT vectorised, undag Kernel, single
-/////////////////////////////////////////////////////////////////
-#undef KERNEL_DAG
-#define INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplFH>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#define INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplFH>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#define EXTERIOR
-template<> void
-WilsonKernels<WilsonImplF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplFH>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-
-/////////////////////////////////////////////////////////////////
-// XYZT vectorised, dag Kernel, single
-/////////////////////////////////////////////////////////////////
-#define KERNEL_DAG
-#define INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#define INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#define EXTERIOR
-template<> void
-WilsonKernels<WilsonImplF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-
-// undefine
-#include <simd/Fujitsu_A64FX_undef.h>
-
-///////////////////////////////////////////////////////////
-// If we are A64FX specialise the double precision routine
-///////////////////////////////////////////////////////////
-
-#if defined(DSLASHINTRIN)
-#include <simd/Fujitsu_A64FX_intrin_double.h>
-#else
-#include <simd/Fujitsu_A64FX_asm_double.h>
-#endif
-
-// former KNL
-//#define MAYBEPERM(A,perm) if (perm) { A ; }
-//#define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
-//#define COMPLEX_SIGNS(isigns) vComplexD *isigns = &signsD[0];
-
-
-#define INTERIOR_AND_EXTERIOR
-#undef  INTERIOR
-#undef  EXTERIOR
-
-/////////////////////////////////////////////////////////////////
-// XYZT vectorised, undag Kernel, double
-/////////////////////////////////////////////////////////////////
-#undef KERNEL_DAG
-#define INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplD>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplD>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplDF>::AsmDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#define INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplD>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplD>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplDF>::AsmDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#define EXTERIOR
-template<> void
-WilsonKernels<WilsonImplD>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplD>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplDF>::AsmDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-/////////////////////////////////////////////////////////////////
-// XYZT vectorised, dag Kernel, double
-/////////////////////////////////////////////////////////////////
-#define KERNEL_DAG
-#define INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplD>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplD>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#define INTERIOR
-#undef EXTERIOR
-template<> void
-WilsonKernels<WilsonImplD>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-#undef INTERIOR_AND_EXTERIOR
-#undef INTERIOR
-#define EXTERIOR
-template<> void
-WilsonKernels<WilsonImplD>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<WilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-template<> void
-WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf,
-						int ss,int ssU,int Ls,int Ns,const FermionFieldView &in, FermionFieldView &out)
-#if defined (WILSONKERNELSASMBODYA64FX)
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h>
-#else
-#include <qcd/action/fermion/implementation/WilsonKernelsAsmBody.h>
-#endif
-
-
-
-// undefs
-#undef WILSONKERNELSASMBODYA64FX
-#include <simd/Fujitsu_A64FX_undef.h>
-
-#endif //A64FXASM

Grid/qcd/action/fermion/implementation/WilsonKernelsAsmBodyA64FX.h

@@ -1,380 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: WilsonKernelsAsmBodyA64FX.h
-
-    Copyright (C) 2020
-
-Author:  Nils Meyer  <nils.meyer@ur.de>  Regensburg University
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifdef KERNEL_DAG
-#define DIR0_PROJ    XP_PROJ
-#define DIR1_PROJ    YP_PROJ
-#define DIR2_PROJ    ZP_PROJ
-#define DIR3_PROJ    TP_PROJ
-#define DIR4_PROJ    XM_PROJ
-#define DIR5_PROJ    YM_PROJ
-#define DIR6_PROJ    ZM_PROJ
-#define DIR7_PROJ    TM_PROJ
-#define DIR0_RECON   XP_RECON
-#define DIR1_RECON   YP_RECON_ACCUM
-#define DIR2_RECON   ZP_RECON_ACCUM
-#define DIR3_RECON   TP_RECON_ACCUM
-#define DIR4_RECON   XM_RECON_ACCUM
-#define DIR5_RECON   YM_RECON_ACCUM
-#define DIR6_RECON   ZM_RECON_ACCUM
-#define DIR7_RECON   TM_RECON_ACCUM
-#else
-#define DIR0_PROJ    XM_PROJ
-#define DIR1_PROJ    YM_PROJ
-#define DIR2_PROJ    ZM_PROJ
-#define DIR3_PROJ    TM_PROJ
-#define DIR4_PROJ    XP_PROJ
-#define DIR5_PROJ    YP_PROJ
-#define DIR6_PROJ    ZP_PROJ
-#define DIR7_PROJ    TP_PROJ
-#define DIR0_RECON   XM_RECON
-#define DIR1_RECON   YM_RECON_ACCUM
-#define DIR2_RECON   ZM_RECON_ACCUM
-#define DIR3_RECON   TM_RECON_ACCUM
-#define DIR4_RECON   XP_RECON_ACCUM
-#define DIR5_RECON   YP_RECON_ACCUM
-#define DIR6_RECON   ZP_RECON_ACCUM
-#define DIR7_RECON   TP_RECON_ACCUM
-#endif
-
-//using namespace std;
-
-#undef SHOW
-//#define SHOW
-
-#undef WHERE
-
-#ifdef INTERIOR_AND_EXTERIOR
-#define WHERE "INT_AND_EXT"
-#endif
-
-#ifdef INTERIOR
-#define WHERE "INT"
-#endif
-
-#ifdef EXTERIOR
-#define WHERE "EXT"
-#endif
-
-//#pragma message("here")
-
-
-
-////////////////////////////////////////////////////////////////////////////////
-// Comms then compute kernel
-////////////////////////////////////////////////////////////////////////////////
-#ifdef INTERIOR_AND_EXTERIOR
-
-#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-      basep = st.GetPFInfo(nent,plocal); nent++;			\
-      if ( local ) {							            \
-    LOAD_CHIMU(base);                                       \
-    LOAD_TABLE(PERMUTE_DIR);                                \
-    PROJ;							                        \
-    MAYBEPERM(PERMUTE_DIR,perm);					        \
-      } else {								                \
-	LOAD_CHI(base);							                \
-      }									                    \
-      base = st.GetInfo(ptype,local,perm,NxtDir,ent,plocal); ent++;	\
-    MULT_2SPIN_1(Dir);					                    \
-    PREFETCH_CHIMU(base);                                   \
-    PREFETCH_CHIMU_L2(basep);                               \
-    /* PREFETCH_GAUGE_L1(NxtDir); */                        \
-    MULT_2SPIN_2;					                        \
-    if (s == 0) {                                           \
-      if ((Dir == 0) || (Dir == 4)) { PREFETCH_GAUGE_L2(Dir); } \
-    }                                                       \
-    RECON;								                    \
-
-#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)	    \
-  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++; \
-  PREFETCH1_CHIMU(base);						            \
-  ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)
-
-#define RESULT(base,basep) SAVE_RESULT(base,basep);
-
-#endif
-
-////////////////////////////////////////////////////////////////////////////////
-// Pre comms kernel -- prefetch like normal because it is mostly right
-////////////////////////////////////////////////////////////////////////////////
-#ifdef INTERIOR
-
-#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-      basep = st.GetPFInfo(nent,plocal); nent++;			\
-      if ( local ) {							            \
-    LOAD_CHIMU(base);                                       \
-    LOAD_TABLE(PERMUTE_DIR);                                \
-    PROJ;							                        \
-    MAYBEPERM(PERMUTE_DIR,perm);					        \
-      }else if ( st.same_node[Dir] ) {LOAD_CHI(base);}	    \
-      base = st.GetInfo(ptype,local,perm,NxtDir,ent,plocal); ent++;	\
-      if ( local || st.same_node[Dir] ) {				    \
-    MULT_2SPIN_1(Dir);					                    \
-    PREFETCH_CHIMU(base);                                   \
-    /* PREFETCH_GAUGE_L1(NxtDir); */                        \
-    MULT_2SPIN_2;					                        \
-    if (s == 0) {                                           \
-       if ((Dir == 0) || (Dir == 4)) { PREFETCH_GAUGE_L2(Dir); } \
-    }                                                       \
-    RECON;								                    \
-    PREFETCH_CHIMU_L2(basep);                               \
-      } else { PREFETCH_CHIMU(base); }								                    \
-
-#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++;		\
-  PREFETCH1_CHIMU(base);						\
-  ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)
-
-#define RESULT(base,basep) SAVE_RESULT(base,basep);
-
-#endif
-////////////////////////////////////////////////////////////////////////////////
-// Post comms kernel
-////////////////////////////////////////////////////////////////////////////////
-#ifdef EXTERIOR
-
-
-#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++; \
-  if((!local)&&(!st.same_node[Dir]) ) {					    \
-    LOAD_CHI(base);							                \
-    MULT_2SPIN_1(Dir);					                    \
-    PREFETCH_CHIMU(base);                                   \
-    /* PREFETCH_GAUGE_L1(NxtDir); */                        \
-    MULT_2SPIN_2;					                        \
-    if (s == 0) {                                           \
-      if ((Dir == 0) || (Dir == 4)) { PREFETCH_GAUGE_L2(Dir); } \
-    }                                                       \
-    RECON;								                    \
-    nmu++;								                    \
-  }
-
-#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)	    \
-  nmu=0;								                    \
-  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++;\
-  if((!local)&&(!st.same_node[Dir]) ) {					    \
-    LOAD_CHI(base);							                \
-    MULT_2SPIN_1(Dir);					                    \
-    PREFETCH_CHIMU(base);                                   \
-    /* PREFETCH_GAUGE_L1(NxtDir); */                        \
-    MULT_2SPIN_2;					                        \
-    if (s == 0) {                                           \
-      if ((Dir == 0) || (Dir == 4)) { PREFETCH_GAUGE_L2(Dir); } \
-    }                                                       \
-    RECON;								                    \
-    nmu++;								                    \
-  }
-
-#define RESULT(base,basep) if (nmu){ ADD_RESULT(base,base);}
-
-#endif
-{
-  int nmu;
-  int local,perm, ptype;
-  uint64_t base;
-  uint64_t basep;
-  const uint64_t plocal =(uint64_t) & in[0];
-
-  MASK_REGS;
-  int nmax=U.oSites();
-  for(int site=0;site<Ns;site++) {
-#ifndef EXTERIOR
-    //    int sU =lo.Reorder(ssU);
-    int sU =ssU;
-    int ssn=ssU+1;     if(ssn>=nmax) ssn=0;
-    //    int sUn=lo.Reorder(ssn);
-    int sUn=ssn;
-    LOCK_GAUGE(0);
-#else
-    int sU =ssU;
-    int ssn=ssU+1;     if(ssn>=nmax) ssn=0;
-    int sUn=ssn;
-#endif
-    for(int s=0;s<Ls;s++) {
-      ss =sU*Ls+s;
-      ssn=sUn*Ls+s;
-      int  ent=ss*8;// 2*Ndim
-      int nent=ssn*8;
-
-      uint64_t delta_base, delta_base_p;
-
-   ASM_LEG_XP(Xp,Yp,PERMUTE_DIR3,DIR0_PROJ,DIR0_RECON);
-
-#ifdef SHOW
-      float rescale = 64. * 12.;
-      std::cout << "=================================================================" << std::endl;
-      std::cout << "ss = " << ss << "   ssn = " << ssn << std::endl;
-      std::cout << "sU = " << sU << "   ssU = " << ssU << std::endl;
-      std::cout << " " << std::endl;
-
-
-      std::cout << "Dir = " << Xp << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Xp] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-      ASM_LEG(Yp,Zp,PERMUTE_DIR2,DIR1_PROJ,DIR1_RECON);
-
-#ifdef SHOW
-      std::cout << "Dir = " << Yp << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Yp] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-      ASM_LEG(Zp,Tp,PERMUTE_DIR1,DIR2_PROJ,DIR2_RECON);
-
-#ifdef SHOW
-      std::cout << "Dir = " << Zp << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Zp] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-      ASM_LEG(Tp,Xm,PERMUTE_DIR0,DIR3_PROJ,DIR3_RECON);
-
-#ifdef SHOW
-      std::cout << "Dir = " << Tp << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Tp] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-      ASM_LEG(Xm,Ym,PERMUTE_DIR3,DIR4_PROJ,DIR4_RECON);
-
-#ifdef SHOW
-      std::cout << "Dir = " << Xm << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Xm] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-      ASM_LEG(Ym,Zm,PERMUTE_DIR2,DIR5_PROJ,DIR5_RECON);
-
-#ifdef SHOW
-      std::cout << "Dir = " << Ym << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Ym] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-      ASM_LEG(Zm,Tm,PERMUTE_DIR1,DIR6_PROJ,DIR6_RECON);
-
-#ifdef SHOW
-      std::cout << "Dir = " << Zm << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Zm] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-      ASM_LEG(Tm,Xp,PERMUTE_DIR0,DIR7_PROJ,DIR7_RECON);
-
-#ifdef SHOW
-      std::cout << "Dir = " << Tm << "        "  << WHERE<< std::endl;
-
-      std::cout << "ent  nent  local  perm       = " << ent << "  " << nent << "  " << local << "  "  << perm << std::endl;
-      std::cout << "st.same_node[Dir] = " << st.same_node[Tm] << std::endl;
-      std::cout << "base              = " << (base - plocal)/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-#ifdef EXTERIOR
-      if (nmu==0) break;
-      //      if (nmu!=0) std::cout << "EXT "<<sU<<std::endl;
-#endif
-      base = (uint64_t) &out[ss];
-      basep= st.GetPFInfo(nent,plocal); ent++;
-      basep = (uint64_t) &out[ssn];
-      RESULT(base,basep);
-
-#ifdef SHOW
-      std::cout << "Dir = FINAL        " <<  WHERE<< std::endl;;
-
-      base_ss = base;
-      std::cout << "base              = " << (base - (uint64_t) &out[0])/rescale << std::endl;
-      std::cout << "Basep             = " << (basep - plocal)/rescale << std::endl;
-      //printf("U                 = %llu\n", (uint64_t)&[sU](Dir));
-      std::cout << "----------------------------------------------------" << std::endl;
-#endif
-
-    }
-    ssU++;
-    UNLOCK_GAUGE(0);
-  }
-}
-
-#undef DIR0_PROJ
-#undef DIR1_PROJ
-#undef DIR2_PROJ
-#undef DIR3_PROJ
-#undef DIR4_PROJ
-#undef DIR5_PROJ
-#undef DIR6_PROJ
-#undef DIR7_PROJ
-#undef DIR0_RECON
-#undef DIR1_RECON
-#undef DIR2_RECON
-#undef DIR3_RECON
-#undef DIR4_RECON
-#undef DIR5_RECON
-#undef DIR6_RECON
-#undef DIR7_RECON
-#undef ASM_LEG
-#undef ASM_LEG_XP
-#undef RESULT

Grid/qcd/action/fermion/implementation/WilsonKernelsHandGparityImplementation.h

@@ -646,7 +646,7 @@ NAMESPACE_BEGIN(Grid);
   HAND_RESULT_EXT(ss,F)
 
 #define HAND_SPECIALISE_GPARITY(IMPL)					\
-  template<> accelerator_inline void						\
+  template<> void						\
   WilsonKernels<IMPL>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor  *buf, \
 				    int ss,int sU,const FermionFieldView &in, FermionFieldView &out) \
   {									\
@@ -662,7 +662,7 @@ NAMESPACE_BEGIN(Grid);
     HAND_DOP_SITE(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
   }									\
 									\
-  template<> accelerator_inline void						\
+  template<> void						\
   WilsonKernels<IMPL>::HandDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf, \
 				       int ss,int sU,const FermionFieldView &in, FermionFieldView &out) \
   {									\
@@ -678,7 +678,7 @@ NAMESPACE_BEGIN(Grid);
     HAND_DOP_SITE_DAG(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
   }									\
 									\
-  template<> accelerator_inline void						\
+  template<> void						\
   WilsonKernels<IMPL>::HandDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor  *buf, \
 				       int ss,int sU,const FermionFieldView &in, FermionFieldView &out) \
   {									\
@@ -694,7 +694,7 @@ NAMESPACE_BEGIN(Grid);
     HAND_DOP_SITE_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
   }									\
 									\
-  template<> accelerator_inline void						\
+  template<> void						\
   WilsonKernels<IMPL>::HandDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf, \
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out) \
   {									\
@@ -710,7 +710,7 @@ NAMESPACE_BEGIN(Grid);
     HAND_DOP_SITE_DAG_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
   }									\
 									\
-  template<> accelerator_inline void							\
+  template<> void							\
   WilsonKernels<IMPL>::HandDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor  *buf, \
 				       int ss,int sU,const FermionFieldView &in, FermionFieldView &out) \
   {									\
@@ -727,7 +727,7 @@ NAMESPACE_BEGIN(Grid);
     nmu = 0;								\
     HAND_DOP_SITE_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
   }									\
-  template<> accelerator_inline void						\
+  template<> void						\
   WilsonKernels<IMPL>::HandDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf, \
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out) \
   {									\

Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h

@@ -495,7 +495,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 NAMESPACE_BEGIN(Grid);
 
-template<class Impl> accelerator_inline void 
+template<class Impl> void 
 WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 				  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
@@ -519,7 +519,7 @@ WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,Site
   HAND_RESULT(ss);
 }
 
-template<class Impl>  accelerator_inline
+template<class Impl>
 void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
@@ -542,7 +542,7 @@ void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView
   HAND_RESULT(ss);
 }
 
-template<class Impl>  accelerator_inline void 
+template<class Impl> void 
 WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
@@ -566,7 +566,7 @@ WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,Si
   HAND_RESULT(ss);
 }
 
-template<class Impl> accelerator_inline
+template<class Impl>
 void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
@@ -589,7 +589,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldVi
   HAND_RESULT(ss);
 }
 
-template<class Impl>  accelerator_inline void 
+template<class Impl> void 
 WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
@@ -614,7 +614,7 @@ WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,Si
   HAND_RESULT_EXT(ss);
 }
 
-template<class Impl>  accelerator_inline
+template<class Impl>
 void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {

Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h.debug

@@ -1,943 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: ./lib/qcd/action/fermion/WilsonKernelsHand.cc
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-
-#pragma once
-
-#include <Grid/qcd/action/fermion/FermionCore.h>
-
-
-#undef LOAD_CHIMU
-#undef LOAD_CHI
-#undef MULT_2SPIN
-#undef PERMUTE_DIR
-#undef XP_PROJ
-#undef YP_PROJ
-#undef ZP_PROJ
-#undef TP_PROJ
-#undef XM_PROJ
-#undef YM_PROJ
-#undef ZM_PROJ
-#undef TM_PROJ
-#undef XP_RECON
-#undef XP_RECON_ACCUM
-#undef XM_RECON
-#undef XM_RECON_ACCUM
-#undef YP_RECON_ACCUM
-#undef YM_RECON_ACCUM
-#undef ZP_RECON_ACCUM
-#undef ZM_RECON_ACCUM
-#undef TP_RECON_ACCUM
-#undef TM_RECON_ACCUM
-#undef ZERO_RESULT
-#undef Chimu_00
-#undef Chimu_01
-#undef Chimu_02
-#undef Chimu_10
-#undef Chimu_11
-#undef Chimu_12
-#undef Chimu_20
-#undef Chimu_21
-#undef Chimu_22
-#undef Chimu_30
-#undef Chimu_31
-#undef Chimu_32
-#undef HAND_STENCIL_LEG
-#undef HAND_STENCIL_LEG_INT
-#undef HAND_STENCIL_LEG_EXT
-#undef HAND_RESULT
-#undef HAND_RESULT_INT
-#undef HAND_RESULT_EXT
-
-#define REGISTER
-
-#define LOAD_CHIMU \
-  {const SiteSpinor & ref (in[offset]);	\
-    Chimu_00=ref()(0)(0);\
-    Chimu_01=ref()(0)(1);\
-    Chimu_02=ref()(0)(2);\
-    Chimu_10=ref()(1)(0);\
-    Chimu_11=ref()(1)(1);\
-    Chimu_12=ref()(1)(2);\
-    Chimu_20=ref()(2)(0);\
-    Chimu_21=ref()(2)(1);\
-    Chimu_22=ref()(2)(2);\
-    Chimu_30=ref()(3)(0);\
-    Chimu_31=ref()(3)(1);\
-    Chimu_32=ref()(3)(2);\
-    std::cout << std::endl << "DEBUG -- LOAD_CHIMU" << std::endl; \
-    std::cout << "Chimu_00 -- " <<  Chimu_00 << std::endl; \
-    std::cout << "Chimu_01 -- " <<  Chimu_01 << std::endl; \
-    std::cout << "Chimu_02 -- " <<  Chimu_02 << std::endl; \
-    std::cout << "Chimu_10 -- " <<  Chimu_10 << std::endl; \
-    std::cout << "Chimu_11 -- " <<  Chimu_11 << std::endl; \
-    std::cout << "Chimu_12 -- " <<  Chimu_12 << std::endl; \
-    std::cout << "Chimu_20 -- " <<  Chimu_20 << std::endl; \
-    std::cout << "Chimu_21 -- " <<  Chimu_21 << std::endl; \
-    std::cout << "Chimu_22 -- " <<  Chimu_22 << std::endl; \
-    std::cout << "Chimu_30 -- " <<  Chimu_30 << std::endl; \
-    std::cout << "Chimu_31 -- " <<  Chimu_31 << std::endl; \
-    std::cout << "Chimu_32 -- " <<  Chimu_32 << std::endl; \
-}
-
-#define LOAD_CHI\
-  {const SiteHalfSpinor &ref(buf[offset]);	\
-    Chi_00 = ref()(0)(0);\
-    Chi_01 = ref()(0)(1);\
-    Chi_02 = ref()(0)(2);\
-    Chi_10 = ref()(1)(0);\
-    Chi_11 = ref()(1)(1);\
-    Chi_12 = ref()(1)(2);\
-    std::cout << std::endl << "DEBUG -- LOAD_CHI" << std::endl; \
-    std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-    std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-    std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-    std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-    std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-    std::cout << "Chi_12 -- " <<  Chi_12 << std::endl; \
-  }
-
-// To splat or not to splat depends on the implementation
-#define MULT_2SPIN(A)\
-  {auto & ref(U[sU](A));			\
-   Impl::loadLinkElement(U_00,ref()(0,0));	\
-   Impl::loadLinkElement(U_10,ref()(1,0));	\
-   Impl::loadLinkElement(U_20,ref()(2,0));	\
-   Impl::loadLinkElement(U_01,ref()(0,1));	\
-   Impl::loadLinkElement(U_11,ref()(1,1));	\
-   Impl::loadLinkElement(U_21,ref()(2,1));	\
-    UChi_00 = U_00*Chi_00;\
-    UChi_10 = U_00*Chi_10;\
-    UChi_01 = U_10*Chi_00;\
-    UChi_11 = U_10*Chi_10;\
-    UChi_02 = U_20*Chi_00;\
-    UChi_12 = U_20*Chi_10;\
-    UChi_00+= U_01*Chi_01;\
-    UChi_10+= U_01*Chi_11;\
-    UChi_01+= U_11*Chi_01;\
-    UChi_11+= U_11*Chi_11;\
-    UChi_02+= U_21*Chi_01;\
-    UChi_12+= U_21*Chi_11;\
-    Impl::loadLinkElement(U_00,ref()(0,2));	\
-    Impl::loadLinkElement(U_10,ref()(1,2));	\
-    Impl::loadLinkElement(U_20,ref()(2,2));	\
-    UChi_00+= U_00*Chi_02;\
-    UChi_10+= U_00*Chi_12;\
-    UChi_01+= U_10*Chi_02;\
-    UChi_11+= U_10*Chi_12;\
-    UChi_02+= U_20*Chi_02;\
-    UChi_12+= U_20*Chi_12;\
-    std::cout << std::endl << "DEBUG -- MULT_2SPIN" << std::endl; \
-    std::cout << "UChi_00 -- " <<  UChi_00 << std::endl; \
-    std::cout << "UChi_01 -- " <<  UChi_01 << std::endl; \
-    std::cout << "UChi_02 -- " <<  UChi_02 << std::endl; \
-    std::cout << "UChi_10 -- " <<  UChi_10 << std::endl; \
-    std::cout << "UChi_11 -- " <<  UChi_11 << std::endl; \
-    std::cout << "UChi_12 -- " <<  UChi_12 << std::endl; \
-    }
-
-
-#define PERMUTE_DIR(dir)			\
-std::cout << std::endl << "DEBUG -- PERM PRE" << std::endl; \
-std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-std::cout << "Chi_12 -- " <<  Chi_12 << std::endl; \
-      permute##dir(Chi_00,Chi_00);\
-      permute##dir(Chi_01,Chi_01);\
-      permute##dir(Chi_02,Chi_02);\
-      permute##dir(Chi_10,Chi_10);\
-      permute##dir(Chi_11,Chi_11);\
-      permute##dir(Chi_12,Chi_12);\
-  std::cout << std::endl << "DEBUG -- PERM POST" << std::endl; \
-  std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-  std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-  std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-  std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-  std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-  std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-//      hspin(0)=fspin(0)+timesI(fspin(3));
-//      hspin(1)=fspin(1)+timesI(fspin(2));
-#define XP_PROJ \
-    Chi_00 = Chimu_00+timesI(Chimu_30);\
-    Chi_01 = Chimu_01+timesI(Chimu_31);\
-    Chi_02 = Chimu_02+timesI(Chimu_32);\
-    Chi_10 = Chimu_10+timesI(Chimu_20);\
-    Chi_11 = Chimu_11+timesI(Chimu_21);\
-    Chi_12 = Chimu_12+timesI(Chimu_22);\
-    std::cout << std::endl << "DEBUG -- XP_PROJ" << std::endl; \
-    std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-    std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-    std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-    std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-    std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-    std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-#define YP_PROJ \
-    Chi_00 = Chimu_00-Chimu_30;\
-    Chi_01 = Chimu_01-Chimu_31;\
-    Chi_02 = Chimu_02-Chimu_32;\
-    Chi_10 = Chimu_10+Chimu_20;\
-    Chi_11 = Chimu_11+Chimu_21;\
-    Chi_12 = Chimu_12+Chimu_22;\
-    std::cout << std::endl << "DEBUG -- YP_PROJ" << std::endl; \
-    std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-    std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-    std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-    std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-    std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-    std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-#define ZP_PROJ \
-  Chi_00 = Chimu_00+timesI(Chimu_20);		\
-  Chi_01 = Chimu_01+timesI(Chimu_21);		\
-  Chi_02 = Chimu_02+timesI(Chimu_22);		\
-  Chi_10 = Chimu_10-timesI(Chimu_30);		\
-  Chi_11 = Chimu_11-timesI(Chimu_31);		\
-  Chi_12 = Chimu_12-timesI(Chimu_32);\
-  std::cout << std::endl << "DEBUG -- ZP_PROJ" << std::endl; \
-  std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-  std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-  std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-  std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-  std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-  std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-#define TP_PROJ \
-  Chi_00 = Chimu_00+Chimu_20;		\
-  Chi_01 = Chimu_01+Chimu_21;		\
-  Chi_02 = Chimu_02+Chimu_22;		\
-  Chi_10 = Chimu_10+Chimu_30;		\
-  Chi_11 = Chimu_11+Chimu_31;		\
-  Chi_12 = Chimu_12+Chimu_32;\
-  std::cout << std::endl << "DEBUG -- TP_PROJ" << std::endl; \
-  std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-  std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-  std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-  std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-  std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-  std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-
-//      hspin(0)=fspin(0)-timesI(fspin(3));
-//      hspin(1)=fspin(1)-timesI(fspin(2));
-#define XM_PROJ \
-    Chi_00 = Chimu_00-timesI(Chimu_30);\
-    Chi_01 = Chimu_01-timesI(Chimu_31);\
-    Chi_02 = Chimu_02-timesI(Chimu_32);\
-    Chi_10 = Chimu_10-timesI(Chimu_20);\
-    Chi_11 = Chimu_11-timesI(Chimu_21);\
-    Chi_12 = Chimu_12-timesI(Chimu_22);\
-    std::cout << std::endl << "DEBUG -- XM_PROJ" << std::endl; \
-    std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-    std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-    std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-    std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-    std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-    std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-#define YM_PROJ \
-    Chi_00 = Chimu_00+Chimu_30;\
-    Chi_01 = Chimu_01+Chimu_31;\
-    Chi_02 = Chimu_02+Chimu_32;\
-    Chi_10 = Chimu_10-Chimu_20;\
-    Chi_11 = Chimu_11-Chimu_21;\
-    Chi_12 = Chimu_12-Chimu_22;\
-    std::cout << std::endl << "DEBUG -- YM_PROJ" << std::endl; \
-    std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-    std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-    std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-    std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-    std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-    std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-#define ZM_PROJ \
-  Chi_00 = Chimu_00-timesI(Chimu_20);		\
-  Chi_01 = Chimu_01-timesI(Chimu_21);		\
-  Chi_02 = Chimu_02-timesI(Chimu_22);		\
-  Chi_10 = Chimu_10+timesI(Chimu_30);		\
-  Chi_11 = Chimu_11+timesI(Chimu_31);		\
-  Chi_12 = Chimu_12+timesI(Chimu_32);\
-  std::cout << std::endl << "DEBUG -- ZM_PROJ" << std::endl; \
-  std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-  std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-  std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-  std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-  std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-  std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-#define TM_PROJ \
-  Chi_00 = Chimu_00-Chimu_20;		\
-  Chi_01 = Chimu_01-Chimu_21;		\
-  Chi_02 = Chimu_02-Chimu_22;		\
-  Chi_10 = Chimu_10-Chimu_30;		\
-  Chi_11 = Chimu_11-Chimu_31;		\
-  Chi_12 = Chimu_12-Chimu_32;\
-  std::cout << std::endl << "DEBUG -- TM_PROJ" << std::endl; \
-  std::cout << "Chi_00 -- " <<  Chi_00 << std::endl; \
-  std::cout << "Chi_01 -- " <<  Chi_01 << std::endl; \
-  std::cout << "Chi_02 -- " <<  Chi_02 << std::endl; \
-  std::cout << "Chi_10 -- " <<  Chi_10 << std::endl; \
-  std::cout << "Chi_11 -- " <<  Chi_11 << std::endl; \
-  std::cout << "Chi_12 -- " <<  Chi_12 << std::endl;
-
-//      fspin(0)=hspin(0);
-//      fspin(1)=hspin(1);
-//      fspin(2)=timesMinusI(hspin(1));
-//      fspin(3)=timesMinusI(hspin(0));
-#define XP_RECON\
-  result_00 = UChi_00;\
-  result_01 = UChi_01;\
-  result_02 = UChi_02;\
-  result_10 = UChi_10;\
-  result_11 = UChi_11;\
-  result_12 = UChi_12;\
-  result_20 = timesMinusI(UChi_10);\
-  result_21 = timesMinusI(UChi_11);\
-  result_22 = timesMinusI(UChi_12);\
-  result_30 = timesMinusI(UChi_00);\
-  result_31 = timesMinusI(UChi_01);\
-  result_32 = timesMinusI(UChi_02);\
-  std::cout << std::endl << "DEBUG -- XP_RECON" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define XP_RECON_ACCUM\
-  result_00+=UChi_00;\
-  result_01+=UChi_01;\
-  result_02+=UChi_02;\
-  result_10+=UChi_10;\
-  result_11+=UChi_11;\
-  result_12+=UChi_12;\
-  result_20-=timesI(UChi_10);\
-  result_21-=timesI(UChi_11);\
-  result_22-=timesI(UChi_12);\
-  result_30-=timesI(UChi_00);\
-  result_31-=timesI(UChi_01);\
-  result_32-=timesI(UChi_02);\
-  std::cout << std::endl << "DEBUG -- XP_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define XM_RECON\
-  result_00 = UChi_00;\
-  result_01 = UChi_01;\
-  result_02 = UChi_02;\
-  result_10 = UChi_10;\
-  result_11 = UChi_11;\
-  result_12 = UChi_12;\
-  result_20 = timesI(UChi_10);\
-  result_21 = timesI(UChi_11);\
-  result_22 = timesI(UChi_12);\
-  result_30 = timesI(UChi_00);\
-  result_31 = timesI(UChi_01);\
-  result_32 = timesI(UChi_02);\
-  std::cout << std::endl << "DEBUG -- XM_RECON" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define XM_RECON_ACCUM\
-  result_00+= UChi_00;\
-  result_01+= UChi_01;\
-  result_02+= UChi_02;\
-  result_10+= UChi_10;\
-  result_11+= UChi_11;\
-  result_12+= UChi_12;\
-  result_20+= timesI(UChi_10);\
-  result_21+= timesI(UChi_11);\
-  result_22+= timesI(UChi_12);\
-  result_30+= timesI(UChi_00);\
-  result_31+= timesI(UChi_01);\
-  result_32+= timesI(UChi_02);\
-  std::cout << std::endl << "DEBUG -- XM_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define YP_RECON_ACCUM\
-  result_00+= UChi_00;\
-  result_01+= UChi_01;\
-  result_02+= UChi_02;\
-  result_10+= UChi_10;\
-  result_11+= UChi_11;\
-  result_12+= UChi_12;\
-  result_20+= UChi_10;\
-  result_21+= UChi_11;\
-  result_22+= UChi_12;\
-  result_30-= UChi_00;\
-  result_31-= UChi_01;\
-  result_32-= UChi_02;\
-  std::cout << std::endl << "DEBUG -- YP_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define YM_RECON_ACCUM\
-  result_00+= UChi_00;\
-  result_01+= UChi_01;\
-  result_02+= UChi_02;\
-  result_10+= UChi_10;\
-  result_11+= UChi_11;\
-  result_12+= UChi_12;\
-  result_20-= UChi_10;\
-  result_21-= UChi_11;\
-  result_22-= UChi_12;\
-  result_30+= UChi_00;\
-  result_31+= UChi_01;\
-  result_32+= UChi_02;\
-  std::cout << std::endl << "DEBUG -- YM_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define ZP_RECON_ACCUM\
-  result_00+= UChi_00;\
-  result_01+= UChi_01;\
-  result_02+= UChi_02;\
-  result_10+= UChi_10;\
-  result_11+= UChi_11;\
-  result_12+= UChi_12;\
-  result_20-= timesI(UChi_00);			\
-  result_21-= timesI(UChi_01);			\
-  result_22-= timesI(UChi_02);			\
-  result_30+= timesI(UChi_10);			\
-  result_31+= timesI(UChi_11);			\
-  result_32+= timesI(UChi_12);\
-  std::cout << std::endl << "DEBUG -- ZP_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define ZM_RECON_ACCUM\
-  result_00+= UChi_00;\
-  result_01+= UChi_01;\
-  result_02+= UChi_02;\
-  result_10+= UChi_10;\
-  result_11+= UChi_11;\
-  result_12+= UChi_12;\
-  result_20+= timesI(UChi_00);			\
-  result_21+= timesI(UChi_01);			\
-  result_22+= timesI(UChi_02);			\
-  result_30-= timesI(UChi_10);			\
-  result_31-= timesI(UChi_11);			\
-  result_32-= timesI(UChi_12);\
-  std::cout << std::endl << "DEBUG -- ZM_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define TP_RECON_ACCUM\
-  result_00+= UChi_00;\
-  result_01+= UChi_01;\
-  result_02+= UChi_02;\
-  result_10+= UChi_10;\
-  result_11+= UChi_11;\
-  result_12+= UChi_12;\
-  result_20+= UChi_00;			\
-  result_21+= UChi_01;			\
-  result_22+= UChi_02;			\
-  result_30+= UChi_10;			\
-  result_31+= UChi_11;			\
-  result_32+= UChi_12;\
-  std::cout << std::endl << "DEBUG -- TP_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define TM_RECON_ACCUM\
-  result_00+= UChi_00;\
-  result_01+= UChi_01;\
-  result_02+= UChi_02;\
-  result_10+= UChi_10;\
-  result_11+= UChi_11;\
-  result_12+= UChi_12;\
-  result_20-= UChi_00;	\
-  result_21-= UChi_01;	\
-  result_22-= UChi_02;	\
-  result_30-= UChi_10;	\
-  result_31-= UChi_11;	\
-  result_32-= UChi_12;\
-  std::cout << std::endl << "DEBUG -- TM_RECON_ACCUM" << std::endl; \
-  std::cout << "result_00 -- " <<  result_00 << std::endl; \
-  std::cout << "result_01 -- " <<  result_01 << std::endl; \
-  std::cout << "result_02 -- " <<  result_02 << std::endl; \
-  std::cout << "result_10 -- " <<  result_10 << std::endl; \
-  std::cout << "result_11 -- " <<  result_11 << std::endl; \
-  std::cout << "result_12 -- " <<  result_12 << std::endl; \
-  std::cout << "result_20 -- " <<  result_20 << std::endl; \
-  std::cout << "result_21 -- " <<  result_21 << std::endl; \
-  std::cout << "result_22 -- " <<  result_22 << std::endl; \
-  std::cout << "result_30 -- " <<  result_30 << std::endl; \
-  std::cout << "result_31 -- " <<  result_31 << std::endl; \
-  std::cout << "result_32 -- " <<  result_32 << std::endl;
-
-#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON)	\
-  SE=st.GetEntry(ptype,DIR,ss);			\
-  offset = SE->_offset;				\
-  local  = SE->_is_local;			\
-  perm   = SE->_permute;			\
-  if ( local ) {				\
-    LOAD_CHIMU;					\
-    PROJ;					\
-    if ( perm) {				\
-      PERMUTE_DIR(PERM);			\
-    }						\
-  } else {					\
-    LOAD_CHI;					\
-  }						\
-  MULT_2SPIN(DIR);				\
-  RECON;
-
-#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON)	\
-  SE=st.GetEntry(ptype,DIR,ss);			\
-  offset = SE->_offset;				\
-  local  = SE->_is_local;			\
-  perm   = SE->_permute;			\
-  if ( local ) {				\
-    LOAD_CHIMU;					\
-    PROJ;					\
-    if ( perm) {				\
-      PERMUTE_DIR(PERM);			\
-    }						\
-  } else if ( st.same_node[DIR] ) {		\
-    LOAD_CHI;					\
-  }						\
-  if (local || st.same_node[DIR] ) {		\
-    MULT_2SPIN(DIR);				\
-    RECON;					\
-  }
-
-#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON)	\
-  SE=st.GetEntry(ptype,DIR,ss);			\
-  offset = SE->_offset;				\
-  if((!SE->_is_local)&&(!st.same_node[DIR]) ) {	\
-    LOAD_CHI;					\
-    MULT_2SPIN(DIR);				\
-    RECON;					\
-    nmu++;					\
-  }
-
-#define HAND_RESULT(ss)				\
-  {						\
-    SiteSpinor & ref (out[ss]);		\
-    vstream(ref()(0)(0),result_00);		\
-    vstream(ref()(0)(1),result_01);		\
-    vstream(ref()(0)(2),result_02);		\
-    vstream(ref()(1)(0),result_10);		\
-    vstream(ref()(1)(1),result_11);		\
-    vstream(ref()(1)(2),result_12);		\
-    vstream(ref()(2)(0),result_20);		\
-    vstream(ref()(2)(1),result_21);		\
-    vstream(ref()(2)(2),result_22);		\
-    vstream(ref()(3)(0),result_30);		\
-    vstream(ref()(3)(1),result_31);		\
-    vstream(ref()(3)(2),result_32);		\
-    std::cout << std::endl << "DEBUG -- RESULT" << std::endl; \
-    std::cout << "result_00 -- " <<  result_00 << std::endl; \
-    std::cout << "result_01 -- " <<  result_01 << std::endl; \
-    std::cout << "result_02 -- " <<  result_02 << std::endl; \
-    std::cout << "result_10 -- " <<  result_10 << std::endl; \
-    std::cout << "result_11 -- " <<  result_11 << std::endl; \
-    std::cout << "result_12 -- " <<  result_12 << std::endl; \
-    std::cout << "result_20 -- " <<  result_20 << std::endl; \
-    std::cout << "result_21 -- " <<  result_21 << std::endl; \
-    std::cout << "result_22 -- " <<  result_22 << std::endl; \
-    std::cout << "result_30 -- " <<  result_30 << std::endl; \
-    std::cout << "result_31 -- " <<  result_31 << std::endl; \
-    std::cout << "result_32 -- " <<  result_32 << std::endl;\
-  }
-
-#define HAND_RESULT_EXT(ss)			\
-  if (nmu){					\
-    SiteSpinor & ref (out[ss]);		\
-    ref()(0)(0)+=result_00;		\
-    ref()(0)(1)+=result_01;		\
-    ref()(0)(2)+=result_02;		\
-    ref()(1)(0)+=result_10;		\
-    ref()(1)(1)+=result_11;		\
-    ref()(1)(2)+=result_12;		\
-    ref()(2)(0)+=result_20;		\
-    ref()(2)(1)+=result_21;		\
-    ref()(2)(2)+=result_22;		\
-    ref()(3)(0)+=result_30;		\
-    ref()(3)(1)+=result_31;		\
-    ref()(3)(2)+=result_32;		\
-    std::cout << std::endl << "DEBUG -- RESULT EXT" << std::endl; \
-    std::cout << "result_00 -- " <<  result_00 << std::endl; \
-    std::cout << "result_01 -- " <<  result_01 << std::endl; \
-    std::cout << "result_02 -- " <<  result_02 << std::endl; \
-    std::cout << "result_10 -- " <<  result_10 << std::endl; \
-    std::cout << "result_11 -- " <<  result_11 << std::endl; \
-    std::cout << "result_12 -- " <<  result_12 << std::endl; \
-    std::cout << "result_20 -- " <<  result_20 << std::endl; \
-    std::cout << "result_21 -- " <<  result_21 << std::endl; \
-    std::cout << "result_22 -- " <<  result_22 << std::endl; \
-    std::cout << "result_30 -- " <<  result_30 << std::endl; \
-    std::cout << "result_31 -- " <<  result_31 << std::endl; \
-    std::cout << "result_32 -- " <<  result_32 << std::endl;\
-  }
-
-
-#define HAND_DECLARATIONS(a)			\
-  Simd result_00;				\
-  Simd result_01;				\
-  Simd result_02;				\
-  Simd result_10;				\
-  Simd result_11;				\
-  Simd result_12;				\
-  Simd result_20;				\
-  Simd result_21;				\
-  Simd result_22;				\
-  Simd result_30;				\
-  Simd result_31;				\
-  Simd result_32;				\
-  Simd Chi_00;					\
-  Simd Chi_01;					\
-  Simd Chi_02;					\
-  Simd Chi_10;					\
-  Simd Chi_11;					\
-  Simd Chi_12;					\
-  Simd UChi_00;					\
-  Simd UChi_01;					\
-  Simd UChi_02;					\
-  Simd UChi_10;					\
-  Simd UChi_11;					\
-  Simd UChi_12;					\
-  Simd U_00;					\
-  Simd U_10;					\
-  Simd U_20;					\
-  Simd U_01;					\
-  Simd U_11;					\
-  Simd U_21;\
-  Simd debugreg;\
-  svbool_t pg1;        \
-  pg1 = svptrue_b64();        \
-
-#define ZERO_RESULT				\
-  result_00=Zero();				\
-  result_01=Zero();				\
-  result_02=Zero();				\
-  result_10=Zero();				\
-  result_11=Zero();				\
-  result_12=Zero();				\
-  result_20=Zero();				\
-  result_21=Zero();				\
-  result_22=Zero();				\
-  result_30=Zero();				\
-  result_31=Zero();				\
-  result_32=Zero();
-
-#define Chimu_00 Chi_00
-#define Chimu_01 Chi_01
-#define Chimu_02 Chi_02
-#define Chimu_10 Chi_10
-#define Chimu_11 Chi_11
-#define Chimu_12 Chi_12
-#define Chimu_20 UChi_00
-#define Chimu_21 UChi_01
-#define Chimu_22 UChi_02
-#define Chimu_30 UChi_10
-#define Chimu_31 UChi_11
-#define Chimu_32 UChi_12
-
-NAMESPACE_BEGIN(Grid);
-
-template<class Impl> void
-WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
-				  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
-{
-// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  int offset,local,perm, ptype;
-  StencilEntry *SE;
-
-  HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
-  HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-  HAND_RESULT(ss);
-}
-
-template<class Impl>
-void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
-					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
-{
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  StencilEntry *SE;
-  int offset,local,perm, ptype;
-
-  HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON);
-  HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG(TP_PROJ,0,Tp,TP_RECON_ACCUM);
-  HAND_STENCIL_LEG(XM_PROJ,3,Xm,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG(YM_PROJ,2,Ym,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG(TM_PROJ,0,Tm,TM_RECON_ACCUM);
-  HAND_RESULT(ss);
-}
-
-template<class Impl> void
-WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
-					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
-{
-// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  int offset,local,perm, ptype;
-  StencilEntry *SE;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-  HAND_RESULT(ss);
-}
-
-template<class Impl>
-void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
-						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
-{
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  StencilEntry *SE;
-  int offset,local,perm, ptype;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
-  HAND_RESULT(ss);
-}
-
-template<class Impl> void
-WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
-					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
-{
-// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  int offset, ptype;
-  StencilEntry *SE;
-  int nmu=0;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-  HAND_RESULT_EXT(ss);
-}
-
-template<class Impl>
-void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
-						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
-{
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  StencilEntry *SE;
-  int offset, ptype;
-  int nmu=0;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
-  HAND_RESULT_EXT(ss);
-}
-
-////////////// Wilson ; uses this implementation /////////////////////
-
-NAMESPACE_END(Grid);
-#undef LOAD_CHIMU
-#undef LOAD_CHI
-#undef MULT_2SPIN
-#undef PERMUTE_DIR
-#undef XP_PROJ
-#undef YP_PROJ
-#undef ZP_PROJ
-#undef TP_PROJ
-#undef XM_PROJ
-#undef YM_PROJ
-#undef ZM_PROJ
-#undef TM_PROJ
-#undef XP_RECON
-#undef XP_RECON_ACCUM
-#undef XM_RECON
-#undef XM_RECON_ACCUM
-#undef YP_RECON_ACCUM
-#undef YM_RECON_ACCUM
-#undef ZP_RECON_ACCUM
-#undef ZM_RECON_ACCUM
-#undef TP_RECON_ACCUM
-#undef TM_RECON_ACCUM
-#undef ZERO_RESULT
-#undef Chimu_00
-#undef Chimu_01
-#undef Chimu_02
-#undef Chimu_10
-#undef Chimu_11
-#undef Chimu_12
-#undef Chimu_20
-#undef Chimu_21
-#undef Chimu_22
-#undef Chimu_30
-#undef Chimu_31
-#undef Chimu_32
-#undef HAND_STENCIL_LEG
-#undef HAND_STENCIL_LEG_INT
-#undef HAND_STENCIL_LEG_EXT
-#undef HAND_RESULT
-#undef HAND_RESULT_INT
-#undef HAND_RESULT_EXT

Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h

@@ -114,7 +114,7 @@ accelerator_inline void get_stencil(StencilEntry * mem, StencilEntry &chip)
   ////////////////////////////////////////////////////////////////////
   // All legs kernels ; comms then compute
   ////////////////////////////////////////////////////////////////////
-template <class Impl> accelerator_inline
+template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U,
 					     SiteHalfSpinor *buf, int sF,
 					     int sU, const FermionFieldView &in, FermionFieldView &out)
@@ -140,7 +140,7 @@ void WilsonKernels<Impl>::GenericDhopSiteDag(StencilView &st, DoubledGaugeFieldV
   coalescedWrite(out[sF],result,lane);
 };
 
-template <class Impl> accelerator_inline
+template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSite(StencilView &st, DoubledGaugeFieldView &U,
 					  SiteHalfSpinor *buf, int sF,
 					  int sU, const FermionFieldView &in, FermionFieldView &out) 
@@ -169,7 +169,7 @@ void WilsonKernels<Impl>::GenericDhopSite(StencilView &st, DoubledGaugeFieldView
   ////////////////////////////////////////////////////////////////////
   // Interior kernels
   ////////////////////////////////////////////////////////////////////
-template <class Impl> accelerator_inline
+template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSiteDagInt(StencilView &st,  DoubledGaugeFieldView &U,
 						SiteHalfSpinor *buf, int sF,
 						int sU, const FermionFieldView &in, FermionFieldView &out)
@@ -197,7 +197,7 @@ void WilsonKernels<Impl>::GenericDhopSiteDagInt(StencilView &st,  DoubledGaugeFi
   coalescedWrite(out[sF], result,lane);
 };
 
-template <class Impl> accelerator_inline
+template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSiteInt(StencilView &st,  DoubledGaugeFieldView &U,
 							 SiteHalfSpinor *buf, int sF,
 							 int sU, const FermionFieldView &in, FermionFieldView &out) 
@@ -227,7 +227,7 @@ void WilsonKernels<Impl>::GenericDhopSiteInt(StencilView &st,  DoubledGaugeField
 ////////////////////////////////////////////////////////////////////
 // Exterior kernels
 ////////////////////////////////////////////////////////////////////
-template <class Impl> accelerator_inline
+template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSiteDagExt(StencilView &st,  DoubledGaugeFieldView &U,
 						SiteHalfSpinor *buf, int sF,
 						int sU, const FermionFieldView &in, FermionFieldView &out)
@@ -258,7 +258,7 @@ void WilsonKernels<Impl>::GenericDhopSiteDagExt(StencilView &st,  DoubledGaugeFi
   }
 };
 
-template <class Impl> accelerator_inline
+template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSiteExt(StencilView &st,  DoubledGaugeFieldView &U,
 					     SiteHalfSpinor *buf, int sF,
 					     int sU, const FermionFieldView &in, FermionFieldView &out) 
@@ -290,7 +290,7 @@ void WilsonKernels<Impl>::GenericDhopSiteExt(StencilView &st,  DoubledGaugeField
 };
 
 #define DhopDirMacro(Dir,spProj,spRecon)	\
-  template <class Impl> accelerator_inline				\
+  template <class Impl>							\
   void WilsonKernels<Impl>::DhopDir##Dir(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int sF, \
 					 int sU, const FermionFieldView &in, FermionFieldView &out, int dir) \
   {									\
@@ -318,7 +318,7 @@ DhopDirMacro(Ym,spProjYm,spReconYm);
 DhopDirMacro(Zm,spProjZm,spReconZm);
 DhopDirMacro(Tm,spProjTm,spReconTm);
 
-template <class Impl> accelerator_inline
+template <class Impl> 
 void WilsonKernels<Impl>::DhopDirK( StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int sF,
 				    int sU, const FermionFieldView &in, FermionFieldView &out, int dir, int gamma) 
 {
@@ -501,3 +501,4 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st,  DoubledGaugeField
 #undef ASM_CALL
 
 NAMESPACE_END(Grid);
+

Grid/qcd/action/fermion/instantiation/WilsonAdjImplD/WilsonKernelsInstantiationWilsonAdjImplD.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonAdjImplD/WilsonKernelsInstantiationWilsonAdjImplD.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonAdjImplF/WilsonKernelsInstantiationWilsonAdjImplF.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonAdjImplF/WilsonKernelsInstantiationWilsonAdjImplF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonImplD/WilsonKernelsInstantiationWilsonImplD.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonImplD/WilsonKernelsInstantiationWilsonImplD.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonImplDF/WilsonKernelsInstantiationWilsonImplDF.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonImplDF/WilsonKernelsInstantiationWilsonImplDF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonImplF/WilsonKernelsInstantiationWilsonImplF.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonImplF/WilsonKernelsInstantiationWilsonImplF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonKernelsInstantiationWilsonImplFH.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonKernelsInstantiationWilsonImplFH.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonKernelsInstantiation.cc.master

@@ -4,12 +4,11 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
 
-Copyright (C) 2015, 2020
+Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
 Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -35,13 +34,9 @@ directory
 
 #ifndef AVX512
 #ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
 #include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
 #endif
 #endif
-#endif
-#endif
 
 NAMESPACE_BEGIN(Grid);
 
@@ -49,3 +44,4 @@ NAMESPACE_BEGIN(Grid);
 template class WilsonKernels<IMPLEMENTATION>; 
 
 NAMESPACE_END(Grid);
+

Grid/qcd/action/fermion/instantiation/WilsonKernelsInstantiationAsm.cc

@@ -37,7 +37,6 @@ directory
 ////////////////////////////////////////////////////////////////////////
 NAMESPACE_BEGIN(Grid);
 #include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmAvx512.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmA64FX.h>
 #include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmQPX.h>
 NAMESPACE_END(Grid);
 

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexAntiSymmetricImplD/WilsonKernelsInstantiationWilsonTwoIndexAntiSymmetricImplD.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexAntiSymmetricImplD/WilsonKernelsInstantiationWilsonTwoIndexAntiSymmetricImplD.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexAntiSymmetricImplF/WilsonKernelsInstantiationWilsonTwoIndexAntiSymmetricImplF.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexAntiSymmetricImplF/WilsonKernelsInstantiationWilsonTwoIndexAntiSymmetricImplF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexSymmetricImplD/WilsonKernelsInstantiationWilsonTwoIndexSymmetricImplD.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexSymmetricImplD/WilsonKernelsInstantiationWilsonTwoIndexSymmetricImplD.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexSymmetricImplF/WilsonKernelsInstantiationWilsonTwoIndexSymmetricImplF.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonTwoIndexSymmetricImplF/WilsonKernelsInstantiationWilsonTwoIndexSymmetricImplF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/ZWilsonImplD/WilsonKernelsInstantiationZWilsonImplD.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/ZWilsonImplD/WilsonKernelsInstantiationZWilsonImplD.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonKernelsInstantiationZWilsonImplDF.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonKernelsInstantiationZWilsonImplDF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/ZWilsonImplF/WilsonKernelsInstantiationZWilsonImplF.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/ZWilsonImplF/WilsonKernelsInstantiationZWilsonImplF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonKernelsInstantiationZWilsonImplFH.cc

@@ -1,51 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/action/fermion/WilsonKernels.cc
-
-Copyright (C) 2015, 2020
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Nils Meyer <nils.meyer@ur.de> Regensburg University
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/qcd/action/fermion/FermionCore.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h>
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h>
-
-#ifndef AVX512
-#ifndef QPX
-#ifndef A64FX
-#ifndef A64FXFIXEDSIZE
-#include <Grid/qcd/action/fermion/implementation/WilsonKernelsAsmImplementation.h>
-#endif
-#endif
-#endif
-#endif
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class WilsonKernels<IMPLEMENTATION>;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonKernelsInstantiationZWilsonImplFH.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/gauge/GaugeImplementations.h

@@ -59,7 +59,7 @@ public:
   }
   static inline GaugeLinkField
   CovShiftIdentityBackward(const GaugeLinkField &Link, int mu) {
-    return Cshift(adj(Link), mu, -1);
+    return Cshift(closure(adj(Link)), mu, -1);
   }
   static inline GaugeLinkField
   CovShiftIdentityForward(const GaugeLinkField &Link, int mu) {

Grid/qcd/hmc/integrators/Integrator.h

@@ -301,9 +301,9 @@ public:
       t_P[level] = 0;
     }
 
-    for (int stp = 0; stp < Params.MDsteps; ++stp) {  // MD step
-      int first_step = (stp == 0);
-      int last_step = (stp == Params.MDsteps - 1);
+    for (int step = 0; step < Params.MDsteps; ++step) {  // MD step
+      int first_step = (step == 0);
+      int last_step = (step == Params.MDsteps - 1);
       this->step(U, 0, first_step, last_step);
     }
 

Grid/qcd/utils/A2Autils.h

@@ -1119,33 +1119,39 @@ void A2Autils<FImpl>::ContractFourQuarkColourDiagonal(const PropagatorField &WWV
   assert(gamma0.size()==gamma1.size());
   int Ng = gamma0.size();
 
+  // Make device accessible copy
+  Vector<Gamma> Gamma0v (Ng);
+  Vector<Gamma> Gamma1v (Ng);
+  Gamma *Gamma0 = & Gamma0v[0];
+  Gamma *Gamma1 = & Gamma1v[0];
+  for(int g=0;g<Ng;g++) {
+    Gamma0[g]=gamma0[g];
+    Gamma1[g]=gamma1[g];
+  }
+  
   GridBase *grid = WWVV0.Grid();
 
-  autoView(WWVV0_v , WWVV0,CpuRead);
-  autoView(WWVV1_v , WWVV1,CpuRead);
-  autoView(O_trtr_v, O_trtr,CpuWrite);
-  autoView(O_fig8_v, O_fig8,CpuWrite);
-  thread_for(ss,grid->oSites(),{
-
   typedef typename ComplexField::vector_object vobj;
+  autoView(WWVV0_v , WWVV0,AcceleratorRead);
+  autoView(WWVV1_v , WWVV1,AcceleratorRead);
+  autoView(O_trtr_v, O_trtr,AcceleratorWrite);
+  autoView(O_fig8_v, O_fig8,AcceleratorWrite);
+  accelerator_for(ss,grid->oSites(),vobj::Nsimd(),{
 
-    vobj v_trtr;
-    vobj v_fig8;
-
-    auto VV0 = WWVV0_v[ss];
-    auto VV1 = WWVV1_v[ss];
+    auto VV0 = WWVV0_v(ss);
+    auto VV1 = WWVV1_v(ss);
     
     for(int g=0;g<Ng;g++){
 
-      v_trtr = trace(VV0 * gamma0[g])* trace(VV1*gamma1[g]);
-      v_fig8 = trace(VV0 * gamma0[g] * VV1 * gamma1[g]);
+      auto v_trtr = trace(VV0 * gamma0[g])* trace(VV1*gamma1[g]);
+      auto v_fig8 = trace(VV0 * gamma0[g] * VV1 * gamma1[g]);
 
       if ( g==0 ) {
-	O_trtr_v[ss] = v_trtr; 
-	O_fig8_v[ss] = v_fig8;
+	coalescedWrite(O_trtr_v[ss], v_trtr); 
+	coalescedWrite(O_fig8_v[ss], v_fig8);
       } else { 
-	O_trtr_v[ss]+= v_trtr; 
-	O_fig8_v[ss]+= v_fig8;
+	coalescedWrite(O_trtr_v[ss], O_trtr_v(ss)+v_trtr); 
+	coalescedWrite(O_fig8_v[ss], O_fig8_v(ss)+v_fig8);
       }
       
     }
@@ -1165,25 +1171,36 @@ void A2Autils<FImpl>::ContractFourQuarkColourMix(const PropagatorField &WWVV0,
 
   GridBase *grid = WWVV0.Grid();
 
-  autoView( WWVV0_v , WWVV0,CpuRead);
-  autoView( WWVV1_v , WWVV1,CpuRead);
-  autoView( O_trtr_v, O_trtr,CpuWrite);
-  autoView( O_fig8_v, O_fig8,CpuWrite);
+  // Make device accessible copy
+  Vector<Gamma> Gamma0v (Ng);
+  Vector<Gamma> Gamma1v (Ng);
+  Gamma *Gamma0 = & Gamma0v[0];
+  Gamma *Gamma1 = & Gamma1v[0];
+  for(int g=0;g<Ng;g++) {
+    Gamma0[g]=gamma0[g];
+    Gamma1[g]=gamma1[g];
+  }
 
-  thread_for(ss,grid->oSites(),{
+  autoView( WWVV0_v , WWVV0,AcceleratorRead);
+  autoView( WWVV1_v , WWVV1,AcceleratorRead);
+  autoView( O_trtr_v, O_trtr,AcceleratorWrite);
+  autoView( O_fig8_v, O_fig8,AcceleratorWrite);
 
   typedef typename ComplexField::vector_object vobj;
+  accelerator_for(ss,grid->oSites(),vobj::Nsimd(),{
 
-    auto VV0 = WWVV0_v[ss];
-    auto VV1 = WWVV1_v[ss];
+    auto VV0 = WWVV0_v(ss);
+    auto VV1 = WWVV1_v(ss);
+
+    typedef decltype(trace(VV0)) scalar;
 
     for(int g=0;g<Ng;g++){
 
       auto VV0G = VV0 * gamma0[g];  // Spin multiply
       auto VV1G = VV1 * gamma1[g];
 
-      vobj v_trtr=Zero();
-      vobj v_fig8=Zero();
+      scalar v_trtr=Zero();
+      scalar v_fig8=Zero();
 
       /////////////////////////////////////////
       // Colour mixed
@@ -1234,11 +1251,11 @@ Bag [8,4]  fig8 (-227.58,3.58808e-17) trtr (-32.5776,1.83286e-17)     //  - 1602
       }}}}
 
       if ( g==0 ) {
-	O_trtr_v[ss] = v_trtr; 
-	O_fig8_v[ss] = v_fig8;
+	coalescedWrite(O_trtr_v[ss] , v_trtr); 
+	coalescedWrite(O_fig8_v[ss] , v_fig8);
       } else { 
-	O_trtr_v[ss]+= v_trtr; 
-	O_fig8_v[ss]+= v_fig8;
+	coalescedWrite(O_trtr_v[ss],O_trtr_v(ss) + v_trtr); 
+	coalescedWrite(O_fig8_v[ss],O_fig8_v(ss) + v_fig8);
       }
       
     }

Grid/qcd/utils/BaryonUtils.h

@@ -51,7 +51,7 @@ public:
 
   private: 
   template <class mobj, class robj>
-  static void BaryonSite(const mobj &D1,
+  static void baryon_site(const mobj &D1,
 				 const mobj &D2,
 				 const mobj &D3,
 				 const Gamma GammaA_left,
@@ -61,18 +61,8 @@ public:
 				 const int parity,
 				 const bool * wick_contractions,
   				 robj &result);
-  template <class mobj, class robj>
-  static void BaryonSiteMatrix(const mobj &D1,
-         const mobj &D2,
-         const mobj &D3,
-         const Gamma GammaA_left,
-         const Gamma GammaB_left,
-         const Gamma GammaA_right,
-         const Gamma GammaB_right,
-         const bool * wick_contractions,
-           robj &result);
   public:
-  static void WickContractions(std::string qi, 
+  static void Wick_Contractions(std::string qi, 
                  std::string qf, 
                  bool* wick_contractions);
   static void ContractBaryons(const PropagatorField &q1_left,
@@ -85,17 +75,8 @@ public:
 				 const bool* wick_contractions,
 				 const int parity,
 				 ComplexField &baryon_corr);
-  static void ContractBaryonsMatrix(const PropagatorField &q1_left,
-         const PropagatorField &q2_left,
-         const PropagatorField &q3_left,
-         const Gamma GammaA_left,
-         const Gamma GammaB_left,
-         const Gamma GammaA_right,
-         const Gamma GammaB_right,
-         const bool* wick_contractions,
-         SpinMatrixField &baryon_corr);
   template <class mobj, class robj>
-  static void ContractBaryonsSliced(const mobj &D1,
+  static void ContractBaryons_Sliced(const mobj &D1,
 				 const mobj &D2,
 				 const mobj &D3,
 				 const Gamma GammaA_left,
@@ -106,20 +87,9 @@ public:
 				 const int parity,
 				 const int nt,
 				 robj &result);
-  template <class mobj, class robj>
-  static void ContractBaryonsSlicedMatrix(const mobj &D1,
-         const mobj &D2,
-         const mobj &D3,
-         const Gamma GammaA_left,
-         const Gamma GammaB_left,
-         const Gamma GammaA_right,
-         const Gamma GammaB_right,
-         const bool* wick_contractions,
-         const int nt,
-         robj &result);
   private:
   template <class mobj, class mobj2, class robj>
-  static void BaryonGamma3ptGroup1Site(
+  static void Baryon_Gamma_3pt_Group1_Site(
            const mobj &Dq1_ti,
            const mobj2 &Dq2_spec,
            const mobj2 &Dq3_spec,
@@ -131,7 +101,7 @@ public:
            robj &result);
 
   template <class mobj, class mobj2, class robj>
-  static void BaryonGamma3ptGroup2Site(
+  static void Baryon_Gamma_3pt_Group2_Site(
            const mobj2 &Dq1_spec,
            const mobj &Dq2_ti,
            const mobj2 &Dq3_spec,
@@ -143,7 +113,7 @@ public:
            robj &result);
 
   template <class mobj, class mobj2, class robj>
-  static void BaryonGamma3ptGroup3Site(
+  static void Baryon_Gamma_3pt_Group3_Site(
            const mobj2 &Dq1_spec,
            const mobj2 &Dq2_spec,
            const mobj &Dq3_ti,
@@ -155,7 +125,7 @@ public:
            robj &result);
   public:
   template <class mobj>
-  static void BaryonGamma3pt(
+  static void Baryon_Gamma_3pt(
            const PropagatorField &q_ti,
            const mobj &Dq_spec1,
            const mobj &Dq_spec2,
@@ -168,7 +138,7 @@ public:
            SpinMatrixField &stn_corr);
   private: 
   template <class mobj, class mobj2, class robj>
-  static void SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
+  static void Sigma_to_Nucleon_Q1_Eye_site(const mobj &Dq_loop,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
 						 const mobj &Ds_ti,
@@ -177,7 +147,7 @@ public:
 		                 		 const Gamma GammaB_nucl,
 						 robj &result);
   template <class mobj, class mobj2, class robj>
-  static void SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
+  static void Sigma_to_Nucleon_Q1_NonEye_site(const mobj &Du_ti,
 						 const mobj &Du_tf,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
@@ -189,7 +159,7 @@ public:
 
 
   template <class mobj, class mobj2, class robj>
-  static void SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
+  static void Sigma_to_Nucleon_Q2_Eye_site(const mobj &Dq_loop,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
 						 const mobj &Ds_ti,
@@ -198,7 +168,7 @@ public:
 		                 		 const Gamma GammaB_nucl,
 						 robj &result);
   template <class mobj, class mobj2, class robj>
-  static void SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
+  static void Sigma_to_Nucleon_Q2_NonEye_site(const mobj &Du_ti,
 						 const mobj &Du_tf,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
@@ -209,7 +179,7 @@ public:
 						 robj &result);
   public:
   template <class mobj>
-  static void SigmaToNucleonEye(const PropagatorField &qq_loop,
+  static void Sigma_to_Nucleon_Eye(const PropagatorField &qq_loop,
 				 const mobj &Du_spec,
 				 const PropagatorField &qd_tf,
 				 const PropagatorField &qs_ti,
@@ -219,7 +189,7 @@ public:
 		                 const std::string op,
 				 SpinMatrixField &stn_corr);
   template <class mobj>
-  static void SigmaToNucleonNonEye(const PropagatorField &qq_ti,
+  static void Sigma_to_Nucleon_NonEye(const PropagatorField &qq_ti,
 				 const PropagatorField &qq_tf,
 				 const mobj &Du_spec,
 				 const PropagatorField &qd_tf,
@@ -247,7 +217,7 @@ const Real BaryonUtils<FImpl>::epsilon_sgn[6] = {1.,1.,1.,-1.,-1.,-1.};
 //This is the old version
 template <class FImpl>
 template <class mobj, class robj>
-void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
+void BaryonUtils<FImpl>::baryon_site(const mobj &D1,
                 const mobj &D2,
                 const mobj &D3,
                          const Gamma GammaA_i,
@@ -359,132 +329,12 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
     }}
 }
 
-//New version without parity projection or trace
-template <class FImpl>
-template <class mobj, class robj>
-void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
-                const mobj &D2,
-                const mobj &D3,
-                         const Gamma GammaA_i,
-                         const Gamma GammaB_i,
-                         const Gamma GammaA_f,
-                         const Gamma GammaB_f,
-                const bool * wick_contraction,
-                robj &result)
-{
-
-    auto D1_GAi =  D1 * GammaA_i;
-    auto GAf_D1_GAi = GammaA_f * D1_GAi;
-    auto GBf_D1_GAi = GammaB_f * D1_GAi;
-
-    auto D2_GBi = D2 * GammaB_i;
-    auto GBf_D2_GBi = GammaB_f * D2_GBi;
-    auto GAf_D2_GBi = GammaA_f * D2_GBi;
-
-    auto GBf_D3 = GammaB_f * D3;
-    auto GAf_D3 = GammaA_f * D3;
-
-    for (int ie_f=0; ie_f < 6 ; ie_f++){
-        int a_f = epsilon[ie_f][0]; //a
-        int b_f = epsilon[ie_f][1]; //b
-        int c_f = epsilon[ie_f][2]; //c
-    for (int ie_i=0; ie_i < 6 ; ie_i++){
-        int a_i = epsilon[ie_i][0]; //a'
-        int b_i = epsilon[ie_i][1]; //b'
-        int c_i = epsilon[ie_i][2]; //c'
-
-        Real ee = epsilon_sgn[ie_f] * epsilon_sgn[ie_i];
-        //This is the \delta_{456}^{123} part
-        if (wick_contraction[0]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int rho_f=0; rho_f<Ns; rho_f++){
-                auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
-                for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()(rho_f,rho_i)() += ee  * GAf_D1_GAi_rr_cc
-                                        * D2_GBi    ()(alpha_f,beta_i)(a_f,a_i)
-                                        * GBf_D3    ()(alpha_f,beta_i)(b_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{231} part
-        if (wick_contraction[1]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto GBf_D2_GBi_ab_ba = GBf_D2_GBi ()(alpha_f,beta_i)(b_f,a_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() += ee  * D1_GAi_ar_ac
-                                        * GBf_D2_GBi_ab_ba
-                                        * GAf_D3        ()(rho_f,beta_i)(c_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{312} part
-        if (wick_contraction[2]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto D3_ab_ab = D3 ()(alpha_f,beta_i)(a_f,b_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() += ee  * GBf_D1_GAi_ar_bc
-                                        * GAf_D2_GBi    ()(rho_f,beta_i)(c_f,a_i)
-                                        * D3_ab_ab;
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{132} part
-        if (wick_contraction[3]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int rho_f=0; rho_f<Ns; rho_f++){
-                auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
-                for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()(rho_f,rho_i)() -= ee  * GAf_D1_GAi_rr_cc
-                                        * GBf_D2_GBi    ()(alpha_f,beta_i)(b_f,a_i)
-                                        * D3            ()(alpha_f,beta_i)(a_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{321} part
-        if (wick_contraction[4]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto D2_GBi_ab_aa = D2_GBi()(alpha_f,beta_i)(a_f,a_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() -= ee  * GBf_D1_GAi_ar_bc
-                                        * D2_GBi_ab_aa
-                                        * GAf_D3    ()(rho_f,beta_i)(c_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{213} part
-        if (wick_contraction[5]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto GBf_D3_ab_bb = GBf_D3()(alpha_f,beta_i)(b_f,b_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() -= ee  * D1_GAi_ar_ac
-                                        * GAf_D2_GBi    ()(rho_f,beta_i)(c_f,a_i)
-                                        * GBf_D3_ab_bb;
-                }}
-            }}
-        }
-    }}
-}
-
 /* Computes which wick contractions should be performed for a    *
  * baryon 2pt function given the initial and finals state quark  *
  * flavours.                                                     *
  * The array wick_contractions must be of length 6               */
 template<class FImpl>
-void BaryonUtils<FImpl>::WickContractions(std::string qi, std::string qf, bool* wick_contractions) {
+void BaryonUtils<FImpl>::Wick_Contractions(std::string qi, std::string qf, bool* wick_contractions) {
     const int epsilon[6][3] = {{0,1,2},{1,2,0},{2,0,1},{0,2,1},{2,1,0},{1,0,2}};
     for (int ie=0; ie < 6 ; ie++) {
         wick_contractions[ie] = (qi.size() == 3 && qf.size() == 3
@@ -515,6 +365,11 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
   assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
 
+  std::cout << "GammaA (left) " << (GammaA_left.g) <<  std::endl;
+  std::cout << "GammaB (left) " << (GammaB_left.g) <<  std::endl;
+  std::cout << "GammaA (right) " << (GammaA_right.g) <<  std::endl;
+  std::cout << "GammaB (right) " << (GammaB_right.g) <<  std::endl;
+ 
   assert(parity==1 || parity == -1 && "Parity must be +1 or -1");
 
   GridBase *grid = q1_left.Grid();
@@ -542,62 +397,13 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
     auto D2 = v2[ss];
     auto D3 = v3[ss];
     vobj result=Zero();
-    BaryonSite(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,parity,wick_contractions,result);
+    baryon_site(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,parity,wick_contractions,result);
     vbaryon_corr[ss] = result; 
   }  );//end loop over lattice sites
 
   t += usecond();
 
-  std::cout << GridLogDebug << std::setw(10) << bytes/t*1.0e6/1024/1024/1024 << " GB/s " << std::endl;
-}
-
-template<class FImpl>
-void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
-             const PropagatorField &q2_left,
-             const PropagatorField &q3_left,
-                         const Gamma GammaA_left,
-                         const Gamma GammaB_left,
-                         const Gamma GammaA_right,
-                         const Gamma GammaB_right,
-             const bool* wick_contractions,
-             SpinMatrixField &baryon_corr)
-{
-
-  assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
-  assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
- 
-  GridBase *grid = q1_left.Grid();
-  
-  autoView(vbaryon_corr, baryon_corr,CpuWrite);
-  autoView( v1 , q1_left, CpuRead);
-  autoView( v2 , q2_left, CpuRead);
-  autoView( v3 , q3_left, CpuRead);
-
-  // Real bytes =0.;
-  // bytes += grid->oSites() * (432.*sizeof(vComplex) + 126.*sizeof(int) + 36.*sizeof(Real));
-  // for (int ie=0; ie < 6 ; ie++){
-  //   if(ie==0 or ie==3){
-  //      bytes += grid->oSites() * (4.*sizeof(int) + 4752.*sizeof(vComplex)) * wick_contractions[ie];
-  //   }
-  //   else{
-  //      bytes += grid->oSites() * (64.*sizeof(int) + 5184.*sizeof(vComplex)) * wick_contractions[ie];
-  //   }
-  // }
-  // Real t=0.;
-  // t =-usecond();
-
-  accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto D1 = v1[ss];
-    auto D2 = v2[ss];
-    auto D3 = v3[ss];
-    sobj result=Zero();
-    BaryonSiteMatrix(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,wick_contractions,result);
-    vbaryon_corr[ss] = result; 
-  }  );//end loop over lattice sites
-
-  // t += usecond();
-
-  // std::cout << GridLogDebug << std::setw(10) << bytes/t*1.0e6/1024/1024/1024 << " GB/s " << std::endl;
+  std::cout << std::setw(10) << bytes/t*1.0e6/1024/1024/1024 << " GB/s " << std::endl;
 
 }
 
@@ -608,7 +414,7 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
  * Wick_Contractions function above                               */
 template <class FImpl>
 template <class mobj, class robj>
-void BaryonUtils<FImpl>::ContractBaryonsSliced(const mobj &D1,
+void BaryonUtils<FImpl>::ContractBaryons_Sliced(const mobj &D1,
 						 const mobj &D2,
 						 const mobj &D3,
 				                 const Gamma GammaA_left,
@@ -624,32 +430,15 @@ void BaryonUtils<FImpl>::ContractBaryonsSliced(const mobj &D1,
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
   assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
 
+  std::cout << "GammaA (left) " << (GammaA_left.g) <<  std::endl;
+  std::cout << "GammaB (left) " << (GammaB_left.g) <<  std::endl;
+  std::cout << "GammaA (right) " << (GammaA_right.g) <<  std::endl;
+  std::cout << "GammaB (right) " << (GammaB_right.g) <<  std::endl;
+ 
   assert(parity==1 || parity == -1 && "Parity must be +1 or -1");
 
   for (int t=0; t<nt; t++) {
-    BaryonSite(D1[t],D2[t],D3[t],GammaA_left,GammaB_left,GammaA_right,GammaB_right,parity,wick_contractions,result[t]);
-  }
-}
-
-template <class FImpl>
-template <class mobj, class robj>
-void BaryonUtils<FImpl>::ContractBaryonsSlicedMatrix(const mobj &D1,
-             const mobj &D2,
-             const mobj &D3,
-                         const Gamma GammaA_left,
-                         const Gamma GammaB_left,
-                         const Gamma GammaA_right,
-                         const Gamma GammaB_right,
-             const bool* wick_contractions,
-             const int nt,
-             robj &result)
-{
-
-  assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
-  assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
-
-  for (int t=0; t<nt; t++) {
-    BaryonSiteMatrix(D1[t],D2[t],D3[t],GammaA_left,GammaB_left,GammaA_right,GammaB_right,wick_contractions,result[t]);
+    baryon_site(D1[t],D2[t],D3[t],GammaA_left,GammaB_left,GammaA_right,GammaB_right,parity,wick_contractions,result[t]);
   }
 }
 
@@ -665,7 +454,7 @@ void BaryonUtils<FImpl>::ContractBaryonsSlicedMatrix(const mobj &D1,
  * Dq4_tf is a quark line from t_f to t_J */
 template<class FImpl>
 template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
+void BaryonUtils<FImpl>::Baryon_Gamma_3pt_Group1_Site(
                         const mobj &Dq1_ti,
                         const mobj2 &Dq2_spec,
                         const mobj2 &Dq3_spec,
@@ -757,7 +546,7 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
  * Dq4_tf is a quark line from t_f to t_J */
 template<class FImpl>
 template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
+void BaryonUtils<FImpl>::Baryon_Gamma_3pt_Group2_Site(
                         const mobj2 &Dq1_spec,
                         const mobj &Dq2_ti,
                         const mobj2 &Dq3_spec,
@@ -847,7 +636,7 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
  * Dq4_tf is a quark line from t_f to t_J */
 template<class FImpl>
 template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
+void BaryonUtils<FImpl>::Baryon_Gamma_3pt_Group3_Site(
                         const mobj2 &Dq1_spec,
                         const mobj2 &Dq2_spec,
                         const mobj &Dq3_ti,
@@ -939,7 +728,7 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
  * https://aportelli.github.io/Hadrons-doc/#/mcontraction        */
 template<class FImpl>
 template <class mobj>
-void BaryonUtils<FImpl>::BaryonGamma3pt(
+void BaryonUtils<FImpl>::Baryon_Gamma_3pt(
                         const PropagatorField &q_ti,
                         const mobj &Dq_spec1,
                         const mobj &Dq_spec2,
@@ -962,7 +751,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
             auto Dq_ti = vq_ti[ss];
             auto Dq_tf = vq_tf[ss];
             sobj result=Zero();
-            BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+            Baryon_Gamma_3pt_Group1_Site(Dq_ti,Dq_spec1,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
             vcorr[ss] += result; 
         });//end loop over lattice sites
     } else if (group == 2) {
@@ -970,7 +759,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
             auto Dq_ti = vq_ti[ss];
             auto Dq_tf = vq_tf[ss];
             sobj result=Zero();
-            BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+            Baryon_Gamma_3pt_Group2_Site(Dq_spec1,Dq_ti,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
             vcorr[ss] += result; 
         });//end loop over lattice sites
     } else if (group == 3) {
@@ -978,7 +767,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
             auto Dq_ti = vq_ti[ss];
             auto Dq_tf = vq_tf[ss];
             sobj result=Zero();
-            BaryonGamma3ptGroup3Site(Dq_spec1,Dq_spec2,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+            Baryon_Gamma_3pt_Group3_Site(Dq_spec1,Dq_spec2,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
 
             vcorr[ss] += result; 
         });//end loop over lattice sites
@@ -998,7 +787,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
+void BaryonUtils<FImpl>::Sigma_to_Nucleon_Q1_Eye_site(const mobj &Dq_loop,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
 						 const mobj &Ds_ti,
@@ -1049,7 +838,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
+void BaryonUtils<FImpl>::Sigma_to_Nucleon_Q1_NonEye_site(const mobj &Du_ti,
 						 const mobj &Du_tf,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
@@ -1108,7 +897,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
+void BaryonUtils<FImpl>::Sigma_to_Nucleon_Q2_Eye_site(const mobj &Dq_loop,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
 						 const mobj &Ds_ti,
@@ -1159,7 +948,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
+void BaryonUtils<FImpl>::Sigma_to_Nucleon_Q2_NonEye_site(const mobj &Du_ti,
 						 const mobj &Du_tf,
 						 const mobj2 &Du_spec,
 						 const mobj &Dd_tf,
@@ -1213,7 +1002,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
 
 template<class FImpl>
 template <class mobj>
-void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
+void BaryonUtils<FImpl>::Sigma_to_Nucleon_Eye(const PropagatorField &qq_loop,
 						 const mobj &Du_spec,
 						 const PropagatorField &qd_tf,
 						 const PropagatorField &qs_ti,
@@ -1240,9 +1029,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
     auto Ds_ti = vs_ti[ss];
     sobj result=Zero();
     if(op == "Q1"){
-      SigmaToNucleonQ1EyeSite(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      Sigma_to_Nucleon_Q1_Eye_site(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else if(op == "Q2"){
-      SigmaToNucleonQ2EyeSite(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      Sigma_to_Nucleon_Q2_Eye_site(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else {
       assert(0 && "Weak Operator not correctly specified");
     }
@@ -1252,7 +1041,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
 
 template<class FImpl>
 template <class mobj>
-void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
+void BaryonUtils<FImpl>::Sigma_to_Nucleon_NonEye(const PropagatorField &qq_ti,
 						 const PropagatorField &qq_tf,
 						 const mobj &Du_spec,
 						 const PropagatorField &qd_tf,
@@ -1282,9 +1071,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
     auto Ds_ti = vs_ti[ss];
     sobj result=Zero();
     if(op == "Q1"){
-      SigmaToNucleonQ1NonEyeSite(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      Sigma_to_Nucleon_Q1_NonEye_site(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else if(op == "Q2"){
-      SigmaToNucleonQ2NonEyeSite(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      Sigma_to_Nucleon_Q2_NonEye_site(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else {
       assert(0 && "Weak Operator not correctly specified");
     }

Grid/qcd/utils/CovariantCshift.h

@@ -53,21 +53,23 @@ namespace PeriodicBC {
     return Cshift(tmp,mu,-1);// moves towards positive mu
   }
 
-  template<class gauge,class Expr,typename std::enable_if<is_lattice_expr<Expr>::value,void>::type * = nullptr>
-    auto  CovShiftForward(const Lattice<gauge> &Link, 
+  template<class gauge,typename Op, typename T1> auto
+    CovShiftForward(const Lattice<gauge> &Link, 
 		    int mu,
-			  const Expr &expr) -> decltype(closure(expr))
+		    const LatticeUnaryExpression<Op,T1> &expr)
+    -> Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> 
   {
-    auto arg = closure(expr);
+    Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> arg(expr);
     return CovShiftForward(Link,mu,arg);
   }
-  template<class gauge,class Expr,typename std::enable_if<is_lattice_expr<Expr>::value,void>::type * = nullptr>
-    auto  CovShiftBackward(const Lattice<gauge> &Link, 
+  template<class gauge,typename Op, typename T1> auto
+    CovShiftBackward(const Lattice<gauge> &Link, 
 		     int mu,
-			   const Expr &expr) -> decltype(closure(expr))
+		     const LatticeUnaryExpression<Op,T1> &expr)
+    -> Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> 
   {
-    auto arg = closure(expr);
-    return CovShiftBackward(Link,mu,arg);
+    Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> arg(expr);
+    return CovShiftForward(Link,mu,arg);
   }
 
 }
@@ -140,23 +142,26 @@ namespace ConjugateBC {
     return Cshift(tmp,mu,-1);// moves towards positive mu
   }
 
-  template<class gauge,class Expr,typename std::enable_if<is_lattice_expr<Expr>::value,void>::type * = nullptr>
-    auto  CovShiftForward(const Lattice<gauge> &Link, 
+  template<class gauge,typename Op, typename T1> auto
+    CovShiftForward(const Lattice<gauge> &Link, 
 		    int mu,
-			  const Expr &expr) -> decltype(closure(expr))
+		    const LatticeUnaryExpression<Op,T1> &expr)
+    -> Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> 
   {
-    auto arg = closure(expr);
+    Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> arg(expr);
     return CovShiftForward(Link,mu,arg);
   }
-  template<class gauge,class Expr,typename std::enable_if<is_lattice_expr<Expr>::value,void>::type * = nullptr>
-    auto  CovShiftBackward(const Lattice<gauge> &Link, 
+  template<class gauge,typename Op, typename T1> auto
+    CovShiftBackward(const Lattice<gauge> &Link, 
 		     int mu,
-			   const Expr &expr)  -> decltype(closure(expr))
+		     const LatticeUnaryExpression<Op,T1> &expr)
+    -> Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> 
   {
-    auto arg = closure(expr);
-    return CovShiftBackward(Link,mu,arg);
+    Lattice<decltype(expr.op.func(eval(0, expr.arg1)))> arg(expr);
+    return CovShiftForward(Link,mu,arg);
   }
 
+
 }
 
 

Grid/qcd/utils/SUn.h

@@ -449,8 +449,7 @@ public:
     LatticeReal alpha(grid);
 
     //    std::cout<<GridLogMessage<<"xi "<<xi <<std::endl;
-    xi = 2.0 *xi;
-    alpha = toReal(xi);
+    alpha = toReal(2.0 * xi);
 
     do {
       // A. Generate two uniformly distributed pseudo-random numbers R and R',

Grid/qcd/utils/SUnAdjoint.h

@@ -47,9 +47,14 @@ public:
   typedef Lattice<vAMatrixF> LatticeAdjMatrixF;
   typedef Lattice<vAMatrixD> LatticeAdjMatrixD;
 
-  typedef Lattice<iVector<iScalar<iMatrix<vComplex, Dimension> >, Nd> >  LatticeAdjField;
-  typedef Lattice<iVector<iScalar<iMatrix<vComplexF, Dimension> >, Nd> > LatticeAdjFieldF;
-  typedef Lattice<iVector<iScalar<iMatrix<vComplexD, Dimension> >, Nd> > LatticeAdjFieldD;
+  typedef Lattice<iVector<iScalar<iMatrix<vComplex, Dimension> >, Nd> >
+  LatticeAdjField;
+  typedef Lattice<iVector<iScalar<iMatrix<vComplexF, Dimension> >, Nd> >
+  LatticeAdjFieldF;
+  typedef Lattice<iVector<iScalar<iMatrix<vComplexD, Dimension> >, Nd> >
+  LatticeAdjFieldD;
+
+
 
 
   template <class cplx>
@@ -123,9 +128,7 @@ public:
   }
 
   // Projects the algebra components a lattice matrix (of dimension ncol*ncol -1 )
-  static void projectOnAlgebra(typename SU<ncolour>::LatticeAlgebraVector &h_out, const LatticeAdjMatrix &in, Real scale = 1.0) 
-  {
-    
+  static void projectOnAlgebra(typename SU<ncolour>::LatticeAlgebraVector &h_out, const LatticeAdjMatrix &in, Real scale = 1.0) {
     conformable(h_out, in);
     h_out = Zero();
     AMatrix iTa;
@@ -133,7 +136,7 @@ public:
 
     for (int a = 0; a < Dimension; a++) {
       generator(a, iTa);
-      LatticeComplex tmp = real(trace(iTa * in)) * coefficient;
+      auto tmp = real(trace(iTa * in)) * coefficient;
       pokeColour(h_out, tmp, a);
     }
   }

Grid/qcd/utils/WilsonLoops.h

@@ -485,7 +485,7 @@ public:
 
         // Up staple    ___ ___
         //             |       |
-        tmp = Cshift(adj(U[nu]), nu, -1);
+        tmp = Cshift(closure(adj(U[nu])), nu, -1);
         tmp = adj(U2[mu]) * tmp;
         tmp = Cshift(tmp, mu, -2);
 
@@ -519,7 +519,7 @@ public:
         //
         //      |  |
 
-        tmp = Cshift(adj(U2[nu]), nu, -2);
+        tmp = Cshift(closure(adj(U2[nu])), nu, -2);
         tmp = Gimpl::CovShiftBackward(U[mu], mu, tmp);
         tmp = U2[nu] * Cshift(tmp, nu, 2);
         Stap += Cshift(tmp, mu, 1);

Grid/serialisation/JSON_IO.cc

@@ -26,7 +26,7 @@
     *************************************************************************************/
     /*  END LEGAL */
 #include <Grid/Grid.h>
-#if (!defined(GRID_CUDA)) && (!defined(GRID_HIP))
+#ifndef __NVCC__
 
 NAMESPACE_BEGIN(Grid);
 

Grid/simd/Fujitsu_A64FX_asm_double.h

@@ -1,779 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: Fujitsu_A64FX_asm_double.h
-
-    Copyright (C) 2020
-
-Author: Nils Meyer <nils.meyer@ur.de>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#define LOAD_CHIMU(base)               LOAD_CHIMU_INTERLEAVED_A64FXd(base)  
-#define PREFETCH_CHIMU_L1(A)           PREFETCH_CHIMU_L1_INTERNAL_A64FXd(A)  
-#define PREFETCH_GAUGE_L1(A)           PREFETCH_GAUGE_L1_INTERNAL_A64FXd(A)  
-#define PREFETCH_CHIMU_L2(A)           PREFETCH_CHIMU_L2_INTERNAL_A64FXd(A)  
-#define PREFETCH_GAUGE_L2(A)           PREFETCH_GAUGE_L2_INTERNAL_A64FXd(A)  
-#define PF_GAUGE(A)  
-#define PREFETCH_RESULT_L2_STORE(A)    PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXd(A)  
-#define PREFETCH_RESULT_L1_STORE(A)    PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXd(A)  
-#define PREFETCH1_CHIMU(A)             PREFETCH_CHIMU_L1(A)  
-#define PREFETCH_CHIMU(A)              PREFETCH_CHIMU_L1(A)  
-#define LOCK_GAUGE(A)  
-#define UNLOCK_GAUGE(A)  
-#define MASK_REGS                      DECLARATIONS_A64FXd  
-#define SAVE_RESULT(A,B)               RESULT_A64FXd(A); PREFETCH_RESULT_L2_STORE(B)  
-#define MULT_2SPIN_1(Dir)              MULT_2SPIN_1_A64FXd(Dir)  
-#define MULT_2SPIN_2                   MULT_2SPIN_2_A64FXd  
-#define LOAD_CHI(base)                 LOAD_CHI_A64FXd(base)  
-#define ADD_RESULT(base,basep)         LOAD_CHIMU(base); ADD_RESULT_INTERNAL_A64FXd; RESULT_A64FXd(base)  
-#define XP_PROJ                        XP_PROJ_A64FXd  
-#define YP_PROJ                        YP_PROJ_A64FXd  
-#define ZP_PROJ                        ZP_PROJ_A64FXd  
-#define TP_PROJ                        TP_PROJ_A64FXd  
-#define XM_PROJ                        XM_PROJ_A64FXd  
-#define YM_PROJ                        YM_PROJ_A64FXd  
-#define ZM_PROJ                        ZM_PROJ_A64FXd  
-#define TM_PROJ                        TM_PROJ_A64FXd  
-#define XP_RECON                       XP_RECON_A64FXd  
-#define XM_RECON                       XM_RECON_A64FXd  
-#define XM_RECON_ACCUM                 XM_RECON_ACCUM_A64FXd  
-#define YM_RECON_ACCUM                 YM_RECON_ACCUM_A64FXd  
-#define ZM_RECON_ACCUM                 ZM_RECON_ACCUM_A64FXd  
-#define TM_RECON_ACCUM                 TM_RECON_ACCUM_A64FXd  
-#define XP_RECON_ACCUM                 XP_RECON_ACCUM_A64FXd  
-#define YP_RECON_ACCUM                 YP_RECON_ACCUM_A64FXd  
-#define ZP_RECON_ACCUM                 ZP_RECON_ACCUM_A64FXd  
-#define TP_RECON_ACCUM                 TP_RECON_ACCUM_A64FXd  
-#define PERMUTE_DIR0                   0  
-#define PERMUTE_DIR1                   1  
-#define PERMUTE_DIR2                   2  
-#define PERMUTE_DIR3                   3  
-#define PERMUTE                        PERMUTE_A64FXd;  
-#define LOAD_TABLE(Dir)                if (Dir == 0) { LOAD_TABLE0; } else if (Dir == 1) { LOAD_TABLE1; } else if (Dir == 2) { LOAD_TABLE2; }  
-#define MAYBEPERM(Dir,perm)            if (Dir != 3) { if (perm) { PERMUTE; } }  
-// DECLARATIONS
-#define DECLARATIONS_A64FXd  \
-    const uint64_t lut[4][8] = { \
-        {4, 5, 6, 7, 0, 1, 2, 3}, \
-        {2, 3, 0, 1, 6, 7, 4, 5}, \
-        {1, 0, 3, 2, 5, 4, 7, 6}, \
-        {0, 1, 2, 4, 5, 6, 7, 8} };\
-asm ( \
-    "fmov z31.d , 0 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// RESULT
-#define RESULT_A64FXd(base)  \
-{ \
-asm ( \
-    "str z0, [%[storeptr], -6, mul vl] \n\t" \
-    "str z1, [%[storeptr], -5, mul vl] \n\t" \
-    "str z2, [%[storeptr], -4, mul vl] \n\t" \
-    "str z3, [%[storeptr], -3, mul vl] \n\t" \
-    "str z4, [%[storeptr], -2, mul vl] \n\t" \
-    "str z5, [%[storeptr], -1, mul vl] \n\t" \
-    "str z6, [%[storeptr], 0, mul vl] \n\t" \
-    "str z7, [%[storeptr], 1, mul vl] \n\t" \
-    "str z8, [%[storeptr], 2, mul vl] \n\t" \
-    "str z9, [%[storeptr], 3, mul vl] \n\t" \
-    "str z10, [%[storeptr], 4, mul vl] \n\t" \
-    "str z11, [%[storeptr], 5, mul vl] \n\t" \
-    :  \
-    : [storeptr] "r" (base + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_CHIMU_L2 (prefetch to L2)
-#define PREFETCH_CHIMU_L2_INTERNAL_A64FXd(base)  \
-{ \
-asm ( \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_CHIMU_L1 (prefetch to L1)
-#define PREFETCH_CHIMU_L1_INTERNAL_A64FXd(base)  \
-{ \
-asm ( \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_GAUGE_L2 (prefetch to L2)
-#define PREFETCH_GAUGE_L2_INTERNAL_A64FXd(A)  \
-{ \
-    const auto & ref(U[sUn](A)); uint64_t baseU = (uint64_t)&ref + 3 * 3 * 64; \
-asm ( \
-    "prfd PLDL2STRM, p5, [%[fetchptr], -4, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 12, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 16, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 20, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 24, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 28, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_GAUGE_L1 (prefetch to L1)
-#define PREFETCH_GAUGE_L1_INTERNAL_A64FXd(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-asm ( \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHI
-#define LOAD_CHI_A64FXd(base)  \
-{ \
-asm ( \
-    "ldr z12, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], 2, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], 5, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHIMU
-#define LOAD_CHIMU_INTERLEAVED_A64FXd(base)  \
-{ \
-asm ( \
-    "ptrue p5.d \n\t" \
-    "ldr z12, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z21, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z18, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z22, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z19, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z23, [%[fetchptr], 5, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z20, [%[fetchptr], 2, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHIMU_0213
-#define LOAD_CHIMU_0213_A64FXd  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-asm ( \
-    "ptrue p5.d \n\t" \
-    "ldr z12, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z18, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z19, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z20, [%[fetchptr], 2, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z21, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z22, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z23, [%[fetchptr], 5, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (&ref[2][0]) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHIMU_0312
-#define LOAD_CHIMU_0312_A64FXd  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-asm ( \
-    "ptrue p5.d \n\t" \
-    "ldr z12, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z21, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z22, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z23, [%[fetchptr], 5, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z18, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z19, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z20, [%[fetchptr], 2, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (&ref[2][0]) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_TABLE0
-#define LOAD_TABLE0  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (0) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_TABLE1
-#define LOAD_TABLE1  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (1) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_TABLE2
-#define LOAD_TABLE2  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (2) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_TABLE3
-#define LOAD_TABLE3  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (3) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// PERMUTE
-#define PERMUTE_A64FXd  \
-asm ( \
-    "tbl z12.d, { z12.d }, z30.d \n\t"  \
-    "tbl z13.d, { z13.d }, z30.d \n\t"  \
-    "tbl z14.d, { z14.d }, z30.d \n\t"  \
-    "tbl z15.d, { z15.d }, z30.d \n\t"  \
-    "tbl z16.d, { z16.d }, z30.d \n\t"  \
-    "tbl z17.d, { z17.d }, z30.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_GAUGE
-#define LOAD_GAUGE  \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-{ \
-asm ( \
-    "ptrue p5.d \n\t" \
-    "ldr z24, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z25, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z26, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z27, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z28, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z29, [%[fetchptr], 1, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// MULT_2SPIN
-#define MULT_2SPIN_1_A64FXd(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-asm ( \
-    "ldr z24, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z25, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z26, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z27, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z28, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z29, [%[fetchptr], 1, mul vl] \n\t" \
-    "movprfx z18.d, p5/m, z31.d \n\t" \
-    "fcmla z18.d, p5/m, z24.d, z12.d, 0 \n\t" \
-    "movprfx z21.d, p5/m, z31.d \n\t" \
-    "fcmla z21.d, p5/m, z24.d, z15.d, 0 \n\t" \
-    "movprfx z19.d, p5/m, z31.d \n\t" \
-    "fcmla z19.d, p5/m, z25.d, z12.d, 0 \n\t" \
-    "movprfx z22.d, p5/m, z31.d \n\t" \
-    "fcmla z22.d, p5/m, z25.d, z15.d, 0 \n\t" \
-    "movprfx z20.d, p5/m, z31.d \n\t" \
-    "fcmla z20.d, p5/m, z26.d, z12.d, 0 \n\t" \
-    "movprfx z23.d, p5/m, z31.d \n\t" \
-    "fcmla z23.d, p5/m, z26.d, z15.d, 0 \n\t" \
-    "fcmla z18.d, p5/m, z24.d, z12.d, 90 \n\t" \
-    "fcmla z21.d, p5/m, z24.d, z15.d, 90 \n\t" \
-    "fcmla z19.d, p5/m, z25.d, z12.d, 90 \n\t" \
-    "fcmla z22.d, p5/m, z25.d, z15.d, 90 \n\t" \
-    "fcmla z20.d, p5/m, z26.d, z12.d, 90 \n\t" \
-    "fcmla z23.d, p5/m, z26.d, z15.d, 90 \n\t" \
-    "ldr z24, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z25, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z26, [%[fetchptr], 2, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// MULT_2SPIN_BACKEND
-#define MULT_2SPIN_2_A64FXd  \
-{ \
-asm ( \
-    "fcmla z18.d, p5/m, z27.d, z13.d, 0 \n\t" \
-    "fcmla z21.d, p5/m, z27.d, z16.d, 0 \n\t" \
-    "fcmla z19.d, p5/m, z28.d, z13.d, 0 \n\t" \
-    "fcmla z22.d, p5/m, z28.d, z16.d, 0 \n\t" \
-    "fcmla z20.d, p5/m, z29.d, z13.d, 0 \n\t" \
-    "fcmla z23.d, p5/m, z29.d, z16.d, 0 \n\t" \
-    "fcmla z18.d, p5/m, z27.d, z13.d, 90 \n\t" \
-    "fcmla z21.d, p5/m, z27.d, z16.d, 90 \n\t" \
-    "fcmla z19.d, p5/m, z28.d, z13.d, 90 \n\t" \
-    "fcmla z22.d, p5/m, z28.d, z16.d, 90 \n\t" \
-    "fcmla z20.d, p5/m, z29.d, z13.d, 90 \n\t" \
-    "fcmla z23.d, p5/m, z29.d, z16.d, 90 \n\t" \
-    "fcmla z18.d, p5/m, z24.d, z14.d, 0 \n\t" \
-    "fcmla z21.d, p5/m, z24.d, z17.d, 0 \n\t" \
-    "fcmla z19.d, p5/m, z25.d, z14.d, 0 \n\t" \
-    "fcmla z22.d, p5/m, z25.d, z17.d, 0 \n\t" \
-    "fcmla z20.d, p5/m, z26.d, z14.d, 0 \n\t" \
-    "fcmla z23.d, p5/m, z26.d, z17.d, 0 \n\t" \
-    "fcmla z18.d, p5/m, z24.d, z14.d, 90 \n\t" \
-    "fcmla z21.d, p5/m, z24.d, z17.d, 90 \n\t" \
-    "fcmla z19.d, p5/m, z25.d, z14.d, 90 \n\t" \
-    "fcmla z22.d, p5/m, z25.d, z17.d, 90 \n\t" \
-    "fcmla z20.d, p5/m, z26.d, z14.d, 90 \n\t" \
-    "fcmla z23.d, p5/m, z26.d, z17.d, 90 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XP_PROJ
-#define XP_PROJ_A64FXd  \
-{ \
-asm ( \
-    "fcadd z12.d, p5/m, z12.d, z21.d, 90 \n\t" \
-    "fcadd z13.d, p5/m, z13.d, z22.d, 90 \n\t" \
-    "fcadd z14.d, p5/m, z14.d, z23.d, 90 \n\t" \
-    "fcadd z15.d, p5/m, z15.d, z18.d, 90 \n\t" \
-    "fcadd z16.d, p5/m, z16.d, z19.d, 90 \n\t" \
-    "fcadd z17.d, p5/m, z17.d, z20.d, 90 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XP_RECON
-#define XP_RECON_A64FXd  \
-asm ( \
-    "movprfx z6.d, p5/m, z31.d \n\t" \
-    "fcadd z6.d, p5/m, z6.d, z21.d, 270 \n\t" \
-    "movprfx z7.d, p5/m, z31.d \n\t" \
-    "fcadd z7.d, p5/m, z7.d, z22.d, 270 \n\t" \
-    "movprfx z8.d, p5/m, z31.d \n\t" \
-    "fcadd z8.d, p5/m, z8.d, z23.d, 270 \n\t" \
-    "movprfx z9.d, p5/m, z31.d \n\t" \
-    "fcadd z9.d, p5/m, z9.d, z18.d, 270 \n\t" \
-    "movprfx z10.d, p5/m, z31.d \n\t" \
-    "fcadd z10.d, p5/m, z10.d, z19.d, 270 \n\t" \
-    "movprfx z11.d, p5/m, z31.d \n\t" \
-    "fcadd z11.d, p5/m, z11.d, z20.d, 270 \n\t" \
-    "mov z0.d, p5/m, z18.d \n\t" \
-    "mov z1.d, p5/m, z19.d \n\t" \
-    "mov z2.d, p5/m, z20.d \n\t" \
-    "mov z3.d, p5/m, z21.d \n\t" \
-    "mov z4.d, p5/m, z22.d \n\t" \
-    "mov z5.d, p5/m, z23.d \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// XP_RECON_ACCUM
-#define XP_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fcadd z9.d, p5/m, z9.d, z18.d, 270 \n\t" \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fcadd z10.d, p5/m, z10.d, z19.d, 270 \n\t" \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fcadd z11.d, p5/m, z11.d, z20.d, 270 \n\t" \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fcadd z6.d, p5/m, z6.d, z21.d, 270 \n\t" \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fcadd z7.d, p5/m, z7.d, z22.d, 270 \n\t" \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fcadd z8.d, p5/m, z8.d, z23.d, 270 \n\t" \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YP_PROJ
-#define YP_PROJ_A64FXd  \
-{ \
-asm ( \
-    "fsub z12.d, p5/m, z12.d, z21.d \n\t" \
-    "fsub z13.d, p5/m, z13.d, z22.d \n\t" \
-    "fsub z14.d, p5/m, z14.d, z23.d \n\t" \
-    "fadd z15.d, p5/m, z15.d, z18.d \n\t"  \
-    "fadd z16.d, p5/m, z16.d, z19.d \n\t"  \
-    "fadd z17.d, p5/m, z17.d, z20.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// ZP_PROJ
-#define ZP_PROJ_A64FXd  \
-{ \
-asm ( \
-    "fcadd z12.d, p5/m, z12.d, z18.d, 90 \n\t" \
-    "fcadd z13.d, p5/m, z13.d, z19.d, 90 \n\t" \
-    "fcadd z14.d, p5/m, z14.d, z20.d, 90 \n\t" \
-    "fcadd z15.d, p5/m, z15.d, z21.d, 270 \n\t" \
-    "fcadd z16.d, p5/m, z16.d, z22.d, 270 \n\t" \
-    "fcadd z17.d, p5/m, z17.d, z23.d, 270 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// TP_PROJ
-#define TP_PROJ_A64FXd  \
-{ \
-asm ( \
-    "fadd z12.d, p5/m, z12.d, z18.d \n\t"  \
-    "fadd z13.d, p5/m, z13.d, z19.d \n\t"  \
-    "fadd z14.d, p5/m, z14.d, z20.d \n\t"  \
-    "fadd z15.d, p5/m, z15.d, z21.d \n\t"  \
-    "fadd z16.d, p5/m, z16.d, z22.d \n\t"  \
-    "fadd z17.d, p5/m, z17.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XM_PROJ
-#define XM_PROJ_A64FXd  \
-{ \
-asm ( \
-    "fcadd z12.d, p5/m, z12.d, z21.d, 270 \n\t" \
-    "fcadd z13.d, p5/m, z13.d, z22.d, 270 \n\t" \
-    "fcadd z14.d, p5/m, z14.d, z23.d, 270 \n\t" \
-    "fcadd z15.d, p5/m, z15.d, z18.d, 270 \n\t" \
-    "fcadd z16.d, p5/m, z16.d, z19.d, 270 \n\t" \
-    "fcadd z17.d, p5/m, z17.d, z20.d, 270 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XM_RECON
-#define XM_RECON_A64FXd  \
-asm ( \
-    "movprfx z6.d, p5/m, z31.d \n\t" \
-    "fcadd z6.d, p5/m, z6.d, z21.d, 90 \n\t" \
-    "movprfx z7.d, p5/m, z31.d \n\t" \
-    "fcadd z7.d, p5/m, z7.d, z22.d, 90 \n\t" \
-    "movprfx z8.d, p5/m, z31.d \n\t" \
-    "fcadd z8.d, p5/m, z8.d, z23.d, 90 \n\t" \
-    "movprfx z9.d, p5/m, z31.d \n\t" \
-    "fcadd z9.d, p5/m, z9.d, z18.d, 90 \n\t" \
-    "movprfx z10.d, p5/m, z31.d \n\t" \
-    "fcadd z10.d, p5/m, z10.d, z19.d, 90 \n\t" \
-    "movprfx z11.d, p5/m, z31.d \n\t" \
-    "fcadd z11.d, p5/m, z11.d, z20.d, 90 \n\t" \
-    "mov z0.d, p5/m, z18.d \n\t" \
-    "mov z1.d, p5/m, z19.d \n\t" \
-    "mov z2.d, p5/m, z20.d \n\t" \
-    "mov z3.d, p5/m, z21.d \n\t" \
-    "mov z4.d, p5/m, z22.d \n\t" \
-    "mov z5.d, p5/m, z23.d \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YM_PROJ
-#define YM_PROJ_A64FXd  \
-{ \
-asm ( \
-    "fadd z12.d, p5/m, z12.d, z21.d \n\t"  \
-    "fadd z13.d, p5/m, z13.d, z22.d \n\t"  \
-    "fadd z14.d, p5/m, z14.d, z23.d \n\t"  \
-    "fsub z15.d, p5/m, z15.d, z18.d \n\t" \
-    "fsub z16.d, p5/m, z16.d, z19.d \n\t" \
-    "fsub z17.d, p5/m, z17.d, z20.d \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// ZM_PROJ
-#define ZM_PROJ_A64FXd  \
-{ \
-asm ( \
-    "fcadd z12.d, p5/m, z12.d, z18.d, 270 \n\t" \
-    "fcadd z13.d, p5/m, z13.d, z19.d, 270 \n\t" \
-    "fcadd z14.d, p5/m, z14.d, z20.d, 270 \n\t" \
-    "fcadd z15.d, p5/m, z15.d, z21.d, 90 \n\t" \
-    "fcadd z16.d, p5/m, z16.d, z22.d, 90 \n\t" \
-    "fcadd z17.d, p5/m, z17.d, z23.d, 90 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// TM_PROJ
-#define TM_PROJ_A64FXd  \
-{ \
-asm ( \
-    "ptrue p5.d \n\t" \
-    "fsub z12.d, p5/m, z12.d, z18.d \n\t" \
-    "fsub z13.d, p5/m, z13.d, z19.d \n\t" \
-    "fsub z14.d, p5/m, z14.d, z20.d \n\t" \
-    "fsub z15.d, p5/m, z15.d, z21.d \n\t" \
-    "fsub z16.d, p5/m, z16.d, z22.d \n\t" \
-    "fsub z17.d, p5/m, z17.d, z23.d \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XM_RECON_ACCUM
-#define XM_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fcadd z9.d, p5/m, z9.d, z18.d, 90 \n\t" \
-    "fcadd z10.d, p5/m, z10.d, z19.d, 90 \n\t" \
-    "fcadd z11.d, p5/m, z11.d, z20.d, 90 \n\t" \
-    "fcadd z6.d, p5/m, z6.d, z21.d, 90 \n\t" \
-    "fcadd z7.d, p5/m, z7.d, z22.d, 90 \n\t" \
-    "fcadd z8.d, p5/m, z8.d, z23.d, 90 \n\t" \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YP_RECON_ACCUM
-#define YP_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fsub z9.d, p5/m, z9.d, z18.d \n\t" \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fsub z10.d, p5/m, z10.d, z19.d \n\t" \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fsub z11.d, p5/m, z11.d, z20.d \n\t" \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fadd z6.d, p5/m, z6.d, z21.d \n\t"  \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fadd z7.d, p5/m, z7.d, z22.d \n\t"  \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    "fadd z8.d, p5/m, z8.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YM_RECON_ACCUM
-#define YM_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fadd z9.d, p5/m, z9.d, z18.d \n\t"  \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fadd z10.d, p5/m, z10.d, z19.d \n\t"  \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fadd z11.d, p5/m, z11.d, z20.d \n\t"  \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fsub z6.d, p5/m, z6.d, z21.d \n\t" \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fsub z7.d, p5/m, z7.d, z22.d \n\t" \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    "fsub z8.d, p5/m, z8.d, z23.d \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// ZP_RECON_ACCUM
-#define ZP_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fcadd z6.d, p5/m, z6.d, z18.d, 270 \n\t" \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fcadd z7.d, p5/m, z7.d, z19.d, 270 \n\t" \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fcadd z8.d, p5/m, z8.d, z20.d, 270 \n\t" \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fcadd z9.d, p5/m, z9.d, z21.d, 90 \n\t" \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fcadd z10.d, p5/m, z10.d, z22.d, 90 \n\t" \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fcadd z11.d, p5/m, z11.d, z23.d, 90 \n\t" \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// ZM_RECON_ACCUM
-#define ZM_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fcadd z6.d, p5/m, z6.d, z18.d, 90 \n\t" \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fcadd z7.d, p5/m, z7.d, z19.d, 90 \n\t" \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fcadd z8.d, p5/m, z8.d, z20.d, 90 \n\t" \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fcadd z9.d, p5/m, z9.d, z21.d, 270 \n\t" \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fcadd z10.d, p5/m, z10.d, z22.d, 270 \n\t" \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fcadd z11.d, p5/m, z11.d, z23.d, 270 \n\t" \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// TP_RECON_ACCUM
-#define TP_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fadd z6.d, p5/m, z6.d, z18.d \n\t"  \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fadd z7.d, p5/m, z7.d, z19.d \n\t"  \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fadd z8.d, p5/m, z8.d, z20.d \n\t"  \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fadd z9.d, p5/m, z9.d, z21.d \n\t"  \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fadd z10.d, p5/m, z10.d, z22.d \n\t"  \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    "fadd z11.d, p5/m, z11.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// TM_RECON_ACCUM
-#define TM_RECON_ACCUM_A64FXd  \
-asm ( \
-    "fadd z0.d, p5/m, z0.d, z18.d \n\t"  \
-    "fsub z6.d, p5/m, z6.d, z18.d \n\t" \
-    "fadd z1.d, p5/m, z1.d, z19.d \n\t"  \
-    "fsub z7.d, p5/m, z7.d, z19.d \n\t" \
-    "fadd z2.d, p5/m, z2.d, z20.d \n\t"  \
-    "fsub z8.d, p5/m, z8.d, z20.d \n\t" \
-    "fadd z3.d, p5/m, z3.d, z21.d \n\t"  \
-    "fsub z9.d, p5/m, z9.d, z21.d \n\t" \
-    "fadd z4.d, p5/m, z4.d, z22.d \n\t"  \
-    "fsub z10.d, p5/m, z10.d, z22.d \n\t" \
-    "fadd z5.d, p5/m, z5.d, z23.d \n\t"  \
-    "fsub z11.d, p5/m, z11.d, z23.d \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// ZERO_PSI
-#define ZERO_PSI_A64FXd  \
-asm ( \
-    "ptrue p5.d \n\t" \
-    "fmov z0.d , 0 \n\t" \
-    "fmov z1.d , 0 \n\t" \
-    "fmov z2.d , 0 \n\t" \
-    "fmov z3.d , 0 \n\t" \
-    "fmov z4.d , 0 \n\t" \
-    "fmov z5.d , 0 \n\t" \
-    "fmov z6.d , 0 \n\t" \
-    "fmov z7.d , 0 \n\t" \
-    "fmov z8.d , 0 \n\t" \
-    "fmov z9.d , 0 \n\t" \
-    "fmov z10.d , 0 \n\t" \
-    "fmov z11.d , 0 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// PREFETCH_RESULT_L2_STORE (prefetch store to L2)
-#define PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXd(base)  \
-{ \
-asm ( \
-    "prfd PSTL2STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PSTL2STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PSTL2STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_RESULT_L1_STORE (prefetch store to L1)
-#define PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXd(base)  \
-{ \
-asm ( \
-    "prfd PSTL1STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PSTL1STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PSTL1STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// ADD_RESULT_INTERNAL
-#define ADD_RESULT_INTERNAL_A64FXd  \
-asm ( \
-    "fadd z0.d, p5/m, z0.d, z12.d \n\t"  \
-    "fadd z1.d, p5/m, z1.d, z13.d \n\t"  \
-    "fadd z2.d, p5/m, z2.d, z14.d \n\t"  \
-    "fadd z3.d, p5/m, z3.d, z15.d \n\t"  \
-    "fadd z4.d, p5/m, z4.d, z16.d \n\t"  \
-    "fadd z5.d, p5/m, z5.d, z17.d \n\t"  \
-    "fadd z6.d, p5/m, z6.d, z18.d \n\t"  \
-    "fadd z7.d, p5/m, z7.d, z19.d \n\t"  \
-    "fadd z8.d, p5/m, z8.d, z20.d \n\t"  \
-    "fadd z9.d, p5/m, z9.d, z21.d \n\t"  \
-    "fadd z10.d, p5/m, z10.d, z22.d \n\t"  \
-    "fadd z11.d, p5/m, z11.d, z23.d \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-

Grid/simd/Fujitsu_A64FX_asm_single.h

@@ -1,779 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: Fujitsu_A64FX_asm_single.h
-
-    Copyright (C) 2020
-
-Author: Nils Meyer <nils.meyer@ur.de>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#define LOAD_CHIMU(base)               LOAD_CHIMU_INTERLEAVED_A64FXf(base)  
-#define PREFETCH_CHIMU_L1(A)           PREFETCH_CHIMU_L1_INTERNAL_A64FXf(A)  
-#define PREFETCH_GAUGE_L1(A)           PREFETCH_GAUGE_L1_INTERNAL_A64FXf(A)  
-#define PREFETCH_CHIMU_L2(A)           PREFETCH_CHIMU_L2_INTERNAL_A64FXf(A)  
-#define PREFETCH_GAUGE_L2(A)           PREFETCH_GAUGE_L2_INTERNAL_A64FXf(A)  
-#define PF_GAUGE(A)  
-#define PREFETCH_RESULT_L2_STORE(A)    PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXf(A)  
-#define PREFETCH_RESULT_L1_STORE(A)    PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXf(A)  
-#define PREFETCH1_CHIMU(A)             PREFETCH_CHIMU_L1(A)  
-#define PREFETCH_CHIMU(A)              PREFETCH_CHIMU_L1(A)  
-#define LOCK_GAUGE(A)  
-#define UNLOCK_GAUGE(A)  
-#define MASK_REGS                      DECLARATIONS_A64FXf  
-#define SAVE_RESULT(A,B)               RESULT_A64FXf(A); PREFETCH_RESULT_L2_STORE(B)  
-#define MULT_2SPIN_1(Dir)              MULT_2SPIN_1_A64FXf(Dir)  
-#define MULT_2SPIN_2                   MULT_2SPIN_2_A64FXf  
-#define LOAD_CHI(base)                 LOAD_CHI_A64FXf(base)  
-#define ADD_RESULT(base,basep)         LOAD_CHIMU(base); ADD_RESULT_INTERNAL_A64FXf; RESULT_A64FXf(base)  
-#define XP_PROJ                        XP_PROJ_A64FXf  
-#define YP_PROJ                        YP_PROJ_A64FXf  
-#define ZP_PROJ                        ZP_PROJ_A64FXf  
-#define TP_PROJ                        TP_PROJ_A64FXf  
-#define XM_PROJ                        XM_PROJ_A64FXf  
-#define YM_PROJ                        YM_PROJ_A64FXf  
-#define ZM_PROJ                        ZM_PROJ_A64FXf  
-#define TM_PROJ                        TM_PROJ_A64FXf  
-#define XP_RECON                       XP_RECON_A64FXf  
-#define XM_RECON                       XM_RECON_A64FXf  
-#define XM_RECON_ACCUM                 XM_RECON_ACCUM_A64FXf  
-#define YM_RECON_ACCUM                 YM_RECON_ACCUM_A64FXf  
-#define ZM_RECON_ACCUM                 ZM_RECON_ACCUM_A64FXf  
-#define TM_RECON_ACCUM                 TM_RECON_ACCUM_A64FXf  
-#define XP_RECON_ACCUM                 XP_RECON_ACCUM_A64FXf  
-#define YP_RECON_ACCUM                 YP_RECON_ACCUM_A64FXf  
-#define ZP_RECON_ACCUM                 ZP_RECON_ACCUM_A64FXf  
-#define TP_RECON_ACCUM                 TP_RECON_ACCUM_A64FXf  
-#define PERMUTE_DIR0                   0  
-#define PERMUTE_DIR1                   1  
-#define PERMUTE_DIR2                   2  
-#define PERMUTE_DIR3                   3  
-#define PERMUTE                        PERMUTE_A64FXf;  
-#define LOAD_TABLE(Dir)                if (Dir == 0) { LOAD_TABLE0; } else if (Dir == 1) { LOAD_TABLE1 } else if (Dir == 2) { LOAD_TABLE2; } else if (Dir == 3) { LOAD_TABLE3; }  
-#define MAYBEPERM(A,perm)              if (perm) { PERMUTE; }  
-// DECLARATIONS
-#define DECLARATIONS_A64FXf  \
-    const uint32_t lut[4][16] = { \
-        {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}, \
-        {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11}, \
-        {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}, \
-        {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14} }; \
-asm ( \
-    "fmov z31.s , 0 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// RESULT
-#define RESULT_A64FXf(base)  \
-{ \
-asm ( \
-    "str z0, [%[storeptr], -6, mul vl] \n\t" \
-    "str z1, [%[storeptr], -5, mul vl] \n\t" \
-    "str z2, [%[storeptr], -4, mul vl] \n\t" \
-    "str z3, [%[storeptr], -3, mul vl] \n\t" \
-    "str z4, [%[storeptr], -2, mul vl] \n\t" \
-    "str z5, [%[storeptr], -1, mul vl] \n\t" \
-    "str z6, [%[storeptr], 0, mul vl] \n\t" \
-    "str z7, [%[storeptr], 1, mul vl] \n\t" \
-    "str z8, [%[storeptr], 2, mul vl] \n\t" \
-    "str z9, [%[storeptr], 3, mul vl] \n\t" \
-    "str z10, [%[storeptr], 4, mul vl] \n\t" \
-    "str z11, [%[storeptr], 5, mul vl] \n\t" \
-    :  \
-    : [storeptr] "r" (base + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_CHIMU_L2 (prefetch to L2)
-#define PREFETCH_CHIMU_L2_INTERNAL_A64FXf(base)  \
-{ \
-asm ( \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_CHIMU_L1 (prefetch to L1)
-#define PREFETCH_CHIMU_L1_INTERNAL_A64FXf(base)  \
-{ \
-asm ( \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_GAUGE_L2 (prefetch to L2)
-#define PREFETCH_GAUGE_L2_INTERNAL_A64FXf(A)  \
-{ \
-    const auto & ref(U[sUn](A)); uint64_t baseU = (uint64_t)&ref + 3 * 3 * 64; \
-asm ( \
-    "prfd PLDL2STRM, p5, [%[fetchptr], -4, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 12, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 16, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 20, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 24, mul vl] \n\t" \
-    "prfd PLDL2STRM, p5, [%[fetchptr], 28, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_GAUGE_L1 (prefetch to L1)
-#define PREFETCH_GAUGE_L1_INTERNAL_A64FXf(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-asm ( \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PLDL1STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHI
-#define LOAD_CHI_A64FXf(base)  \
-{ \
-asm ( \
-    "ldr z12, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], 2, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], 5, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHIMU
-#define LOAD_CHIMU_INTERLEAVED_A64FXf(base)  \
-{ \
-asm ( \
-    "ptrue p5.s \n\t" \
-    "ldr z12, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z21, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z18, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z22, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z19, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z23, [%[fetchptr], 5, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z20, [%[fetchptr], 2, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHIMU_0213
-#define LOAD_CHIMU_0213_A64FXf  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-asm ( \
-    "ptrue p5.s \n\t" \
-    "ldr z12, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z18, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z19, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z20, [%[fetchptr], 2, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z21, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z22, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z23, [%[fetchptr], 5, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (&ref[2][0]) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_CHIMU_0312
-#define LOAD_CHIMU_0312_A64FXf  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-asm ( \
-    "ptrue p5.s \n\t" \
-    "ldr z12, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z21, [%[fetchptr], 3, mul vl] \n\t" \
-    "ldr z13, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z22, [%[fetchptr], 4, mul vl] \n\t" \
-    "ldr z14, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z23, [%[fetchptr], 5, mul vl] \n\t" \
-    "ldr z15, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z18, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z16, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z19, [%[fetchptr], 1, mul vl] \n\t" \
-    "ldr z17, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z20, [%[fetchptr], 2, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (&ref[2][0]) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// LOAD_TABLE0
-#define LOAD_TABLE0  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (0) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_TABLE1
-#define LOAD_TABLE1  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (1) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_TABLE2
-#define LOAD_TABLE2  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (2) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_TABLE3
-#define LOAD_TABLE3  \
-asm ( \
-    "ldr z30, [%[tableptr], %[index], mul vl] \n\t" \
-    :  \
-    : [tableptr] "r" (&lut[0]),[index] "i" (3) \
-    : "memory","cc","p5","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// PERMUTE
-#define PERMUTE_A64FXf  \
-asm ( \
-    "tbl z12.s, { z12.s }, z30.s \n\t"  \
-    "tbl z13.s, { z13.s }, z30.s \n\t"  \
-    "tbl z14.s, { z14.s }, z30.s \n\t"  \
-    "tbl z15.s, { z15.s }, z30.s \n\t"  \
-    "tbl z16.s, { z16.s }, z30.s \n\t"  \
-    "tbl z17.s, { z17.s }, z30.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// LOAD_GAUGE
-#define LOAD_GAUGE  \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-{ \
-asm ( \
-    "ptrue p5.s \n\t" \
-    "ldr z24, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z25, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z26, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z27, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z28, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z29, [%[fetchptr], 1, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// MULT_2SPIN
-#define MULT_2SPIN_1_A64FXf(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-asm ( \
-    "ldr z24, [%[fetchptr], -6, mul vl] \n\t" \
-    "ldr z25, [%[fetchptr], -3, mul vl] \n\t" \
-    "ldr z26, [%[fetchptr], 0, mul vl] \n\t" \
-    "ldr z27, [%[fetchptr], -5, mul vl] \n\t" \
-    "ldr z28, [%[fetchptr], -2, mul vl] \n\t" \
-    "ldr z29, [%[fetchptr], 1, mul vl] \n\t" \
-    "movprfx z18.s, p5/m, z31.s \n\t" \
-    "fcmla z18.s, p5/m, z24.s, z12.s, 0 \n\t" \
-    "movprfx z21.s, p5/m, z31.s \n\t" \
-    "fcmla z21.s, p5/m, z24.s, z15.s, 0 \n\t" \
-    "movprfx z19.s, p5/m, z31.s \n\t" \
-    "fcmla z19.s, p5/m, z25.s, z12.s, 0 \n\t" \
-    "movprfx z22.s, p5/m, z31.s \n\t" \
-    "fcmla z22.s, p5/m, z25.s, z15.s, 0 \n\t" \
-    "movprfx z20.s, p5/m, z31.s \n\t" \
-    "fcmla z20.s, p5/m, z26.s, z12.s, 0 \n\t" \
-    "movprfx z23.s, p5/m, z31.s \n\t" \
-    "fcmla z23.s, p5/m, z26.s, z15.s, 0 \n\t" \
-    "fcmla z18.s, p5/m, z24.s, z12.s, 90 \n\t" \
-    "fcmla z21.s, p5/m, z24.s, z15.s, 90 \n\t" \
-    "fcmla z19.s, p5/m, z25.s, z12.s, 90 \n\t" \
-    "fcmla z22.s, p5/m, z25.s, z15.s, 90 \n\t" \
-    "fcmla z20.s, p5/m, z26.s, z12.s, 90 \n\t" \
-    "fcmla z23.s, p5/m, z26.s, z15.s, 90 \n\t" \
-    "ldr z24, [%[fetchptr], -4, mul vl] \n\t" \
-    "ldr z25, [%[fetchptr], -1, mul vl] \n\t" \
-    "ldr z26, [%[fetchptr], 2, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (baseU + 2 * 3 * 64) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// MULT_2SPIN_BACKEND
-#define MULT_2SPIN_2_A64FXf  \
-{ \
-asm ( \
-    "fcmla z18.s, p5/m, z27.s, z13.s, 0 \n\t" \
-    "fcmla z21.s, p5/m, z27.s, z16.s, 0 \n\t" \
-    "fcmla z19.s, p5/m, z28.s, z13.s, 0 \n\t" \
-    "fcmla z22.s, p5/m, z28.s, z16.s, 0 \n\t" \
-    "fcmla z20.s, p5/m, z29.s, z13.s, 0 \n\t" \
-    "fcmla z23.s, p5/m, z29.s, z16.s, 0 \n\t" \
-    "fcmla z18.s, p5/m, z27.s, z13.s, 90 \n\t" \
-    "fcmla z21.s, p5/m, z27.s, z16.s, 90 \n\t" \
-    "fcmla z19.s, p5/m, z28.s, z13.s, 90 \n\t" \
-    "fcmla z22.s, p5/m, z28.s, z16.s, 90 \n\t" \
-    "fcmla z20.s, p5/m, z29.s, z13.s, 90 \n\t" \
-    "fcmla z23.s, p5/m, z29.s, z16.s, 90 \n\t" \
-    "fcmla z18.s, p5/m, z24.s, z14.s, 0 \n\t" \
-    "fcmla z21.s, p5/m, z24.s, z17.s, 0 \n\t" \
-    "fcmla z19.s, p5/m, z25.s, z14.s, 0 \n\t" \
-    "fcmla z22.s, p5/m, z25.s, z17.s, 0 \n\t" \
-    "fcmla z20.s, p5/m, z26.s, z14.s, 0 \n\t" \
-    "fcmla z23.s, p5/m, z26.s, z17.s, 0 \n\t" \
-    "fcmla z18.s, p5/m, z24.s, z14.s, 90 \n\t" \
-    "fcmla z21.s, p5/m, z24.s, z17.s, 90 \n\t" \
-    "fcmla z19.s, p5/m, z25.s, z14.s, 90 \n\t" \
-    "fcmla z22.s, p5/m, z25.s, z17.s, 90 \n\t" \
-    "fcmla z20.s, p5/m, z26.s, z14.s, 90 \n\t" \
-    "fcmla z23.s, p5/m, z26.s, z17.s, 90 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XP_PROJ
-#define XP_PROJ_A64FXf  \
-{ \
-asm ( \
-    "fcadd z12.s, p5/m, z12.s, z21.s, 90 \n\t" \
-    "fcadd z13.s, p5/m, z13.s, z22.s, 90 \n\t" \
-    "fcadd z14.s, p5/m, z14.s, z23.s, 90 \n\t" \
-    "fcadd z15.s, p5/m, z15.s, z18.s, 90 \n\t" \
-    "fcadd z16.s, p5/m, z16.s, z19.s, 90 \n\t" \
-    "fcadd z17.s, p5/m, z17.s, z20.s, 90 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XP_RECON
-#define XP_RECON_A64FXf  \
-asm ( \
-    "movprfx z6.s, p5/m, z31.s \n\t" \
-    "fcadd z6.s, p5/m, z6.s, z21.s, 270 \n\t" \
-    "movprfx z7.s, p5/m, z31.s \n\t" \
-    "fcadd z7.s, p5/m, z7.s, z22.s, 270 \n\t" \
-    "movprfx z8.s, p5/m, z31.s \n\t" \
-    "fcadd z8.s, p5/m, z8.s, z23.s, 270 \n\t" \
-    "movprfx z9.s, p5/m, z31.s \n\t" \
-    "fcadd z9.s, p5/m, z9.s, z18.s, 270 \n\t" \
-    "movprfx z10.s, p5/m, z31.s \n\t" \
-    "fcadd z10.s, p5/m, z10.s, z19.s, 270 \n\t" \
-    "movprfx z11.s, p5/m, z31.s \n\t" \
-    "fcadd z11.s, p5/m, z11.s, z20.s, 270 \n\t" \
-    "mov z0.s, p5/m, z18.s \n\t" \
-    "mov z1.s, p5/m, z19.s \n\t" \
-    "mov z2.s, p5/m, z20.s \n\t" \
-    "mov z3.s, p5/m, z21.s \n\t" \
-    "mov z4.s, p5/m, z22.s \n\t" \
-    "mov z5.s, p5/m, z23.s \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// XP_RECON_ACCUM
-#define XP_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fcadd z9.s, p5/m, z9.s, z18.s, 270 \n\t" \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fcadd z10.s, p5/m, z10.s, z19.s, 270 \n\t" \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fcadd z11.s, p5/m, z11.s, z20.s, 270 \n\t" \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fcadd z6.s, p5/m, z6.s, z21.s, 270 \n\t" \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fcadd z7.s, p5/m, z7.s, z22.s, 270 \n\t" \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fcadd z8.s, p5/m, z8.s, z23.s, 270 \n\t" \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YP_PROJ
-#define YP_PROJ_A64FXf  \
-{ \
-asm ( \
-    "fsub z12.s, p5/m, z12.s, z21.s \n\t" \
-    "fsub z13.s, p5/m, z13.s, z22.s \n\t" \
-    "fsub z14.s, p5/m, z14.s, z23.s \n\t" \
-    "fadd z15.s, p5/m, z15.s, z18.s \n\t"  \
-    "fadd z16.s, p5/m, z16.s, z19.s \n\t"  \
-    "fadd z17.s, p5/m, z17.s, z20.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// ZP_PROJ
-#define ZP_PROJ_A64FXf  \
-{ \
-asm ( \
-    "fcadd z12.s, p5/m, z12.s, z18.s, 90 \n\t" \
-    "fcadd z13.s, p5/m, z13.s, z19.s, 90 \n\t" \
-    "fcadd z14.s, p5/m, z14.s, z20.s, 90 \n\t" \
-    "fcadd z15.s, p5/m, z15.s, z21.s, 270 \n\t" \
-    "fcadd z16.s, p5/m, z16.s, z22.s, 270 \n\t" \
-    "fcadd z17.s, p5/m, z17.s, z23.s, 270 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// TP_PROJ
-#define TP_PROJ_A64FXf  \
-{ \
-asm ( \
-    "fadd z12.s, p5/m, z12.s, z18.s \n\t"  \
-    "fadd z13.s, p5/m, z13.s, z19.s \n\t"  \
-    "fadd z14.s, p5/m, z14.s, z20.s \n\t"  \
-    "fadd z15.s, p5/m, z15.s, z21.s \n\t"  \
-    "fadd z16.s, p5/m, z16.s, z22.s \n\t"  \
-    "fadd z17.s, p5/m, z17.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XM_PROJ
-#define XM_PROJ_A64FXf  \
-{ \
-asm ( \
-    "fcadd z12.s, p5/m, z12.s, z21.s, 270 \n\t" \
-    "fcadd z13.s, p5/m, z13.s, z22.s, 270 \n\t" \
-    "fcadd z14.s, p5/m, z14.s, z23.s, 270 \n\t" \
-    "fcadd z15.s, p5/m, z15.s, z18.s, 270 \n\t" \
-    "fcadd z16.s, p5/m, z16.s, z19.s, 270 \n\t" \
-    "fcadd z17.s, p5/m, z17.s, z20.s, 270 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XM_RECON
-#define XM_RECON_A64FXf  \
-asm ( \
-    "movprfx z6.s, p5/m, z31.s \n\t" \
-    "fcadd z6.s, p5/m, z6.s, z21.s, 90 \n\t" \
-    "movprfx z7.s, p5/m, z31.s \n\t" \
-    "fcadd z7.s, p5/m, z7.s, z22.s, 90 \n\t" \
-    "movprfx z8.s, p5/m, z31.s \n\t" \
-    "fcadd z8.s, p5/m, z8.s, z23.s, 90 \n\t" \
-    "movprfx z9.s, p5/m, z31.s \n\t" \
-    "fcadd z9.s, p5/m, z9.s, z18.s, 90 \n\t" \
-    "movprfx z10.s, p5/m, z31.s \n\t" \
-    "fcadd z10.s, p5/m, z10.s, z19.s, 90 \n\t" \
-    "movprfx z11.s, p5/m, z31.s \n\t" \
-    "fcadd z11.s, p5/m, z11.s, z20.s, 90 \n\t" \
-    "mov z0.s, p5/m, z18.s \n\t" \
-    "mov z1.s, p5/m, z19.s \n\t" \
-    "mov z2.s, p5/m, z20.s \n\t" \
-    "mov z3.s, p5/m, z21.s \n\t" \
-    "mov z4.s, p5/m, z22.s \n\t" \
-    "mov z5.s, p5/m, z23.s \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YM_PROJ
-#define YM_PROJ_A64FXf  \
-{ \
-asm ( \
-    "fadd z12.s, p5/m, z12.s, z21.s \n\t"  \
-    "fadd z13.s, p5/m, z13.s, z22.s \n\t"  \
-    "fadd z14.s, p5/m, z14.s, z23.s \n\t"  \
-    "fsub z15.s, p5/m, z15.s, z18.s \n\t" \
-    "fsub z16.s, p5/m, z16.s, z19.s \n\t" \
-    "fsub z17.s, p5/m, z17.s, z20.s \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// ZM_PROJ
-#define ZM_PROJ_A64FXf  \
-{ \
-asm ( \
-    "fcadd z12.s, p5/m, z12.s, z18.s, 270 \n\t" \
-    "fcadd z13.s, p5/m, z13.s, z19.s, 270 \n\t" \
-    "fcadd z14.s, p5/m, z14.s, z20.s, 270 \n\t" \
-    "fcadd z15.s, p5/m, z15.s, z21.s, 90 \n\t" \
-    "fcadd z16.s, p5/m, z16.s, z22.s, 90 \n\t" \
-    "fcadd z17.s, p5/m, z17.s, z23.s, 90 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// TM_PROJ
-#define TM_PROJ_A64FXf  \
-{ \
-asm ( \
-    "ptrue p5.s \n\t" \
-    "fsub z12.s, p5/m, z12.s, z18.s \n\t" \
-    "fsub z13.s, p5/m, z13.s, z19.s \n\t" \
-    "fsub z14.s, p5/m, z14.s, z20.s \n\t" \
-    "fsub z15.s, p5/m, z15.s, z21.s \n\t" \
-    "fsub z16.s, p5/m, z16.s, z22.s \n\t" \
-    "fsub z17.s, p5/m, z17.s, z23.s \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); \
-}
-// XM_RECON_ACCUM
-#define XM_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fcadd z9.s, p5/m, z9.s, z18.s, 90 \n\t" \
-    "fcadd z10.s, p5/m, z10.s, z19.s, 90 \n\t" \
-    "fcadd z11.s, p5/m, z11.s, z20.s, 90 \n\t" \
-    "fcadd z6.s, p5/m, z6.s, z21.s, 90 \n\t" \
-    "fcadd z7.s, p5/m, z7.s, z22.s, 90 \n\t" \
-    "fcadd z8.s, p5/m, z8.s, z23.s, 90 \n\t" \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YP_RECON_ACCUM
-#define YP_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fsub z9.s, p5/m, z9.s, z18.s \n\t" \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fsub z10.s, p5/m, z10.s, z19.s \n\t" \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fsub z11.s, p5/m, z11.s, z20.s \n\t" \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fadd z6.s, p5/m, z6.s, z21.s \n\t"  \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fadd z7.s, p5/m, z7.s, z22.s \n\t"  \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    "fadd z8.s, p5/m, z8.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// YM_RECON_ACCUM
-#define YM_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fadd z9.s, p5/m, z9.s, z18.s \n\t"  \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fadd z10.s, p5/m, z10.s, z19.s \n\t"  \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fadd z11.s, p5/m, z11.s, z20.s \n\t"  \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fsub z6.s, p5/m, z6.s, z21.s \n\t" \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fsub z7.s, p5/m, z7.s, z22.s \n\t" \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    "fsub z8.s, p5/m, z8.s, z23.s \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// ZP_RECON_ACCUM
-#define ZP_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fcadd z6.s, p5/m, z6.s, z18.s, 270 \n\t" \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fcadd z7.s, p5/m, z7.s, z19.s, 270 \n\t" \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fcadd z8.s, p5/m, z8.s, z20.s, 270 \n\t" \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fcadd z9.s, p5/m, z9.s, z21.s, 90 \n\t" \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fcadd z10.s, p5/m, z10.s, z22.s, 90 \n\t" \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fcadd z11.s, p5/m, z11.s, z23.s, 90 \n\t" \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// ZM_RECON_ACCUM
-#define ZM_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fcadd z6.s, p5/m, z6.s, z18.s, 90 \n\t" \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fcadd z7.s, p5/m, z7.s, z19.s, 90 \n\t" \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fcadd z8.s, p5/m, z8.s, z20.s, 90 \n\t" \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fcadd z9.s, p5/m, z9.s, z21.s, 270 \n\t" \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fcadd z10.s, p5/m, z10.s, z22.s, 270 \n\t" \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fcadd z11.s, p5/m, z11.s, z23.s, 270 \n\t" \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// TP_RECON_ACCUM
-#define TP_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fadd z6.s, p5/m, z6.s, z18.s \n\t"  \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fadd z7.s, p5/m, z7.s, z19.s \n\t"  \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fadd z8.s, p5/m, z8.s, z20.s \n\t"  \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fadd z9.s, p5/m, z9.s, z21.s \n\t"  \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fadd z10.s, p5/m, z10.s, z22.s \n\t"  \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    "fadd z11.s, p5/m, z11.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// TM_RECON_ACCUM
-#define TM_RECON_ACCUM_A64FXf  \
-asm ( \
-    "fadd z0.s, p5/m, z0.s, z18.s \n\t"  \
-    "fsub z6.s, p5/m, z6.s, z18.s \n\t" \
-    "fadd z1.s, p5/m, z1.s, z19.s \n\t"  \
-    "fsub z7.s, p5/m, z7.s, z19.s \n\t" \
-    "fadd z2.s, p5/m, z2.s, z20.s \n\t"  \
-    "fsub z8.s, p5/m, z8.s, z20.s \n\t" \
-    "fadd z3.s, p5/m, z3.s, z21.s \n\t"  \
-    "fsub z9.s, p5/m, z9.s, z21.s \n\t" \
-    "fadd z4.s, p5/m, z4.s, z22.s \n\t"  \
-    "fsub z10.s, p5/m, z10.s, z22.s \n\t" \
-    "fadd z5.s, p5/m, z5.s, z23.s \n\t"  \
-    "fsub z11.s, p5/m, z11.s, z23.s \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// ZERO_PSI
-#define ZERO_PSI_A64FXf  \
-asm ( \
-    "ptrue p5.s \n\t" \
-    "fmov z0.s , 0 \n\t" \
-    "fmov z1.s , 0 \n\t" \
-    "fmov z2.s , 0 \n\t" \
-    "fmov z3.s , 0 \n\t" \
-    "fmov z4.s , 0 \n\t" \
-    "fmov z5.s , 0 \n\t" \
-    "fmov z6.s , 0 \n\t" \
-    "fmov z7.s , 0 \n\t" \
-    "fmov z8.s , 0 \n\t" \
-    "fmov z9.s , 0 \n\t" \
-    "fmov z10.s , 0 \n\t" \
-    "fmov z11.s , 0 \n\t" \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-
-// PREFETCH_RESULT_L2_STORE (prefetch store to L2)
-#define PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXf(base)  \
-{ \
-asm ( \
-    "prfd PSTL2STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PSTL2STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PSTL2STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// PREFETCH_RESULT_L1_STORE (prefetch store to L1)
-#define PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXf(base)  \
-{ \
-asm ( \
-    "prfd PSTL1STRM, p5, [%[fetchptr], 0, mul vl] \n\t" \
-    "prfd PSTL1STRM, p5, [%[fetchptr], 4, mul vl] \n\t" \
-    "prfd PSTL1STRM, p5, [%[fetchptr], 8, mul vl] \n\t" \
-    :  \
-    : [fetchptr] "r" (base) \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31","memory" \
-); \
-}
-// ADD_RESULT_INTERNAL
-#define ADD_RESULT_INTERNAL_A64FXf  \
-asm ( \
-    "fadd z0.s, p5/m, z0.s, z12.s \n\t"  \
-    "fadd z1.s, p5/m, z1.s, z13.s \n\t"  \
-    "fadd z2.s, p5/m, z2.s, z14.s \n\t"  \
-    "fadd z3.s, p5/m, z3.s, z15.s \n\t"  \
-    "fadd z4.s, p5/m, z4.s, z16.s \n\t"  \
-    "fadd z5.s, p5/m, z5.s, z17.s \n\t"  \
-    "fadd z6.s, p5/m, z6.s, z18.s \n\t"  \
-    "fadd z7.s, p5/m, z7.s, z19.s \n\t"  \
-    "fadd z8.s, p5/m, z8.s, z20.s \n\t"  \
-    "fadd z9.s, p5/m, z9.s, z21.s \n\t"  \
-    "fadd z10.s, p5/m, z10.s, z22.s \n\t"  \
-    "fadd z11.s, p5/m, z11.s, z23.s \n\t"  \
-    :  \
-    :  \
-    : "p5","cc","z0","z1","z2","z3","z4","z5","z6","z7","z8","z9","z10","z11","z12","z13","z14","z15","z16","z17","z18","z19","z20","z21","z22","z23","z24","z25","z26","z27","z28","z29","z30","z31" \
-); 
-

Grid/simd/Fujitsu_A64FX_intrin_double.h

@@ -1,601 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: Fujitsu_A64FX_intrin_double.h
-
-    Copyright (C) 2020
-
-Author: Nils Meyer <nils.meyer@ur.de>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#define LOAD_CHIMU(base)               LOAD_CHIMU_INTERLEAVED_A64FXd(base)  
-#define PREFETCH_CHIMU_L1(A)           PREFETCH_CHIMU_L1_INTERNAL_A64FXd(A)  
-#define PREFETCH_GAUGE_L1(A)           PREFETCH_GAUGE_L1_INTERNAL_A64FXd(A)  
-#define PREFETCH_CHIMU_L2(A)           PREFETCH_CHIMU_L2_INTERNAL_A64FXd(A)  
-#define PREFETCH_GAUGE_L2(A)           PREFETCH_GAUGE_L2_INTERNAL_A64FXd(A)  
-#define PF_GAUGE(A)  
-#define PREFETCH_RESULT_L2_STORE(A)    PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXd(A)  
-#define PREFETCH_RESULT_L1_STORE(A)    PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXd(A)  
-#define PREFETCH1_CHIMU(A)             PREFETCH_CHIMU_L1(A)  
-#define PREFETCH_CHIMU(A)              PREFETCH_CHIMU_L1(A)  
-#define LOCK_GAUGE(A)  
-#define UNLOCK_GAUGE(A)  
-#define MASK_REGS                      DECLARATIONS_A64FXd  
-#define SAVE_RESULT(A,B)               RESULT_A64FXd(A); PREFETCH_RESULT_L2_STORE(B)  
-#define MULT_2SPIN_1(Dir)              MULT_2SPIN_1_A64FXd(Dir)  
-#define MULT_2SPIN_2                   MULT_2SPIN_2_A64FXd  
-#define LOAD_CHI(base)                 LOAD_CHI_A64FXd(base)  
-#define ADD_RESULT(base,basep)         LOAD_CHIMU(base); ADD_RESULT_INTERNAL_A64FXd; RESULT_A64FXd(base)  
-#define XP_PROJ                        XP_PROJ_A64FXd  
-#define YP_PROJ                        YP_PROJ_A64FXd  
-#define ZP_PROJ                        ZP_PROJ_A64FXd  
-#define TP_PROJ                        TP_PROJ_A64FXd  
-#define XM_PROJ                        XM_PROJ_A64FXd  
-#define YM_PROJ                        YM_PROJ_A64FXd  
-#define ZM_PROJ                        ZM_PROJ_A64FXd  
-#define TM_PROJ                        TM_PROJ_A64FXd  
-#define XP_RECON                       XP_RECON_A64FXd  
-#define XM_RECON                       XM_RECON_A64FXd  
-#define XM_RECON_ACCUM                 XM_RECON_ACCUM_A64FXd  
-#define YM_RECON_ACCUM                 YM_RECON_ACCUM_A64FXd  
-#define ZM_RECON_ACCUM                 ZM_RECON_ACCUM_A64FXd  
-#define TM_RECON_ACCUM                 TM_RECON_ACCUM_A64FXd  
-#define XP_RECON_ACCUM                 XP_RECON_ACCUM_A64FXd  
-#define YP_RECON_ACCUM                 YP_RECON_ACCUM_A64FXd  
-#define ZP_RECON_ACCUM                 ZP_RECON_ACCUM_A64FXd  
-#define TP_RECON_ACCUM                 TP_RECON_ACCUM_A64FXd  
-#define PERMUTE_DIR0                   0  
-#define PERMUTE_DIR1                   1  
-#define PERMUTE_DIR2                   2  
-#define PERMUTE_DIR3                   3  
-#define PERMUTE                        PERMUTE_A64FXd;  
-#define LOAD_TABLE(Dir)                if (Dir == 0) { LOAD_TABLE0; } else if (Dir == 1) { LOAD_TABLE1; } else if (Dir == 2) { LOAD_TABLE2; }  
-#define MAYBEPERM(Dir,perm)            if (Dir != 3) { if (perm) { PERMUTE; } }  
-// DECLARATIONS
-#define DECLARATIONS_A64FXd  \
-    const uint64_t lut[4][8] = { \
-        {4, 5, 6, 7, 0, 1, 2, 3}, \
-        {2, 3, 0, 1, 6, 7, 4, 5}, \
-        {1, 0, 3, 2, 5, 4, 7, 6}, \
-        {0, 1, 2, 4, 5, 6, 7, 8} };\
-    svfloat64_t result_00;        \
-    svfloat64_t result_01;        \
-    svfloat64_t result_02;        \
-    svfloat64_t result_10;        \
-    svfloat64_t result_11;        \
-    svfloat64_t result_12;        \
-    svfloat64_t result_20;        \
-    svfloat64_t result_21;        \
-    svfloat64_t result_22;        \
-    svfloat64_t result_30;        \
-    svfloat64_t result_31;        \
-    svfloat64_t result_32;        \
-    svfloat64_t Chi_00;        \
-    svfloat64_t Chi_01;        \
-    svfloat64_t Chi_02;        \
-    svfloat64_t Chi_10;        \
-    svfloat64_t Chi_11;        \
-    svfloat64_t Chi_12;        \
-    svfloat64_t UChi_00;        \
-    svfloat64_t UChi_01;        \
-    svfloat64_t UChi_02;        \
-    svfloat64_t UChi_10;        \
-    svfloat64_t UChi_11;        \
-    svfloat64_t UChi_12;        \
-    svfloat64_t U_00;        \
-    svfloat64_t U_10;        \
-    svfloat64_t U_20;        \
-    svfloat64_t U_01;        \
-    svfloat64_t U_11;        \
-    svfloat64_t U_21;        \
-    svbool_t pg1;        \
-    pg1 = svptrue_b64();        \
-    svuint64_t table0; \
-    svfloat64_t zero0;        \
-    zero0 = svdup_f64(0.); 
-
-#define Chimu_00 Chi_00  
-#define Chimu_01 Chi_01  
-#define Chimu_02 Chi_02  
-#define Chimu_10 Chi_10  
-#define Chimu_11 Chi_11  
-#define Chimu_12 Chi_12  
-#define Chimu_20 UChi_00  
-#define Chimu_21 UChi_01  
-#define Chimu_22 UChi_02  
-#define Chimu_30 UChi_10  
-#define Chimu_31 UChi_11  
-#define Chimu_32 UChi_12  
-// RESULT
-#define RESULT_A64FXd(base)  \
-{ \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + -6 * 64), result_00);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + -5 * 64), result_01);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + -4 * 64), result_02);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + -3 * 64), result_10);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + -2 * 64), result_11);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + -1 * 64), result_12);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + 0 * 64), result_20);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + 1 * 64), result_21);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + 2 * 64), result_22);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + 3 * 64), result_30);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + 4 * 64), result_31);  \
-    svst1(pg1, (float64_t*)(base + 2 * 3 * 64 + 5 * 64), result_32);  \
-}
-// PREFETCH_CHIMU_L2 (prefetch to L2)
-#define PREFETCH_CHIMU_L2_INTERNAL_A64FXd(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PLDL2STRM); \
-}
-// PREFETCH_CHIMU_L1 (prefetch to L1)
-#define PREFETCH_CHIMU_L1_INTERNAL_A64FXd(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PLDL1STRM); \
-}
-// PREFETCH_GAUGE_L2 (prefetch to L2)
-#define PREFETCH_GAUGE_L2_INTERNAL_A64FXd(A)  \
-{ \
-    const auto & ref(U[sUn](A)); uint64_t baseU = (uint64_t)&ref + 3 * 3 * 64; \
-    svprfd(pg1, (int64_t*)(baseU + -256), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 0), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 256), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 512), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 768), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1024), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1280), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1536), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1792), SV_PLDL2STRM); \
-}
-// PREFETCH_GAUGE_L1 (prefetch to L1)
-#define PREFETCH_GAUGE_L1_INTERNAL_A64FXd(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-    svprfd(pg1, (int64_t*)(baseU + 0), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 256), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 512), SV_PLDL1STRM); \
-}
-// LOAD_CHI
-#define LOAD_CHI_A64FXd(base)  \
-{ \
-    Chi_00 = svld1(pg1, (float64_t*)(base + 0 * 64));  \
-    Chi_01 = svld1(pg1, (float64_t*)(base + 1 * 64));  \
-    Chi_02 = svld1(pg1, (float64_t*)(base + 2 * 64));  \
-    Chi_10 = svld1(pg1, (float64_t*)(base + 3 * 64));  \
-    Chi_11 = svld1(pg1, (float64_t*)(base + 4 * 64));  \
-    Chi_12 = svld1(pg1, (float64_t*)(base + 5 * 64));  \
-}
-// LOAD_CHIMU
-#define LOAD_CHIMU_INTERLEAVED_A64FXd(base)  \
-{ \
-    Chimu_00 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -6 * 64));  \
-    Chimu_30 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 3 * 64));  \
-    Chimu_10 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -3 * 64));  \
-    Chimu_20 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 0 * 64));  \
-    Chimu_01 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -5 * 64));  \
-    Chimu_31 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 4 * 64));  \
-    Chimu_11 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -2 * 64));  \
-    Chimu_21 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 1 * 64));  \
-    Chimu_02 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -4 * 64));  \
-    Chimu_32 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 5 * 64));  \
-    Chimu_12 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -1 * 64));  \
-    Chimu_22 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 2 * 64));  \
-}
-// LOAD_CHIMU_0213
-#define LOAD_CHIMU_0213_A64FXd  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-    Chimu_00 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -6 * 64));  \
-    Chimu_20 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 0 * 64));  \
-    Chimu_01 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -5 * 64));  \
-    Chimu_21 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 1 * 64));  \
-    Chimu_02 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -4 * 64));  \
-    Chimu_22 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 2 * 64));  \
-    Chimu_10 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -3 * 64));  \
-    Chimu_30 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 3 * 64));  \
-    Chimu_11 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -2 * 64));  \
-    Chimu_31 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 4 * 64));  \
-    Chimu_12 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -1 * 64));  \
-    Chimu_32 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 5 * 64));  \
-}
-// LOAD_CHIMU_0312
-#define LOAD_CHIMU_0312_A64FXd  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-    Chimu_00 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -6 * 64));  \
-    Chimu_30 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 3 * 64));  \
-    Chimu_01 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -5 * 64));  \
-    Chimu_31 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 4 * 64));  \
-    Chimu_02 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -4 * 64));  \
-    Chimu_32 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 5 * 64));  \
-    Chimu_10 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -3 * 64));  \
-    Chimu_20 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 0 * 64));  \
-    Chimu_11 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -2 * 64));  \
-    Chimu_21 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 1 * 64));  \
-    Chimu_12 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + -1 * 64));  \
-    Chimu_22 = svld1(pg1, (float64_t*)(base + 2 * 3 * 64 + 2 * 64));  \
-}
-// LOAD_TABLE0
-#define LOAD_TABLE0  \
-    table0 = svld1(pg1, (uint64_t*)&lut[0]);  
-
-// LOAD_TABLE1
-#define LOAD_TABLE1  \
-    table0 = svld1(pg1, (uint64_t*)&lut[1]);  
-
-// LOAD_TABLE2
-#define LOAD_TABLE2  \
-    table0 = svld1(pg1, (uint64_t*)&lut[2]);  
-
-// LOAD_TABLE3
-#define LOAD_TABLE3  \
-    table0 = svld1(pg1, (uint64_t*)&lut[3]);  
-
-// PERMUTE
-#define PERMUTE_A64FXd  \
-    Chi_00 = svtbl(Chi_00, table0);    \
-    Chi_01 = svtbl(Chi_01, table0);    \
-    Chi_02 = svtbl(Chi_02, table0);    \
-    Chi_10 = svtbl(Chi_10, table0);    \
-    Chi_11 = svtbl(Chi_11, table0);    \
-    Chi_12 = svtbl(Chi_12, table0);    
-
-// LOAD_GAUGE
-#define LOAD_GAUGE  \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-{ \
-    U_00 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -6 * 64));  \
-    U_10 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -3 * 64));  \
-    U_20 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + 0 * 64));  \
-    U_01 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -5 * 64));  \
-    U_11 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -2 * 64));  \
-    U_21 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + 1 * 64));  \
-}
-// MULT_2SPIN
-#define MULT_2SPIN_1_A64FXd(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-    U_00 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -6 * 64));  \
-    U_10 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -3 * 64));  \
-    U_20 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + 0 * 64));  \
-    U_01 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -5 * 64));  \
-    U_11 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -2 * 64));  \
-    U_21 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + 1 * 64));  \
-    UChi_00 = svcmla_x(pg1, zero0, U_00, Chi_00, 0); \
-    UChi_10 = svcmla_x(pg1, zero0, U_00, Chi_10, 0); \
-    UChi_01 = svcmla_x(pg1, zero0, U_10, Chi_00, 0); \
-    UChi_11 = svcmla_x(pg1, zero0, U_10, Chi_10, 0); \
-    UChi_02 = svcmla_x(pg1, zero0, U_20, Chi_00, 0); \
-    UChi_12 = svcmla_x(pg1, zero0, U_20, Chi_10, 0); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_00, Chi_00, 90); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_00, Chi_10, 90); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_10, Chi_00, 90); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_10, Chi_10, 90); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_20, Chi_00, 90); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_20, Chi_10, 90); \
-    U_00 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -4 * 64));  \
-    U_10 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + -1 * 64));  \
-    U_20 = svld1(pg1, (float64_t*)(baseU + 2 * 3 * 64 + 2 * 64));  \
-}
-// MULT_2SPIN_BACKEND
-#define MULT_2SPIN_2_A64FXd  \
-{ \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_01, Chi_01, 0); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_01, Chi_11, 0); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_11, Chi_01, 0); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_11, Chi_11, 0); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_21, Chi_01, 0); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_21, Chi_11, 0); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_01, Chi_01, 90); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_01, Chi_11, 90); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_11, Chi_01, 90); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_11, Chi_11, 90); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_21, Chi_01, 90); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_21, Chi_11, 90); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_00, Chi_02, 0); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_00, Chi_12, 0); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_10, Chi_02, 0); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_10, Chi_12, 0); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_20, Chi_02, 0); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_20, Chi_12, 0); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_00, Chi_02, 90); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_00, Chi_12, 90); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_10, Chi_02, 90); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_10, Chi_12, 90); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_20, Chi_02, 90); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_20, Chi_12, 90); \
-}
-// XP_PROJ
-#define XP_PROJ_A64FXd  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_30, 90);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_31, 90);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_32, 90);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_20, 90);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_21, 90);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_22, 90);   \
-}
-// XP_RECON
-#define XP_RECON_A64FXd  \
-    result_20 = svcadd_x(pg1, zero0, UChi_10, 270);   \
-    result_21 = svcadd_x(pg1, zero0, UChi_11, 270);   \
-    result_22 = svcadd_x(pg1, zero0, UChi_12, 270);   \
-    result_30 = svcadd_x(pg1, zero0, UChi_00, 270);   \
-    result_31 = svcadd_x(pg1, zero0, UChi_01, 270);   \
-    result_32 = svcadd_x(pg1, zero0, UChi_02, 270);   \
-    result_00 = UChi_00;        \
-    result_01 = UChi_01;        \
-    result_02 = UChi_02;        \
-    result_10 = UChi_10;        \
-    result_11 = UChi_11;        \
-    result_12 = UChi_12;        
-
-// XP_RECON_ACCUM
-#define XP_RECON_ACCUM_A64FXd  \
-    result_30 = svcadd_x(pg1, result_30, UChi_00, 270);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_31 = svcadd_x(pg1, result_31, UChi_01, 270);   \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_32 = svcadd_x(pg1, result_32, UChi_02, 270);   \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_20 = svcadd_x(pg1, result_20, UChi_10, 270);   \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_21 = svcadd_x(pg1, result_21, UChi_11, 270);   \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_22 = svcadd_x(pg1, result_22, UChi_12, 270);   \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// YP_PROJ
-#define YP_PROJ_A64FXd  \
-{ \
-    Chi_00 = svsub_x(pg1, Chimu_00, Chimu_30);  \
-    Chi_01 = svsub_x(pg1, Chimu_01, Chimu_31);  \
-    Chi_02 = svsub_x(pg1, Chimu_02, Chimu_32);  \
-    Chi_10 = svadd_x(pg1, Chimu_10, Chimu_20);  \
-    Chi_11 = svadd_x(pg1, Chimu_11, Chimu_21);  \
-    Chi_12 = svadd_x(pg1, Chimu_12, Chimu_22);  \
-}
-// ZP_PROJ
-#define ZP_PROJ_A64FXd  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_20, 90);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_21, 90);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_22, 90);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_30, 270);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_31, 270);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_32, 270);   \
-}
-// TP_PROJ
-#define TP_PROJ_A64FXd  \
-{ \
-    Chi_00 = svadd_x(pg1, Chimu_00, Chimu_20);  \
-    Chi_01 = svadd_x(pg1, Chimu_01, Chimu_21);  \
-    Chi_02 = svadd_x(pg1, Chimu_02, Chimu_22);  \
-    Chi_10 = svadd_x(pg1, Chimu_10, Chimu_30);  \
-    Chi_11 = svadd_x(pg1, Chimu_11, Chimu_31);  \
-    Chi_12 = svadd_x(pg1, Chimu_12, Chimu_32);  \
-}
-// XM_PROJ
-#define XM_PROJ_A64FXd  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_30, 270);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_31, 270);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_32, 270);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_20, 270);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_21, 270);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_22, 270);   \
-}
-// XM_RECON
-#define XM_RECON_A64FXd  \
-    result_20 = svcadd_x(pg1, zero0, UChi_10, 90);   \
-    result_21 = svcadd_x(pg1, zero0, UChi_11, 90);   \
-    result_22 = svcadd_x(pg1, zero0, UChi_12, 90);   \
-    result_30 = svcadd_x(pg1, zero0, UChi_00, 90);   \
-    result_31 = svcadd_x(pg1, zero0, UChi_01, 90);   \
-    result_32 = svcadd_x(pg1, zero0, UChi_02, 90);   \
-    result_00 = UChi_00;        \
-    result_01 = UChi_01;        \
-    result_02 = UChi_02;        \
-    result_10 = UChi_10;        \
-    result_11 = UChi_11;        \
-    result_12 = UChi_12;        
-
-// YM_PROJ
-#define YM_PROJ_A64FXd  \
-{ \
-    Chi_00 = svadd_x(pg1, Chimu_00, Chimu_30);  \
-    Chi_01 = svadd_x(pg1, Chimu_01, Chimu_31);  \
-    Chi_02 = svadd_x(pg1, Chimu_02, Chimu_32);  \
-    Chi_10 = svsub_x(pg1, Chimu_10, Chimu_20);  \
-    Chi_11 = svsub_x(pg1, Chimu_11, Chimu_21);  \
-    Chi_12 = svsub_x(pg1, Chimu_12, Chimu_22);  \
-}
-// ZM_PROJ
-#define ZM_PROJ_A64FXd  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_20, 270);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_21, 270);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_22, 270);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_30, 90);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_31, 90);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_32, 90);   \
-}
-// TM_PROJ
-#define TM_PROJ_A64FXd  \
-{ \
-    Chi_00 = svsub_x(pg1, Chimu_00, Chimu_20);  \
-    Chi_01 = svsub_x(pg1, Chimu_01, Chimu_21);  \
-    Chi_02 = svsub_x(pg1, Chimu_02, Chimu_22);  \
-    Chi_10 = svsub_x(pg1, Chimu_10, Chimu_30);  \
-    Chi_11 = svsub_x(pg1, Chimu_11, Chimu_31);  \
-    Chi_12 = svsub_x(pg1, Chimu_12, Chimu_32);  \
-}
-// XM_RECON_ACCUM
-#define XM_RECON_ACCUM_A64FXd  \
-    result_30 = svcadd_x(pg1, result_30, UChi_00, 90);   \
-    result_31 = svcadd_x(pg1, result_31, UChi_01, 90);   \
-    result_32 = svcadd_x(pg1, result_32, UChi_02, 90);   \
-    result_20 = svcadd_x(pg1, result_20, UChi_10, 90);   \
-    result_21 = svcadd_x(pg1, result_21, UChi_11, 90);   \
-    result_22 = svcadd_x(pg1, result_22, UChi_12, 90);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// YP_RECON_ACCUM
-#define YP_RECON_ACCUM_A64FXd  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_30 = svsub_x(pg1, result_30, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_31 = svsub_x(pg1, result_31, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_32 = svsub_x(pg1, result_32, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_20 = svadd_x(pg1, result_20, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_21 = svadd_x(pg1, result_21, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_22 = svadd_x(pg1, result_22, UChi_12); 
-
-// YM_RECON_ACCUM
-#define YM_RECON_ACCUM_A64FXd  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_30 = svadd_x(pg1, result_30, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_31 = svadd_x(pg1, result_31, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_32 = svadd_x(pg1, result_32, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_20 = svsub_x(pg1, result_20, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_21 = svsub_x(pg1, result_21, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_22 = svsub_x(pg1, result_22, UChi_12); 
-
-// ZP_RECON_ACCUM
-#define ZP_RECON_ACCUM_A64FXd  \
-    result_20 = svcadd_x(pg1, result_20, UChi_00, 270);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_21 = svcadd_x(pg1, result_21, UChi_01, 270);   \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_22 = svcadd_x(pg1, result_22, UChi_02, 270);   \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_30 = svcadd_x(pg1, result_30, UChi_10, 90);   \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_31 = svcadd_x(pg1, result_31, UChi_11, 90);   \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_32 = svcadd_x(pg1, result_32, UChi_12, 90);   \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// ZM_RECON_ACCUM
-#define ZM_RECON_ACCUM_A64FXd  \
-    result_20 = svcadd_x(pg1, result_20, UChi_00, 90);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_21 = svcadd_x(pg1, result_21, UChi_01, 90);   \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_22 = svcadd_x(pg1, result_22, UChi_02, 90);   \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_30 = svcadd_x(pg1, result_30, UChi_10, 270);   \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_31 = svcadd_x(pg1, result_31, UChi_11, 270);   \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_32 = svcadd_x(pg1, result_32, UChi_12, 270);   \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// TP_RECON_ACCUM
-#define TP_RECON_ACCUM_A64FXd  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_20 = svadd_x(pg1, result_20, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_21 = svadd_x(pg1, result_21, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_22 = svadd_x(pg1, result_22, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_30 = svadd_x(pg1, result_30, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_31 = svadd_x(pg1, result_31, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_32 = svadd_x(pg1, result_32, UChi_12); 
-
-// TM_RECON_ACCUM
-#define TM_RECON_ACCUM_A64FXd  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_20 = svsub_x(pg1, result_20, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_21 = svsub_x(pg1, result_21, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_22 = svsub_x(pg1, result_22, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_30 = svsub_x(pg1, result_30, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_31 = svsub_x(pg1, result_31, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_32 = svsub_x(pg1, result_32, UChi_12); 
-
-// ZERO_PSI
-#define ZERO_PSI_A64FXd  \
-    result_00 = svdup_f64(0.); \
-    result_01 = svdup_f64(0.); \
-    result_02 = svdup_f64(0.); \
-    result_10 = svdup_f64(0.); \
-    result_11 = svdup_f64(0.); \
-    result_12 = svdup_f64(0.); \
-    result_20 = svdup_f64(0.); \
-    result_21 = svdup_f64(0.); \
-    result_22 = svdup_f64(0.); \
-    result_30 = svdup_f64(0.); \
-    result_31 = svdup_f64(0.); \
-    result_32 = svdup_f64(0.); 
-
-// PREFETCH_RESULT_L2_STORE (prefetch store to L2)
-#define PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXd(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PSTL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PSTL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PSTL2STRM); \
-}
-// PREFETCH_RESULT_L1_STORE (prefetch store to L1)
-#define PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXd(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PSTL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PSTL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PSTL1STRM); \
-}
-// ADD_RESULT_INTERNAL
-#define ADD_RESULT_INTERNAL_A64FXd  \
-    result_00 = svadd_x(pg1, result_00, Chimu_00); \
-    result_01 = svadd_x(pg1, result_01, Chimu_01); \
-    result_02 = svadd_x(pg1, result_02, Chimu_02); \
-    result_10 = svadd_x(pg1, result_10, Chimu_10); \
-    result_11 = svadd_x(pg1, result_11, Chimu_11); \
-    result_12 = svadd_x(pg1, result_12, Chimu_12); \
-    result_20 = svadd_x(pg1, result_20, Chimu_20); \
-    result_21 = svadd_x(pg1, result_21, Chimu_21); \
-    result_22 = svadd_x(pg1, result_22, Chimu_22); \
-    result_30 = svadd_x(pg1, result_30, Chimu_30); \
-    result_31 = svadd_x(pg1, result_31, Chimu_31); \
-    result_32 = svadd_x(pg1, result_32, Chimu_32); 
-

Grid/simd/Fujitsu_A64FX_intrin_single.h

@@ -1,601 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: Fujitsu_A64FX_intrin_single.h
-
-    Copyright (C) 2020
-
-Author: Nils Meyer <nils.meyer@ur.de>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#define LOAD_CHIMU(base)               LOAD_CHIMU_INTERLEAVED_A64FXf(base)  
-#define PREFETCH_CHIMU_L1(A)           PREFETCH_CHIMU_L1_INTERNAL_A64FXf(A)  
-#define PREFETCH_GAUGE_L1(A)           PREFETCH_GAUGE_L1_INTERNAL_A64FXf(A)  
-#define PREFETCH_CHIMU_L2(A)           PREFETCH_CHIMU_L2_INTERNAL_A64FXf(A)  
-#define PREFETCH_GAUGE_L2(A)           PREFETCH_GAUGE_L2_INTERNAL_A64FXf(A)  
-#define PF_GAUGE(A)  
-#define PREFETCH_RESULT_L2_STORE(A)    PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXf(A)  
-#define PREFETCH_RESULT_L1_STORE(A)    PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXf(A)  
-#define PREFETCH1_CHIMU(A)             PREFETCH_CHIMU_L1(A)  
-#define PREFETCH_CHIMU(A)              PREFETCH_CHIMU_L1(A)  
-#define LOCK_GAUGE(A)  
-#define UNLOCK_GAUGE(A)  
-#define MASK_REGS                      DECLARATIONS_A64FXf  
-#define SAVE_RESULT(A,B)               RESULT_A64FXf(A); PREFETCH_RESULT_L2_STORE(B)  
-#define MULT_2SPIN_1(Dir)              MULT_2SPIN_1_A64FXf(Dir)  
-#define MULT_2SPIN_2                   MULT_2SPIN_2_A64FXf  
-#define LOAD_CHI(base)                 LOAD_CHI_A64FXf(base)  
-#define ADD_RESULT(base,basep)         LOAD_CHIMU(base); ADD_RESULT_INTERNAL_A64FXf; RESULT_A64FXf(base)  
-#define XP_PROJ                        XP_PROJ_A64FXf  
-#define YP_PROJ                        YP_PROJ_A64FXf  
-#define ZP_PROJ                        ZP_PROJ_A64FXf  
-#define TP_PROJ                        TP_PROJ_A64FXf  
-#define XM_PROJ                        XM_PROJ_A64FXf  
-#define YM_PROJ                        YM_PROJ_A64FXf  
-#define ZM_PROJ                        ZM_PROJ_A64FXf  
-#define TM_PROJ                        TM_PROJ_A64FXf  
-#define XP_RECON                       XP_RECON_A64FXf  
-#define XM_RECON                       XM_RECON_A64FXf  
-#define XM_RECON_ACCUM                 XM_RECON_ACCUM_A64FXf  
-#define YM_RECON_ACCUM                 YM_RECON_ACCUM_A64FXf  
-#define ZM_RECON_ACCUM                 ZM_RECON_ACCUM_A64FXf  
-#define TM_RECON_ACCUM                 TM_RECON_ACCUM_A64FXf  
-#define XP_RECON_ACCUM                 XP_RECON_ACCUM_A64FXf  
-#define YP_RECON_ACCUM                 YP_RECON_ACCUM_A64FXf  
-#define ZP_RECON_ACCUM                 ZP_RECON_ACCUM_A64FXf  
-#define TP_RECON_ACCUM                 TP_RECON_ACCUM_A64FXf  
-#define PERMUTE_DIR0                   0  
-#define PERMUTE_DIR1                   1  
-#define PERMUTE_DIR2                   2  
-#define PERMUTE_DIR3                   3  
-#define PERMUTE                        PERMUTE_A64FXf;  
-#define LOAD_TABLE(Dir)                if (Dir == 0) { LOAD_TABLE0; } else if (Dir == 1) { LOAD_TABLE1 } else if (Dir == 2) { LOAD_TABLE2; } else if (Dir == 3) { LOAD_TABLE3; }  
-#define MAYBEPERM(A,perm)              if (perm) { PERMUTE; }  
-// DECLARATIONS
-#define DECLARATIONS_A64FXf  \
-    const uint32_t lut[4][16] = { \
-        {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}, \
-        {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11}, \
-        {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}, \
-        {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14} }; \
-    svfloat32_t result_00;        \
-    svfloat32_t result_01;        \
-    svfloat32_t result_02;        \
-    svfloat32_t result_10;        \
-    svfloat32_t result_11;        \
-    svfloat32_t result_12;        \
-    svfloat32_t result_20;        \
-    svfloat32_t result_21;        \
-    svfloat32_t result_22;        \
-    svfloat32_t result_30;        \
-    svfloat32_t result_31;        \
-    svfloat32_t result_32;        \
-    svfloat32_t Chi_00;        \
-    svfloat32_t Chi_01;        \
-    svfloat32_t Chi_02;        \
-    svfloat32_t Chi_10;        \
-    svfloat32_t Chi_11;        \
-    svfloat32_t Chi_12;        \
-    svfloat32_t UChi_00;        \
-    svfloat32_t UChi_01;        \
-    svfloat32_t UChi_02;        \
-    svfloat32_t UChi_10;        \
-    svfloat32_t UChi_11;        \
-    svfloat32_t UChi_12;        \
-    svfloat32_t U_00;        \
-    svfloat32_t U_10;        \
-    svfloat32_t U_20;        \
-    svfloat32_t U_01;        \
-    svfloat32_t U_11;        \
-    svfloat32_t U_21;        \
-    svbool_t pg1;        \
-    pg1 = svptrue_b32();        \
-    svuint32_t table0; \
-    svfloat32_t zero0;        \
-    zero0 = svdup_f32(0.); 
-
-#define Chimu_00 Chi_00  
-#define Chimu_01 Chi_01  
-#define Chimu_02 Chi_02  
-#define Chimu_10 Chi_10  
-#define Chimu_11 Chi_11  
-#define Chimu_12 Chi_12  
-#define Chimu_20 UChi_00  
-#define Chimu_21 UChi_01  
-#define Chimu_22 UChi_02  
-#define Chimu_30 UChi_10  
-#define Chimu_31 UChi_11  
-#define Chimu_32 UChi_12  
-// RESULT
-#define RESULT_A64FXf(base)  \
-{ \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + -6 * 64), result_00);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + -5 * 64), result_01);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + -4 * 64), result_02);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + -3 * 64), result_10);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + -2 * 64), result_11);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + -1 * 64), result_12);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + 0 * 64), result_20);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + 1 * 64), result_21);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + 2 * 64), result_22);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + 3 * 64), result_30);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + 4 * 64), result_31);  \
-    svst1(pg1, (float32_t*)(base + 2 * 3 * 64 + 5 * 64), result_32);  \
-}
-// PREFETCH_CHIMU_L2 (prefetch to L2)
-#define PREFETCH_CHIMU_L2_INTERNAL_A64FXf(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PLDL2STRM); \
-}
-// PREFETCH_CHIMU_L1 (prefetch to L1)
-#define PREFETCH_CHIMU_L1_INTERNAL_A64FXf(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PLDL1STRM); \
-}
-// PREFETCH_GAUGE_L2 (prefetch to L2)
-#define PREFETCH_GAUGE_L2_INTERNAL_A64FXf(A)  \
-{ \
-    const auto & ref(U[sUn](A)); uint64_t baseU = (uint64_t)&ref + 3 * 3 * 64; \
-    svprfd(pg1, (int64_t*)(baseU + -256), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 0), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 256), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 512), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 768), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1024), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1280), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1536), SV_PLDL2STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 1792), SV_PLDL2STRM); \
-}
-// PREFETCH_GAUGE_L1 (prefetch to L1)
-#define PREFETCH_GAUGE_L1_INTERNAL_A64FXf(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-    svprfd(pg1, (int64_t*)(baseU + 0), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 256), SV_PLDL1STRM); \
-    svprfd(pg1, (int64_t*)(baseU + 512), SV_PLDL1STRM); \
-}
-// LOAD_CHI
-#define LOAD_CHI_A64FXf(base)  \
-{ \
-    Chi_00 = svld1(pg1, (float32_t*)(base + 0 * 64));  \
-    Chi_01 = svld1(pg1, (float32_t*)(base + 1 * 64));  \
-    Chi_02 = svld1(pg1, (float32_t*)(base + 2 * 64));  \
-    Chi_10 = svld1(pg1, (float32_t*)(base + 3 * 64));  \
-    Chi_11 = svld1(pg1, (float32_t*)(base + 4 * 64));  \
-    Chi_12 = svld1(pg1, (float32_t*)(base + 5 * 64));  \
-}
-// LOAD_CHIMU
-#define LOAD_CHIMU_INTERLEAVED_A64FXf(base)  \
-{ \
-    Chimu_00 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -6 * 64));  \
-    Chimu_30 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 3 * 64));  \
-    Chimu_10 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -3 * 64));  \
-    Chimu_20 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 0 * 64));  \
-    Chimu_01 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -5 * 64));  \
-    Chimu_31 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 4 * 64));  \
-    Chimu_11 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -2 * 64));  \
-    Chimu_21 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 1 * 64));  \
-    Chimu_02 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -4 * 64));  \
-    Chimu_32 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 5 * 64));  \
-    Chimu_12 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -1 * 64));  \
-    Chimu_22 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 2 * 64));  \
-}
-// LOAD_CHIMU_0213
-#define LOAD_CHIMU_0213_A64FXf  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-    Chimu_00 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -6 * 64));  \
-    Chimu_20 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 0 * 64));  \
-    Chimu_01 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -5 * 64));  \
-    Chimu_21 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 1 * 64));  \
-    Chimu_02 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -4 * 64));  \
-    Chimu_22 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 2 * 64));  \
-    Chimu_10 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -3 * 64));  \
-    Chimu_30 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 3 * 64));  \
-    Chimu_11 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -2 * 64));  \
-    Chimu_31 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 4 * 64));  \
-    Chimu_12 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -1 * 64));  \
-    Chimu_32 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 5 * 64));  \
-}
-// LOAD_CHIMU_0312
-#define LOAD_CHIMU_0312_A64FXf  \
-{ \
-    const SiteSpinor & ref(in[offset]); \
-    Chimu_00 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -6 * 64));  \
-    Chimu_30 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 3 * 64));  \
-    Chimu_01 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -5 * 64));  \
-    Chimu_31 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 4 * 64));  \
-    Chimu_02 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -4 * 64));  \
-    Chimu_32 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 5 * 64));  \
-    Chimu_10 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -3 * 64));  \
-    Chimu_20 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 0 * 64));  \
-    Chimu_11 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -2 * 64));  \
-    Chimu_21 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 1 * 64));  \
-    Chimu_12 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + -1 * 64));  \
-    Chimu_22 = svld1(pg1, (float32_t*)(base + 2 * 3 * 64 + 2 * 64));  \
-}
-// LOAD_TABLE0
-#define LOAD_TABLE0  \
-    table0 = svld1(pg1, (uint32_t*)&lut[0]);  
-
-// LOAD_TABLE1
-#define LOAD_TABLE1  \
-    table0 = svld1(pg1, (uint32_t*)&lut[1]);  
-
-// LOAD_TABLE2
-#define LOAD_TABLE2  \
-    table0 = svld1(pg1, (uint32_t*)&lut[2]);  
-
-// LOAD_TABLE3
-#define LOAD_TABLE3  \
-    table0 = svld1(pg1, (uint32_t*)&lut[3]);  
-
-// PERMUTE
-#define PERMUTE_A64FXf  \
-    Chi_00 = svtbl(Chi_00, table0);    \
-    Chi_01 = svtbl(Chi_01, table0);    \
-    Chi_02 = svtbl(Chi_02, table0);    \
-    Chi_10 = svtbl(Chi_10, table0);    \
-    Chi_11 = svtbl(Chi_11, table0);    \
-    Chi_12 = svtbl(Chi_12, table0);    
-
-// LOAD_GAUGE
-#define LOAD_GAUGE  \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-{ \
-    U_00 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -6 * 64));  \
-    U_10 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -3 * 64));  \
-    U_20 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + 0 * 64));  \
-    U_01 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -5 * 64));  \
-    U_11 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -2 * 64));  \
-    U_21 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + 1 * 64));  \
-}
-// MULT_2SPIN
-#define MULT_2SPIN_1_A64FXf(A)  \
-{ \
-    const auto & ref(U[sU](A)); uint64_t baseU = (uint64_t)&ref; \
-    U_00 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -6 * 64));  \
-    U_10 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -3 * 64));  \
-    U_20 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + 0 * 64));  \
-    U_01 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -5 * 64));  \
-    U_11 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -2 * 64));  \
-    U_21 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + 1 * 64));  \
-    UChi_00 = svcmla_x(pg1, zero0, U_00, Chi_00, 0); \
-    UChi_10 = svcmla_x(pg1, zero0, U_00, Chi_10, 0); \
-    UChi_01 = svcmla_x(pg1, zero0, U_10, Chi_00, 0); \
-    UChi_11 = svcmla_x(pg1, zero0, U_10, Chi_10, 0); \
-    UChi_02 = svcmla_x(pg1, zero0, U_20, Chi_00, 0); \
-    UChi_12 = svcmla_x(pg1, zero0, U_20, Chi_10, 0); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_00, Chi_00, 90); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_00, Chi_10, 90); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_10, Chi_00, 90); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_10, Chi_10, 90); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_20, Chi_00, 90); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_20, Chi_10, 90); \
-    U_00 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -4 * 64));  \
-    U_10 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + -1 * 64));  \
-    U_20 = svld1(pg1, (float32_t*)(baseU + 2 * 3 * 64 + 2 * 64));  \
-}
-// MULT_2SPIN_BACKEND
-#define MULT_2SPIN_2_A64FXf  \
-{ \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_01, Chi_01, 0); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_01, Chi_11, 0); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_11, Chi_01, 0); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_11, Chi_11, 0); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_21, Chi_01, 0); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_21, Chi_11, 0); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_01, Chi_01, 90); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_01, Chi_11, 90); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_11, Chi_01, 90); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_11, Chi_11, 90); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_21, Chi_01, 90); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_21, Chi_11, 90); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_00, Chi_02, 0); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_00, Chi_12, 0); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_10, Chi_02, 0); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_10, Chi_12, 0); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_20, Chi_02, 0); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_20, Chi_12, 0); \
-    UChi_00 = svcmla_x(pg1, UChi_00, U_00, Chi_02, 90); \
-    UChi_10 = svcmla_x(pg1, UChi_10, U_00, Chi_12, 90); \
-    UChi_01 = svcmla_x(pg1, UChi_01, U_10, Chi_02, 90); \
-    UChi_11 = svcmla_x(pg1, UChi_11, U_10, Chi_12, 90); \
-    UChi_02 = svcmla_x(pg1, UChi_02, U_20, Chi_02, 90); \
-    UChi_12 = svcmla_x(pg1, UChi_12, U_20, Chi_12, 90); \
-}
-// XP_PROJ
-#define XP_PROJ_A64FXf  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_30, 90);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_31, 90);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_32, 90);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_20, 90);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_21, 90);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_22, 90);   \
-}
-// XP_RECON
-#define XP_RECON_A64FXf  \
-    result_20 = svcadd_x(pg1, zero0, UChi_10, 270);   \
-    result_21 = svcadd_x(pg1, zero0, UChi_11, 270);   \
-    result_22 = svcadd_x(pg1, zero0, UChi_12, 270);   \
-    result_30 = svcadd_x(pg1, zero0, UChi_00, 270);   \
-    result_31 = svcadd_x(pg1, zero0, UChi_01, 270);   \
-    result_32 = svcadd_x(pg1, zero0, UChi_02, 270);   \
-    result_00 = UChi_00;        \
-    result_01 = UChi_01;        \
-    result_02 = UChi_02;        \
-    result_10 = UChi_10;        \
-    result_11 = UChi_11;        \
-    result_12 = UChi_12;        
-
-// XP_RECON_ACCUM
-#define XP_RECON_ACCUM_A64FXf  \
-    result_30 = svcadd_x(pg1, result_30, UChi_00, 270);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_31 = svcadd_x(pg1, result_31, UChi_01, 270);   \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_32 = svcadd_x(pg1, result_32, UChi_02, 270);   \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_20 = svcadd_x(pg1, result_20, UChi_10, 270);   \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_21 = svcadd_x(pg1, result_21, UChi_11, 270);   \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_22 = svcadd_x(pg1, result_22, UChi_12, 270);   \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// YP_PROJ
-#define YP_PROJ_A64FXf  \
-{ \
-    Chi_00 = svsub_x(pg1, Chimu_00, Chimu_30);  \
-    Chi_01 = svsub_x(pg1, Chimu_01, Chimu_31);  \
-    Chi_02 = svsub_x(pg1, Chimu_02, Chimu_32);  \
-    Chi_10 = svadd_x(pg1, Chimu_10, Chimu_20);  \
-    Chi_11 = svadd_x(pg1, Chimu_11, Chimu_21);  \
-    Chi_12 = svadd_x(pg1, Chimu_12, Chimu_22);  \
-}
-// ZP_PROJ
-#define ZP_PROJ_A64FXf  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_20, 90);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_21, 90);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_22, 90);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_30, 270);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_31, 270);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_32, 270);   \
-}
-// TP_PROJ
-#define TP_PROJ_A64FXf  \
-{ \
-    Chi_00 = svadd_x(pg1, Chimu_00, Chimu_20);  \
-    Chi_01 = svadd_x(pg1, Chimu_01, Chimu_21);  \
-    Chi_02 = svadd_x(pg1, Chimu_02, Chimu_22);  \
-    Chi_10 = svadd_x(pg1, Chimu_10, Chimu_30);  \
-    Chi_11 = svadd_x(pg1, Chimu_11, Chimu_31);  \
-    Chi_12 = svadd_x(pg1, Chimu_12, Chimu_32);  \
-}
-// XM_PROJ
-#define XM_PROJ_A64FXf  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_30, 270);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_31, 270);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_32, 270);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_20, 270);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_21, 270);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_22, 270);   \
-}
-// XM_RECON
-#define XM_RECON_A64FXf  \
-    result_20 = svcadd_x(pg1, zero0, UChi_10, 90);   \
-    result_21 = svcadd_x(pg1, zero0, UChi_11, 90);   \
-    result_22 = svcadd_x(pg1, zero0, UChi_12, 90);   \
-    result_30 = svcadd_x(pg1, zero0, UChi_00, 90);   \
-    result_31 = svcadd_x(pg1, zero0, UChi_01, 90);   \
-    result_32 = svcadd_x(pg1, zero0, UChi_02, 90);   \
-    result_00 = UChi_00;        \
-    result_01 = UChi_01;        \
-    result_02 = UChi_02;        \
-    result_10 = UChi_10;        \
-    result_11 = UChi_11;        \
-    result_12 = UChi_12;        
-
-// YM_PROJ
-#define YM_PROJ_A64FXf  \
-{ \
-    Chi_00 = svadd_x(pg1, Chimu_00, Chimu_30);  \
-    Chi_01 = svadd_x(pg1, Chimu_01, Chimu_31);  \
-    Chi_02 = svadd_x(pg1, Chimu_02, Chimu_32);  \
-    Chi_10 = svsub_x(pg1, Chimu_10, Chimu_20);  \
-    Chi_11 = svsub_x(pg1, Chimu_11, Chimu_21);  \
-    Chi_12 = svsub_x(pg1, Chimu_12, Chimu_22);  \
-}
-// ZM_PROJ
-#define ZM_PROJ_A64FXf  \
-{ \
-    Chi_00 = svcadd_x(pg1, Chimu_00, Chimu_20, 270);   \
-    Chi_01 = svcadd_x(pg1, Chimu_01, Chimu_21, 270);   \
-    Chi_02 = svcadd_x(pg1, Chimu_02, Chimu_22, 270);   \
-    Chi_10 = svcadd_x(pg1, Chimu_10, Chimu_30, 90);   \
-    Chi_11 = svcadd_x(pg1, Chimu_11, Chimu_31, 90);   \
-    Chi_12 = svcadd_x(pg1, Chimu_12, Chimu_32, 90);   \
-}
-// TM_PROJ
-#define TM_PROJ_A64FXf  \
-{ \
-    Chi_00 = svsub_x(pg1, Chimu_00, Chimu_20);  \
-    Chi_01 = svsub_x(pg1, Chimu_01, Chimu_21);  \
-    Chi_02 = svsub_x(pg1, Chimu_02, Chimu_22);  \
-    Chi_10 = svsub_x(pg1, Chimu_10, Chimu_30);  \
-    Chi_11 = svsub_x(pg1, Chimu_11, Chimu_31);  \
-    Chi_12 = svsub_x(pg1, Chimu_12, Chimu_32);  \
-}
-// XM_RECON_ACCUM
-#define XM_RECON_ACCUM_A64FXf  \
-    result_30 = svcadd_x(pg1, result_30, UChi_00, 90);   \
-    result_31 = svcadd_x(pg1, result_31, UChi_01, 90);   \
-    result_32 = svcadd_x(pg1, result_32, UChi_02, 90);   \
-    result_20 = svcadd_x(pg1, result_20, UChi_10, 90);   \
-    result_21 = svcadd_x(pg1, result_21, UChi_11, 90);   \
-    result_22 = svcadd_x(pg1, result_22, UChi_12, 90);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// YP_RECON_ACCUM
-#define YP_RECON_ACCUM_A64FXf  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_30 = svsub_x(pg1, result_30, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_31 = svsub_x(pg1, result_31, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_32 = svsub_x(pg1, result_32, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_20 = svadd_x(pg1, result_20, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_21 = svadd_x(pg1, result_21, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_22 = svadd_x(pg1, result_22, UChi_12); 
-
-// YM_RECON_ACCUM
-#define YM_RECON_ACCUM_A64FXf  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_30 = svadd_x(pg1, result_30, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_31 = svadd_x(pg1, result_31, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_32 = svadd_x(pg1, result_32, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_20 = svsub_x(pg1, result_20, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_21 = svsub_x(pg1, result_21, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_22 = svsub_x(pg1, result_22, UChi_12); 
-
-// ZP_RECON_ACCUM
-#define ZP_RECON_ACCUM_A64FXf  \
-    result_20 = svcadd_x(pg1, result_20, UChi_00, 270);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_21 = svcadd_x(pg1, result_21, UChi_01, 270);   \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_22 = svcadd_x(pg1, result_22, UChi_02, 270);   \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_30 = svcadd_x(pg1, result_30, UChi_10, 90);   \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_31 = svcadd_x(pg1, result_31, UChi_11, 90);   \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_32 = svcadd_x(pg1, result_32, UChi_12, 90);   \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// ZM_RECON_ACCUM
-#define ZM_RECON_ACCUM_A64FXf  \
-    result_20 = svcadd_x(pg1, result_20, UChi_00, 90);   \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_21 = svcadd_x(pg1, result_21, UChi_01, 90);   \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_22 = svcadd_x(pg1, result_22, UChi_02, 90);   \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_30 = svcadd_x(pg1, result_30, UChi_10, 270);   \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_31 = svcadd_x(pg1, result_31, UChi_11, 270);   \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_32 = svcadd_x(pg1, result_32, UChi_12, 270);   \
-    result_12 = svadd_x(pg1, result_12, UChi_12); 
-
-// TP_RECON_ACCUM
-#define TP_RECON_ACCUM_A64FXf  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_20 = svadd_x(pg1, result_20, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_21 = svadd_x(pg1, result_21, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_22 = svadd_x(pg1, result_22, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_30 = svadd_x(pg1, result_30, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_31 = svadd_x(pg1, result_31, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_32 = svadd_x(pg1, result_32, UChi_12); 
-
-// TM_RECON_ACCUM
-#define TM_RECON_ACCUM_A64FXf  \
-    result_00 = svadd_x(pg1, result_00, UChi_00); \
-    result_20 = svsub_x(pg1, result_20, UChi_00); \
-    result_01 = svadd_x(pg1, result_01, UChi_01); \
-    result_21 = svsub_x(pg1, result_21, UChi_01); \
-    result_02 = svadd_x(pg1, result_02, UChi_02); \
-    result_22 = svsub_x(pg1, result_22, UChi_02); \
-    result_10 = svadd_x(pg1, result_10, UChi_10); \
-    result_30 = svsub_x(pg1, result_30, UChi_10); \
-    result_11 = svadd_x(pg1, result_11, UChi_11); \
-    result_31 = svsub_x(pg1, result_31, UChi_11); \
-    result_12 = svadd_x(pg1, result_12, UChi_12); \
-    result_32 = svsub_x(pg1, result_32, UChi_12); 
-
-// ZERO_PSI
-#define ZERO_PSI_A64FXf  \
-    result_00 = svdup_f32(0.); \
-    result_01 = svdup_f32(0.); \
-    result_02 = svdup_f32(0.); \
-    result_10 = svdup_f32(0.); \
-    result_11 = svdup_f32(0.); \
-    result_12 = svdup_f32(0.); \
-    result_20 = svdup_f32(0.); \
-    result_21 = svdup_f32(0.); \
-    result_22 = svdup_f32(0.); \
-    result_30 = svdup_f32(0.); \
-    result_31 = svdup_f32(0.); \
-    result_32 = svdup_f32(0.); 
-
-// PREFETCH_RESULT_L2_STORE (prefetch store to L2)
-#define PREFETCH_RESULT_L2_STORE_INTERNAL_A64FXf(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PSTL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PSTL2STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PSTL2STRM); \
-}
-// PREFETCH_RESULT_L1_STORE (prefetch store to L1)
-#define PREFETCH_RESULT_L1_STORE_INTERNAL_A64FXf(base)  \
-{ \
-    svprfd(pg1, (int64_t*)(base + 0), SV_PSTL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 256), SV_PSTL1STRM); \
-    svprfd(pg1, (int64_t*)(base + 512), SV_PSTL1STRM); \
-}
-// ADD_RESULT_INTERNAL
-#define ADD_RESULT_INTERNAL_A64FXf  \
-    result_00 = svadd_x(pg1, result_00, Chimu_00); \
-    result_01 = svadd_x(pg1, result_01, Chimu_01); \
-    result_02 = svadd_x(pg1, result_02, Chimu_02); \
-    result_10 = svadd_x(pg1, result_10, Chimu_10); \
-    result_11 = svadd_x(pg1, result_11, Chimu_11); \
-    result_12 = svadd_x(pg1, result_12, Chimu_12); \
-    result_20 = svadd_x(pg1, result_20, Chimu_20); \
-    result_21 = svadd_x(pg1, result_21, Chimu_21); \
-    result_22 = svadd_x(pg1, result_22, Chimu_22); \
-    result_30 = svadd_x(pg1, result_30, Chimu_30); \
-    result_31 = svadd_x(pg1, result_31, Chimu_31); \
-    result_32 = svadd_x(pg1, result_32, Chimu_32); 
-

Grid/simd/Fujitsu_A64FX_undef.h

@@ -1,76 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: Fujitsu_A64FX_undef.h
-
-    Copyright (C) 2020
-
-Author: Nils Meyer <nils.meyer@ur.de>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#undef LOAD_CHIMU
-#undef PREFETCH_CHIMU_L1
-#undef PREFETCH_GAUGE_L1
-#undef PREFETCH_CHIMU_L2
-#undef PREFETCH_GAUGE_L2
-#undef PREFETCH_GAUGE_L1_INTERNAL
-#undef PREFETCH1_CHIMU
-#undef PREFETCH_CHIMU
-#undef PREFETCH_RESULT_L2_STORE
-#undef PREFETCH_RESULT_L1_STORE
-#undef LOAD_GAUGE
-#undef LOCK_GAUGE
-#undef UNLOCK_GAUGE
-#undef MASK_REGS
-#undef SAVE_RESULT
-#undef ADD_RESULT
-#undef MULT_2SPIN_1
-#undef MULT_2SPIN_2
-#undef MAYBEPERM
-#undef LOAD_CHI
-#undef XP_PROJ
-#undef YP_PROJ
-#undef ZP_PROJ
-#undef TP_PROJ
-#undef XM_PROJ
-#undef YM_PROJ
-#undef ZM_PROJ
-#undef TM_PROJ
-#undef XP_RECON
-#undef XM_RECON
-#undef XM_RECON_ACCUM
-#undef YM_RECON_ACCUM
-#undef ZM_RECON_ACCUM
-#undef TM_RECON_ACCUM
-#undef XP_RECON_ACCUM
-#undef YP_RECON_ACCUM
-#undef ZP_RECON_ACCUM
-#undef TP_RECON_ACCUM
-#undef PERMUTE
-#undef PERMUTE_DIR0
-#undef PERMUTE_DIR1
-#undef PERMUTE_DIR2
-#undef PERMUTE_DIR3
-#undef LOAD_TABLE
-#undef LOAD_TABLE0
-#undef LOAD_TABLE1
-#undef LOAD_TABLE2
-#undef LOAD_TABLE3

Grid/simd/Grid_a64fx-2.h

@@ -1,942 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: Grid_a64fx-2.h
-
-    Copyright (C) 2020
-
-    Author: Nils Meyer          <nils.meyer@ur.de>
-
-    with support from Arm
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-
-/////////////////////////////////////////////////////
-// Using SVE ACLE
-/////////////////////////////////////////////////////
-
-static_assert(GEN_SIMD_WIDTH % 64u == 0, "A64FX SIMD vector size is 64 bytes");
-
-NAMESPACE_BEGIN(Grid);
-NAMESPACE_BEGIN(Optimization);
-
-  // type traits giving the number of elements for each vector type
-  template <typename T> struct W;
-  template <> struct W<double> {
-    constexpr static unsigned int c = GEN_SIMD_WIDTH/16u;
-    constexpr static unsigned int r = GEN_SIMD_WIDTH/8u;
-  };
-  template <> struct W<float> {
-    constexpr static unsigned int c = GEN_SIMD_WIDTH/8u;
-    constexpr static unsigned int r = GEN_SIMD_WIDTH/4u;
-  };
-  template <> struct W<Integer> {
-    constexpr static unsigned int r = GEN_SIMD_WIDTH/4u;
-  };
-  template <> struct W<uint16_t> {
-    constexpr static unsigned int c = GEN_SIMD_WIDTH/4u;
-    constexpr static unsigned int r = GEN_SIMD_WIDTH/2u;
-  };
-  template <> struct W<uint64_t> {
-    constexpr static unsigned int c = GEN_SIMD_WIDTH/16u;
-    constexpr static unsigned int r = GEN_SIMD_WIDTH/8u;
-  };
-
-  #ifdef ARMCLANGCOMPAT
-  // SIMD vector immediate types
-  template <typename T>
-  struct vec_imm {
-    alignas(GEN_SIMD_WIDTH) T v[W<T>::r];
-  };
-
-  // SIMD vector types
-  template <typename T>
-  struct vec {
-    alignas(GEN_SIMD_WIDTH) T v[W<T>::r];
-    vec() = default;
-    vec(const vec &rhs) { this->operator=(rhs); }
-    vec(const vec_imm<T> &rhs) {
-      // v = rhs.v
-      svst1(svptrue_b8(), (T*)this, svld1(svptrue_b8(), (T*)rhs.v));
-    }
-
-    inline vec &operator=(const vec &rhs) {
-      // v = rhs.v
-      svst1(svptrue_b8(), (T*)this, svld1(svptrue_b8(), (T*)rhs.v));
-      return *this;
-    };
-  };
-
-  #else //  no ARMCLANGCOMPAT
-  #define vec_imm vec
-  // SIMD vector types
-  template <typename T>
-  struct vec {
-    alignas(GEN_SIMD_WIDTH) T v[W<T>::r];
-  };
-  #endif
-
-  typedef vec<float>     vecf;
-  typedef vec<double>    vecd;
-  typedef vec<uint16_t>  vech; // half precision comms
-  typedef vec<Integer>   veci;
-
-NAMESPACE_END(Optimization)
-NAMESPACE_END(Grid)
-
-// low-level API
-NAMESPACE_BEGIN(Grid);
-NAMESPACE_BEGIN(Optimization);
-
-template <typename T>
-struct acle{};
-
-template <>
-struct acle<double>{
-  typedef svfloat64_t vt;
-  typedef svfloat64x2_t vt2;
-  typedef svfloat64x4_t vt4;
-  typedef float64_t pt;
-  typedef uint64_t uint;
-  typedef svuint64_t svuint;
-
-  static inline svbool_t pg1(){return svptrue_b64();}
-  static inline svbool_t pg2(){return svptrue_pat_b64(SV_VL4);}
-  static inline svbool_t pg4(){return svptrue_pat_b64(SV_VL2);}
-  static inline vec<uint64_t> tbl_swap(){
-    //const vec<uint64_t> t = {1, 0, 3, 2, 5, 4, 7, 6};
-    const vec_imm<uint64_t> t = {1, 0, 3, 2, 5, 4, 7, 6};
-    return t;
-  }
-  static inline vec<uint64_t> tbl0(){
-    //const vec<uint64_t> t = {4, 5, 6, 7, 0, 1, 2, 3};
-    const vec_imm<uint64_t> t = {4, 5, 6, 7, 0, 1, 2, 3};
-    return t;
-  }
-  static inline vec<uint64_t> tbl1(){
-    //const vec<uint64_t> t = {2, 3, 0, 1, 6, 7, 4, 5};
-    const vec_imm<uint64_t> t = {2, 3, 0, 1, 6, 7, 4, 5};
-    return t;
-  }
-  static inline vec<uint64_t> tbl_exch1a(){ // Exchange1
-    //const vec<uint64_t> t = {0, 1, 4, 5, 2, 3, 6, 7};
-    const vec_imm<uint64_t> t = {0, 1, 4, 5, 2, 3, 6, 7};
-    return t;
-  }
-  static inline vec<uint64_t> tbl_exch1b(){ // Exchange1
-    //const vec<uint64_t> t = {2, 3, 6, 7, 0, 1, 4, 5};
-    const vec_imm<uint64_t> t = {2, 3, 6, 7, 0, 1, 4, 5};
-    return t;
-  }
-  static inline vec<uint64_t> tbl_exch1c(){ // Exchange1
-    //const vec<uint64_t> t = {4, 5, 0, 1, 6, 7, 2, 3};
-    const vec_imm<uint64_t> t = {4, 5, 0, 1, 6, 7, 2, 3};
-    return t;
-  }
-  static inline svbool_t pg_even(){return svzip1_b64(svptrue_b64(), svpfalse_b());}
-  static inline svbool_t pg_odd() {return svzip1_b64(svpfalse_b(), svptrue_b64());}
-  static inline svfloat64_t zero(){return svdup_f64(0.);}
-};
-
-template <>
-struct acle<float>{
-  typedef svfloat32_t vt;
-  typedef svfloat32x2_t vt2;
-  typedef float32_t pt;
-  typedef uint32_t uint;
-  typedef svuint32_t svuint;
-
-  static inline svbool_t pg1(){return svptrue_b32();}
-  static inline svbool_t pg2(){return svptrue_pat_b32(SV_VL8);}
-  // exchange neighboring elements
-  static inline vec<uint32_t> tbl_swap(){
-    //const vec<uint32_t> t = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14};
-    const vec_imm<uint32_t> t = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14};
-    return t;
-  }
-  static inline vec<uint32_t> tbl0(){
-    //const vec<uint32_t> t = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7};
-    const vec_imm<uint32_t> t = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7};
-    return t;
-  }
-  static inline vec<uint32_t> tbl1(){
-    //const vec<uint32_t> t = {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11};
-    const vec_imm<uint32_t> t = {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11};
-    return t;
-  }
-  static inline vec<uint32_t> tbl2(){
-    //const vec<uint32_t> t = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
-    const vec_imm<uint32_t> t = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
-    return t;
-  }
-  static inline vec<uint32_t> tbl_exch1a(){ // Exchange1
-    //const vec<uint32_t> t = {0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15 };
-    const vec_imm<uint32_t> t = {0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15 };
-    return t;
-  }
-  static inline vec<uint32_t> tbl_exch1b(){ // Exchange1
-    //const vec<uint32_t> t = {4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 8, 9, 10, 11 };
-    const vec_imm<uint32_t> t = {4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 8, 9, 10, 11 };
-    return t;
-  }
-  static inline vec<uint32_t> tbl_exch1c(){ // Exchange1
-    //const vec<uint32_t> t = {8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7};
-    const vec_imm<uint32_t> t = {8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7};
-    return t;
-  }
-  static inline svbool_t pg_even(){return svzip1_b32(svptrue_b32(), svpfalse_b());}
-  static inline svbool_t pg_odd() {return svzip1_b32(svpfalse_b(), svptrue_b32());}
-  static inline svfloat32_t zero(){return svdup_f32(0.);}
-};
-
-template <>
-struct acle<uint16_t>{
-  typedef svfloat16_t vt;
-  typedef float16_t pt;
-  typedef uint16_t uint;
-  typedef svuint16_t svuint;
-
-  static inline svbool_t pg1(){return svptrue_b16();}
-  static inline svbool_t pg2(){return svptrue_pat_b16(SV_VL16);}
-  static inline svbool_t pg_even(){return svzip1_b16(svptrue_b16(), svpfalse_b());}
-  static inline svbool_t pg_odd() {return svzip1_b16(svpfalse_b(), svptrue_b16());}
-  static inline svfloat16_t zero(){return svdup_f16(0.);}
-};
-
-template <>
-struct acle<Integer>{
-  typedef svuint32_t vt;
-  typedef svuint32x2_t vt2;
-  typedef Integer pt;
-  typedef uint32_t uint;
-  typedef svuint32_t svuint;
-
-  //static inline svbool_t pg1(){return svptrue_b16();}
-  static inline svbool_t pg1(){return svptrue_b32();}
-  static inline svbool_t pg2(){return svptrue_pat_b32(SV_VL8);}
-  static inline svbool_t pg_even(){return svzip1_b32(svptrue_b32(), svpfalse_b());}
-  static inline svbool_t pg_odd() {return svzip1_b32(svpfalse_b(), svptrue_b32());}
-};
-
-// ---------------------------------------------------
-
-struct Vsplat{
-  // Complex float
-  inline vecf operator()(float a, float b){
-    vecf out;
-    svbool_t pg1 = acle<float>::pg1();
-    typename acle<float>::vt a_v = svdup_f32(a);
-    typename acle<float>::vt b_v = svdup_f32(b);
-    typename acle<float>::vt r_v = svzip1(a_v, b_v);
-    svst1(pg1, out.v, r_v);
-    return out;
-  }
-
-  // Real float
-  inline vecf operator()(float a){
-    vecf out;
-    svbool_t pg1 = acle<float>::pg1();
-    typename acle<float>::vt r_v = svdup_f32(a);
-    svst1(pg1, out.v, r_v);
-    return out;
-  }
-
- // Complex double
-  inline vecd operator()(double a, double b){
-    vecd out;
-    svbool_t pg1 = acle<double>::pg1();
-    typename acle<double>::vt a_v = svdup_f64(a);
-    typename acle<double>::vt b_v = svdup_f64(b);
-    typename acle<double>::vt r_v = svzip1(a_v, b_v);
-    svst1(pg1, out.v, r_v);
-    return out;
-  }
-
-  // Real double
-  inline vecd operator()(double a){
-    vecd out;
-    svbool_t pg1 = acle<double>::pg1();
-    typename acle<double>::vt r_v = svdup_f64(a);
-    svst1(pg1, out.v, r_v);
-    return out;
-  }
-
-  // Integer
-  inline vec<Integer> operator()(Integer a){
-    vec<Integer> out;
-    svbool_t pg1 = acle<Integer>::pg1();
-    // Add check whether Integer is really a uint32_t???
-    typename acle<Integer>::vt r_v = svdup_u32(a);
-    svst1(pg1, out.v, r_v);
-    return out;
-  }
-};
-
-struct Vstore{
-  // Real
-  template <typename T>
-  inline void operator()(vec<T> a, T *D){
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, (typename acle<T>::pt*)&a.v);
-    svst1(pg1, D, a_v);
-  }
-};
-
-struct Vstream{
-  // Real
-  template <typename T>
-  inline void operator()(T * a, vec<T> b){
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    svstnt1(pg1, a, b_v);
-    //svst1(pg1, a, b_v);
-  }
-};
-
-  struct Vset{
-    // Complex
-    template <typename T>
-    inline vec<T> operator()(std::complex<T> *a){
-      vec<T> out;
-      svbool_t pg1 = acle<T>::pg1();
-      typename acle<T>::vt a_v = svld1(pg1, (T*)a);
-      svst1(pg1, out.v, a_v);
-
-      return out;
-    }
-
-    // Real
-    template <typename T>
-    inline vec<T> operator()(T *a){
-      vec<T> out;
-      svbool_t pg1 = acle<T>::pg1();
-      typename acle<T>::vt a_v = svld1(pg1, a);
-      svst1(pg1, out.v, a_v);
-
-      return out;
-    }
-  };
-
-/////////////////////////////////////////////////////
-// Arithmetic operations
-/////////////////////////////////////////////////////
-
-struct Sum{
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    typename acle<T>::vt r_v = svadd_x(pg1, a_v, b_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct Sub{
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    typename acle<T>::vt r_v = svsub_x(pg1, a_v, b_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct Mult{
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b, vec<T> c){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    typename acle<T>::vt c_v = svld1(pg1, c.v);
-    typename acle<T>::vt r_v = svmla_x(pg1, c_v, a_v, b_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    typename acle<T>::vt r_v = svmul_x(pg1, a_v, b_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct MultRealPart{
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v  = svld1(pg1, a.v);
-    typename acle<T>::vt b_v  = svld1(pg1, b.v);
-
-    // using FCMLA
-    typename acle<T>::vt z_v = acle<T>::zero();
-    typename acle<T>::vt r_v = svcmla_x(pg1, z_v, a_v, b_v, 0);
-
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct MaddRealPart{
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b, vec<T> c){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v  = svld1(pg1, a.v);
-    typename acle<T>::vt b_v  = svld1(pg1, b.v);
-    typename acle<T>::vt c_v  = svld1(pg1, c.v);
-
-    // using FCMLA
-    typename acle<T>::vt r_v = svcmla_x(pg1, c_v, a_v, b_v, 0);
-
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct MultComplex{
-  // Complex a*b
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    typename acle<T>::vt z_v = acle<T>::zero();
-
-    // using FCMLA
-    typename acle<T>::vt r_v = svcmla_x(pg1, z_v, a_v, b_v, 0);
-    r_v = svcmla_x(pg1, r_v, a_v, b_v, 90);
-
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct MultAddComplex{
-  // Complex a*b+c
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b, vec<T> c){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    typename acle<T>::vt c_v = svld1(pg1, c.v);;
-
-    // using FCMLA
-    typename acle<T>::vt r_v = svcmla_x(pg1, c_v, a_v, b_v, 0);
-    r_v = svcmla_x(pg1, r_v, a_v, b_v, 90);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct Div{
-  // Real
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    typename acle<T>::vt b_v = svld1(pg1, b.v);
-    typename acle<T>::vt r_v = svdiv_x(pg1, a_v, b_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct Conj{
-  // Complex
-  template <typename T>
-  inline vec<T> operator()(vec<T> a){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    svbool_t pg_odd = acle<T>::pg_odd();
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    //typename acle<T>::vt r_v = svneg_x(pg_odd, a_v);
-    typename acle<T>::vt r_v = svneg_m(a_v, pg_odd, a_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct TimesMinusI{
-  // Complex
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    const vec<typename acle<T>::uint> tbl_swap = acle<T>::tbl_swap();
-    svbool_t pg1 = acle<T>::pg1();
-    svbool_t pg_odd = acle<T>::pg_odd();
-
-    typename acle<T>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    a_v = svtbl(a_v, tbl_swap_v);
-    typename acle<T>::vt r_v = svneg_m(a_v, pg_odd, a_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct TimesI{
-  // Complex
-  template <typename T>
-  inline vec<T> operator()(vec<T> a, vec<T> b){
-    vec<T> out;
-    const vec<typename acle<T>::uint> tbl_swap = acle<T>::tbl_swap();
-    svbool_t pg1 = acle<T>::pg1();
-    svbool_t pg_even = acle<T>::pg_even();
-
-    typename acle<T>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
-    typename acle<T>::vt a_v = svld1(pg1, a.v);
-    a_v = svtbl(a_v, tbl_swap_v);
-    //typename acle<T>::vt r_v = svneg_x(pg_even, a_v);
-    typename acle<T>::vt r_v = svneg_m(a_v, pg_even, a_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-};
-
-struct PrecisionChange {
-  static inline vech StoH (const vecf &sa,const vecf &sb) {
-    vech ret;
-    svbool_t pg1s = acle<float>::pg1();
-    svbool_t pg1h = acle<uint16_t>::pg1();
-    typename acle<float>::vt sa_v = svld1(pg1s, sa.v);
-    typename acle<float>::vt sb_v = svld1(pg1s, sb.v);
-    typename acle<uint16_t>::vt ha_v = svcvt_f16_x(pg1s, sa_v);
-    typename acle<uint16_t>::vt hb_v = svcvt_f16_x(pg1s, sb_v);
-    typename acle<uint16_t>::vt r_v = svuzp1(ha_v, hb_v);
-    svst1(pg1h, (typename acle<uint16_t>::pt*)&ret.v, r_v);
-
-    return ret;
-  }
-  static inline void HtoS(vech h,vecf &sa,vecf &sb) {
-    svbool_t pg1h = acle<uint16_t>::pg1();
-    svbool_t pg1s = acle<float>::pg1();
-    typename acle<uint16_t>::vt h_v = svld1(pg1h, (typename acle<uint16_t>::pt*)&h.v);
-    typename acle<uint16_t>::vt ha_v = svzip1(h_v, h_v);
-    typename acle<uint16_t>::vt hb_v = svzip2(h_v, h_v);
-    typename acle<float>::vt sa_v = svcvt_f32_x(pg1s, ha_v);
-    typename acle<float>::vt sb_v = svcvt_f32_x(pg1s, hb_v);
-    svst1(pg1s, sa.v, sa_v);
-    svst1(pg1s, sb.v, sb_v);
-  }
-  static inline vecf DtoS (vecd a,vecd b) {
-    vecf ret;
-    svbool_t pg1d = acle<double>::pg1();
-    svbool_t pg1s = acle<float>::pg1();
-    typename acle<double>::vt a_v = svld1(pg1d, a.v);
-    typename acle<double>::vt b_v = svld1(pg1d, b.v);
-    typename acle<float>::vt sa_v = svcvt_f32_x(pg1d, a_v);
-    typename acle<float>::vt sb_v = svcvt_f32_x(pg1d, b_v);
-    typename acle<float>::vt r_v = svuzp1(sa_v, sb_v);
-    svst1(pg1s, ret.v, r_v);
-
-    return ret;
-  }
-  static inline void StoD (vecf s,vecd &a,vecd &b) {
-    svbool_t pg1s = acle<float>::pg1();
-    svbool_t pg1d = acle<double>::pg1();
-    typename acle<float>::vt s_v = svld1(pg1s, s.v);
-    typename acle<float>::vt sa_v = svzip1(s_v, s_v);
-    typename acle<float>::vt sb_v = svzip2(s_v, s_v);
-    typename acle<double>::vt a_v = svcvt_f64_x(pg1d, sa_v);
-    typename acle<double>::vt b_v = svcvt_f64_x(pg1d, sb_v);
-    svst1(pg1d, a.v, a_v);
-    svst1(pg1d, b.v, b_v);
-  }
-  static inline vech DtoH (vecd a,vecd b,vecd c,vecd d) {
-    vech ret;
-    svbool_t pg1d = acle<double>::pg1();
-    svbool_t pg1h = acle<uint16_t>::pg1();
-    typename acle<double>::vt a_v = svld1(pg1d, a.v);
-    typename acle<double>::vt b_v = svld1(pg1d, b.v);
-    typename acle<double>::vt c_v = svld1(pg1d, c.v);
-    typename acle<double>::vt d_v = svld1(pg1d, d.v);
-    typename acle<uint16_t>::vt ha_v = svcvt_f16_x(pg1d, a_v);
-    typename acle<uint16_t>::vt hb_v = svcvt_f16_x(pg1d, b_v);
-    typename acle<uint16_t>::vt hc_v = svcvt_f16_x(pg1d, c_v);
-    typename acle<uint16_t>::vt hd_v = svcvt_f16_x(pg1d, d_v);
-    typename acle<uint16_t>::vt hab_v = svuzp1(ha_v, hb_v);
-    typename acle<uint16_t>::vt hcd_v = svuzp1(hc_v, hd_v);
-    typename acle<uint16_t>::vt r_v = svuzp1(hab_v, hcd_v);
-    svst1(pg1h, (typename acle<uint16_t>::pt*)&ret.v, r_v);
-
-    return ret;
-/*
-    vecf sa,sb;
-    sa = DtoS(a,b);
-    sb = DtoS(c,d);
-    return StoH(sa,sb);
-*/
-  }
-  static inline void HtoD(vech h,vecd &a,vecd &b,vecd &c,vecd &d) {
-    svbool_t pg1h = acle<uint16_t>::pg1();
-    svbool_t pg1d = acle<double>::pg1();
-    typename acle<uint16_t>::vt h_v = svld1(pg1h, (typename acle<uint16_t>::pt*)&h.v);
-    typename acle<uint16_t>::vt sa_v = svzip1(h_v, h_v);
-    typename acle<uint16_t>::vt sb_v = svzip2(h_v, h_v);
-    typename acle<uint16_t>::vt da_v = svzip1(sa_v, sa_v);
-    typename acle<uint16_t>::vt db_v = svzip2(sa_v, sa_v);
-    typename acle<uint16_t>::vt dc_v = svzip1(sb_v, sb_v);
-    typename acle<uint16_t>::vt dd_v = svzip2(sb_v, sb_v);
-    typename acle<double>::vt a_v = svcvt_f64_x(pg1d, da_v);
-    typename acle<double>::vt b_v = svcvt_f64_x(pg1d, db_v);
-    typename acle<double>::vt c_v = svcvt_f64_x(pg1d, dc_v);
-    typename acle<double>::vt d_v = svcvt_f64_x(pg1d, dd_v);
-    svst1(pg1d, a.v, a_v);
-    svst1(pg1d, b.v, b_v);
-    svst1(pg1d, c.v, c_v);
-    svst1(pg1d, d.v, d_v);
-/*
-    vecf sa,sb;
-    HtoS(h,sa,sb);
-    StoD(sa,a,b);
-    StoD(sb,c,d);
-*/
-  }
-};
-
-struct Exchange{
-
-  // Exchange0 is valid for arbitrary SVE vector length
-  template <typename T>
-  static inline void Exchange0(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a1_v = svld1(pg1, in1.v);
-    typename acle<T>::vt a2_v = svld1(pg1, in2.v);
-    typename acle<T>::vt r1_v = svext(a1_v, a1_v, (uint64_t)W<T>::c);
-    r1_v = svext(r1_v, a2_v, (uint64_t)W<T>::c);
-    typename acle<T>::vt r2_v = svext(a2_v, a2_v, (uint64_t)W<T>::c);
-    r2_v = svext(a1_v, r2_v, (uint64_t)W<T>::c);
-    svst1(pg1, out1.v, r1_v);
-    svst1(pg1, out2.v, r2_v);
-  }
-
-  template <typename T>
-  static inline void Exchange1(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
-    // this one is tricky; svtrn2q* from SVE2 fits best, but it is not available in SVE1
-    // alternative: use 4-el structure; expect translation into ldp + stp -> SFI
-    svbool_t pg1 = acle<T>::pg1();
-    const vec<typename acle<T>::uint> tbl_exch1a = acle<T>::tbl_exch1a();
-    const vec<typename acle<T>::uint> tbl_exch1b = acle<T>::tbl_exch1b();
-    const vec<typename acle<T>::uint> tbl_exch1c = acle<T>::tbl_exch1c();
-
-    typename acle<T>::svuint tbl_exch1a_v = svld1(pg1, tbl_exch1a.v);
-    typename acle<T>::svuint tbl_exch1b_v = svld1(pg1, tbl_exch1b.v);
-    typename acle<T>::svuint tbl_exch1c_v = svld1(pg1, tbl_exch1c.v);
-
-    typename acle<T>::vt in1_v  = svld1(pg1, in1.v);
-    typename acle<T>::vt in2_v  = svld1(pg1, in2.v);
-
-    typename acle<T>::vt a1_v   = svtbl(in1_v, tbl_exch1a_v);
-    typename acle<T>::vt a2_v   = svtbl(in2_v, tbl_exch1b_v);
-    typename acle<T>::vt b1_v   = svext(a2_v, a1_v, (uint64_t)(W<T>::r / 2u));
-    typename acle<T>::vt b2_v   = svext(a1_v, a2_v, (uint64_t)(W<T>::r / 2u));
-    typename acle<T>::vt out1_v = svtbl(b1_v, tbl_exch1c_v);
-    typename acle<T>::vt out2_v = svtbl(b2_v, tbl_exch1a_v);
-
-    svst1(pg1, out1.v, out1_v);
-    svst1(pg1, out2.v, out2_v);
-  }
-
-  template <typename T>
-  static inline void Exchange2(vec<T> &out1, vec<T> &out2, const vec<T> &in1, const vec<T> &in2){
-    svbool_t pg1 = acle<double>::pg1();
-    typename acle<double>::vt a1_v = svld1(pg1, (typename acle<double>::pt*)in1.v);
-    typename acle<double>::vt a2_v = svld1(pg1, (typename acle<double>::pt*)in2.v);
-    typename acle<double>::vt r1_v = svtrn1(a1_v, a2_v);
-    typename acle<double>::vt r2_v = svtrn2(a1_v, a2_v);
-    svst1(pg1, (typename acle<double>::pt*)out1.v, r1_v);
-    svst1(pg1, (typename acle<double>::pt*)out2.v, r2_v);
-  }
-
-  static inline void Exchange3(vecf &out1, vecf &out2, const vecf &in1, const vecf &in2){
-    svbool_t pg1 = acle<float>::pg1();
-    typename acle<float>::vt a1_v = svld1(pg1, in1.v);
-    typename acle<float>::vt a2_v = svld1(pg1, in2.v);
-    typename acle<float>::vt r1_v = svtrn1(a1_v, a2_v);
-    typename acle<float>::vt r2_v = svtrn2(a1_v, a2_v);
-    svst1(pg1, out1.v, r1_v);
-    svst1(pg1, out2.v, r2_v);
-  }
-
-  static inline void Exchange3(vecd &out1, vecd &out2, const vecd &in1, const vecd &in2){
-    assert(0);
-    return;
-  }
-};
-
-struct Permute{
-
-  // Permute0 is valid for any SVE vector width
-  template <typename T>
-  static inline vec<T> Permute0(vec<T> in) {
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, in.v);
-    typename acle<T>::vt r_v = svext(a_v, a_v, (uint64_t)(W<T>::r / 2u));
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-
-  static inline vecd Permute1(vecd in) {
-    vecd out;
-    const vec<typename acle<double>::uint> tbl_swap = acle<double>::tbl1();
-    svbool_t pg1 = acle<double>::pg1();
-    typename acle<double>::vt a_v = svld1(pg1, in.v);
-    typename acle<double>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
-    typename acle<double>::vt r_v = svtbl(a_v, tbl_swap_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-
-  static inline vecf Permute1(vecf in) {
-    vecf out;
-    const vec<typename acle<float>::uint> tbl_swap = acle<float>::tbl1();
-    svbool_t pg1 = acle<float>::pg1();
-    typename acle<float>::vt a_v = svld1(pg1, in.v);
-    typename acle<float>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
-    typename acle<float>::vt r_v = svtbl(a_v, tbl_swap_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-
-  static inline vecd Permute2(vecd in) {
-    vecd out;
-    const vec<typename acle<double>::uint> tbl_swap = acle<double>::tbl_swap();
-    svbool_t pg1 = acle<double>::pg1();
-    typename acle<double>::vt a_v = svld1(pg1, in.v);
-    typename acle<double>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
-    typename acle<double>::vt r_v = svtbl(a_v, tbl_swap_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-
-  static inline vecf Permute2(vecf in) {
-    vecf out;
-    const vec<typename acle<float>::uint> tbl_swap = acle<float>::tbl2();
-    svbool_t pg1 = acle<float>::pg1();
-    typename acle<float>::vt a_v = svld1(pg1, in.v);
-    typename acle<float>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
-    typename acle<float>::vt r_v = svtbl(a_v, tbl_swap_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-
-  static inline vecf Permute3(vecf in) {
-    vecf out;
-    const vec<typename acle<float>::uint> tbl_swap = acle<float>::tbl_swap();
-    svbool_t pg1 = acle<float>::pg1();
-    typename acle<float>::vt a_v = svld1(pg1, in.v);
-    typename acle<float>::svuint tbl_swap_v = svld1(pg1, tbl_swap.v);
-    typename acle<float>::vt r_v = svtbl(a_v, tbl_swap_v);
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-
-  static inline vecd Permute3(vecd in) {
-    return in;
-  }
-
-};
-
-struct Rotate{
-
-  template <int n, typename T> static inline vec<T> tRotate(vec<T> in){
-    vec<T> out;
-    svbool_t pg1 = acle<T>::pg1();
-    typename acle<T>::vt a_v = svld1(pg1, in.v);
-    typename acle<T>::vt r_v = svext(a_v, a_v, (uint64_t)(n%W<T>::r));
-    svst1(pg1, out.v, r_v);
-
-    return out;
-  }
-
-  template <typename T>
-  static inline vec<T> rotate(vec<T> in, int n){
-
-    switch(n){
-    case 0:  return tRotate<0,  T>(in); break;
-    case 1:  return tRotate<1,  T>(in); break;
-    case 2:  return tRotate<2,  T>(in); break;
-    case 3:  return tRotate<3,  T>(in); break;
-    case 4:  return tRotate<4,  T>(in); break;
-    case 5:  return tRotate<5,  T>(in); break;
-    case 6:  return tRotate<6,  T>(in); break;
-    case 7:  return tRotate<7,  T>(in); break;
-
-    case 8:  return tRotate<8,  T>(in); break;
-    case 9:  return tRotate<9,  T>(in); break;
-    case 10: return tRotate<10, T>(in); break;
-    case 11: return tRotate<11, T>(in); break;
-    case 12: return tRotate<12, T>(in); break;
-    case 13: return tRotate<13, T>(in); break;
-    case 14: return tRotate<14, T>(in); break;
-    case 15: return tRotate<15, T>(in); break;
-    default: assert(0);
-    }
-  }
-};
-
-// tree-based reduction
-#define svred(pg, v)\
-svaddv(pg, v);
-
-// left-to-right reduction
-// #define svred(pg, v)\
-// svadda(pg, 0, v)
-
-template <typename Out_type, typename In_type>
-struct Reduce{
-  //Need templated class to overload output type
-  //General form must generate error if compiled
-  inline Out_type operator()(In_type in){
-    printf("Error, using wrong Reduce function\n");
-    exit(1);
-    return 0;
-  }
-};
-
-//Complex float Reduce
-template <>
-inline Grid::ComplexF Reduce<Grid::ComplexF, vecf>::operator()(vecf in){
-  svbool_t pg1 = acle<float>::pg1();
-  svbool_t pg_even = acle<float>::pg_even();
-  svbool_t pg_odd  = acle<float>::pg_odd();
-  typename acle<float>::vt a_v = svld1(pg1, in.v);
-  float a = svred(pg_even, a_v);
-  float b = svred(pg_odd, a_v);
-
-  return Grid::ComplexF(a, b);
-
-}
-
-//Real float Reduce
-template <>
-inline Grid::RealF Reduce<Grid::RealF, vecf>::operator()(vecf in){
-  svbool_t pg1 = acle<float>::pg1();
-  typename acle<float>::vt a_v = svld1(pg1, in.v);
-  float a = svred(pg1, a_v);
-
-  return a;
-}
-
-//Complex double Reduce
-template <>
-inline Grid::ComplexD Reduce<Grid::ComplexD, vecd>::operator()(vecd in){
-  svbool_t pg1 = acle<double>::pg1();
-  svbool_t pg_even = acle<double>::pg_even();
-  svbool_t pg_odd  = acle<double>::pg_odd();
-  typename acle<double>::vt a_v = svld1(pg1, in.v);
-  double a = svred(pg_even, a_v);
-  double b = svred(pg_odd, a_v);
-
-  return Grid::ComplexD(a, b);
-}
-
-//Real double Reduce
-template <>
-inline Grid::RealD Reduce<Grid::RealD, vecd>::operator()(vecd in){
-  svbool_t pg1 = acle<double>::pg1();
-  typename acle<double>::vt a_v = svld1(pg1, in.v);
-  double a = svred(pg1, a_v);
-
-  return a;
-}
-
-//Integer Reduce
-template <>
-inline Integer Reduce<Integer, veci>::operator()(veci in){
-  svbool_t pg1 = acle<Integer>::pg1();
-  typename acle<Integer>::vt a_v = svld1(pg1, in.v);
-  Integer a = svred(pg1, a_v);
-
-  return a;
-}
-
-#undef svred
-#undef vec_imm
-
-NAMESPACE_END(Optimization)
-
-//////////////////////////////////////////////////////////////////////////////////////
-// Here assign types
-
-typedef Optimization::vech SIMD_Htype; // Reduced precision type
-typedef Optimization::vecf SIMD_Ftype; // Single precision type
-typedef Optimization::vecd SIMD_Dtype; // Double precision type
-typedef Optimization::veci SIMD_Itype; // Integer type
-
-// prefetch utilities
-inline void v_prefetch0(int size, const char *ptr){};
-inline void prefetch_HINT_T0(const char *ptr){};
-
-// Function name aliases
-typedef Optimization::Vsplat   VsplatSIMD;
-typedef Optimization::Vstore   VstoreSIMD;
-typedef Optimization::Vset     VsetSIMD;
-typedef Optimization::Vstream  VstreamSIMD;
-template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
-
-// Arithmetic operations
-typedef Optimization::Sum            SumSIMD;
-typedef Optimization::Sub            SubSIMD;
-typedef Optimization::Div            DivSIMD;
-typedef Optimization::Mult           MultSIMD;
-typedef Optimization::MultComplex    MultComplexSIMD;
-typedef Optimization::MultAddComplex MultAddComplexSIMD;
-typedef Optimization::MultRealPart   MultRealPartSIMD;
-typedef Optimization::MaddRealPart   MaddRealPartSIMD;
-typedef Optimization::Conj           ConjSIMD;
-typedef Optimization::TimesMinusI    TimesMinusISIMD;
-typedef Optimization::TimesI         TimesISIMD;
-
-NAMESPACE_END(Grid)

Grid/simd/Grid_a64fx-fixedsize.h

@@ -1,769 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: Grid_a64fx-fixedsize.h
-
-    Copyright (C) 2020
-
-    Author: Nils Meyer         <nils.meyer@ur.de>           Regensburg University
-
-    with support from Arm
-            Richard Sandiford  <richard.sandiford@arm.com>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-
-/////////////////////////////////////////////////////
-// Using SVE ACLE with fixed-size data types
-/////////////////////////////////////////////////////
-
-
-// gcc 10 features
-#if __ARM_FEATURE_SVE_BITS==512
-/* gcc 10.0.1 and gcc 10.1 bug using ACLE data types  CAS-159553-Y1K4C6
-   workaround: use gcc's internal data types, bugfix expected for gcc 10.2
-typedef svbool_t    pred __attribute__((arm_sve_vector_bits(512)));
-typedef svfloat16_t vech __attribute__((arm_sve_vector_bits(512)));
-typedef svfloat32_t vecf __attribute__((arm_sve_vector_bits(512)));
-typedef svfloat64_t vecd __attribute__((arm_sve_vector_bits(512)));
-typedef svuint32_t  veci __attribute__((arm_sve_vector_bits(512)));
-typedef svuint32_t  lutf __attribute__((arm_sve_vector_bits(512))); // LUTs for float
-typedef svuint64_t  lutd __attribute__((arm_sve_vector_bits(512))); // LUTs for double
-*/
-typedef __SVBool_t    pred __attribute__((arm_sve_vector_bits(512)));
-typedef __SVFloat16_t vech __attribute__((arm_sve_vector_bits(512)));
-typedef __SVFloat32_t vecf __attribute__((arm_sve_vector_bits(512)));
-typedef __SVFloat64_t vecd __attribute__((arm_sve_vector_bits(512)));
-typedef __SVUint32_t  veci __attribute__((arm_sve_vector_bits(512)));
-typedef __SVUint32_t  lutf __attribute__((arm_sve_vector_bits(512))); // LUTs for float
-typedef __SVUint64_t  lutd __attribute__((arm_sve_vector_bits(512))); // LUTs for double
-#else
-#pragma error("Oops. Illegal SVE vector size!?")
-#endif /* __ARM_FEATURE_SVE_BITS */
-
-// low-level API
-NAMESPACE_BEGIN(Grid);
-NAMESPACE_BEGIN(Optimization);
-
-// convenience union types for tables eliminating loads
-union ulutf {
-  lutf v;
-  uint32_t s[16];
-};
-union ulutd {
-  lutd v;
-  uint64_t s[8];
-};
-
-template <typename T>
-struct acle{};
-
-template <>
-struct acle<double>{
-  static inline lutd tbl_swap(){
-    const ulutd t = { .s = {1, 0, 3, 2, 5, 4, 7, 6} };
-    return t.v;
-  }
-  static inline lutd tbl0(){
-    const ulutd t = { .s = {4, 5, 6, 7, 0, 1, 2, 3} };
-    return t.v;
-  }
-  static inline lutd tbl1(){
-    const ulutd t = { .s = {2, 3, 0, 1, 6, 7, 4, 5} };
-    return t.v;
-  }
-  static inline lutd tbl_exch1a(){ // Exchange1
-    const ulutd t = { .s = {0, 1, 4, 5, 2, 3, 6, 7} };
-    return t.v;
-  }
-  static inline lutd tbl_exch1b(){ // Exchange1
-    const ulutd t = { .s = {2, 3, 6, 7, 0, 1, 4, 5} };
-    return t.v;
-  }
-  static inline lutd tbl_exch1c(){ // Exchange1
-    const ulutd t = { .s = {4, 5, 0, 1, 6, 7, 2, 3} };
-    return t.v;
-  }
-  static inline pred pg1(){return svptrue_b64();}
-  static inline pred pg_even(){return svzip1_b64(svptrue_b64(), svpfalse_b());}
-  static inline pred pg_odd() {return svzip1_b64(svpfalse_b(), svptrue_b64());}
-  static inline vecd zero(){return svdup_f64(0.);}
-};
-
-template <>
-struct acle<float>{
-  // exchange neighboring elements
-  static inline lutf tbl_swap(){
-    const ulutf t = { .s = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14} };
-    return t.v;
-  }
-  static inline lutf tbl0(){
-    const ulutf t = { .s = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7} };
-    return t.v;
-  }
-  static inline lutf tbl1(){
-    const ulutf t = { .s = {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11} };
-    return t.v;
-  }
-  static inline lutf tbl2(){
-    const ulutf t = { .s = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13} };
-    return t.v;
-  }
-  static inline lutf tbl_exch1a(){ // Exchange1
-    const ulutf t = { .s = {0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15 } };
-    return t.v;
-  }
-  static inline lutf tbl_exch1b(){ // Exchange1
-    const ulutf t = { .s = {4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 8, 9, 10, 11 } };
-    return t.v;
-  }
-  static inline lutf tbl_exch1c(){ // Exchange1
-    const ulutf t = { .s = {8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7} };
-    return t.v;
-  }
-  static inline pred pg1(){return svptrue_b32();}
-  static inline pred pg_even(){return svzip1_b32(svptrue_b32(), svpfalse_b());}
-  static inline pred pg_odd() {return svzip1_b32(svpfalse_b(), svptrue_b32());}
-  static inline vecf zero(){return svdup_f32(0.);}
-};
-
-template <>
-struct acle<uint16_t>{
-  static inline pred pg1(){return svptrue_b16();}
-  static inline pred pg_even(){return svzip1_b16(svptrue_b16(), svpfalse_b());}
-  static inline pred pg_odd() {return svzip1_b16(svpfalse_b(), svptrue_b16());}
-  static inline vech zero(){return svdup_f16(0.);}
-};
-
-template <>
-struct acle<Integer>{
-  //static inline svbool_t pg1(){return svptrue_b16();}
-  static inline pred pg1(){return svptrue_b32();}
-  static inline pred pg_even(){return svzip1_b32(svptrue_b32(), svpfalse_b());}
-  static inline pred pg_odd() {return svzip1_b32(svpfalse_b(), svptrue_b32());}
-};
-
-// ---------------------------------------------------
-
-struct Vsplat{
-  // Complex float
-  inline vecf operator()(float a, float b){
-    vecf a_v = svdup_f32(a);
-    vecf b_v = svdup_f32(b);
-    return svzip1(a_v, b_v);
-  }
-  // Real float
-  inline vecf operator()(float a){
-    return svdup_f32(a);
-  }
-  // Complex double
-  inline vecd operator()(double a, double b){
-    vecd a_v = svdup_f64(a);
-    vecd b_v = svdup_f64(b);
-    return svzip1(a_v, b_v);
-  }
-  // Real double
-  inline vecd operator()(double a){
-    return svdup_f64(a);
-  }
-  // Integer
-  inline veci operator()(Integer a){
-    return svdup_u32(a);
-  }
-};
-
-struct Vstore{
-  // Real float
-  inline void operator()(vecf a, float *D){
-    pred pg1 = acle<float>::pg1();
-    svst1(pg1, D, a);
-  }
-  // Real double
-  inline void operator()(vecd a, double *D){
-    pred pg1 = acle<double>::pg1();
-    svst1(pg1, D, a);
-  }
-  // Real float
-  inline void operator()(veci a, Integer *D){
-    pred pg1 = acle<Integer>::pg1();
-    svst1(pg1, D, a);
-  }
-};
-
-struct Vstream{
-  // Real float
-  inline void operator()(float * a, vecf b){
-    pred pg1 = acle<float>::pg1();
-    svstnt1(pg1, a, b);
-    //svst1(pg1, a, b);
-  }
-  // Real double
-  inline void operator()(double * a, vecd b){
-    pred pg1 = acle<double>::pg1();
-    svstnt1(pg1, a, b);
-    //svst1(pg1, a, b);
-  }
-};
-
-struct Vset{
-  // Complex float
-  inline vecf operator()(Grid::ComplexF *a){
-    pred pg1 = acle<float>::pg1();
-    return svld1(pg1, (float*)a);
-  }
-  // Complex double
-  inline vecd operator()(Grid::ComplexD *a){
-    pred pg1 = acle<double>::pg1();
-    return svld1(pg1, (double*)a);
-  }
-  // Real float
-  inline vecf operator()(float *a){
-    pred pg1 = acle<float>::pg1();
-    return svld1(pg1, a);
-  }
-  // Real double
-  inline vecd operator()(double *a){
-    pred pg1 = acle<double>::pg1();
-    return svld1(pg1, a);
-  }
-  // Integer
-  inline veci operator()(Integer *a){
-    pred pg1 = acle<Integer>::pg1();
-    return svld1(pg1, a);
-  }
-};
-
-/////////////////////////////////////////////////////
-// Arithmetic operations
-/////////////////////////////////////////////////////
-
-struct Sum{
-  // Complex/real float
-  inline vecf operator()(vecf a, vecf b){
-    pred pg1 = acle<float>::pg1();
-    return svadd_x(pg1, a, b);
-  }
-  // Complex/real double
-  inline vecd operator()(vecd a, vecd b){
-    pred pg1 = acle<double>::pg1();
-    return svadd_x(pg1, a, b);
-  }
-  // Integer
-  inline veci operator()(veci a, veci b){
-    pred pg1 = acle<Integer>::pg1();
-    return svadd_x(pg1, a, b);
-  }
-};
-
-struct Sub{
-  // Complex/real float
-  inline vecf operator()(vecf a, vecf b){
-    pred pg1 = acle<float>::pg1();
-    return svsub_x(pg1, a, b);
-  }
-  // Complex/real double
-  inline vecd operator()(vecd a, vecd b){
-    pred pg1 = acle<double>::pg1();
-    return svsub_x(pg1, a, b);
-  }
-  // Integer
-  inline veci operator()(veci a, veci b){
-    pred pg1 = acle<Integer>::pg1();
-    return svsub_x(pg1, a, b);
-  }
-
-};
-
-struct Mult{
-  // Real float fma
-  inline vecf operator()(vecf a, vecf b, vecf c){
-    pred pg1 = acle<float>::pg1();
-    return svmad_x(pg1, b, c, a);
-  }
-  // Real double fma
-  inline vecd operator()(vecd a, vecd b, vecd c){
-    pred pg1 = acle<double>::pg1();
-    return svmad_x(pg1, b, c, a);
-  }
-  // Real float
-  inline vecf operator()(vecf a, vecf b){
-    pred pg1 = acle<float>::pg1();
-    return svmul_x(pg1, a, b);
-  }
-  // Real double
-  inline vecd operator()(vecd a, vecd b){
-    pred pg1 = acle<double>::pg1();
-    return svmul_x(pg1, a, b);
-  }
-  // Integer
-  inline veci operator()(veci a, veci b){
-    pred pg1 = acle<Integer>::pg1();
-    return svmul_x(pg1, a, b);
-  }
-};
-
-struct MultRealPart{
-  // Complex float
-  inline vecf operator()(vecf a, vecf b){
-    pred pg1 = acle<float>::pg1();
-    // using FCMLA
-    vecf z_v = acle<float>::zero();
-    return svcmla_x(pg1, z_v, a, b, 0);
-  }
-  // Complex double
-  inline vecd operator()(vecd a, vecd b){
-    pred pg1 = acle<double>::pg1();
-    // using FCMLA
-    vecd z_v = acle<double>::zero();
-    return svcmla_x(pg1, z_v, a, b, 0);
-  }
-};
-
-struct MaddRealPart{
-  // Complex float
-  inline vecf operator()(vecf a, vecf b, vecf c){
-    pred pg1 = acle<float>::pg1();
-    // using FCMLA
-    return svcmla_x(pg1, c, a, b, 0);
-  }
-  // Complex double
-  inline vecd operator()(vecd a, vecd b, vecd c){
-    pred pg1 = acle<double>::pg1();
-    // using FCMLA
-    return svcmla_x(pg1, c, a, b, 0);
-  }
-};
-
-struct MultComplex{
-  // Complex a*b
-  // Complex float
-  inline vecf operator()(vecf a, vecf b){
-    pred pg1 = acle<float>::pg1();
-    vecf z = acle<float>::zero();
-    // using FCMLA
-    vecf r_v = svcmla_x(pg1, z, a, b, 0);
-    return svcmla_x(pg1, r_v, a, b, 90);
-  }
-  // Complex double
-  inline vecd operator()(vecd a, vecd b){
-    pred pg1 = acle<double>::pg1();
-    vecd z = acle<double>::zero();
-    // using FCMLA
-    vecd r_v = svcmla_x(pg1, z, a, b, 0);
-    return svcmla_x(pg1, r_v, a, b, 90);
-  }
-};
-
-struct MultAddComplex{
-  // Complex a*b+c
-  // Complex float
-  inline vecf operator()(vecf a, vecf b, vecf c){
-    pred pg1 = acle<float>::pg1();
-    // using FCMLA
-    vecf r_v = svcmla_x(pg1, c, a, b, 0);
-    return svcmla_x(pg1, r_v, a, b, 90);
-  }
-  // Complex double
-  inline vecd operator()(vecd a, vecd b, vecd c){
-    pred pg1 = acle<double>::pg1();
-    // using FCMLA
-    vecd r_v = svcmla_x(pg1, c, a, b, 0);
-    return svcmla_x(pg1, r_v, a, b, 90);
-  }
-};
-
-struct Div{
-  // Real float
-  inline vecf operator()(vecf a, vecf b){
-    pred pg1 = acle<float>::pg1();
-    return svdiv_x(pg1, a, b);
-  }
-  // Real double
-  inline vecd operator()(vecd a, vecd b){
-    pred pg1 = acle<double>::pg1();
-    return svdiv_x(pg1, a, b);
-  }
-};
-
-struct Conj{
-  // Complex float
-  inline vecf operator()(vecf a){
-    pred pg_odd = acle<float>::pg_odd();
-    //return svneg_x(pg_odd, a);  this is unsafe
-    return svneg_m(a, pg_odd, a);
-  }
-  // Complex double
-  inline vecd operator()(vecd a){
-    pred pg_odd = acle<double>::pg_odd();
-    //return svneg_x(pg_odd, a);  this is unsafe
-    return svneg_m(a, pg_odd, a);
-  }
-};
-
-struct TimesMinusI{
-  // Complex float
-  inline vecf operator()(vecf a, vecf b){
-    lutf tbl_swap = acle<float>::tbl_swap();
-    pred pg1 = acle<float>::pg1();
-    pred pg_odd = acle<float>::pg_odd();
-
-    vecf a_v = svtbl(a, tbl_swap);
-    //return svneg_x(pg_odd, a_v);  this is unsafe
-    return svneg_m(a_v, pg_odd, a_v);
-  }
-  // Complex double
-  inline vecd operator()(vecd a, vecd b){
-    lutd tbl_swap = acle<double>::tbl_swap();
-    pred pg1 = acle<double>::pg1();
-    pred pg_odd = acle<double>::pg_odd();
-
-    vecd a_v = svtbl(a, tbl_swap);
-    //return svneg_x(pg_odd, a_v);  this is unsafe
-    return svneg_m(a_v, pg_odd, a_v);
-  }
-};
-
-struct TimesI{
-  // Complex float
-  inline vecf operator()(vecf a, vecf b){
-    lutf tbl_swap = acle<float>::tbl_swap();
-    pred pg1 = acle<float>::pg1();
-    pred pg_even = acle<float>::pg_even();
-
-    vecf a_v = svtbl(a, tbl_swap);
-    //return svneg_x(pg_even, a_v);  this is unsafe
-    return svneg_m(a_v, pg_even, a_v);
-  }
-  // Complex double
-  inline vecd operator()(vecd a, vecd b){
-    lutd tbl_swap = acle<double>::tbl_swap();
-    pred pg1 = acle<double>::pg1();
-    pred pg_even = acle<double>::pg_even();
-
-    vecd a_v = svtbl(a, tbl_swap);
-    //return svneg_x(pg_even, a_v);  this is unsafe
-    return svneg_m(a_v, pg_even, a_v);
-  }
-};
-
-struct PrecisionChange {
-  static inline vech StoH (vecf sa, vecf sb) {
-    pred pg1s = acle<float>::pg1();
-    vech ha_v = svcvt_f16_x(pg1s, sa);
-    vech hb_v = svcvt_f16_x(pg1s, sb);
-    return svuzp1(ha_v, hb_v);
-  }
-  static inline void HtoS(vech h,vecf &sa,vecf &sb) {
-    pred pg1s = acle<float>::pg1();
-    vech ha_v = svzip1(h, h);
-    vech hb_v = svzip2(h, h);
-    sa = svcvt_f32_x(pg1s, ha_v);
-    sb = svcvt_f32_x(pg1s, hb_v);
-  }
-  static inline vecf DtoS (vecd a,vecd b) {
-    pred pg1d = acle<double>::pg1();
-    vecf sa_v = svcvt_f32_x(pg1d, a);
-    vecf sb_v = svcvt_f32_x(pg1d, b);
-    return svuzp1(sa_v, sb_v);
-  }
-  static inline void StoD (vecf s,vecd &a,vecd &b) {
-    pred pg1d = acle<double>::pg1();
-    vecf sa_v = svzip1(s, s);
-    vecf sb_v = svzip2(s, s);
-    a = svcvt_f64_x(pg1d, sa_v);
-    b = svcvt_f64_x(pg1d, sb_v);
-  }
-  static inline vech DtoH (vecd a,vecd b,vecd c,vecd d) {
-    pred pg1d = acle<double>::pg1();
-    pred pg1h = acle<uint16_t>::pg1();
-    vech ha_v = svcvt_f16_x(pg1d, a);
-    vech hb_v = svcvt_f16_x(pg1d, b);
-    vech hc_v = svcvt_f16_x(pg1d, c);
-    vech hd_v = svcvt_f16_x(pg1d, d);
-    vech hab_v = svuzp1(ha_v, hb_v);
-    vech hcd_v = svuzp1(hc_v, hd_v);
-    return svuzp1(hab_v, hcd_v);
-
-/*
-    vecf sa,sb;
-    sa = DtoS(a,b);
-    sb = DtoS(c,d);
-    return StoH(sa,sb);
-*/
-  }
-  static inline void HtoD(vech h,vecd &a,vecd &b,vecd &c,vecd &d) {
-    pred pg1h = acle<uint16_t>::pg1();
-    pred pg1d = acle<double>::pg1();
-    vech sa_v = svzip1(h, h);
-    vech sb_v = svzip2(h, h);
-    vech da_v = svzip1(sa_v, sa_v);
-    vech db_v = svzip2(sa_v, sa_v);
-    vech dc_v = svzip1(sb_v, sb_v);
-    vech dd_v = svzip2(sb_v, sb_v);
-    a = svcvt_f64_x(pg1d, da_v);
-    b = svcvt_f64_x(pg1d, db_v);
-    c = svcvt_f64_x(pg1d, dc_v);
-    d = svcvt_f64_x(pg1d, dd_v);
-
-/*
-    vecf sa,sb;
-    HtoS(h,sa,sb);
-    StoD(sa,a,b);
-    StoD(sb,c,d);
-*/
-  }
-};
-
-struct Exchange{
-  // float
-  static inline void Exchange0(vecf &out1, vecf &out2, vecf in1, vecf in2){
-    vecf r1_v = svext(in1, in1, (uint64_t)8u);
-    vecf r2_v = svext(in2, in2, (uint64_t)8u);
-    out1 = svext(r1_v, in2, (uint64_t)8u);
-    out2 = svext(in1, r2_v, (uint64_t)8u);
-  }
-  static inline void Exchange1(vecf &out1, vecf &out2, vecf in1, vecf in2){
-    // this one is tricky; svtrn2q* from SVE2 fits best, but it is not available in SVE1
-    // alternative: use 4-el structure; expect translation into 4x ldp + 4x stp -> SFI
-    lutf tbl_exch1a = acle<float>::tbl_exch1a();
-    lutf tbl_exch1b = acle<float>::tbl_exch1b();
-    lutf tbl_exch1c = acle<float>::tbl_exch1c();
-
-    vecf a1_v = svtbl(in1, tbl_exch1a);
-    vecf a2_v = svtbl(in2, tbl_exch1b);
-    vecf b1_v  = svext(a2_v, a1_v, (uint64_t)8u);
-    vecf b2_v  = svext(a1_v, a2_v, (uint64_t)8u);
-    out1 = svtbl(b1_v, tbl_exch1c);
-    out2 = svtbl(b2_v, tbl_exch1a);
-  }
-  static inline void Exchange2(vecf &out1, vecf &out2, vecf in1, vecf in2){
-    out1 = (vecf)svtrn1((vecd)in1, (vecd)in2);
-    out2 = (vecf)svtrn2((vecd)in1, (vecd)in2);
-  }
-  static inline void Exchange3(vecf &out1, vecf &out2, vecf in1, vecf in2){
-    out1 = svtrn1(in1, in2);
-    out2 = svtrn2(in1, in2);
-  }
-
-  // double
-  static inline void Exchange0(vecd &out1, vecd &out2, vecd in1, vecd in2){
-    vecd r1_v = svext(in1, in1, (uint64_t)4u);
-    vecd r2_v = svext(in2, in2, (uint64_t)4u);
-    out1 = svext(r1_v, in2, (uint64_t)4u);
-    out2 = svext(in1, r2_v, (uint64_t)4u);
-  }
-  static inline void Exchange1(vecd &out1, vecd &out2, vecd in1, vecd in2){
-    // this one is tricky; svtrn2q* from SVE2 fits best, but it is not available in SVE1
-    // alternative: use 4-el structure; expect translation into 4x ldp + 4x stp -> SFI
-    lutd tbl_exch1a = acle<double>::tbl_exch1a();
-    lutd tbl_exch1b = acle<double>::tbl_exch1b();
-    lutd tbl_exch1c = acle<double>::tbl_exch1c();
-
-    vecd a1_v = svtbl(in1, tbl_exch1a);
-    vecd a2_v = svtbl(in2, tbl_exch1b);
-    vecd b1_v = svext(a2_v, a1_v, (uint64_t)4u);
-    vecd b2_v = svext(a1_v, a2_v, (uint64_t)4u);
-    out1 = svtbl(b1_v, tbl_exch1c);
-    out2 = svtbl(b2_v, tbl_exch1a);
-  }
-  static inline void Exchange2(vecd &out1, vecd &out2, vecd in1, vecd in2){
-    out1 = svtrn1(in1, in2);
-    out2 = svtrn2(in1, in2);
-  }
-  static inline void Exchange3(vecd &out1, vecd &out2, vecd in1, vecd in2){
-    assert(0);
-    return;
-  }
-};
-
-#undef VECTOR_FOR
-
-struct Permute{
-  // float
-  static inline vecf Permute0(vecf in) {
-    return svext(in, in, (uint64_t)8u);
-  }
-  static inline vecf Permute1(vecf in) {
-    lutf tbl_swap = acle<float>::tbl1();
-    return svtbl(in, tbl_swap);
-  }
-  static inline vecf Permute2(vecf in) {
-    lutf tbl_swap = acle<float>::tbl2();
-    return svtbl(in, tbl_swap);
-  }
-  static inline vecf Permute3(vecf in) {
-    lutf tbl_swap = acle<float>::tbl_swap();
-    return svtbl(in, tbl_swap);
-  }
-
-  // double
-  static inline vecd Permute0(vecd in) {
-    return svext(in, in, (uint64_t)(8u / 2u));
-  }
-  static inline vecd Permute1(vecd in) {
-    lutd tbl_swap = acle<double>::tbl1();
-    return svtbl(in, tbl_swap);
-  }
-  static inline vecd Permute2(vecd in) {
-    lutd tbl_swap = acle<double>::tbl_swap();
-    return svtbl(in, tbl_swap);
-  }
-  static inline vecd Permute3(vecd in) {
-    return in;
-  }
-};
-
-struct Rotate{
-
-  static inline vecf rotate(vecf in, int n){
-    switch(n){
-    case 0:  return tRotate<0>(in); break;
-    case 1:  return tRotate<1>(in); break;
-    case 2:  return tRotate<2>(in); break;
-    case 3:  return tRotate<3>(in); break;
-    case 4:  return tRotate<4>(in); break;
-    case 5:  return tRotate<5>(in); break;
-    case 6:  return tRotate<6>(in); break;
-    case 7:  return tRotate<7>(in); break;
-
-    case 8:  return tRotate<8>(in); break;
-    case 9:  return tRotate<9>(in); break;
-    case 10: return tRotate<10>(in); break;
-    case 11: return tRotate<11>(in); break;
-    case 12: return tRotate<12>(in); break;
-    case 13: return tRotate<13>(in); break;
-    case 14: return tRotate<14>(in); break;
-    case 15: return tRotate<15>(in); break;
-    default: assert(0);
-    }
-  }
-  static inline vecd rotate(vecd in, int n){
-    switch(n){
-    case 0:  return tRotate<0>(in); break;
-    case 1:  return tRotate<1>(in); break;
-    case 2:  return tRotate<2>(in); break;
-    case 3:  return tRotate<3>(in); break;
-    case 4:  return tRotate<4>(in); break;
-    case 5:  return tRotate<5>(in); break;
-    case 6:  return tRotate<6>(in); break;
-    case 7:  return tRotate<7>(in); break;
-    default: assert(0);
-    }
-  }
-
-  template <int n> static inline vecf tRotate(vecf in){
-    return svext(in, in, (uint64_t)n);
-  }
-  template <int n> static inline vecd tRotate(vecd in){
-    return svext(in, in, (uint64_t)n);
-  }
-};
-
-// tree-based reduction
-#define svred(pg, v)\
-svaddv(pg, v);
-
-// left-to-right reduction
-// #define svred(pg, v)\
-// svadda(pg, 0, v)
-
-template <typename Out_type, typename In_type>
-struct Reduce{
-  //Need templated class to overload output type
-  //General form must generate error if compiled
-  inline Out_type operator()(In_type in){
-    printf("Error, using wrong Reduce function\n");
-    //exit(1);
-    return 0;
-  }
-};
-//Complex float Reduce
-template <>
-inline Grid::ComplexF Reduce<Grid::ComplexF, vecf>::operator()(vecf in){
-  pred pg_even = acle<float>::pg_even();
-  pred pg_odd  = acle<float>::pg_odd();
-  float a = svred(pg_even, in);
-  float b = svred(pg_odd, in);
-  return Grid::ComplexF(a, b);
-}
-//Real float Reduce
-template <>
-inline Grid::RealF Reduce<Grid::RealF, vecf>::operator()(vecf in){
-  pred pg1 = acle<float>::pg1();
-  return svred(pg1, in);
-}
-//Complex double Reduce
-template <>
-inline Grid::ComplexD Reduce<Grid::ComplexD, vecd>::operator()(vecd in){
-  pred pg_even = acle<double>::pg_even();
-  pred pg_odd  = acle<double>::pg_odd();
-  double a = svred(pg_even, in);
-  double b = svred(pg_odd, in);
-  return Grid::ComplexD(a, b);
-}
-//Real double Reduce
-template <>
-inline Grid::RealD Reduce<Grid::RealD, vecd>::operator()(vecd in){
-  pred pg1 = acle<double>::pg1();
-  return svred(pg1, in);
-}
-//Integer Reduce
-template <>
-inline Integer Reduce<Integer, veci>::operator()(veci in){
-  pred pg1 = acle<Integer>::pg1();
-  return svred(pg1, in);
-}
-
-#undef svred
-
-NAMESPACE_END(Optimization);
-
-//////////////////////////////////////////////////////////////////////////////////////
-// Here assign types
-
-typedef vech SIMD_Htype; // Reduced precision type
-typedef vecf SIMD_Ftype; // Single precision type
-typedef vecd SIMD_Dtype; // Double precision type
-typedef veci SIMD_Itype; // Integer type
-
-// prefetch utilities
-inline void v_prefetch0(int size, const char *ptr){};
-inline void prefetch_HINT_T0(const char *ptr){};
-
-// Function name aliases
-typedef Optimization::Vsplat   VsplatSIMD;
-typedef Optimization::Vstore   VstoreSIMD;
-typedef Optimization::Vset     VsetSIMD;
-typedef Optimization::Vstream  VstreamSIMD;
-template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
-
-// Arithmetic operations
-typedef Optimization::Sum            SumSIMD;
-typedef Optimization::Sub            SubSIMD;
-typedef Optimization::Div            DivSIMD;
-typedef Optimization::Mult           MultSIMD;
-typedef Optimization::MultComplex    MultComplexSIMD;
-typedef Optimization::MultAddComplex MultAddComplexSIMD;
-typedef Optimization::MultRealPart   MultRealPartSIMD;
-typedef Optimization::MaddRealPart   MaddRealPartSIMD;
-typedef Optimization::Conj           ConjSIMD;
-typedef Optimization::TimesMinusI    TimesMinusISIMD;
-typedef Optimization::TimesI         TimesISIMD;
-
-NAMESPACE_END(Grid);

Grid/simd/Grid_gpu_vec.h

@@ -41,11 +41,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
-#if (!defined(GRID_CUDA)) && (!defined(GRID_HIP))
-typedef struct { uint16_t x;} half;
-#endif
-typedef struct Half2_t { half x; half y; } Half2;
-
 #define COALESCE_GRANULARITY ( GEN_SIMD_WIDTH )
 
 template<class pair>
@@ -130,14 +125,14 @@ inline accelerator GpuVector<N,datum> operator/(const GpuVector<N,datum> l,const
 }
 
 constexpr int NSIMD_RealH    = COALESCE_GRANULARITY / sizeof(half);
-constexpr int NSIMD_ComplexH = COALESCE_GRANULARITY / sizeof(Half2);
+constexpr int NSIMD_ComplexH = COALESCE_GRANULARITY / sizeof(half2);
 constexpr int NSIMD_RealF    = COALESCE_GRANULARITY / sizeof(float);
 constexpr int NSIMD_ComplexF = COALESCE_GRANULARITY / sizeof(float2);
 constexpr int NSIMD_RealD    = COALESCE_GRANULARITY / sizeof(double);
 constexpr int NSIMD_ComplexD = COALESCE_GRANULARITY / sizeof(double2);
 constexpr int NSIMD_Integer  = COALESCE_GRANULARITY / sizeof(Integer);
 
-typedef GpuComplex<Half2  > GpuComplexH;
+typedef GpuComplex<half2  > GpuComplexH;
 typedef GpuComplex<float2 > GpuComplexF;
 typedef GpuComplex<double2> GpuComplexD;
 
@@ -152,9 +147,11 @@ typedef GpuVector<NSIMD_Integer,  Integer     > GpuVectorI;
 accelerator_inline float half2float(half h)
 {
   float f;
-#if defined(GRID_CUDA) || defined(GRID_HIP)
+#ifdef GRID_SIMT
   f = __half2float(h);
 #else 
+  //f = __half2float(h);
+  __half_raw hr(h);
   Grid_half hh; 
   hh.x = hr.x;
   f=  sfw_half_to_float(hh);
@@ -164,11 +161,13 @@ accelerator_inline float half2float(half h)
 accelerator_inline half float2half(float f)
 {
   half h;
-#if defined(GRID_CUDA) || defined(GRID_HIP)
+#ifdef GRID_SIMT
   h = __float2half(f);
 #else
   Grid_half hh = sfw_float_to_half(f);
-  h.x = hh.x;
+  __half_raw hr;  
+  hr.x = hh.x;
+  h = __half(hr);
 #endif
   return h;
 }

Grid/simd/Grid_vector_types.h

@@ -110,63 +110,9 @@ accelerator_inline Grid_half sfw_float_to_half(float ff) {
 #ifdef GPU_VEC
 #include "Grid_gpu_vec.h"
 #endif
-/*
 #ifdef GEN
 #include "Grid_generic.h"
 #endif
-*/
-
-#ifdef GEN
-  #if defined(A64FX) || defined(A64FXFIXEDSIZE) // breakout A64FX SVE ACLE here
-    #include <arm_sve.h>
-    #if defined(A64FX) // VLA
-      #pragma message("building A64FX / SVE ACLE VLA")
-      #if defined(ARMCLANGCOMPAT)
-        #pragma message("applying data types patch")
-      #endif
-      #include "Grid_a64fx-2.h"
-    #endif
-    #if defined(A64FXFIXEDSIZE) // fixed size data types
-      #pragma message("building for A64FX / SVE ACLE fixed size")
-      #include "Grid_a64fx-fixedsize.h"
-    #endif
-  #else
-    //#pragma message("building GEN") // generic
-    #include "Grid_generic.h"
-  #endif
-#endif
-
-#ifdef A64FX
-  #include <arm_sve.h>
-  #ifdef __ARM_FEATURE_SVE_BITS
-    //#pragma message("building A64FX SVE VLS")
-    #include "Grid_a64fx-fixedsize.h"
-  #else
-    #pragma message("building A64FX SVE VLA")
-    #if defined(ARMCLANGCOMPAT)
-      #pragma message("applying data types patch")
-    #endif
-    #include "Grid_a64fx-2.h"
-  #endif
-#endif
-
-/*
-#ifdef A64FXVLA
-#pragma message("building A64FX VLA")
-#if defined(ARMCLANGCOMPAT)
-  #pragma message("applying data types patch")
-#endif
-#include <arm_sve.h>
-#include "Grid_a64fx-2.h"
-#endif
-
-#ifdef A64FXVLS
-#pragma message("building A64FX VLS")
-#include <arm_sve.h>
-#include "Grid_a64fx-fixedsize.h"
-#endif
-*/
-
 #ifdef SSE4
 #include "Grid_sse4.h"
 #endif
@@ -217,12 +163,6 @@ template <typename T> struct is_complex : public std::false_type {};
 template <> struct is_complex<ComplexD> : public std::true_type {};
 template <> struct is_complex<ComplexF> : public std::true_type {};
 
-template <typename T> struct is_ComplexD : public std::false_type {};
-template <> struct is_ComplexD<ComplexD> : public std::true_type {};
-
-template <typename T> struct is_ComplexF : public std::false_type {};
-template <> struct is_ComplexF<ComplexF> : public std::true_type {};
-
 template<typename T, typename V=void> struct is_real : public std::false_type {};
 template<typename T> struct is_real<T, typename std::enable_if<std::is_floating_point<T>::value,
   void>::type> : public std::true_type {};
@@ -283,69 +223,6 @@ public:
     return sizeof(Vector_type) / sizeof(Scalar_type);
   }
 
-  #ifdef ARMCLANGCOMPAT
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<!is_complex<S>::value, S>::type, Vector_type> &&rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b8(), (Scalar_type*)this, svld1(svptrue_b8(), (Scalar_type*)&(rhs.v)));
-      return *this;
-    };
-
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<!is_complex<S>::value, S>::type, Vector_type> &rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b8(), (Scalar_type*)this, svld1(svptrue_b8(), (Scalar_type*)&(rhs.v)));
-      return *this;
-    };
-
-    /*
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<is_complex<S>::value, S>::type, Vector_type> &&rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b8(), (int8_t*)this, svld1(svptrue_b8(), (int8_t*)&(rhs.v)));
-      return *this;
-    };
-
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<is_complex<S>::value, S>::type, Vector_type> &rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b8(), (int8_t*)this, svld1(svptrue_b8(), (int8_t*)&(rhs.v)));
-      return *this;
-    };
-    */
-
-    // ComplexF
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<is_ComplexF<S>::value, S>::type, Vector_type> &&rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b32(), (float*)this, svld1(svptrue_b32(), (float*)&(rhs.v)));
-      return *this;
-    };
-
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<is_ComplexF<S>::value, S>::type, Vector_type> &rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b32(), (float*)this, svld1(svptrue_b32(), (float*)&(rhs.v)));
-      return *this;
-    };
-
-    // ComplexD
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<is_ComplexD<S>::value, S>::type, Vector_type> &&rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b64(), (double*)this, svld1(svptrue_b64(), (double*)&(rhs.v)));
-      return *this;
-    };
-
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd &operator=(const Grid_simd<typename std::enable_if<is_ComplexD<S>::value, S>::type, Vector_type> &rhs) {
-      //v = rhs.v;
-      svst1(svptrue_b64(), (double*)this, svld1(svptrue_b64(), (double*)&(rhs.v)));
-      return *this;
-    };
-
-  #else
-
   accelerator_inline Grid_simd &operator=(const Grid_simd &&rhs) {
     v = rhs.v;
     return *this;
@@ -355,23 +232,10 @@ public:
     return *this;
   };  // faster than not declaring it and leaving to the compiler
 
-  #endif
 
   accelerator Grid_simd() = default;
-
-  #ifdef ARMCLANGCOMPAT
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd(const Grid_simd<typename std::enable_if<!is_complex<S>::value, S>::type, Vector_type> &rhs) { this->operator=(rhs); }
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd(const Grid_simd<typename std::enable_if<!is_complex<S>::value, S>::type, Vector_type> &&rhs) { this->operator=(rhs); }
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd(const Grid_simd<typename std::enable_if<is_complex<S>::value, S>::type, Vector_type> &rhs) { this->operator=(rhs); }
-    template <class S = Scalar_type>
-    accelerator_inline Grid_simd(const Grid_simd<typename std::enable_if<is_complex<S>::value, S>::type, Vector_type> &&rhs) { this->operator=(rhs); }
-  #else
   accelerator_inline Grid_simd(const Grid_simd &rhs) : v(rhs.v){};  // compiles in movaps
   accelerator_inline Grid_simd(const Grid_simd &&rhs) : v(rhs.v){};
-  #endif
   accelerator_inline Grid_simd(const Real a) { vsplat(*this, Scalar_type(a)); };
   // Enable if complex type
   template <typename S = Scalar_type> accelerator_inline
@@ -394,20 +258,11 @@ public:
   ///////////////////////////////////////////////
 
   // FIXME -- alias this to an accelerator_inline MAC struct.
-
-  #if defined(A64FX) || defined(A64FXFIXEDSIZE)
-  friend accelerator_inline void mac(Grid_simd *__restrict__ y,
-				     const Grid_simd *__restrict__ a,
-				     const Grid_simd *__restrict__ x) {
-    *y = fxmac((*a), (*x), (*y));
-  };
-  #else
   friend accelerator_inline void mac(Grid_simd *__restrict__ y,
 				     const Grid_simd *__restrict__ a,
 				     const Grid_simd *__restrict__ x) {
     *y = (*a) * (*x) + (*y);
   };
-  #endif
   
   friend accelerator_inline void mult(Grid_simd *__restrict__ y,
 				      const Grid_simd *__restrict__ l,
@@ -886,27 +741,6 @@ accelerator_inline Grid_simd<S, V> operator*(Grid_simd<S, V> a, Grid_simd<S, V>
   return ret;
 };
 
-// ---------------- A64FX MAC -------------------
-// Distinguish between complex types and others
-#if defined(A64FX) || defined(A64FXFIXEDSIZE)
-template <class S, class V, IfComplex<S> = 0>
-accelerator_inline Grid_simd<S, V> fxmac(Grid_simd<S, V> a, Grid_simd<S, V> b, Grid_simd<S, V> c) {
-  Grid_simd<S, V> ret;
-  ret.v = trinary<V>(a.v, b.v, c.v, MultAddComplexSIMD());
-  return ret;
-};
-
-// Real/Integer types
-template <class S, class V, IfNotComplex<S> = 0>
-accelerator_inline Grid_simd<S, V> fxmac(Grid_simd<S, V> a, Grid_simd<S, V> b, Grid_simd<S, V> c) {
-  Grid_simd<S, V> ret;
-  ret.v = trinary<V>(a.v, b.v, c.v, MultSIMD());
-  return ret;
-};
-#endif
-// ----------------------------------------------
-
-
 // Distinguish between complex types and others
 template <class S, class V, IfComplex<S> = 0>
 accelerator_inline Grid_simd<S, V> operator/(Grid_simd<S, V> a, Grid_simd<S, V> b) {
@@ -1085,14 +919,6 @@ accelerator_inline void precisionChange(vRealD    *out,vRealF    *in,int nvec)
   for(int m=0;m*2<nvec;m++){
     int n=m*2;
     Optimization::PrecisionChange::StoD(in[m].v,out[n].v,out[n+1].v);
-    // Bug in gcc 10.0.1 and gcc 10.1 using fixed-size SVE ACLE data types  CAS-159553-Y1K4C6
-    // function call results in compile-time error:
-    // In function ‘void Grid::precisionChange(Grid::vRealD*, Grid::vRealF*, int)’:
-    // .../Grid_vector_types.h:961:56: error:
-    // cannot bind non-const lvalue reference of type ‘vecd&’ {aka ‘svfloat64_t&’}
-    // to an rvalue of type ‘vecd’ {aka ‘svfloat64_t’}
-    // 961 |     Optimization::PrecisionChange::StoD(in[m].v,out[n].v,out[n+1].v);
-    //  |                                                 ~~~~~~~^
   }
 }
 accelerator_inline void precisionChange(vRealD    *out,vRealH    *in,int nvec)

Grid/simd/Grid_vector_unops.h

@@ -125,6 +125,14 @@ accelerator_inline Grid_simd<S, V> sqrt(const Grid_simd<S, V> &r) {
   return SimdApply(SqrtRealFunctor<S>(), r);
 }
 template <class S, class V>
+accelerator_inline Grid_simd<S, V> rsqrt(const Grid_simd<S, V> &r) {
+  return SimdApply(RSqrtRealFunctor<S>(), r);
+}
+template <class Scalar>
+accelerator_inline Scalar rsqrt(const Scalar &r) {
+  return (RSqrtRealFunctor<Scalar>(), r);
+}
+template <class S, class V>
 accelerator_inline Grid_simd<S, V> cos(const Grid_simd<S, V> &r) {
   return SimdApply(CosRealFunctor<S>(), r);
 }

Grid/simd/Simd.h

@@ -93,11 +93,6 @@ accelerator_inline ComplexF pow(const ComplexF& r,RealF y){ return(std::pow(r,y)
 using std::abs;
 using std::pow;
 using std::sqrt;
-using std::log;
-using std::exp;
-using std::sin;
-using std::cos;
-
 
 accelerator_inline RealF    conjugate(const RealF  & r){ return r; }
 accelerator_inline RealD    conjugate(const RealD  & r){ return r; }

Grid/stencil/Stencil.h

@@ -1307,8 +1307,8 @@ public:
 	std::cout << GridLogMessage << " Stencil SHM " << (shm_bytes)/gatheralltime/1000.*NP/NN << " GB/s per node"<<std::endl;
 
 	auto all_bytes = comms_bytes+shm_bytes;
-	std::cout << GridLogMessage << " Stencil SHM all " << (all_bytes)/gatheralltime/1000. << " GB/s per rank"<<std::endl;
-	std::cout << GridLogMessage << " Stencil SHM all " << (all_bytes)/gatheralltime/1000.*NP/NN << " GB/s per node"<<std::endl;
+	std::cout << GridLogMessage << " Stencil SHM all" << (all_bytes)/gatheralltime/1000. << " GB/s per rank"<<std::endl;
+	std::cout << GridLogMessage << " Stencil SHM all" << (all_bytes)/gatheralltime/1000.*NP/NN << " GB/s per node"<<std::endl;
 
 	auto membytes = (shm_bytes + comms_bytes/2) // read/write
 	              + (shm_bytes+comms_bytes)/2 * sizeof(vobj)/sizeof(cobj);

Grid/tensors/Tensor_Ta.h

@@ -92,22 +92,17 @@ accelerator_inline iMatrix<vtype,N> ProjectOnGroup(const iMatrix<vtype,N> &arg)
 {
   // need a check for the group type?
   iMatrix<vtype,N> ret(arg);
-  vtype rnrm;
   vtype nrm;
   vtype inner;
   for(int c1=0;c1<N;c1++){
-
-    // Normalises row c1
     zeroit(inner);	
     for(int c2=0;c2<N;c2++)
       inner += innerProduct(ret._internal[c1][c2],ret._internal[c1][c2]);
 
-    nrm = sqrt(inner);
-    nrm = 1.0/nrm;
+    nrm = rsqrt(inner);
     for(int c2=0;c2<N;c2++)
       ret._internal[c1][c2]*= nrm;
       
-    // Remove c1 from rows c1+1...N-1
     for (int b=c1+1; b<N; ++b){
       decltype(ret._internal[b][b]*ret._internal[b][b]) pr;
       zeroit(pr);

Grid/tensors/Tensor_index.h

@@ -272,7 +272,7 @@ public:
   static auto traceIndex(const iVector<vtype,N> arg) ->  iScalar<RemoveCRV(arg._internal[0])>
   {
     iScalar<RemoveCRV(arg._internal[0])> ret;
-    zeroit(ret);
+    ret._internal=Zero();
     for(int i=0;i<N;i++){
       ret._internal = ret._internal+ arg._internal[i];
     }

Grid/tensors/Tensor_traits.h

@@ -190,36 +190,6 @@ NAMESPACE_BEGIN(Grid);
     typedef ComplexD DoublePrecision;
     typedef ComplexD DoublePrecision2;
   };
-
-#ifdef GRID_CUDA
-  template<> struct GridTypeMapper<std::complex<float> > : public GridTypeMapper_Base {
-    typedef std::complex<float> scalar_type;
-    typedef std::complex<double> scalar_typeD;
-    typedef scalar_type vector_type;
-    typedef scalar_typeD vector_typeD;
-    typedef scalar_type tensor_reduced;
-    typedef scalar_type scalar_object;
-    typedef scalar_typeD scalar_objectD;
-    typedef scalar_type Complexified;
-    typedef RealF Realified;
-    typedef scalar_typeD DoublePrecision;
-    typedef scalar_typeD DoublePrecision2;
-  };
-  template<> struct GridTypeMapper<std::complex<double> > : public GridTypeMapper_Base {
-    typedef std::complex<double> scalar_type;
-    typedef std::complex<double> scalar_typeD;
-    typedef scalar_type vector_type;
-    typedef scalar_typeD vector_typeD;
-    typedef scalar_type tensor_reduced;
-    typedef scalar_type scalar_object;
-    typedef scalar_typeD scalar_objectD;
-    typedef scalar_type Complexified;
-    typedef RealD Realified;
-    typedef scalar_typeD DoublePrecision;
-    typedef scalar_typeD DoublePrecision2;
-  };
-#endif
-
   template<> struct GridTypeMapper<ComplexD2> : public GridTypeMapper_Base {
     typedef ComplexD2 scalar_type;
     typedef ComplexD2 scalar_typeD;

Grid/tensors/Tensor_unary.h

@@ -84,6 +84,7 @@ NAMESPACE_BEGIN(Grid);
   }
 
 UNARY(sqrt);
+UNARY(rsqrt);
 UNARY(sin);
 UNARY(cos);
 UNARY(asin);

Grid/threads/Accelerator.cc

@@ -16,54 +16,40 @@ void acceleratorInit(void)
   char * localRankStr = NULL;
   int rank = 0, world_rank=0; 
 #define ENV_LOCAL_RANK_OMPI    "OMPI_COMM_WORLD_LOCAL_RANK"
-#define ENV_RANK_OMPI          "OMPI_COMM_WORLD_RANK"
-#define ENV_LOCAL_RANK_SLURM   "SLURM_LOCALID"
-#define ENV_RANK_SLURM         "SLURM_PROCID"
 #define ENV_LOCAL_RANK_MVAPICH "MV2_COMM_WORLD_LOCAL_RANK"
+#define ENV_RANK_OMPI          "OMPI_COMM_WORLD_RANK"
 #define ENV_RANK_MVAPICH       "MV2_COMM_WORLD_RANK"
   // We extract the local rank initialization using an environment variable
-  if ((localRankStr = getenv(ENV_LOCAL_RANK_OMPI)) != NULL) {
-    printf("OPENMPI detected\n");
+  if ((localRankStr = getenv(ENV_LOCAL_RANK_OMPI)) != NULL)
+  {
     rank = atoi(localRankStr);		
-  } else if ((localRankStr = getenv(ENV_LOCAL_RANK_MVAPICH)) != NULL) {
-    printf("MVAPICH detected\n");
+  }
+  if ((localRankStr = getenv(ENV_LOCAL_RANK_MVAPICH)) != NULL)
+  {
     rank = atoi(localRankStr);		
-  } else if ((localRankStr = getenv(ENV_LOCAL_RANK_SLURM)) != NULL) {
-    printf("SLURM detected\n");
-    rank = atoi(localRankStr);		
-  } else { 
-    printf("MPI version is unknown - bad things may happen\n");
   }
   if ((localRankStr = getenv(ENV_RANK_OMPI   )) != NULL) { world_rank = atoi(localRankStr);}
   if ((localRankStr = getenv(ENV_RANK_MVAPICH)) != NULL) { world_rank = atoi(localRankStr);}
-  if ((localRankStr = getenv(ENV_RANK_SLURM  )) != NULL) { world_rank = atoi(localRankStr);}
 
   size_t totalDeviceMem=0;
   for (int i = 0; i < nDevices; i++) {
 
-#define GPU_PROP_FMT(canMapHostMemory,FMT)     printf("AcceleratorCudaInit[%d]:   " #canMapHostMemory ": " FMT" \n",rank,prop.canMapHostMemory);
+#define GPU_PROP_FMT(canMapHostMemory,FMT)     printf("AcceleratorCudaInit:   " #canMapHostMemory ": " FMT" \n",prop.canMapHostMemory);
 #define GPU_PROP(canMapHostMemory)             GPU_PROP_FMT(canMapHostMemory,"%d");
     cudaGetDeviceProperties(&gpu_props[i], i);
     cudaDeviceProp prop; 
     prop = gpu_props[i];
     totalDeviceMem = prop.totalGlobalMem;
     if ( world_rank == 0) {
-#ifndef GRID_DEFAULT_GPU
-      if ( i==rank ) {
-	printf("AcceleratorCudaInit[%d]: ========================\n",rank);
-	printf("AcceleratorCudaInit[%d]: Device Number    : %d\n", rank,i);
-	printf("AcceleratorCudaInit[%d]: ========================\n",rank);
-	printf("AcceleratorCudaInit[%d]: Device identifier: %s\n",rank, prop.name);
-
+      printf("AcceleratorCudaInit: ========================\n");
+      printf("AcceleratorCudaInit: Device Number    : %d\n", i);
+      printf("AcceleratorCudaInit: ========================\n");
+      printf("AcceleratorCudaInit: Device identifier: %s\n", prop.name);
 
       GPU_PROP_FMT(totalGlobalMem,"%lld");
       GPU_PROP(managedMemory);
       GPU_PROP(isMultiGpuBoard);
       GPU_PROP(warpSize);
-	GPU_PROP(pciBusID);
-	GPU_PROP(pciDeviceID);
-      }
-#endif
       //      GPU_PROP(unifiedAddressing);
       //      GPU_PROP(l2CacheSize);
       //      GPU_PROP(singleToDoublePrecisionPerfRatio);
@@ -73,17 +59,11 @@ void acceleratorInit(void)
 #undef GPU_PROP_FMT    
 #undef GPU_PROP
 
-#ifdef GRID_DEFAULT_GPU
+#ifdef GRID_IBM_SUMMIT
   // IBM Jsrun makes cuda Device numbering screwy and not match rank
-  if ( world_rank == 0 ) {
-    printf("AcceleratorCudaInit: using default device \n");
-    printf("AcceleratorCudaInit: assume user either uses a) IBM jsrun, or \n");
-    printf("AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding \n");
-    printf("AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no \n");
-  }
+  if ( world_rank == 0 )  printf("AcceleratorCudaInit: IBM Summit or similar - NOT setting device to node rank\n");
 #else
-  printf("AcceleratorCudaInit: rank %d setting device to node rank %d\n",world_rank,rank);
-  printf("AcceleratorCudaInit: Configure options --enable-select-gpu=yes \n");
+  if ( world_rank == 0 )  printf("AcceleratorCudaInit: setting device to node rank\n");
   cudaSetDevice(rank);
 #endif
   if ( world_rank == 0 )  printf("AcceleratorCudaInit: ================================================\n");
@@ -116,24 +96,20 @@ void acceleratorInit(void)
   if ((localRankStr = getenv(ENV_RANK_OMPI   )) != NULL) { world_rank = atoi(localRankStr);}
   if ((localRankStr = getenv(ENV_RANK_MVAPICH)) != NULL) { world_rank = atoi(localRankStr);}
 
-  printf("world_rank %d has %d devices\n",world_rank,nDevices);
-  size_t totalDeviceMem=0;
   for (int i = 0; i < nDevices; i++) {
 
 #define GPU_PROP_FMT(canMapHostMemory,FMT)     printf("AcceleratorHipInit:   " #canMapHostMemory ": " FMT" \n",prop.canMapHostMemory);
 #define GPU_PROP(canMapHostMemory)             GPU_PROP_FMT(canMapHostMemory,"%d");
     
     hipGetDeviceProperties(&gpu_props[i], i);
+    if ( world_rank == 0) {
       hipDeviceProp_t prop; 
       prop = gpu_props[i];
-    totalDeviceMem = prop.totalGlobalMem;
-    if ( world_rank == 0) {
       printf("AcceleratorHipInit: ========================\n");
       printf("AcceleratorHipInit: Device Number    : %d\n", i);
       printf("AcceleratorHipInit: ========================\n");
       printf("AcceleratorHipInit: Device identifier: %s\n", prop.name);
 
-      GPU_PROP_FMT(totalGlobalMem,"%lu");
       //      GPU_PROP(managedMemory);
       GPU_PROP(isMultiGpuBoard);
       GPU_PROP(warpSize);
@@ -142,21 +118,13 @@ void acceleratorInit(void)
       //      GPU_PROP(singleToDoublePrecisionPerfRatio);
     }
   }
-  MemoryManager::DeviceMaxBytes = (8*totalDeviceMem)/10; // Assume 80% ours
 #undef GPU_PROP_FMT    
 #undef GPU_PROP
-
-#ifdef GRID_DEFAULT_GPU
-  if ( world_rank == 0 ) {
-    printf("AcceleratorHipInit: using default device \n");
-    printf("AcceleratorHipInit: assume user either uses a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding \n");
-    printf("AcceleratorHipInit: Configure options --enable-summit, --enable-select-gpu=no \n");
-  }
+#ifdef GRID_IBM_SUMMIT
+  // IBM Jsrun makes cuda Device numbering screwy and not match rank
+  if ( world_rank == 0 )  printf("AcceleratorHipInit: IBM Summit or similar - NOT setting device to node rank\n");
 #else
-  if ( world_rank == 0 ) {
-    printf("AcceleratorHipInit: rank %d setting device to node rank %d\n",world_rank,rank);
-    printf("AcceleratorHipInit: Configure options --enable-select-gpu=yes \n");
-  }
+  if ( world_rank == 0 )  printf("AcceleratorHipInit: setting device to node rank\n");
   hipSetDevice(rank);
 #endif
   if ( world_rank == 0 )  printf("AcceleratorHipInit: ================================================\n");