Merge 37d1d87c3c into 73c0b29535

For some reason Dirichlet benchmark fails on several nodes; need to
debug this.

Grid/allocator/MemoryManager.cc

@@ -222,6 +222,9 @@ void MemoryManager::InitMessage(void) {
 #ifdef GRID_SYCL
   std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_shared"<<std::endl;
 #endif
+#ifdef GRID_OMPTARGET
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using OMPTARGET managed memory"<<std::endl;
+#endif
 #else
   std::cout << GridLogMessage<< "MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory"<<std::endl;
 #ifdef GRID_CUDA
@@ -233,6 +236,9 @@ void MemoryManager::InitMessage(void) {
 #ifdef GRID_SYCL
   std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_device"<<std::endl;
 #endif
+#ifdef GRID_OMPTARGET
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using OMPTARGET omp_alloc_device"<<std::endl;
+#endif
 #endif
 
 }

Grid/communicator/SharedMemoryNone.cc

@@ -68,7 +68,8 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
   // Each MPI rank should allocate our own buffer
   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-  ShmCommBuf = acceleratorAllocDevice(bytes);
+  ShmCommBuf = acceleratorAllocShared(bytes);
+  //ShmCommBuf = acceleratorAllocDevice(bytes);
 
   if (ShmCommBuf == (void *)NULL ) {
     std::cerr << " SharedMemoryNone.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;

Grid/cshift/Cshift_common.h

@@ -29,8 +29,27 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 NAMESPACE_BEGIN(Grid);
 
-extern Vector<std::pair<int,int> > Cshift_table; 
+extern std::vector<std::pair<int,int> > Cshift_table; 
+extern commVector<std::pair<int,int> > Cshift_table_device; 
 
+inline std::pair<int,int> *MapCshiftTable(void)
+{
+  // GPU version
+#ifdef ACCELERATOR_CSHIFT    
+  uint64_t sz=Cshift_table.size();
+  if (Cshift_table_device.size()!=sz )    {
+    Cshift_table_device.resize(sz);
+  }
+  acceleratorCopyToDevice((void *)&Cshift_table[0],
+			  (void *)&Cshift_table_device[0],
+			  sizeof(Cshift_table[0])*sz);
+
+  return &Cshift_table_device[0];
+#else 
+  return &Cshift_table[0];
+#endif
+  // CPU version use identify map
+}
 ///////////////////////////////////////////////////////////////////
 // Gather for when there is no need to SIMD split 
 ///////////////////////////////////////////////////////////////////
@@ -74,8 +93,8 @@ Gather_plane_simple (const Lattice<vobj> &rhs,cshiftVector<vobj> &buffer,int dim
   }
   {
     auto buffer_p = & buffer[0];
-    auto table = &Cshift_table[0];
-#ifdef ACCELERATOR_CSHIFT    
+    auto table = MapCshiftTable();
+#ifdef ACCELERATOR_CSHIFT
     autoView(rhs_v , rhs, AcceleratorRead);
     accelerator_for(i,ent,vobj::Nsimd(),{
 	coalescedWrite(buffer_p[table[i].first],coalescedRead(rhs_v[table[i].second]));
@@ -225,7 +244,7 @@ template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,cshiftVector<
   
   {
     auto buffer_p = & buffer[0];
-    auto table = &Cshift_table[0];
+    auto table = MapCshiftTable();
 #ifdef ACCELERATOR_CSHIFT    
     autoView( rhs_v, rhs, AcceleratorWrite);
     accelerator_for(i,ent,vobj::Nsimd(),{
@@ -297,30 +316,6 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
   }
 }
 
-#if (defined(GRID_CUDA) || defined(GRID_HIP)) && defined(ACCELERATOR_CSHIFT)
-
-template <typename T>
-T iDivUp(T a, T b) // Round a / b to nearest higher integer value
-{ return (a % b != 0) ? (a / b + 1) : (a / b); }
-
-template <typename T>
-__global__ void populate_Cshift_table(T* vector, T lo, T ro, T e1, T e2, T stride)
-{
-    int idx = blockIdx.x*blockDim.x + threadIdx.x;
-    if (idx >= e1*e2) return;
-
-    int n, b, o;
-
-    n = idx / e2;
-    b = idx % e2;
-    o = n*stride + b;
-
-    vector[2*idx + 0] = lo + o;
-    vector[2*idx + 1] = ro + o;
-}
-
-#endif
-
 //////////////////////////////////////////////////////
 // local to node block strided copies
 //////////////////////////////////////////////////////
@@ -345,20 +340,12 @@ template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs
   int ent=0;
 
   if(cbmask == 0x3 ){
-#if (defined(GRID_CUDA) || defined(GRID_HIP)) && defined(ACCELERATOR_CSHIFT)
-    ent = e1*e2;
-    dim3 blockSize(acceleratorThreads());
-    dim3 gridSize(iDivUp((unsigned int)ent, blockSize.x));
-    populate_Cshift_table<<<gridSize, blockSize>>>(&Cshift_table[0].first, lo, ro, e1, e2, stride);
-    accelerator_barrier();
-#else
     for(int n=0;n<e1;n++){
       for(int b=0;b<e2;b++){
         int o =n*stride+b;
 	Cshift_table[ent++] = std::pair<int,int>(lo+o,ro+o);
       }
     }
-#endif
   } else { 
     for(int n=0;n<e1;n++){
       for(int b=0;b<e2;b++){
@@ -372,7 +359,7 @@ template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs
   }
 
   {
-    auto table = &Cshift_table[0];
+    auto table = MapCshiftTable();
 #ifdef ACCELERATOR_CSHIFT    
     autoView(rhs_v , rhs, AcceleratorRead);
     autoView(lhs_v , lhs, AcceleratorWrite);
@@ -409,19 +396,11 @@ template<class vobj> void Copy_plane_permute(Lattice<vobj>& lhs,const Lattice<vo
   int ent=0;
 
   if ( cbmask == 0x3 ) {
-#if (defined(GRID_CUDA) || defined(GRID_HIP)) && defined(ACCELERATOR_CSHIFT)
-    ent = e1*e2;
-    dim3 blockSize(acceleratorThreads());
-    dim3 gridSize(iDivUp((unsigned int)ent, blockSize.x));
-    populate_Cshift_table<<<gridSize, blockSize>>>(&Cshift_table[0].first, lo, ro, e1, e2, stride);
-    accelerator_barrier();
-#else
     for(int n=0;n<e1;n++){
     for(int b=0;b<e2;b++){
       int o  =n*stride;
       Cshift_table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
     }}
-#endif
   } else {
     for(int n=0;n<e1;n++){
     for(int b=0;b<e2;b++){
@@ -432,7 +411,7 @@ template<class vobj> void Copy_plane_permute(Lattice<vobj>& lhs,const Lattice<vo
   }
 
   {
-    auto table = &Cshift_table[0];
+    auto table = MapCshiftTable();
 #ifdef ACCELERATOR_CSHIFT    
     autoView( rhs_v, rhs, AcceleratorRead);
     autoView( lhs_v, lhs, AcceleratorWrite);

Grid/cshift/Cshift_mpi.h

@@ -52,7 +52,8 @@ template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension
   int comm_dim        = rhs.Grid()->_processors[dimension] >1 ;
   int splice_dim      = rhs.Grid()->_simd_layout[dimension]>1 && (comm_dim);
 
-
+  RealD t1,t0;
+  t0=usecond();
   if ( !comm_dim ) {
     //std::cout << "CSHIFT: Cshift_local" <<std::endl;
     Cshift_local(ret,rhs,dimension,shift); // Handles checkerboarding
@@ -63,6 +64,8 @@ template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension
     //std::cout << "CSHIFT: Cshift_comms" <<std::endl;
     Cshift_comms(ret,rhs,dimension,shift);
   }
+  t1=usecond();
+  //  std::cout << GridLogPerformance << "Cshift took "<< (t1-t0)/1e3 << " ms"<<std::endl;
   return ret;
 }
 
@@ -127,16 +130,20 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
     
   int cb= (cbmask==0x2)? Odd : Even;
   int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
-
+  RealD tcopy=0.0;
+  RealD tgather=0.0;
+  RealD tscatter=0.0;
+  RealD tcomms=0.0;
+  uint64_t xbytes=0;
   for(int x=0;x<rd;x++){       
 
     int sx        =  (x+sshift)%rd;
     int comm_proc = ((x+sshift)/rd)%pd;
     
     if (comm_proc==0) {
-
+      tcopy-=usecond();
       Copy_plane(ret,rhs,dimension,x,sx,cbmask); 
-
+      tcopy+=usecond();
     } else {
 
       int words = buffer_size;
@@ -144,26 +151,39 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
 
       int bytes = words * sizeof(vobj);
 
+      tgather-=usecond();
       Gather_plane_simple (rhs,send_buf,dimension,sx,cbmask);
+      tgather+=usecond();
 
       //      int rank           = grid->_processor;
       int recv_from_rank;
       int xmit_to_rank;
       grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
-
-      grid->Barrier();
+      
+      tcomms-=usecond();
+      //      grid->Barrier();
 
       grid->SendToRecvFrom((void *)&send_buf[0],
 			   xmit_to_rank,
 			   (void *)&recv_buf[0],
 			   recv_from_rank,
 			   bytes);
+      xbytes+=bytes;
+      //      grid->Barrier();
+      tcomms+=usecond();
 
-      grid->Barrier();
-
+      tscatter-=usecond();
       Scatter_plane_simple (ret,recv_buf,dimension,x,cbmask);
+      tscatter+=usecond();
     }
   }
+  /*
+  std::cout << GridLogPerformance << " Cshift copy    "<<tcopy/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift gather  "<<tgather/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift scatter "<<tscatter/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift comm    "<<tcomms/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift BW      "<<(2.0*xbytes)/tcomms<<" MB/s "<<2*xbytes<< " Bytes "<<std::endl;
+  */
 }
 
 template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
@@ -190,6 +210,12 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   assert(shift>=0);
   assert(shift<fd);
 
+  RealD tcopy=0.0;
+  RealD tgather=0.0;
+  RealD tscatter=0.0;
+  RealD tcomms=0.0;
+  uint64_t xbytes=0;
+  
   int permute_type=grid->PermuteType(dimension);
 
   ///////////////////////////////////////////////
@@ -227,7 +253,9 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
       pointers[i] = &send_buf_extract[i][0];
     }
     int sx   = (x+sshift)%rd;
+    tgather-=usecond();
     Gather_plane_extract(rhs,pointers,dimension,sx,cbmask);
+    tgather+=usecond();
 
     for(int i=0;i<Nsimd;i++){
       
@@ -252,7 +280,8 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
       if(nbr_proc){
 	grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
 
-	grid->Barrier();
+	tcomms-=usecond();
+	//	grid->Barrier();
 
 	send_buf_extract_mpi = &send_buf_extract[nbr_lane][0];
 	recv_buf_extract_mpi = &recv_buf_extract[i][0];
@@ -262,7 +291,9 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
 			     recv_from_rank,
 			     bytes);
 
-	grid->Barrier();
+	xbytes+=bytes;
+	//	grid->Barrier();
+	tcomms+=usecond();
 
 	rpointers[i] = &recv_buf_extract[i][0];
       } else { 
@@ -270,9 +301,17 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
       }
 
     }
+    tscatter-=usecond();
     Scatter_plane_merge(ret,rpointers,dimension,x,cbmask);
+    tscatter+=usecond();
   }
-
+  /*
+  std::cout << GridLogPerformance << " Cshift (s) copy    "<<tcopy/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift (s) gather  "<<tgather/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift (s) scatter "<<tscatter/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift (s) comm    "<<tcomms/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift BW      "<<(2.0*xbytes)/tcomms<<" MB/s "<<2*xbytes<< " Bytes "<<std::endl;
+  */
 }
 #else 
 template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
@@ -292,6 +331,11 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
   assert(comm_dim==1);
   assert(shift>=0);
   assert(shift<fd);
+  RealD tcopy=0.0;
+  RealD tgather=0.0;
+  RealD tscatter=0.0;
+  RealD tcomms=0.0;
+  uint64_t xbytes=0;
   
   int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
   static cshiftVector<vobj> send_buf_v; send_buf_v.resize(buffer_size);
@@ -315,7 +359,9 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
     
     if (comm_proc==0) {
 
+      tcopy-=usecond();
       Copy_plane(ret,rhs,dimension,x,sx,cbmask); 
+      tcopy+=usecond();
 
     } else {
 
@@ -324,7 +370,9 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
 
       int bytes = words * sizeof(vobj);
 
+      tgather-=usecond();
       Gather_plane_simple (rhs,send_buf_v,dimension,sx,cbmask);
+      tgather+=usecond();
 
       //      int rank           = grid->_processor;
       int recv_from_rank;
@@ -332,7 +380,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
       grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
 
 
-      grid->Barrier();
+      tcomms-=usecond();
+      //      grid->Barrier();
 
       acceleratorCopyDeviceToDevice((void *)&send_buf_v[0],(void *)&send_buf[0],bytes);
       grid->SendToRecvFrom((void *)&send_buf[0],
@@ -340,13 +389,24 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
 			   (void *)&recv_buf[0],
 			   recv_from_rank,
 			   bytes);
+      xbytes+=bytes;
       acceleratorCopyDeviceToDevice((void *)&recv_buf[0],(void *)&recv_buf_v[0],bytes);
 
-      grid->Barrier();
+      //      grid->Barrier();
+      tcomms+=usecond();
 
+      tscatter-=usecond();
       Scatter_plane_simple (ret,recv_buf_v,dimension,x,cbmask);
+      tscatter+=usecond();
     }
   }
+  /*
+  std::cout << GridLogPerformance << " Cshift copy    "<<tcopy/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift gather  "<<tgather/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift scatter "<<tscatter/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift comm    "<<tcomms/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift BW      "<<(2.0*xbytes)/tcomms<<" MB/s "<<2*xbytes<< " Bytes "<<std::endl;
+  */
 }
 
 template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
@@ -372,6 +432,11 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   assert(simd_layout==2);
   assert(shift>=0);
   assert(shift<fd);
+  RealD tcopy=0.0;
+  RealD tgather=0.0;
+  RealD tscatter=0.0;
+  RealD tcomms=0.0;
+  uint64_t xbytes=0;
 
   int permute_type=grid->PermuteType(dimension);
 
@@ -414,8 +479,10 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
     for(int i=0;i<Nsimd;i++){       
       pointers[i] = &send_buf_extract[i][0];
     }
+    tgather-=usecond();
     int sx   = (x+sshift)%rd;
     Gather_plane_extract(rhs,pointers,dimension,sx,cbmask);
+    tgather+=usecond();
 
     for(int i=0;i<Nsimd;i++){
       
@@ -440,7 +507,8 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
       if(nbr_proc){
 	grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
 
-	grid->Barrier();
+	tcomms-=usecond();
+	//	grid->Barrier();
 
 	acceleratorCopyDeviceToDevice((void *)&send_buf_extract[nbr_lane][0],(void *)send_buf_extract_mpi,bytes);
 	grid->SendToRecvFrom((void *)send_buf_extract_mpi,
@@ -449,17 +517,28 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
 			     recv_from_rank,
 			     bytes);
 	acceleratorCopyDeviceToDevice((void *)recv_buf_extract_mpi,(void *)&recv_buf_extract[i][0],bytes);
+	xbytes+=bytes;
 
-	grid->Barrier();
+	//	grid->Barrier();
+	tcomms+=usecond();
 	rpointers[i] = &recv_buf_extract[i][0];
       } else { 
 	rpointers[i] = &send_buf_extract[nbr_lane][0];
       }
 
     }
+    tscatter-=usecond();
     Scatter_plane_merge(ret,rpointers,dimension,x,cbmask);
-  }
+    tscatter+=usecond();
 
+  }
+  /*
+  std::cout << GridLogPerformance << " Cshift (s) copy    "<<tcopy/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift (s) gather  "<<tgather/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift (s) scatter "<<tscatter/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift (s) comm    "<<tcomms/1e3<<" ms"<<std::endl;
+  std::cout << GridLogPerformance << " Cshift BW      "<<(2.0*xbytes)/tcomms<<" MB/s"<<std::endl;
+  */
 }
 #endif
 NAMESPACE_END(Grid); 

Grid/cshift/Cshift_table.cc

@@ -1,4 +1,5 @@
 #include <Grid/GridCore.h>       
 NAMESPACE_BEGIN(Grid);
-Vector<std::pair<int,int> > Cshift_table; 
+std::vector<std::pair<int,int> > Cshift_table; 
+commVector<std::pair<int,int> > Cshift_table_device; 
 NAMESPACE_END(Grid);

Grid/lattice/Lattice_arith.h

@@ -270,5 +270,42 @@ RealD axpby_norm(Lattice<vobj> &ret,sobj a,sobj b,const Lattice<vobj> &x,const L
     return axpby_norm_fast(ret,a,b,x,y);
 }
 
+/// Trace product
+template<class obj> auto traceProduct(const Lattice<obj> &rhs_1,const Lattice<obj> &rhs_2)
+  -> Lattice<decltype(trace(obj()))>
+{
+  typedef decltype(trace(obj())) robj;
+  Lattice<robj> ret_i(rhs_1.Grid());
+  autoView( rhs1 , rhs_1, AcceleratorRead);
+  autoView( rhs2 , rhs_2, AcceleratorRead);
+  autoView( ret , ret_i, AcceleratorWrite);
+  ret.Checkerboard() = rhs_1.Checkerboard();
+  accelerator_for(ss,rhs1.size(),obj::Nsimd(),{
+      coalescedWrite(ret[ss],traceProduct(rhs1(ss),rhs2(ss)));
+  });
+  return ret_i;
+}
+
+template<class obj1,class obj2> auto traceProduct(const Lattice<obj1> &rhs_1,const obj2 &rhs2)
+  -> Lattice<decltype(trace(obj1()))>
+{
+  typedef decltype(trace(obj1())) robj;
+  Lattice<robj> ret_i(rhs_1.Grid());
+  autoView( rhs1 , rhs_1, AcceleratorRead);
+  autoView( ret , ret_i, AcceleratorWrite);
+  ret.Checkerboard() = rhs_1.Checkerboard();
+  accelerator_for(ss,rhs1.size(),obj1::Nsimd(),{
+      coalescedWrite(ret[ss],traceProduct(rhs1(ss),rhs2));
+  });
+  return ret_i;
+}
+template<class obj1,class obj2> auto traceProduct(const obj2 &rhs_2,const Lattice<obj1> &rhs_1)
+  -> Lattice<decltype(trace(obj1()))>
+{
+  return traceProduct(rhs_1,rhs_2);
+}
+
+
+
 NAMESPACE_END(Grid);
 #endif

Grid/lattice/Lattice_reduction.h

@@ -251,10 +251,10 @@ inline ComplexD rankInnerProduct(const Lattice<vobj> &left,const Lattice<vobj> &
     autoView( right_v,right, AcceleratorRead);
     // This code could read coalesce
     // GPU - SIMT lane compliance...
-    accelerator_for( ss, sites, nsimd,{
-	auto x_l = left_v(ss);
-	auto y_l = right_v(ss);
-	coalescedWrite(inner_tmp_v[ss],innerProductD(x_l,y_l));
+    accelerator_for( ss, sites, 1,{
+        auto x_l = left_v[ss];
+        auto y_l = right_v[ss];
+        inner_tmp_v[ss]=innerProductD(x_l,y_l);
     });
   }
 #else
@@ -267,11 +267,18 @@ inline ComplexD rankInnerProduct(const Lattice<vobj> &left,const Lattice<vobj> &
     autoView( right_v,right, AcceleratorRead);
 
     // GPU - SIMT lane compliance...
-    accelerator_for( ss, sites, nsimd,{
-	auto x_l = left_v(ss);
-	auto y_l = right_v(ss);
-	coalescedWrite(inner_tmp_v[ss],innerProduct(x_l,y_l));
-    });
+    //accelerator_for( ss, sites, nsimd,{
+    //    auto x_l = left_v(ss);
+    //    auto y_l = right_v(ss);
+    //    coalescedWrite(inner_tmp_v[ss],innerProduct(x_l,y_l));
+    //});
+    #pragma omp target map ( to:left_v, right_v ) map ( tofrom:inner_tmp_v )
+    #pragma omp teams distribute parallel for thread_limit(THREAD_LIMIT) //nowait
+    for ( uint64_t ss=0;ss<sites;ss++) { 
+        auto x_l = left_v[ss];
+        auto y_l = right_v[ss];
+        coalescedWrite(inner_tmp_v[ss],innerProduct(x_l,y_l));
+    }
   }
 #endif
   // This is in single precision and fails some tests

Grid/lattice/Lattice_reduction_gpu.h

@@ -30,9 +30,12 @@ int getNumBlocksAndThreads(const Iterator n, const size_t sizeofsobj, Iterator &
   cudaGetDevice(&device);
 #endif
 #ifdef GRID_HIP
-  hipGetDevice(&device);
+  auto r=hipGetDevice(&device);
 #endif
-  
+#ifdef GRID_OMPTARGET
+  device = omp_get_device_num();  
+#endif
+
   Iterator warpSize            = gpu_props[device].warpSize;
   Iterator sharedMemPerBlock   = gpu_props[device].sharedMemPerBlock;
   Iterator maxThreadsPerBlock  = gpu_props[device].maxThreadsPerBlock;

Grid/lattice/Lattice_rng.h

@@ -152,6 +152,7 @@ public:
 #ifdef RNG_FAST_DISCARD
   static void Skip(RngEngine &eng,uint64_t site)
   {
+#if 0
     /////////////////////////////////////////////////////////////////////////////////////
     // Skip by 2^40 elements between successive lattice sites
     // This goes by 10^12.
@@ -162,9 +163,9 @@ public:
     // tens of seconds per trajectory so this is clean in all reasonable cases,
     // and margin of safety is orders of magnitude.
     // We could hack Sitmo to skip in the higher order words of state if necessary
-      //
-      // Replace with 2^30 ; avoid problem on large volumes
-      //
+    //
+    // Replace with 2^30 ; avoid problem on large volumes
+    //
     /////////////////////////////////////////////////////////////////////////////////////
     //      uint64_t skip = site+1;  //   Old init Skipped then drew.  Checked compat with faster init
     const int shift = 30;
@@ -179,6 +180,9 @@ public:
     assert((skip >> shift)==site); // check for overflow
 
     eng.discard(skip);
+#else
+    eng.discardhi(site);
+#endif
     //      std::cout << " Engine  " <<site << " state " <<eng<<std::endl;
   } 
 #endif

Grid/lattice/Lattice_view.h

@@ -79,7 +79,7 @@ public:
   accelerator_inline uint64_t end(void)   const { return this->_odata_size; };
   accelerator_inline uint64_t size(void)  const { return this->_odata_size; };
 
-  LatticeView(const LatticeAccelerator<vobj> &refer_to_me) : LatticeAccelerator<vobj> (refer_to_me){}
+  LatticeView(const LatticeAccelerator<vobj> &refer_to_me) : LatticeAccelerator<vobj> (refer_to_me){ }
   LatticeView(const LatticeView<vobj> &refer_to_me) = default; // Trivially copyable
   LatticeView(const LatticeAccelerator<vobj> &refer_to_me,ViewMode mode) : LatticeAccelerator<vobj> (refer_to_me)
   {

Grid/qcd/action/ActionBase.h

@@ -129,6 +129,22 @@ public:
   virtual ~Action(){}
 };
 
+template <class GaugeField >
+class EmptyAction : public Action <GaugeField>
+{
+  virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { assert(0);}; // refresh pseudofermions
+  virtual RealD S(const GaugeField& U) { return 0.0;};                             // evaluate the action
+  virtual void deriv(const GaugeField& U, GaugeField& dSdU) { assert(0); };        // evaluate the action derivative
+
+  ///////////////////////////////
+  // Logging
+  ///////////////////////////////
+  virtual std::string action_name()    { return std::string("Level Force Log"); };
+  virtual std::string LogParameters()  { return std::string("No parameters");};
+};
+
+
+
 NAMESPACE_END(Grid);
 
 #endif // ACTION_BASE_H

Grid/qcd/hmc/integrators/Integrator.h

@@ -87,6 +87,8 @@ public:
 
   const ActionSet<Field, RepresentationPolicy> as;
 
+  ActionSet<Field,RepresentationPolicy> LevelForces;
+  
   //Get a pointer to a shared static instance of the "do-nothing" momentum filter to serve as a default
   static MomentumFilterBase<MomentaField> const* getDefaultMomFilter(){ 
     static MomentumFilterNone<MomentaField> filter;
@@ -124,6 +126,9 @@ public:
     // input U actually not used in the fundamental case
     // Fundamental updates, include smearing
 
+    assert(as.size()==LevelForces.size());
+    
+    Field level_force(U.Grid()); level_force =Zero();
     for (int a = 0; a < as[level].actions.size(); ++a) {
 
       double start_full = usecond();
@@ -144,7 +149,10 @@ public:
       MomFilter->applyFilter(force);
 
       std::cout << GridLogIntegrator << " update_P : Level [" << level <<"]["<<a <<"] "<<name<<" dt "<<ep<<  std::endl;
-      
+
+      // track the total
+      level_force = level_force+force;
+
       Real force_abs   = std::sqrt(norm2(force)/U.Grid()->gSites()); //average per-site norm.  nb. norm2(latt) = \sum_x norm2(latt[x]) 
       Real impulse_abs = force_abs * ep * HMC_MOMENTUM_DENOMINATOR;    
 
@@ -167,6 +175,16 @@ public:
 
     }
 
+    {
+      // total force
+      Real force_abs   = std::sqrt(norm2(level_force)/U.Grid()->gSites()); //average per-site norm.  nb. norm2(latt) = \sum_x norm2(latt[x]) 
+      Real impulse_abs = force_abs * ep * HMC_MOMENTUM_DENOMINATOR;    
+
+      Real force_max   = std::sqrt(maxLocalNorm2(level_force));
+      Real impulse_max = force_max * ep * HMC_MOMENTUM_DENOMINATOR;    
+      LevelForces[level].actions.at(0)->deriv_log(force_abs,force_max,impulse_abs,impulse_max);
+    }
+
     // Force from the other representations
     as[level].apply(update_P_hireps, Representations, Mom, U, ep);
 
@@ -216,6 +234,16 @@ public:
 
     //Default the momentum filter to "do-nothing"
     MomFilter = getDefaultMomFilter();
+
+    for (int level = 0; level < as.size(); ++level) {
+      int multiplier = as.at(level).multiplier;
+      ActionLevel<Field> * Level = new ActionLevel<Field>(multiplier);
+      Level->push_back(new EmptyAction<Field>); 
+      LevelForces.push_back(*Level);
+      // does it copy by value or reference??
+      // - answer it copies by value, BUT the action level contains a reference that is NOT updated.
+      // Unsafe code in Guido's area
+    }
   };
 
   virtual ~Integrator() {}
@@ -233,10 +261,14 @@ public:
 
   void reset_timer(void)
   {
+    assert(as.size()==LevelForces.size());
     for (int level = 0; level < as.size(); ++level) {
       for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
         as[level].actions.at(actionID)->reset_timer();
       }
+      int actionID=0;
+      assert(LevelForces.at(level).actions.size()==1);
+      LevelForces.at(level).actions.at(actionID)->reset_timer();
     }
   }
   void print_timer(void)
@@ -298,6 +330,16 @@ public:
 		  <<" calls "     << as[level].actions.at(actionID)->deriv_num
 		  << std::endl;
       }
+      int actionID=0;
+      std::cout << GridLogMessage 
+		  << LevelForces[level].actions.at(actionID)->action_name()
+		  <<"["<<level<<"]["<< actionID<<"] :\n\t\t "
+		  <<" force max " << LevelForces[level].actions.at(actionID)->deriv_max_average()
+		  <<" norm "      << LevelForces[level].actions.at(actionID)->deriv_norm_average()
+		  <<" Fdt max  "  << LevelForces[level].actions.at(actionID)->Fdt_max_average()
+		  <<" Fdt norm "  << LevelForces[level].actions.at(actionID)->Fdt_norm_average()
+		  <<" calls "     << LevelForces[level].actions.at(actionID)->deriv_num
+		  << std::endl;
     }
     std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::"<< std::endl;
   }
@@ -319,6 +361,13 @@ public:
 	std::cout << as[level].actions.at(actionID)->LogParameters();
       }
     }
+    std::cout << " [Integrator] Total Force loggers: "<< LevelForces.size() <<std::endl;
+    for (int level = 0; level < LevelForces.size(); ++level) {
+      std::cout << GridLogMessage << "[Integrator] ---- Level: "<< level << std::endl;
+      for (int actionID = 0; actionID < LevelForces[level].actions.size(); ++actionID) {
+	std::cout << GridLogMessage << "["<< LevelForces[level].actions.at(actionID)->action_name() << "] ID: " << actionID << std::endl;
+      }
+    }
     std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::"<< std::endl;
   }
 
@@ -400,6 +449,7 @@ public:
   RealD S(Field& U) 
   {  // here also U not used
 
+    assert(as.size()==LevelForces.size());
     std::cout << GridLogIntegrator << "Integrator action\n";
 
     RealD H = - FieldImplementation::FieldSquareNorm(P)/HMC_MOMENTUM_DENOMINATOR; // - trace (P*P)/denom

Grid/qcd/smearing/GaugeConfigurationMasked.h

@@ -1,3 +1,4 @@
+
 /*!
   @file GaugeConfiguration.h
   @brief Declares the GaugeConfiguration class
@@ -6,6 +7,15 @@
 
 NAMESPACE_BEGIN(Grid);
 
+
+template<class T> void Dump(const Lattice<T> & lat,
+			    std::string s,
+			    Coordinate site = Coordinate({0,0,0,0}))
+{
+  typename T::scalar_object tmp;
+  peekSite(tmp,lat,site);
+  std::cout << " Dump "<<s<<" "<<tmp<<std::endl;
+}
 /*!
   @brief Smeared configuration masked container
   Modified for a multi-subset smearing (aka Luscher Flowed HMC)
@@ -28,6 +38,101 @@ private:
   typedef typename SU3Adjoint::LatticeAdjMatrix  AdjMatrixField;
   typedef typename SU3Adjoint::LatticeAdjVector  AdjVectorField;
 
+  void BaseSmearDerivative(GaugeField& SigmaTerm,
+			   const GaugeField& iLambda,
+			   const GaugeField& U,
+			   int mmu, RealD rho)
+  {
+    // Reference
+    // Morningstar, Peardon, Phys.Rev.D69,054501(2004)
+    // Equation 75
+    // Computing Sigma_mu, derivative of S[fat links] with respect to the thin links
+    // Output SigmaTerm
+
+    GridBase *grid = U.Grid();
+
+    WilsonLoops<Gimpl> WL;
+    GaugeLinkField staple(grid), u_tmp(grid);
+    GaugeLinkField iLambda_mu(grid), iLambda_nu(grid);
+    GaugeLinkField U_mu(grid), U_nu(grid);
+    GaugeLinkField sh_field(grid), temp_Sigma(grid);
+    Real rho_munu, rho_numu;
+
+    rho_munu = rho;
+    rho_numu = rho;
+    for(int mu = 0; mu < Nd; ++mu){
+      U_mu       = peekLorentz(      U, mu);
+      iLambda_mu = peekLorentz(iLambda, mu);
+
+      for(int nu = 0; nu < Nd; ++nu){
+	if(nu==mu) continue;
+
+	U_nu       = peekLorentz(      U, nu);
+
+	// Nd(nd-1) = 12 staples normally.
+	// We must compute 6 of these
+	// in FTHMC case
+	if ( (mu==mmu)||(nu==mmu) )
+	  WL.StapleUpper(staple, U, mu, nu);
+	
+	if(nu==mmu) {
+	  iLambda_nu = peekLorentz(iLambda, nu);
+
+	  temp_Sigma = -rho_numu*staple*iLambda_nu;  //ok
+	  //-r_numu*U_nu(x+mu)*Udag_mu(x+nu)*Udag_nu(x)*Lambda_nu(x)
+	  Gimpl::AddLink(SigmaTerm, temp_Sigma, mu);
+
+	  sh_field = Cshift(iLambda_nu, mu, 1);// general also for Gparity?
+
+	  temp_Sigma = rho_numu*sh_field*staple; //ok
+	  //r_numu*Lambda_nu(mu)*U_nu(x+mu)*Udag_mu(x+nu)*Udag_nu(x)
+	  Gimpl::AddLink(SigmaTerm, temp_Sigma, mu);
+	}
+
+	if ( mu == mmu ) { 
+	  sh_field = Cshift(iLambda_mu, nu, 1);
+
+	  temp_Sigma = -rho_munu*staple*U_nu*sh_field*adj(U_nu); //ok
+	  //-r_munu*U_nu(x+mu)*Udag_mu(x+nu)*Lambda_mu(x+nu)*Udag_nu(x)
+	  Gimpl::AddLink(SigmaTerm, temp_Sigma, mu);
+	}
+
+	//	staple = Zero();
+	sh_field = Cshift(U_nu, mu, 1);
+
+	temp_Sigma = Zero();
+
+	if ( mu == mmu )
+	  temp_Sigma = -rho_munu*adj(sh_field)*adj(U_mu)*iLambda_mu*U_nu;
+
+	if ( nu == mmu ) {
+	  temp_Sigma += rho_numu*adj(sh_field)*adj(U_mu)*iLambda_nu*U_nu;
+
+	  u_tmp = adj(U_nu)*iLambda_nu;
+	  sh_field = Cshift(u_tmp, mu, 1);
+	  temp_Sigma += -rho_numu*sh_field*adj(U_mu)*U_nu;
+	}
+	
+	sh_field = Cshift(temp_Sigma, nu, -1);
+	Gimpl::AddLink(SigmaTerm, sh_field, mu);
+
+      }
+    }
+  }
+  
+  void BaseSmear(GaugeLinkField& Cup, const GaugeField& U,int mu,RealD rho) {
+    GridBase *grid = U.Grid();
+    GaugeLinkField tmp_stpl(grid);
+    WilsonLoops<Gimpl> WL;
+    Cup = Zero();
+    for(int nu=0; nu<Nd; ++nu){
+      if (nu != mu) {
+	// get the staple in direction mu, nu
+	WL.Staple(tmp_stpl, U, mu, nu);  //nb staple conventions of IroIro and Grid differ by a dagger
+	Cup += adj(tmp_stpl*rho);
+      }
+    }
+  }
   // Adjoint vector to GaugeField force
   void InsertForce(GaugeField &Fdet,AdjVectorField &Fdet_nu,int nu)
   {
@@ -47,27 +152,54 @@ private:
     GaugeLinkField UtaU(PlaqL.Grid());
     GaugeLinkField D(PlaqL.Grid());
     AdjMatrixField Dbc(PlaqL.Grid());
+    AdjMatrixField Dbc_opt(PlaqL.Grid());
     LatticeComplex tmp(PlaqL.Grid());
     const int Ngen = SU3Adjoint::Dimension;
     Complex ci(0,1);
     ColourMatrix   ta,tb,tc;
-    
+    RealD t=0;
+    RealD tp=0;
+    RealD tta=0;
+    RealD tpk=0;
+    t-=usecond();
     for(int a=0;a<Ngen;a++) {
+      tta-=usecond();
       SU3::generator(a, ta);
+      ta = 2.0 * ci * ta;
       // Qlat Tb = 2i Tb^Grid
-      UtaU= 2.0*ci*adj(PlaqL)*ta*PlaqR;
+      UtaU= adj(PlaqL)*ta*PlaqR; // 6ms
+      tta+=usecond();
+      ////////////////////////////////////////////
+      // Could add this entire C-loop to a projection routine
+      // for performance. Could also pick checkerboard on UtaU
+      // and set checkerboard on result for 2x perf
+      ////////////////////////////////////////////
       for(int c=0;c<Ngen;c++) {
 	SU3::generator(c, tc);
-	D = Ta( (2.0)*ci*tc *UtaU);
+	tc = 2.0*ci*tc;
+	tp-=usecond(); 
+	D = Ta( tc *UtaU); // 2ms
+#if 1
+	SU3::LieAlgebraProject(Dbc_opt,D,c); // 5.5ms
+#else
 	for(int b=0;b<Ngen;b++){
 	  SU3::generator(b, tb);
 	  tmp =-trace(ci*tb*D); 
 	  PokeIndex<ColourIndex>(Dbc,tmp,b,c);  // Adjoint rep
 	}
+#endif
+	tp+=usecond();
       }
-      tmp = trace(MpInvJx * Dbc);
+      //      Dump(Dbc_opt,"Dbc_opt");
+      //      Dump(Dbc,"Dbc");
+      tpk-=usecond();
+      tmp = trace(MpInvJx * Dbc_opt);
       PokeIndex<ColourIndex>(Fdet2,tmp,a);
+      tpk+=usecond();
     }
+    t+=usecond();
+    std::cout << GridLogPerformance << " Compute_MpInvJx_dNxxdSy " << t/1e3 << " ms  proj "<<tp/1e3<< " ms"
+	      << " ta "<<tta/1e3<<" ms" << " poke "<<tpk/1e3<< " ms"<<std::endl;
   }
   
   void ComputeNxy(const GaugeLinkField &PlaqL,const GaugeLinkField &PlaqR,AdjMatrixField &NxAd)
@@ -79,12 +211,17 @@ private:
     ColourMatrix   tc;
     for(int b=0;b<Ngen;b++) {
       SU3::generator(b, tb);
-      Nx = (2.0)*Ta( adj(PlaqL)*ci*tb * PlaqR );
+      tb = 2.0 * ci * tb;
+      Nx = Ta( adj(PlaqL)*tb * PlaqR );
+#if 1
+      SU3::LieAlgebraProject(NxAd,Nx,b);
+#else
       for(int c=0;c<Ngen;c++) {
 	SU3::generator(c, tc);
 	auto tmp =closure( -trace(ci*tc*Nx)); 
 	PokeIndex<ColourIndex>(NxAd,tmp,c,b); 
       }
+#endif
     }
   }
   void ApplyMask(GaugeField &U,int smr)
@@ -164,8 +301,7 @@ public:
     // Computes ALL the staples -- could compute one only and do it here
     RealD time;
     time=-usecond();
-    this->StoutSmearing->BaseSmear(C, U);
-    Cmu = peekLorentz(C, mu);
+    BaseSmear(Cmu, U,mu,rho);
 
     //////////////////////////////////////////////////////////////////
     // Assemble Luscher exp diff map J matrix 
@@ -209,6 +345,36 @@ public:
     // dJ(x)/dxe
     //////////////////////////////////////
     time=-usecond();
+#if 1
+    std::vector<AdjMatrixField>  dJdX;    dJdX.resize(8,grid);
+    std::vector<AdjMatrix> TRb_s; TRb_s.resize(8);
+    AdjMatrixField tbXn(grid);
+    AdjMatrixField sumXtbX(grid);
+    AdjMatrixField t2(grid);
+    AdjMatrixField dt2(grid);
+    AdjMatrixField t3(grid);
+    AdjMatrixField dt3(grid);
+    AdjMatrixField aunit(grid);
+
+    for(int b=0;b<8;b++){
+      SU3Adjoint::generator(b, TRb_s[b]);
+      dJdX[b] = TRb_s[b];
+    }
+    aunit = ComplexD(1.0);
+    // Could put into an accelerator_for
+    X  = (-1.0)*ZxAd; 
+    t2 = X;
+    for (int j = 12; j > 1; --j) {
+      t3  = t2*(1.0 / (j + 1))  + aunit;
+      t2  = X * t3;
+      for(int b=0;b<8;b++){
+	dJdX[b]= TRb_s[b] * t3 + X * dJdX[b]*(1.0 / (j + 1));
+      }
+    }
+    for(int b=0;b<8;b++){
+      dJdX[b] = -dJdX[b];
+    }
+#else
     std::vector<AdjMatrixField>  dJdX;    dJdX.resize(8,grid);
     AdjMatrixField tbXn(grid);
     AdjMatrixField sumXtbX(grid);
@@ -224,14 +390,15 @@ public:
       X  = (-1.0)*ZxAd; 
       t2 = X;
       dt2 = TRb;
-      for (int j = 20; j > 1; --j) {
-	t3 = t2*(1.0 / (j + 1))  + aunit;
+      for (int j = 12; j > 1; --j) {
+	t3  = t2*(1.0 / (j + 1))  + aunit;
 	dt3 = dt2*(1.0 / (j + 1));
 	t2 = X * t3;
 	dt2 = TRb * t3 + X * dt3;
       }
       dJdX[b] = -dt2; 
     }
+#endif  
     time+=usecond();
     std::cout << GridLogMessage << "dJx took "<<time<< " us"<<std::endl;
     /////////////////////////////////////////////////////////////////
@@ -281,8 +448,8 @@ public:
     
     for(int e =0 ; e<8 ; e++){
       LatticeComplexD tr(grid);
-      ColourMatrix te;
-      SU3::generator(e, te);
+      //      ColourMatrix te;
+      //      SU3::generator(e, te);
       tr = trace(dJdX[e] * nMpInv);
       pokeColour(dJdXe_nMpInv,tr,e);
     }
@@ -493,20 +660,25 @@ public:
     //////////////////////////////////////////////////////////////////
     // Assemble the N matrix
     //////////////////////////////////////////////////////////////////
-    // Computes ALL the staples -- could compute one only here
-    this->StoutSmearing->BaseSmear(C, U);
-    Cmu = peekLorentz(C, mu);
+    double rho=this->StoutSmearing->SmearRho[1];
+    BaseSmear(Cmu, U,mu,rho);
+
     Umu = peekLorentz(U, mu);
     Complex ci(0,1);
     for(int b=0;b<Ngen;b++) {
       SU3::generator(b, Tb);
       // Qlat Tb = 2i Tb^Grid
       Nb = (2.0)*Ta( ci*Tb * Umu * adj(Cmu));
+      // FIXME -- replace this with LieAlgebraProject
+#if 0
+      SU3::LieAlgebraProject(Ncb,tmp,b);
+#else
       for(int c=0;c<Ngen;c++) {
 	SU3::generator(c, Tc);
 	auto tmp = -trace(ci*Tc*Nb); // Luchang's norm: (2Tc) (2Td) N^db = -2 delta cd N^db // - was important
 	PokeIndex<ColourIndex>(Ncb,tmp,c,b); 
       }
+#endif
     }      
 
     //////////////////////////////////////////////////////////////////
@@ -693,15 +865,19 @@ private:
 					  const GaugeField& GaugeK,int level) 
   {
     GridBase* grid = GaugeK.Grid();
-    GaugeField C(grid), SigmaK(grid), iLambda(grid);
+    GaugeField SigmaK(grid), iLambda(grid);
     GaugeField SigmaKPrimeA(grid);
     GaugeField SigmaKPrimeB(grid);
     GaugeLinkField iLambda_mu(grid);
     GaugeLinkField iQ(grid), e_iQ(grid);
     GaugeLinkField SigmaKPrime_mu(grid);
     GaugeLinkField GaugeKmu(grid), Cmu(grid);
-    
-    this->StoutSmearing->BaseSmear(C, GaugeK);
+
+    int mmu= (level/2) %Nd;
+    int cb= (level%2);
+    double rho=this->StoutSmearing->SmearRho[1];
+
+    // Can override this to do one direction only.
     SigmaK = Zero();
     iLambda = Zero();
 
@@ -712,18 +888,38 @@ private:
     // Could get away with computing only one polarisation here
     // int mu= (smr/2) %Nd;
     // SigmaKprime_A has only one component
-    for (int mu = 0; mu < Nd; mu++)
+#if 0
+    BaseSmear(Cmu, GaugeK,mu,rho);
+    GaugeKmu = peekLorentz(GaugeK, mu);
+    SigmaKPrime_mu = peekLorentz(SigmaKPrimeA, mu);
+    iQ = Ta(Cmu * adj(GaugeKmu));
+    this->set_iLambda(iLambda_mu, e_iQ, iQ, SigmaKPrime_mu, GaugeKmu);
+    pokeLorentz(SigmaK, SigmaKPrime_mu * e_iQ + adj(Cmu) * iLambda_mu, mu);
+    pokeLorentz(iLambda, iLambda_mu, mu);
+    BaseSmearDerivative(SigmaK, iLambda,GaugeK,mu,rho);  // derivative of SmearBase
+#else
+    //    GaugeField C(grid);
+    //    this->StoutSmearing->BaseSmear(C, GaugeK);
+    //    for (int mu = 0; mu < Nd; mu++)
+    int mu =mmu;
+    BaseSmear(Cmu, GaugeK,mu,rho);
     {
-      Cmu = peekLorentz(C, mu);
+      // Cmu = peekLorentz(C, mu);
       GaugeKmu = peekLorentz(GaugeK, mu);
       SigmaKPrime_mu = peekLorentz(SigmaKPrimeA, mu);
       iQ = Ta(Cmu * adj(GaugeKmu));
       this->set_iLambda(iLambda_mu, e_iQ, iQ, SigmaKPrime_mu, GaugeKmu);
       pokeLorentz(SigmaK, SigmaKPrime_mu * e_iQ + adj(Cmu) * iLambda_mu, mu);
       pokeLorentz(iLambda, iLambda_mu, mu);
+      std::cout << " mu "<<mu<<" SigmaKPrime_mu"<<norm2(SigmaKPrime_mu)<< " iLambda_mu " <<norm2(iLambda_mu)<<std::endl;
     }
-    this->StoutSmearing->derivative(SigmaK, iLambda,GaugeK);  // derivative of SmearBase
-
+    //    GaugeField SigmaKcopy(grid);
+    //    SigmaKcopy = SigmaK;
+    BaseSmearDerivative(SigmaK, iLambda,GaugeK,mu,rho);  // derivative of SmearBase
+    //    this->StoutSmearing->derivative(SigmaK, iLambda,GaugeK);  // derivative of SmearBase
+    //    SigmaKcopy = SigmaKcopy - SigmaK;
+    //    std::cout << " BaseSmearDerivative fast path error" <<norm2(SigmaKcopy)<<std::endl;
+#endif
     ////////////////////////////////////////////////////////////////////////////////////
     // propagate the rest of the force as identity map, just add back
     ////////////////////////////////////////////////////////////////////////////////////

Grid/qcd/smearing/StoutSmearing.h

@@ -69,7 +69,7 @@ public:
   /*! Construct stout smearing object from explicitly specified rho matrix */
   Smear_Stout(const std::vector<double>& rho_)
     : OwnedBase{new Smear_APE<Gimpl>(rho_)}, SmearBase{OwnedBase.get()} {
-    std::cout << GridLogDebug << "Stout smearing constructor : Smear_Stout(const std::vector<double>& " << rho_ << " )" << std::endl
+    std::cout << GridLogDebug << "Stout smearing constructor : Smear_Stout(const std::vector<double>& " << rho_ << " )" << std::endl;
     assert(Nc == 3 && "Stout smearing currently implemented only for Nc==3");
     }
 

Grid/qcd/utils/GaugeGroup.h

@@ -100,6 +100,9 @@ class GaugeGroup {
   using iGroupMatrix = iScalar<iScalar<iMatrix<vtype, ncolour> > >;
   template <typename vtype>
   using iAlgebraVector = iScalar<iScalar<iVector<vtype, AdjointDimension> > >;
+  template <typename vtype>
+  using iSUnAlgebraMatrix =
+    iScalar<iScalar<iMatrix<vtype, AdjointDimension> > >;
   static int su2subgroups(void) { return su2subgroups(group_name()); }
 
   //////////////////////////////////////////////////////////////////////////////////////////////////
@@ -128,10 +131,19 @@ class GaugeGroup {
   typedef Lattice<vMatrix> LatticeMatrix;
   typedef Lattice<vMatrixF> LatticeMatrixF;
   typedef Lattice<vMatrixD> LatticeMatrixD;
-
+  
   typedef Lattice<vAlgebraVector> LatticeAlgebraVector;
   typedef Lattice<vAlgebraVectorF> LatticeAlgebraVectorF;
   typedef Lattice<vAlgebraVectorD> LatticeAlgebraVectorD;
+   
+  typedef iSUnAlgebraMatrix<vComplex>  vAlgebraMatrix;
+  typedef iSUnAlgebraMatrix<vComplexF> vAlgebraMatrixF;
+  typedef iSUnAlgebraMatrix<vComplexD> vAlgebraMatrixD;
+
+  typedef Lattice<vAlgebraMatrix>  LatticeAlgebraMatrix;
+  typedef Lattice<vAlgebraMatrixF> LatticeAlgebraMatrixF;
+  typedef Lattice<vAlgebraMatrixD> LatticeAlgebraMatrixD;
+  
 
   typedef iSU2Matrix<Complex> SU2Matrix;
   typedef iSU2Matrix<ComplexF> SU2MatrixF;
@@ -160,7 +172,7 @@ class GaugeGroup {
     return generator(lieIndex, ta, group_name());
   }
 
-  static void su2SubGroupIndex(int &i1, int &i2, int su2_index) {
+  static accelerator_inline void su2SubGroupIndex(int &i1, int &i2, int su2_index) {
     return su2SubGroupIndex(i1, i2, su2_index, group_name());
   }
 
@@ -389,6 +401,52 @@ class GaugeGroup {
     }
   }
 
+// Ta are hermitian (?)
+// Anti herm is i Ta basis
+static void LieAlgebraProject(LatticeAlgebraMatrix &out,const LatticeMatrix &in, int b)
+{
+  conformable(in, out);
+  GridBase *grid = out.Grid();
+  LatticeComplex tmp(grid);
+  Matrix ta;
+  // Using Luchang's projection convention
+  //  2 Tr{Ta Tb} A_b= 2/2 delta ab A_b = A_a
+  autoView(out_v,out,AcceleratorWrite);
+  autoView(in_v,in,AcceleratorRead);
+  int N = ncolour;
+  int NNm1 = N * (N - 1);
+  int hNNm1= NNm1/2;
+  RealD sqrt_2 = sqrt(2.0);
+  Complex ci(0.0,1.0);
+  for(int su2Index=0;su2Index<hNNm1;su2Index++){
+    int i1, i2;
+    su2SubGroupIndex(i1, i2, su2Index);
+    int ax = su2Index*2;
+    int ay = su2Index*2+1;
+    accelerator_for(ss,grid->oSites(),1,{
+	// in is traceless ANTI-hermitian whereas Grid generators are Hermitian.
+	// trace( Ta x Ci in)
+	// Bet I need to move to real part with mult by -i
+	out_v[ss]()()(ax,b) = 0.5*(real(in_v[ss]()()(i2,i1)) - real(in_v[ss]()()(i1,i2)));
+	out_v[ss]()()(ay,b) = 0.5*(imag(in_v[ss]()()(i1,i2)) + imag(in_v[ss]()()(i2,i1)));
+      });
+  }
+  for(int diagIndex=0;diagIndex<N-1;diagIndex++){
+    int k = diagIndex + 1; // diagIndex starts from 0
+    int a = NNm1+diagIndex;
+    RealD scale = 1.0/sqrt(2.0*k*(k+1));
+    accelerator_for(ss,grid->oSites(),vComplex::Nsimd(),{
+	auto tmp = in_v[ss]()()(0,0);
+	for(int i=1;i<k;i++){
+	  tmp=tmp+in_v[ss]()()(i,i);
+	}
+	tmp = tmp - in_v[ss]()()(k,k)*k;
+	out_v[ss]()()(a,b) =imag(tmp) * scale;
+      });
+    }
+}
+
+  
 };
     
 template <int ncolour>

Grid/qcd/utils/SUn.impl.h

@@ -10,6 +10,7 @@
 // doesn't get found by the scripts/filelist during bootstrapping.
 
 private:
+
 template <ONLY_IF_SU>
 static int su2subgroups(GroupName::SU) { return (ncolour * (ncolour - 1)) / 2; }
 ////////////////////////////////////////////////////////////////////////
@@ -576,3 +577,4 @@ static void RandomGaugeTransform(GridParallelRNG &pRNG, typename Gimpl::GaugeFie
   LieRandomize(pRNG,g,1.0);
   GaugeTransform<Gimpl>(Umu,g);
 }
+

Grid/sitmo_rng/sitmo_prng_engine.hpp

@@ -218,6 +218,10 @@ public:
     // -------------------------------------------------
     // misc
     // -------------------------------------------------
+    void discardhi(uint64_t z) {
+      _s[3] += z;
+      encrypt_counter();
+    }
     
     // req: 26.5.1.4 Random number engine requirements, p.908 table 117, row 9
     // Advances e’s state ei to ei+z by any means equivalent to z
@@ -387,4 +391,4 @@ private:
 #undef MIXK
 #undef MIX2
 
-#endif
+#endif

Grid/tensors/Tensor_trace.h

@@ -69,6 +69,35 @@ accelerator_inline auto trace(const iVector<vtype,N> &arg) -> iVector<decltype(t
   }
   return ret;
 }
+////////////////////////////
+// Fast path traceProduct
+////////////////////////////
+template<class S1 , class S2, IfNotGridTensor<S1> = 0, IfNotGridTensor<S2> = 0>
+accelerator_inline auto traceProduct( const S1 &arg1,const S2 &arg2)
+  -> decltype(arg1*arg2)
+{
+  return arg1*arg2;
+}
+
+template<class vtype,class rtype,int N >
+accelerator_inline auto traceProduct(const iMatrix<vtype,N> &arg1,const iMatrix<rtype,N> &arg2) -> iScalar<decltype(trace(arg1._internal[0][0]*arg2._internal[0][0]))>
+{
+  iScalar<decltype( trace(arg1._internal[0][0]*arg2._internal[0][0] )) > ret;
+  zeroit(ret._internal);
+  for(int i=0;i<N;i++){
+  for(int j=0;j<N;j++){
+    ret._internal=ret._internal+traceProduct(arg1._internal[i][j],arg2._internal[j][i]);
+  }}
+  return ret;
+}
+
+template<class vtype,class rtype >
+accelerator_inline auto traceProduct(const iScalar<vtype> &arg1,const iScalar<rtype> &arg2) -> iScalar<decltype(trace(arg1._internal*arg2._internal))>
+{
+  iScalar<decltype(trace(arg1._internal*arg2._internal))> ret;
+  ret._internal=traceProduct(arg1._internal,arg2._internal);
+  return ret;
+}
 
 NAMESPACE_END(Grid);
 

Grid/tensors/Tensor_traits.h

@@ -34,9 +34,12 @@ NAMESPACE_BEGIN(Grid);
 
   // These are the Grid tensors
   template<typename T>     struct isGridTensor                : public std::false_type { static constexpr bool notvalue = true; };
-  template<class T>        struct isGridTensor<iScalar<T>>    : public std::true_type  { static constexpr bool notvalue = false; };
-  template<class T, int N> struct isGridTensor<iVector<T, N>> : public std::true_type  { static constexpr bool notvalue = false; };
-  template<class T, int N> struct isGridTensor<iMatrix<T, N>> : public std::true_type  { static constexpr bool notvalue = false; };
+  template<class T>        struct isGridTensor<iScalar<T> >   : public std::true_type  { static constexpr bool notvalue = false; };
+  template<class T, int N> struct isGridTensor<iVector<T, N> >: public std::true_type  { static constexpr bool notvalue = false; };
+  template<class T, int N> struct isGridTensor<iMatrix<T, N> >: public std::true_type  { static constexpr bool notvalue = false; };
+
+  template <typename T>  using IfGridTensor    = Invoke<std::enable_if<isGridTensor<T>::value, int> >;
+  template <typename T>  using IfNotGridTensor = Invoke<std::enable_if<!isGridTensor<T>::value, int> >;
 
   // Traits to identify scalars
   template<typename T>     struct isGridScalar                : public std::false_type { static constexpr bool notvalue = true; };

Grid/threads/Accelerator.cc

@@ -7,6 +7,8 @@ uint32_t accelerator_threads=2;
 uint32_t acceleratorThreads(void)       {return accelerator_threads;};
 void     acceleratorThreads(uint32_t t) {accelerator_threads = t;};
 
+#define ENV_LOCAL_RANK_PALS    "PALS_LOCAL_RANKID"
+#define ENV_RANK_PALS          "PALS_RANKID"
 #define ENV_LOCAL_RANK_OMPI    "OMPI_COMM_WORLD_LOCAL_RANK"
 #define ENV_RANK_OMPI          "OMPI_COMM_WORLD_RANK"
 #define ENV_LOCAL_RANK_SLURM   "SLURM_LOCALID"
@@ -14,7 +16,10 @@ void     acceleratorThreads(uint32_t t) {accelerator_threads = t;};
 #define ENV_LOCAL_RANK_MVAPICH "MV2_COMM_WORLD_LOCAL_RANK"
 #define ENV_RANK_MVAPICH       "MV2_COMM_WORLD_RANK"
 
-#ifdef GRID_CUDA
+
+// fold omptarget into device specific acceleratorInit()
+#if defined(GRID_CUDA) || (defined(GRID_OMPTARGET) && defined(__CUDA_ARCH__))
+#include <cuda_runtime_api.h>
 cudaDeviceProp *gpu_props;
 cudaStream_t copyStream;
 cudaStream_t computeStream;
@@ -113,7 +118,7 @@ void acceleratorInit(void)
 }
 #endif
 
-#ifdef GRID_HIP
+#if defined(GRID_HIP) || (defined(GRID_OMPTARGET) && defined(__HIP_DEVICE_COMPILE__))
 hipDeviceProp_t *gpu_props;
 hipStream_t copyStream;
 hipStream_t computeStream;
@@ -147,7 +152,7 @@ void acceleratorInit(void)
 #define GPU_PROP_FMT(canMapHostMemory,FMT)     printf("AcceleratorHipInit:   " #canMapHostMemory ": " FMT" \n",prop.canMapHostMemory);
 #define GPU_PROP(canMapHostMemory)             GPU_PROP_FMT(canMapHostMemory,"%d");
     
-    hipGetDeviceProperties(&gpu_props[i], i);
+    auto r=hipGetDeviceProperties(&gpu_props[i], i);
     hipDeviceProp_t prop; 
     prop = gpu_props[i];
     totalDeviceMem = prop.totalGlobalMem;
@@ -198,7 +203,7 @@ void acceleratorInit(void)
 #endif
 
 
-#ifdef GRID_SYCL
+#if defined(GRID_SYCL) //|| (defined(GRID_OMPTARGET) && defined(__SYCL_DEVICE_ONLY__))
 
 cl::sycl::queue *theGridAccelerator;
 cl::sycl::queue *theCopyAccelerator;
@@ -228,8 +233,17 @@ void acceleratorInit(void)
   {
     rank = atoi(localRankStr);		
   }
+  if ((localRankStr = getenv(ENV_LOCAL_RANK_PALS)) != NULL)
+  {
+    rank = atoi(localRankStr);		
+  }
   if ((localRankStr = getenv(ENV_RANK_OMPI   )) != NULL) { world_rank = atoi(localRankStr);}
   if ((localRankStr = getenv(ENV_RANK_MVAPICH)) != NULL) { world_rank = atoi(localRankStr);}
+  if ((localRankStr = getenv(ENV_RANK_PALS   )) != NULL) { world_rank = atoi(localRankStr);}
+
+  char hostname[HOST_NAME_MAX+1];
+  gethostname(hostname, HOST_NAME_MAX+1);
+  if ( rank==0 ) printf(" acceleratorInit world_rank %d is host %s \n",world_rank,hostname);
 
   auto devices = cl::sycl::device::get_devices();
   for(int d = 0;d<devices.size();d++){
@@ -241,9 +255,10 @@ void acceleratorInit(void)
     printf("AcceleratorSyclInit:   " #prop ": " FMT" \n",devices[d].get_info<cl::sycl::info::device::prop>());
 
 #define GPU_PROP(prop)             GPU_PROP_FMT(prop,"%ld");
+    if ( world_rank == 0) {
 
-    GPU_PROP_STR(vendor);
-    GPU_PROP_STR(version);
+      GPU_PROP_STR(vendor);
+      GPU_PROP_STR(version);
     //    GPU_PROP_STR(device_type);
     /*
     GPU_PROP(max_compute_units);
@@ -259,7 +274,8 @@ void acceleratorInit(void)
     GPU_PROP(single_fp_config);
     */
     //    GPU_PROP(double_fp_config);
-    GPU_PROP(global_mem_size);
+      GPU_PROP(global_mem_size);
+    }
 
   }
   if ( world_rank == 0 ) {
@@ -270,7 +286,7 @@ void acceleratorInit(void)
 }
 #endif
 
-#if (!defined(GRID_CUDA)) && (!defined(GRID_SYCL))&& (!defined(GRID_HIP))
+#if (!defined(GRID_CUDA)) && (!defined(GRID_SYCL))&& (!defined(GRID_HIP))// && (!defined(GRID_OMPTARGET))
 void acceleratorInit(void){}
 #endif
 

Grid/threads/Accelerator.h

@@ -26,8 +26,11 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
 *************************************************************************************/
 /*  END LEGAL */
-#pragma once
 
+#ifndef ACCELERATOR_H
+#define ACCELERATOR_H
+
+#pragma once
 #include <string.h>
 
 #ifdef HAVE_MALLOC_MALLOC_H
@@ -405,7 +408,7 @@ void LambdaApply(uint64_t numx, uint64_t numy, uint64_t numz, lambda Lambda)
 
 #define accelerator_barrier(dummy)				\
   {								\
-    hipStreamSynchronize(computeStream);			\
+    auto r=hipStreamSynchronize(computeStream);			\
     auto err = hipGetLastError();				\
     if ( err != hipSuccess ) {					\
       printf("After hipDeviceSynchronize() : HIP error %s \n", hipGetErrorString( err )); \
@@ -438,19 +441,19 @@ inline void *acceleratorAllocDevice(size_t bytes)
   return ptr;
 };
 
-inline void acceleratorFreeShared(void *ptr){ hipFree(ptr);};
-inline void acceleratorFreeDevice(void *ptr){ hipFree(ptr);};
-inline void acceleratorCopyToDevice(void *from,void *to,size_t bytes)  { hipMemcpy(to,from,bytes, hipMemcpyHostToDevice);}
-inline void acceleratorCopyFromDevice(void *from,void *to,size_t bytes){ hipMemcpy(to,from,bytes, hipMemcpyDeviceToHost);}
+inline void acceleratorFreeShared(void *ptr){ auto r=hipFree(ptr);};
+inline void acceleratorFreeDevice(void *ptr){ auto r=hipFree(ptr);};
+inline void acceleratorCopyToDevice(void *from,void *to,size_t bytes)  { auto r=hipMemcpy(to,from,bytes, hipMemcpyHostToDevice);}
+inline void acceleratorCopyFromDevice(void *from,void *to,size_t bytes){ auto r=hipMemcpy(to,from,bytes, hipMemcpyDeviceToHost);}
 //inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes)  { hipMemcpy(to,from,bytes, hipMemcpyDeviceToDevice);}
 //inline void acceleratorCopySynchronise(void) {  }
-inline void acceleratorMemSet(void *base,int value,size_t bytes) { hipMemset(base,value,bytes);}
+inline void acceleratorMemSet(void *base,int value,size_t bytes) { auto r=hipMemset(base,value,bytes);}
 
 inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes) // Asynch
 {
-  hipMemcpyDtoDAsync(to,from,bytes, copyStream);
+  auto r=hipMemcpyDtoDAsync(to,from,bytes, copyStream);
 }
-inline void acceleratorCopySynchronise(void) { hipStreamSynchronize(copyStream); };
+inline void acceleratorCopySynchronise(void) { auto r=hipStreamSynchronize(copyStream); };
 
 #endif
 
@@ -474,14 +477,155 @@ inline void acceleratorCopySynchronise(void) { hipStreamSynchronize(copyStream);
 #endif
 
 //////////////////////////////////////////////
-// CPU Target - No accelerator just thread instead
+// OpenMP Target acceleration
 //////////////////////////////////////////////
+#ifdef GRID_OMPTARGET
+//TODO GRID_SIMT for OMPTARGET
+#define GRID_ACCELERATED
+#include<omp.h>
+#ifdef __CUDA_ARCH__
+#include <cuda_runtime_api.h>
+#elif defined __HIP_DEVICE_COMPILE__
+#include <hip/hip_runtime.h>
+#elif defined __SYCL_DEVICE_ONLY__
+#include <CL/sycl.hpp>
+#include <CL/sycl/usm.hpp>
+#endif
+extern "C" void *llvm_omp_target_alloc_host  (size_t Size, int DeviceNum);
+extern "C" void *llvm_omp_target_alloc_device(size_t Size, int DeviceNum);
+extern "C" void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum);
+//TODO: Dynamic Shared Memory
 
-#if ( (!defined(GRID_SYCL)) && (!defined(GRID_CUDA)) && (!defined(GRID_HIP)) )
+#define THREAD_LIMIT acceleratorThreads()
 
-#undef GRID_SIMT
+#define accelerator
+#define accelerator_inline strong_inline
+#ifdef THREAD_LIMIT
+#define accelerator_for(i,num,nsimd, ... ) \
+	_Pragma("omp target teams distribute parallel for thread_limit(THREAD_LIMIT)") \
+	for ( uint64_t i=0;i<num;i++) { __VA_ARGS__ } ; 
+#define accelerator_forNB(i,num,nsimd, ... ) \
+	_Pragma("omp target teams distribute parallel for thread_limit(THREAD_LIMIT) nowait") \
+        for ( uint64_t i=0;i<num;i++) { __VA_ARGS__ } ;
+#define accelerator_barrier(dummy) _Pragma("omp barrier") 
+#define accelerator_for2d(iter1, num1, iter2, num2, nsimd, ... ) \
+	_Pragma("omp target teams distribute parallel for thread_limit(THREAD_LIMIT) collapse(2)") \
+        for ( uint64_t iter1=0;iter1<num1;iter1++) \
+	for ( uint64_t iter2=0;iter2<num2;iter2++) { __VA_ARGS__ } ;
+#else
+#define accelerator_for(i,num,nsimd, ... ) \
+        _Pragma("omp target teams distribute parallel for") \
+        for ( uint64_t i=0;i<num;i++) { __VA_ARGS__ } ;
+#define accelerator_forNB(i,num,nsimd, ... ) \
+        _Pragma("omp target teams distribute parallel for nowait") \
+        for ( uint64_t i=0;i<num;i++) { __VA_ARGS__ } ;
+#define accelerator_barrier(dummy) _Pragma("omp barrier")
+#define accelerator_for2d(iter1, num1, iter2, num2, nsimd, ... ) \
+        _Pragma("omp target teams distribute parallel for collapse(2)") \
+        for ( uint64_t iter1=0;iter1<num1;iter1++) \
+        for ( uint64_t iter2=0;iter2<num2;iter2++) { __VA_ARGS__ } ;
+#endif
 
+accelerator_inline int acceleratorSIMTlane(int Nsimd) { return 0; } // CUDA specific
+inline void acceleratorCopyToDevice(void *from,void *to,size_t bytes)
+{
+  int devc = omp_get_default_device();
+  int host = omp_get_initial_device();
+  if( omp_target_memcpy( to, from, bytes, 0, 0, devc, host ) ) {
+    printf(" omp_target_memcpy host to device failed for %ld in device %d \n",bytes,devc);
+  }
+};
+inline void acceleratorCopyFromDevice(void *from,void *to,size_t bytes)
+{
+  int devc = omp_get_default_device();
+  int host = omp_get_initial_device();
+  if( omp_target_memcpy( to, from, bytes, 0, 0, host, devc ) ) {
+    printf(" omp_target_memcpy device to host failed for %ld in device %d \n",bytes,devc);
+  }
+};
+inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes) 
+{ 
+#ifdef __CUDA_ARCH__
+  extern cudaStream_t copyStream;
+  cudaMemcpyAsync(to,from,bytes, cudaMemcpyDeviceToDevice,copyStream);
+#elif defined __HIP_DEVICE_COMPILE__
+  extern hipStream_t copyStream;
+  hipMemcpyDtoDAsync(to,from,bytes, copyStream);
+#elif defined __SYCL_DEVICE_ONLY__
+  theCopyAccelerator->memcpy(to,from,bytes);
+#endif
+};
+inline void acceleratorCopySynchronise(void) 
+{
+  //#pragma omp barrier
+#ifdef __CUDA_ARCH__
+  extern cudaStream_t copyStream;
+  cudaStreamSynchronize(copyStream);
+#elif defined __HIP_DEVICE_COMPILE__
+  extern hipStream_t copyStream;
+  hipStreamSynchronize(copyStream);
+#elif defined __SYCL_DEVICE_ONLY__
+  theCopyAccelerator->wait();
+#endif
+};
+inline int  acceleratorIsCommunicable(void *ptr){ return 1; }
+inline void acceleratorMemSet(void *base,int value,size_t bytes)
+{
+  void *base_host = memalign(GRID_ALLOC_ALIGN,bytes);
+  memset(base_host,value,bytes);
+  int devc = omp_get_default_device();
+  int host = omp_get_initial_device();
+  if( omp_target_memcpy( base, base_host, bytes, 0, 0, devc, host ) ) {
+    printf(" omp_target_memcpy device to host failed in MemSet for %ld in device %d \n",bytes,devc);
+  }
+};
+inline void *acceleratorAllocShared(size_t bytes)
+{
+#ifdef __CUDA_ARCH__
+  void *ptr=NULL;
+  auto err = cudaMallocManaged((void **)&ptr,bytes);
+  if( err != cudaSuccess ) {
+    ptr = (void *) NULL;
+    printf(" cudaMallocManaged failed for %d %s \n",bytes,cudaGetErrorString(err));
+  }
+  return ptr;
+#elif defined __HIP_DEVICE_COMPILE__
+  void *ptr=NULL;
+  auto err = hipMallocManaged((void **)&ptr,bytes);
+  if( err != hipSuccess ) {
+    ptr = (void *) NULL;
+    printf(" hipMallocManaged failed for %d %s \n",bytes,cudaGetErrorString(err));
+  }
+  return ptr;
+#elif defined __SYCL_DEVICE_ONLY__
+  queue q;
+  //void *ptr = malloc_shared<void *>(bytes, q);
+  return ptr;
+#else
+  int devc = omp_get_default_device();
+  void *ptr=NULL;
+  ptr = (void *) llvm_omp_target_alloc_shared(bytes, devc);
+  if( ptr == NULL ) {
+    printf(" llvm_omp_target_alloc_shared failed for %ld in device %d \n",bytes,devc);
+  }
+  return ptr;
+#endif
+};
+inline void *acceleratorAllocDevice(size_t bytes)
+{
+  int devc = omp_get_default_device();
+  void *ptr=NULL;
+  ptr = (void *) omp_target_alloc(bytes, devc);
+  if( ptr == NULL ) {
+    printf(" omp_target_alloc failed for %ld in device %d \n",bytes,devc);
+  }
+  return ptr;
+};
+inline void acceleratorFreeShared(void *ptr){omp_target_free(ptr, omp_get_default_device());};
+inline void acceleratorFreeDevice(void *ptr){omp_target_free(ptr, omp_get_default_device());};
 
+//OpenMP CPU threads
+#else
 
 #define accelerator 
 #define accelerator_inline strong_inline
@@ -510,7 +654,14 @@ inline void *acceleratorAllocDevice(size_t bytes){return memalign(GRID_ALLOC_ALI
 inline void acceleratorFreeShared(void *ptr){free(ptr);};
 inline void acceleratorFreeDevice(void *ptr){free(ptr);};
 #endif
+#endif
 
+//////////////////////////////////////////////
+// CPU Target - No accelerator just thread instead
+//////////////////////////////////////////////
+
+#if ( (!defined(GRID_SYCL)) && (!defined(GRID_CUDA)) && (!defined(GRID_HIP)) ) && (!defined(GRID_OMPTARGET))
+#undef GRID_SIMT
 #endif // CPU target
 
 #ifdef HAVE_MM_MALLOC_H
@@ -575,4 +726,13 @@ accelerator_inline void acceleratorFence(void)
   return;
 }
 
+inline void acceleratorCopyDeviceToDevice(void *from,void *to,size_t bytes)
+{
+  acceleratorCopyDeviceToDeviceAsynch(from,to,bytes);
+  acceleratorCopySynchronise();
+}
+
+
 NAMESPACE_END(Grid);
+#endif
+

Grid/threads/Threads.h

@@ -46,7 +46,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #endif
 
 #ifdef GRID_OMP
-#define DO_PRAGMA_(x) _Pragma (#x)
+#define DO_PRAGMA_(x) _Pragma ("x")
 #define DO_PRAGMA(x) DO_PRAGMA_(x)
 #define thread_num(a) omp_get_thread_num()
 #define thread_max(a) omp_get_max_threads()

HMC/FTHMC2p1f.cc

@@ -54,15 +54,16 @@ int main(int argc, char **argv)
   //  MD.name    = std::string("Force Gradient");
   typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;
   MD.name    = std::string("MinimumNorm2");
-  MD.MDsteps = 12;
+  MD.MDsteps = 24;
   MD.trajL   = 1.0;
 
   HMCparameters HMCparams;
-  HMCparams.StartTrajectory  = 0;
+  HMCparams.StartTrajectory  = 104;
   HMCparams.Trajectories     = 200;
   HMCparams.NoMetropolisUntil=  20;
   // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
-  HMCparams.StartingType     =std::string("HotStart");
+  //  HMCparams.StartingType     =std::string("HotStart");
+  HMCparams.StartingType     =std::string("CheckpointStart");
   HMCparams.MD = MD;
   HMCWrapper TheHMC(HMCparams);
 
@@ -87,6 +88,7 @@ int main(int argc, char **argv)
   // here there is too much indirection
   typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
   TheHMC.Resources.AddObservable<PlaqObs>();
+
   //////////////////////////////////////////////
 
   const int Ls      = 16;
@@ -134,7 +136,6 @@ int main(int argc, char **argv)
   ////////////////////////////////////
   ActionLevel<HMCWrapper::Field> Level1(1);
   ActionLevel<HMCWrapper::Field> Level2(2);
-  ActionLevel<HMCWrapper::Field> Level3(4);
 
   ////////////////////////////////////
   // Strange action
@@ -191,7 +192,7 @@ int main(int argc, char **argv)
   Smear_Stout<HMCWrapper::ImplPolicy> Stout(rho);
   SmearedConfigurationMasked<HMCWrapper::ImplPolicy> SmearingPolicy(GridPtr, Nstep, Stout);
   JacobianAction<HMCWrapper::ImplPolicy> Jacobian(&SmearingPolicy);
-  if( ApplySmearing ) Level2.push_back(&Jacobian);
+  if( ApplySmearing ) Level1.push_back(&Jacobian);
   std::cout << GridLogMessage << " Built the Jacobian "<< std::endl;
 
 
@@ -200,7 +201,7 @@ int main(int argc, char **argv)
   /////////////////////////////////////////////////////////////
   //  GaugeAction.is_smeared = ApplySmearing;
   GaugeAction.is_smeared = true;
-  Level3.push_back(&GaugeAction);
+  Level2.push_back(&GaugeAction);
 
   std::cout << GridLogMessage << " ************************************************"<< std::endl;
   std::cout << GridLogMessage << " Action complete -- NO FERMIONS FOR NOW -- FIXME"<< std::endl;
@@ -210,10 +211,11 @@ int main(int argc, char **argv)
 
 
   std::cout << GridLogMessage << " Running the FT HMC "<< std::endl;
-
   TheHMC.TheAction.push_back(Level1);
   TheHMC.TheAction.push_back(Level2);
-  TheHMC.TheAction.push_back(Level3);
+
+  TheHMC.ReadCommandLine(argc,argv);  // params on CML or from param file
+  TheHMC.initializeGaugeFieldAndRNGs(U);
 
   TheHMC.Run(SmearingPolicy); // for smearing
 

HMC/FTHMC2p1f_3GeV.cc

@@ -0,0 +1,226 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Copyright (C) 2023
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/qcd/smearing/GaugeConfigurationMasked.h>
+#include <Grid/qcd/smearing/JacobianAction.h>
+
+using namespace Grid;
+
+int main(int argc, char **argv)
+{
+  std::cout << std::setprecision(12);
+  
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef WilsonImplR FermionImplPolicy;
+  typedef MobiusFermionD FermionAction;
+  typedef typename FermionAction::FermionField FermionField;
+
+  typedef Grid::XmlReader       Serialiser;
+
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper;
+  //  MD.name    = std::string("Leap Frog");
+  //  typedef GenericHMCRunner<ForceGradient> HMCWrapper;
+  //  MD.name    = std::string("Force Gradient");
+  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;
+  MD.name    = std::string("MinimumNorm2");
+  MD.MDsteps = 24;
+  MD.trajL   = 1.0;
+
+  HMCparameters HMCparams;
+  HMCparams.StartTrajectory  = 0;
+  HMCparams.Trajectories     = 200;
+  HMCparams.NoMetropolisUntil=  20;
+  // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+  //  HMCparams.StartingType     =std::string("HotStart");
+  HMCparams.StartingType     =std::string("ColdStart");
+  //  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.MD = MD;
+  HMCWrapper TheHMC(HMCparams);
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_EODWF_lat";
+  CPparams.smeared_prefix = "ckpoint_EODWF_lat_smr";
+  CPparams.rng_prefix    = "ckpoint_EODWF_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.saveSmeared   = true;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+
+  //////////////////////////////////////////////
+
+  const int Ls      = 12;
+  Real beta         = 2.37;
+  Real light_mass   = 0.0047;
+  Real strange_mass = 0.0186;
+  Real pv_mass      = 1.0;
+  RealD M5  = 1.8;
+  RealD b   = 1.0; // Scale factor one, Shamir
+  RealD c   = 0.0;
+
+  OneFlavourRationalParams OFRp;
+  OFRp.lo       = 1.0e-2;
+  OFRp.hi       = 64;
+  OFRp.MaxIter  = 10000;
+  OFRp.tolerance= 1.0e-10;
+  OFRp.degree   = 14;
+  OFRp.precision= 40;
+
+  std::vector<Real> hasenbusch({ 0.05, 0.1, 0.25, 0.5 });
+
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+  LatticeGaugeField Uhot(GridPtr);
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+
+  double StoppingCondition = 1e-10;
+  double MaxCGIterations = 30000;
+  ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
+
+  bool ApplySmearing = true;
+  
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(2);
+
+  ////////////////////////////////////
+  // Strange action
+  ////////////////////////////////////
+
+  MobiusEOFAFermionD Strange_Op_L (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionD Strange_Op_R (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 CG,
+	 CG, CG,
+	 CG, CG, 
+	 OFRp, false);
+
+  EOFA.is_smeared = ApplySmearing;
+  Level1.push_back(&EOFA);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass);
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]);
+    light_num.push_back(hasenbusch[h]);
+  }
+  light_num.push_back(pv_mass);
+
+  std::vector<FermionAction *> Numerators;
+  std::vector<FermionAction *> Denominators;
+  std::vector<TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> *> Quotients;
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    std::cout << GridLogMessage << " 2f quotient Action  "<< light_num[h] << " / " << light_den[h]<< std::endl;
+    Numerators.push_back  (new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, Params));
+    Denominators.push_back(new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, Params));
+    Quotients.push_back   (new TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],CG,CG));
+  }
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    Quotients[h]->is_smeared = ApplySmearing;
+    Level1.push_back(Quotients[h]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // lnDetJacobianAction
+  /////////////////////////////////////////////////////////////
+  double rho = 0.1;  // smearing parameter
+  int Nsmear = 1;    // number of smearing levels - must be multiple of 2Nd
+  int Nstep  = 8*Nsmear;    // number of smearing levels - must be multiple of 2Nd
+  Smear_Stout<HMCWrapper::ImplPolicy> Stout(rho);
+  SmearedConfigurationMasked<HMCWrapper::ImplPolicy> SmearingPolicy(GridPtr, Nstep, Stout);
+  JacobianAction<HMCWrapper::ImplPolicy> Jacobian(&SmearingPolicy);
+  if( ApplySmearing ) Level1.push_back(&Jacobian);
+  std::cout << GridLogMessage << " Built the Jacobian "<< std::endl;
+
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  GaugeAction.is_smeared = ApplySmearing;
+  Level2.push_back(&GaugeAction);
+
+  std::cout << GridLogMessage << " ************************************************"<< std::endl;
+  std::cout << GridLogMessage << " Action complete -- NO FERMIONS FOR NOW -- FIXME"<< std::endl;
+  std::cout << GridLogMessage << " ************************************************"<< std::endl;
+  std::cout << GridLogMessage <<  std::endl;
+  std::cout << GridLogMessage <<  std::endl;
+
+
+  std::cout << GridLogMessage << " Running the FT HMC "<< std::endl;
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+
+  TheHMC.ReadCommandLine(argc,argv);  // params on CML or from param file
+  TheHMC.initializeGaugeFieldAndRNGs(U);
+
+  TheHMC.Run(SmearingPolicy); // for smearing
+
+  Grid_finalize();
+} // main
+
+
+

HMC/HMC2p1f_3GeV.cc

@@ -0,0 +1,226 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Copyright (C) 2023
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/qcd/smearing/GaugeConfigurationMasked.h>
+#include <Grid/qcd/smearing/JacobianAction.h>
+
+using namespace Grid;
+
+int main(int argc, char **argv)
+{
+  std::cout << std::setprecision(12);
+  
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef WilsonImplR FermionImplPolicy;
+  typedef MobiusFermionD FermionAction;
+  typedef typename FermionAction::FermionField FermionField;
+
+  typedef Grid::XmlReader       Serialiser;
+
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper;
+  //  MD.name    = std::string("Leap Frog");
+  //  typedef GenericHMCRunner<ForceGradient> HMCWrapper;
+  //  MD.name    = std::string("Force Gradient");
+  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;
+  MD.name    = std::string("MinimumNorm2");
+  MD.MDsteps = 24;
+  MD.trajL   = 1.0;
+
+  HMCparameters HMCparams;
+  HMCparams.StartTrajectory  = 0;
+  HMCparams.Trajectories     = 200;
+  HMCparams.NoMetropolisUntil=  20;
+  // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+  //  HMCparams.StartingType     =std::string("HotStart");
+  HMCparams.StartingType     =std::string("ColdStart");
+  //  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.MD = MD;
+  HMCWrapper TheHMC(HMCparams);
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_EODWF_lat";
+  CPparams.smeared_prefix = "ckpoint_EODWF_lat_smr";
+  CPparams.rng_prefix    = "ckpoint_EODWF_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.saveSmeared   = true;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+
+  //////////////////////////////////////////////
+
+  const int Ls      = 12;
+  Real beta         = 2.37;
+  Real light_mass   = 0.0047;
+  Real strange_mass = 0.0186;
+  Real pv_mass      = 1.0;
+  RealD M5  = 1.8;
+  RealD b   = 1.0; // Scale factor one, Shamir
+  RealD c   = 0.0;
+
+  OneFlavourRationalParams OFRp;
+  OFRp.lo       = 1.0e-2;
+  OFRp.hi       = 64;
+  OFRp.MaxIter  = 10000;
+  OFRp.tolerance= 1.0e-10;
+  OFRp.degree   = 14;
+  OFRp.precision= 40;
+
+  std::vector<Real> hasenbusch({ 0.05, 0.1, 0.25, 0.5 });
+
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+  LatticeGaugeField Uhot(GridPtr);
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+
+  double StoppingCondition = 1e-10;
+  double MaxCGIterations = 30000;
+  ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
+
+  bool ApplySmearing = false;
+  
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(2);
+
+  ////////////////////////////////////
+  // Strange action
+  ////////////////////////////////////
+
+  MobiusEOFAFermionD Strange_Op_L (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionD Strange_Op_R (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 CG,
+	 CG, CG,
+	 CG, CG, 
+	 OFRp, false);
+
+  EOFA.is_smeared = ApplySmearing;
+  Level1.push_back(&EOFA);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass);
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]);
+    light_num.push_back(hasenbusch[h]);
+  }
+  light_num.push_back(pv_mass);
+
+  std::vector<FermionAction *> Numerators;
+  std::vector<FermionAction *> Denominators;
+  std::vector<TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> *> Quotients;
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    std::cout << GridLogMessage << " 2f quotient Action  "<< light_num[h] << " / " << light_den[h]<< std::endl;
+    Numerators.push_back  (new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, Params));
+    Denominators.push_back(new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, Params));
+    Quotients.push_back   (new TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],CG,CG));
+  }
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    Quotients[h]->is_smeared = ApplySmearing;
+    Level1.push_back(Quotients[h]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // lnDetJacobianAction
+  /////////////////////////////////////////////////////////////
+  double rho = 0.1;  // smearing parameter
+  int Nsmear = 1;    // number of smearing levels - must be multiple of 2Nd
+  int Nstep  = 8*Nsmear;    // number of smearing levels - must be multiple of 2Nd
+  Smear_Stout<HMCWrapper::ImplPolicy> Stout(rho);
+  SmearedConfigurationMasked<HMCWrapper::ImplPolicy> SmearingPolicy(GridPtr, Nstep, Stout);
+  JacobianAction<HMCWrapper::ImplPolicy> Jacobian(&SmearingPolicy);
+  if( ApplySmearing ) Level1.push_back(&Jacobian);
+  std::cout << GridLogMessage << " Built the Jacobian "<< std::endl;
+
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  GaugeAction.is_smeared = ApplySmearing;
+  Level2.push_back(&GaugeAction);
+
+  std::cout << GridLogMessage << " ************************************************"<< std::endl;
+  std::cout << GridLogMessage << " Action complete -- NO FERMIONS FOR NOW -- FIXME"<< std::endl;
+  std::cout << GridLogMessage << " ************************************************"<< std::endl;
+  std::cout << GridLogMessage <<  std::endl;
+  std::cout << GridLogMessage <<  std::endl;
+
+
+  std::cout << GridLogMessage << " Running the FT HMC "<< std::endl;
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+
+  TheHMC.ReadCommandLine(argc,argv);  // params on CML or from param file
+  TheHMC.initializeGaugeFieldAndRNGs(U);
+
+  TheHMC.Run(SmearingPolicy); // for smearing
+
+  Grid_finalize();
+} // main
+
+
+

HMC/Mobius2p1f_DD_EOFA_96I_double.cc

@@ -0,0 +1,350 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_EODWFRatio.cc
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+int main(int argc, char **argv) {
+  using namespace Grid;
+
+  Grid_init(&argc, &argv);
+
+  CartesianCommunicator::BarrierWorld();
+  std::cout << GridLogMessage << " Clock skew check" <<std::endl;
+  
+  int threads = GridThread::GetThreads();
+
+   // Typedefs to simplify notation
+  typedef WilsonImplD FermionImplPolicy;
+  typedef MobiusFermionD FermionAction;
+  typedef MobiusEOFAFermionD FermionEOFAAction;
+  typedef typename FermionAction::FermionField FermionField;
+
+  typedef Grid::XmlReader       Serialiser;
+
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper;
+  //  MD.name    = std::string("Leap Frog");
+  typedef GenericHMCRunner<ForceGradient> HMCWrapper;
+  MD.name    = std::string("Force Gradient");
+  //typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;
+  // MD.name    = std::string("MinimumNorm2");
+  // TrajL = 2
+  // 4/2 => 0.6 dH
+  // 3/3 => 0.8 dH .. depth 3, slower
+  //MD.MDsteps =  4;
+  MD.MDsteps =  3;
+  MD.trajL   = 0.5;
+
+  HMCparameters HMCparams;
+  HMCparams.StartTrajectory  = 1077;
+  HMCparams.Trajectories     = 1;
+  HMCparams.NoMetropolisUntil=  0;
+  // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+  //  HMCparams.StartingType     =std::string("ColdStart");
+  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.MD = MD;
+  HMCWrapper TheHMC(HMCparams);
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_DDHMC_lat";
+  CPparams.rng_prefix    = "ckpoint_DDHMC_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+  std::cout << "loaded NERSC checpointer"<<std::endl;
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  const int Ls      = 12;
+  RealD M5  = 1.8;
+  RealD b   = 1.5;
+  RealD c   = 0.5;
+  Real beta         = 2.13;
+  //  Real light_mass   = 5.4e-4;
+  Real light_mass     = 7.8e-4;
+  Real light_mass_dir = 0.01;
+  Real strange_mass = 0.0362;
+  Real pv_mass      = 1.0;
+  std::vector<Real> hasenbusch({ 0.01, 0.045, 0.108, 0.25, 0.51 , pv_mass });
+  //  std::vector<Real> hasenbusch({ light_mass, 0.01, 0.045, 0.108, 0.25, 0.51 , pv_mass });
+  //  std::vector<Real> hasenbusch({ light_mass, 0.005, 0.0145, 0.045, 0.108, 0.25, 0.51 , pv_mass }); // Updated
+  //  std::vector<Real> hasenbusch({ light_mass, 0.0145, 0.045, 0.108, 0.25, 0.51 , 0.75 , pv_mass });
+
+  int SP_iters=9000;
+  
+  RationalActionParams OFRp; // Up/down
+  OFRp.lo       = 6.0e-5;
+  OFRp.hi       = 90.0;
+  OFRp.inv_pow  = 2;
+  OFRp.MaxIter  = SP_iters; // get most shifts by 2000, stop sharing space
+  OFRp.action_tolerance= 1.0e-8;
+  OFRp.action_degree   = 18;
+  OFRp.md_tolerance= 1.0e-7;
+  OFRp.md_degree   = 14;
+  //  OFRp.degree   = 20; converges
+  //  OFRp.degree   = 16;
+  OFRp.precision= 80;
+  OFRp.BoundsCheckFreq=0;
+  std::vector<RealD> ActionTolByPole({
+      //      1.0e-8,1.0e-8,1.0e-8,1.0e-8,
+      3.0e-7,1.0e-7,1.0e-8,1.0e-8,
+      1.0e-8,1.0e-8,1.0e-8,1.0e-8,
+      1.0e-8,1.0e-8,1.0e-8,1.0e-8,
+      1.0e-8,1.0e-8,1.0e-8,1.0e-8,
+      1.0e-8,1.0e-8
+    });
+  std::vector<RealD> MDTolByPole({
+      //      1.6e-5,5.0e-6,1.0e-6,3.0e-7, // soften convergence more more
+      //      1.0e-6,3.0e-7,1.0e-7,1.0e-7,
+      1.0e-5,1.0e-6,1.0e-7,1.0e-7, // soften convergence
+      1.0e-8,1.0e-8,1.0e-8,1.0e-8,
+      1.0e-8,1.0e-8,1.0e-8,1.0e-8,
+      1.0e-8,1.0e-8
+    });
+
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+
+  typedef SchurDiagMooeeOperator<FermionAction ,FermionField > LinearOperatorD;
+  typedef SchurDiagMooeeOperator<FermionEOFAAction ,FermionField > LinearOperatorEOFAD;
+
+  ////////////////////////////////////////////////////////////////
+  // Domain decomposed
+  ////////////////////////////////////////////////////////////////
+  Coordinate latt4  = GridPtr->GlobalDimensions();
+  Coordinate mpi    = GridPtr->ProcessorGrid();
+  Coordinate shm;
+
+  GlobalSharedMemory::GetShmDims(mpi,shm);
+  
+  Coordinate CommDim(Nd);
+  for(int d=0;d<Nd;d++) CommDim[d]= (mpi[d]/shm[d])>1 ? 1 : 0;
+
+  Coordinate NonDirichlet(Nd+1,0);
+  Coordinate Dirichlet(Nd+1,0);
+  Dirichlet[1] = CommDim[0]*latt4[0]/mpi[0] * shm[0];
+  Dirichlet[2] = CommDim[1]*latt4[1]/mpi[1] * shm[1];
+  Dirichlet[3] = CommDim[2]*latt4[2]/mpi[2] * shm[2];
+  Dirichlet[4] = CommDim[3]*latt4[3]/mpi[3] * shm[3];
+  //Dirichlet[1] = 0;
+  //Dirichlet[2] = 0;
+  //Dirichlet[3] = 0;
+
+  // 
+  Coordinate Block4(Nd);
+  Block4[0] = Dirichlet[1];
+  Block4[1] = Dirichlet[2];
+  Block4[2] = Dirichlet[3];
+  Block4[3] = Dirichlet[4];
+
+  int Width=4;
+  TheHMC.Resources.SetMomentumFilter(new DDHMCFilter<WilsonImplD::Field>(Block4,Width));
+
+  //////////////////////////
+  // Fermion Grids
+  //////////////////////////
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeFieldD  U(GridPtr); U=Zero();
+
+  std::cout << GridLogMessage << " Running the HMC "<< std::endl;
+  TheHMC.ReadCommandLine(argc,argv);  // params on CML or from param file
+  TheHMC.initializeGaugeFieldAndRNGs(U);
+  std::cout << "loaded NERSC gauge field"<<std::endl;
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+  FermionAction::ImplParams ParamsDir(boundary);
+
+  Params.dirichlet=NonDirichlet;
+  ParamsDir.dirichlet=Dirichlet;
+  ParamsDir.partialDirichlet=0;
+  std::cout << GridLogMessage<< "Partial Dirichlet depth is "<<dwf_compressor_depth<<std::endl;
+
+  //  double StoppingCondition = 1e-14;
+  //  double MDStoppingCondition = 1e-9;
+  double StoppingCondition = 1e-8;
+  double MDStoppingCondition = 1e-8;
+  double MDStoppingConditionLoose = 1e-8;
+  double MDStoppingConditionStrange = 1e-8;
+  double MaxCGIterations = 300000;
+  ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
+  ConjugateGradient<FermionField>  MDCG(MDStoppingCondition,MaxCGIterations);
+
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(3);
+  ActionLevel<HMCWrapper::Field> Level3(15);
+
+  ////////////////////////////////////
+  // Strange action
+  ////////////////////////////////////
+  FermionAction StrangeOp (U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,strange_mass,M5,b,c, Params);
+  FermionAction StrangePauliVillarsOp(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,pv_mass,  M5,b,c, Params);
+
+  // Probably dominates the force - back to EOFA.
+  OneFlavourRationalParams SFRp;
+  SFRp.lo       = 0.1;
+  SFRp.hi       = 25.0;
+  SFRp.MaxIter  = 10000;
+  SFRp.tolerance= 1.0e-8;
+  SFRp.mdtolerance= 2.0e-6;
+  SFRp.degree   = 12;
+  SFRp.precision= 50;
+  
+  MobiusEOFAFermionD Strange_Op_L (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionD Strange_Op_R (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+  ConjugateGradient<FermionField>      ActionCG(StoppingCondition,MaxCGIterations);
+  ConjugateGradient<FermionField>  DerivativeCG(MDStoppingCondition,MaxCGIterations);
+  LinearOperatorEOFAD Strange_LinOp_L (Strange_Op_L);
+  LinearOperatorEOFAD Strange_LinOp_R (Strange_Op_R);
+
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 ActionCG, 
+	 ActionCG, ActionCG,
+	 DerivativeCG, DerivativeCG,
+	 SFRp, true);
+  Level2.push_back(&EOFA);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+  std::vector<int> dirichlet_den;
+  std::vector<int> dirichlet_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass);  dirichlet_den.push_back(0);
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]); dirichlet_den.push_back(1);
+  }
+
+  for(int h=0;h<n_hasenbusch;h++){
+    light_num.push_back(hasenbusch[h]); dirichlet_num.push_back(1);
+  }
+  light_num.push_back(pv_mass);  dirichlet_num.push_back(0);
+
+  std::vector<FermionAction *> Numerators;
+  std::vector<FermionAction *> Denominators;
+  std::vector<TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> *> Quotients;
+  
+  std::vector<GeneralEvenOddRatioRationalPseudoFermionAction<FermionImplPolicy> *> Bdys;
+
+  typedef SchurDiagMooeeOperator<FermionAction ,FermionField > LinearOperatorD;
+  std::vector<LinearOperatorD *> LinOpD;
+  
+  for(int h=0;h<n_hasenbusch+1;h++){
+    std::cout << GridLogMessage
+	      << " 2f quotient Action ";
+    std::cout << "det D("<<light_den[h]<<")";
+    if ( dirichlet_den[h] ) std::cout << "^dirichlet    ";
+    std::cout << "/ det D("<<light_num[h]<<")";
+    if ( dirichlet_num[h] ) std::cout << "^dirichlet    ";
+    std::cout << std::endl;
+
+    FermionAction::ImplParams ParamsNum(boundary);
+    FermionAction::ImplParams ParamsDen(boundary);
+    
+    if ( dirichlet_num[h]==1) ParamsNum.dirichlet = Dirichlet;
+    else                      ParamsNum.dirichlet = NonDirichlet;
+
+    if ( dirichlet_den[h]==1) ParamsDen.dirichlet = Dirichlet;
+    else                      ParamsDen.dirichlet = NonDirichlet;
+
+    if ( dirichlet_num[h]==1) ParamsNum.partialDirichlet = 1;
+    else                      ParamsNum.partialDirichlet = 0;
+
+    if ( dirichlet_den[h]==1) ParamsDen.partialDirichlet = 1;
+    else                      ParamsDen.partialDirichlet = 0;
+    
+    Numerators.push_back  (new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, ParamsNum));
+    Denominators.push_back(new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, ParamsDen));
+
+    LinOpD.push_back(new LinearOperatorD(*Denominators[h]));
+
+    double conv  = MDStoppingCondition;
+    if (h<3) conv= MDStoppingConditionLoose; // Relax on first two hasenbusch factors
+    
+    if(h!=0) {
+      Quotients.push_back (new TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],MDCG,CG));
+    } else {
+      Bdys.push_back( new GeneralEvenOddRatioRationalPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],OFRp));
+      Bdys.push_back( new GeneralEvenOddRatioRationalPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],OFRp));
+    }
+  }
+  for(int h=0;h<Bdys.size();h++){
+    Bdys[h]->SetTolerances(ActionTolByPole,MDTolByPole);
+  }
+  int nquo=Quotients.size();
+  Level1.push_back(Bdys[0]);
+  Level1.push_back(Bdys[1]);
+  Level2.push_back(Quotients[0]);
+  for(int h=1;h<nquo-1;h++){
+    Level2.push_back(Quotients[h]);
+  }
+  Level2.push_back(Quotients[nquo-1]);
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  Level3.push_back(&GaugeAction);
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+  TheHMC.TheAction.push_back(Level3);
+  std::cout << GridLogMessage << " Action complete "<< std::endl;
+  /////////////////////////////////////////////////////////////
+
+  TheHMC.Run();  // no smearing
+
+  Grid_finalize();
+} // main
+
+
+

HMC/Mobius2p1f_DD_EOFA_96I_mixed.cc

@@ -343,7 +343,7 @@ int main(int argc, char **argv) {
   // Probably dominates the force - back to EOFA.
   OneFlavourRationalParams SFRp;
   SFRp.lo       = 0.1;
-  SFRp.hi       = 25.0;
+  SFRp.hi       = 30.0;
   SFRp.MaxIter  = 10000;
   SFRp.tolerance= 1.0e-5;
   SFRp.mdtolerance= 2.0e-4;

HMC/Mobius2p1f_EOFA_96I_hmc.cc

@@ -128,7 +128,7 @@ template<class FermionOperatorD, class FermionOperatorF, class SchurOperatorD, c
       ////////////////////////////////////////////////////////////////////////////////////
       // Make a mixed precision conjugate gradient
       ////////////////////////////////////////////////////////////////////////////////////
-#if 1
+#if 0
       RealD delta=1.e-4;
       std::cout << GridLogMessage << "Calling reliable update Conjugate Gradient" <<std::endl;
       ConjugateGradientReliableUpdate<FieldD,FieldF> MPCG(Tolerance,MaxInnerIterations*MaxOuterIterations,delta,SinglePrecGrid5,LinOpF,LinOpD);
@@ -180,7 +180,7 @@ int main(int argc, char **argv) {
   // 4/2 => 0.6 dH
   // 3/3 => 0.8 dH .. depth 3, slower
   //MD.MDsteps =  4;
-  MD.MDsteps =  14;
+  MD.MDsteps =  12;
   MD.trajL   = 0.5;
 
   HMCparameters HMCparams;
@@ -204,7 +204,7 @@ int main(int argc, char **argv) {
   TheHMC.Resources.LoadNerscCheckpointer(CPparams);
   std::cout << "loaded NERSC checpointer"<<std::endl;
   RNGModuleParameters RNGpar;
-  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.serial_seeds = "1 2 3 4 5 6 7 8 9 10";
   RNGpar.parallel_seeds = "6 7 8 9 10";
   TheHMC.Resources.SetRNGSeeds(RNGpar);
 
@@ -218,15 +218,14 @@ int main(int argc, char **argv) {
   RealD M5  = 1.8;
   RealD b   = 1.5;
   RealD c   = 0.5;
-  Real beta         = 2.13;
+  RealD beta         = 2.13;
   //  Real light_mass   = 5.4e-4;
   Real light_mass     = 7.8e-4;
+  //  Real light_mass     = 7.8e-3;
   Real strange_mass = 0.0362;
   Real pv_mass      = 1.0;
-  //  std::vector<Real> hasenbusch({ 0.01, 0.045, 0.108, 0.25, 0.51 , pv_mass });
-  //  std::vector<Real> hasenbusch({ light_mass, 0.01, 0.045, 0.108, 0.25, 0.51 , pv_mass });
-  std::vector<Real> hasenbusch({ 0.005, 0.0145, 0.045, 0.108, 0.25, 0.51 }); // Updated
-  //  std::vector<Real> hasenbusch({ light_mass, 0.0145, 0.045, 0.108, 0.25, 0.51 , 0.75 , pv_mass });
+  std::vector<Real> hasenbusch({ 0.005, 0.0145, 0.045, 0.108, 0.25, 0.35 , 0.51, 0.6, 0.8 }); // Updated
+  //std::vector<Real> hasenbusch({ 0.0145, 0.045, 0.108, 0.25, 0.35 , 0.51, 0.6, 0.8 }); // Updated
 
   auto GridPtr   = TheHMC.Resources.GetCartesian();
   auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
@@ -277,20 +276,20 @@ int main(int argc, char **argv) {
 
   //  double StoppingCondition = 1e-14;
   //  double MDStoppingCondition = 1e-9;
-  double StoppingCondition = 1e-9;
-  double MDStoppingCondition = 1e-8;
-  double MDStoppingConditionLoose = 1e-8;
-  double MDStoppingConditionStrange = 1e-8;
-  double MaxCGIterations = 300000;
+  double StoppingCondition = 1e-14;
+  double MDStoppingCondition = 1e-9;
+  double MDStoppingConditionLoose = 1e-9;
+  double MDStoppingConditionStrange = 1e-9;
+  double MaxCGIterations = 50000;
   ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
   ConjugateGradient<FermionField>  MDCG(MDStoppingCondition,MaxCGIterations);
 
   ////////////////////////////////////
   // Collect actions
   ////////////////////////////////////
-  //  ActionLevel<HMCWrapper::Field> Level1(1);
-  ActionLevel<HMCWrapper::Field> Level2(1);
-  ActionLevel<HMCWrapper::Field> Level3(15);
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(2);
+  ActionLevel<HMCWrapper::Field> Level3(4);
 
   ////////////////////////////////////
   // Strange action
@@ -300,11 +299,11 @@ int main(int argc, char **argv) {
 
   // Probably dominates the force - back to EOFA.
   OneFlavourRationalParams SFRp;
-  SFRp.lo       = 0.1;
+  SFRp.lo       = 0.8;
   SFRp.hi       = 30.0;
   SFRp.MaxIter  = 10000;
-  SFRp.tolerance= 1.0e-8;
-  SFRp.mdtolerance= 2.0e-6;
+  SFRp.tolerance= 1.0e-12;
+  SFRp.mdtolerance= 1.0e-9;
   SFRp.degree   = 10;
   SFRp.precision= 50;
   
@@ -355,8 +354,10 @@ int main(int argc, char **argv) {
   ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
     EOFA(Strange_Op_L, Strange_Op_R, 
 	 ActionCG, 
-	 ActionCGL, ActionCGR,
-	 DerivativeCGL, DerivativeCGR,
+	 //	 ActionCGL, ActionCGR,
+	 //	 DerivativeCGL, DerivativeCGR,
+	 ActionCG, ActionCG,
+	 DerivativeCG, DerivativeCG,
 	 SFRp, true);
   Level2.push_back(&EOFA);
 
@@ -443,13 +444,14 @@ int main(int argc, char **argv) {
   }
   int nquo=Quotients.size();
   for(int h=0;h<nquo;h++){
-    Level2.push_back(Quotients[h]);
+    Level1.push_back(Quotients[h]);
   }
 
   /////////////////////////////////////////////////////////////
   // Gauge action
   /////////////////////////////////////////////////////////////
   Level3.push_back(&GaugeAction);
+  TheHMC.TheAction.push_back(Level1);
   TheHMC.TheAction.push_back(Level2);
   TheHMC.TheAction.push_back(Level3);
   std::cout << GridLogMessage << " Action complete "<< std::endl;

HMC/Mobius2p1f_EOFA_96I_hmc_double.cc

@@ -0,0 +1,268 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_EODWFRatio.cc
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+
+
+int main(int argc, char **argv) {
+  using namespace Grid;
+
+  std::cout << " Grid Initialise "<<std::endl;
+  
+  Grid_init(&argc, &argv);
+
+  CartesianCommunicator::BarrierWorld();
+  std::cout << GridLogMessage << " Clock skew check" <<std::endl;
+  
+  int threads = GridThread::GetThreads();
+
+   // Typedefs to simplify notation
+  typedef WilsonImplD FermionImplPolicy;
+  typedef MobiusFermionD FermionAction;
+  typedef MobiusEOFAFermionD FermionEOFAAction;
+  typedef typename FermionAction::FermionField FermionField;
+
+  typedef WilsonImplF FermionImplPolicyF;
+  typedef MobiusFermionF FermionActionF;
+  typedef MobiusEOFAFermionF FermionEOFAActionF;
+  typedef typename FermionActionF::FermionField FermionFieldF;
+
+  typedef Grid::XmlReader       Serialiser;
+
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper;
+  //  MD.name    = std::string("Leap Frog");
+  typedef GenericHMCRunner<ForceGradient> HMCWrapper;
+  MD.name    = std::string("Force Gradient");
+  //  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;
+  //  MD.name    = std::string("MinimumNorm2");
+  // TrajL = 2
+  // 4/2 => 0.6 dH
+  // 3/3 => 0.8 dH .. depth 3, slower
+  //MD.MDsteps =  4;
+  MD.MDsteps =  8;
+  MD.trajL   = 0.5;
+
+  HMCparameters HMCparams;
+  HMCparams.StartTrajectory  = 1077;
+  HMCparams.Trajectories     = 20;
+  HMCparams.NoMetropolisUntil=  0;
+  // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+  HMCparams.StartingType     =std::string("ColdStart");
+  //  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.MD = MD;
+  HMCWrapper TheHMC(HMCparams);
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_HMC_lat";
+  CPparams.rng_prefix    = "ckpoint_HMC_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+  std::cout << "loaded NERSC checpointer"<<std::endl;
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5 6 7 8 9 10";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  const int Ls      = 12;
+  RealD M5  = 1.8;
+  RealD b   = 1.5;
+  RealD c   = 0.5;
+  RealD beta         = 2.13;
+  //  Real light_mass   = 5.4e-4;
+  Real light_mass     = 7.8e-4;
+  //  Real light_mass     = 7.8e-3;
+  Real strange_mass = 0.0362;
+  Real pv_mass      = 1.0;
+  std::vector<Real> hasenbusch({ 0.005, 0.0145, 0.045, 0.108, 0.25, 0.35 , 0.51, 0.6, 0.8 }); // Updated
+  //std::vector<Real> hasenbusch({ 0.0145, 0.045, 0.108, 0.25, 0.35 , 0.51, 0.6, 0.8 }); // Updated
+
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+
+  typedef SchurDiagMooeeOperator<FermionAction ,FermionField > LinearOperatorD;
+  typedef SchurDiagMooeeOperator<FermionEOFAAction ,FermionField > LinearOperatorEOFAD;
+
+  ////////////////////////////////////////////////////////////////
+  // Domain decomposed
+  ////////////////////////////////////////////////////////////////
+  Coordinate latt4  = GridPtr->GlobalDimensions();
+  Coordinate mpi    = GridPtr->ProcessorGrid();
+  Coordinate shm;
+
+  GlobalSharedMemory::GetShmDims(mpi,shm);
+
+  //////////////////////////
+  // Fermion Grids
+  //////////////////////////
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeFieldD  U(GridPtr); U=Zero();
+
+  std::cout << GridLogMessage << " Running the HMC "<< std::endl;
+  TheHMC.ReadCommandLine(argc,argv);  // params on CML or from param file
+  TheHMC.initializeGaugeFieldAndRNGs(U);
+  std::cout << "loaded NERSC gauge field"<<std::endl;
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+
+  //  double StoppingCondition = 1e-14;
+  //  double MDStoppingCondition = 1e-9;
+  double StoppingCondition = 1e-14;
+  double MDStoppingCondition = 1e-9;
+  double MDStoppingConditionLoose = 1e-9;
+  double MDStoppingConditionStrange = 1e-9;
+  double MaxCGIterations = 50000;
+  ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
+  ConjugateGradient<FermionField>  MDCG(MDStoppingCondition,MaxCGIterations);
+
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(2);
+  ActionLevel<HMCWrapper::Field> Level3(4);
+
+  ////////////////////////////////////
+  // Strange action
+  ////////////////////////////////////
+  FermionAction StrangeOp (U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,strange_mass,M5,b,c, Params);
+  FermionAction StrangePauliVillarsOp(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,pv_mass,  M5,b,c, Params);
+
+  // Probably dominates the force - back to EOFA.
+  OneFlavourRationalParams SFRp;
+  SFRp.lo       = 0.8;
+  SFRp.hi       = 30.0;
+  SFRp.MaxIter  = 10000;
+  SFRp.tolerance= 1.0e-12;
+  SFRp.mdtolerance= 1.0e-9;
+  SFRp.degree   = 10;
+  SFRp.precision= 50;
+  
+  MobiusEOFAFermionD Strange_Op_L (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionD Strange_Op_R (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+  ConjugateGradient<FermionField>      ActionCG(StoppingCondition,MaxCGIterations);
+  ConjugateGradient<FermionField>  DerivativeCG(MDStoppingCondition,MaxCGIterations);
+  LinearOperatorEOFAD Strange_LinOp_L (Strange_Op_L);
+  LinearOperatorEOFAD Strange_LinOp_R (Strange_Op_R);
+
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 ActionCG, 
+	 ActionCG, ActionCG,
+	 DerivativeCG, DerivativeCG,
+	 SFRp, true);
+  Level2.push_back(&EOFA);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass); 
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]);
+  }
+
+  for(int h=0;h<n_hasenbusch;h++){
+    light_num.push_back(hasenbusch[h]);
+  }
+  light_num.push_back(pv_mass);
+
+  std::vector<FermionAction *> Numerators;
+  std::vector<FermionAction *> Denominators;
+  std::vector<TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> *> Quotients;
+  
+  std::vector<OneFlavourEvenOddRatioRationalPseudoFermionAction<FermionImplPolicy> *> Bdys;
+
+  typedef SchurDiagMooeeOperator<FermionAction ,FermionField > LinearOperatorD;
+  std::vector<LinearOperatorD *> LinOpD;
+  
+  for(int h=0;h<n_hasenbusch+1;h++){
+    std::cout << GridLogMessage
+	      << " 2f quotient Action ";
+    std::cout << "det D("<<light_den[h]<<")";
+    std::cout << "/ det D("<<light_num[h]<<")";
+    std::cout << std::endl;
+
+    FermionAction::ImplParams ParamsNum(boundary);
+    FermionAction::ImplParams ParamsDen(boundary);
+    
+    Numerators.push_back  (new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, ParamsNum));
+    Denominators.push_back(new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, ParamsDen));
+
+    LinOpD.push_back(new LinearOperatorD(*Denominators[h]));
+
+    double conv  = MDStoppingCondition;
+    if (h<3) conv= MDStoppingConditionLoose; // Relax on first two hasenbusch factors
+    
+    Quotients.push_back (new TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],MDCG,CG,CG));
+  }
+  int nquo=Quotients.size();
+  for(int h=0;h<nquo;h++){
+    Level1.push_back(Quotients[h]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  Level3.push_back(&GaugeAction);
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+  TheHMC.TheAction.push_back(Level3);
+  std::cout << GridLogMessage << " Action complete "<< std::endl;
+  /////////////////////////////////////////////////////////////
+
+  TheHMC.Run();  // no smearing
+
+  Grid_finalize();
+} // main
+
+
+

HOWTO

@@ -0,0 +1,25 @@
+1. on Cori GPU, load necessary modules
+source ./load_cgpu_modules.sh 
+
+2. run bootstrap scrip
+./bootstrap.sh
+
+3. Create a build directory, for example, 
+mkdir build-cgpu
+
+3. run configure script in the build directory 
+cd build-cgpu
+sh config-command
+
+Example config-command for single-GPU omp offload: 
+
+../configure \
+    --enable-comms=none \
+    --enable-simd=GEN \
+    --enable-gen-simd-width=16 \
+    CXX=clang++ \
+    LDFLAGS="-L${CUDA_ROOT}/lib64 -lcudart" \
+    CXXFLAGS="-Wno-unknown-cuda-version -I${CUDA_ROOT}/include -fopenmp -std=c++14 -fopenmp-cuda-mode  -O3 -g -fopenmp-targets=nvptx64-nvidia-cuda -Wformat -DOMPTARGET -DOMPTARGET_MANAGED"
+
+4. compile 
+make -j8

MPI_benchmark/bench2.pbs

@@ -0,0 +1,22 @@
+#!/bin/bash
+#PBS -q EarlyAppAccess
+#PBS -l select=2
+#PBS -l walltime=01:00:00
+#PBS -A LatticeQCD_aesp_CNDA
+
+export TZ='/usr/share/zoneinfo/US/Central'
+export OMP_PROC_BIND=spread
+export OMP_NUM_THREADS=3
+unset OMP_PLACES
+
+cd $PBS_O_WORKDIR
+
+NNODES=`wc -l < $PBS_NODEFILE`
+NRANKS=12         # Number of MPI ranks per node
+NDEPTH=4          # Number of hardware threads per rank, spacing between MPI ranks on a node
+NTHREADS=$OMP_NUM_THREADS # Number of OMP threads per rank, given to OMP_NUM_THREADS
+
+NTOTRANKS=$(( NNODES * NRANKS ))
+
+CMD="mpiexec -np 2 -ppn 1  -envall ./gpu_tile_compact.sh ./halo_mpi --mpi 2.1.1.1"
+$CMD

MPI_benchmark/compile-command

@@ -0,0 +1 @@
+mpicxx  -fsycl halo_mpi.cc -o halo_mpi

MPI_benchmark/gpu_tile_compact.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+export NUMA_PMAP=(2 2 2 3 3 3 2 2 2 3 3 3 )
+export NUMA_MAP=(0 0 0 1 1 1 0 0 0 1 1 1 )
+export  GPU_MAP=(0 1 2 3 4 5 0 1 2 3 4 5 )
+export TILE_MAP=(0 0 0 0 0 0 1 1 1 1 1 1 )
+
+export PNUMA=${NUMA_PMAP[$PALS_LOCAL_RANKID]}
+export NUMA=${NUMA_MAP[$PALS_LOCAL_RANKID]}
+export gpu_id=${GPU_MAP[$PALS_LOCAL_RANKID]}
+export tile_id=${TILE_MAP[$PALS_LOCAL_RANKID]}
+  
+
+export ZE_AFFINITY_MASK=$gpu_id.$tile_id
+export ONEAPI_DEVICE_FILTER=gpu,level_zero
+
+#unset EnableWalkerPartition
+#export EnableImplicitScaling=0
+#export GRID_MPICH_NIC_BIND=$NIC
+#export ONEAPI_DEVICE_SELECTOR=level_zero:$gpu_id.$tile_id
+#export ZE_ENABLE_PCI_ID_DEVICE_ORDER=1
+#export SYCL_PI_LEVEL_ZERO_DEVICE_SCOPE_EVENTS=0
+#export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+#export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE=0:2
+#export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE_FOR_D2D_COPY=1
+#export SYCL_PI_LEVEL_ZERO_USM_RESIDENT=1
+
+echo "rank $PALS_RANKID ; local rank $PALS_LOCAL_RANKID ; ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK ; NUMA $NUMA "
+
+numactl -m $PNUMA -N $NUMA  "$@"

MPI_benchmark/halo_mpi.cc

@@ -0,0 +1,333 @@
+#include <cassert>
+#include <complex>
+#include <memory>
+#include <vector>
+#include <algorithm>
+#include <array>
+#include <string>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <ctime>
+#include <sys/time.h>
+
+#include <mpi.h>
+
+/**************************************************************
+ * GPU - GPU memory cartesian halo exchange benchmark
+ * Config: what is the target
+ **************************************************************
+ */
+#undef ACC_CUDA
+#undef  ACC_HIP
+#define  ACC_SYCL
+#undef  ACC_NONE
+
+/**************************************************************
+ * Some MPI globals
+ **************************************************************
+ */
+MPI_Comm WorldComm;
+MPI_Comm WorldShmComm;
+
+int WorldSize;
+int WorldRank;
+
+int WorldShmSize;
+int WorldShmRank;
+
+/**************************************************************
+ * Allocate buffers on the GPU, SYCL needs an init call and context
+ **************************************************************
+ */
+#ifdef ACC_CUDA
+#include <cuda.h>
+void acceleratorInit(void){}
+void *acceleratorAllocDevice(size_t bytes)
+{
+  void *ptr=NULL;
+  auto err = cudaMalloc((void **)&ptr,bytes);
+  assert(err==cudaSuccess);
+  return ptr;
+}
+void acceleratorFreeDevice(void *ptr){  cudaFree(ptr);}
+#endif
+#ifdef ACC_HIP
+#include <hip/hip_runtime.h>
+void acceleratorInit(void){}
+inline void *acceleratorAllocDevice(size_t bytes)
+{
+  void *ptr=NULL;
+  auto err = hipMalloc((void **)&ptr,bytes);
+  if( err != hipSuccess ) {
+    ptr = (void *) NULL;
+    printf(" hipMalloc failed for %ld %s \n",bytes,hipGetErrorString(err));
+  }
+  return ptr;
+};
+inline void acceleratorFreeDevice(void *ptr){ auto r=hipFree(ptr);};
+#endif
+#ifdef ACC_SYCL
+#include <sycl/CL/sycl.hpp>
+#include <sycl/usm.hpp>
+cl::sycl::queue *theAccelerator;
+void acceleratorInit(void)
+{
+  int nDevices = 1;
+#if 1
+  cl::sycl::gpu_selector selector;
+  cl::sycl::device selectedDevice { selector };
+  theAccelerator = new sycl::queue (selectedDevice);
+#else
+  cl::sycl::device selectedDevice {cl::sycl::gpu_selector_v  };
+  theAccelerator = new sycl::queue (selectedDevice);
+#endif
+  auto name = theAccelerator->get_device().get_info<sycl::info::device::name>();
+  printf("AcceleratorSyclInit: Selected device is %s\n",name.c_str()); fflush(stdout);
+}
+inline void *acceleratorAllocDevice(size_t bytes){ return malloc_device(bytes,*theAccelerator);};
+inline void acceleratorFreeDevice(void *ptr){free(ptr,*theAccelerator);};
+#endif
+#ifdef ACC_NONE
+void acceleratorInit(void){}
+inline void *acceleratorAllocDevice(size_t bytes){ return malloc(bytes);};
+inline void acceleratorFreeDevice(void *ptr){free(ptr);};
+#endif
+
+
+/**************************************************************
+ * Microsecond timer
+ **************************************************************
+ */
+inline double usecond(void) {
+  struct timeval tv;
+  gettimeofday(&tv,NULL);
+  return 1.0e6*tv.tv_sec + 1.0*tv.tv_usec;
+}
+/**************************************************************
+ * Main benchmark routine
+ **************************************************************
+ */
+void Benchmark(int64_t L,std::vector<int> cart_geom,bool use_device,int ncall)
+{
+  int64_t words = 3*4*2;
+  int64_t face,vol;
+  int Nd=cart_geom.size();
+  
+  /**************************************************************
+   * L^Nd volume, L^(Nd-1) faces, 12 complex per site
+   * Allocate memory for these
+   **************************************************************
+   */
+  face=1; for( int d=0;d<Nd-1;d++) face = face*L;
+  vol=1;  for( int d=0;d<Nd;d++) vol = vol*L;
+
+  
+  std::vector<void *> send_bufs;
+  std::vector<void *> recv_bufs;
+  size_t vw = face*words;
+  size_t bytes = face*words*sizeof(double);
+
+  if ( use_device ) {
+    for(int d=0;d<2*Nd;d++){
+      send_bufs.push_back(acceleratorAllocDevice(bytes));
+      recv_bufs.push_back(acceleratorAllocDevice(bytes));
+    }
+  } else {
+    for(int d=0;d<2*Nd;d++){
+      send_bufs.push_back(malloc(bytes));
+      recv_bufs.push_back(malloc(bytes));
+    }
+  }
+  /*********************************************************
+   * Build cartesian communicator
+   *********************************************************
+   */
+  int ierr;
+  int rank;
+  std::vector<int> coor(Nd);
+  MPI_Comm communicator;
+  std::vector<int> periodic(Nd,1);
+  MPI_Cart_create(WorldComm,Nd,&cart_geom[0],&periodic[0],0,&communicator);
+  MPI_Comm_rank(communicator,&rank);
+  MPI_Cart_coords(communicator,rank,Nd,&coor[0]);
+
+  static int reported;
+  if ( ! reported ) { 
+    printf("World Rank %d Shm Rank %d CartCoor %d %d %d %d\n",WorldRank,WorldShmRank,
+	 coor[0],coor[1],coor[2],coor[3]); fflush(stdout);
+    reported =1 ;
+  }
+  /*********************************************************
+   * Perform halo exchanges
+   *********************************************************
+   */
+  for(int d=0;d<Nd;d++){
+    if ( cart_geom[d]>1 ) {
+      double t0=usecond();
+
+      int from,to;
+      
+      MPI_Barrier(communicator);
+      for(int n=0;n<ncall;n++){
+	
+	void *xmit = (void *)send_bufs[d];
+	void *recv = (void *)recv_bufs[d];
+	
+	ierr=MPI_Cart_shift(communicator,d,1,&from,&to);
+	assert(ierr==0);
+	
+	ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,to,rank,
+			  recv,bytes,MPI_CHAR,from, from,
+			  communicator,MPI_STATUS_IGNORE);
+	assert(ierr==0);
+	
+	xmit = (void *)send_bufs[Nd+d];
+	recv = (void *)recv_bufs[Nd+d];
+	
+	ierr=MPI_Cart_shift(communicator,d,-1,&from,&to);
+	assert(ierr==0);
+	
+	ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,to,rank,
+			  recv,bytes,MPI_CHAR,from, from,
+			  communicator,MPI_STATUS_IGNORE);
+	assert(ierr==0);
+      }
+      MPI_Barrier(communicator);
+
+      double t1=usecond();
+      
+      double dbytes    = bytes*WorldShmSize;
+      double xbytes    = dbytes*2.0*ncall;
+      double rbytes    = xbytes;
+      double bidibytes = xbytes+rbytes;
+
+      if ( ! WorldRank ) {
+	printf("\t%12ld\t %12ld %16.0lf\n",L,bytes,bidibytes/(t1-t0)); fflush(stdout);
+      }
+    }
+  }
+  /*********************************************************
+   * Free memory
+   *********************************************************
+   */
+  if ( use_device ) {
+    for(int d=0;d<2*Nd;d++){
+      acceleratorFreeDevice(send_bufs[d]);
+      acceleratorFreeDevice(recv_bufs[d]);
+    }
+  } else {
+    for(int d=0;d<2*Nd;d++){
+      free(send_bufs[d]);
+      free(recv_bufs[d]);
+    }
+  }
+
+}
+
+/**************************************
+ * Command line junk
+ **************************************/
+
+std::string CmdOptionPayload(char ** begin, char ** end, const std::string & option)
+{
+  char ** itr = std::find(begin, end, option);
+  if (itr != end && ++itr != end) {
+    std::string payload(*itr);
+    return payload;
+  }
+  return std::string("");
+}
+bool CmdOptionExists(char** begin, char** end, const std::string& option)
+{
+  return std::find(begin, end, option) != end;
+}
+void CmdOptionIntVector(const std::string &str,std::vector<int> & vec)
+{
+  vec.resize(0);
+  std::stringstream ss(str);
+  int i;
+  while (ss >> i){
+    vec.push_back(i);
+    if(std::ispunct(ss.peek()))
+      ss.ignore();
+  }
+  return;
+}
+/**************************************
+ * Command line junk
+ **************************************/
+int main(int argc, char **argv)
+{
+  std::string arg;
+
+  acceleratorInit();
+
+  MPI_Init(&argc,&argv);
+
+  WorldComm = MPI_COMM_WORLD;
+  
+  MPI_Comm_split_type(WorldComm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&WorldShmComm);
+
+  MPI_Comm_rank(WorldComm     ,&WorldRank);
+  MPI_Comm_size(WorldComm     ,&WorldSize);
+
+  MPI_Comm_rank(WorldShmComm     ,&WorldShmRank);
+  MPI_Comm_size(WorldShmComm     ,&WorldShmSize);
+
+  if ( WorldSize/WorldShmSize > 2) {
+    printf("This benchmark is meant to run on at most two nodes only\n");
+  }
+
+  auto mpi =std::vector<int>({1,1,1,1});
+
+  if( CmdOptionExists(argv,argv+argc,"--mpi") ){
+    arg = CmdOptionPayload(argv,argv+argc,"--mpi");
+    CmdOptionIntVector(arg,mpi);
+  } else {
+    printf("Must specify --mpi <n1.n2.n3.n4> command line argument\n");
+    exit(0);
+  }
+
+  if( !WorldRank ) {
+    printf("***********************************\n");
+    printf("%d ranks\n",WorldSize); 
+    printf("%d ranks-per-node\n",WorldShmSize);
+    printf("%d nodes\n",WorldSize/WorldShmSize);fflush(stdout);
+    printf("Cartesian layout: ");
+    for(int d=0;d<mpi.size();d++){
+      printf("%d ",mpi[d]);
+    }
+    printf("\n");fflush(stdout);
+    printf("***********************************\n");
+  }
+
+  
+  if( !WorldRank ) {
+    printf("=========================================================\n");
+    printf("= Benchmarking HOST memory MPI performance               \n");
+    printf("=========================================================\n");fflush(stdout);
+    printf("= L\t pkt bytes\t MB/s           \n");
+    printf("=========================================================\n");fflush(stdout);
+  }
+
+  for(int L=16;L<=64;L+=4){
+    Benchmark(L,mpi,false,100);
+  }  
+
+  if( !WorldRank ) {
+    printf("=========================================================\n");
+    printf("= Benchmarking DEVICE memory MPI performance             \n");
+    printf("=========================================================\n");fflush(stdout);
+  }
+  for(int L=16;L<=64;L+=4){
+    Benchmark(L,mpi,true,100);
+  }  
+
+  if( !WorldRank ) {
+    printf("=========================================================\n");
+    printf("= DONE   \n");
+    printf("=========================================================\n");
+  }
+  MPI_Finalize();
+}

amd-omp-stack-err/README

@@ -0,0 +1,27 @@
+module load rocm/5.5.1
+
+mkdir build-amd-err && cd build-amd-err
+
+cp ../amd-omp-stack-err/Test.cc ../amd-omp-stack-err/WilsonFermionInstantiationWilsonImplD.cc .
+
+../configure CXX=amdclang++ --enable-comms=none --enable-simd=GEN --enable-accelerator-cshift=no --enable-shm=no --disable-unified --enable-unified=no --enable-fermion-reps=no --enable-gen-simd-width=16 CXXFLAGS="-Wno-unknown-cuda-version -fopenmp --offload-arch=gfx90a -std=c++14 -fopenmp-cuda-mode -O3 -g -Wformat -DEIGEN_NO_CUDA -DEIGEN_DONT_VECTORIZE -DOMPTARGET"
+
+amdclang++ -c Test.cc -o Test.o -I/autofs/nccs-svm1_home1/atif/Grid -I/autofs/nccs-svm1_home1/atif/Grid/build-amd-err/Grid/ -O3 -Wno-unknown-cuda-version -fopenmp --offload-arch=gfx90a -std=c++14 -fopenmp-cuda-mode -O3 -Wformat -DEIGEN_NO_CUDA -DOMPTARGET -fno-strict-aliasing
+
+amdclang++ -c WilsonFermionInstantiationWilsonImplD.cc -o WilsonFails.o -I/autofs/nccs-svm1_home1/atif/Grid -I/autofs/nccs-svm1_home1/atif/Grid/build-amd-err/Grid/ -O3 -Wno-unknown-cuda-version -fopenmp --offload-arch=gfx90a -std=c++14 -fopenmp-cuda-mode -O3 -Wformat -DEIGEN_NO_CUDA -DOMPTARGET -fno-strict-aliasing
+
+ar cru libWilsonFails.a WilsonFails.o
+
+ranlib libWilsonFails.a
+
+amdclang++ -o Test -I/autofs/nccs-svm1_home1/atif/Grid -I/autofs/nccs-svm1_home1/atif/Grid/build-amd-err/Grid/   -O3 -Wno-unknown-cuda-version -fopenmp --offload-arch=gfx90a -std=c++14 -fopenmp-cuda-mode -O3 -Wformat -DEIGEN_NO_CUDA -DOMPTARGET  -fno-strict-aliasing Test.o -L./  -lWilsonFails
+
+error: stack frame size (149840) exceeds limit (131056) in function '__omp_offloading_72_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE_l190'
+error: stack frame size (149840) exceeds limit (131056) in function '__omp_offloading_72_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS_23LatticeBinaryExpressionINS_10BinaryOrOrENS0_INS1_INS3_IjNS7_IjEEEEEEEESL_EESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE_l190'
+error: stack frame size (149840) exceeds limit (131056) in function '__omp_offloading_72_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryAddESD_NS_24LatticeTrinaryExpressionINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EEEERSD_RKNS_23LatticeBinaryExpressionIT_T0_T1_EE_l166'
+clang-16: error: amdgcn-link command failed with exit code 1 (use -v to see invocation)
+
+
+llvm-objdump -t libWilsonFermionWorks2.a > objdump_works2.txt
+llvm-cxxfilt < objdump_works2.txt > cxxfilt_works2.txt
+

amd-omp-stack-err/Test.cc

@@ -0,0 +1,37 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_rng.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+ ;
+
+int main (int argc, char ** argv)
+{
+  std::cout << "atif1 " << __FILE__ << ":" << __LINE__ << std::endl;
+}

amd-omp-stack-err/WilsonFermionInstantiationWilsonImplD.cc

@@ -0,0 +1,615 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/WilsonFermion.cc
+
+Copyright (C) 2022
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Fabian Joswig <fabian.joswig@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/WilsonFermion.h>
+
+NAMESPACE_BEGIN(Grid);
+
+/////////////////////////////////
+// Constructor and gauge import
+/////////////////////////////////
+
+//template <class Impl>
+//WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
+//                                   GridRedBlackCartesian &Hgrid, RealD _mass,
+//                                   const ImplParams &p,
+//                                   const WilsonAnisotropyCoefficients &anis)
+//  :
+//    Kernels(p),
+//    _grid(&Fgrid),
+//    _cbgrid(&Hgrid),
+//    Stencil(&Fgrid, npoint, Even, directions, displacements,p),
+//    StencilEven(&Hgrid, npoint, Even, directions,displacements,p),  // source is Even
+//    StencilOdd(&Hgrid, npoint, Odd, directions,displacements,p),  // source is Odd
+//    mass(_mass),
+//    Lebesgue(_grid),
+//    LebesgueEvenOdd(_cbgrid),
+//    Umu(&Fgrid),
+//    UmuEven(&Hgrid),
+//    UmuOdd(&Hgrid),
+//      _tmp(&Hgrid),
+//      anisotropyCoeff(anis)
+//{
+//  Stencil.lo     = &Lebesgue;
+//  StencilEven.lo = &LebesgueEvenOdd;
+//  StencilOdd.lo  = &LebesgueEvenOdd;
+//  // Allocate the required comms buffer
+//  ImportGauge(_Umu);
+//  if  (anisotropyCoeff.isAnisotropic){
+//    diag_mass = mass + 1.0 + (Nd-1)*(anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//  } else {
+//    diag_mass = 4.0 + mass;
+//  }
+//
+//  int vol4;
+//  vol4=Fgrid.oSites();
+//  Stencil.BuildSurfaceList(1,vol4);
+//  vol4=Hgrid.oSites();
+//  StencilEven.BuildSurfaceList(1,vol4);
+//  StencilOdd.BuildSurfaceList(1,vol4);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::ImportGauge(const GaugeField &_Umu)
+//{
+//  GaugeField HUmu(_Umu.Grid());
+//
+//  //Here multiply the anisotropy coefficients
+//  if (anisotropyCoeff.isAnisotropic)
+//  {
+//
+//    for (int mu = 0; mu < Nd; mu++)
+//    {
+//      GaugeLinkField U_dir = (-0.5)*PeekIndex<LorentzIndex>(_Umu, mu);
+//      if (mu != anisotropyCoeff.t_direction)
+//        U_dir *= (anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//
+//      PokeIndex<LorentzIndex>(HUmu, U_dir, mu);
+//    }
+//  }
+//  else
+//  {
+//    HUmu = _Umu * (-0.5);
+//  }
+//  Impl::DoubleStore(GaugeGrid(), Umu, HUmu);
+//  pickCheckerboard(Even, UmuEven, Umu);
+//  pickCheckerboard(Odd, UmuOdd, Umu);
+//}
+//
+///////////////////////////////
+//// Implement the interface
+///////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::M(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerNo);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerYes);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Meooe(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerNo);
+//  } else {
+//    DhopOE(in, out, DaggerNo);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerYes);
+//  } else {
+//    DhopOE(in, out, DaggerYes);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mooee(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  typename FermionField::scalar_type scal(diag_mass);
+//  out = scal * in;
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MooeeDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Mooee(in, out);
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInv(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  out = (1.0/(diag_mass))*in;
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  MooeeInv(in,out);
+//}
+//template<class Impl>
+//void WilsonFermion<Impl>::MomentumSpacePropagator(FermionField &out, const FermionField &in,RealD _m,std::vector<double> twist)
+//{
+//  typedef typename FermionField::vector_type vector_type;
+//  typedef typename FermionField::scalar_type ScalComplex;
+//  typedef Lattice<iSinglet<vector_type> > LatComplex;
+//
+//  // what type LatticeComplex
+//  conformable(_grid,out.Grid());
+//
+//  Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT
+//  };
+//
+//  Coordinate latt_size   = _grid->_fdimensions;
+//
+//  FermionField   num  (_grid); num  = Zero();
+//  LatComplex    wilson(_grid); wilson= Zero();
+//  LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
+//
+//  LatComplex denom(_grid); denom= Zero();
+//  LatComplex kmu(_grid);
+//  ScalComplex ci(0.0,1.0);
+//  // momphase = n * 2pi / L
+//  for(int mu=0;mu<Nd;mu++) {
+//
+//    LatticeCoordinate(kmu,mu);
+//
+//    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+//
+//    kmu = TwoPiL * kmu;
+//    kmu = kmu + TwoPiL * one * twist[mu];//momentum for twisted boundary conditions
+//
+//    wilson = wilson + 2.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
+//
+//    num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);    // derivative term
+//
+//    denom=denom + sin(kmu)*sin(kmu);
+//  }
+//
+//  wilson = wilson + _m;     // 2 sin^2 k/2 + m
+//
+//  num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
+//
+//  denom= denom+wilson*wilson; // sin^2 k + (2 sin^2 k/2 + m)^2
+//
+//  denom= one/denom;
+//
+//  out = num*denom; // [ -i gmu sin k + 2 sin^2 k/2 + m] / [ sin^2 k + (2 sin^2 k/2 + m)^2 ]
+//
+//}
+//
+//
+/////////////////////////////////////
+//// Internal
+/////////////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
+//                                        GaugeField &mat, const FermionField &A,
+//                                        const FermionField &B, int dag) {
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//
+//  FermionField Btilde(B.Grid());
+//  FermionField Atilde(B.Grid());
+//  Atilde = A;
+//
+//  st.HaloExchange(B, compressor);
+//
+//  for (int mu = 0; mu < Nd; mu++) {
+//    ////////////////////////////////////////////////////////////////////////
+//    // Flip gamma (1+g)<->(1-g) if dag
+//    ////////////////////////////////////////////////////////////////////////
+//    int gamma = mu;
+//    if (!dag) gamma += Nd;
+//
+//    int Ls=1;
+//    Kernels::DhopDirKernel(st, U, st.CommBuf(), Ls, B.Grid()->oSites(), B, Btilde, mu, gamma);
+//
+//    //////////////////////////////////////////////////
+//    // spin trace outer product
+//    //////////////////////////////////////////////////
+//    Impl::InsertForce4D(mat, Btilde, Atilde, mu);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _grid);
+//  conformable(U.Grid(), V.Grid());
+//  conformable(U.Grid(), mat.Grid());
+//
+//  mat.Checkerboard() = U.Checkerboard();
+//
+//  DerivInternal(Stencil, Umu, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid()); not general, leaving as a comment (Guido)
+//  // Motivation: look at the SchurDiff operator
+//
+//  assert(V.Checkerboard() == Even);
+//  assert(U.Checkerboard() == Odd);
+//  mat.Checkerboard() = Odd;
+//
+//  DerivInternal(StencilEven, UmuOdd, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid());
+//
+//  assert(V.Checkerboard() == Odd);
+//  assert(U.Checkerboard() == Even);
+//  mat.Checkerboard() = Even;
+//
+//  DerivInternal(StencilOdd, UmuEven, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _grid);  // verifies full grid
+//  conformable(in.Grid(), out.Grid());
+//
+//  out.Checkerboard() = in.Checkerboard();
+//
+//  DhopInternal(Stencil, Lebesgue, Umu, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Even);
+//  out.Checkerboard() = Odd;
+//
+//  DhopInternal(StencilEven, LebesgueEvenOdd, UmuOdd, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Odd);
+//  out.Checkerboard() = Even;
+//
+//  DhopInternal(StencilOdd, LebesgueEvenOdd, UmuEven, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  DhopDir(in, out, dir, disp);
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::MdirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  DhopDirAll(in, out);
+//}
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  int skip = (disp == 1) ? 0 : 1;
+//  int dirdisp = dir + skip * 4;
+//  int gamma = dir + (1 - skip) * 4;
+//
+//  DhopDirCalc(in, out, dirdisp, gamma, DaggerNo);
+//};
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  assert((out.size()==8)||(out.size()==9));
+//  for(int dir=0;dir<Nd;dir++){
+//    for(int disp=-1;disp<=1;disp+=2){
+//
+//      int skip = (disp == 1) ? 0 : 1;
+//      int dirdisp = dir + skip * 4;
+//      int gamma = dir + (1 - skip) * 4;
+//
+//      DhopDirCalc(in, out[dirdisp], dirdisp, gamma, DaggerNo);
+//    }
+//  }
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirCalc(const FermionField &in, FermionField &out,int dirdisp, int gamma, int dag)
+//{
+//  int Ls=1;
+//  uint64_t Nsite=in.oSites();
+//  Kernels::DhopDirKernel(Stencil, Umu, Stencil.CommBuf(), Ls, Nsite, in, out, dirdisp, gamma);
+//};
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//#ifdef GRID_OMP
+//  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute )
+//    DhopInternalOverlappedComms(st,lo,U,in,out,dag);
+//  else
+//#endif
+//    DhopInternalSerial(st,lo,U,in,out,dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalOverlappedComms(StencilImpl &st, LebesgueOrder &lo,
+//						      DoubledGaugeField &U,
+//						      const FermionField &in,
+//						      FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopOverlapped");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//  int len =  U.Grid()->oSites();
+//
+//  /////////////////////////////
+//  // Start comms  // Gather intranode and extra node differentiated??
+//  /////////////////////////////
+//  std::vector<std::vector<CommsRequest_t> > requests;
+//  st.Prepare();
+//  {
+//    GRID_TRACE("Gather");
+//    st.HaloGather(in,compressor);
+//  }
+//
+//  tracePush("Communication");
+//  st.CommunicateBegin(requests);
+//
+//  /////////////////////////////
+//  // Overlap with comms
+//  /////////////////////////////
+//  {
+//    GRID_TRACE("MergeSHM");
+//    st.CommsMergeSHM(compressor);
+//  }
+//
+//  /////////////////////////////
+//  // do the compute interior
+//  /////////////////////////////
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagInterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  } else {
+//    GRID_TRACE("DhopInterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  }
+//
+//  /////////////////////////////
+//  // Complete comms
+//  /////////////////////////////
+//  st.CommunicateComplete(requests);
+//  tracePop("Communication");
+//
+//  {
+//    GRID_TRACE("Merge");
+//    st.CommsMerge(compressor);
+//  }
+//  /////////////////////////////
+//  // do the compute exterior
+//  /////////////////////////////
+//
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagExterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  } else {
+//    GRID_TRACE("DhopExterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  }
+//};
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalSerial(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopSerial");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//  Compressor compressor(dag);
+//  {
+//    GRID_TRACE("HaloExchange");
+//    st.HaloExchange(in, compressor);
+//  }
+//
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDag");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  } else {
+//    GRID_TRACE("Dhop");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  }
+//};
+///*Change ends */
+//
+///*******************************************************************************
+// * Conserved current utilities for Wilson fermions, for contracting propagators
+// * to make a conserved current sink or inserting the conserved current
+// * sequentially.
+// ******************************************************************************/
+//template <class Impl>
+//void WilsonFermion<Impl>::ContractConservedCurrent(PropagatorField &q_in_1,
+//                                                   PropagatorField &q_in_2,
+//                                                   PropagatorField &q_out,
+//                                                   PropagatorField &src,
+//                                                   Current curr_type,
+//                                                   unsigned int mu)
+//{
+//  if(curr_type != Current::Vector)
+//  {
+//    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+//    exit(1);
+//  }
+//
+//  Gamma g5(Gamma::Algebra::Gamma5);
+//  conformable(_grid, q_in_1.Grid());
+//  conformable(_grid, q_in_2.Grid());
+//  conformable(_grid, q_out.Grid());
+//  auto UGrid= this->GaugeGrid();
+//
+//  PropagatorField tmp_shifted(UGrid);
+//  PropagatorField g5Lg5(UGrid);
+//  PropagatorField R(UGrid);
+//  PropagatorField gmuR(UGrid);
+//
+//    Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT,
+//  };
+//  Gamma gmu=Gamma(Gmu[mu]);
+//
+//  g5Lg5=g5*q_in_1*g5;
+//  tmp_shifted=Cshift(q_in_2,mu,1);
+//  Impl::multLinkField(R,this->Umu,tmp_shifted,mu);
+//  gmuR=gmu*R;
+//
+//  q_out=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//
+//  tmp_shifted=Cshift(q_in_1,mu,1);
+//  Impl::multLinkField(g5Lg5,this->Umu,tmp_shifted,mu);
+//  g5Lg5=g5*g5Lg5*g5;
+//  R=q_in_2;
+//  gmuR=gmu*R;
+//
+//  q_out-=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//}
+//
+template <class Impl>
+void WilsonFermion<Impl>::SeqConservedCurrent(PropagatorField &q_in,
+                                              PropagatorField &q_out,
+                                              PropagatorField &src,
+                                              Current curr_type,
+                                              unsigned int mu,
+                                              unsigned int tmin,
+                                              unsigned int tmax,
+					      ComplexField &lattice_cmplx)
+{
+  if(curr_type != Current::Vector)
+  {
+    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+    exit(1);
+  }
+
+  int tshift = (mu == Nd-1) ? 1 : 0;
+  unsigned int LLt    = GridDefaultLatt()[Tp];
+  conformable(_grid, q_in.Grid());
+  conformable(_grid, q_out.Grid());
+  auto UGrid= this->GaugeGrid();
+
+  PropagatorField tmp(UGrid);
+  PropagatorField Utmp(UGrid);
+  PropagatorField L(UGrid);
+  PropagatorField zz (UGrid);
+  zz=Zero();
+  LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
+
+    Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT,
+  };
+  Gamma gmu=Gamma(Gmu[mu]);
+
+  tmp = Cshift(q_in,mu,1);
+  Impl::multLinkField(Utmp,this->Umu,tmp,mu);
+  tmp = ( Utmp*lattice_cmplx - gmu*Utmp*lattice_cmplx ); // Forward hop
+  tmp = where((lcoor>=tmin),tmp,zz); // Mask the time
+//  q_out = where((lcoor<=tmax),tmp,zz); // Position of current complicated
+//
+//  tmp = q_in *lattice_cmplx;
+//  tmp = Cshift(tmp,mu,-1);
+//  Impl::multLinkField(Utmp,this->Umu,tmp,mu+Nd); // Adjoint link
+//  tmp = -( Utmp + gmu*Utmp );
+//  // Mask the time
+//  if (tmax == LLt - 1 && tshift == 1){ // quick fix to include timeslice 0 if tmax + tshift is over the last timeslice
+//    unsigned int t0 = 0;
+//    tmp = where(((lcoor==t0) || (lcoor>=tmin+tshift)),tmp,zz);
+//  } else {
+//    tmp = where((lcoor>=tmin+tshift),tmp,zz);
+//  }
+//  q_out+= where((lcoor<=tmax+tshift),tmp,zz); // Position of current complicated
+}
+
+template class WilsonFermion<WilsonImplD>; 
+
+NAMESPACE_END(Grid);

amd-omp-stack-err/WilsonFermionWorks1.cc

@@ -0,0 +1,615 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/WilsonFermion.cc
+
+Copyright (C) 2022
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Fabian Joswig <fabian.joswig@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/WilsonFermion.h>
+
+NAMESPACE_BEGIN(Grid);
+
+/////////////////////////////////
+// Constructor and gauge import
+/////////////////////////////////
+
+//template <class Impl>
+//WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
+//                                   GridRedBlackCartesian &Hgrid, RealD _mass,
+//                                   const ImplParams &p,
+//                                   const WilsonAnisotropyCoefficients &anis)
+//  :
+//    Kernels(p),
+//    _grid(&Fgrid),
+//    _cbgrid(&Hgrid),
+//    Stencil(&Fgrid, npoint, Even, directions, displacements,p),
+//    StencilEven(&Hgrid, npoint, Even, directions,displacements,p),  // source is Even
+//    StencilOdd(&Hgrid, npoint, Odd, directions,displacements,p),  // source is Odd
+//    mass(_mass),
+//    Lebesgue(_grid),
+//    LebesgueEvenOdd(_cbgrid),
+//    Umu(&Fgrid),
+//    UmuEven(&Hgrid),
+//    UmuOdd(&Hgrid),
+//      _tmp(&Hgrid),
+//      anisotropyCoeff(anis)
+//{
+//  Stencil.lo     = &Lebesgue;
+//  StencilEven.lo = &LebesgueEvenOdd;
+//  StencilOdd.lo  = &LebesgueEvenOdd;
+//  // Allocate the required comms buffer
+//  ImportGauge(_Umu);
+//  if  (anisotropyCoeff.isAnisotropic){
+//    diag_mass = mass + 1.0 + (Nd-1)*(anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//  } else {
+//    diag_mass = 4.0 + mass;
+//  }
+//
+//  int vol4;
+//  vol4=Fgrid.oSites();
+//  Stencil.BuildSurfaceList(1,vol4);
+//  vol4=Hgrid.oSites();
+//  StencilEven.BuildSurfaceList(1,vol4);
+//  StencilOdd.BuildSurfaceList(1,vol4);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::ImportGauge(const GaugeField &_Umu)
+//{
+//  GaugeField HUmu(_Umu.Grid());
+//
+//  //Here multiply the anisotropy coefficients
+//  if (anisotropyCoeff.isAnisotropic)
+//  {
+//
+//    for (int mu = 0; mu < Nd; mu++)
+//    {
+//      GaugeLinkField U_dir = (-0.5)*PeekIndex<LorentzIndex>(_Umu, mu);
+//      if (mu != anisotropyCoeff.t_direction)
+//        U_dir *= (anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//
+//      PokeIndex<LorentzIndex>(HUmu, U_dir, mu);
+//    }
+//  }
+//  else
+//  {
+//    HUmu = _Umu * (-0.5);
+//  }
+//  Impl::DoubleStore(GaugeGrid(), Umu, HUmu);
+//  pickCheckerboard(Even, UmuEven, Umu);
+//  pickCheckerboard(Odd, UmuOdd, Umu);
+//}
+//
+///////////////////////////////
+//// Implement the interface
+///////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::M(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerNo);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerYes);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Meooe(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerNo);
+//  } else {
+//    DhopOE(in, out, DaggerNo);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerYes);
+//  } else {
+//    DhopOE(in, out, DaggerYes);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mooee(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  typename FermionField::scalar_type scal(diag_mass);
+//  out = scal * in;
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MooeeDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Mooee(in, out);
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInv(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  out = (1.0/(diag_mass))*in;
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  MooeeInv(in,out);
+//}
+//template<class Impl>
+//void WilsonFermion<Impl>::MomentumSpacePropagator(FermionField &out, const FermionField &in,RealD _m,std::vector<double> twist)
+//{
+//  typedef typename FermionField::vector_type vector_type;
+//  typedef typename FermionField::scalar_type ScalComplex;
+//  typedef Lattice<iSinglet<vector_type> > LatComplex;
+//
+//  // what type LatticeComplex
+//  conformable(_grid,out.Grid());
+//
+//  Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT
+//  };
+//
+//  Coordinate latt_size   = _grid->_fdimensions;
+//
+//  FermionField   num  (_grid); num  = Zero();
+//  LatComplex    wilson(_grid); wilson= Zero();
+//  LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
+//
+//  LatComplex denom(_grid); denom= Zero();
+//  LatComplex kmu(_grid);
+//  ScalComplex ci(0.0,1.0);
+//  // momphase = n * 2pi / L
+//  for(int mu=0;mu<Nd;mu++) {
+//
+//    LatticeCoordinate(kmu,mu);
+//
+//    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+//
+//    kmu = TwoPiL * kmu;
+//    kmu = kmu + TwoPiL * one * twist[mu];//momentum for twisted boundary conditions
+//
+//    wilson = wilson + 2.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
+//
+//    num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);    // derivative term
+//
+//    denom=denom + sin(kmu)*sin(kmu);
+//  }
+//
+//  wilson = wilson + _m;     // 2 sin^2 k/2 + m
+//
+//  num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
+//
+//  denom= denom+wilson*wilson; // sin^2 k + (2 sin^2 k/2 + m)^2
+//
+//  denom= one/denom;
+//
+//  out = num*denom; // [ -i gmu sin k + 2 sin^2 k/2 + m] / [ sin^2 k + (2 sin^2 k/2 + m)^2 ]
+//
+//}
+//
+//
+/////////////////////////////////////
+//// Internal
+/////////////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
+//                                        GaugeField &mat, const FermionField &A,
+//                                        const FermionField &B, int dag) {
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//
+//  FermionField Btilde(B.Grid());
+//  FermionField Atilde(B.Grid());
+//  Atilde = A;
+//
+//  st.HaloExchange(B, compressor);
+//
+//  for (int mu = 0; mu < Nd; mu++) {
+//    ////////////////////////////////////////////////////////////////////////
+//    // Flip gamma (1+g)<->(1-g) if dag
+//    ////////////////////////////////////////////////////////////////////////
+//    int gamma = mu;
+//    if (!dag) gamma += Nd;
+//
+//    int Ls=1;
+//    Kernels::DhopDirKernel(st, U, st.CommBuf(), Ls, B.Grid()->oSites(), B, Btilde, mu, gamma);
+//
+//    //////////////////////////////////////////////////
+//    // spin trace outer product
+//    //////////////////////////////////////////////////
+//    Impl::InsertForce4D(mat, Btilde, Atilde, mu);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _grid);
+//  conformable(U.Grid(), V.Grid());
+//  conformable(U.Grid(), mat.Grid());
+//
+//  mat.Checkerboard() = U.Checkerboard();
+//
+//  DerivInternal(Stencil, Umu, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid()); not general, leaving as a comment (Guido)
+//  // Motivation: look at the SchurDiff operator
+//
+//  assert(V.Checkerboard() == Even);
+//  assert(U.Checkerboard() == Odd);
+//  mat.Checkerboard() = Odd;
+//
+//  DerivInternal(StencilEven, UmuOdd, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid());
+//
+//  assert(V.Checkerboard() == Odd);
+//  assert(U.Checkerboard() == Even);
+//  mat.Checkerboard() = Even;
+//
+//  DerivInternal(StencilOdd, UmuEven, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _grid);  // verifies full grid
+//  conformable(in.Grid(), out.Grid());
+//
+//  out.Checkerboard() = in.Checkerboard();
+//
+//  DhopInternal(Stencil, Lebesgue, Umu, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Even);
+//  out.Checkerboard() = Odd;
+//
+//  DhopInternal(StencilEven, LebesgueEvenOdd, UmuOdd, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Odd);
+//  out.Checkerboard() = Even;
+//
+//  DhopInternal(StencilOdd, LebesgueEvenOdd, UmuEven, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  DhopDir(in, out, dir, disp);
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::MdirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  DhopDirAll(in, out);
+//}
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  int skip = (disp == 1) ? 0 : 1;
+//  int dirdisp = dir + skip * 4;
+//  int gamma = dir + (1 - skip) * 4;
+//
+//  DhopDirCalc(in, out, dirdisp, gamma, DaggerNo);
+//};
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  assert((out.size()==8)||(out.size()==9));
+//  for(int dir=0;dir<Nd;dir++){
+//    for(int disp=-1;disp<=1;disp+=2){
+//
+//      int skip = (disp == 1) ? 0 : 1;
+//      int dirdisp = dir + skip * 4;
+//      int gamma = dir + (1 - skip) * 4;
+//
+//      DhopDirCalc(in, out[dirdisp], dirdisp, gamma, DaggerNo);
+//    }
+//  }
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirCalc(const FermionField &in, FermionField &out,int dirdisp, int gamma, int dag)
+//{
+//  int Ls=1;
+//  uint64_t Nsite=in.oSites();
+//  Kernels::DhopDirKernel(Stencil, Umu, Stencil.CommBuf(), Ls, Nsite, in, out, dirdisp, gamma);
+//};
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//#ifdef GRID_OMP
+//  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute )
+//    DhopInternalOverlappedComms(st,lo,U,in,out,dag);
+//  else
+//#endif
+//    DhopInternalSerial(st,lo,U,in,out,dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalOverlappedComms(StencilImpl &st, LebesgueOrder &lo,
+//						      DoubledGaugeField &U,
+//						      const FermionField &in,
+//						      FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopOverlapped");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//  int len =  U.Grid()->oSites();
+//
+//  /////////////////////////////
+//  // Start comms  // Gather intranode and extra node differentiated??
+//  /////////////////////////////
+//  std::vector<std::vector<CommsRequest_t> > requests;
+//  st.Prepare();
+//  {
+//    GRID_TRACE("Gather");
+//    st.HaloGather(in,compressor);
+//  }
+//
+//  tracePush("Communication");
+//  st.CommunicateBegin(requests);
+//
+//  /////////////////////////////
+//  // Overlap with comms
+//  /////////////////////////////
+//  {
+//    GRID_TRACE("MergeSHM");
+//    st.CommsMergeSHM(compressor);
+//  }
+//
+//  /////////////////////////////
+//  // do the compute interior
+//  /////////////////////////////
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagInterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  } else {
+//    GRID_TRACE("DhopInterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  }
+//
+//  /////////////////////////////
+//  // Complete comms
+//  /////////////////////////////
+//  st.CommunicateComplete(requests);
+//  tracePop("Communication");
+//
+//  {
+//    GRID_TRACE("Merge");
+//    st.CommsMerge(compressor);
+//  }
+//  /////////////////////////////
+//  // do the compute exterior
+//  /////////////////////////////
+//
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagExterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  } else {
+//    GRID_TRACE("DhopExterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  }
+//};
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalSerial(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopSerial");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//  Compressor compressor(dag);
+//  {
+//    GRID_TRACE("HaloExchange");
+//    st.HaloExchange(in, compressor);
+//  }
+//
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDag");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  } else {
+//    GRID_TRACE("Dhop");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  }
+//};
+///*Change ends */
+//
+///*******************************************************************************
+// * Conserved current utilities for Wilson fermions, for contracting propagators
+// * to make a conserved current sink or inserting the conserved current
+// * sequentially.
+// ******************************************************************************/
+//template <class Impl>
+//void WilsonFermion<Impl>::ContractConservedCurrent(PropagatorField &q_in_1,
+//                                                   PropagatorField &q_in_2,
+//                                                   PropagatorField &q_out,
+//                                                   PropagatorField &src,
+//                                                   Current curr_type,
+//                                                   unsigned int mu)
+//{
+//  if(curr_type != Current::Vector)
+//  {
+//    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+//    exit(1);
+//  }
+//
+//  Gamma g5(Gamma::Algebra::Gamma5);
+//  conformable(_grid, q_in_1.Grid());
+//  conformable(_grid, q_in_2.Grid());
+//  conformable(_grid, q_out.Grid());
+//  auto UGrid= this->GaugeGrid();
+//
+//  PropagatorField tmp_shifted(UGrid);
+//  PropagatorField g5Lg5(UGrid);
+//  PropagatorField R(UGrid);
+//  PropagatorField gmuR(UGrid);
+//
+//    Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT,
+//  };
+//  Gamma gmu=Gamma(Gmu[mu]);
+//
+//  g5Lg5=g5*q_in_1*g5;
+//  tmp_shifted=Cshift(q_in_2,mu,1);
+//  Impl::multLinkField(R,this->Umu,tmp_shifted,mu);
+//  gmuR=gmu*R;
+//
+//  q_out=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//
+//  tmp_shifted=Cshift(q_in_1,mu,1);
+//  Impl::multLinkField(g5Lg5,this->Umu,tmp_shifted,mu);
+//  g5Lg5=g5*g5Lg5*g5;
+//  R=q_in_2;
+//  gmuR=gmu*R;
+//
+//  q_out-=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//}
+//
+template <class Impl>
+void WilsonFermion<Impl>::SeqConservedCurrent(PropagatorField &q_in,
+                                              PropagatorField &q_out,
+                                              PropagatorField &src,
+                                              Current curr_type,
+                                              unsigned int mu,
+                                              unsigned int tmin,
+                                              unsigned int tmax,
+					      ComplexField &lattice_cmplx)
+{
+  if(curr_type != Current::Vector)
+  {
+    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+    exit(1);
+  }
+
+  int tshift = (mu == Nd-1) ? 1 : 0;
+  unsigned int LLt    = GridDefaultLatt()[Tp];
+  conformable(_grid, q_in.Grid());
+  conformable(_grid, q_out.Grid());
+  auto UGrid= this->GaugeGrid();
+
+  PropagatorField tmp(UGrid);
+  PropagatorField Utmp(UGrid);
+  PropagatorField L(UGrid);
+  PropagatorField zz (UGrid);
+  zz=Zero();
+  LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
+
+    Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT,
+  };
+  Gamma gmu=Gamma(Gmu[mu]);
+
+  tmp = Cshift(q_in,mu,1);
+  Impl::multLinkField(Utmp,this->Umu,tmp,mu);
+  tmp = ( Utmp*lattice_cmplx - gmu*Utmp*lattice_cmplx ); // Forward hop
+  tmp = where((lcoor>=tmin),tmp,zz); // Mask the time
+//  q_out = where((lcoor<=tmax),tmp,zz); // Position of current complicated
+//
+//  tmp = q_in *lattice_cmplx;
+//  tmp = Cshift(tmp,mu,-1);
+//  Impl::multLinkField(Utmp,this->Umu,tmp,mu+Nd); // Adjoint link
+//  tmp = -( Utmp + gmu*Utmp );
+//  // Mask the time
+//  if (tmax == LLt - 1 && tshift == 1){ // quick fix to include timeslice 0 if tmax + tshift is over the last timeslice
+//    unsigned int t0 = 0;
+//    tmp = where(((lcoor==t0) || (lcoor>=tmin+tshift)),tmp,zz);
+//  } else {
+//    tmp = where((lcoor>=tmin+tshift),tmp,zz);
+//  }
+//  q_out+= where((lcoor<=tmax+tshift),tmp,zz); // Position of current complicated
+}
+
+//template class WilsonFermion<WilsonImplD>; 
+
+NAMESPACE_END(Grid);

amd-omp-stack-err/WilsonFermionWorks2.cc

@@ -0,0 +1,615 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/WilsonFermion.cc
+
+Copyright (C) 2022
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Fabian Joswig <fabian.joswig@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/WilsonFermion.h>
+
+NAMESPACE_BEGIN(Grid);
+
+/////////////////////////////////
+// Constructor and gauge import
+/////////////////////////////////
+
+//template <class Impl>
+//WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
+//                                   GridRedBlackCartesian &Hgrid, RealD _mass,
+//                                   const ImplParams &p,
+//                                   const WilsonAnisotropyCoefficients &anis)
+//  :
+//    Kernels(p),
+//    _grid(&Fgrid),
+//    _cbgrid(&Hgrid),
+//    Stencil(&Fgrid, npoint, Even, directions, displacements,p),
+//    StencilEven(&Hgrid, npoint, Even, directions,displacements,p),  // source is Even
+//    StencilOdd(&Hgrid, npoint, Odd, directions,displacements,p),  // source is Odd
+//    mass(_mass),
+//    Lebesgue(_grid),
+//    LebesgueEvenOdd(_cbgrid),
+//    Umu(&Fgrid),
+//    UmuEven(&Hgrid),
+//    UmuOdd(&Hgrid),
+//      _tmp(&Hgrid),
+//      anisotropyCoeff(anis)
+//{
+//  Stencil.lo     = &Lebesgue;
+//  StencilEven.lo = &LebesgueEvenOdd;
+//  StencilOdd.lo  = &LebesgueEvenOdd;
+//  // Allocate the required comms buffer
+//  ImportGauge(_Umu);
+//  if  (anisotropyCoeff.isAnisotropic){
+//    diag_mass = mass + 1.0 + (Nd-1)*(anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//  } else {
+//    diag_mass = 4.0 + mass;
+//  }
+//
+//  int vol4;
+//  vol4=Fgrid.oSites();
+//  Stencil.BuildSurfaceList(1,vol4);
+//  vol4=Hgrid.oSites();
+//  StencilEven.BuildSurfaceList(1,vol4);
+//  StencilOdd.BuildSurfaceList(1,vol4);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::ImportGauge(const GaugeField &_Umu)
+//{
+//  GaugeField HUmu(_Umu.Grid());
+//
+//  //Here multiply the anisotropy coefficients
+//  if (anisotropyCoeff.isAnisotropic)
+//  {
+//
+//    for (int mu = 0; mu < Nd; mu++)
+//    {
+//      GaugeLinkField U_dir = (-0.5)*PeekIndex<LorentzIndex>(_Umu, mu);
+//      if (mu != anisotropyCoeff.t_direction)
+//        U_dir *= (anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//
+//      PokeIndex<LorentzIndex>(HUmu, U_dir, mu);
+//    }
+//  }
+//  else
+//  {
+//    HUmu = _Umu * (-0.5);
+//  }
+//  Impl::DoubleStore(GaugeGrid(), Umu, HUmu);
+//  pickCheckerboard(Even, UmuEven, Umu);
+//  pickCheckerboard(Odd, UmuOdd, Umu);
+//}
+//
+///////////////////////////////
+//// Implement the interface
+///////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::M(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerNo);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerYes);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Meooe(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerNo);
+//  } else {
+//    DhopOE(in, out, DaggerNo);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerYes);
+//  } else {
+//    DhopOE(in, out, DaggerYes);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mooee(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  typename FermionField::scalar_type scal(diag_mass);
+//  out = scal * in;
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MooeeDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Mooee(in, out);
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInv(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  out = (1.0/(diag_mass))*in;
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  MooeeInv(in,out);
+//}
+//template<class Impl>
+//void WilsonFermion<Impl>::MomentumSpacePropagator(FermionField &out, const FermionField &in,RealD _m,std::vector<double> twist)
+//{
+//  typedef typename FermionField::vector_type vector_type;
+//  typedef typename FermionField::scalar_type ScalComplex;
+//  typedef Lattice<iSinglet<vector_type> > LatComplex;
+//
+//  // what type LatticeComplex
+//  conformable(_grid,out.Grid());
+//
+//  Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT
+//  };
+//
+//  Coordinate latt_size   = _grid->_fdimensions;
+//
+//  FermionField   num  (_grid); num  = Zero();
+//  LatComplex    wilson(_grid); wilson= Zero();
+//  LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
+//
+//  LatComplex denom(_grid); denom= Zero();
+//  LatComplex kmu(_grid);
+//  ScalComplex ci(0.0,1.0);
+//  // momphase = n * 2pi / L
+//  for(int mu=0;mu<Nd;mu++) {
+//
+//    LatticeCoordinate(kmu,mu);
+//
+//    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+//
+//    kmu = TwoPiL * kmu;
+//    kmu = kmu + TwoPiL * one * twist[mu];//momentum for twisted boundary conditions
+//
+//    wilson = wilson + 2.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
+//
+//    num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);    // derivative term
+//
+//    denom=denom + sin(kmu)*sin(kmu);
+//  }
+//
+//  wilson = wilson + _m;     // 2 sin^2 k/2 + m
+//
+//  num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
+//
+//  denom= denom+wilson*wilson; // sin^2 k + (2 sin^2 k/2 + m)^2
+//
+//  denom= one/denom;
+//
+//  out = num*denom; // [ -i gmu sin k + 2 sin^2 k/2 + m] / [ sin^2 k + (2 sin^2 k/2 + m)^2 ]
+//
+//}
+//
+//
+/////////////////////////////////////
+//// Internal
+/////////////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
+//                                        GaugeField &mat, const FermionField &A,
+//                                        const FermionField &B, int dag) {
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//
+//  FermionField Btilde(B.Grid());
+//  FermionField Atilde(B.Grid());
+//  Atilde = A;
+//
+//  st.HaloExchange(B, compressor);
+//
+//  for (int mu = 0; mu < Nd; mu++) {
+//    ////////////////////////////////////////////////////////////////////////
+//    // Flip gamma (1+g)<->(1-g) if dag
+//    ////////////////////////////////////////////////////////////////////////
+//    int gamma = mu;
+//    if (!dag) gamma += Nd;
+//
+//    int Ls=1;
+//    Kernels::DhopDirKernel(st, U, st.CommBuf(), Ls, B.Grid()->oSites(), B, Btilde, mu, gamma);
+//
+//    //////////////////////////////////////////////////
+//    // spin trace outer product
+//    //////////////////////////////////////////////////
+//    Impl::InsertForce4D(mat, Btilde, Atilde, mu);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _grid);
+//  conformable(U.Grid(), V.Grid());
+//  conformable(U.Grid(), mat.Grid());
+//
+//  mat.Checkerboard() = U.Checkerboard();
+//
+//  DerivInternal(Stencil, Umu, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid()); not general, leaving as a comment (Guido)
+//  // Motivation: look at the SchurDiff operator
+//
+//  assert(V.Checkerboard() == Even);
+//  assert(U.Checkerboard() == Odd);
+//  mat.Checkerboard() = Odd;
+//
+//  DerivInternal(StencilEven, UmuOdd, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid());
+//
+//  assert(V.Checkerboard() == Odd);
+//  assert(U.Checkerboard() == Even);
+//  mat.Checkerboard() = Even;
+//
+//  DerivInternal(StencilOdd, UmuEven, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _grid);  // verifies full grid
+//  conformable(in.Grid(), out.Grid());
+//
+//  out.Checkerboard() = in.Checkerboard();
+//
+//  DhopInternal(Stencil, Lebesgue, Umu, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Even);
+//  out.Checkerboard() = Odd;
+//
+//  DhopInternal(StencilEven, LebesgueEvenOdd, UmuOdd, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Odd);
+//  out.Checkerboard() = Even;
+//
+//  DhopInternal(StencilOdd, LebesgueEvenOdd, UmuEven, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  DhopDir(in, out, dir, disp);
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::MdirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  DhopDirAll(in, out);
+//}
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  int skip = (disp == 1) ? 0 : 1;
+//  int dirdisp = dir + skip * 4;
+//  int gamma = dir + (1 - skip) * 4;
+//
+//  DhopDirCalc(in, out, dirdisp, gamma, DaggerNo);
+//};
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  assert((out.size()==8)||(out.size()==9));
+//  for(int dir=0;dir<Nd;dir++){
+//    for(int disp=-1;disp<=1;disp+=2){
+//
+//      int skip = (disp == 1) ? 0 : 1;
+//      int dirdisp = dir + skip * 4;
+//      int gamma = dir + (1 - skip) * 4;
+//
+//      DhopDirCalc(in, out[dirdisp], dirdisp, gamma, DaggerNo);
+//    }
+//  }
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirCalc(const FermionField &in, FermionField &out,int dirdisp, int gamma, int dag)
+//{
+//  int Ls=1;
+//  uint64_t Nsite=in.oSites();
+//  Kernels::DhopDirKernel(Stencil, Umu, Stencil.CommBuf(), Ls, Nsite, in, out, dirdisp, gamma);
+//};
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//#ifdef GRID_OMP
+//  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute )
+//    DhopInternalOverlappedComms(st,lo,U,in,out,dag);
+//  else
+//#endif
+//    DhopInternalSerial(st,lo,U,in,out,dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalOverlappedComms(StencilImpl &st, LebesgueOrder &lo,
+//						      DoubledGaugeField &U,
+//						      const FermionField &in,
+//						      FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopOverlapped");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//  int len =  U.Grid()->oSites();
+//
+//  /////////////////////////////
+//  // Start comms  // Gather intranode and extra node differentiated??
+//  /////////////////////////////
+//  std::vector<std::vector<CommsRequest_t> > requests;
+//  st.Prepare();
+//  {
+//    GRID_TRACE("Gather");
+//    st.HaloGather(in,compressor);
+//  }
+//
+//  tracePush("Communication");
+//  st.CommunicateBegin(requests);
+//
+//  /////////////////////////////
+//  // Overlap with comms
+//  /////////////////////////////
+//  {
+//    GRID_TRACE("MergeSHM");
+//    st.CommsMergeSHM(compressor);
+//  }
+//
+//  /////////////////////////////
+//  // do the compute interior
+//  /////////////////////////////
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagInterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  } else {
+//    GRID_TRACE("DhopInterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  }
+//
+//  /////////////////////////////
+//  // Complete comms
+//  /////////////////////////////
+//  st.CommunicateComplete(requests);
+//  tracePop("Communication");
+//
+//  {
+//    GRID_TRACE("Merge");
+//    st.CommsMerge(compressor);
+//  }
+//  /////////////////////////////
+//  // do the compute exterior
+//  /////////////////////////////
+//
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagExterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  } else {
+//    GRID_TRACE("DhopExterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  }
+//};
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalSerial(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopSerial");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//  Compressor compressor(dag);
+//  {
+//    GRID_TRACE("HaloExchange");
+//    st.HaloExchange(in, compressor);
+//  }
+//
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDag");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  } else {
+//    GRID_TRACE("Dhop");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  }
+//};
+///*Change ends */
+//
+///*******************************************************************************
+// * Conserved current utilities for Wilson fermions, for contracting propagators
+// * to make a conserved current sink or inserting the conserved current
+// * sequentially.
+// ******************************************************************************/
+//template <class Impl>
+//void WilsonFermion<Impl>::ContractConservedCurrent(PropagatorField &q_in_1,
+//                                                   PropagatorField &q_in_2,
+//                                                   PropagatorField &q_out,
+//                                                   PropagatorField &src,
+//                                                   Current curr_type,
+//                                                   unsigned int mu)
+//{
+//  if(curr_type != Current::Vector)
+//  {
+//    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+//    exit(1);
+//  }
+//
+//  Gamma g5(Gamma::Algebra::Gamma5);
+//  conformable(_grid, q_in_1.Grid());
+//  conformable(_grid, q_in_2.Grid());
+//  conformable(_grid, q_out.Grid());
+//  auto UGrid= this->GaugeGrid();
+//
+//  PropagatorField tmp_shifted(UGrid);
+//  PropagatorField g5Lg5(UGrid);
+//  PropagatorField R(UGrid);
+//  PropagatorField gmuR(UGrid);
+//
+//    Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT,
+//  };
+//  Gamma gmu=Gamma(Gmu[mu]);
+//
+//  g5Lg5=g5*q_in_1*g5;
+//  tmp_shifted=Cshift(q_in_2,mu,1);
+//  Impl::multLinkField(R,this->Umu,tmp_shifted,mu);
+//  gmuR=gmu*R;
+//
+//  q_out=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//
+//  tmp_shifted=Cshift(q_in_1,mu,1);
+//  Impl::multLinkField(g5Lg5,this->Umu,tmp_shifted,mu);
+//  g5Lg5=g5*g5Lg5*g5;
+//  R=q_in_2;
+//  gmuR=gmu*R;
+//
+//  q_out-=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//}
+//
+template <class Impl>
+void WilsonFermion<Impl>::SeqConservedCurrent(PropagatorField &q_in,
+                                              PropagatorField &q_out,
+                                              PropagatorField &src,
+                                              Current curr_type,
+                                              unsigned int mu,
+                                              unsigned int tmin,
+                                              unsigned int tmax,
+					      ComplexField &lattice_cmplx)
+{
+  if(curr_type != Current::Vector)
+  {
+    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+    exit(1);
+  }
+
+  int tshift = (mu == Nd-1) ? 1 : 0;
+  unsigned int LLt    = GridDefaultLatt()[Tp];
+  conformable(_grid, q_in.Grid());
+  conformable(_grid, q_out.Grid());
+  auto UGrid= this->GaugeGrid();
+
+  PropagatorField tmp(UGrid);
+  PropagatorField Utmp(UGrid);
+  PropagatorField L(UGrid);
+  PropagatorField zz (UGrid);
+  zz=Zero();
+  LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
+
+    Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT,
+  };
+  Gamma gmu=Gamma(Gmu[mu]);
+
+  tmp = Cshift(q_in,mu,1);
+  Impl::multLinkField(Utmp,this->Umu,tmp,mu);
+  tmp = ( Utmp*lattice_cmplx - gmu*Utmp*lattice_cmplx ); // Forward hop
+//  tmp = where((lcoor>=tmin),tmp,zz); // Mask the time
+//  q_out = where((lcoor<=tmax),tmp,zz); // Position of current complicated
+//
+//  tmp = q_in *lattice_cmplx;
+//  tmp = Cshift(tmp,mu,-1);
+//  Impl::multLinkField(Utmp,this->Umu,tmp,mu+Nd); // Adjoint link
+//  tmp = -( Utmp + gmu*Utmp );
+//  // Mask the time
+//  if (tmax == LLt - 1 && tshift == 1){ // quick fix to include timeslice 0 if tmax + tshift is over the last timeslice
+//    unsigned int t0 = 0;
+//    tmp = where(((lcoor==t0) || (lcoor>=tmin+tshift)),tmp,zz);
+//  } else {
+//    tmp = where((lcoor>=tmin+tshift),tmp,zz);
+//  }
+//  q_out+= where((lcoor<=tmax+tshift),tmp,zz); // Position of current complicated
+}
+
+template class WilsonFermion<WilsonImplD>; 
+
+NAMESPACE_END(Grid);

amd-omp-stack-err/WilsonFermionWorks3.cc

@@ -0,0 +1,615 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/WilsonFermion.cc
+
+Copyright (C) 2022
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Fabian Joswig <fabian.joswig@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/WilsonFermion.h>
+
+NAMESPACE_BEGIN(Grid);
+
+/////////////////////////////////
+// Constructor and gauge import
+/////////////////////////////////
+
+//template <class Impl>
+//WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
+//                                   GridRedBlackCartesian &Hgrid, RealD _mass,
+//                                   const ImplParams &p,
+//                                   const WilsonAnisotropyCoefficients &anis)
+//  :
+//    Kernels(p),
+//    _grid(&Fgrid),
+//    _cbgrid(&Hgrid),
+//    Stencil(&Fgrid, npoint, Even, directions, displacements,p),
+//    StencilEven(&Hgrid, npoint, Even, directions,displacements,p),  // source is Even
+//    StencilOdd(&Hgrid, npoint, Odd, directions,displacements,p),  // source is Odd
+//    mass(_mass),
+//    Lebesgue(_grid),
+//    LebesgueEvenOdd(_cbgrid),
+//    Umu(&Fgrid),
+//    UmuEven(&Hgrid),
+//    UmuOdd(&Hgrid),
+//      _tmp(&Hgrid),
+//      anisotropyCoeff(anis)
+//{
+//  Stencil.lo     = &Lebesgue;
+//  StencilEven.lo = &LebesgueEvenOdd;
+//  StencilOdd.lo  = &LebesgueEvenOdd;
+//  // Allocate the required comms buffer
+//  ImportGauge(_Umu);
+//  if  (anisotropyCoeff.isAnisotropic){
+//    diag_mass = mass + 1.0 + (Nd-1)*(anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//  } else {
+//    diag_mass = 4.0 + mass;
+//  }
+//
+//  int vol4;
+//  vol4=Fgrid.oSites();
+//  Stencil.BuildSurfaceList(1,vol4);
+//  vol4=Hgrid.oSites();
+//  StencilEven.BuildSurfaceList(1,vol4);
+//  StencilOdd.BuildSurfaceList(1,vol4);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::ImportGauge(const GaugeField &_Umu)
+//{
+//  GaugeField HUmu(_Umu.Grid());
+//
+//  //Here multiply the anisotropy coefficients
+//  if (anisotropyCoeff.isAnisotropic)
+//  {
+//
+//    for (int mu = 0; mu < Nd; mu++)
+//    {
+//      GaugeLinkField U_dir = (-0.5)*PeekIndex<LorentzIndex>(_Umu, mu);
+//      if (mu != anisotropyCoeff.t_direction)
+//        U_dir *= (anisotropyCoeff.nu / anisotropyCoeff.xi_0);
+//
+//      PokeIndex<LorentzIndex>(HUmu, U_dir, mu);
+//    }
+//  }
+//  else
+//  {
+//    HUmu = _Umu * (-0.5);
+//  }
+//  Impl::DoubleStore(GaugeGrid(), Umu, HUmu);
+//  pickCheckerboard(Even, UmuEven, Umu);
+//  pickCheckerboard(Odd, UmuOdd, Umu);
+//}
+//
+///////////////////////////////
+//// Implement the interface
+///////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::M(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerNo);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Dhop(in, out, DaggerYes);
+//  axpy(out, diag_mass, in, out);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Meooe(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerNo);
+//  } else {
+//    DhopOE(in, out, DaggerNo);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out)
+//{
+//  if (in.Checkerboard() == Odd) {
+//    DhopEO(in, out, DaggerYes);
+//  } else {
+//    DhopOE(in, out, DaggerYes);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mooee(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  typename FermionField::scalar_type scal(diag_mass);
+//  out = scal * in;
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::MooeeDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  Mooee(in, out);
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInv(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  out = (1.0/(diag_mass))*in;
+//}
+//
+//template<class Impl>
+//void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in, FermionField &out)
+//{
+//  out.Checkerboard() = in.Checkerboard();
+//  MooeeInv(in,out);
+//}
+//template<class Impl>
+//void WilsonFermion<Impl>::MomentumSpacePropagator(FermionField &out, const FermionField &in,RealD _m,std::vector<double> twist)
+//{
+//  typedef typename FermionField::vector_type vector_type;
+//  typedef typename FermionField::scalar_type ScalComplex;
+//  typedef Lattice<iSinglet<vector_type> > LatComplex;
+//
+//  // what type LatticeComplex
+//  conformable(_grid,out.Grid());
+//
+//  Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT
+//  };
+//
+//  Coordinate latt_size   = _grid->_fdimensions;
+//
+//  FermionField   num  (_grid); num  = Zero();
+//  LatComplex    wilson(_grid); wilson= Zero();
+//  LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
+//
+//  LatComplex denom(_grid); denom= Zero();
+//  LatComplex kmu(_grid);
+//  ScalComplex ci(0.0,1.0);
+//  // momphase = n * 2pi / L
+//  for(int mu=0;mu<Nd;mu++) {
+//
+//    LatticeCoordinate(kmu,mu);
+//
+//    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+//
+//    kmu = TwoPiL * kmu;
+//    kmu = kmu + TwoPiL * one * twist[mu];//momentum for twisted boundary conditions
+//
+//    wilson = wilson + 2.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
+//
+//    num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);    // derivative term
+//
+//    denom=denom + sin(kmu)*sin(kmu);
+//  }
+//
+//  wilson = wilson + _m;     // 2 sin^2 k/2 + m
+//
+//  num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
+//
+//  denom= denom+wilson*wilson; // sin^2 k + (2 sin^2 k/2 + m)^2
+//
+//  denom= one/denom;
+//
+//  out = num*denom; // [ -i gmu sin k + 2 sin^2 k/2 + m] / [ sin^2 k + (2 sin^2 k/2 + m)^2 ]
+//
+//}
+//
+//
+/////////////////////////////////////
+//// Internal
+/////////////////////////////////////
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
+//                                        GaugeField &mat, const FermionField &A,
+//                                        const FermionField &B, int dag) {
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//
+//  FermionField Btilde(B.Grid());
+//  FermionField Atilde(B.Grid());
+//  Atilde = A;
+//
+//  st.HaloExchange(B, compressor);
+//
+//  for (int mu = 0; mu < Nd; mu++) {
+//    ////////////////////////////////////////////////////////////////////////
+//    // Flip gamma (1+g)<->(1-g) if dag
+//    ////////////////////////////////////////////////////////////////////////
+//    int gamma = mu;
+//    if (!dag) gamma += Nd;
+//
+//    int Ls=1;
+//    Kernels::DhopDirKernel(st, U, st.CommBuf(), Ls, B.Grid()->oSites(), B, Btilde, mu, gamma);
+//
+//    //////////////////////////////////////////////////
+//    // spin trace outer product
+//    //////////////////////////////////////////////////
+//    Impl::InsertForce4D(mat, Btilde, Atilde, mu);
+//  }
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _grid);
+//  conformable(U.Grid(), V.Grid());
+//  conformable(U.Grid(), mat.Grid());
+//
+//  mat.Checkerboard() = U.Checkerboard();
+//
+//  DerivInternal(Stencil, Umu, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid()); not general, leaving as a comment (Guido)
+//  // Motivation: look at the SchurDiff operator
+//
+//  assert(V.Checkerboard() == Even);
+//  assert(U.Checkerboard() == Odd);
+//  mat.Checkerboard() = Odd;
+//
+//  DerivInternal(StencilEven, UmuOdd, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U, const FermionField &V, int dag)
+//{
+//  conformable(U.Grid(), _cbgrid);
+//  conformable(U.Grid(), V.Grid());
+//  //conformable(U.Grid(), mat.Grid());
+//
+//  assert(V.Checkerboard() == Odd);
+//  assert(U.Checkerboard() == Even);
+//  mat.Checkerboard() = Even;
+//
+//  DerivInternal(StencilOdd, UmuEven, mat, U, V, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _grid);  // verifies full grid
+//  conformable(in.Grid(), out.Grid());
+//
+//  out.Checkerboard() = in.Checkerboard();
+//
+//  DhopInternal(Stencil, Lebesgue, Umu, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Even);
+//  out.Checkerboard() = Odd;
+//
+//  DhopInternal(StencilEven, LebesgueEvenOdd, UmuOdd, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
+//{
+//  conformable(in.Grid(), _cbgrid);    // verifies half grid
+//  conformable(in.Grid(), out.Grid());  // drops the cb check
+//
+//  assert(in.Checkerboard() == Odd);
+//  out.Checkerboard() = Even;
+//
+//  DhopInternal(StencilOdd, LebesgueEvenOdd, UmuEven, in, out, dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::Mdir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  DhopDir(in, out, dir, disp);
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::MdirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  DhopDirAll(in, out);
+//}
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDir(const FermionField &in, FermionField &out, int dir, int disp)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  int skip = (disp == 1) ? 0 : 1;
+//  int dirdisp = dir + skip * 4;
+//  int gamma = dir + (1 - skip) * 4;
+//
+//  DhopDirCalc(in, out, dirdisp, gamma, DaggerNo);
+//};
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirAll(const FermionField &in, std::vector<FermionField> &out)
+//{
+//  Compressor compressor(DaggerNo);
+//  Stencil.HaloExchange(in, compressor);
+//
+//  assert((out.size()==8)||(out.size()==9));
+//  for(int dir=0;dir<Nd;dir++){
+//    for(int disp=-1;disp<=1;disp+=2){
+//
+//      int skip = (disp == 1) ? 0 : 1;
+//      int dirdisp = dir + skip * 4;
+//      int gamma = dir + (1 - skip) * 4;
+//
+//      DhopDirCalc(in, out[dirdisp], dirdisp, gamma, DaggerNo);
+//    }
+//  }
+//}
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopDirCalc(const FermionField &in, FermionField &out,int dirdisp, int gamma, int dag)
+//{
+//  int Ls=1;
+//  uint64_t Nsite=in.oSites();
+//  Kernels::DhopDirKernel(Stencil, Umu, Stencil.CommBuf(), Ls, Nsite, in, out, dirdisp, gamma);
+//};
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//#ifdef GRID_OMP
+//  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute )
+//    DhopInternalOverlappedComms(st,lo,U,in,out,dag);
+//  else
+//#endif
+//    DhopInternalSerial(st,lo,U,in,out,dag);
+//}
+//
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalOverlappedComms(StencilImpl &st, LebesgueOrder &lo,
+//						      DoubledGaugeField &U,
+//						      const FermionField &in,
+//						      FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopOverlapped");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//
+//  Compressor compressor(dag);
+//  int len =  U.Grid()->oSites();
+//
+//  /////////////////////////////
+//  // Start comms  // Gather intranode and extra node differentiated??
+//  /////////////////////////////
+//  std::vector<std::vector<CommsRequest_t> > requests;
+//  st.Prepare();
+//  {
+//    GRID_TRACE("Gather");
+//    st.HaloGather(in,compressor);
+//  }
+//
+//  tracePush("Communication");
+//  st.CommunicateBegin(requests);
+//
+//  /////////////////////////////
+//  // Overlap with comms
+//  /////////////////////////////
+//  {
+//    GRID_TRACE("MergeSHM");
+//    st.CommsMergeSHM(compressor);
+//  }
+//
+//  /////////////////////////////
+//  // do the compute interior
+//  /////////////////////////////
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagInterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  } else {
+//    GRID_TRACE("DhopInterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+//  }
+//
+//  /////////////////////////////
+//  // Complete comms
+//  /////////////////////////////
+//  st.CommunicateComplete(requests);
+//  tracePop("Communication");
+//
+//  {
+//    GRID_TRACE("Merge");
+//    st.CommsMerge(compressor);
+//  }
+//  /////////////////////////////
+//  // do the compute exterior
+//  /////////////////////////////
+//
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDagExterior");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  } else {
+//    GRID_TRACE("DhopExterior");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,0,1);
+//  }
+//};
+////
+//template <class Impl>
+//void WilsonFermion<Impl>::DhopInternalSerial(StencilImpl &st, LebesgueOrder &lo,
+//                                       DoubledGaugeField &U,
+//                                       const FermionField &in,
+//                                       FermionField &out, int dag)
+//{
+//  GRID_TRACE("DhopSerial");
+//  assert((dag == DaggerNo) || (dag == DaggerYes));
+//  Compressor compressor(dag);
+//  {
+//    GRID_TRACE("HaloExchange");
+//    st.HaloExchange(in, compressor);
+//  }
+//
+//  int Opt = WilsonKernelsStatic::Opt;
+//  if (dag == DaggerYes) {
+//    GRID_TRACE("DhopDag");
+//    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  } else {
+//    GRID_TRACE("Dhop");
+//    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out);
+//  }
+//};
+///*Change ends */
+//
+///*******************************************************************************
+// * Conserved current utilities for Wilson fermions, for contracting propagators
+// * to make a conserved current sink or inserting the conserved current
+// * sequentially.
+// ******************************************************************************/
+//template <class Impl>
+//void WilsonFermion<Impl>::ContractConservedCurrent(PropagatorField &q_in_1,
+//                                                   PropagatorField &q_in_2,
+//                                                   PropagatorField &q_out,
+//                                                   PropagatorField &src,
+//                                                   Current curr_type,
+//                                                   unsigned int mu)
+//{
+//  if(curr_type != Current::Vector)
+//  {
+//    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+//    exit(1);
+//  }
+//
+//  Gamma g5(Gamma::Algebra::Gamma5);
+//  conformable(_grid, q_in_1.Grid());
+//  conformable(_grid, q_in_2.Grid());
+//  conformable(_grid, q_out.Grid());
+//  auto UGrid= this->GaugeGrid();
+//
+//  PropagatorField tmp_shifted(UGrid);
+//  PropagatorField g5Lg5(UGrid);
+//  PropagatorField R(UGrid);
+//  PropagatorField gmuR(UGrid);
+//
+//    Gamma::Algebra Gmu [] = {
+//    Gamma::Algebra::GammaX,
+//    Gamma::Algebra::GammaY,
+//    Gamma::Algebra::GammaZ,
+//    Gamma::Algebra::GammaT,
+//  };
+//  Gamma gmu=Gamma(Gmu[mu]);
+//
+//  g5Lg5=g5*q_in_1*g5;
+//  tmp_shifted=Cshift(q_in_2,mu,1);
+//  Impl::multLinkField(R,this->Umu,tmp_shifted,mu);
+//  gmuR=gmu*R;
+//
+//  q_out=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//
+//  tmp_shifted=Cshift(q_in_1,mu,1);
+//  Impl::multLinkField(g5Lg5,this->Umu,tmp_shifted,mu);
+//  g5Lg5=g5*g5Lg5*g5;
+//  R=q_in_2;
+//  gmuR=gmu*R;
+//
+//  q_out-=adj(g5Lg5)*R;
+//  q_out-=adj(g5Lg5)*gmuR;
+//}
+using Impl = WilsonImplD;
+template <class Impl>
+void WilsonFermion<Impl>::SeqConservedCurrent(PropagatorField &q_in,
+                                              PropagatorField &q_out,
+                                              PropagatorField &src,
+                                              Current curr_type,
+                                              unsigned int mu,
+                                              unsigned int tmin,
+                                              unsigned int tmax,
+					      ComplexField &lattice_cmplx)
+{
+  if(curr_type != Current::Vector)
+  {
+    std::cout << GridLogError << "Only the conserved vector current is implemented so far." << std::endl;
+    exit(1);
+  }
+
+  int tshift = (mu == Nd-1) ? 1 : 0;
+  unsigned int LLt    = GridDefaultLatt()[Tp];
+  conformable(_grid, q_in.Grid());
+  conformable(_grid, q_out.Grid());
+  auto UGrid= this->GaugeGrid();
+
+  PropagatorField tmp(UGrid);
+  PropagatorField Utmp(UGrid);
+  PropagatorField L(UGrid);
+  PropagatorField zz (UGrid);
+  zz=Zero();
+  LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
+
+    Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT,
+  };
+  Gamma gmu=Gamma(Gmu[mu]);
+
+  tmp = Cshift(q_in,mu,1);
+  Impl::multLinkField(Utmp,this->Umu,tmp,mu);
+  tmp = ( Utmp*lattice_cmplx - gmu*Utmp*lattice_cmplx ); // Forward hop
+  tmp = where((lcoor>=tmin),tmp,zz); // Mask the time
+//  q_out = where((lcoor<=tmax),tmp,zz); // Position of current complicated
+//
+//  tmp = q_in *lattice_cmplx;
+//  tmp = Cshift(tmp,mu,-1);
+//  Impl::multLinkField(Utmp,this->Umu,tmp,mu+Nd); // Adjoint link
+//  tmp = -( Utmp + gmu*Utmp );
+//  // Mask the time
+//  if (tmax == LLt - 1 && tshift == 1){ // quick fix to include timeslice 0 if tmax + tshift is over the last timeslice
+//    unsigned int t0 = 0;
+//    tmp = where(((lcoor==t0) || (lcoor>=tmin+tshift)),tmp,zz);
+//  } else {
+//    tmp = where((lcoor>=tmin+tshift),tmp,zz);
+//  }
+//  q_out+= where((lcoor<=tmax+tshift),tmp,zz); // Position of current complicated
+}
+
+//template class WilsonFermion<WilsonImplD>; 
+
+NAMESPACE_END(Grid);

amd-omp-stack-err/objdump_fails.txt

@@ -0,0 +1,586 @@
+
+libWilsonFails.a(WilsonFails.o):	file format elf64-x86-64
+
+SYMBOL TABLE:
+0000000000000000 l    df *ABS*	0000000000000000 WilsonFermionInstantiationWilsonImplD.cc
+0000000000000000 l    d  .text	0000000000000000 .text
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv
+0000000000000000 l    d  .text._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	0000000000000000 .text._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	0000000000000000 .gcc_except_table._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_
+0000000000000000 l       .gcc_except_table._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	0000000000000000 GCC_except_table1
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	0000000000000000 GCC_except_table3
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv
+0000000000000000 l    d  .text._ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv	0000000000000000 .text._ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l       .rodata.cst8	0000000000000000 .LCPI20_0
+0000000000000008 l       .rodata.cst8	0000000000000000 .LCPI20_1
+0000000000000000 l       .rodata.cst16	0000000000000000 .LCPI20_2
+0000000000000010 l       .rodata.cst16	0000000000000000 .LCPI20_3
+0000000000000020 l       .rodata.cst16	0000000000000000 .LCPI20_4
+0000000000000030 l       .rodata.cst16	0000000000000000 .LCPI20_5
+0000000000000040 l       .rodata.cst16	0000000000000000 .LCPI20_6
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE
+0000000000000000 l    d  .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000000 .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE
+0000000000000000 l       .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000000 GCC_except_table20
+0000000000000010 l       .rodata.cst8	0000000000000000 .LCPI21_0
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d
+0000000000000000 l    d  .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000000 .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d
+0000000000000000 l       .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000000 GCC_except_table21
+0000000000000050 l       .rodata.cst16	0000000000000000 .LCPI22_0
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l    d  .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l       .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 GCC_except_table22
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4GridlsERSoRNS_6LoggerE	0000000000000000 .text._ZN4GridlsERSoRNS_6LoggerE
+0000000000000000 l    d  .gcc_except_table._ZN4GridlsERSoRNS_6LoggerE	0000000000000000 .gcc_except_table._ZN4GridlsERSoRNS_6LoggerE
+0000000000000000 l       .gcc_except_table._ZN4GridlsERSoRNS_6LoggerE	0000000000000000 GCC_except_table31
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l    d  .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l       .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 GCC_except_table35
+0000000000000000 l    d  .text._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	0000000000000000 .text._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii
+0000000000000000 l    d  .gcc_except_table._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	0000000000000000 .gcc_except_table._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii
+0000000000000000 l       .gcc_except_table._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	0000000000000000 GCC_except_table36
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 GCC_except_table38
+0000000000000000 l    d  .text._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000000 .text._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i
+0000000000000000 l     F .text	000000000000010d .omp_outlined.
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000000 .gcc_except_table._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i
+0000000000000000 l       .gcc_except_table._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000000 GCC_except_table39
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE
+0000000000001160 l     F .text	00000000000000e5 .omp_outlined..44
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE
+000000000000bf90 l     F .text	00000000000000e5 .omp_outlined..52
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEED2Ev	0000000000000000 GCC_except_table42
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table43
+0000000000000000 l    d  .text._ZN4Grid6Logger10backgroundB5cxx11Ev	0000000000000000 .text._ZN4Grid6Logger10backgroundB5cxx11Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid6Logger10backgroundB5cxx11Ev	0000000000000000 .gcc_except_table._ZN4Grid6Logger10backgroundB5cxx11Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid6Logger10backgroundB5cxx11Ev	0000000000000000 GCC_except_table44
+0000000000000000 l    d  .text._ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE	0000000000000000 .text._ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE
+0000000000000000 l    d  .text._ZN4Grid6Logger8evidenceB5cxx11Ev	0000000000000000 .text._ZN4Grid6Logger8evidenceB5cxx11Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid6Logger8evidenceB5cxx11Ev	0000000000000000 .gcc_except_table._ZN4Grid6Logger8evidenceB5cxx11Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid6Logger8evidenceB5cxx11Ev	0000000000000000 GCC_except_table46
+0000000000000000 l    d  .text._ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_	0000000000000000 .text._ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_
+0000000000000100 l     O .rodata	0000000000000001 _ZStL19piecewise_construct
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_
+0000000000000000 l    d  .gcc_except_table._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	0000000000000000 .gcc_except_table._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_
+0000000000000000 l       .gcc_except_table._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	0000000000000000 GCC_except_table49
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 GCC_except_table55
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 GCC_except_table56
+0000000000000000 l    d  .text._ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z	0000000000000000 .text._ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	0000000000000000 GCC_except_table63
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	0000000000000000 GCC_except_table64
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table69
+0000000000000000 l    d  .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii	0000000000000000 .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii
+0000000000000060 l       .rodata.cst16	0000000000000000 .LCPI71_0
+0000000000000000 l    d  .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii	0000000000000000 .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii
+0000000000000000 l    d  .text._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000000 .text._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii
+0000000000000000 l    d  .gcc_except_table._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000000 .gcc_except_table._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii
+0000000000000000 l       .gcc_except_table._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000000 GCC_except_table72
+0000000000000000 l    d  .text._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	0000000000000000 .text._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	0000000000000000 .gcc_except_table._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii
+0000000000000000 l       .gcc_except_table._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	0000000000000000 GCC_except_table73
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	0000000000000000 GCC_except_table74
+0000000000000000 l    d  .text._ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm	0000000000000000 .text._ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm
+0000000000000000 l    d  .text._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	0000000000000000 .text._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	0000000000000000 GCC_except_table76
+0000000000000000 l    d  .text._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .text._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l       .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 GCC_except_table77
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE
+0000000000000110 l     F .text	000000000000104c .omp_outlined..41
+0000000000000070 l       .rodata.cst16	0000000000000000 .LCPI80_0
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	0000000000000000 GCC_except_table81
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE
+0000000000001250 l     F .text	000000000000ad36 .omp_outlined..45
+0000000000000080 l       .rodata.cst16	0000000000000000 .LCPI87_0
+0000000000000000 l    d  .rodata	0000000000000000 .rodata
+0000000000000000 l    d  .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000000 .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+0000000000000000 l    d  .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_	0000000000000000 .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EE
+0000000000000090 l       .rodata.cst16	0000000000000000 .LCPI92_0
+00000000000000a0 l       .rodata.cst16	0000000000000000 .LCPI92_1
+0000000000000000 l    d  .text._ZN4Grid12LSComparisonINS_3vgeINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEjEESA_jEENS_7LatticeIS8_EET_RKNSC_IT0_EERKT1_	0000000000000000 .text._ZN4Grid12LSComparisonINS_3vgeINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEjEESA_jEENS_7LatticeIS8_EET_RKNSC_IT0_EERKT1_
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12LSComparisonINS_3vgeINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEjEESA_jEENS_7LatticeIS8_EET_RKNSC_IT0_EERKT1_	0000000000000000 .gcc_except_table._ZN4Grid12LSComparisonINS_3vgeINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEjEESA_jEENS_7LatticeIS8_EET_RKNSC_IT0_EERKT1_
+0000000000000000 l       .gcc_except_table._ZN4Grid12LSComparisonINS_3vgeINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEjEESA_jEENS_7LatticeIS8_EET_RKNSC_IT0_EERKT1_	0000000000000000 GCC_except_table92
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8allocateEmPKv	0000000000000000 GCC_except_table97
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE10deallocateEPS7_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE10deallocateEPS7_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE10deallocateEPS7_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE10deallocateEPS7_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE10deallocateEPS7_m	0000000000000000 GCC_except_table98
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8ViewOpenENS_8ViewModeE
+000000000000c080 l     F .text	000000000000046a .omp_outlined..53
+0000000000000000 l    d  .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEELPS9_0EEEvRiRKT_	0000000000000000 .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEELPS9_0EEEvRiRKT_
+0000000000000000 l    d  .text._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000000 .text._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000000 GCC_except_table104
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 GCC_except_table105
+0000000000000000 l    d  .text._ZN4Grid12uvmAllocatorIiE10deallocateEPim	0000000000000000 .text._ZN4Grid12uvmAllocatorIiE10deallocateEPim
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12uvmAllocatorIiE10deallocateEPim	0000000000000000 .gcc_except_table._ZN4Grid12uvmAllocatorIiE10deallocateEPim
+0000000000000000 l       .gcc_except_table._ZN4Grid12uvmAllocatorIiE10deallocateEPim	0000000000000000 GCC_except_table106
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	0000000000000000 GCC_except_table107
+0000000000000000 l    d  .text._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .text._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .gcc_except_table._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l       .gcc_except_table._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 GCC_except_table108
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 GCC_except_table109
+0000000000000000 l    d  .text._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .text._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .gcc_except_table._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l       .gcc_except_table._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 GCC_except_table110
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 GCC_except_table111
+0000000000000000 l    d  .text.startup	0000000000000000 .text.startup
+0000000000000000 l     F .text.startup	0000000000000001 __cxx_global_var_init.56
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 GCC_except_table116
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE
+00000000000000b0 l       .rodata.cst16	0000000000000000 .LCPI123_0
+0000000000000000 l    d  .text._ZN4Grid3FFTC2EPNS_13GridCartesianE	0000000000000000 .text._ZN4Grid3FFTC2EPNS_13GridCartesianE
+0000000000000000 l    d  .gcc_except_table._ZN4Grid3FFTC2EPNS_13GridCartesianE	0000000000000000 .gcc_except_table._ZN4Grid3FFTC2EPNS_13GridCartesianE
+0000000000000000 l       .gcc_except_table._ZN4Grid3FFTC2EPNS_13GridCartesianE	0000000000000000 GCC_except_table123
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000 l    d  .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l    d  .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l       .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 GCC_except_table127
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE
+000000000000c4f0 l     F .text	00000000000000e5 .omp_outlined..63
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE
+000000000000c730 l     F .text	00000000000000e5 .omp_outlined..67
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE
+000000000000cdd0 l     F .text	00000000000000e5 .omp_outlined..79
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table131
+00000000000000c0 l       .rodata.cst16	0000000000000000 .LCPI132_0
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_	0000000000000000 .text._ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_
+0000000000000000 l    d  .text._ZN4Grid13GridCartesianD0Ev	0000000000000000 .text._ZN4Grid13GridCartesianD0Ev
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian14CheckerBoardedEi	0000000000000000 .text._ZN4Grid13GridCartesian14CheckerBoardedEi
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian23CheckerBoardDestinationEiii	0000000000000000 .text._ZN4Grid13GridCartesian23CheckerBoardDestinationEiii
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian17CheckerBoardShiftEiiii	0000000000000000 .text._ZN4Grid13GridCartesian17CheckerBoardShiftEiiii
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii	0000000000000000 .text._ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian22CheckerBoardFromOindexEi	0000000000000000 .text._ZN4Grid13GridCartesian22CheckerBoardFromOindexEi
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi	0000000000000000 .text._ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi
+0000000000000000 l    d  .text._ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	0000000000000000 GCC_except_table145
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	0000000000000000 GCC_except_table146
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table150
+000000000000c5e0 l     F .text	000000000000014d .omp_outlined..64
+0000000000000018 l       .rodata.cst8	0000000000000000 .LCPI152_0
+00000000000000d0 l       .rodata.cst16	0000000000000000 .LCPI152_1
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+000000000000c820 l     F .text	0000000000000346 .omp_outlined..68
+0000000000000020 l       .rodata.cst8	0000000000000000 .LCPI155_0
+0000000000000028 l       .rodata.cst8	0000000000000000 .LCPI155_1
+0000000000000000 l    d  .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000000 .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+0000000000000000 l    d  .text._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	0000000000000000 .text._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi
+0000000000000000 l    d  .gcc_except_table._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	0000000000000000 .gcc_except_table._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi
+0000000000000000 l       .gcc_except_table._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	0000000000000000 GCC_except_table157
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_
+000000000000cb70 l     F .text	00000000000000e5 .omp_outlined..71
+0000000000000000 l    d  .text._ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE	0000000000000000 .text._ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE
+000000000000cc60 l     F .text	0000000000000166 .omp_outlined..72
+000000000000cec0 l     F .text	0000000000000462 .omp_outlined..80
+00000000000000e0 l       .rodata.cst16	0000000000000000 .LCPI163_0
+0000000000000010 l     F .text.startup	0000000000000021 _GLOBAL__sub_I_WilsonFermionInstantiationWilsonImplD.cc
+0000000000000000 l     O .bss	0000000000000001 _ZStL8__ioinit
+0000000000000040 l     F .text.startup	000000000000000a .omp_offloading.requires_reg
+0000000000000000 l    d  .bss	0000000000000000 .bss
+0000000000000001 l     O .bss	0000000000000001 _ZN5EigenL4lastE
+0000000000000002 l     O .bss	0000000000000002 _ZN5EigenL6lastp1E
+0000000000000004 l     O .bss	0000000000000001 _ZN5EigenL3fixILi1EEE
+0000000000000005 l     O .bss	0000000000000001 _ZN5EigenL3allE
+0000000000000000 l    d  .rodata.str1.1	0000000000000000 .rodata.str1.1
+0000000000000000 l     O .rodata.str1.16	0000000000000114 .omp_offloading.entry_name
+0000000000000000 l    d  .rodata.str1.16	0000000000000000 .rodata.str1.16
+0000000000000120 l     O .rodata.str1.16	0000000000000131 .omp_offloading.entry_name.84
+0000000000000260 l     O .rodata.str1.16	0000000000000102 .omp_offloading.entry_name.85
+0000000000000370 l     O .rodata.str1.16	00000000000000ee .omp_offloading.entry_name.86
+0000000000000460 l     O .rodata.str1.16	000000000000013b .omp_offloading.entry_name.87
+00000000000005a0 l     O .rodata.str1.16	000000000000009e .omp_offloading.entry_name.88
+0000000000000640 l     O .rodata.str1.16	0000000000000130 .omp_offloading.entry_name.89
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv	0000000000000009 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv
+0000000000000000  w    F .text._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	000000000000008b _ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	000000000000100e _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000  w    F .text.__clang_call_terminate	000000000000000b .hidden __clang_call_terminate
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000         *UND*	0000000000000000 _Unwind_Resume
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000009 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	000000000000015b _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev
+0000000000000000  w    O .rodata._ZTVN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	00000000000001a8 _ZTVN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    F .text._ZN4Grid12uvmAllocatorIiE10deallocateEPim	000000000000100f _ZN4Grid12uvmAllocatorIiE10deallocateEPim
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	000000000000100e _ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m
+0000000000000000  w    F .text._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000256 _ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev	0000000000000012 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev
+0000000000000000         *UND*	0000000000000000 _ZdlPv
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv	0000000000000009 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv	0000000000000009 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv
+0000000000000000  w    F .text._ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv	0000000000000006 _ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv	0000000000000006 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE	000000000000001a _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000         *UND*	0000000000000000 __assert_fail
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000c _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000c _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000c _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_	00000000000000e3 _ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000e2f _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE
+0000000000000000  w    F .text._ZN4Grid3FFTC2EPNS_13GridCartesianE	000000000000024b _ZN4Grid3FFTC2EPNS_13GridCartesianE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_	00000000000000f7 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000  w    F .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	000000000000020a _ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000         *UND*	0000000000000000 acos
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE	00000000000000ab _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE	00000000000003d2 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	00000000000000ab _ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE	00000000000003d2 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	00000000000000ce _ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi
+0000000000000000         *UND*	0000000000000000 _Znwm
+0000000000000000         *UND*	0000000000000000 memmove
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE	00000000000003d2 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	000000000000100f _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m
+0000000000000000         *UND*	0000000000000000 _ZSt17__throw_bad_allocv
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000222 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000508 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid15GridDefaultLattEv
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000320 _ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000  w    F .text._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	000000000000018c _ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_	0000000000000168 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	000000000000100e _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000  w    F .text._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000277 _ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	00000000000000ab _ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE	0000000000000592 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid12LSComparisonINS_3vgeINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEjEESA_jEENS_7LatticeIS8_EET_RKNSC_IT0_EERKT1_	0000000000000132 _ZN4Grid12LSComparisonINS_3vgeINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEjEESA_jEENS_7LatticeIS8_EET_RKNSC_IT0_EERKT1_
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EE	00000000000000ab _ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE	00000000000004ab _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE10deallocateEPS7_m	000000000000100f _ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE10deallocateEPS7_m
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	000000000000100f _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m
+0000000000000000         *UND*	0000000000000000 _ZSt4cout
+0000000000000000         *UND*	0000000000000000 _ZN4Grid12GridLogErrorE
+0000000000000000  w    F .text._ZN4GridlsERSoRNS_6LoggerE	0000000000000348 _ZN4GridlsERSoRNS_6LoggerE
+0000000000000000         *UND*	0000000000000000 _ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
+0000000000000000         *UND*	0000000000000000 _ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_
+0000000000000000         *UND*	0000000000000000 exit
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_	0000000000000266 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid6Logger10backgroundB5cxx11Ev	00000000000000e1 _ZN4Grid6Logger10backgroundB5cxx11Ev
+0000000000000000         *UND*	0000000000000000 _ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
+0000000000000000         *UND*	0000000000000000 _ZN4Grid6Logger9timestampE
+0000000000000000         *UND*	0000000000000000 _ZNSt6chrono3_V212system_clock3nowEv
+0000000000000000  w    F .text._ZN4Grid6Logger8evidenceB5cxx11Ev	00000000000000e1 _ZN4Grid6Logger8evidenceB5cxx11Ev
+0000000000000000  w    F .text._ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE	00000000000001b5 _ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid6Logger7devnullE
+0000000000000000         *UND*	0000000000000000 _ZNSolsEi
+0000000000000000         *UND*	0000000000000000 _ZNKSt5ctypeIcE13_M_widen_initEv
+0000000000000000         *UND*	0000000000000000 _ZNSo3putEc
+0000000000000000         *UND*	0000000000000000 _ZNSo5flushEv
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZSt16__throw_bad_castv
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 memset
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager9ViewCloseEPvNS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicator8ThisRankEv
+0000000000000000  w    F .text._ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE	000000000000027d _ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii	00000000000001b4 _ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid18acceleratorThreadsEv
+0000000000000150  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118af7__ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i_l116.region_id
+0000000000000000         *UND*	0000000000000000 __tgt_target_kernel
+0000000000000000         *UND*	0000000000000000 __kmpc_global_thread_num
+0000000000000000         *UND*	0000000000000000 __kmpc_push_num_teams
+0000000000000000         *UND*	0000000000000000 __kmpc_fork_teams
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000164 _ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+0000000000000000  w    F .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_	0000000000000164 _ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_
+0000000000000000         *UND*	0000000000000000 memcpy
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE	000000000000021d _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+00000000000001c0  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE_l166.region_id
+0000000000000000  w    F .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEELPS9_0EEEvRiRKT_	0000000000000164 _ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEELPS9_0EEEvRiRKT_
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8ViewOpenENS_8ViewModeE	000000000000021d _ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8ViewOpenENS_8ViewModeE
+00000000000001e8  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE_l190.region_id
+0000000000000000  w    F .text._ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_	000000000000013d _ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_
+0000000000000000         *UND*	0000000000000000 _ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE9_M_createERmm
+0000000000000000         *UND*	0000000000000000 _ZNSo9_M_insertIlEERSoT_
+0000000000000000         *UND*	0000000000000000 __cxa_begin_catch
+0000000000000000         *UND*	0000000000000000 _ZSt9terminatev
+0000000000000000         *UND*	0000000000000000 memcmp
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	000000000000017e _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_	00000000000002b4 _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_
+0000000000000000         *UND*	0000000000000000 _ZSt29_Rb_tree_insert_and_rebalancebPSt18_Rb_tree_node_baseS0_RS_
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E	0000000000000031 _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E
+0000000000000000         *UND*	0000000000000000 __cxa_rethrow
+0000000000000000         *UND*	0000000000000000 __cxa_end_catch
+0000000000000000         *UND*	0000000000000000 _ZSt18_Rb_tree_decrementPSt18_Rb_tree_node_base
+0000000000000000         *UND*	0000000000000000 _ZSt18_Rb_tree_incrementPSt18_Rb_tree_node_base
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_	000000000000012f _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	000000000000103f _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid14MemoryProfiler5statsE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid14MemoryProfiler5debugE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid12GridLogDebugE
+0000000000000000         *UND*	0000000000000000 vsnprintf
+0000000000000000  w    F .text._ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z	0000000000000104 _ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z
+0000000000000000         *UND*	0000000000000000 _ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE9_M_appendEPKcm
+0000000000000000         *UND*	0000000000000000 _ZN4Grid10sizeStringB5cxx11Em
+0000000000000000         *UND*	0000000000000000 _ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE10_M_replaceEmmPKcm
+0000000000000000         *UND*	0000000000000000 _ZNSo9_M_insertIPKvEERSoT_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager11CpuAllocateEm
+0000000000000000         *UND*	0000000000000000 _ZSt20__throw_length_errorPKc
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager7CpuFreeEPvm
+0000000000000000         *UND*	0000000000000000 _ZNSo9_M_insertImEERSoT_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager8ViewOpenEPvmNS_8ViewModeENS_10ViewAdviseE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	000000000000103e _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE	000000000000021d _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicator21ProcessorCoorFromRankEiRNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii	000000000000051b _ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii
+0000000000000000  w    F .text._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	000000000000059a _ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii
+0000000000000000  w    F .text._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000b8a _ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii
+0000000000000000         *UND*	0000000000000000 _ZN4Grid12Cshift_tableE
+0000000000000000  w    F .text._ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm	00000000000001d0 _ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm
+0000000000000000  w    F .text._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	000000000000103e _ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	000000000000100f _ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager14SharedAllocateEm
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager10SharedFreeEPvm
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 __kmpc_for_static_init_8u
+0000000000000000         *UND*	0000000000000000 __kmpc_fork_call
+0000000000000000         *UND*	0000000000000000 __kmpc_for_static_fini
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE4ViewENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEC2ERKNS_18LatticeAcceleratorIS7_EENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8allocateEmPKv	000000000000103e _ZN4Grid16alignedAllocatorINS_7iScalarINS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	000000000000100e _ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	000000000000100e _ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000  w    F .text._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	000000000000100f _ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager15AcceleratorFreeEPvm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	000000000000103f _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    O .rodata._ZTVN4Grid13GridCartesianE	0000000000000068 _ZTVN4Grid13GridCartesianE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicatorC2ERKNS_17AcceleratorVectorIiLi8EEERKS0_Ri
+0000000000000000  w    F .text._ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_	0000000000000451 _ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev
+00000000000001e9  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE_l166.region_id
+0000000000000000  w    F .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000164 _ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+00000000000001ea  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE_l166.region_id
+0000000000000228  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE_l166.region_id
+0000000000000000  w    F .text._ZN4Grid13GridCartesianD0Ev	0000000000000012 _ZN4Grid13GridCartesianD0Ev
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicatorD2Ev
+0000000000000000  w    F .text._ZN4Grid13GridCartesian14CheckerBoardedEi	0000000000000003 _ZN4Grid13GridCartesian14CheckerBoardedEi
+0000000000000000  w    F .text._ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE	0000000000000003 _ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid13GridCartesian23CheckerBoardDestinationEiii	0000000000000003 _ZN4Grid13GridCartesian23CheckerBoardDestinationEiii
+0000000000000000  w    F .text._ZN4Grid13GridCartesian17CheckerBoardShiftEiiii	0000000000000003 _ZN4Grid13GridCartesian17CheckerBoardShiftEiiii
+0000000000000000  w    F .text._ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii	0000000000000003 _ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii
+0000000000000000  w    F .text._ZN4Grid13GridCartesian22CheckerBoardFromOindexEi	0000000000000003 _ZN4Grid13GridCartesian22CheckerBoardFromOindexEi
+0000000000000000  w    F .text._ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi	0000000000000003 _ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi
+0000000000000000  w    F .text._ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000096 _ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000096 _ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	000000000000103e _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 cexp
+0000000000000000  w    F .text._ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE	0000000000000174 _ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE
+0000000000000208  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD__l331.region_id
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitC1Ev
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitD1Ev
+0000000000000000         *UND*	0000000000000000 .hidden __dso_handle
+0000000000000000         *UND*	0000000000000000 __cxa_atexit
+0000000000000000         *UND*	0000000000000000 __tgt_register_requires
+0000000000000000  w    O .rodata._ZTIN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000038 _ZTIN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE1MERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MdagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MdirERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_ii
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE7MdirAllERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSt6vectorISK_SaISK_EE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MeooeERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MooeeERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooeeInvERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeooeDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooeeDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11MooeeInvDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4DhopERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DhopOEERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DhopEOERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE7DhopDirERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_ii
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DhopDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11DhopDerivEOERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11DhopDerivOEERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23MomentumSpacePropagatorERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERKSK_dSt6vectorIdSaIdEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ImportGaugeERKNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE24ContractConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_SL_NS_7CurrentEj
+0000000000000000  w    O .rodata._ZTSN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000089 _ZTSN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000089 _ZTSN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	000000000000008b _ZTSN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	0000000000000099 _ZTSN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	000000000000008b _ZTSN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000  w    O .rodata._ZTIN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	0000000000000010 _ZTIN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000         *UND*	0000000000000000 _ZTVN10__cxxabiv117__class_type_infoE
+0000000000000000  w    O .rodata._ZTIN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	0000000000000018 _ZTIN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000         *UND*	0000000000000000 _ZTVN10__cxxabiv120__si_class_type_infoE
+0000000000000000  w    O .rodata._ZTSN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE	0000000000000074 _ZTSN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE
+0000000000000000  w    O .rodata._ZTSN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE	0000000000000072 _ZTSN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE	0000000000000059 _ZTSN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE
+0000000000000000  w    O .rodata._ZTIN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE	0000000000000010 _ZTIN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE
+0000000000000000  w    O .rodata._ZTIN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE	0000000000000018 _ZTIN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE
+0000000000000000  w    O .rodata._ZTIN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE	0000000000000018 _ZTIN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE
+0000000000000000  w    O .rodata._ZTIN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000038 _ZTIN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000         *UND*	0000000000000000 _ZTVN10__cxxabiv121__vmi_class_type_infoE
+0000000000000000  w    O .rodata._ZTSN4Grid19WilsonKernelsStaticE	000000000000001d _ZTSN4Grid19WilsonKernelsStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid19WilsonKernelsStaticE	0000000000000010 _ZTIN4Grid19WilsonKernelsStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000038 _ZTIN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid19WilsonFermionStaticE	000000000000001d _ZTSN4Grid19WilsonFermionStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid19WilsonFermionStaticE	0000000000000010 _ZTIN4Grid19WilsonFermionStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid13GridCartesianE	0000000000000018 _ZTIN4Grid13GridCartesianE
+0000000000000000  w    O .rodata._ZTSN4Grid13GridCartesianE	0000000000000017 _ZTSN4Grid13GridCartesianE
+0000000000000000  w    O .rodata._ZTSN4Grid8GridBaseE	0000000000000011 _ZTSN4Grid8GridBaseE
+0000000000000000  w    O .rodata._ZTSN4Grid10GridThreadE	0000000000000014 _ZTSN4Grid10GridThreadE
+0000000000000000  w    O .rodata._ZTIN4Grid10GridThreadE	0000000000000010 _ZTIN4Grid10GridThreadE
+0000000000000000  w    O .rodata._ZTIN4Grid8GridBaseE	0000000000000038 _ZTIN4Grid8GridBaseE
+0000000000000000         *UND*	0000000000000000 _ZTIN4Grid21CartesianCommunicatorE
+0000000000000000  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118af7__ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i_l116
+0000000000000020  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE_l166
+0000000000000040  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_12TrinaryWhereENS0_INS1_INS3_IjNS7_IjEEEEEEEESD_SD_EERSD_RKNS_24LatticeTrinaryExpressionIT_T0_T1_T2_EE_l190
+0000000000000060  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE_l166
+0000000000000080  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE_l166
+00000000000000a0  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD__l331
+00000000000000c0  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE_l166
+0000000000000000         *UND*	0000000000000000 __gxx_personality_v0

amd-omp-stack-err/objdump_works1.txt

@@ -0,0 +1,16 @@
+
+libWilsonFermionWorks1.a(WilsonFermionWorks1.o):	file format elf64-x86-64
+
+SYMBOL TABLE:
+0000000000000000 l    df *ABS*	0000000000000000 WilsonFermionWorks1.cc
+0000000000000000 l    d  .text.startup	0000000000000000 .text.startup
+0000000000000000 l     F .text.startup	0000000000000021 _GLOBAL__sub_I_WilsonFermionWorks1.cc
+0000000000000000 l     O .bss	0000000000000001 _ZStL8__ioinit
+0000000000000000 l    d  .bss	0000000000000000 .bss
+0000000000000001 l     O .bss	0000000000000001 _ZN5EigenL4lastE
+0000000000000002 l     O .bss	0000000000000002 _ZN5EigenL6lastp1E
+0000000000000004 l     O .bss	0000000000000001 _ZN5EigenL3allE
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitC1Ev
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitD1Ev
+0000000000000000         *UND*	0000000000000000 .hidden __dso_handle
+0000000000000000         *UND*	0000000000000000 __cxa_atexit

amd-omp-stack-err/objdump_works2.txt

@@ -0,0 +1,545 @@
+
+libWilsonFermionWorks2.a(WilsonFermionWorks2.o):	file format elf64-x86-64
+
+SYMBOL TABLE:
+0000000000000000 l    df *ABS*	0000000000000000 WilsonFermionWorks2.cc
+0000000000000000 l    d  .text	0000000000000000 .text
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv
+0000000000000000 l    d  .text._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	0000000000000000 .text._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	0000000000000000 .gcc_except_table._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_
+0000000000000000 l       .gcc_except_table._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	0000000000000000 GCC_except_table1
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	0000000000000000 GCC_except_table3
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv
+0000000000000000 l    d  .text._ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv	0000000000000000 .text._ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000 l       .rodata.cst8	0000000000000000 .LCPI20_0
+0000000000000008 l       .rodata.cst8	0000000000000000 .LCPI20_1
+0000000000000000 l       .rodata.cst16	0000000000000000 .LCPI20_2
+0000000000000010 l       .rodata.cst16	0000000000000000 .LCPI20_3
+0000000000000020 l       .rodata.cst16	0000000000000000 .LCPI20_4
+0000000000000030 l       .rodata.cst16	0000000000000000 .LCPI20_5
+0000000000000040 l       .rodata.cst16	0000000000000000 .LCPI20_6
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE
+0000000000000000 l    d  .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000000 .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE
+0000000000000000 l       .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000000 GCC_except_table20
+0000000000000010 l       .rodata.cst8	0000000000000000 .LCPI21_0
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d
+0000000000000000 l    d  .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000000 .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d
+0000000000000000 l       .gcc_except_table._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000000 GCC_except_table21
+0000000000000050 l       .rodata.cst16	0000000000000000 .LCPI22_0
+0000000000000000 l    d  .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l    d  .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l       .gcc_except_table._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 GCC_except_table22
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000000 .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000 l    d  .text._ZN4GridlsERSoRNS_6LoggerE	0000000000000000 .text._ZN4GridlsERSoRNS_6LoggerE
+0000000000000000 l    d  .gcc_except_table._ZN4GridlsERSoRNS_6LoggerE	0000000000000000 .gcc_except_table._ZN4GridlsERSoRNS_6LoggerE
+0000000000000000 l       .gcc_except_table._ZN4GridlsERSoRNS_6LoggerE	0000000000000000 GCC_except_table31
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l    d  .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l       .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 GCC_except_table35
+0000000000000000 l    d  .text._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	0000000000000000 .text._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii
+0000000000000000 l    d  .gcc_except_table._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	0000000000000000 .gcc_except_table._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii
+0000000000000000 l       .gcc_except_table._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	0000000000000000 GCC_except_table36
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 GCC_except_table38
+0000000000000000 l    d  .text._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000000 .text._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i
+0000000000000000 l     F .text	000000000000010d .omp_outlined.
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000000 .gcc_except_table._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i
+0000000000000000 l       .gcc_except_table._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000000 GCC_except_table39
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE
+0000000000001160 l     F .text	00000000000000e5 .omp_outlined..44
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table41
+0000000000000000 l    d  .text._ZN4Grid6Logger10backgroundB5cxx11Ev	0000000000000000 .text._ZN4Grid6Logger10backgroundB5cxx11Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid6Logger10backgroundB5cxx11Ev	0000000000000000 .gcc_except_table._ZN4Grid6Logger10backgroundB5cxx11Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid6Logger10backgroundB5cxx11Ev	0000000000000000 GCC_except_table42
+0000000000000000 l    d  .text._ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE	0000000000000000 .text._ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE
+0000000000000000 l    d  .text._ZN4Grid6Logger8evidenceB5cxx11Ev	0000000000000000 .text._ZN4Grid6Logger8evidenceB5cxx11Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid6Logger8evidenceB5cxx11Ev	0000000000000000 .gcc_except_table._ZN4Grid6Logger8evidenceB5cxx11Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid6Logger8evidenceB5cxx11Ev	0000000000000000 GCC_except_table44
+0000000000000000 l    d  .text._ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_	0000000000000000 .text._ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_
+0000000000000100 l     O .rodata	0000000000000001 _ZStL19piecewise_construct
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_
+0000000000000000 l    d  .gcc_except_table._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	0000000000000000 .gcc_except_table._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_
+0000000000000000 l       .gcc_except_table._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	0000000000000000 GCC_except_table47
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E
+0000000000000000 l    d  .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_	0000000000000000 .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 GCC_except_table53
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 GCC_except_table54
+0000000000000000 l    d  .text._ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z	0000000000000000 .text._ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	0000000000000000 GCC_except_table61
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	0000000000000000 GCC_except_table62
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table67
+0000000000000000 l    d  .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii	0000000000000000 .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii
+0000000000000060 l       .rodata.cst16	0000000000000000 .LCPI69_0
+0000000000000000 l    d  .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii	0000000000000000 .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii
+0000000000000000 l    d  .text._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000000 .text._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii
+0000000000000000 l    d  .gcc_except_table._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000000 .gcc_except_table._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii
+0000000000000000 l       .gcc_except_table._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000000 GCC_except_table70
+0000000000000000 l    d  .text._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	0000000000000000 .text._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	0000000000000000 .gcc_except_table._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii
+0000000000000000 l       .gcc_except_table._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	0000000000000000 GCC_except_table71
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	0000000000000000 GCC_except_table72
+0000000000000000 l    d  .text._ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm	0000000000000000 .text._ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm
+0000000000000000 l    d  .text._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	0000000000000000 .text._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	0000000000000000 GCC_except_table74
+0000000000000000 l    d  .text._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .text._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l       .gcc_except_table._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 GCC_except_table75
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE
+0000000000000110 l     F .text	000000000000104c .omp_outlined..41
+0000000000000070 l       .rodata.cst16	0000000000000000 .LCPI78_0
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	0000000000000000 GCC_except_table79
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE
+0000000000001250 l     F .text	000000000000ad36 .omp_outlined..45
+0000000000000080 l       .rodata.cst16	0000000000000000 .LCPI85_0
+0000000000000000 l    d  .rodata	0000000000000000 .rodata
+0000000000000000 l    d  .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000000 .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+0000000000000000 l    d  .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_	0000000000000000 .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000000 .text._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000000 GCC_except_table89
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000000 GCC_except_table90
+0000000000000000 l    d  .text._ZN4Grid12uvmAllocatorIiE10deallocateEPim	0000000000000000 .text._ZN4Grid12uvmAllocatorIiE10deallocateEPim
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12uvmAllocatorIiE10deallocateEPim	0000000000000000 .gcc_except_table._ZN4Grid12uvmAllocatorIiE10deallocateEPim
+0000000000000000 l       .gcc_except_table._ZN4Grid12uvmAllocatorIiE10deallocateEPim	0000000000000000 GCC_except_table91
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	0000000000000000 GCC_except_table92
+0000000000000000 l    d  .text._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .text._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .gcc_except_table._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l       .gcc_except_table._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 GCC_except_table93
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	0000000000000000 GCC_except_table94
+0000000000000000 l    d  .text._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .text._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 .gcc_except_table._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000 l       .gcc_except_table._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	0000000000000000 GCC_except_table95
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	0000000000000000 GCC_except_table96
+0000000000000000 l    d  .text.startup	0000000000000000 .text.startup
+0000000000000000 l     F .text.startup	0000000000000001 __cxx_global_var_init.52
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	0000000000000000 GCC_except_table101
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE
+0000000000000090 l       .rodata.cst16	0000000000000000 .LCPI108_0
+0000000000000000 l    d  .text._ZN4Grid3FFTC2EPNS_13GridCartesianE	0000000000000000 .text._ZN4Grid3FFTC2EPNS_13GridCartesianE
+0000000000000000 l    d  .gcc_except_table._ZN4Grid3FFTC2EPNS_13GridCartesianE	0000000000000000 .gcc_except_table._ZN4Grid3FFTC2EPNS_13GridCartesianE
+0000000000000000 l       .gcc_except_table._ZN4Grid3FFTC2EPNS_13GridCartesianE	0000000000000000 GCC_except_table108
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000 l    d  .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l    d  .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000 l       .gcc_except_table._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000000 GCC_except_table112
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE
+000000000000bf90 l     F .text	00000000000000e5 .omp_outlined..59
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE
+000000000000c1d0 l     F .text	00000000000000e5 .omp_outlined..63
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE
+000000000000c870 l     F .text	00000000000000e5 .omp_outlined..75
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table116
+00000000000000a0 l       .rodata.cst16	0000000000000000 .LCPI117_0
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_	0000000000000000 .text._ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_
+0000000000000000 l    d  .text._ZN4Grid13GridCartesianD0Ev	0000000000000000 .text._ZN4Grid13GridCartesianD0Ev
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian14CheckerBoardedEi	0000000000000000 .text._ZN4Grid13GridCartesian14CheckerBoardedEi
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian23CheckerBoardDestinationEiii	0000000000000000 .text._ZN4Grid13GridCartesian23CheckerBoardDestinationEiii
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian17CheckerBoardShiftEiiii	0000000000000000 .text._ZN4Grid13GridCartesian17CheckerBoardShiftEiiii
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii	0000000000000000 .text._ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian22CheckerBoardFromOindexEi	0000000000000000 .text._ZN4Grid13GridCartesian22CheckerBoardFromOindexEi
+0000000000000000 l    d  .text._ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi	0000000000000000 .text._ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi
+0000000000000000 l    d  .text._ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000000 .text._ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	0000000000000000 GCC_except_table130
+0000000000000000 l    d  .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	0000000000000000 .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m
+0000000000000000 l    d  .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	0000000000000000 .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m
+0000000000000000 l       .gcc_except_table._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	0000000000000000 GCC_except_table131
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	0000000000000000 .text._ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000 l    d  .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	0000000000000000 .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev
+0000000000000000 l    d  .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	0000000000000000 .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev
+0000000000000000 l       .gcc_except_table._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	0000000000000000 GCC_except_table135
+000000000000c080 l     F .text	000000000000014d .omp_outlined..60
+0000000000000018 l       .rodata.cst8	0000000000000000 .LCPI137_0
+00000000000000b0 l       .rodata.cst16	0000000000000000 .LCPI137_1
+0000000000000000 l    d  .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	0000000000000000 .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+000000000000c2c0 l     F .text	0000000000000346 .omp_outlined..64
+0000000000000020 l       .rodata.cst8	0000000000000000 .LCPI140_0
+0000000000000028 l       .rodata.cst8	0000000000000000 .LCPI140_1
+0000000000000000 l    d  .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000000 .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+0000000000000000 l    d  .text._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	0000000000000000 .text._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi
+0000000000000000 l    d  .gcc_except_table._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	0000000000000000 .gcc_except_table._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi
+0000000000000000 l       .gcc_except_table._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	0000000000000000 GCC_except_table142
+0000000000000000 l    d  .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_	0000000000000000 .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_
+000000000000c610 l     F .text	00000000000000e5 .omp_outlined..67
+0000000000000000 l    d  .text._ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE	0000000000000000 .text._ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE
+000000000000c700 l     F .text	0000000000000166 .omp_outlined..68
+000000000000c960 l     F .text	0000000000000462 .omp_outlined..76
+00000000000000c0 l       .rodata.cst16	0000000000000000 .LCPI148_0
+0000000000000010 l     F .text.startup	0000000000000021 _GLOBAL__sub_I_WilsonFermionWorks2.cc
+0000000000000000 l     O .bss	0000000000000001 _ZStL8__ioinit
+0000000000000040 l     F .text.startup	000000000000000a .omp_offloading.requires_reg
+0000000000000000 l    d  .bss	0000000000000000 .bss
+0000000000000001 l     O .bss	0000000000000001 _ZN5EigenL4lastE
+0000000000000002 l     O .bss	0000000000000002 _ZN5EigenL6lastp1E
+0000000000000004 l     O .bss	0000000000000001 _ZN5EigenL3fixILi1EEE
+0000000000000005 l     O .bss	0000000000000001 _ZN5EigenL3allE
+0000000000000000 l    d  .rodata.str1.1	0000000000000000 .rodata.str1.1
+0000000000000000 l     O .rodata.str1.16	0000000000000114 .omp_offloading.entry_name
+0000000000000000 l    d  .rodata.str1.16	0000000000000000 .rodata.str1.16
+0000000000000120 l     O .rodata.str1.16	0000000000000131 .omp_offloading.entry_name.80
+0000000000000260 l     O .rodata.str1.16	00000000000000ee .omp_offloading.entry_name.81
+0000000000000350 l     O .rodata.str1.16	000000000000013b .omp_offloading.entry_name.82
+0000000000000490 l     O .rodata.str1.16	000000000000009e .omp_offloading.entry_name.83
+0000000000000530 l     O .rodata.str1.16	0000000000000130 .omp_offloading.entry_name.84
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv	0000000000000009 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4GridEv
+0000000000000000  w    F .text._ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_	000000000000008b _ZN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE5MdagMERKSE_RSE_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	000000000000100e _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000  w    F .text.__clang_call_terminate	000000000000000b .hidden __clang_call_terminate
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000         *UND*	0000000000000000 _Unwind_Resume
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000009 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MdiagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev	000000000000015b _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED2Ev
+0000000000000000  w    O .rodata._ZTVN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	00000000000001a8 _ZTVN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    F .text._ZN4Grid12uvmAllocatorIiE10deallocateEPim	000000000000100f _ZN4Grid12uvmAllocatorIiE10deallocateEPim
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m	000000000000100e _ZN4Grid16alignedAllocatorINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE10deallocateEPSD_m
+0000000000000000  w    F .text._ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev	0000000000000256 _ZN4Grid16CartesianStencilINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEENS1_INS2_ISA_Li2EEEEENS_16WilsonImplParamsEED2Ev
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev	0000000000000012 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEED0Ev
+0000000000000000         *UND*	0000000000000000 _ZdlPv
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv	0000000000000009 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE12RedBlackGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv	0000000000000009 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MassEv
+0000000000000000  w    F .text._ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv	0000000000000006 _ZN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEE7ConstEEEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv	0000000000000006 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11isTrivialEEEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE3tmpEv
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE	000000000000001a _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14DirichletBlockERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000         *UND*	0000000000000000 __assert_fail
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11FermionGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19FermionRedBlackGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9GaugeGridEv
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv	0000000000000008 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE17GaugeRedBlackGridEv
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000c _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6MDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000c _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MoeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000c _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeoDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_	00000000000000e3 _ZN4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEaSINS_4ZeroEEERSE_RKT_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeeDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE	0000000000000e2f _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_dSt6vectorIS4_SaIS4_EESO_IdSaIdEE
+0000000000000000  w    F .text._ZN4Grid3FFTC2EPNS_13GridCartesianE	000000000000024b _ZN4Grid3FFTC2EPNS_13GridCartesianE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_	00000000000000f7 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_4ZeroEEERSC_RKT_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000  w    F .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi	000000000000020a _ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000         *UND*	0000000000000000 acos
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE	00000000000000ab _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE	00000000000003d2 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	00000000000000ab _ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE	00000000000003d2 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi	00000000000000ce _ZN4Grid3FFT12FFT_dim_maskINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSG_NS_17AcceleratorVectorIiLi8EEEi
+0000000000000000         *UND*	0000000000000000 _Znwm
+0000000000000000         *UND*	0000000000000000 memmove
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE	00000000000003d2 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m	000000000000100f _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE10deallocateEPSB_m
+0000000000000000         *UND*	0000000000000000 _ZSt17__throw_bad_allocv
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d	0000000000000222 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE14FreePropagatorERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_d
+0000000000000000  w    F .text._ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE	0000000000000466 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE19SeqConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_NS_7CurrentEjjjRNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid15GridDefaultLattEv
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_4ZeroEEERSD_RKT_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE	000000000000021c _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2EPNS_8GridBaseENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi	0000000000000320 _ZN4Grid17LatticeCoordinateINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEvRNS_7LatticeIT_EEi
+0000000000000000  w    F .text._ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii	000000000000018c _ZN4Grid6CshiftINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEENS_7LatticeIT_EERKSF_ii
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_	0000000000000168 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSEOSD_
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m	000000000000100e _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE10deallocateEPSC_m
+0000000000000000  w    F .text._ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i	0000000000000277 _ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE	00000000000000ab _ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE	0000000000000592 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m	000000000000100f _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE10deallocateEPS9_m
+0000000000000000         *UND*	0000000000000000 _ZSt4cout
+0000000000000000         *UND*	0000000000000000 _ZN4Grid12GridLogErrorE
+0000000000000000  w    F .text._ZN4GridlsERSoRNS_6LoggerE	0000000000000348 _ZN4GridlsERSoRNS_6LoggerE
+0000000000000000         *UND*	0000000000000000 _ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
+0000000000000000         *UND*	0000000000000000 _ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_
+0000000000000000         *UND*	0000000000000000 exit
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev	0000000000000037 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE	000000000000000e _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ContractJ5qERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEERNSE_INSF_INSF_INSF_IS8_EEEEEEEE
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DminusERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_	0000000000000266 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DminusDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ImportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23ImportUnphysicalFermionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE29ExportPhysicalFermionSolutionERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_	0000000000000008 _ZN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE27ExportPhysicalFermionSourceERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000  w    F .text._ZN4Grid6Logger10backgroundB5cxx11Ev	00000000000000e1 _ZN4Grid6Logger10backgroundB5cxx11Ev
+0000000000000000         *UND*	0000000000000000 _ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
+0000000000000000         *UND*	0000000000000000 _ZN4Grid6Logger9timestampE
+0000000000000000         *UND*	0000000000000000 _ZNSt6chrono3_V212system_clock3nowEv
+0000000000000000  w    F .text._ZN4Grid6Logger8evidenceB5cxx11Ev	00000000000000e1 _ZN4Grid6Logger8evidenceB5cxx11Ev
+0000000000000000  w    F .text._ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE	00000000000001b5 _ZN4GridlsERSoRKNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid6Logger7devnullE
+0000000000000000         *UND*	0000000000000000 _ZNSolsEi
+0000000000000000         *UND*	0000000000000000 _ZNKSt5ctypeIcE13_M_widen_initEv
+0000000000000000         *UND*	0000000000000000 _ZNSo3putEc
+0000000000000000         *UND*	0000000000000000 _ZNSo5flushEv
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZSt16__throw_bad_castv
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 memset
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager9ViewCloseEPvNS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicator8ThisRankEv
+0000000000000000  w    F .text._ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE	000000000000027d _ZN4Grid8GridBase21RankIndexToGlobalCoorEiiiRNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii	00000000000001b4 _ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_ii
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE4ViewENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid18acceleratorThreadsEv
+0000000000000150  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118af7__ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i_l116.region_id
+0000000000000000         *UND*	0000000000000000 __tgt_target_kernel
+0000000000000000         *UND*	0000000000000000 __kmpc_global_thread_num
+0000000000000000         *UND*	0000000000000000 __kmpc_push_num_teams
+0000000000000000         *UND*	0000000000000000 __kmpc_fork_teams
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEED2Ev
+0000000000000000  w    F .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000164 _ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iMatrixINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+0000000000000000  w    F .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_	0000000000000164 _ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEELPSD_0EEEvRiRKT_
+0000000000000000         *UND*	0000000000000000 memcpy
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE	000000000000021d _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+00000000000001c0  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE_l166.region_id
+0000000000000000  w    F .text._ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_	000000000000013d _ZNSt3mapINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St4lessIS5_ESaISt4pairIKS5_S5_EEEixEOS5_
+0000000000000000         *UND*	0000000000000000 _ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE9_M_createERmm
+0000000000000000         *UND*	0000000000000000 _ZNSo9_M_insertIlEERSoT_
+0000000000000000         *UND*	0000000000000000 __cxa_begin_catch
+0000000000000000         *UND*	0000000000000000 _ZSt9terminatev
+0000000000000000         *UND*	0000000000000000 memcmp
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_	000000000000017e _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE22_M_emplace_hint_uniqueIJRKSt21piecewise_construct_tSt5tupleIJOS5_EESJ_IJEEEEESt17_Rb_tree_iteratorIS8_ESt23_Rb_tree_const_iteratorIS8_EDpOT_
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_	00000000000002b4 _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE29_M_get_insert_hint_unique_posESt23_Rb_tree_const_iteratorIS8_ERS7_
+0000000000000000         *UND*	0000000000000000 _ZSt29_Rb_tree_insert_and_rebalancebPSt18_Rb_tree_node_baseS0_RS_
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E	0000000000000031 _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE12_M_drop_nodeEPSt13_Rb_tree_nodeIS8_E
+0000000000000000         *UND*	0000000000000000 __cxa_rethrow
+0000000000000000         *UND*	0000000000000000 __cxa_end_catch
+0000000000000000         *UND*	0000000000000000 _ZSt18_Rb_tree_decrementPSt18_Rb_tree_node_base
+0000000000000000         *UND*	0000000000000000 _ZSt18_Rb_tree_incrementPSt18_Rb_tree_node_base
+0000000000000000  w    F .text._ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_	000000000000012f _ZNSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4pairIKS5_S5_ESt10_Select1stIS8_ESt4lessIS5_ESaIS8_EE24_M_get_insert_unique_posERS7_
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	000000000000103f _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid14MemoryProfiler5statsE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid14MemoryProfiler5debugE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid12GridLogDebugE
+0000000000000000         *UND*	0000000000000000 vsnprintf
+0000000000000000  w    F .text._ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z	0000000000000104 _ZN9__gnu_cxx12__to_xstringINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEcEET_PFiPT0_mPKS8_P13__va_list_tagEmSB_z
+0000000000000000         *UND*	0000000000000000 _ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE9_M_appendEPKcm
+0000000000000000         *UND*	0000000000000000 _ZN4Grid10sizeStringB5cxx11Em
+0000000000000000         *UND*	0000000000000000 _ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE10_M_replaceEmmPKcm
+0000000000000000         *UND*	0000000000000000 _ZNSo9_M_insertIPKvEERSoT_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager11CpuAllocateEm
+0000000000000000         *UND*	0000000000000000 _ZSt20__throw_length_errorPKc
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager7CpuFreeEPvm
+0000000000000000         *UND*	0000000000000000 _ZNSo9_M_insertImEERSoT_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager8ViewOpenEPvmNS_8ViewModeENS_10ViewAdviseE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv	000000000000103e _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorIS9_EENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE	000000000000021d _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdIjNS_12Optimization3vecIjEEEEEEEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicator21ProcessorCoorFromRankEiRNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii	000000000000051b _ZN4Grid12Cshift_localINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iii
+0000000000000000  w    F .text._ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii	000000000000059a _ZN4Grid10Copy_planeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiii
+0000000000000000  w    F .text._ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii	0000000000000b8a _ZN4Grid18Copy_plane_permuteINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEvRNS_7LatticeIT_EERKSF_iiiii
+0000000000000000         *UND*	0000000000000000 _ZN4Grid12Cshift_tableE
+0000000000000000  w    F .text._ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm	00000000000001d0 _ZNSt6vectorISt4pairIiiEN4Grid12uvmAllocatorIS1_EEE17_M_default_appendEm
+0000000000000000  w    F .text._ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv	000000000000103e _ZN4Grid12uvmAllocatorISt4pairIiiEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m	000000000000100f _ZN4Grid12uvmAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager14SharedAllocateEm
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager10SharedFreeEPvm
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 __kmpc_for_static_init_8u
+0000000000000000         *UND*	0000000000000000 __kmpc_fork_call
+0000000000000000         *UND*	0000000000000000 __kmpc_for_static_fini
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi8EEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m	000000000000100e _ZN4Grid12devAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m	000000000000100e _ZN4Grid16alignedAllocatorINS_12StencilEntryEE10deallocateEPS1_m
+0000000000000000  w    F .text._ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m	000000000000100f _ZN4Grid12devAllocatorISt4pairIiiEE10deallocateEPS2_m
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13MemoryManager15AcceleratorFreeEPvm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv	000000000000103f _ZN4Grid16alignedAllocatorINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEC2ERKNS_18LatticeAcceleratorISC_EENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE4ViewENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEEC2ERKNS_18LatticeAcceleratorISD_EENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE	0000000000000225 _ZN4Grid11LatticeViewINS_7iVectorINS_7iScalarINS_7iMatrixINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEEEELi4EEEE8ViewOpenENS_8ViewModeE
+0000000000000000  w    O .rodata._ZTVN4Grid13GridCartesianE	0000000000000068 _ZTVN4Grid13GridCartesianE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicatorC2ERKNS_17AcceleratorVectorIiLi8EEERKS0_Ri
+0000000000000000  w    F .text._ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_	0000000000000451 _ZN4Grid13GridCartesian4InitERKNS_17AcceleratorVectorIiLi8EEES4_S4_
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm	00000000000000e3 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE6resizeEm
+0000000000000000  w    F .text._ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE	00000000000001d8 _ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE11SetViewModeENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE4ViewENS_8ViewModeE
+0000000000000000  w    F .text._ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE	00000000000000b6 _ZNK4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEE4ViewENS_8ViewModeE
+0000000000000000  w    F .text._ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev	000000000000001a _ZN4Grid10ViewCloserINS_11LatticeViewINS_7iScalarINS2_INS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEEED2Ev
+00000000000001e8  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE_l166.region_id
+0000000000000000  w    F .text._ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_	0000000000000164 _ZN4Grid16CBFromExpressionINS_11LatticeViewINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEELPSE_0EEEvRiRKT_
+00000000000001e9  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE_l166.region_id
+0000000000000228  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE_l166.region_id
+0000000000000000  w    F .text._ZN4Grid13GridCartesianD0Ev	0000000000000012 _ZN4Grid13GridCartesianD0Ev
+0000000000000000         *UND*	0000000000000000 _ZN4Grid21CartesianCommunicatorD2Ev
+0000000000000000  w    F .text._ZN4Grid13GridCartesian14CheckerBoardedEi	0000000000000003 _ZN4Grid13GridCartesian14CheckerBoardedEi
+0000000000000000  w    F .text._ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE	0000000000000003 _ZN4Grid13GridCartesian12CheckerBoardERKNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid13GridCartesian23CheckerBoardDestinationEiii	0000000000000003 _ZN4Grid13GridCartesian23CheckerBoardDestinationEiii
+0000000000000000  w    F .text._ZN4Grid13GridCartesian17CheckerBoardShiftEiiii	0000000000000003 _ZN4Grid13GridCartesian17CheckerBoardShiftEiiii
+0000000000000000  w    F .text._ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii	0000000000000003 _ZN4Grid13GridCartesian22CheckerBoardShiftForCBEiiii
+0000000000000000  w    F .text._ZN4Grid13GridCartesian22CheckerBoardFromOindexEi	0000000000000003 _ZN4Grid13GridCartesian22CheckerBoardFromOindexEi
+0000000000000000  w    F .text._ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi	0000000000000003 _ZN4Grid13GridCartesian27CheckerBoardFromOindexTableEi
+0000000000000000  w    F .text._ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000096 _ZN4Grid8GridBase6oIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE	0000000000000096 _ZN4Grid8GridBase6iIndexERNS_17AcceleratorVectorIiLi8EEE
+0000000000000000  w    F .text._ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv	000000000000103e _ZN4Grid16alignedAllocatorINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEE8allocateEmPKv
+0000000000000000  w    F .text._ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE	00000000000000c5 _ZN4Grid11LatticeViewINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEC2ERKNS_18LatticeAcceleratorISB_EENS_8ViewModeE
+0000000000000000         *UND*	0000000000000000 cexp
+0000000000000000  w    F .text._ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE	0000000000000174 _ZN4Grid11conformableINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEESC_EEvRKNS_7LatticeIT_EERKNSD_IT0_EE
+0000000000000208  w    O .rodata	0000000000000001 .__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD__l331.region_id
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitC1Ev
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitD1Ev
+0000000000000000         *UND*	0000000000000000 .hidden __dso_handle
+0000000000000000         *UND*	0000000000000000 __cxa_atexit
+0000000000000000         *UND*	0000000000000000 __tgt_register_requires
+0000000000000000  w    O .rodata._ZTIN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000038 _ZTIN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE1MERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MdagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4MdirERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_ii
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE7MdirAllERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSt6vectorISK_SaISK_EE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MeooeERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE5MooeeERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooeeInvERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MeooeDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE8MooeeDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11MooeeInvDagERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE4DhopERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DhopOEERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE6DhopEOERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE7DhopDirERKNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERSK_ii
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE9DhopDerivERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11DhopDerivEOERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11DhopDerivOEERNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEERKNSE_INSG_INSF_INSF_IS8_Li3EEELi4EEEEEEESS_i
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE23MomentumSpacePropagatorERNS_7LatticeINS_7iScalarINS_7iVectorINSG_IS8_Li3EEELi4EEEEEEERKSK_dSt6vectorIdSaIdEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE11ImportGaugeERKNS_7LatticeINS_7iVectorINS_7iScalarINS_7iMatrixIS8_Li3EEEEELi4EEEEE
+0000000000000000         *UND*	0000000000000000 _ZN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEE24ContractConservedCurrentERNS_7LatticeINS_7iScalarINS_7iMatrixINSG_IS8_Li3EEELi4EEEEEEESL_SL_SL_NS_7CurrentEj
+0000000000000000  w    O .rodata._ZTSN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000089 _ZTSN4Grid13WilsonFermionINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000089 _ZTSN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	000000000000008b _ZTSN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	0000000000000099 _ZTSN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	000000000000008b _ZTSN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000  w    O .rodata._ZTIN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	0000000000000010 _ZTIN4Grid16SparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000         *UND*	0000000000000000 _ZTVN10__cxxabiv117__class_type_infoE
+0000000000000000  w    O .rodata._ZTIN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE	0000000000000018 _ZTIN4Grid30CheckerBoardedSparseMatrixBaseINS_7LatticeINS_7iScalarINS_7iVectorINS3_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEEEE
+0000000000000000         *UND*	0000000000000000 _ZTVN10__cxxabiv120__si_class_type_infoE
+0000000000000000  w    O .rodata._ZTSN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE	0000000000000074 _ZTSN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE
+0000000000000000  w    O .rodata._ZTSN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE	0000000000000072 _ZTSN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE	0000000000000059 _ZTSN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE
+0000000000000000  w    O .rodata._ZTIN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE	0000000000000010 _ZTIN4Grid14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEE
+0000000000000000  w    O .rodata._ZTIN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE	0000000000000018 _ZTIN4Grid17PeriodicGaugeImplINS_14GaugeImplTypesINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3ELi12EEEEE
+0000000000000000  w    O .rodata._ZTIN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE	0000000000000018 _ZTIN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEE
+0000000000000000  w    O .rodata._ZTIN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000038 _ZTIN4Grid15FermionOperatorINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000         *UND*	0000000000000000 _ZTVN10__cxxabiv121__vmi_class_type_infoE
+0000000000000000  w    O .rodata._ZTSN4Grid19WilsonKernelsStaticE	000000000000001d _ZTSN4Grid19WilsonKernelsStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid19WilsonKernelsStaticE	0000000000000010 _ZTIN4Grid19WilsonKernelsStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE	0000000000000038 _ZTIN4Grid13WilsonKernelsINS_10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEEEEE
+0000000000000000  w    O .rodata._ZTSN4Grid19WilsonFermionStaticE	000000000000001d _ZTSN4Grid19WilsonFermionStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid19WilsonFermionStaticE	0000000000000010 _ZTIN4Grid19WilsonFermionStaticE
+0000000000000000  w    O .rodata._ZTIN4Grid13GridCartesianE	0000000000000018 _ZTIN4Grid13GridCartesianE
+0000000000000000  w    O .rodata._ZTSN4Grid13GridCartesianE	0000000000000017 _ZTSN4Grid13GridCartesianE
+0000000000000000  w    O .rodata._ZTSN4Grid8GridBaseE	0000000000000011 _ZTSN4Grid8GridBaseE
+0000000000000000  w    O .rodata._ZTSN4Grid10GridThreadE	0000000000000014 _ZTSN4Grid10GridThreadE
+0000000000000000  w    O .rodata._ZTIN4Grid10GridThreadE	0000000000000010 _ZTIN4Grid10GridThreadE
+0000000000000000  w    O .rodata._ZTIN4Grid8GridBaseE	0000000000000038 _ZTIN4Grid8GridBaseE
+0000000000000000         *UND*	0000000000000000 _ZTIN4Grid21CartesianCommunicatorE
+0000000000000000  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118af7__ZN4Grid10WilsonImplINS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEENS_14FundamentalRepILi3EEENS_9CoeffRealEE13multLinkFieldINS_7LatticeINS_7iScalarINS_7iMatrixINSF_IS7_Li3EEELi4EEEEEEEEEvRT_RKNSD_INS_7iVectorINSE_ISG_EELi8EEEEERKSK_i_l116
+0000000000000020  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iMatrixINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinarySubENS_23LatticeBinaryExpressionINS_9BinaryMulESD_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEENSG_ISH_NSG_ISH_NS_5GammaESD_EESL_EEEERSD_RKNSG_IT_T0_T1_EE_l166
+0000000000000040  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS1_INS1_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEEEEEEEEEaSINS_9BinaryAddESC_NS_23LatticeBinaryExpressionINS_9BinaryMulENSF_ISG_dSC_EEdEEEERSC_RKNSF_IT_T0_T1_EE_l166
+0000000000000060  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulENS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_NSI_ISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEdEEEESD_EERSD_RKNSI_IT_T0_T1_EE_l166
+0000000000000080  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSERKSD__l331
+00000000000000a0  w    O omp_offloading_entries	0000000000000020 .omp_offloading.entry.__omp_offloading_73_1e118ab9__ZN4Grid7LatticeINS_7iScalarINS_7iVectorINS2_INS_9Grid_simdISt7complexIdENS_12Optimization3vecIdEEEELi3EEELi4EEEEEEaSINS_9BinaryMulESD_NS_22LatticeUnaryExpressionINS_8UnaryExpENS_23LatticeBinaryExpressionISF_S5_NS0_INS1_INS1_INS1_IS9_EEEEEEEEEEEEEERSD_RKNSI_IT_T0_T1_EE_l166
+0000000000000000         *UND*	0000000000000000 __gxx_personality_v0

amd-omp-stack-err/objdump_works3.txt

@@ -0,0 +1,16 @@
+
+libWilsonFermionWorks3.a(WilsonFermionWorks3.o):	file format elf64-x86-64
+
+SYMBOL TABLE:
+0000000000000000 l    df *ABS*	0000000000000000 WilsonFermionWorks3.cc
+0000000000000000 l    d  .text.startup	0000000000000000 .text.startup
+0000000000000000 l     F .text.startup	0000000000000021 _GLOBAL__sub_I_WilsonFermionWorks3.cc
+0000000000000000 l     O .bss	0000000000000001 _ZStL8__ioinit
+0000000000000000 l    d  .bss	0000000000000000 .bss
+0000000000000001 l     O .bss	0000000000000001 _ZN5EigenL4lastE
+0000000000000002 l     O .bss	0000000000000002 _ZN5EigenL6lastp1E
+0000000000000004 l     O .bss	0000000000000001 _ZN5EigenL3allE
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitC1Ev
+0000000000000000         *UND*	0000000000000000 _ZNSt8ios_base4InitD1Ev
+0000000000000000         *UND*	0000000000000000 .hidden __dso_handle
+0000000000000000         *UND*	0000000000000000 __cxa_atexit

benchmarks/Benchmark_dwf.cc

@@ -84,7 +84,7 @@ int main (int argc, char ** argv)
   GridParallelRNG          RNG5(FGrid);  RNG5.SeedUniqueString(std::string("The 5D RNG"));
   std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
 
-  LatticeFermion src   (FGrid); random(RNG5,src);
+  LatticeFermion src   (FGrid);  random(RNG5,src);
 #if 0
   src = Zero();
   {
@@ -96,7 +96,9 @@ int main (int argc, char ** argv)
     pokeSite(tmp,src,origin);
   }
 #else
+  std::cout << GridLogMessage << "Drawing gauge field1" << std::endl;
   RealD N2 = 1.0/::sqrt(norm2(src));
+  std::cout << GridLogMessage << "Drawing gauge field3" << std::endl;
   src = src*N2;
 #endif
 
@@ -215,8 +217,12 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
     std::cout<<GridLogMessage << "RF  GiB/s (base 2) =   "<< 1000000. * data_rf/((t1-t0))<<std::endl;
     std::cout<<GridLogMessage << "mem GiB/s (base 2) =   "<< 1000000. * data_mem/((t1-t0))<<std::endl;
-    err = ref-result;
-    std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+//#pragma omp target is_device_ptr ( err.View(CpuWrite), ref.View(CpuWrite), result.View(CpuWrite) )
+    ref-result;
+    //err = ref-result;
+    
+    std::cout<<GridLogMessage << "norm diff 0  "<<std::endl;
+    std::cout<<GridLogMessage << "norm diff   "<< norm2(err) << " norm diff 1 " <<std::endl;
     //exit(0);
 
     if(( norm2(err)>1.0e-4) ) {

benchmarks/Benchmark_dwf_fp32.cc

@@ -90,11 +90,11 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "++++++++++++++++++++++++++++++++++++++++++++++++" <<std::endl;
 
   for(int d=0;d<Nd;d++) CommDim[d]= (mpi[d]/shm[d])>1 ? 1 : 0;
-  Dirichlet[0] = 0;
-  Dirichlet[1] = CommDim[0]*latt4[0]/mpi[0] * shm[0];
-  Dirichlet[2] = CommDim[1]*latt4[1]/mpi[1] * shm[1];
-  Dirichlet[3] = CommDim[2]*latt4[2]/mpi[2] * shm[2];
-  Dirichlet[4] = CommDim[3]*latt4[3]/mpi[3] * shm[3];
+  //  Dirichlet[0] = 0;
+  //  Dirichlet[1] = CommDim[0]*latt4[0]/mpi[0] * shm[0];
+  //  Dirichlet[2] = CommDim[1]*latt4[1]/mpi[1] * shm[1];
+  //  Dirichlet[3] = CommDim[2]*latt4[2]/mpi[2] * shm[2];
+  //  Dirichlet[4] = CommDim[3]*latt4[3]/mpi[3] * shm[3];
 
   Benchmark(Ls,Dirichlet);
 
@@ -105,11 +105,11 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "++++++++++++++++++++++++++++++++++++++++++++++++" <<std::endl;
 
   for(int d=0;d<Nd;d++) CommDim[d]= mpi[d]>1 ? 1 : 0;
-  Dirichlet[0] = 0;
-  Dirichlet[1] = CommDim[0]*latt4[0]/mpi[0];
-  Dirichlet[2] = CommDim[1]*latt4[1]/mpi[1];
-  Dirichlet[3] = CommDim[2]*latt4[2]/mpi[2];
-  Dirichlet[4] = CommDim[3]*latt4[3]/mpi[3];
+  //  Dirichlet[0] = 0;
+  //  Dirichlet[1] = CommDim[0]*latt4[0]/mpi[0];
+  //  Dirichlet[2] = CommDim[1]*latt4[1]/mpi[1];
+  //  Dirichlet[3] = CommDim[2]*latt4[2]/mpi[2];
+  //  Dirichlet[4] = CommDim[3]*latt4[3]/mpi[3];
   
   Benchmark(Ls,Dirichlet);
 
@@ -185,6 +185,7 @@ void Benchmark(int Ls, Coordinate Dirichlet)
   GaugeField Umu(UGrid);
   GaugeField UmuCopy(UGrid);
   SU<Nc>::HotConfiguration(RNG4,Umu);
+  //  SU<Nc>::ColdConfiguration(Umu);
   UmuCopy=Umu;
   std::cout << GridLogMessage << "Random gauge initialised " << std::endl;
 
@@ -307,6 +308,14 @@ void Benchmark(int Ls, Coordinate Dirichlet)
     if(( n2e>1.0e-4) ) {
       std::cout<<GridLogMessage << "WRONG RESULT" << std::endl;
       FGrid->Barrier();
+      std::cout<<GridLogMessage << "RESULT" << std::endl;
+      //      std::cout << result<<std::endl;
+      std::cout << norm2(result)<<std::endl;
+      std::cout<<GridLogMessage << "REF" << std::endl;
+      std::cout << norm2(ref)<<std::endl;
+      std::cout<<GridLogMessage << "ERR" << std::endl;
+      std::cout << norm2(err)<<std::endl;
+      FGrid->Barrier();
       exit(-1);
     }
     assert (n2e< 1.0e-4 );

benchmarks/Benchmark_su3.cc

@@ -36,19 +36,22 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
-#define LMAX (40)
+#define LMAX (8)
 #define LMIN (8)
 #define LADD (8)
 
-  int64_t Nwarm=10;
-  int64_t Nloop=100;
+  int64_t Nwarm=500;
+  int64_t Nloop=1500;
 
   Coordinate simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   Coordinate mpi_layout  = GridDefaultMpi();
 
   int64_t threads = GridThread::GetThreads();
-  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+  int64_t accelerator_threads = acceleratorThreads();
 
+  std::cout<<GridLogMessage << "Grid is setup with LMAX="<< LMAX << ", LMIN=" << LMIN << ", LADD=" << LADD << ", Nwarm, Nloop =" << Nwarm <<"," << Nloop <<std::endl;
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+  std::cout<<GridLogMessage << "Grid is setup to use "<<accelerator_threads<<" GPU threads"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking SU3xSU3  x= x*y"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
@@ -222,6 +225,7 @@ int main (int argc, char ** argv)
 
     }
 
+#if 1
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking SU3xSU3  CovShiftForward(z,x,y)"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
@@ -254,7 +258,9 @@ int main (int argc, char ** argv)
 	    std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t" << flops/time<<std::endl;
       }
   }
-#if 1
+#endif
+
+#if 0
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking SU3xSU3  z= x * Cshift(y)"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;

bootstrap.sh

@@ -1,25 +1,26 @@
 #!/usr/bin/env bash
 set -e
 
-EIGEN_URL='https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.bz2'
+EIGEN_URL='https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.bz2'
+##EIGEN_URL='https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.bz2'
 EIGEN_SHA256SUM='685adf14bd8e9c015b78097c1dc22f2f01343756f196acdc76a678e1ae352e11'
 
 
 echo "-- deploying Eigen source..."
 ARC=`basename ${EIGEN_URL}`
 wget ${EIGEN_URL} --no-check-certificate
-if command -v sha256sum; then
-   echo "$EIGEN_SHA256SUM  $(basename "$EIGEN_URL")" \
-      | sha256sum --check || exit 1
-else
-   echo "WARNING: could not verify checksum, please install sha256sum" >&2
-fi
+#if command -v sha256sum; then
+#   echo "$EIGEN_SHA256SUM  $(basename "$EIGEN_URL")" \
+#      | sha256sum --check || exit 1
+#else
+#   echo "WARNING: could not verify checksum, please install sha256sum" >&2
+#fi
 ./scripts/update_eigen.sh ${ARC}
 rm ${ARC}
 # patch for non-portable includes in Eigen 3.3.5
 # apparently already fixed in Eigen HEAD so it should not be 
 # a problem in the future (A.P.)
-patch Eigen/unsupported/Eigen/CXX11/Tensor scripts/eigen-3.3.5.Tensor.patch
+# patch Eigen/unsupported/Eigen/CXX11/Tensor scripts/eigen-3.3.5.Tensor.patch
 
 echo '-- generating Make.inc files...'
 ./scripts/filelist

config-command

@@ -0,0 +1,8 @@
+../configure \
+    --enable-comms=none \
+    --enable-simd=GEN \
+    --enable-gen-simd-width=16 \
+    CXX=clang++ \
+    LDFLAGS="-lcudart " \
+    CXXFLAGS="-fopenmp -std=c++14 -fopenmp-cuda-mode  -O3 -target x86_64-pc-linux-gnu -fopenmp-targets=nvptx64-nvidia-cuda -lcudart -Xopenmp-target=nvptx64-nvidia-cuda -march=sm_70 -DOMPTARGET -DOMPTARGET_MANAGED"
+

configure.ac

@@ -247,13 +247,13 @@ case ${ac_ACC_CSHIFT} in
 esac
 
 
-############### SYCL/CUDA/HIP/none
+############### SYCL/CUDA/HIP/OpenMP/none
 AC_ARG_ENABLE([accelerator],
-    [AS_HELP_STRING([--enable-accelerator=cuda|sycl|hip|none],[enable none,cuda,sycl,hip acceleration])],
+    [AS_HELP_STRING([--enable-accelerator=cuda|sycl|hip|openmp|none],[enable none,openmp,cuda,sycl,hip acceleration])],
     [ac_ACCELERATOR=${enable_accelerator}], [ac_ACCELERATOR=none])
 case ${ac_ACCELERATOR} in
     cuda)
-      echo CUDA acceleration
+      echo CUDA acceleration ${ac_ACCELERATOR} ${enable_accelerator}
       LIBS="${LIBS} -lcuda"
       AC_DEFINE([GRID_CUDA],[1],[Use CUDA offload]);;
     sycl)
@@ -262,12 +262,15 @@ case ${ac_ACCELERATOR} in
     hip)
       echo HIP acceleration
       AC_DEFINE([GRID_HIP],[1],[Use HIP offload]);;
+    openmp)
+      echo OMPTARGET acceleration
+      AC_DEFINE([GRID_OMPTARGET],[1],[Use OMPTARGET offload]);;
     none)
       echo NO acceleration    ;;
     no)
       echo NO acceleration    ;;
     *)
-      AC_MSG_ERROR(["Acceleration not suppoorted ${ac_ACCELERATOR}"]);;
+      AC_MSG_ERROR(["1Acceleration not suppoorted ${ac_ACCELERATOR}"]);;
 esac
 
 ############### UNIFIED MEMORY

load_cgpu_modules.sh

@@ -0,0 +1,6 @@
+#!/bin/sh -f
+
+module load cgpu 
+module load gcc/8.3.0
+module load cuda/11.0.3
+module load llvm/14.0.0-git_20211104b

systems/Aurora/benchmarks/bench1024.pbs

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+## qsub -q EarlyAppAccess -A Aurora_Deployment -I -l select=1 -l walltime=60:00
+
+#PBS -q EarlyAppAccess
+#PBS -l select=1024
+#PBS -l walltime=01:00:00
+#PBS -A LatticeQCD_aesp_CNDA
+
+#export OMP_PROC_BIND=spread
+#unset OMP_PLACES
+
+cd $PBS_O_WORKDIR
+
+source ../sourceme.sh
+
+cat $PBS_NODEFILE
+
+export OMP_NUM_THREADS=3
+export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
+
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE
+#unset MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST
+
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
+export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
+export MPICH_OFI_NIC_POLICY=GPU
+
+# 12 ppn, 32 nodes, 384 ranks
+#
+CMD="mpiexec -np 12288 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_comms_host_device --mpi 8.6.16.16 --grid 64.48.64.284 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32"
+
+$CMD 
+
+CMD="mpiexec -np 12288 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 8.8.8.24 --grid 128.128.128.384 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 1024node.dwf.small
+
+CMD="mpiexec -np 12288 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 16.8.8.12 --grid 256.256.256.384 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 1024node.dwf
+
+

systems/Aurora/benchmarks/bench12.pbs

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+## qsub -q EarlyAppAccess -A Aurora_Deployment -I -l select=1 -l walltime=60:00
+
+#PBS -q EarlyAppAccess
+#PBS -l select=2
+#PBS -l walltime=01:00:00
+#PBS -A LatticeQCD_aesp_CNDA
+
+#export OMP_PROC_BIND=spread
+#unset OMP_PLACES
+
+cd $PBS_O_WORKDIR
+
+source ../sourceme.sh
+
+export OMP_NUM_THREADS=3
+export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
+
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE
+#unset MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST
+
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
+export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
+export MPICH_OFI_NIC_POLICY=GPU
+
+CMD="mpiexec -np 24 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_comms_host_device --mpi 2.3.2.2 --grid 32.24.32.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32"
+
+$CMD 
+
+CMD="mpiexec -np 24 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 2.3.2.2 --grid 64.96.64.64 --comms-overlap \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32"
+
+$CMD 

systems/Aurora/benchmarks/bench2048.pbs

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+## qsub -q EarlyAppAccess -A Aurora_Deployment -I -l select=1 -l walltime=60:00
+
+#PBS -q EarlyAppAccess
+#PBS -l select=2048
+#PBS -l walltime=01:00:00
+#PBS -A LatticeQCD_aesp_CNDA
+
+#export OMP_PROC_BIND=spread
+#unset OMP_PLACES
+
+cd $PBS_O_WORKDIR
+
+source ../sourceme.sh
+
+cat $PBS_NODEFILE
+
+export OMP_NUM_THREADS=3
+export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
+
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE
+#unset MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST
+
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
+export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
+export MPICH_OFI_NIC_POLICY=GPU
+
+# 12 ppn, 32 nodes, 384 ranks
+#
+CMD="mpiexec -np 24576 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_comms_host_device --mpi 8.12.16.16 --grid 64.48.64.284 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32"
+
+$CMD 
+
+CMD="mpiexec -np 24576 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 16.8.8.24 --grid 128.128.128.384 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 2048node.dwf.small
+
+CMD="mpiexec -np 24576 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 16.8.8.24 --grid 256.256.256.768 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 2048node.dwf
+
+

systems/Aurora/benchmarks/bench256.pbs

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+## qsub -q EarlyAppAccess -A Aurora_Deployment -I -l select=1 -l walltime=60:00
+
+#PBS -q EarlyAppAccess
+#PBS -l select=256
+#PBS -l walltime=01:00:00
+#PBS -A LatticeQCD_aesp_CNDA
+
+#export OMP_PROC_BIND=spread
+#unset OMP_PLACES
+
+cd $PBS_O_WORKDIR
+
+source ../sourceme.sh
+
+cat $PBS_NODEFILE
+
+export OMP_NUM_THREADS=3
+export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
+
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE
+#unset MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST
+
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
+export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
+export MPICH_OFI_NIC_POLICY=GPU
+
+# 12 ppn, 32 nodes, 384 ranks
+#
+CMD="mpiexec -np 3072 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_comms_host_device --mpi 8.6.8.8 --grid 32.24.32.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32"
+
+$CMD 
+
+CMD="mpiexec -np 3072 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 8.8.4.12 --grid 128.128.128.768 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 256node.dwf.large

systems/Aurora/benchmarks/bench512.pbs

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+## qsub -q EarlyAppAccess -A Aurora_Deployment -I -l select=1 -l walltime=60:00
+
+#PBS -q EarlyAppAccess
+#PBS -l select=512
+#PBS -l walltime=01:00:00
+#PBS -A LatticeQCD_aesp_CNDA
+
+#export OMP_PROC_BIND=spread
+#unset OMP_PLACES
+
+cd $PBS_O_WORKDIR
+
+source ../sourceme.sh
+
+cat $PBS_NODEFILE
+
+export OMP_NUM_THREADS=3
+export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
+
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE
+#unset MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST
+
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
+export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
+export MPICH_OFI_NIC_POLICY=GPU
+
+# 12 ppn, 32 nodes, 384 ranks
+#
+CMD="mpiexec -np 6144 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_comms_host_device --mpi 8.6.8.16 --grid 32.24.32.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32"
+
+$CMD 
+
+CMD="mpiexec -np 6144 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 8.8.8.12 --grid 256.128.128.768 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 512node.dwf.large

systems/Aurora/benchmarks/bench_scaling.pbs

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+## qsub -q EarlyAppAccess -A Aurora_Deployment -I -l select=1 -l walltime=60:00
+
+#PBS -q EarlyAppAccess
+#PBS -l select=32
+#PBS -l walltime=01:00:00
+#PBS -A LatticeQCD_aesp_CNDA
+
+#export OMP_PROC_BIND=spread
+#unset OMP_PLACES
+
+cd $PBS_O_WORKDIR
+
+source ../sourceme.sh
+
+cat $PBS_NODEFILE
+
+export OMP_NUM_THREADS=3
+export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
+
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE
+#unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE
+#unset MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST
+
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
+export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
+export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
+export MPICH_OFI_NIC_POLICY=GPU
+
+# 12 ppn, 32 nodes, 384 ranks
+#
+CMD="mpiexec -np 384 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_comms_host_device --mpi 4.6.4.4 --grid 32.24.32.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32"
+
+$CMD 
+
+CMD="mpiexec -np 12 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 1.2.2.3 --grid 16.64.64.96 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 1node.dwf
+
+
+CMD="mpiexec -np 24 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 2.2.2.3 --grid 32.64.64.96 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 2node.dwf
+
+CMD="mpiexec -np 48 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 2.2.2.6 --grid 32.64.64.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 4node.dwf
+
+CMD="mpiexec -np 96 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 2.2.4.6 --grid 32.64.128.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 8node.dwf
+
+CMD="mpiexec -np 192 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 2.4.4.6 --grid 32.128.128.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 16node.dwf
+
+
+CMD="mpiexec -np 384 -ppn 12  -envall \
+	     ./gpu_tile_compact.sh \
+	     ./Benchmark_dwf_fp32 --mpi 4.4.4.6 --grid 64.128.128.192 \
+		--shm-mpi 1 --shm 2048 --device-mem 32000 --accelerator-threads 32 --comms-overlap"
+$CMD | tee 32node.dwf

systems/Aurora/benchmarks/gpu_tile_compact.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+export NUMA_MAP=(2 2 2 3 3 3 2 2 2 3 3 3 )
+#export NUMA_MAP=(0 0 0 1 1 1 0 0 0 1 1 1 )
+export NUMA_PMAP=(0 0 0 1 1 1 0 0 0 1 1 1 )
+export  NIC_MAP=(0 1 2 4 5 6 0 1 2 4 5 6 )
+export  GPU_MAP=(0 1 2 3 4 5 0 1 2 3 4 5 )
+export TILE_MAP=(0 0 0 0 0 0 1 1 1 1 1 1 )
+
+export NUMA=${NUMA_MAP[$PALS_LOCAL_RANKID]}
+export NUMAP=${NUMA_PMAP[$PALS_LOCAL_RANKID]}
+export NIC=${NIC_MAP[$PALS_LOCAL_RANKID]}
+export gpu_id=${GPU_MAP[$PALS_LOCAL_RANKID]}
+export tile_id=${TILE_MAP[$PALS_LOCAL_RANKID]}
+  
+#export GRID_MPICH_NIC_BIND=$NIC
+#export ONEAPI_DEVICE_SELECTOR=level_zero:$gpu_id.$tile_id
+
+unset EnableWalkerPartition
+export EnableImplicitScaling=0
+export ZE_AFFINITY_MASK=$gpu_id.$tile_id
+export ONEAPI_DEVICE_FILTER=gpu,level_zero
+
+#export ZE_ENABLE_PCI_ID_DEVICE_ORDER=1
+#export SYCL_PI_LEVEL_ZERO_DEVICE_SCOPE_EVENTS=0
+#export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+#export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE=0:2
+#export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE_FOR_D2D_COPY=1
+#export SYCL_PI_LEVEL_ZERO_USM_RESIDENT=1
+
+#echo "rank $PALS_RANKID ; local rank $PALS_LOCAL_RANKID ; ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK ; NUMA $NUMA "
+
+numactl -m $NUMA -N $NUMAP  "$@"

systems/Aurora/benchmarks/gpu_tile_compact4.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+export  NUMA_MAP=(2 2 3 3  2 2  3 3  )
+export  PROC_MAP=(0 0 1 1  0 0  1 1  )
+export  NIC_MAP=(0 0  4 4  1 1  5 5  )
+export  GPU_MAP=(0 1  3 4  0 1  3 4  )
+export TILE_MAP=(0 0  0 0  1 1  1 1  )
+export NUMA=${NUMA_MAP[$PALS_LOCAL_RANKID]}
+export NIC=${NIC_MAP[$PALS_LOCAL_RANKID]}
+export gpu_id=${GPU_MAP[$PALS_LOCAL_RANKID]}
+export tile_id=${TILE_MAP[$PALS_LOCAL_RANKID]}
+  
+#export GRID_MPICH_NIC_BIND=$NIC
+
+unset EnableWalkerPartition
+export EnableImplicitScaling=0
+export ZE_ENABLE_PCI_ID_DEVICE_ORDER=1
+export ZE_AFFINITY_MASK=$gpu_id.$tile_id
+#export ONEAPI_DEVICE_SELECTOR=level_zero:$gpu_id.$tile_id
+export ONEAPI_DEVICE_FILTER=gpu,level_zero
+export SYCL_PI_LEVEL_ZERO_DEVICE_SCOPE_EVENTS=0
+export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE=0:2
+export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE_FOR_D2D_COPY=1
+#export SYCL_PI_LEVEL_ZERO_USM_RESIDENT=1
+
+echo "rank $PALS_RANKID ; local rank $PALS_LOCAL_RANKID ; ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK ; NIC $GRID_MPICH_NIC_BIND ; NUMA domain $NUMA"
+
+numactl -m $NUMA -N $PROC_MAP  "$@"

systems/Aurora/config-command

@@ -0,0 +1,16 @@
+TOOLS=$HOME/tools
+../../configure \
+	--enable-simd=GPU \
+	--enable-gen-simd-width=64 \
+	--enable-comms=mpi-auto \
+	--enable-accelerator-cshift \
+	--disable-gparity \
+	--disable-fermion-reps \
+	--enable-shm=nvlink \
+	--enable-accelerator=sycl \
+	--enable-unified=no \
+	MPICXX=mpicxx \
+	CXX=icpx \
+	LDFLAGS="-fiopenmp -fsycl -fsycl-device-code-split=per_kernel -fsycl-device-lib=all -lze_loader -L$TOOLS/lib64/" \
+	CXXFLAGS="-fiopenmp -fsycl-unnamed-lambda -fsycl -I$INSTALL/include -Wno-tautological-compare -I$HOME/ -I$TOOLS/include"
+

systems/Aurora/proxies.sh

@@ -0,0 +1,9 @@
+export HTTP_PROXY=http://proxy.alcf.anl.gov:3128
+export HTTPS_PROXY=http://proxy.alcf.anl.gov:3128
+export http_proxy=http://proxy.alcf.anl.gov:3128
+export https_proxy=http://proxy.alcf.anl.gov:3128
+export MPIR_CVAR_CH4_OFI_ENABLE_HMEM=1
+git config --global http.proxy http://proxy.alcf.anl.gov:3128
+module use /soft/modulefiles
+module load intel_compute_runtime/release/agama-devel-682.22
+ 

systems/Aurora/sourceme.sh

@@ -0,0 +1,12 @@
+#export ONEAPI_DEVICE_SELECTOR=level_zero:0.0
+
+module use /soft/modulefiles
+module load intel_compute_runtime/release/agama-devel-682.22
+
+export HTTP_PROXY=http://proxy.alcf.anl.gov:3128
+export HTTPS_PROXY=http://proxy.alcf.anl.gov:3128
+export http_proxy=http://proxy.alcf.anl.gov:3128
+export https_proxy=http://proxy.alcf.anl.gov:3128
+#export MPIR_CVAR_CH4_OFI_ENABLE_HMEM=1
+git config --global http.proxy http://proxy.alcf.anl.gov:3128
+ 

systems/Frontier/config-command

@@ -0,0 +1,23 @@
+CLIME=`spack find --paths c-lime@2-3-9 | grep c-lime| cut -c 15-`
+../../configure --enable-comms=mpi-auto \
+--with-lime=$CLIME \
+--enable-unified=no \
+--enable-shm=nvlink \
+--enable-tracing=timer \
+--enable-accelerator=hip \
+--enable-gen-simd-width=64 \
+--disable-gparity \
+--disable-fermion-reps \
+--enable-simd=GPU \
+--enable-accelerator-cshift \
+--with-gmp=$OLCF_GMP_ROOT \
+--with-fftw=$FFTW_DIR/.. \
+--with-mpfr=/opt/cray/pe/gcc/mpfr/3.1.4/ \
+--disable-fermion-reps \
+CXX=hipcc MPICXX=mpicxx \
+CXXFLAGS="-fPIC -I{$ROCM_PATH}/include/ -I${MPICH_DIR}/include -L/lib64 -fgpu-sanitize" \
+ LDFLAGS="-L/lib64 -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa -lamdhip64 "
+
+
+
+

systems/Frontier/sourceme.sh

@@ -0,0 +1,13 @@
+. /autofs/nccs-svm1_home1/paboyle/Crusher/Grid/spack/share/spack/setup-env.sh
+spack load c-lime
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/crusher/spack-envs/base/opt/cray-sles15-zen3/gcc-11.2.0/gperftools-2.9.1-72ubwtuc5wcz2meqltbfdb76epufgzo2/lib
+module load emacs 
+module load PrgEnv-gnu
+module load rocm
+module load cray-mpich/8.1.23
+module load gmp
+module load cray-fftw
+module load craype-accel-amd-gfx90a
+export LD_LIBRARY_PATH=/opt/gcc/mpfr/3.1.4/lib:$LD_LIBRARY_PATH
+#Hack for lib
+#export LD_LIBRARY_PATH=`pwd`:$LD_LIBRARY_PATH

systems/Lumi/HMC/32cube/fthmc3gev.slurm

@@ -0,0 +1,57 @@
+#!/bin/bash -l
+#SBATCH --job-name=fthmc3ge
+#SBATCH --partition=small-g
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+##SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-node=8
+#SBATCH --time=2:00:00
+#SBATCH --account=project_465000546
+#SBATCH --gpu-bind=none
+#SBATCH --exclusive
+#SBATCH --mem=0
+
+
+#sbatch --dependency=afterany:$SLURM_JOBID fthmc3gev.slurm
+
+CPU_BIND="map_ldom:3,3,1,1,0,0,2,2"
+MEM_BIND="map_mem:3,3,1,1,0,0,2,2"
+echo $CPU_BIND
+
+cat << EOF > ./select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3 4 5 6 7)
+export NUMA_MAP=(3 3 1 1 0 0 2 2)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export NUM=\${NUMA_MAP[\$SLURM_LOCALID]}
+#export HIP_VISIBLE_DEVICES=\$GPU
+export ROCR_VISIBLE_DEVICES=\$GPU
+echo RANK \$SLURM_LOCALID using GPU \$GPU    
+echo NUMA \$SLURM_LOCALID using NUMA \${NUM}
+echo numactl -m \$NUM -N \$NUM \$*
+exec numactl -m \$NUM -N \$NUM \$*
+EOF
+cat ./select_gpu
+
+chmod +x ./select_gpu
+
+root=/scratch/project_465000546/boylepet/Grid/systems/Lumi
+source ${root}/sourceme.sh
+
+export OMP_NUM_THREADS=7
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+#cfg=`ls -rt ckpoint_*lat* | tail -n 1  `
+#traj="${cfg#*.}"
+#cfg=`ls -rt ckpoint_*lat* | tail -n 1  `
+traj=0
+
+vol=32.32.32.64
+mpi=1.2.2.2
+PARAMS="--mpi $mpi --accelerator-threads 16 --comms-sequential --shm 2048 --shm-mpi 0 --grid $vol"
+#HMCPARAMS="--StartingType CheckpointStart --StartingTrajectory $traj --Trajectories 200"
+HMCPARAMS="--StartingType ColdStart --StartingTrajectory $traj --Trajectories 20"
+
+srun ./select_gpu ../FTHMC2p1f_3GeV $HMCPARAMS $PARAMS
+

systems/Lumi/config-command

@@ -23,7 +23,7 @@ echo mpfr X$MPFR
 --disable-fermion-reps \
 --disable-gparity \
 CXX=hipcc MPICXX=mpicxx \
-  CXXFLAGS="-fPIC --offload-arch=gfx90a -I/opt/rocm/include/ -std=c++14 -I/opt/cray/pe/mpich/8.1.23/ofi/gnu/9.1/include" \
+  CXXFLAGS="-fPIC --offload-arch=gfx90a -I/opt/rocm/include/ -std=c++17 -I/opt/cray/pe/mpich/8.1.23/ofi/gnu/9.1/include" \
   LDFLAGS="-L/opt/cray/pe/mpich/8.1.23/ofi/gnu/9.1/lib -lmpi -L/opt/cray/pe/mpich/8.1.23/gtl/lib -lmpi_gtl_hsa -lamdhip64 -fopenmp" 
 
 

systems/SDCC-A100/bench.slurm

@@ -0,0 +1,42 @@
+#!/bin/bash
+#SBATCH --partition csi
+#SBATCH --time=00:10:00
+#SBATCH -A csigeneral
+#SBATCH --exclusive
+#SBATCH --nodes=1
+#SBATCH --ntasks=4
+#SBATCH --qos csi
+#SBATCH --gres=gpu:4
+
+source sourceme.sh
+
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export CUDA_VISIBLE_DEVICES=\$GPU
+unset ROCR_VISIBLE_DEVICES
+echo RANK \$SLURM_LOCALID using GPU \$GPU    
+exec \$*
+EOF
+chmod +x ./select_gpu
+
+
+export OMP_NUM_THREADS=4
+export OMPI_MCA_btl=^uct,openib
+export UCX_TLS=cuda,gdr_copy,rc,rc_x,sm,cuda_copy,cuda_ipc
+export UCX_RNDV_SCHEME=put_zcopy
+export UCX_RNDV_THRESH=16384
+export UCX_IB_GPU_DIRECT_RDMA=no
+export UCX_MEMTYPE_CACHE=n
+
+export OMP_NUM_THREAD=8
+#srun -N1 -n1 nvidia-smi
+#srun -N1 -n1 numactl -H > numa.txt
+srun -N1 -n1 lstopo A100-topo.pdf
+
+# 4.35 TF/s
+#srun -N1 -n1 ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.1.1 --grid 16.32.32.32 --shm 2048 --shm-mpi 0  --accelerator-threads 16
+
+srun -N1 -n4 ./select_gpu ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.2.2 --grid 32.32.64.64 --shm 2048 --shm-mpi 0  --accelerator-threads 16
+

systems/SDCC-A100/config-command

@@ -0,0 +1,17 @@
+../../configure \
+--enable-comms=mpi-auto \
+--enable-unified=no \
+--enable-shm=nvlink \
+--enable-accelerator=cuda \
+--enable-gen-simd-width=64 \
+--enable-simd=GPU \
+--disable-accelerator-cshift \
+--disable-fermion-reps \
+--disable-gparity \
+CXX=nvcc \
+MPICXX=mpicxx \
+LDFLAGS="-cudart shared " \
+CXXFLAGS="-ccbin mpicxx -gencode arch=compute_80,code=sm_80 -std=c++17 -cudart shared"
+
+
+

systems/SDCC-A100/sourceme.sh

@@ -0,0 +1,2 @@
+module load cuda/12.2
+module load openmpi

systems/SDCC-ARM/config-command-mpi

@@ -0,0 +1,6 @@
+HDF=$HOME/paboyle/install
+
+LDFLAGS=-L$HDF/lib CXX=clang++ ../../configure --enable-simd=NEONv8 --enable-comms=none --enable-unified=yes --disable-fermion-reps --disable-gparity --disable-debug --with-hdf5=$HDF 
+#LDFLAGS=-L$HDF/lib CXX=clang++ ../../configure --enable-simd=GEN --enable-comms=none --enable-unified=yes --disable-fermion-reps --disable-gparity --disable-debug --with-hdf5=$HDF 
+
+

systems/SDCC-ICE/bench.slurm

@@ -0,0 +1,31 @@
+#!/bin/bash
+#SBATCH --partition lqcd
+#SBATCH --time=00:20:00
+#SBATCH -A lqcdtest
+#SBATCH --exclusive
+#SBATCH --nodes=1
+#SBATCH --ntasks=2
+#SBATCH --qos lqcd
+
+source sourceme.sh
+
+export OMP_NUM_THREAD=24
+#srun -N1 -n1 numactl -H > numa.txt
+#srun -N1 -n1 lstopo ice-topo.pdf
+
+cat << EOF > select_socket
+#!/bin/bash
+export NUM_MAP=(0 1)
+export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
+exec \$*
+EOF
+chmod +x ./select_socket
+
+#for vol in 8.8.8.16 8.8.8.32 8.8.8.64
+#for vol in 8.8.16.16 8.8.16.32 8.8.16.64
+for vol in 8.16.16.16 8.16.16.32 8.16.16.64 16.16.16.32 16.16.16.64 24.24.24.64 32.32.32.32
+do
+srun --cpu-bind=ldoms -N1 -n2 ./select_socket ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.1.2 --grid $vol --dslash-asm > $vol.2socket.out
+srun --cpu-bind=ldoms -N1 -n1 ./select_socket ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.1.1 --grid $vol --dslash-asm > $vol.1socket.out
+done
+

systems/SDCC-ICE/config-command

@@ -0,0 +1,19 @@
+../../configure \
+--enable-debug \
+--enable-comms=mpi-auto \
+--enable-unified=yes \
+--enable-shm=shmopen \
+--enable-shm-fast-path=shmopen \
+--enable-accelerator=none \
+--enable-simd=AVX512 \
+--disable-accelerator-cshift \
+--disable-fermion-reps \
+--disable-gparity \
+CXX=clang++ \
+MPICXX=mpicxx \
+LDFLAGS=-L/direct/sdcc+u/paboyle/spack/opt/spack/linux-almalinux8-icelake/gcc-8.5.0/hwloc-2.9.1-hgkscnt5pferhtde4ahctlupb6qf3vtl/lib/ \
+LIBS=-lhwloc \
+CXXFLAGS="-std=c++17"
+
+
+

systems/SDCC-ICE/sourceme.sh

@@ -0,0 +1,2 @@
+export LD_LIBRARY_PATH=/direct/sdcc+u/paboyle/spack/opt/spack/linux-almalinux8-icelake/gcc-8.5.0/llvm-12.0.1-agey6vtuw3e375rewhhobvkznjh5ltz4/lib/:$LD_LIBRARY_PATH
+module load openmpi