Merge pull request #50 from waterret/develop

use sha256 based splittable rng

.gitignore

@@ -47,9 +47,7 @@ Config.h.in
 config.log
 config.status
 .deps
-Make.inc
-eigen.inc
-Eigen.inc
+*.inc
 
 # http://www.gnu.org/software/autoconf #
 ########################################

Makefile.am

@@ -1,12 +1,5 @@
 # additional include paths necessary to compile the C++ library
 SUBDIRS = lib benchmarks tests
 
-include $(top_srcdir)/doxygen.inc
-
-tests: all
-	$(MAKE) -C tests tests
-
-.PHONY: tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
-
 AM_CXXFLAGS += -I$(top_builddir)/include
 ACLOCAL_AMFLAGS = -I m4

README

@@ -1 +0,0 @@
-README.md

README

@@ -0,0 +1,44 @@
+This library provides data parallel C++ container classes with internal memory layout
+that is transformed to map efficiently to SIMD architectures. CSHIFT facilities
+are provided, similar to HPF and cmfortran, and user control is given over the mapping of
+array indices to both MPI tasks and SIMD processing elements.
+
+* Identically shaped arrays then be processed with perfect data parallelisation.
+* Such identically shapped arrays are called conformable arrays.
+
+The transformation is based on the observation that Cartesian array processing involves
+identical processing to be performed on different regions of the Cartesian array.
+
+The library will (eventually) both geometrically decompose into MPI tasks and across SIMD lanes.
+
+Data parallel array operations can then be specified with a SINGLE data parallel paradigm, but
+optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a significant simplification
+for most programmers.
+
+The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture.
+Presently SSE2 (128 bit) AVX, AVX2 (256 bit) and IMCI and AVX512 (512 bit) targets are supported.
+
+These are presented as 
+
+  vRealF, vRealD, vComplexF, vComplexD 
+
+internal vector data types. These may be useful in themselves for other programmers.
+The corresponding scalar types are named
+
+  RealF, RealD, ComplexF, ComplexD
+
+MPI parallelism is UNIMPLEMENTED and for now only OpenMP and SIMD parallelism is present in the library.
+
+   You can give `configure' initial values for configuration parameters
+by setting variables in the command line or in the environment.  Here
+is are examples:
+
+     ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -msse4" --enable-simd=SSE4
+
+     ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -mavx" --enable-simd=AVX1
+
+     ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -mavx2" --enable-simd=AVX2
+
+     ./configure CXX=icpc CXXFLAGS="-std=c++11 -O3 -mmic" --enable-simd=AVX512 --host=none
+     
+     

README.md

@@ -16,27 +16,11 @@
 
 **Data parallel C++ mathematical object library.**
 
+Please send all pull requests to the `develop` branch.
+
 License: GPL v2.
 
-Last update Nov 2016.
-
-_Please do not send pull requests to the `master` branch which is reserved for releases._
-
-### Bug report
-
-_To help us tracking and solving more efficiently issues with Grid, please report problems using the issue system of GitHub rather than sending emails to Grid developers._
-
-When you file an issue, please go though the following checklist:
-
-1. Check that the code is pointing to the `HEAD` of `develop` or any commit in `master` which is tagged with a version number. 
-2. Give a description of the target platform (CPU, network, compiler). Please give the full CPU part description, using for example `cat /proc/cpuinfo | grep 'model name' | uniq` (Linux) or `sysctl machdep.cpu.brand_string` (macOS) and the full output the `--version` option of your compiler.
-3. Give the exact `configure` command used.
-4. Attach `config.log`.
-5. Attach `config.summary`.
-6. Attach the output of `make V=1`.
-7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example.
-
-
+Last update 2016/08/03.
 
 ### Description
 This library provides data parallel C++ container classes with internal memory layout
@@ -45,7 +29,7 @@ are provided, similar to HPF and cmfortran, and user control is given over the m
 array indices to both MPI tasks and SIMD processing elements.
 
 * Identically shaped arrays then be processed with perfect data parallelisation.
-* Such identically shaped arrays are called conformable arrays.
+* Such identically shapped arrays are called conformable arrays.
 
 The transformation is based on the observation that Cartesian array processing involves
 identical processing to be performed on different regions of the Cartesian array.
@@ -58,7 +42,7 @@ optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a signifi
 for most programmers.
 
 The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture.
-Presently SSE4 (128 bit) AVX, AVX2, QPX (256 bit), IMCI, and AVX512 (512 bit) targets are supported (ARM NEON on the way).
+Presently SSE4 (128 bit) AVX, AVX2 (256 bit) and IMCI and AVX512 (512 bit) targets are supported (ARM NEON and BG/Q QPX on the way).
 
 These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. These may be useful in themselves for other programmers.
 The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`.
@@ -66,7 +50,7 @@ The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `Compl
 MPI, OpenMP, and SIMD parallelism are present in the library.
 Please see https://arxiv.org/abs/1512.03487 for more detail.
 
-### Quick start
+### Installation
 First, start by cloning the repository:
 
 ``` bash
@@ -87,10 +71,12 @@ mkdir build; cd build
 ../configure --enable-precision=double --enable-simd=AVX --enable-comms=mpi-auto --prefix=<path>
 ```
 
-where `--enable-precision=` set the default precision,
-`--enable-simd=` set the SIMD type, `--enable-
-comms=`, and `<path>` should be replaced by the prefix path where you want to
-install Grid. Other options are detailed in the next section, you can also use `configure
+where `--enable-precision=` set the default precision (`single` or `double`),
+`--enable-simd=` set the SIMD type (see possible values below), `--enable-
+comms=` set the protocol used for communications (`none`, `mpi`, `mpi-auto` or
+`shmem`), and `<path>` should be replaced by the prefix path where you want to
+install Grid. The `mpi-auto` communication option set `configure` to determine
+automatically how to link to MPI. Other options are available, use `configure
 --help` to display them. Like with any other program using GNU autotool, the
 `CXX`, `CXXFLAGS`, `LDFLAGS`, ... environment variables can be modified to
 customise the build.
@@ -106,91 +92,25 @@ To minimise the build time, only the tests at the root of the `tests` directory
 ``` bash
 make -C tests/<subdir> tests
 ```
-If you want to build all the tests at once just use `make tests`.
-
-### Build configuration options
-
-- `--prefix=<path>`: installation prefix for Grid.
-- `--with-gmp=<path>`: look for GMP in the UNIX prefix `<path>`
-- `--with-mpfr=<path>`: look for MPFR in the UNIX prefix `<path>`
-- `--with-fftw=<path>`: look for FFTW in the UNIX prefix `<path>`
-- `--enable-lapack[=<path>]`: enable LAPACK support in Lanczos eigensolver. A UNIX prefix containing the library can be specified (optional).
-- `--enable-mkl[=<path>]`: use Intel MKL for FFT (and LAPACK if enabled) routines. A UNIX prefix containing the library can be specified (optional).
-- `--enable-numa`: enable NUMA first touch optimisation
-- `--enable-simd=<code>`: setup Grid for the SIMD target `<code>` (default: `GEN`). A list of possible SIMD targets is detailed in a section below.
-- `--enable-gen-simd-width=<size>`: select the size (in bytes) of the generic SIMD vector type (default: 32 bytes).
-- `--enable-precision={single|double}`: set the default precision (default: `double`).
-- `--enable-precision=<comm>`: Use `<comm>` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below.
-- `--enable-rng={ranlux48|mt19937}`: choose the RNG (default: `ranlux48 `).
-- `--disable-timers`: disable system dependent high-resolution timers.
-- `--enable-chroma`: enable Chroma regression tests.
-- `--enable-doxygen-doc`: enable the Doxygen documentation generation (build with `make doxygen-doc`)
-
-### Possible communication interfaces
-
-The following options can be use with the `--enable-comms=` option to target different communication interfaces:
-
-| `<comm>`       | Description                                                   |
-| -------------- | ------------------------------------------------------------- |
-| `none`         | no communications                                             |
-| `mpi[-auto]`   | MPI communications                                            |
-| `mpi3[-auto]`  | MPI communications using MPI 3 shared memory                  |
-| `mpi3l[-auto]` | MPI communications using MPI 3 shared memory and leader model |
-| `shmem `       | Cray SHMEM communications                                     |
-
-For the MPI interfaces the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names). The `-auto` suffix is not supported by the Cray environment wrapper scripts. Use the standard versions instead.  
 
 ### Possible SIMD types
 
 The following options can be use with the `--enable-simd=` option to target different SIMD instruction sets:
 
-| `<code>`    | Description                            |
+| String      | Description                            |
 | ----------- | -------------------------------------- |
 | `GEN`       | generic portable vector code           |
 | `SSE4`      | SSE 4.2 (128 bit)                      |
 | `AVX`       | AVX (256 bit)                          |
-| `AVXFMA`    | AVX (256 bit) + FMA                    |
-| `AVXFMA4`   | AVX (256 bit) + FMA4                   |
+| `AVXFMA4`   | AVX (256 bit) + FMA                    |
 | `AVX2`      | AVX 2 (256 bit)                        |
 | `AVX512`    | AVX 512 bit                            |
-| `QPX`       | QPX (256 bit)                          |
+| `AVX512MIC` | AVX 512 bit for Intel MIC architecture |
+| `ICMI`      | Intel ICMI instructions (512 bit)      |
 
 Alternatively, some CPU codenames can be directly used:
 
-| `<code>`    | Description                            |
+| String      | Description                            |
 | ----------- | -------------------------------------- |
-| `KNC`       | [Intel Xeon Phi codename Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) |
-| `KNL`       | [Intel Xeon Phi codename Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |
-| `BGQ`       | Blue Gene/Q                            |
-
-#### Notes:
-- We currently support AVX512 only for the Intel compiler. Support for GCC and clang will appear in future versions of Grid when the AVX512 support within GCC and clang will be more advanced.
-- For BG/Q only [bgclang](http://trac.alcf.anl.gov/projects/llvm-bgq) is supported. We do not presently plan to support more compilers for this platform.
-- BG/Q performances are currently rather poor. This is being investigated for future versions.
-- The vector size for the `GEN` target can be specified with the `configure` script option `--enable-gen-simd-width`.
-
-### Build setup for Intel Knights Landing platform
-
-The following configuration is recommended for the Intel Knights Landing platform:
-
-``` bash
-../configure --enable-precision=double\
-             --enable-simd=KNL        \
-             --enable-comms=mpi-auto \
-             --with-gmp=<path>        \
-             --with-mpfr=<path>       \
-             --enable-mkl             \
-             CXX=icpc MPICXX=mpiicpc
-```
-
-where `<path>` is the UNIX prefix where GMP and MPFR are installed. If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
-
-``` bash
-../configure --enable-precision=double\
-             --enable-simd=KNL        \
-             --enable-comms=mpi       \
-             --with-gmp=<path>        \
-             --with-mpfr=<path>       \
-             --enable-mkl             \
-             CXX=CC CC=cc
-```
+| `KNC`       | [Intel Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) |
+| `KNL`       | [Intel Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |

VERSION

@@ -1,6 +1,4 @@
-Version : 0.6.0
+Version : 0.5.0
 
 - AVX512, AVX2, AVX, SSE good
 - Clang 3.5 and above, ICPC v16 and above, GCC 4.9 and above
-- MPI and MPI3
-- HiRep, Smearing, Generic gauge group

benchmarks/Benchmark_comms.cc

@@ -42,14 +42,15 @@ int main (int argc, char ** argv)
 
   int Nloop=10;
   int nmu=0;
-  for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
-
+  for(int mu=0;mu<4;mu++) if (mpi_layout[mu]>1) nmu++;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-  int maxlat=16;
-  for(int lat=4;lat<=maxlat;lat+=2){
+
+
+
+  for(int lat=4;lat<=32;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
@@ -124,7 +125,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
 
 
-  for(int lat=4;lat<=maxlat;lat+=2){
+  for(int lat=4;lat<=32;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
       std::vector<int> latt_size  ({lat,lat,lat,lat});
@@ -193,51 +194,45 @@ int main (int argc, char ** argv)
     }
   }  
 
+#if 0
 
-  Nloop=100;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "= Benchmarking concurrent STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking sequential persistent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
 
-  for(int lat=4;lat<=maxlat;lat+=2){
+
+  for(int lat=4;lat<=32;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
-      std::vector<int> latt_size  ({lat*mpi_layout[0],
-      				    lat*mpi_layout[1],
-      				    lat*mpi_layout[2],
-      				    lat*mpi_layout[3]});
+      std::vector<int> latt_size  ({lat,lat,lat,lat});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
-      std::vector<HalfSpinColourVectorD *> xbuf(8);
-      std::vector<HalfSpinColourVectorD *> rbuf(8);
-      Grid.ShmBufferFreeAll();
-      for(int d=0;d<8;d++){
-	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
-	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
-      }
+      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
-      for(int i=0;i<Nloop;i++){
 
-	std::vector<CartesianCommunicator::CommsRequest_t> requests;
+      std::vector<CartesianCommunicator::CommsRequest_t> empty;
+      std::vector<std::vector<CartesianCommunicator::CommsRequest_t> > requests_fwd(Nd,empty);
+      std::vector<std::vector<CartesianCommunicator::CommsRequest_t> > requests_bwd(Nd,empty);
 
-	ncomm=0;
       for(int mu=0;mu<4;mu++){
-	
+	ncomm=0;
 	if (mpi_layout[mu]>1 ) {
-	  
 	  ncomm++;
-	    int comm_proc=1;
+
+	  int comm_proc;
 	  int xmit_to_rank;
 	  int recv_from_rank;
 
+	  comm_proc=1;
 	  Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
+	  Grid.SendToRecvFromInit(requests_fwd[mu],
 				  (void *)&xbuf[mu][0],
 				  xmit_to_rank,
 				  (void *)&rbuf[mu][0],
@@ -245,9 +240,8 @@ int main (int argc, char ** argv)
 				  bytes);
 
 	  comm_proc = mpi_layout[mu]-1;
-	  
 	  Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
+	  Grid.SendToRecvFromInit(requests_bwd[mu],
 				  (void *)&xbuf[mu+4][0],
 				  xmit_to_rank,
 				  (void *)&rbuf[mu+4][0],
@@ -256,94 +250,24 @@ int main (int argc, char ** argv)
 
 	}
       }
-	Grid.StencilSendToRecvFromComplete(requests);
-	Grid.Barrier();
-
-      }
-      double stop=usecond();
-
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
-
-      double time = stop-start; // microseconds
-
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
-    }
-  }    
-
-
-
-  Nloop=100;
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "= Benchmarking sequential STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-
-  for(int lat=4;lat<=maxlat;lat+=2){
-    for(int Ls=1;Ls<=16;Ls*=2){
-
-      std::vector<int> latt_size  ({lat*mpi_layout[0],
-      				    lat*mpi_layout[1],
-      				    lat*mpi_layout[2],
-      				    lat*mpi_layout[3]});
-
-      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-
-      std::vector<HalfSpinColourVectorD *> xbuf(8);
-      std::vector<HalfSpinColourVectorD *> rbuf(8);
-      Grid.ShmBufferFreeAll();
-      for(int d=0;d<8;d++){
-	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
-	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
-      }
-
-      int ncomm;
-      int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
+      {
 	double start=usecond();
 	for(int i=0;i<Nloop;i++){
 	  
-	std::vector<CartesianCommunicator::CommsRequest_t> requests;
-
-	ncomm=0;
 	  for(int mu=0;mu<4;mu++){
 	    
 	    if (mpi_layout[mu]>1 ) {
 	      
-	    ncomm++;
-	    int comm_proc=1;
-	    int xmit_to_rank;
-	    int recv_from_rank;
-	    
-	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu][0],
-					    recv_from_rank,
-					    bytes);
-	    //	    Grid.StencilSendToRecvFromComplete(requests);
-	    //	    requests.resize(0);
-
-	    comm_proc = mpi_layout[mu]-1;
-	  
-	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu+4][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu+4][0],
-					    recv_from_rank,
-					    bytes);
-	    Grid.StencilSendToRecvFromComplete(requests);
-	    requests.resize(0);
-	  
+	      Grid.SendToRecvFromBegin(requests_fwd[mu]);
+	      Grid.SendToRecvFromComplete(requests_fwd[mu]);
+	      Grid.SendToRecvFromBegin(requests_bwd[mu]);
+	      Grid.SendToRecvFromComplete(requests_bwd[mu]);
 	    }
 	  }
 	  Grid.Barrier();
-
 	}
+	
 	double stop=usecond();
 	
 	double dbytes    = bytes;
@@ -351,11 +275,47 @@ int main (int argc, char ** argv)
 	double rbytes    = xbytes;
 	double bidibytes = xbytes+rbytes;
 	
-      double time = stop-start; // microseconds
+	double time = stop-start;
 	
 	std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+
+      }
+
+
+      {
+	double start=usecond();
+	for(int i=0;i<Nloop;i++){
+	  
+	  for(int mu=0;mu<4;mu++){
+	    
+	    if (mpi_layout[mu]>1 ) {
+	      
+	      Grid.SendToRecvFromBegin(requests_fwd[mu]);
+	      Grid.SendToRecvFromBegin(requests_bwd[mu]);
+	      Grid.SendToRecvFromComplete(requests_fwd[mu]);
+	      Grid.SendToRecvFromComplete(requests_bwd[mu]);
 	    }
 	  }
+	  Grid.Barrier();
+	}
+	
+	double stop=usecond();
+	
+	double dbytes    = bytes;
+	double xbytes    = Nloop*dbytes*2.0*ncomm;
+	double rbytes    = xbytes;
+	double bidibytes = xbytes+rbytes;
+	
+	double time = stop-start;
+	
+	std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+
+      }
+
+    }
+  }
+
+#endif
 
   Grid_finalize();
 }

benchmarks/Benchmark_dwf.cc

@@ -44,6 +44,7 @@ struct scal {
     Gamma::GammaT
   };
 
+bool overlapComms = false;
 typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
 typedef WilsonFermion5D<DomainWallVec5dImplF> WilsonFermion5DF;
 typedef WilsonFermion5D<DomainWallVec5dImplD> WilsonFermion5DD;
@@ -53,11 +54,15 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
+  if( GridCmdOptionExists(argv,argv+argc,"--asynch") ){
+    overlapComms = true;
+  }
+
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
   std::vector<int> latt4 = GridDefaultLatt();
-  const int Ls=8;
+  const int Ls=16;
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
@@ -121,24 +126,17 @@ int main (int argc, char ** argv)
 
   RealD NP = UGrid->_Nprocessors;
 
+  for(int doasm=1;doasm<2;doasm++){
+
+    QCD::WilsonKernelsStatic::AsmOpt=doasm;
+
   DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
   
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::Dhop                  "<<std::endl;
-  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
-  if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
-  if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-
+  std::cout<<GridLogMessage << "Naive wilson implementation "<<std::endl;
+  std::cout << GridLogMessage<< "Calling Dw"<<std::endl;
   int ncall =100;
   if (1) {
-    FGrid->Barrier();
+
     Dw.ZeroCounters();
     double t0=usecond();
     for(int i=0;i<ncall;i++){
@@ -147,7 +145,6 @@ int main (int argc, char ** argv)
       __SSC_STOP;
     }
     double t1=usecond();
-    FGrid->Barrier();
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=1344*volume*ncall;
@@ -156,26 +153,14 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
     std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
-    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-    assert (norm2(err)< 1.0e-4 );
     Dw.Report();
   }
 
   if (1)
   {
-
-    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-    std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::Dhop "<<std::endl;
-    std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
-    if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
-    if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
-    if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
-    if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
-    if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
-    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-
     typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
     LatticeFermion ssrc(sFGrid);
     LatticeFermion sref(sFGrid);
@@ -194,7 +179,6 @@ int main (int argc, char ** argv)
       pokeSite(tmp,ssrc,site);
     }}}}}
     std::cout<<GridLogMessage<< "src norms "<< norm2(src)<<" " <<norm2(ssrc)<<std::endl;
-    FGrid->Barrier();
     double t0=usecond();
     sDw.ZeroCounters();
     for(int i=0;i<ncall;i++){
@@ -203,13 +187,12 @@ int main (int argc, char ** argv)
       __SSC_STOP;
     }
     double t1=usecond();
-    FGrid->Barrier();
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=1344*volume*ncall;
 
     std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
-    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
     sDw.Report();
   
     if(0){
@@ -225,7 +208,8 @@ int main (int argc, char ** argv)
 
     std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
 
-    RealD sum=0;
+
+    RealF sum=0;
     for(int x=0;x<latt4[0];x++){
     for(int y=0;y<latt4[1];y++){
     for(int z=0;z<latt4[2];z++){
@@ -243,12 +227,12 @@ int main (int argc, char ** argv)
       }
     }}}}}
     std::cout<<GridLogMessage<<" difference between normal and simd is "<<sum<<std::endl;
-    assert (sum< 1.0e-4 );
 
 
     if (1) {
 
       LatticeFermion sr_eo(sFGrid);
+      LatticeFermion serr(sFGrid);
 
       LatticeFermion ssrc_e (sFrbGrid);
       LatticeFermion ssrc_o (sFrbGrid);
@@ -260,21 +244,12 @@ int main (int argc, char ** argv)
 
       setCheckerboard(sr_eo,ssrc_o);
       setCheckerboard(sr_eo,ssrc_e);
+      serr = sr_eo-ssrc; 
+      std::cout<<GridLogMessage << "EO src norm diff   "<< norm2(serr)<<std::endl;
 
       sr_e = zero;
       sr_o = zero;
 
-      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-      std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::DhopEO "<<std::endl;
-      std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
-      if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
-      if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
-      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
-      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
-      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
-      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-
-      FGrid->Barrier();
       sDw.ZeroCounters();
       sDw.stat.init("DhopEO");
       double t0=usecond();
@@ -282,14 +257,13 @@ int main (int argc, char ** argv)
         sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
       }
       double t1=usecond();
-      FGrid->Barrier();
       sDw.stat.print();
 
       double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
       double flops=(1344.0*volume*ncall)/2;
 
       std::cout<<GridLogMessage << "sDeo mflop/s =   "<< flops/(t1-t0)<<std::endl;
-      std::cout<<GridLogMessage << "sDeo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+      std::cout<<GridLogMessage << "sDeo mflop/s per node   "<< flops/(t1-t0)/NP<<std::endl;
       sDw.Report();
 
       sDw.DhopEO(ssrc_o,sr_e,DaggerNo);
@@ -299,18 +273,9 @@ int main (int argc, char ** argv)
       pickCheckerboard(Even,ssrc_e,sresult);
       pickCheckerboard(Odd ,ssrc_o,sresult);
       ssrc_e = ssrc_e - sr_e;
-      RealD error = norm2(ssrc_e);
-
       std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm"<<norm2(sr_e) <<std::endl;
       ssrc_o = ssrc_o - sr_o;
-
-      error+= norm2(ssrc_o);
       std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm"<<norm2(sr_o) <<std::endl;
-      if(error>1.0e-4) { 
-	setCheckerboard(ssrc,ssrc_o);
-	setCheckerboard(ssrc,ssrc_e);
-	std::cout<< ssrc << std::endl;
-      }
     }
 
 
@@ -336,13 +301,13 @@ int main (int argc, char ** argv)
     ref = -0.5*ref;
   }
   Dw.Dhop(src,result,1);
-  std::cout << GridLogMessage << "Compare to naive wilson implementation Dag to verify correctness" << std::endl;
+  std::cout << GridLogMessage << "Naive wilson implementation Dag" << std::endl;
   std::cout<<GridLogMessage << "Called DwDag"<<std::endl;
   std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
   std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
   err = ref-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-  assert(norm2(err)<1.0e-4);
+
   LatticeFermion src_e (FrbGrid);
   LatticeFermion src_o (FrbGrid);
   LatticeFermion r_e   (FrbGrid);
@@ -350,37 +315,26 @@ int main (int argc, char ** argv)
   LatticeFermion r_eo  (FGrid);
 
 
-  std::cout<<GridLogMessage << "Calling Deo and Doe and assert Deo+Doe == Dunprec"<<std::endl;
+  std::cout<<GridLogMessage << "Calling Deo and Doe"<<std::endl;
   pickCheckerboard(Even,src_e,src);
   pickCheckerboard(Odd,src_o,src);
 
   std::cout<<GridLogMessage << "src_e"<<norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "src_o"<<norm2(src_o)<<std::endl;
 
-  std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::DhopEO                "<<std::endl;
-  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
-  if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
-  if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
-  std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
   {
     Dw.ZeroCounters();
-    FGrid->Barrier();
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       Dw.DhopEO(src_o,r_e,DaggerNo);
     }
     double t1=usecond();
-    FGrid->Barrier();
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=(1344.0*volume*ncall)/2;
 
     std::cout<<GridLogMessage << "Deo mflop/s =   "<< flops/(t1-t0)<<std::endl;
-    std::cout<<GridLogMessage << "Deo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "Deo mflop/s per node   "<< flops/(t1-t0)/NP<<std::endl;
     Dw.Report();
   }
   Dw.DhopEO(src_o,r_e,DaggerNo);
@@ -396,14 +350,14 @@ int main (int argc, char ** argv)
 
   err = r_eo-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-  assert(norm2(err)<1.0e-4);
 
   pickCheckerboard(Even,src_e,err);
   pickCheckerboard(Odd,src_o,err);
   std::cout<<GridLogMessage << "norm diff even  "<< norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "norm diff odd   "<< norm2(src_o)<<std::endl;
-  assert(norm2(src_e)<1.0e-4);
-  assert(norm2(src_o)<1.0e-4);
+
+
+  }
 
   Grid_finalize();
 }

benchmarks/Benchmark_dwf_ntpf.cc

@@ -0,0 +1,153 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./benchmarks/Benchmark_dwf.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+  d internal;
+};
+
+  Gamma::GammaMatrix Gmu [] = {
+    Gamma::GammaX,
+    Gamma::GammaY,
+    Gamma::GammaZ,
+    Gamma::GammaT
+  };
+
+bool overlapComms = false;
+
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  if( GridCmdOptionExists(argv,argv+argc,"--asynch") ){
+    overlapComms = true;
+  }
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  std::vector<int> latt4 = GridDefaultLatt();
+  const int Ls=16;
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+
+  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+
+  LatticeFermion src   (FGrid); random(RNG5,src);
+  LatticeFermion result(FGrid); result=zero;
+  LatticeFermion    ref(FGrid);    ref=zero;
+  LatticeFermion    tmp(FGrid);
+  LatticeFermion    err(FGrid);
+
+  ColourMatrix cm = Complex(1.0,0.0);
+
+  LatticeGaugeField Umu(UGrid); 
+  random(RNG4,Umu);
+
+  LatticeGaugeField Umu5d(FGrid); 
+
+  // replicate across fifth dimension
+  for(int ss=0;ss<Umu._grid->oSites();ss++){
+    for(int s=0;s<Ls;s++){
+      Umu5d._odata[Ls*ss+s] = Umu._odata[ss];
+    }
+  }
+
+  ////////////////////////////////////
+  // Naive wilson implementation
+  ////////////////////////////////////
+  std::vector<LatticeColourMatrix> U(4,FGrid);
+  for(int mu=0;mu<Nd;mu++){
+    U[mu] = PeekIndex<LorentzIndex>(Umu5d,mu);
+  }
+
+  if (1)
+  {
+    ref = zero;
+    for(int mu=0;mu<Nd;mu++){
+
+      tmp = U[mu]*Cshift(src,mu+1,1);
+      ref=ref + tmp - Gamma(Gmu[mu])*tmp;
+
+      tmp =adj(U[mu])*src;
+      tmp =Cshift(tmp,mu+1,-1);
+      ref=ref + tmp + Gamma(Gmu[mu])*tmp;
+    }
+    ref = -0.5*ref;
+  }
+
+  RealD mass=0.1;
+  RealD M5  =1.8;
+
+  typename DomainWallFermionR::ImplParams params; 
+  params.overlapCommsCompute = overlapComms;
+  
+  RealD NP = UGrid->_Nprocessors;
+
+
+  QCD::WilsonKernelsStatic::AsmOpt=1;
+
+  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,params);
+  
+  std::cout<<GridLogMessage << "Calling Dw"<<std::endl;
+  int ncall =50;
+  if (1) {
+
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      Dw.Dhop(src,result,0);
+    }
+    double t1=usecond();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
+    err = ref-result; 
+    std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+    //    Dw.Report();
+  }
+  Grid_finalize();
+}

benchmarks/Benchmark_dwf_sweep.cc

@@ -51,26 +51,24 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-
   const int Ls=8;
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  if ( getenv("ASMOPT") )  {
+    QCD::WilsonKernelsStatic::AsmOpt=1;
+  } else { 
+    QCD::WilsonKernelsStatic::AsmOpt=0;
+  }
+
   std::cout<<GridLogMessage << "=========================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking DWF"<<std::endl;
   std::cout<<GridLogMessage << "=========================================================================="<<std::endl;
   std::cout<<GridLogMessage << "Volume \t\t\tProcs \t Dw \t eoDw \t sDw \t eosDw (Mflop/s)  "<<std::endl;
   std::cout<<GridLogMessage << "=========================================================================="<<std::endl;
 
-  int Lmax=16;
-  int dmin=2;
+  int Lmax=32;
+  int dmin=0;
   if ( getenv("LMAX") ) Lmax=atoi(getenv("LMAX"));
   if ( getenv("DMIN") ) dmin=atoi(getenv("DMIN"));
   for (int L=8;L<=Lmax;L*=2){

benchmarks/Benchmark_mooee.cc

@@ -1,183 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./benchmarks/Benchmark_dwf.cc
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#include <Grid/Grid.h>
-
-using namespace std;
-using namespace Grid;
-using namespace Grid::QCD;
-
-
-int main (int argc, char ** argv)
-{
-  Grid_init(&argc,&argv);
-
-  int threads = GridThread::GetThreads();
-  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
-
-  std::vector<int> latt4 = GridDefaultLatt();
-  const int Ls=16;
-  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
-  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
-  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
-  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
-
-  std::cout << GridLogMessage << "Making Vec5d innermost grids"<<std::endl;
-  GridCartesian         * sUGrid   = SpaceTimeGrid::makeFourDimDWFGrid(GridDefaultLatt(),GridDefaultMpi());
-  GridRedBlackCartesian * sUrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(sUGrid);
-  GridCartesian         * sFGrid   = SpaceTimeGrid::makeFiveDimDWFGrid(Ls,UGrid);
-  GridRedBlackCartesian * sFrbGrid = SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(Ls,UGrid);
-
-  std::vector<int> seeds4({1,2,3,4});
-  std::vector<int> seeds5({5,6,7,8});
-
-  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
-  std::cout << GridLogMessage << "Seeded"<<std::endl;
-
-  LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(RNG4,Umu);
-
-  std::cout << GridLogMessage << "made random gauge fields"<<std::endl;
-
-  RealD mass=0.1;
-  RealD M5  =1.8;
-  RealD NP = UGrid->_Nprocessors;
-
-
-  if (1)
-  {
-    const int ncall=1000;
-
-    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-    std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::Dhop "<<std::endl;
-    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-
-    GridParallelRNG RNG5(FGrid);
-    LatticeFermion src(FGrid); random(RNG5,src);
-    LatticeFermion result(FGrid);
-
-    DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-    double t0,t1;
-
-    LatticeFermion r_eo(FGrid);
-    LatticeFermion src_e (FrbGrid);
-    LatticeFermion src_o (FrbGrid);
-    LatticeFermion r_e   (FrbGrid);
-    LatticeFermion r_o   (FrbGrid);
-    
-    pickCheckerboard(Even,src_e,src);
-    pickCheckerboard(Odd,src_o,src);
-    
-    setCheckerboard(r_eo,src_o);
-    setCheckerboard(r_eo,src_e);
-    
-    r_e = zero;
-    r_o = zero;
-
-
-#define BENCH_DW(A,in,out)			\
-    Dw.CayleyZeroCounters();			\
-    Dw. A (in,out);				\
-    FGrid->Barrier();				\
-    t0=usecond();				\
-    for(int i=0;i<ncall;i++){			\
-      Dw. A (in,out);				\
-    }						\
-    t1=usecond();				\
-    FGrid->Barrier();				\
-    Dw.CayleyReport();					\
-    std::cout<<GridLogMessage << "Called " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
-    std::cout<<GridLogMessage << "******************"<<std::endl;
-
-#define BENCH_DW_MEO(A,in,out)			\
-    Dw.CayleyZeroCounters();			\
-    Dw. A (in,out,0);				\
-    FGrid->Barrier();				\
-    t0=usecond();				\
-    for(int i=0;i<ncall;i++){			\
-      Dw. A (in,out,0);				\
-    }						\
-    t1=usecond();				\
-    FGrid->Barrier();				\
-    Dw.CayleyReport();					\
-    std::cout<<GridLogMessage << "Called " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
-    std::cout<<GridLogMessage << "******************"<<std::endl;
-
-    BENCH_DW_MEO(Dhop    ,src,result);
-    BENCH_DW_MEO(DhopEO  ,src_o,r_e);
-    BENCH_DW(Meooe   ,src_o,r_e);
-    BENCH_DW(Mooee   ,src_o,r_o);
-    BENCH_DW(MooeeInv,src_o,r_o);
-
-  }
-
-  if (1)
-  {
-    const int ncall=1000;
-
-    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-    std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionVec5dR::Dhop "<<std::endl;
-    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-
-    GridParallelRNG RNG5(sFGrid);
-    LatticeFermion src(sFGrid); random(RNG5,src);
-    LatticeFermion sref(sFGrid);
-    LatticeFermion result(sFGrid);
-
-    std::cout<<GridLogMessage << "Constructing Vec5D Dw "<<std::endl;
-    DomainWallFermionVec5dR Dw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,mass,M5);
-
-    std::cout<<GridLogMessage << "Calling Dhop "<<std::endl;
-    FGrid->Barrier();
-
-    double t0,t1;
-
-    LatticeFermion r_eo(sFGrid);
-    LatticeFermion src_e (sFrbGrid);
-    LatticeFermion src_o (sFrbGrid);
-    LatticeFermion r_e   (sFrbGrid);
-    LatticeFermion r_o   (sFrbGrid);
-    
-    pickCheckerboard(Even,src_e,src);
-    pickCheckerboard(Odd,src_o,src);
-    
-    setCheckerboard(r_eo,src_o);
-    setCheckerboard(r_eo,src_e);
-    
-    r_e = zero;
-    r_o = zero;
-
-    BENCH_DW_MEO(Dhop    ,src,result);
-    BENCH_DW_MEO(DhopEO  ,src_o,r_e);
-    BENCH_DW(Meooe   ,src_o,r_e);
-    BENCH_DW(Mooee   ,src_o,r_o);
-    BENCH_DW(MooeeInv,src_o,r_o);
-
-  }
-
-  Grid_finalize();
-}

benchmarks/Benchmark_wilson_sweep.cc

@@ -58,19 +58,6 @@ int main (int argc, char ** argv)
   std::vector<int> seeds({1,2,3,4});
   RealD mass = 0.1;
 
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  std::cout << GridLogMessage<< "* Benchmarking WilsonFermionR::Dhop                  "<<std::endl;
-  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
-  if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
-  if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
-  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
-  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-
   std::cout<<GridLogMessage << "============================================================================="<< std::endl;
   std::cout<<GridLogMessage << "= Benchmarking Wilson" << std::endl;
   std::cout<<GridLogMessage << "============================================================================="<< std::endl;

benchmarks/Benchmark_zmm.cc

@@ -0,0 +1,175 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_zmm.cc
+
+    Copyright (C) 2015
+
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+
+using namespace Grid;
+using namespace Grid::QCD;
+
+
+int bench(std::ofstream &os, std::vector<int> &latt4,int Ls);
+
+int main(int argc,char **argv)
+{
+  Grid_init(&argc,&argv);
+  std::ofstream os("zmm.dat");
+
+  os << "#V Ls Lxy Lzt C++ Asm OMP L1 " <<std::endl;
+  std::cout<<GridLogMessage << "====================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking ZMM"<<std::endl;
+  std::cout<<GridLogMessage << "====================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "Volume \t\t\t\tC++DW/MFLOPs\tASM-DW/MFLOPs\tdiff"<<std::endl;
+  std::cout<<GridLogMessage << "====================================================================="<<std::endl;
+  for(int L=4;L<=32;L+=4){
+    for(int m=1;m<=2;m++){
+      for(int Ls=8;Ls<=16;Ls+=8){
+	std::vector<int> grid({L,L,m*L,m*L});
+  std::cout << GridLogMessage <<"\t";
+	for(int i=0;i<4;i++) { 
+	  std::cout << grid[i]<<"x";
+	}
+	std::cout << Ls<<"\t\t";
+	bench(os,grid,Ls);
+      }
+    }
+  }
+}
+
+int bench(std::ofstream &os, std::vector<int> &latt4,int Ls)
+{
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+  int threads = GridThread::GetThreads();
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+
+  GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds4);
+
+  LatticeFermion src (FGrid);
+  LatticeFermion tmp (FGrid);
+  LatticeFermion srce(FrbGrid);
+
+  LatticeFermion resulto(FrbGrid); resulto=zero;
+  LatticeFermion resulta(FrbGrid); resulta=zero;
+  LatticeFermion junk(FrbGrid); junk=zero;
+  LatticeFermion diff(FrbGrid); 
+  LatticeGaugeField Umu(UGrid);
+
+  double mfc, mfa, mfo, mfl1;
+
+  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  random(RNG5,src);
+#if 1
+  random(RNG4,Umu);
+#else
+  int mmu=2;
+  std::vector<LatticeColourMatrix> U(4,UGrid);
+  for(int mu=0;mu<Nd;mu++){
+    U[mu] = PeekIndex<LorentzIndex>(Umu,mu);
+    if ( mu!=mmu ) U[mu] = zero;
+    if ( mu==mmu ) U[mu] = 1.0;
+    PokeIndex<LorentzIndex>(Umu,U[mu],mu);
+  }
+#endif
+ pickCheckerboard(Even,srce,src);
+
+  RealD mass=0.1;
+  RealD M5  =1.8;
+  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+
+  int ncall=50;
+  double t0=usecond();
+  for(int i=0;i<ncall;i++){
+    Dw.DhopOE(srce,resulto,0);
+  }
+  double t1=usecond();
+
+  double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+  double flops=1344*volume/2;
+
+  mfc = flops*ncall/(t1-t0);
+  std::cout<<mfc<<"\t\t";
+
+  QCD::WilsonKernelsStatic::AsmOpt=1;
+  t0=usecond();
+  for(int i=0;i<ncall;i++){
+    Dw.DhopOE(srce,resulta,0);
+  }
+  t1=usecond();
+  mfa = flops*ncall/(t1-t0);
+  std::cout<<mfa<<"\t\t";
+  /*
+  int dag=DaggerNo;
+  t0=usecond();
+  for(int i=0;i<1;i++){
+    Dw.DhopInternalOMPbench(Dw.StencilEven,Dw.LebesgueEvenOdd,Dw.UmuOdd,srce,resulta,dag);
+  }
+  t1=usecond();
+  mfo = flops*100/(t1-t0);
+  std::cout<<GridLogMessage << "Called ASM-OMP Dw"<< " mflop/s =   "<< mfo<<std::endl;
+
+  t0=usecond();
+  for(int i=0;i<1;i++){
+    Dw.DhopInternalL1bench(Dw.StencilEven,Dw.LebesgueEvenOdd,Dw.UmuOdd,srce,resulta,dag);
+  }
+  t1=usecond();
+  mfl1= flops*100/(t1-t0);
+  std::cout<<GridLogMessage << "Called ASM-L1 Dw"<< " mflop/s =   "<< mfl1<<std::endl;
+  os << latt4[0]*latt4[1]*latt4[2]*latt4[3]<< " "<<Ls<<" "<< latt4[0] <<" " <<latt4[2]<< " "
+     << mfc<<" "
+     << mfa<<" "
+     << mfo<<" "
+     << mfl1<<std::endl;
+  */
+
+#if 0
+  for(int i=0;i< PerformanceCounter::NumTypes(); i++ ){
+    Dw.DhopOE(srce,resulta,0);
+    PerformanceCounter Counter(i);
+    Counter.Start();
+    Dw.DhopOE(srce,resulta,0);
+    Counter.Stop();
+    Counter.Report();
+  }
+#endif
+  //resulta = (-0.5) * resulta;
+
+  diff = resulto-resulta;
+  std::cout<<norm2(diff)<<std::endl;
+  return 0;
+}
+
+

benchmarks/Makefile.am

@@ -1,11 +1 @@
 include Make.inc
-
-simple: simple_su3_test.o simple_su3_expr.o simple_simd_test.o
-
-EXTRA_LIBRARIES = libsimple_su3_test.a libsimple_su3_expr.a libsimple_simd_test.a
-
-libsimple_su3_test_a_SOURCES = simple_su3_test.cc
-
-libsimple_su3_expr_a_SOURCES = simple_su3_expr.cc
-
-libsimple_simd_test_a_SOURCES = simple_simd_test.cc

benchmarks/simple_simd_test.cc

@@ -1,11 +0,0 @@
-#include <Grid/Grid.h>
-
-Grid::vRealD add(const Grid::vRealD &x, const Grid::vRealD &y)
-{
-	return x+y;
-}
-
-Grid::vRealD sub(const Grid::vRealD &x, const Grid::vRealD &y)
-{
-	return x-y;
-}

benchmarks/simple_su3_expr.cc

@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 using namespace std;
 using namespace Grid;

benchmarks/simple_su3_test.cc

@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 using namespace std;
 using namespace Grid;

bootstrap.sh

@@ -1,12 +1,18 @@
 #!/usr/bin/env bash
 
 EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2'
+FFTW_URL=http://www.fftw.org/fftw-3.3.4.tar.gz
 
 echo "-- deploying Eigen source..."
 wget ${EIGEN_URL} --no-check-certificate
 ./scripts/update_eigen.sh `basename ${EIGEN_URL}`
 rm `basename ${EIGEN_URL}`
 
+echo "-- copying fftw prototypes..."
+wget ${FFTW_URL}
+./scripts/update_fftw.sh `basename ${FFTW_URL}`
+rm `basename ${FFTW_URL}`
+
 echo '-- generating Make.inc files...'
 ./scripts/filelist
 echo '-- generating configure script...'

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.63])
-AC_INIT([Grid], [0.6.0], [https://github.com/paboyle/Grid], [Grid])
+AC_INIT([Grid], [0.5.1-dev], [https://github.com/paboyle/Grid], [Grid])
 AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
@@ -9,33 +9,22 @@ AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
+
 ############### Checks for programs
+AC_LANG(C++)
 CXXFLAGS="-O3 $CXXFLAGS"
 AC_PROG_CXX
 AC_PROG_RANLIB
 
-############### Get compiler informations
-AC_LANG([C++])
-AX_CXX_COMPILE_STDCXX_11([noext],[mandatory])
-AX_COMPILER_VENDOR
-AC_DEFINE_UNQUOTED([CXX_COMP_VENDOR],["$ax_cv_cxx_compiler_vendor"],
-      [vendor of C++ compiler that will compile the code])
-AX_GXX_VERSION
-AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
-      [version of g++ that will compile the code])
-
-############### Checks for typedefs, structures, and compiler characteristics
-AC_TYPE_SIZE_T
-AC_TYPE_UINT32_T
-AC_TYPE_UINT64_T
-
-############### OpenMP 
+############ openmp  ###############
 AC_OPENMP
+
 ac_openmp=no
+
 if test "${OPENMP_CXXFLAGS}X" != "X"; then
-  ac_openmp=yes
-  AM_CXXFLAGS="$OPENMP_CXXFLAGS $AM_CXXFLAGS"
-  AM_LDFLAGS="$OPENMP_CXXFLAGS $AM_LDFLAGS"
+ac_openmp=yes
+AM_CXXFLAGS="$OPENMP_CXXFLAGS $AM_CXXFLAGS"
+AM_LDFLAGS="$OPENMP_CXXFLAGS $AM_LDFLAGS"
 fi
 
 ############### Checks for header files
@@ -48,7 +37,12 @@ AC_CHECK_HEADERS(execinfo.h)
 AC_CHECK_DECLS([ntohll],[], [], [[#include <arpa/inet.h>]])
 AC_CHECK_DECLS([be64toh],[], [], [[#include <arpa/inet.h>]])
 
-############### GMP and MPFR
+############### Checks for typedefs, structures, and compiler characteristics
+AC_TYPE_SIZE_T
+AC_TYPE_UINT32_T
+AC_TYPE_UINT64_T
+
+############### GMP and MPFR #################
 AC_ARG_WITH([gmp],
     [AS_HELP_STRING([--with-gmp=prefix],
     [try this for a non-standard install prefix of the GMP library])],
@@ -60,17 +54,10 @@ AC_ARG_WITH([mpfr],
     [AM_CXXFLAGS="-I$with_mpfr/include $AM_CXXFLAGS"]
     [AM_LDFLAGS="-L$with_mpfr/lib $AM_LDFLAGS"])
 
-############### FFTW3 
-AC_ARG_WITH([fftw],    
-            [AS_HELP_STRING([--with-fftw=prefix],
-            [try this for a non-standard install prefix of the FFTW3 library])],
-            [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
-            [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
-
-############### lapack 
+################## lapack ####################
 AC_ARG_ENABLE([lapack],
     [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])], 
-    [ac_LAPACK=${enable_lapack}], [ac_LAPACK=no])
+    [ac_LAPACK=${enable_lapack}],[ac_LAPACK=no])
 
 case ${ac_LAPACK} in
     no)
@@ -80,26 +67,10 @@ case ${ac_LAPACK} in
     *)
         AM_CXXFLAGS="-I$ac_LAPACK/include $AM_CXXFLAGS"
         AM_LDFLAGS="-L$ac_LAPACK/lib $AM_LDFLAGS"
-        AC_DEFINE([USE_LAPACK],[1],[use LAPACK]);;
+        AC_DEFINE([USE_LAPACK],[1],[use LAPACK])
 esac
 
-############### MKL
-AC_ARG_ENABLE([mkl],
-    [AC_HELP_STRING([--enable-mkl=yes|no|prefix], [enable Intel MKL for LAPACK & FFTW])],
-    [ac_MKL=${enable_mkl}], [ac_MKL=no])
-
-case ${ac_MKL} in
-    no)
-        ;;
-    yes)
-        AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
-    *)
-        AM_CXXFLAGS="-I$ac_MKL/include $AM_CXXFLAGS"
-        AM_LDFLAGS="-L$ac_MKL/lib $AM_LDFLAGS"
-        AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
-esac
-
-############### first-touch
+################## first-touch ####################
 AC_ARG_ENABLE([numa],
     [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
     [ac_NUMA=${enable_NUMA}],[ac_NUMA=no])
@@ -113,50 +84,56 @@ case ${ac_NUMA} in
         AC_DEFINE([GRID_NUMA],[1],[First touch numa locality]);;
 esac
 
+################## FFTW3 ####################
+AC_ARG_WITH([fftw],    
+            [AS_HELP_STRING([--with-fftw=prefix],
+            [try this for a non-standard install prefix of the FFTW3 library])],
+            [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
+            [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
+
+################ Get compiler informations
+AC_LANG([C++])
+AX_CXX_COMPILE_STDCXX_11([noext],[mandatory])
+AX_COMPILER_VENDOR
+AC_DEFINE_UNQUOTED([CXX_COMP_VENDOR],["$ax_cv_cxx_compiler_vendor"],
+      [vendor of C++ compiler that will compile the code])
+AX_GXX_VERSION
+AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
+      [version of g++ that will compile the code])
+
 ############### Checks for library functions
 CXXFLAGS_CPY=$CXXFLAGS
 LDFLAGS_CPY=$LDFLAGS
 CXXFLAGS="$AM_CXXFLAGS $CXXFLAGS"
 LDFLAGS="$AM_LDFLAGS $LDFLAGS"
-
 AC_CHECK_FUNCS([gettimeofday])
-
-if test "${ac_MKL}x" != "nox"; then
-    AC_SEARCH_LIBS([mkl_set_interface_layer], [mkl_rt], [],
-                   [AC_MSG_ERROR("MKL enabled but library not found")])
-fi
-
-AC_SEARCH_LIBS([__gmpf_init], [gmp],
-               [AC_SEARCH_LIBS([mpfr_init], [mpfr], 
-                               [AC_DEFINE([HAVE_LIBMPFR], [1], 
-                                          [Define to 1 if you have the `MPFR' library])]
-                               [have_mpfr=true], [AC_MSG_ERROR([MPFR library not found])])]
-               [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library])]
-               [have_gmp=true])
+AC_CHECK_LIB([gmp],[__gmpf_init],
+             [AC_CHECK_LIB([mpfr],[mpfr_init],
+                 [AC_DEFINE([HAVE_LIBMPFR], [1], [Define to 1 if you have the `MPFR' library (-lmpfr).])]
+                 [have_mpfr=true]
+                 [LIBS="$LIBS -lmpfr"],
+                 [AC_MSG_ERROR([MPFR library not found])])]
+   	     [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library (-lgmp).])]
+             [have_gmp=true]
+             [LIBS="$LIBS -lgmp"],
+             [AC_MSG_WARN([**** GMP library not found, Grid can still compile but RHMC will not work ****])])
 
 if test "${ac_LAPACK}x" != "nox"; then
-    AC_SEARCH_LIBS([LAPACKE_sbdsdc], [lapack], [],
+    AC_CHECK_LIB([lapack],[LAPACKE_sbdsdc],[],
                  [AC_MSG_ERROR("LAPACK enabled but library not found")])
 fi
-
-AC_SEARCH_LIBS([fftw_execute], [fftw3],
-               [AC_SEARCH_LIBS([fftwf_execute], [fftw3f], [],
-                               [AC_MSG_ERROR("single precision FFTW library not found")])]
-               [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
-               [have_fftw=true])
-
+AC_CHECK_LIB([fftw3],[fftw_execute],
+  [AC_DEFINE([HAVE_FFTW],[1],[Define to 1 if you have the `FFTW' library (-lfftw3).])]
+  [have_fftw=true]
+  [LIBS="$LIBS -lfftw3 -lfftw3f"],
+  [AC_MSG_WARN([**** FFTW library not found, Grid can still compile but FFT-based routines will not work ****])])
 CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
 ############### SIMD instruction selection
-AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=code],
-              [select SIMD target (cf. README.md)])], [ac_SIMD=${enable_simd}], [ac_SIMD=GEN])
-
-AC_ARG_ENABLE([gen-simd-width],
-            [AS_HELP_STRING([--enable-gen-simd-width=size],
-            [size (in bytes) of the generic SIMD vectors (default: 32)])],
-            [ac_gen_simd_width=$enable_gen_simd_width],
-            [ac_gen_simd_width=32])
+AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=SSE4|AVX|AVXFMA4|AVXFMA|AVX2|AVX512|AVX512MIC|IMCI|KNL|KNC],\
+	[Select instructions to be SSE4.0, AVX 1.0, AVX 2.0+FMA, AVX 512, IMCI])],\
+	[ac_SIMD=${enable_simd}],[ac_SIMD=GEN])
 
 case ${ax_cv_cxx_compiler_vendor} in
   clang|gnu)
@@ -176,20 +153,14 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
         SIMD_FLAGS='-mavx2 -mfma';;
-      AVX512)
+      AVX512|AVX512MIC|KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
-      KNC)
+      IMCI|KNC)
         AC_DEFINE([IMCI],[1],[IMCI intrinsics for Knights Corner])
         SIMD_FLAGS='';;
-      KNL)
-        AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
-        SIMD_FLAGS='-march=knl';;
       GEN)
-        AC_DEFINE([GEN],[1],[generic vector code])
-        AC_DEFINE_UNQUOTED([GEN_SIMD_WIDTH],[$ac_gen_simd_width],
-                           [generic SIMD vector width (in bytes)])
-        SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)"
+        AC_DEFINE([GENERIC_VEC],[1],[generic vector code])
         SIMD_FLAGS='';;
       QPX|BGQ)
         AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q])
@@ -205,26 +176,26 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVX)
         AC_DEFINE([AVX1],[1],[AVX intrinsics])
         SIMD_FLAGS='-mavx -xavx';;
+      AVXFMA4)
+        AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
+        SIMD_FLAGS='-mavx -mfma';;
       AVXFMA)
-        AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA3])
-        SIMD_FLAGS='-mavx -fma';;
+        AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA4])
+        SIMD_FLAGS='-mavx -mfma';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
         SIMD_FLAGS='-march=core-avx2 -xcore-avx2';;
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-xcore-avx512';;
-      KNC)
-        AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner])
-        SIMD_FLAGS='';;
-      KNL)
+      AVX512MIC|KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing])
         SIMD_FLAGS='-xmic-avx512';;
+      IMCI|KNC)
+        AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner])
+        SIMD_FLAGS='';;
       GEN)
-        AC_DEFINE([GEN],[1],[generic vector code])
-        AC_DEFINE_UNQUOTED([GEN_SIMD_WIDTH],[$ac_gen_simd_width],
-                           [generic SIMD vector width (in bytes)])
-        SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)"
+        AC_DEFINE([GENERIC_VEC],[1],[generic vector code])
         SIMD_FLAGS='';;
       *)
         AC_MSG_ERROR(["SIMD option ${ac_SIMD} not supported by the Intel compiler"]);;
@@ -237,18 +208,14 @@ AM_CXXFLAGS="$SIMD_FLAGS $AM_CXXFLAGS"
 AM_CFLAGS="$SIMD_FLAGS $AM_CFLAGS"
 
 case ${ac_SIMD} in
-  AVX512|KNL)
+  AVX512|AVX512MIC|KNL)
     AC_DEFINE([TEST_ZMM],[1],[compile ZMM test]);;
   *)
 	;;
 esac
 
-############### Precision selection
-AC_ARG_ENABLE([precision],
-              [AC_HELP_STRING([--enable-precision=single|double],
-                              [Select default word size of Real])],
-              [ac_PRECISION=${enable_precision}],[ac_PRECISION=double])
-
+############### precision selection
+AC_ARG_ENABLE([precision],[AC_HELP_STRING([--enable-precision=single|double],[Select default word size of Real])],[ac_PRECISION=${enable_precision}],[ac_PRECISION=double])
 case ${ac_PRECISION} in
      single)
        AC_DEFINE([GRID_DEFAULT_PRECISION_SINGLE],[1],[GRID_DEFAULT_PRECISION is SINGLE] )
@@ -259,51 +226,34 @@ case ${ac_PRECISION} in
 esac
 
 ############### communication type selection
-AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi3-auto|shmem],
-              [Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
+AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|shmem],[Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
 
 case ${ac_COMMS} in
      none)
        AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
-        comms_type='none'
      ;;
-     mpi3l*)
-       AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] )
-       comms_type='mpi3l'
-     ;;
-     mpi3*)
-        AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
-        comms_type='mpi3'
-     ;;
-     mpi*)
+     mpi-auto)
+       AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
+       LX_FIND_MPI
+       if test "x$have_CXX_mpi" = 'xno'; then AC_MSG_ERROR(["MPI not found"]); fi
+       AM_CXXFLAGS="$MPI_CXXFLAGS $AM_CXXFLAGS"
+       AM_CFLAGS="$MPI_CFLAGS $AM_CFLAGS"
+       AM_LDFLAGS="`echo $MPI_CXXLDFLAGS | sed -E 's/-l@<:@^ @:>@+//g'` $AM_LDFLAGS"
+       LIBS="`echo $MPI_CXXLDFLAGS | sed -E 's/-L@<:@^ @:>@+//g'` $LIBS"
+     ;;
+     mpi)
        AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
-        comms_type='mpi'
      ;;
      shmem)
        AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
-        comms_type='shmem'
      ;;
      *)
      AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); 
      ;;
 esac
-case ${ac_COMMS} in
-    *-auto)
-        LX_FIND_MPI
-        if test "x$have_CXX_mpi" = 'xno'; then AC_MSG_ERROR(["The configure could not find the MPI compilation flags. N.B. The -auto mode is not supported by Cray wrappers. Use the non -auto version in this case."]); fi
-        AM_CXXFLAGS="$MPI_CXXFLAGS $AM_CXXFLAGS"
-        AM_CFLAGS="$MPI_CFLAGS $AM_CFLAGS"
-        AM_LDFLAGS="`echo $MPI_CXXLDFLAGS | sed -E 's/-l@<:@^ @:>@+//g'` $AM_LDFLAGS"
-        LIBS="`echo $MPI_CXXLDFLAGS | sed -E 's/-L@<:@^ @:>@+//g'` $LIBS";;
-    *)
-        ;;
-esac
-
-AM_CONDITIONAL(BUILD_COMMS_SHMEM, [ test "${comms_type}X" == "shmemX" ])
-AM_CONDITIONAL(BUILD_COMMS_MPI,   [ test "${comms_type}X" == "mpiX" ])
-AM_CONDITIONAL(BUILD_COMMS_MPI3,  [ test "${comms_type}X" == "mpi3X" ] )
-AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] )
-AM_CONDITIONAL(BUILD_COMMS_NONE,  [ test "${comms_type}X" == "noneX" ])
+AM_CONDITIONAL(BUILD_COMMS_SHMEM,[ test "X${ac_COMMS}X" == "XshmemX" ])
+AM_CONDITIONAL(BUILD_COMMS_MPI,[ test "X${ac_COMMS}X" == "XmpiX" || test "X${ac_COMMS}X" == "Xmpi-autoX" ])
+AM_CONDITIONAL(BUILD_COMMS_NONE,[ test "X${ac_COMMS}X" == "XnoneX" ])
 
 ############### RNG selection
 AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937],\
@@ -322,11 +272,10 @@ case ${ac_RNG} in
      ;;
 esac
 
-############### Timer option
+############### timer option
 AC_ARG_ENABLE([timers],[AC_HELP_STRING([--enable-timers],\
 	[Enable system dependent high res timers])],\
 	[ac_TIMERS=${enable_timers}],[ac_TIMERS=yes])
-
 case ${ac_TIMERS} in
      yes)
       AC_DEFINE([TIMERS_ON],[1],[TIMERS_ON] )
@@ -340,9 +289,7 @@ case ${ac_TIMERS} in
 esac
 
 ############### Chroma regression test
-AC_ARG_ENABLE([chroma],[AC_HELP_STRING([--enable-chroma],
-              [Expect chroma compiled under c++11 ])],ac_CHROMA=yes,ac_CHROMA=no)
-
+AC_ARG_ENABLE([chroma],[AC_HELP_STRING([--enable-chroma],[Expect chroma compiled under c++11 ])],ac_CHROMA=yes,ac_CHROMA=no)
 case ${ac_CHROMA} in
      yes|no)
      ;;
@@ -350,21 +297,15 @@ case ${ac_CHROMA} in
        AC_MSG_ERROR([${ac_CHROMA} unsupported --enable-chroma option]); 
      ;;
 esac
-
 AM_CONDITIONAL(BUILD_CHROMA_REGRESSION,[ test "X${ac_CHROMA}X" == "XyesX" ])
 
 ############### Doxygen
-DX_DOXYGEN_FEATURE([OFF])
-DX_DOT_FEATURE([OFF])
-DX_HTML_FEATURE([ON])
-DX_CHM_FEATURE([OFF])
-DX_CHI_FEATURE([OFF])
-DX_MAN_FEATURE([OFF])
-DX_RTF_FEATURE([OFF])
-DX_XML_FEATURE([OFF])
-DX_PDF_FEATURE([OFF])
-DX_PS_FEATURE([OFF])
-DX_INIT_DOXYGEN([$PACKAGE_NAME], [doxygen.cfg])
+AC_PROG_DOXYGEN
+
+if test -n "$DOXYGEN"
+then
+AC_CONFIG_FILES([docs/doxy.cfg])
+fi
 
 ############### Ouput
 cwd=`pwd -P`; cd ${srcdir}; abs_srcdir=`pwd -P`; cd ${cwd}
@@ -387,35 +328,35 @@ AC_CONFIG_FILES(tests/qdpxx/Makefile)
 AC_CONFIG_FILES(benchmarks/Makefile)
 AC_OUTPUT
 
-echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+echo "
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Summary of configuration for $PACKAGE v$VERSION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ----- PLATFORM ----------------------------------------
-architecture (build)        : $build_cpu
-os (build)                  : $build_os
-architecture (target)       : $target_cpu
-os (target)                 : $target_os
-compiler vendor             : ${ax_cv_cxx_compiler_vendor}
-compiler version            : ${ax_cv_gxx_version}
+- architecture (build)          : $build_cpu
+- os (build)                    : $build_os
+- architecture (target)         : $target_cpu
+- os (target)                   : $target_os
+- compiler vendor               : ${ax_cv_cxx_compiler_vendor}
+- compiler version              : ${ax_cv_gxx_version}
 ----- BUILD OPTIONS -----------------------------------
-SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
-Threading                   : ${ac_openmp} 
-Communications type         : ${comms_type}
-Default precision           : ${ac_PRECISION}
-RNG choice                  : ${ac_RNG} 
-GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
-LAPACK                      : ${ac_LAPACK}
-FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
-build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
+- SIMD                          : ${ac_SIMD}
+- Threading                     : ${ac_openmp} 
+- Communications type           : ${ac_COMMS}
+- Default precision             : ${ac_PRECISION}
+- RNG choice                    : ${ac_RNG} 
+- GMP                           : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
+- LAPACK                        : ${ac_LAPACK}
+- FFTW                          : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
+- build DOXYGEN documentation   : `if test "x$enable_doc" = xyes; then echo yes; else echo no; fi`
+- graphs and diagrams           : `if test "x$enable_dot" = xyes; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
-CXXFLAGS:
+- CXXFLAGS:
 `echo ${AM_CXXFLAGS} ${CXXFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
-LDFLAGS:
+- LDFLAGS:
 `echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
-LIBS:
+- LIBS:
 `echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
--------------------------------------------------------" > config.summary
-echo ""
-cat config.summary
-echo ""
+-------------------------------------------------------
+"

doxygen.inc

@@ -1,184 +0,0 @@
-# Copyright (C) 2004 Oren Ben-Kiki
-# This file is distributed under the same terms as the Automake macro files.
-
-# Generate automatic documentation using Doxygen. Goals and variables values
-# are controlled by the various DX_COND_??? conditionals set by autoconf.
-#
-# The provided goals are:
-# doxygen-doc: Generate all doxygen documentation.
-# doxygen-run: Run doxygen, which will generate some of the documentation
-#              (HTML, CHM, CHI, MAN, RTF, XML) but will not do the post
-#              processing required for the rest of it (PS, PDF, and some MAN).
-# doxygen-man: Rename some doxygen generated man pages.
-# doxygen-ps: Generate doxygen PostScript documentation.
-# doxygen-pdf: Generate doxygen PDF documentation.
-#
-# Note that by default these are not integrated into the automake goals. If
-# doxygen is used to generate man pages, you can achieve this integration by
-# setting man3_MANS to the list of man pages generated and then adding the
-# dependency:
-#
-#   $(man3_MANS): doxygen-doc
-#
-# This will cause make to run doxygen and generate all the documentation.
-#
-# The following variable is intended for use in Makefile.am:
-#
-# DX_CLEANFILES = everything to clean.
-#
-# This is usually added to MOSTLYCLEANFILES.
-
-## --------------------------------- ##
-## Format-independent Doxygen rules. ##
-## --------------------------------- ##
-
-if DX_COND_doc
-
-## ------------------------------- ##
-## Rules specific for HTML output. ##
-## ------------------------------- ##
-
-if DX_COND_html
-
-DX_CLEAN_HTML = @DX_DOCDIR@/html
-
-endif DX_COND_html
-
-## ------------------------------ ##
-## Rules specific for CHM output. ##
-## ------------------------------ ##
-
-if DX_COND_chm
-
-DX_CLEAN_CHM = @DX_DOCDIR@/chm
-
-if DX_COND_chi
-
-DX_CLEAN_CHI = @DX_DOCDIR@/@PACKAGE@.chi
-
-endif DX_COND_chi
-
-endif DX_COND_chm
-
-## ------------------------------ ##
-## Rules specific for MAN output. ##
-## ------------------------------ ##
-
-if DX_COND_man
-
-DX_CLEAN_MAN = @DX_DOCDIR@/man
-
-endif DX_COND_man
-
-## ------------------------------ ##
-## Rules specific for RTF output. ##
-## ------------------------------ ##
-
-if DX_COND_rtf
-
-DX_CLEAN_RTF = @DX_DOCDIR@/rtf
-
-endif DX_COND_rtf
-
-## ------------------------------ ##
-## Rules specific for XML output. ##
-## ------------------------------ ##
-
-if DX_COND_xml
-
-DX_CLEAN_XML = @DX_DOCDIR@/xml
-
-endif DX_COND_xml
-
-## ----------------------------- ##
-## Rules specific for PS output. ##
-## ----------------------------- ##
-
-if DX_COND_ps
-
-DX_CLEAN_PS = @DX_DOCDIR@/@PACKAGE@.ps
-
-DX_PS_GOAL = doxygen-ps
-
-doxygen-ps: @DX_DOCDIR@/@PACKAGE@.ps
-
-@DX_DOCDIR@/@PACKAGE@.ps: @DX_DOCDIR@/@PACKAGE@.tag
-	cd @DX_DOCDIR@/latex; \
-	rm -f *.aux *.toc *.idx *.ind *.ilg *.log *.out; \
-	$(DX_LATEX) refman.tex; \
-	$(MAKEINDEX_PATH) refman.idx; \
-	$(DX_LATEX) refman.tex; \
-	countdown=5; \
-	while $(DX_EGREP) 'Rerun (LaTeX|to get cross-references right)' \
-	                  refman.log > /dev/null 2>&1 \
-	   && test $$countdown -gt 0; do \
-	    $(DX_LATEX) refman.tex; \
-	    countdown=`expr $$countdown - 1`; \
-	done; \
-	$(DX_DVIPS) -o ../@PACKAGE@.ps refman.dvi
-
-endif DX_COND_ps
-
-## ------------------------------ ##
-## Rules specific for PDF output. ##
-## ------------------------------ ##
-
-if DX_COND_pdf
-
-DX_CLEAN_PDF = @DX_DOCDIR@/@PACKAGE@.pdf
-
-DX_PDF_GOAL = doxygen-pdf
-
-doxygen-pdf: @DX_DOCDIR@/@PACKAGE@.pdf
-
-@DX_DOCDIR@/@PACKAGE@.pdf: @DX_DOCDIR@/@PACKAGE@.tag
-	cd @DX_DOCDIR@/latex; \
-	rm -f *.aux *.toc *.idx *.ind *.ilg *.log *.out; \
-	$(DX_PDFLATEX) refman.tex; \
-	$(DX_MAKEINDEX) refman.idx; \
-	$(DX_PDFLATEX) refman.tex; \
-	countdown=5; \
-	while $(DX_EGREP) 'Rerun (LaTeX|to get cross-references right)' \
-	                  refman.log > /dev/null 2>&1 \
-	   && test $$countdown -gt 0; do \
-	    $(DX_PDFLATEX) refman.tex; \
-	    countdown=`expr $$countdown - 1`; \
-	done; \
-	mv refman.pdf ../@PACKAGE@.pdf
-
-endif DX_COND_pdf
-
-## ------------------------------------------------- ##
-## Rules specific for LaTeX (shared for PS and PDF). ##
-## ------------------------------------------------- ##
-
-if DX_COND_latex
-
-DX_CLEAN_LATEX = @DX_DOCDIR@/latex
-
-endif DX_COND_latex
-
-.INTERMEDIATE: doxygen-run $(DX_PS_GOAL) $(DX_PDF_GOAL)
-
-doxygen-run: @DX_DOCDIR@/@PACKAGE@.tag
-
-doxygen-doc: doxygen-run $(DX_PS_GOAL) $(DX_PDF_GOAL)
-
-@DX_DOCDIR@/@PACKAGE@.tag: $(DX_CONFIG) $(pkginclude_HEADERS)
-	rm -rf @DX_DOCDIR@
-	$(DX_ENV) $(DX_DOXYGEN) $(srcdir)/$(DX_CONFIG)
-
-DX_CLEANFILES = \
-    @DX_DOCDIR@/@PACKAGE@.tag \
-    -r \
-    $(DX_CLEAN_HTML) \
-    $(DX_CLEAN_CHM) \
-    $(DX_CLEAN_CHI) \
-    $(DX_CLEAN_MAN) \
-    $(DX_CLEAN_RTF) \
-    $(DX_CLEAN_XML) \
-    $(DX_CLEAN_PS) \
-    $(DX_CLEAN_PDF) \
-    $(DX_CLEAN_LATEX)
-
-endif DX_COND_doc

lib/AlignedAllocator.h

@@ -40,6 +40,14 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <mm_malloc.h>
 #endif
 
+#ifdef GRID_COMMS_SHMEM
+extern "C" { 
+#include <mpp/shmem.h>
+extern void * shmem_align(size_t, size_t);
+extern void  shmem_free(void *);
+}
+#endif
+
 namespace Grid {
 
 ////////////////////////////////////////////////////////////////////
@@ -57,21 +65,53 @@ public:
   typedef _Tp        value_type;
 
   template<typename _Tp1>  struct rebind { typedef alignedAllocator<_Tp1> other; };
+
   alignedAllocator() throw() { }
+
   alignedAllocator(const alignedAllocator&) throw() { }
+
   template<typename _Tp1> alignedAllocator(const alignedAllocator<_Tp1>&) throw() { }
+
   ~alignedAllocator() throw() { }
+
   pointer       address(reference __x)       const { return &__x; }
+  //  const_pointer address(const_reference __x) const { return &__x; }
+
   size_type  max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
 
   pointer allocate(size_type __n, const void* _p= 0)
   { 
+#ifdef GRID_COMMS_SHMEM
+
+    _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64);
+
+
+#define PARANOID_SYMMETRIC_HEAP
+#ifdef PARANOID_SYMMETRIC_HEAP
+    static void * bcast;
+    static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
+
+    bcast = (void *) ptr;
+    shmem_broadcast32((void *)&bcast,(void *)&bcast,sizeof(void *)/4,0,0,0,shmem_n_pes(),psync);
+
+    if ( bcast != ptr ) {
+      std::printf("inconsistent alloc pe %d %lx %lx \n",shmem_my_pe(),bcast,ptr);std::fflush(stdout);
+      BACKTRACEFILE();
+      exit(0);
+    }
+
+    assert( bcast == (void *) ptr);
+
+#endif 
+#else
+
 #ifdef HAVE_MM_MALLOC_H
     _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128);
 #else
     _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp));
 #endif
 
+#endif
     _Tp tmp;
 #ifdef GRID_NUMA
 #pragma omp parallel for schedule(static)
@@ -83,109 +123,27 @@ public:
   }
 
   void deallocate(pointer __p, size_type) { 
-#ifdef HAVE_MM_MALLOC_H
-    _mm_free((void *)__p); 
-#else
-    free((void *)__p);
-#endif
-  }
-  void construct(pointer __p, const _Tp& __val) { };
-  void construct(pointer __p) { };
-  void destroy(pointer __p) { };
-};
-template<typename _Tp>  inline bool operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
-template<typename _Tp>  inline bool operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
-
-//////////////////////////////////////////////////////////////////////////////////////////
-// MPI3 : comms must use shm region
-// SHMEM: comms must use symmetric heap
-//////////////////////////////////////////////////////////////////////////////////////////
 #ifdef GRID_COMMS_SHMEM
-extern "C" { 
-#include <mpp/shmem.h>
-extern void * shmem_align(size_t, size_t);
-extern void  shmem_free(void *);
-}
-#define PARANOID_SYMMETRIC_HEAP
-#endif
-
-template<typename _Tp>
-class commAllocator {
-public: 
-  typedef std::size_t     size_type;
-  typedef std::ptrdiff_t  difference_type;
-  typedef _Tp*       pointer;
-  typedef const _Tp* const_pointer;
-  typedef _Tp&       reference;
-  typedef const _Tp& const_reference;
-  typedef _Tp        value_type;
-
-  template<typename _Tp1>  struct rebind { typedef commAllocator<_Tp1> other; };
-  commAllocator() throw() { }
-  commAllocator(const commAllocator&) throw() { }
-  template<typename _Tp1> commAllocator(const commAllocator<_Tp1>&) throw() { }
-  ~commAllocator() throw() { }
-  pointer       address(reference __x)       const { return &__x; }
-  size_type  max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
-
-#ifdef GRID_COMMS_SHMEM
-  pointer allocate(size_type __n, const void* _p= 0)
-  {
-#ifdef CRAY
-    _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64);
-#else
-    _Tp *ptr = (_Tp *) shmem_align(64,__n*sizeof(_Tp));
-#endif
-#ifdef PARANOID_SYMMETRIC_HEAP
-    static void * bcast;
-    static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
-
-    bcast = (void *) ptr;
-    shmem_broadcast32((void *)&bcast,(void *)&bcast,sizeof(void *)/4,0,0,0,shmem_n_pes(),psync);
-
-    if ( bcast != ptr ) {
-      std::printf("inconsistent alloc pe %d %lx %lx \n",shmem_my_pe(),bcast,ptr);std::fflush(stdout);
-      //      BACKTRACEFILE();
-      exit(0);
-    }
-    assert( bcast == (void *) ptr);
-#endif 
-    return ptr;
-  }
-  void deallocate(pointer __p, size_type) { 
     shmem_free((void *)__p);
-  }
 #else
-  pointer allocate(size_type __n, const void* _p= 0) 
-  {
-#ifdef HAVE_MM_MALLOC_H
-    _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128);
-#else
-    _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp));
-#endif
-    return ptr;
-  }
-  void deallocate(pointer __p, size_type) { 
 #ifdef HAVE_MM_MALLOC_H
     _mm_free((void *)__p); 
 #else
     free((void *)__p);
 #endif
-  }
 #endif
+  }
   void construct(pointer __p, const _Tp& __val) { };
   void construct(pointer __p) { };
+
   void destroy(pointer __p) { };
 };
-template<typename _Tp>  inline bool operator==(const commAllocator<_Tp>&, const commAllocator<_Tp>&){ return true; }
-template<typename _Tp>  inline bool operator!=(const commAllocator<_Tp>&, const commAllocator<_Tp>&){ return false; }
 
-////////////////////////////////////////////////////////////////////////////////
-// Template typedefs
-////////////////////////////////////////////////////////////////////////////////
-template<class T> using Vector     = std::vector<T,alignedAllocator<T> >;           
-template<class T> using commVector = std::vector<T,commAllocator<T> >;              
-template<class T> using Matrix     = std::vector<std::vector<T,alignedAllocator<T> > >;
+template<typename _Tp>  inline bool
+operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
+
+template<typename _Tp>  inline bool
+operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
     
 }; // namespace Grid
 #endif

lib/Bitwise.cc

@@ -1,49 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/Bitwise.cc
-
-Copyright (C) 2016
-
-Author: Guido Cossu <guido.cossu@ed.ac.uk>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <iostream>
-#include <Bitwise.h>
-#include <bitset>
-#include <climits>
-
-namespace Grid {
-
-void show_binaryrep(const unsigned char* a, size_t size) {
-  const unsigned char* beg = a;
-  const unsigned char* end = a + size;
-  unsigned int ctr = 0;
-  while (beg != end) {
-    std::cout << std::bitset<CHAR_BIT>(*beg++) << ' ';
-    ctr++;
-    if (ctr % GRID_REAL_BYTES == 0) std::cout << '\n';
-  }
-  std::cout << '\n';
-}
-
-} // namespace 
-

lib/Bitwise.h

@@ -1,76 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/Bitwise.h
-
-Copyright (C) 2016
-
-Author: Guido Cossu <guido.cossu@ed.ac.uk>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifndef GRID_BITWISE_H
-#define GRID_BITWISE_H
-
-#include <cassert>
-#include <cfloat>
-#include <bitset>
-#include <climits>
-#include <Config.h>
-
-#ifdef GRID_DEFAULT_PRECISION_SINGLE
-#define GRID_REAL_BYTES 4
-#endif
-#ifdef GRID_DEFAULT_PRECISION_DOUBLE
-#define GRID_REAL_BYTES 8
-#endif
-
-
-namespace Grid {
-
-void show_binaryrep(const unsigned char* a, size_t size);
-
-template <typename T>
-void show_binaryrep(const T& a) {
-  const char* beg = reinterpret_cast<const char*>(&a);
-  const char* end = beg + sizeof(a);
-  unsigned int ctr = 0;
-  while (beg != end) {
-  	std::cout << std::bitset<CHAR_BIT>(*beg++) << ' ';
-  	ctr++;
-  	if (ctr % GRID_REAL_BYTES == 0) std::cout << '\n';
-  }
-  std::cout << '\n';
-}
-
-template <typename T>
-void bitwise_xor(T& l, T& r, unsigned char* xors) {
-  assert(sizeof(l) == sizeof(r));
-  unsigned char* org = reinterpret_cast<unsigned char*>(&l);
-  unsigned char* cur = reinterpret_cast<unsigned char*>(&r);
-  int words = sizeof(l) / sizeof(*org);
-  unsigned char result = 0;
-  for (int w = 0; w < words; w++) xors[w] = (org[w] ^ cur[w]);
-}
-
-}; // namespace 
-
-
-#endif

lib/Cshift.h

@@ -38,14 +38,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/cshift/Cshift_mpi.h>
 #endif 
 
-#ifdef GRID_COMMS_MPI3
-#include <Grid/cshift/Cshift_mpi.h>
-#endif 
-
-#ifdef GRID_COMMS_MPI3L
-#include <Grid/cshift/Cshift_mpi.h>
-#endif 
-
 #ifdef GRID_COMMS_SHMEM
 #include <Grid/cshift/Cshift_mpi.h> // uses same implementation of communicator
 #endif 

lib/FFT.h

@@ -30,14 +30,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #define _GRID_FFT_H_
 
 #ifdef HAVE_FFTW	
-#ifdef USE_MKL
-#include <fftw/fftw3.h>
-#else
 #include <fftw3.h>
 #endif
-#endif
-
-
 namespace Grid {
 
   template<class scalar> struct FFTW { };
@@ -126,7 +120,6 @@ namespace Grid {
 
     double Flops(void) {return flops;}
     double MFlops(void) {return flops/usec;}
-    double USec(void)   {return (double)usec;}    
 
     FFT ( GridCartesian * grid ) : 
       vgrid(grid),
@@ -146,34 +139,10 @@ namespace Grid {
     }
     
     template<class vobj>
-    void FFT_dim_mask(Lattice<vobj> &result,const Lattice<vobj> &source,std::vector<int> mask,int sign){
+    void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int inverse){
 
       conformable(result._grid,vgrid);
       conformable(source._grid,vgrid);
-      Lattice<vobj> tmp(vgrid);
-      tmp = source;
-      for(int d=0;d<Nd;d++){
-	if( mask[d] ) {
-	  FFT_dim(result,tmp,d,sign);
-	  tmp=result;
-	}
-      }
-    }
-
-    template<class vobj>
-    void FFT_all_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int sign){
-      std::vector<int> mask(Nd,1);
-      FFT_dim_mask(result,source,mask,sign);
-    }
-
-
-    template<class vobj>
-    void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int sign){
-#ifndef HAVE_FFTW
-      assert(0);
-#else
-      conformable(result._grid,vgrid);
-      conformable(source._grid,vgrid);
 
       int L = vgrid->_ldimensions[dim];
       int G = vgrid->_fdimensions[dim];
@@ -190,12 +159,17 @@ namespace Grid {
       typedef typename vobj::scalar_object sobj;
       typedef typename sobj::scalar_type   scalar;
 
-      Lattice<sobj> pgbuf(&pencil_g);
-      
+      Lattice<vobj> ssource(vgrid); ssource =source;
+      Lattice<sobj> pgsource(&pencil_g);
+      Lattice<sobj> pgresult(&pencil_g); pgresult=zero;
 
+#ifndef HAVE_FFTW	
+      assert(0);
+#else 
       typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
       typedef typename FFTW<scalar>::FFTW_plan   FFTW_plan;
 
+      {
 	int Ncomp = sizeof(sobj)/sizeof(scalar);
 	int Nlow  = 1;
 	for(int d=0;d<dim;d++){
@@ -210,15 +184,14 @@ namespace Grid {
 	istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */
 	int *inembed = n, *onembed = n;
 
-      scalar div;
-	  if ( sign == backward ) div = 1.0/G;
-	  else if ( sign == forward ) div = 1.0;
-	  else assert(0);
+	
+	int sign = FFTW_FORWARD;
+	if (inverse) sign = FFTW_BACKWARD;
 
 	FFTW_plan p;
 	{
-        FFTW_scalar *in = (FFTW_scalar *)&pgbuf._odata[0];
-        FFTW_scalar *out= (FFTW_scalar *)&pgbuf._odata[0];
+	  FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[0];
+	  FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[0];
 	  p = FFTW<scalar>::fftw_plan_many_dft(rank,n,howmany,
 					       in,inembed,
 					       istride,idist,
@@ -227,79 +200,77 @@ namespace Grid {
 					       sign,FFTW_ESTIMATE);
 	}
 
+	double add,mul,fma;
+	FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
+	flops_call = add+mul+2.0*fma;
+
+	GridStopWatch timer;
+
 	// Barrel shift and collect global pencil
-      std::vector<int> lcoor(Nd), gcoor(Nd);
-      result = source;
 	for(int p=0;p<processors[dim];p++) { 
-        PARALLEL_REGION
-        {
-          std::vector<int> cbuf(Nd);
+
+	  for(int idx=0;idx<sgrid->lSites();idx++) { 
+
+	    std::vector<int> lcoor(Nd);
+    	    sgrid->LocalIndexToLocalCoor(idx,lcoor);
+
 	    sobj s;
 
-          PARALLEL_FOR_LOOP_INTERN
-          for(int idx=0;idx<sgrid->lSites();idx++) {
-            sgrid->LocalIndexToLocalCoor(idx,cbuf);
-            peekLocalSite(s,result,cbuf);
-            cbuf[dim]+=p*L;
-            pokeLocalSite(s,pgbuf,cbuf);
-          }
-        }
-        if (p != processors[dim] - 1)
-        {
-          result = Cshift(result,dim,L);
+	    peekLocalSite(s,ssource,lcoor);
+
+	    lcoor[dim]+=p*L;
+	   
+	    pokeLocalSite(s,pgsource,lcoor);
 	  }
+
+	  ssource = Cshift(ssource,dim,L);
 	}
 	
 	// Loop over orthog coords
 	int NN=pencil_g.lSites();
-      GridStopWatch timer;
-      timer.Start();
-      PARALLEL_REGION
-      {
-        std::vector<int> cbuf(Nd);
 
-        PARALLEL_FOR_LOOP_INTERN
+	GridStopWatch Timer;
+	Timer.Start();
+
+PARALLEL_FOR_LOOP
 	for(int idx=0;idx<NN;idx++) { 
-          pencil_g.LocalIndexToLocalCoor(idx, cbuf);
-          if ( cbuf[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
-            FFTW_scalar *in = (FFTW_scalar *)&pgbuf._odata[idx];
-            FFTW_scalar *out= (FFTW_scalar *)&pgbuf._odata[idx];
+
+	  std::vector<int> lcoor(Nd);
+	  pencil_g.LocalIndexToLocalCoor(idx,lcoor);
+
+	  if ( lcoor[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
+	    FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[idx];
+	    FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[idx];
 	    FFTW<scalar>::fftw_execute_dft(p,in,out);
 	  }
 	}
-      }
-      timer.Stop();
 
-      // performance counting
-      double add,mul,fma;
-      FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
-      flops_call = add+mul+2.0*fma;
-      usec += timer.useconds();
+        Timer.Stop();
+	usec += Timer.useconds();
 	flops+= flops_call*NN;
 
-      // writing out result
         int pc = processor_coor[dim];
-      PARALLEL_REGION
-      {
-        std::vector<int> clbuf(Nd), cgbuf(Nd);
-        sobj s;
-        
-        PARALLEL_FOR_LOOP_INTERN
         for(int idx=0;idx<sgrid->lSites();idx++) { 
-          sgrid->LocalIndexToLocalCoor(idx,clbuf);
-          cgbuf = clbuf;
-          cgbuf[dim] = clbuf[dim]+L*pc;
-          peekLocalSite(s,pgbuf,cgbuf);
-          pokeLocalSite(s,result,clbuf);
+	  std::vector<int> lcoor(Nd);
+	  sgrid->LocalIndexToLocalCoor(idx,lcoor);
+	  std::vector<int> gcoor = lcoor;
+	  // extract the result
+	  sobj s;
+	  gcoor[dim] = lcoor[dim]+L*pc;
+	  peekLocalSite(s,pgresult,gcoor);
+	  pokeLocalSite(s,result,lcoor);
 	}
-      }
-      result = result*div;
       	  
-      // destroying plan
 	FFTW<scalar>::fftw_destroy_plan(p);
-#endif
       }
+#endif
+
+
+    }
+
   };
+
+
 }
 
 #endif

lib/Grid.h

@@ -62,7 +62,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/serialisation/Serialisation.h>
 #include "Config.h"
 #include <Grid/Timer.h>
-#include <Grid/Bitwise.h>
 #include <Grid/PerfCount.h>
 #include <Grid/Log.h>
 #include <Grid/AlignedAllocator.h>
@@ -78,10 +77,11 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/Stencil.h>      
 #include <Grid/Algorithms.h>   
 #include <Grid/parallelIO/BinaryIO.h>
-#include <Grid/FFT.h>
-
 #include <Grid/qcd/QCD.h>
 #include <Grid/parallelIO/NerscIO.h>
+
+#include <Grid/FFT.h>
+
 #include <Grid/qcd/hmc/NerscCheckpointer.h>
 #include <Grid/qcd/hmc/HmcRunner.h>
 

lib/Init.cc

@@ -44,33 +44,9 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid.h>
 #include <algorithm>
 #include <iterator>
-#include <cstdlib>
-#include <memory>
-
-
-#include <fenv.h>
-#ifdef __APPLE__
-static int
-feenableexcept (unsigned int excepts)
-{
-  static fenv_t fenv;
-  unsigned int new_excepts = excepts & FE_ALL_EXCEPT,
-    old_excepts;  // previous masks
-
-  if ( fegetenv (&fenv) ) return -1;
-  old_excepts = fenv.__control & FE_ALL_EXCEPT;
-
-  // unmask
-  fenv.__control &= ~new_excepts;
-  fenv.__mxcsr   &= ~(new_excepts << 7);
-
-  return ( fesetenv (&fenv) ? -1 : old_excepts );
-}
-#endif
 
 namespace Grid {
 
-
 //////////////////////////////////////////////////////
 // Convenience functions to access stadard command line arg
 // driven parallelism controls
@@ -147,13 +123,6 @@ void GridCmdOptionIntVector(std::string &str,std::vector<int> & vec)
   return;
 }
 
-void GridCmdOptionInt(std::string &str,int & val)
-{
-  std::stringstream ss(str);
-  ss>>val;
-  return;
-}
-
 
 void GridParseLayout(char **argv,int argc,
 		     std::vector<int> &latt,
@@ -184,12 +153,14 @@ void GridParseLayout(char **argv,int argc,
     assert(ompthreads.size()==1);
     GridThread::SetThreads(ompthreads[0]);
   }
+
   if( GridCmdOptionExists(argv,argv+argc,"--cores") ){
-    int cores;
+    std::vector<int> cores(0);
     arg= GridCmdOptionPayload(argv,argv+argc,"--cores");
-    GridCmdOptionInt(arg,cores);
-    GridThread::SetCores(cores);
+    GridCmdOptionIntVector(arg,cores);
+    GridThread::SetCores(cores[0]);
   }
+
 }
 
 std::string GridCmdVectorIntToString(const std::vector<int> & vec){
@@ -198,40 +169,33 @@ std::string GridCmdVectorIntToString(const std::vector<int> & vec){
   return oss.str();
 }
 /////////////////////////////////////////////////////////
-// Reinit guard
+//
 /////////////////////////////////////////////////////////
-static int Grid_is_initialised = 0;
-
-
 void Grid_init(int *argc,char ***argv)
 {
+  CartesianCommunicator::Init(argc,argv);
+
+  // Parse command line args.
+
   GridLogger::StopWatch.Start();
 
   std::string arg;
-
-  ////////////////////////////////////
-  // Shared memory block size
-  ////////////////////////////////////
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--shm") ){
-    int MB;
-    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--shm");
-    GridCmdOptionInt(arg,MB);
-    CartesianCommunicator::MAX_MPI_SHM_BYTES = MB*1024*1024;
-  }
-
-  CartesianCommunicator::Init(argc,argv);
-
-  ////////////////////////////////////
-  // Logging
-  ////////////////////////////////////
-
   std::vector<std::string> logstreams;
   std::string defaultLog("Error,Warning,Message,Performance");
+
   GridCmdOptionCSL(defaultLog,logstreams);
   GridLogConfigure(logstreams);
 
-  if( !GridCmdOptionExists(*argv,*argv+*argc,"--debug-stdout") ){
-    Grid_quiesce_nodes();
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--help") ){
+    std::cout<<GridLogMessage<<"--help : this message"<<std::endl;
+    std::cout<<GridLogMessage<<"--debug-signals : catch sigsegv and print a blame report"<<std::endl;
+    std::cout<<GridLogMessage<<"--debug-stdout  : print stdout from EVERY node"<<std::endl;    
+    std::cout<<GridLogMessage<<"--decomposition : report on default omp,mpi and simd decomposition"<<std::endl;    
+    std::cout<<GridLogMessage<<"--mpi n.n.n.n   : default MPI decomposition"<<std::endl;    
+    std::cout<<GridLogMessage<<"--threads n     : default number of OMP threads"<<std::endl;
+    std::cout<<GridLogMessage<<"--grid n.n.n.n  : default Grid size"<<std::endl;    
+    std::cout<<GridLogMessage<<"--log list      : comma separted list of streams from Error,Warning,Message,Performance,Iterative,Integrator,Debug,Colours"<<std::endl;
+    exit(EXIT_SUCCESS);
   }
 
   if( GridCmdOptionExists(*argv,*argv+*argc,"--log") ){
@@ -240,39 +204,35 @@ void Grid_init(int *argc,char ***argv)
     GridLogConfigure(logstreams);
   }
 
-  ////////////////////////////////////
-  // Help message
-  ////////////////////////////////////
-
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--help") ){
-    std::cout<<GridLogMessage<<"  --help : this message"<<std::endl;
-    std::cout<<GridLogMessage<<std::endl;
-    std::cout<<GridLogMessage<<"Geometry:"<<std::endl;
-    std::cout<<GridLogMessage<<"  --mpi n.n.n.n   : default MPI decomposition"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --threads n     : default number of OMP threads"<<std::endl;
-    std::cout<<GridLogMessage<<"  --grid n.n.n.n  : default Grid size"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --shm  M        : allocate M megabytes of shared memory for comms"<<std::endl;    
-    std::cout<<GridLogMessage<<std::endl;
-    std::cout<<GridLogMessage<<"Verbose and debug:"<<std::endl;
-    std::cout<<GridLogMessage<<"  --log list      : comma separted list of streams from Error,Warning,Message,Performance,Iterative,Integrator,Debug,Colours"<<std::endl;
-    std::cout<<GridLogMessage<<"  --decomposition : report on default omp,mpi and simd decomposition"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --debug-signals : catch sigsegv and print a blame report"<<std::endl;
-    std::cout<<GridLogMessage<<"  --debug-stdout  : print stdout from EVERY node"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --notimestamp   : suppress millisecond resolution stamps"<<std::endl;    
-    std::cout<<GridLogMessage<<std::endl;
-    std::cout<<GridLogMessage<<"Performance:"<<std::endl;
-    std::cout<<GridLogMessage<<"  --dslash-generic: Wilson kernel for generic Nc"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --dslash-unroll : Wilson kernel for Nc=3"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --dslash-asm    : Wilson kernel for AVX512"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --lebesgue      : Cache oblivious Lebesgue curve/Morton order/Z-graph stencil looping"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --cacheblocking n.m.o.p : Hypercuboidal cache blocking"<<std::endl;    
-    std::cout<<GridLogMessage<<std::endl;
-    exit(EXIT_SUCCESS);
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-signals") ){
+    Grid_debug_handler_init();
+  }
+  if( !GridCmdOptionExists(*argv,*argv+*argc,"--debug-stdout") ){
+    Grid_quiesce_nodes();
+  }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-opt") ){
+    QCD::WilsonKernelsStatic::HandOpt=1;
+  }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--lebesgue") ){
+    LebesgueOrder::UseLebesgueOrder=1;
   }
 
-  ////////////////////////////////////
-  // Banner
-  ////////////////////////////////////
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--cacheblocking") ){
+    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--cacheblocking");
+    GridCmdOptionIntVector(arg,LebesgueOrder::Block);
+  }
+  GridParseLayout(*argv,*argc,
+		  Grid_default_latt,
+		  Grid_default_mpi);
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--decomposition") ){
+    std::cout<<GridLogMessage<<"Grid Decomposition\n";
+    std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
+    std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealD         : "<<sizeof(vRealD)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexF      : "<<sizeof(vComplexF)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexD      : "<<sizeof(vComplexD)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexD::Nsimd()))<<std::endl;
+  }
 
   std::string COL_RED    = GridLogColours.colour["RED"];
   std::string COL_PURPLE = GridLogColours.colour["PURPLE"];
@@ -282,6 +242,7 @@ void Grid_init(int *argc,char ***argv)
   std::string COL_YELLOW = GridLogColours.colour["YELLOW"];
   std::string COL_BACKGROUND = GridLogColours.colour["NORMAL"];
 
+  
   std::cout <<std::endl;
   std::cout <<COL_RED  << "__|__|__|__|__"<<             "|__|__|_"<<COL_PURPLE<<"_|__|__|"<<                "__|__|__|__|__"<<std::endl; 
   std::cout <<COL_RED  << "__|__|__|__|__"<<             "|__|__|_"<<COL_PURPLE<<"_|__|__|"<<                "__|__|__|__|__"<<std::endl; 
@@ -313,63 +274,12 @@ void Grid_init(int *argc,char ***argv)
   std::cout << "GNU General Public License for more details."<<std::endl;
   std::cout << COL_BACKGROUND <<std::endl;
   std::cout << std::endl;
-
-  ////////////////////////////////////
-  // Debug and performance options
-  ////////////////////////////////////
-
-
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-signals") ){
-    Grid_debug_handler_init();
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-unroll") ){
-    QCD::WilsonKernelsStatic::Opt=QCD::WilsonKernelsStatic::OptHandUnroll;
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-asm") ){
-    QCD::WilsonKernelsStatic::Opt=QCD::WilsonKernelsStatic::OptInlineAsm;
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-generic") ){
-    QCD::WilsonKernelsStatic::Opt=QCD::WilsonKernelsStatic::OptGeneric;
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--lebesgue") ){
-    LebesgueOrder::UseLebesgueOrder=1;
-  }
-
-
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--cacheblocking") ){
-    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--cacheblocking");
-    GridCmdOptionIntVector(arg,LebesgueOrder::Block);
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--notimestamp") ){
-    GridLogTimestamp(0);
-  } else { 
-    GridLogTimestamp(1);
-  }
-
-  GridParseLayout(*argv,*argc,
-		  Grid_default_latt,
-		  Grid_default_mpi);
-
-  std::cout << GridLogMessage << "Requesting "<< CartesianCommunicator::MAX_MPI_SHM_BYTES <<" byte stencil comms buffers "<<std::endl;
-
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--decomposition") ){
-    std::cout<<GridLogMessage<<"Grid Decomposition\n";
-    std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
-    std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
-    std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
-    std::cout<<GridLogMessage<<"\tvRealD         : "<<sizeof(vRealD)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealD::Nsimd()))<<std::endl;
-    std::cout<<GridLogMessage<<"\tvComplexF      : "<<sizeof(vComplexF)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexF::Nsimd()))<<std::endl;
-    std::cout<<GridLogMessage<<"\tvComplexD      : "<<sizeof(vComplexD)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexD::Nsimd()))<<std::endl;
-  }
-
-
-  Grid_is_initialised = 1;
 }
 
   
 void Grid_finalize(void)
 {
-#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) 
+#ifdef GRID_COMMS_MPI
   MPI_Finalize();
   Grid_unquiesce_nodes();
 #endif
@@ -416,7 +326,10 @@ void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr)
   exit(0);
   return;
 };
-
+#ifdef GRID_FPE
+#define _GNU_SOURCE
+#include <fenv.h>
+#endif
 void Grid_debug_handler_init(void)
 {
   struct sigaction sa,osa;
@@ -425,9 +338,9 @@ void Grid_debug_handler_init(void)
   sa.sa_flags    = SA_SIGINFO;
   sigaction(SIGSEGV,&sa,NULL);
   sigaction(SIGTRAP,&sa,NULL);
-
+#ifdef GRID_FPE
   feenableexcept( FE_INVALID|FE_OVERFLOW|FE_DIVBYZERO);
-
   sigaction(SIGFPE,&sa,NULL);
+#endif
 }
 }

lib/Init.h

@@ -33,7 +33,6 @@ namespace Grid {
 
   void Grid_init(int *argc,char ***argv);
   void Grid_finalize(void);
-
   // internal, controled with --handle
   void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr);
   void Grid_debug_handler_init(void);
@@ -45,7 +44,6 @@ namespace Grid {
   const std::vector<int> &GridDefaultMpi(void);
   const int              &GridThreads(void)  ;
   void                    GridSetThreads(int t) ;
-  void GridLogTimestamp(int);
 
   // Common parsing chores
   std::string GridCmdOptionPayload(char ** begin, char ** end, const std::string & option);
@@ -54,7 +52,6 @@ namespace Grid {
   void GridCmdOptionCSL(std::string str,std::vector<std::string> & vec);
   void GridCmdOptionIntVector(std::string &str,std::vector<int> & vec);
 
-
   void GridParseLayout(char **argv,int argc,
 		       std::vector<int> &latt,
 		       std::vector<int> &simd,

lib/Log.cc

@@ -31,31 +31,11 @@ directory
 /*  END LEGAL */
 #include <Grid.h>
 
-#include <cxxabi.h>
-
 namespace Grid {
 
-  std::string demangle(const char* name) {
-    
-    int status = -4; // some arbitrary value to eliminate the compiler warning
-    
-    // enable c++11 by passing the flag -std=c++11 to g++
-    std::unique_ptr<char, void(*)(void*)> res {
-      abi::__cxa_demangle(name, NULL, NULL, &status),
-	std::free
-	};
-    
-    return (status==0) ? res.get() : name ;
-  }
-  
 GridStopWatch Logger::StopWatch;
-int Logger::timestamp;
 std::ostream Logger::devnull(0);
 
-void GridLogTimestamp(int on){
-  Logger::Timestamp(on);
-}
-
 Colours GridLogColours(0);
 GridLogger GridLogError(1, "Error", GridLogColours, "RED");
 GridLogger GridLogWarning(1, "Warning", GridLogColours, "YELLOW");
@@ -93,14 +73,14 @@ void GridLogConfigure(std::vector<std::string> &logstreams) {
 ////////////////////////////////////////////////////////////
 void Grid_quiesce_nodes(void) {
   int me = 0;
-#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPI3L)
+#ifdef GRID_COMMS_MPI
   MPI_Comm_rank(MPI_COMM_WORLD, &me);
 #endif
 #ifdef GRID_COMMS_SHMEM
   me = shmem_my_pe();
 #endif
   if (me) {
-    std::cout.setstate(std::ios::badbit);// mute all nodes except 0
+    std::cout.setstate(std::ios::badbit);
   }
 }
 

lib/Log.h

@@ -37,11 +37,10 @@
 #include <execinfo.h>
 #endif
 
-namespace Grid {
+    namespace Grid {
 
-//////////////////////////////////////////////////////////////////////////////////////////////////
 // Dress the output; use std::chrono for time stamping via the StopWatch class
-//////////////////////////////////////////////////////////////////////////////////////////////////
+int Rank(void); // used for early stage debug before library init
 
 
 class Colours{
@@ -56,6 +55,7 @@ public:
 
   void Active(bool activate){
     is_active=activate;
+
     if (is_active){
      colour["BLACK"]  ="\033[30m";
      colour["RED"]    ="\033[31m";
@@ -77,7 +77,10 @@ public:
     colour["WHITE"] ="";
     colour["NORMAL"]="";
   }
-  };
+
+
+};
+
 };
 
 
@@ -85,7 +88,6 @@ class Logger {
 protected:
   Colours &Painter;
   int active;
-  static int timestamp;
   std::string name, topName;
   std::string COLOUR;
 
@@ -97,28 +99,25 @@ public:
   std::string evidence() {return Painter.colour["YELLOW"];}
   std::string colour() {return Painter.colour[COLOUR];}
 
-  Logger(std::string topNm, int on, std::string nm, Colours& col_class, std::string col)  : active(on),
+  Logger(std::string topNm, int on, std::string nm, Colours& col_class, std::string col)
+  : active(on),
   name(nm),
   topName(topNm),
   Painter(col_class),
-    COLOUR(col) {} ;
+  COLOUR(col){} ;
   
   void Active(int on) {active = on;};
   int  isActive(void) {return active;};
-  static void Timestamp(int on) {timestamp = on;};
   
   friend std::ostream& operator<< (std::ostream& stream, Logger& log){
 
     if ( log.active ) {
-      stream << log.background()<< log.topName << log.background()<< " : ";
-      stream << log.colour() <<std::setw(14) << std::left << log.name << log.background() << " : ";
-      if ( log.timestamp ) {
       StopWatch.Stop();
       GridTime now = StopWatch.Elapsed();
       StopWatch.Start();
-	stream << log.evidence()<< now << log.background() << " : " ;
-      }
-      stream << log.colour();
+      stream << log.background()<< log.topName << log.background()<< " : ";
+      stream << log.colour() <<std::setw(14) << std::left << log.name << log.background() << " : ";
+      stream << log.evidence()<< now << log.background() << " : " << log.colour();
       return stream;
     } else { 
       return devnull;
@@ -144,14 +143,13 @@ extern GridLogger GridLogIterative  ;
 extern GridLogger GridLogIntegrator  ;
 extern Colours    GridLogColours;
 
- std::string demangle(const char* name) ;
 
 #define _NBACKTRACE (256)
 extern void * Grid_backtrace_buffer[_NBACKTRACE];
 
 #define BACKTRACEFILE() {\
 char string[20];					\
-std::sprintf(string,"backtrace.%d",CartesianCommunicator::RankWorld()); \
+std::sprintf(string,"backtrace.%d",Rank());				\
 std::FILE * fp = std::fopen(string,"w");				\
 BACKTRACEFP(fp)\
 std::fclose(fp);	    \
@@ -163,7 +161,7 @@ std::fclose(fp);	    \
 int symbols    = backtrace        (Grid_backtrace_buffer,_NBACKTRACE);\
 char **strings = backtrace_symbols(Grid_backtrace_buffer,symbols);\
 for (int i = 0; i < symbols; i++){\
-  std::fprintf (fp,"BackTrace Strings: %d %s\n",i, demangle(strings[i]).c_str()); std::fflush(fp); \
+  std::fprintf (fp,"BackTrace Strings: %d %s\n",i, strings[i]); std::fflush(fp); \
 }\
 }
 #else 

lib/Makefile.am

@@ -1,27 +1,14 @@
 extra_sources=
 if BUILD_COMMS_MPI
   extra_sources+=communicator/Communicator_mpi.cc
-  extra_sources+=communicator/Communicator_base.cc
-endif
-
-if BUILD_COMMS_MPI3
-  extra_sources+=communicator/Communicator_mpi3.cc
-  extra_sources+=communicator/Communicator_base.cc
-endif
-
-if BUILD_COMMS_MPI3L
-  extra_sources+=communicator/Communicator_mpi3_leader.cc
-  extra_sources+=communicator/Communicator_base.cc
 endif
 
 if BUILD_COMMS_SHMEM
   extra_sources+=communicator/Communicator_shmem.cc
-  extra_sources+=communicator/Communicator_base.cc
 endif
 
 if BUILD_COMMS_NONE
   extra_sources+=communicator/Communicator_none.cc
-  extra_sources+=communicator/Communicator_base.cc
 endif
 
 #

lib/PerfCount.h

@@ -43,9 +43,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #else
 #include <sys/syscall.h>
 #endif
-#ifdef __x86_64__
-#include <x86intrin.h>
-#endif
 
 namespace Grid {
 
@@ -89,6 +86,7 @@ inline uint64_t cyclecount(void){
    return tmp;
 }
 #elif defined __x86_64__
+#include <x86intrin.h>
 inline uint64_t cyclecount(void){ 
   return __rdtsc();
   //  unsigned int dummy;

lib/Simd.h

@@ -237,18 +237,6 @@ namespace Grid {
     stream<<">";
     return stream;
   }
-  inline std::ostream& operator<< (std::ostream& stream, const vInteger &o){
-    int nn=vInteger::Nsimd();
-    std::vector<Integer,alignedAllocator<Integer> > buf(nn);
-    vstore(o,&buf[0]);
-    stream<<"<";
-    for(int i=0;i<nn;i++){
-      stream<<buf[i];
-      if(i<nn-1) stream<<",";
-    }
-    stream<<">";
-    return stream;
-  }
 
 
 }

lib/Stencil.h

@@ -68,22 +68,22 @@
  //
  //////////////////////////////////////////////////////////////////////////////////////////
 
-namespace Grid {
+ namespace Grid {
 
-inline void Gather_plane_simple_table_compute (GridBase *grid,int dimension,int plane,int cbmask,
-					       int off,std::vector<std::pair<int,int> > & table)
+template<class vobj,class cobj,class compressor> void 
+Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,std::vector<std::pair<int,int> >& table)
 {
   table.resize(0);
-  int rd = grid->_rdimensions[dimension];
+  int rd = rhs._grid->_rdimensions[dimension];
 
-  if ( !grid->CheckerBoarded(dimension) ) {
+  if ( !rhs._grid->CheckerBoarded(dimension) ) {
     cbmask = 0x3;
   }
-  int so= plane*grid->_ostride[dimension]; // base offset for start of plane 
-  int e1=grid->_slice_nblock[dimension];
-  int e2=grid->_slice_block[dimension];
+  int so= plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
+  int e1=rhs._grid->_slice_nblock[dimension];
+  int e2=rhs._grid->_slice_block[dimension];
 
-  int stride=grid->_slice_stride[dimension];
+  int stride=rhs._grid->_slice_stride[dimension];
   if ( cbmask == 0x3 ) { 
     table.resize(e1*e2);
     for(int n=0;n<e1;n++){
@@ -99,7 +99,7 @@ inline void Gather_plane_simple_table_compute (GridBase *grid,int dimension,int
      for(int n=0;n<e1;n++){
        for(int b=0;b<e2;b++){
 	 int o  = n*stride;
-	 int ocb=1<<grid->CheckerBoardFromOindexTable(o+b);
+	 int ocb=1<<rhs._grid->CheckerBoardFromOindexTable(o+b);
 	 if ( ocb &cbmask ) {
 	   table[bo]=std::pair<int,int>(bo,o+b); bo++;
 	 }
@@ -109,7 +109,8 @@ inline void Gather_plane_simple_table_compute (GridBase *grid,int dimension,int
 }
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice<vobj> &rhs,cobj *buffer,compressor &compress, int off,int so)
+Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,
+			   compressor &compress, int off,int so)
 {
 PARALLEL_FOR_LOOP     
      for(int i=0;i<table.size();i++){
@@ -117,19 +118,31 @@ PARALLEL_FOR_LOOP
      }
 }
 
-struct StencilEntry { 
+template<class vobj,class cobj,class compressor> void 
+Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
+			     double &t_table ,double & t_data )
+{
+  std::vector<std::pair<int,int> > table;
+  Gather_plane_simple_table_compute (rhs, buffer,dimension,plane,cbmask,compress,off,table);
+  int so  = plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
+  Gather_plane_simple_table         (table,rhs,buffer,compress,off,so);
+}
+
+
+
+
+   struct StencilEntry { 
      uint64_t _offset;
      uint64_t _byte_offset;
      uint16_t _is_local;
      uint16_t _permute;
      uint32_t _around_the_world; //256 bits, 32 bytes, 1/2 cacheline
-};
+   };
 
-template<class vobj,class cobj>
-class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal fill in.
+   template<class vobj,class cobj>
+   class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal fill in.
    public:
 
-  typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
        typedef uint32_t StencilInteger;
        typedef typename cobj::vector_type vector_type;
        typedef typename cobj::scalar_type scalar_type;
@@ -145,6 +158,7 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 	 Integer to_rank;
 	 Integer from_rank;
 	 Integer bytes;
+	 volatile Integer done;
        };
 
        std::vector<Packet> Packets;
@@ -152,53 +166,81 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
        int face_table_computed;
        std::vector<std::vector<std::pair<int,int> > > face_table ;
        
+#define SEND_IMMEDIATE
+#define SERIAL_SENDS
+
        void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){
+ #ifdef SEND_IMMEDIATE
+	 commtime-=usecond();
+	 _grid->SendToRecvFrom(xmit,to,rcv,from,bytes);
+	 commtime+=usecond();
+ #endif
 	 Packet p;
 	 p.send_buf = xmit;
 	 p.recv_buf = rcv;
 	 p.to_rank  = to;
 	 p.from_rank= from;
 	 p.bytes    = bytes;
+	 p.done     = 0;
 	 comms_bytes+=2.0*bytes;
 	 Packets.push_back(p);
+
        }
 
-  void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
-  {
-    reqs.resize(Packets.size());
+ #ifdef SERIAL_SENDS
+       void Communicate(void ) { 
 	 commtime-=usecond();
 	 for(int i=0;i<Packets.size();i++){
-	_grid->StencilSendToRecvFromBegin(reqs[i],
+ #ifndef SEND_IMMEDIATE
+	   _grid->SendToRecvFrom(
 				 Packets[i].send_buf,
 				 Packets[i].to_rank,
 				 Packets[i].recv_buf,
 				 Packets[i].from_rank,
 				 Packets[i].bytes);
-	/*
-      }else{
-	_grid->SendToRecvFromBegin(reqs[i],
-				   Packets[i].send_buf,
-				   Packets[i].to_rank,
-				   Packets[i].recv_buf,
-				   Packets[i].from_rank,
-				   Packets[i].bytes);
-      }
-	*/
+ #endif
+	   Packets[i].done = 1;
 	 }
 	 commtime+=usecond();
        }
-  void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs)
-  {
+ #else
+       void Communicate(void ) { 
+	 typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
+	 std::vector<std::vector<CommsRequest_t> > reqs(Packets.size());
 	 commtime-=usecond();
-
-    for(int i=0;i<Packets.size();i++){
-      //      if( ShmDirectCopy ) 
-	_grid->StencilSendToRecvFromComplete(reqs[i]);
-	//      else 
-	//	_grid->SendToRecvFromComplete(reqs[i]);
+	 const int concurrency=2;
+	 for(int i=0;i<Packets.size();i+=concurrency){
+	   for(int ii=0;ii<concurrency;ii++){
+	     int j = i+ii;
+	     if ( j<Packets.size() ) {
+ #ifndef SEND_IMMEDIATE
+	       _grid->SendToRecvFromBegin(reqs[j],
+					  Packets[j].send_buf,
+					  Packets[j].to_rank,
+					  Packets[j].recv_buf,
+					  Packets[j].from_rank,
+					  Packets[j].bytes);
+ #endif
+	     }
+	   }
+	   for(int ii=0;ii<concurrency;ii++){
+	     int j = i+ii;
+	     if ( j<Packets.size() ) {
+ #ifndef SEND_IMMEDIATE
+	       _grid->SendToRecvFromComplete(reqs[i]);
+ #endif
+	     }
+	   }
+	   for(int ii=0;ii<concurrency;ii++){
+	     int j = i+ii;
+	     if ( j<Packets.size() ) {
+	       Packets[j].done = 1;
+	     }
+	   }
 	 }
 	 commtime+=usecond();
        }
+ #endif
 
        ///////////////////////////////////////////
        // Simd merge queue for asynch comms
@@ -218,19 +260,36 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 	 m.rpointers= rpointers;
 	 m.buffer_size = buffer_size;
 	 m.packet_id   = packet_id;
+ #ifdef SEND_IMMEDIATE
+	 mergetime-=usecond();
+ PARALLEL_FOR_LOOP
+	 for(int o=0;o<m.buffer_size;o++){
+	   merge1(m.mpointer[o],m.rpointers,o);
+	 }
+	 mergetime+=usecond();
+ #else
 	 Mergers.push_back(m);
+ #endif
+
        }
 
        void CommsMerge(void ) { 
-
+	 //PARALLEL_NESTED_LOOP2 
 	 for(int i=0;i<Mergers.size();i++){	
 
+	 spintime-=usecond();
+	 int packet_id = Mergers[i].packet_id;
+	 while(! Packets[packet_id].done ); // spin for completion
+	 spintime+=usecond();
+
+ #ifndef SEND_IMMEDIATE
 	 mergetime-=usecond();
-PARALLEL_FOR_LOOP
+ PARALLEL_FOR_LOOP
 	   for(int o=0;o<Mergers[i].buffer_size;o++){
 	     merge1(Mergers[i].mpointer[o],Mergers[i].rpointers,o);
 	   }
 	 mergetime+=usecond();
+ #endif
 
 	 }
        }
@@ -253,19 +312,24 @@ PARALLEL_FOR_LOOP
        // Flat vector, change layout for cache friendly.
        Vector<StencilEntry>  _entries;
 
+       inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; }
+
        void PrecomputeByteOffsets(void){
 	 for(int i=0;i<_entries.size();i++){
 	   if( _entries[i]._is_local ) {
 	     _entries[i]._byte_offset = _entries[i]._offset*sizeof(vobj);
 	   } else { 
+	     // PrecomputeByteOffsets [5] 16384/32768 140735768678528 140735781261056 2581581952
 	     _entries[i]._byte_offset = _entries[i]._offset*sizeof(cobj);
 	   }
 	 }
        };
 
-  inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; }
+       inline uint64_t Touch(int ent) {
+	 //	 _mm_prefetch((char *)&_entries[ent],_MM_HINT_T0);
+       }
        inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
-    uint64_t cbase = (uint64_t)&u_recv_buf_p[0];
+	 uint64_t cbase = (uint64_t)&comm_buf[0];
 	 local = _entries[ent]._is_local;
 	 perm  = _entries[ent]._permute;
 	 if (perm)  ptype = _permute_type[point]; 
@@ -276,33 +340,20 @@ PARALLEL_FOR_LOOP
 	 }
        }
        inline uint64_t GetPFInfo(int ent,uint64_t base) {
-    uint64_t cbase = (uint64_t)&u_recv_buf_p[0];
+	 uint64_t cbase = (uint64_t)&comm_buf[0];
 	 int local = _entries[ent]._is_local;
 	 if (local) return  base + _entries[ent]._byte_offset;
 	 else       return cbase + _entries[ent]._byte_offset;
        }
 
-  ///////////////////////////////////////////////////////////
-  // Unified Comms buffers for all directions
-  ///////////////////////////////////////////////////////////
-  // Vectors that live on the symmetric heap in case of SHMEM
-  //  std::vector<commVector<scalar_object> > u_simd_send_buf_hide;
-  //  std::vector<commVector<scalar_object> > u_simd_recv_buf_hide;
-  //  commVector<cobj>          u_send_buf_hide;
-  //  commVector<cobj>          u_recv_buf_hide;
-
-  // These are used; either SHM objects or refs to the above symmetric heap vectors
-  // depending on comms target
-  cobj* u_recv_buf_p;
-  cobj* u_send_buf_p;
-  std::vector<scalar_object *> u_simd_send_buf;
-  std::vector<scalar_object *> u_simd_recv_buf;
-
+       // Comms buffers
+       std::vector<Vector<scalar_object> > u_simd_send_buf;
+       std::vector<Vector<scalar_object> > u_simd_recv_buf;
+       Vector<cobj>          u_send_buf;
+       Vector<cobj>          comm_buf;
        int u_comm_offset;
        int _unified_buffer_size;
 
-  cobj *CommBuf(void) { return u_recv_buf_p; }
-
        /////////////////////////////////////////
        // Timing info; ugly; possibly temporary
        /////////////////////////////////////////
@@ -384,6 +435,7 @@ PARALLEL_FOR_LOOP
 	 int i = ii; // reverse direction to get SIMD comms done first
 	 int point = i;
 
+
 	 int dimension    = directions[i];
 	 int displacement = distances[i];
 	 int shift = displacement;
@@ -430,25 +482,18 @@ PARALLEL_FOR_LOOP
 	   }
 	 }
        }
-
-    /////////////////////////////////////////////////////////////////////////////////
-    // Try to allocate for receiving in a shared memory region, fall back to buffer
-    /////////////////////////////////////////////////////////////////////////////////
-    const int Nsimd = grid->Nsimd();
-
-    _grid->ShmBufferFreeAll();
-
-    u_simd_send_buf.resize(Nsimd);
-    u_simd_recv_buf.resize(Nsimd);
-
-    u_send_buf_p=(cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj));
-    u_recv_buf_p=(cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj));
-    for(int l=0;l<Nsimd;l++){
-      u_simd_recv_buf[l] = (scalar_object *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(scalar_object));
-      u_simd_send_buf[l] = (scalar_object *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(scalar_object));
-    }
+       u_send_buf.resize(_unified_buffer_size);
+       comm_buf.resize(_unified_buffer_size);
 
        PrecomputeByteOffsets();
+
+       const int Nsimd = grid->Nsimd();
+       u_simd_send_buf.resize(Nsimd);
+       u_simd_recv_buf.resize(Nsimd);
+       for(int l=0;l<Nsimd;l++){
+	 u_simd_send_buf[l].resize(_unified_buffer_size);
+	 u_simd_recv_buf[l].resize(_unified_buffer_size);
+       }
      }
 
      void Local     (int point, int dimension,int shiftpm,int cbmask)
@@ -672,22 +717,38 @@ PARALLEL_FOR_LOOP
        }
      }
 
-  template<class compressor> void HaloExchange(const Lattice<vobj> &source,compressor &compress) 
+
+
+       template<class compressor>
+       void HaloExchange(const Lattice<vobj> &source,compressor &compress) 
        {
-    std::vector<std::vector<CommsRequest_t> > reqs;
 	 calls++;
 	 Mergers.resize(0);
          Packets.resize(0);
-    _grid->StencilBarrier();
          HaloGather(source,compress);
-    this->CommunicateBegin(reqs);
-    _grid->StencilBarrier();
-    this->CommunicateComplete(reqs);
-    _grid->StencilBarrier();
+	 this->Communicate();
 	 CommsMerge(); // spins
        }
-  
-  template<class compressor> void HaloGatherDir(const Lattice<vobj> &source,compressor &compress,int point,int & face_idx)
+#if 0
+       // Overlapping comms and compute typically slows down compute and  is useless
+       // unless memory bandwidth greatly exceeds network
+       template<class compressor>
+       std::thread HaloExchangeBegin(const Lattice<vobj> &source,compressor &compress) {
+	 Mergers.resize(0); 
+	 Packets.resize(0);
+	 HaloGather(source,compress);
+	 return std::thread([&] { this->Communicate(); });
+       }
+       void HaloExchangeComplete(std::thread &thr) 
+       {
+	 CommsMerge(); // spins
+	 jointime-=usecond();
+	 thr.join();
+	 jointime+=usecond();
+       }
+#endif
+       template<class compressor>
+       void HaloGatherDir(const Lattice<vobj> &source,compressor &compress,int point,int & face_idx)
        {
 	   int dimension    = _directions[point];
 	   int displacement = _distances[point];
@@ -745,6 +806,7 @@ PARALLEL_FOR_LOOP
 	 assert(source._grid==_grid);
 	 halogtime-=usecond();
 
+	 assert (comm_buf.size() == _unified_buffer_size );
 	 u_comm_offset=0;
 
 	 // Gather all comms buffers
@@ -801,48 +863,37 @@ PARALLEL_FOR_LOOP
 	       if ( !face_table_computed ) {
 		 t_table-=usecond();
 		 face_table.resize(face_idx+1);
-	  Gather_plane_simple_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,
-					     face_table[face_idx]);
+		 Gather_plane_simple_table_compute (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,face_table[face_idx]);
 		 t_table+=usecond();
 	       }
+	       t_data-=usecond();
+	       Gather_plane_simple_table         (face_table[face_idx],rhs,u_send_buf,compress,u_comm_offset,so);
+	       face_idx++;
+	       t_data+=usecond();
+	       gathertime+=usecond();
 	       
+	       //	       Gather_plane_simple_stencil (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,t_table,t_data);
 
 	       int rank           = _grid->_processor;
 	       int recv_from_rank;
 	       int xmit_to_rank;
 	       _grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
-	
 	       assert (xmit_to_rank   != _grid->ThisRank());
 	       assert (recv_from_rank != _grid->ThisRank());
 
-	/////////////////////////////////////////////////////////
-	// try the direct copy if possible
-	/////////////////////////////////////////////////////////
-
-
-	cobj *send_buf = (cobj *)_grid->ShmBufferTranslate(xmit_to_rank,u_recv_buf_p);
-	if ( send_buf==NULL ) { 
-	  send_buf = u_send_buf_p;
-	}
-	//	std::cout << " send_bufs  "<<std::hex<< send_buf <<" ubp "<<u_send_buf_p <<std::dec<<std::endl;
-	t_data-=usecond();
-	assert(u_send_buf_p!=NULL);
-	assert(send_buf!=NULL);
-	Gather_plane_simple_table         (face_table[face_idx],rhs,send_buf,compress,u_comm_offset,so);  face_idx++;
-	t_data+=usecond();
-	
-	AddPacket((void *)&send_buf[u_comm_offset],
-		  (void *)&u_recv_buf_p[u_comm_offset],
+	       //      FIXME Implement asynchronous send & also avoid buffer copy
+	       AddPacket((void *)&u_send_buf[u_comm_offset],
+			 (void *)  &comm_buf[u_comm_offset],
 			 xmit_to_rank,
 			 recv_from_rank,
 			 bytes);
 
-	gathertime+=usecond();
 	       u_comm_offset+=words;
 	     }
 	   }
 	 }
 
+
        template<class compressor>
 	 void  GatherSimd(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor &compress,int & face_idx)
 	 {
@@ -923,6 +974,10 @@ PARALLEL_FOR_LOOP
 		 auto rp = &u_simd_recv_buf[i       ][u_comm_offset];
 		 auto sp = &u_simd_send_buf[nbr_lane][u_comm_offset];
 
+		 void *vrp = (void *)rp;
+		 void *vsp = (void *)sp;
+
+
 		 if(nbr_proc){
 
 		   int recv_from_rank;
@@ -930,17 +985,9 @@ PARALLEL_FOR_LOOP
 
 		   _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
 
-	    scalar_object *shm = (scalar_object *) _grid->ShmBufferTranslate(recv_from_rank,sp);
-	    //	    if ((ShmDirectCopy==0)||(shm==NULL)) { 
-	    if (shm==NULL) { 
-	      shm = rp;
-	    } 
+		   AddPacket( vsp,vrp,xmit_to_rank,recv_from_rank,bytes);
 
-	    // if Direct, StencilSendToRecvFrom will suppress copy to a peer on node
-	    // assuming above pointer flip
-	    AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
-	    
-	    rpointers[i] = shm;
+		   rpointers[i] = rp;
 
 		 } else { 
 
@@ -949,13 +996,13 @@ PARALLEL_FOR_LOOP
 		 }
 	       }
 
-	AddMerge(&u_recv_buf_p[u_comm_offset],rpointers,buffer_size,Packets.size()-1);
+	       AddMerge(&comm_buf[u_comm_offset],rpointers,buffer_size,Packets.size()-1);
 
 	       u_comm_offset     +=buffer_size;
 	     }
 	   }
 	 }
 
-};
-}
+   };
+ }
 #endif

lib/Threads.h

@@ -38,21 +38,14 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_OMP
 #include <omp.h>
 #ifdef GRID_NUMA
-  #define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(static)")
-  #define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(static)")
-  #else
-  #define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(runtime)")
-  #define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(runtime)")
+#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(static)")
+#else
+#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(runtime)")
 #endif
 #define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)")
-#define PARALLEL_REGION          _Pragma("omp parallel")
-#define PARALLEL_FOR_LOOP_STATIC _Pragma("omp parallel for schedule(static)")
 #else
 #define PARALLEL_FOR_LOOP 
-#define PARALLEL_FOR_LOOP_INTERN
 #define PARALLEL_NESTED_LOOP2
-#define PARALLEL_REGION
-#define PARALLEL_FOR_LOOP_STATIC
 #endif
 
 namespace Grid {
@@ -134,22 +127,6 @@ class GridThread {
     ThreadBarrier();
   };
 
-  static void bcopy(const void *src, void *dst, size_t len) {
-#ifdef GRID_OMP
-#pragma omp parallel 
-    {
-      const char *c_src =(char *) src;
-      char *c_dest=(char *) dst;
-      int me,mywork,myoff;
-      GridThread::GetWorkBarrier(len,me, mywork,myoff);
-      bcopy(&c_src[myoff],&c_dest[myoff],mywork);
-    }
-#else 
-    bcopy(src,dst,len);
-#endif
-  }
-
-
 };
 
 }

lib/algorithms/CoarsenedMatrix.h

@@ -282,7 +282,7 @@ PARALLEL_FOR_LOOP
 	  } else if(SE->_is_local) { 
 	    nbr = in._odata[SE->_offset];
 	  } else {
-	    nbr = Stencil.CommBuf()[SE->_offset];
+	    nbr = Stencil.comm_buf[SE->_offset];
 	  }
 	  res = res + A[point]._odata[ss]*nbr;
 	}

lib/algorithms/iterative/ConjugateGradient.h

@@ -9,7 +9,6 @@ Copyright (C) 2015
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -34,21 +33,6 @@ directory
 
 namespace Grid {
 
-struct CG_state {
-  bool do_repro;
-  std::vector<RealD> residuals;
-
-  CG_state() {reset();}
-
-  void reset(){
-    do_repro = false;
-    residuals.clear();
-  }
-};
-
-
-enum CGexec_mode{ Default, ReproducibilityTest };
-
 /////////////////////////////////////////////////////////////
 // Base classes for iterative processes based on operators
 // single input vec, single output vec.
@@ -61,30 +45,10 @@ class ConjugateGradient : public OperatorFunction<Field> {
                            // Defaults true.
   RealD Tolerance;
   Integer MaxIterations;
-
-  // Reproducibility controls
-  bool ReproTest;
-  CG_state CGState; //to check reproducibility by repeating the CG
-  ReproducibilityState<typename Field::vector_object> ReprTest; // for the inner proucts
-
-  // Constructor
-  ConjugateGradient(RealD tol, Integer maxit, CGexec_mode Mode = Default)
-    : Tolerance(tol),MaxIterations(maxit){
-    switch(Mode)
-    {
-      case Default  : 
-      ErrorOnNoConverge = true;
-      ReproTest = false;
-      case ReproducibilityTest :
-      ErrorOnNoConverge = false;
-      ReproTest = true; 
-    }
-  };
-
-  void set_reproducibility_interval(unsigned int interval){
-    ReprTest.interval = interval;
-  }
-
+  ConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
+      : Tolerance(tol),
+        MaxIterations(maxit),
+        ErrorOnNoConverge(err_on_no_conv){};
 
   void operator()(LinearOperatorBase<Field> &Linop, const Field &src,
                   Field &psi) {
@@ -96,37 +60,34 @@ class ConjugateGradient : public OperatorFunction<Field> {
     Field p(src);
     Field mmp(src);
     Field r(src);
-    Field psi_start(psi);// save for the repro test
-
-    if (CGState.do_repro && ReproTest)
-        std::cout << GridLogMessage << "Starting reproducibility test, full check every "
-                  << ReprTest.interval << " calls" << std::endl;
-
-    if(!ReprTest.do_check)
-        ReprTest.reset();
-    ReprTest.enable_reprocheck=ReproTest;
-
-
 
     // Initial residual computation & set up
-    RealD guess = norm2(psi, ReprTest);
+    RealD guess = norm2(psi);
     assert(std::isnan(guess) == 0);
 
-    Linop.HermOpAndNorm(psi, mmp, d, b);// eventually split this for the norm check
+    
+    Linop.HermOpAndNorm(psi, mmp, d, b);
+    
 
     r = src - mmp;
     p = r;
 
-    a = norm2(p, ReprTest);
+    a = norm2(p);
     cp = a;
-    ssq = norm2(src, ReprTest);
+    ssq = norm2(src);
 
-    std::cout << GridLogIterative << "ConjugateGradient: guess " << guess << std::endl;
-    std::cout << GridLogIterative << "ConjugateGradient:   src " << ssq << std::endl;
-    std::cout << GridLogIterative << "ConjugateGradient:    mp " << d << std::endl;
-    std::cout << GridLogIterative << "ConjugateGradient:   mmp " << b << std::endl;
-    std::cout << GridLogIterative << "ConjugateGradient:  cp,r " << cp << std::endl;
-    std::cout << GridLogIterative << "ConjugateGradient:     p " << a << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient: guess " << guess << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:   src " << ssq << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:    mp " << d << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:   mmp " << b << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:  cp,r " << cp << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:     p " << a << std::endl;
 
     RealD rsq = Tolerance * Tolerance * ssq;
 
@@ -146,10 +107,10 @@ class ConjugateGradient : public OperatorFunction<Field> {
     SolverTimer.Start();
     int k;
     for (k = 1; k <= MaxIterations; k++) {
-      c = cp;// old residual
+      c = cp;
 
       MatrixTimer.Start();
-      Linop.HermOpAndNorm(p, mmp, d, qq);// mmp = Ap, d=pAp
+      Linop.HermOpAndNorm(p, mmp, d, qq);
       MatrixTimer.Stop();
 
       LinalgTimer.Start();
@@ -157,31 +118,14 @@ class ConjugateGradient : public OperatorFunction<Field> {
       //  ComplexD dck  = innerProduct(p,mmp);
 
       a = c / d;
-      b_pred = a * (a * qq - d) / c;// a check
+      b_pred = a * (a * qq - d) / c;
 
-
-      axpy(r, -a, mmp, r);// new residual r = r_old - a * Ap
-      cp = norm2(r, ReprTest); // bookkeeping this norm
-      if (ReproTest && !CGState.do_repro) {
-        CGState.residuals.push_back(cp);  // save residuals state
-                std::cout << GridLogIterative << "ReproTest: Saving state" << std::endl;
-        }
-      if (ReproTest && CGState.do_repro){
-        // check that the residual agrees with the previous run
-        std::cout << GridLogIterative << "ReproTest: Checking state k=" << k << std::endl;
-        if (cp != CGState.residuals[k-1]){
-                std::cout << GridLogMessage << "Failing reproducibility test";
-                std::cout << GridLogMessage << " at k=" << k << std::endl;
-                std::cout << GridLogMessage << "saved residual = " << CGState.residuals[k-1] 
-                        << " cp = " << cp << std::endl;
-                exit(1);  // exit after the first failure
-        }
-      }
+      cp = axpy_norm(r, -a, mmp, r);
       b = cp / c;
 
       // Fuse these loops ; should be really easy
-      psi = a * p + psi; // update solution
-      p = p * b + r;  // update search direction
+      psi = a * p + psi;
+      p = p * b + r;
 
       LinalgTimer.Stop();
       std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
@@ -210,24 +154,8 @@ class ConjugateGradient : public OperatorFunction<Field> {
                   << LinalgTimer.Elapsed();
         std::cout << std::endl;
 
-        if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
+        if (ErrorOnNoConverge) assert(true_residual / Tolerance < 1000.0);
 
-        if (! (CGState.do_repro && ReproTest)){
-                CGState.do_repro = true;
-                ReprTest.do_check = true;
-                ReprTest.reset_counter();
-                this->operator()(Linop, src, psi_start);// run the repro test
-                if (ReprTest.success)
-                	std::cout << GridLogMessage << "Reproducibility test passed" << std::endl;
-                else{
-                	std::cout << GridLogMessage << "Reproducibility test failed" << std::endl;
-                	exit(1);
-                }
-        }
-
-        // Clear state
-        CGState.reset();
-        ReprTest.reset();
         return;
       }
     }

lib/algorithms/iterative/ImplicitlyRestartedLanczos.h

@@ -31,11 +31,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #include <string.h> //memset
 #ifdef USE_LAPACK
-void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
-                   double *vl, double *vu, int *il, int *iu, double *abstol,
-                   int *m, double *w, double *z, int *ldz, int *isuppz,
-                   double *work, int *lwork, int *iwork, int *liwork,
-                   int *info);
+#include <lapacke.h>
 #endif
 #include "DenseMatrix.h"
 #include "EigenSort.h"
@@ -59,7 +55,7 @@ public:
 
     int Nstop;   // Number of evecs checked for convergence
     int Nk;      // Number of converged sought
-    int Np;      // Np -- Number of spare vecs in krylov space
+    int Np;      // Np -- Number of spare vecs in kryloc space
     int Nm;      // Nm -- total number of vectors
 
     RealD eresid;
@@ -1080,10 +1076,10 @@ say con = 2
 **/
 
 template<class T>
-static void Lock(DenseMatrix<T> &H, 	// Hess mtx	
-		 DenseMatrix<T> &Q, 	// Lock Transform
-		 T val, 		// value to be locked
-		 int con, 	// number already locked
+static void Lock(DenseMatrix<T> &H, 	///Hess mtx	
+		 DenseMatrix<T> &Q, 	///Lock Transform
+		 T val, 		///value to be locked
+		 int con, 	///number already locked
 		 RealD small,
 		 int dfg,
 		 bool herm)

lib/cartesian/Cartesian_base.h

@@ -77,7 +77,7 @@ public:
     // GridCartesian / GridRedBlackCartesian
     ////////////////////////////////////////////////////////////////
     virtual int CheckerBoarded(int dim)=0;
-    virtual int CheckerBoard(std::vector<int> &site)=0;
+    virtual int CheckerBoard(std::vector<int> site)=0;
     virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0;
     virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0;
     virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int cb)=0;

lib/cartesian/Cartesian_full.h

@@ -49,7 +49,7 @@ public:
     virtual int CheckerBoarded(int dim){
       return 0;
     }
-    virtual int CheckerBoard(std::vector<int> &site){
+    virtual int CheckerBoard(std::vector<int> site){
         return 0;
     }
     virtual int CheckerBoardDestination(int cb,int shift,int dim){

lib/cartesian/Cartesian_red_black.h

@@ -49,7 +49,7 @@ public:
       if( dim==_checker_dim) return 1;
       else return 0;
     }
-    virtual int CheckerBoard(std::vector<int> &site){
+    virtual int CheckerBoard(std::vector<int> site){
       int linear=0;
       assert(site.size()==_ndimension);
       for(int d=0;d<_ndimension;d++){ 

lib/communicator/Communicator_base.cc

@@ -1,125 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/communicator/Communicator_none.cc
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#include "Grid.h"
-namespace Grid {
-
-///////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////
-void *              CartesianCommunicator::ShmCommBuf;
-uint64_t            CartesianCommunicator::MAX_MPI_SHM_BYTES   = 128*1024*1024; 
-
-/////////////////////////////////
-// Alloc, free shmem region
-/////////////////////////////////
-void *CartesianCommunicator::ShmBufferMalloc(size_t bytes){
-  //  bytes = (bytes+sizeof(vRealD))&(~(sizeof(vRealD)-1));// align up bytes
-  void *ptr = (void *)heap_top;
-  heap_top  += bytes;
-  heap_bytes+= bytes;
-  if (heap_bytes >= MAX_MPI_SHM_BYTES) {
-    std::cout<< " ShmBufferMalloc exceeded shared heap size -- try increasing with --shm <MB> flag" <<std::endl;
-    std::cout<< " Parameter specified in units of MB (megabytes) " <<std::endl;
-    std::cout<< " Current value is " << (MAX_MPI_SHM_BYTES/(1024*1024)) <<std::endl;
-    assert(heap_bytes<MAX_MPI_SHM_BYTES);
-  }
-  return ptr;
-}
-void CartesianCommunicator::ShmBufferFreeAll(void) { 
-  heap_top  =(size_t)ShmBufferSelf();
-  heap_bytes=0;
-}
-
-/////////////////////////////////
-// Grid information queries
-/////////////////////////////////
-int                      CartesianCommunicator::IsBoss(void)            { return _processor==0; };
-int                      CartesianCommunicator::BossRank(void)          { return 0; };
-int                      CartesianCommunicator::ThisRank(void)          { return _processor; };
-const std::vector<int> & CartesianCommunicator::ThisProcessorCoor(void) { return _processor_coor; };
-const std::vector<int> & CartesianCommunicator::ProcessorGrid(void)     { return _processors; };
-int                      CartesianCommunicator::ProcessorCount(void)    { return _Nprocessors; };
-
-
-////////////////////////////////////////////////////////////////////////////////
-// very VERY rarely (Log, serial RNG) we need world without a grid
-////////////////////////////////////////////////////////////////////////////////
-
-void CartesianCommunicator::GlobalSum(ComplexF &c)
-{
-  GlobalSumVector((float *)&c,2);
-}
-void CartesianCommunicator::GlobalSumVector(ComplexF *c,int N)
-{
-  GlobalSumVector((float *)c,2*N);
-}
-void CartesianCommunicator::GlobalSum(ComplexD &c)
-{
-  GlobalSumVector((double *)&c,2);
-}
-void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
-{
-  GlobalSumVector((double *)c,2*N);
-}
-
-#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPI3L)
-
-void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-                                                       void *xmit,
-                                                       int xmit_to_rank,
-                                                       void *recv,
-                                                       int recv_from_rank,
-                                                       int bytes)
-{
-  SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
-}
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall)
-{
-  SendToRecvFromComplete(waitall);
-}
-void CartesianCommunicator::StencilBarrier(void){};
-
-commVector<uint8_t> CartesianCommunicator::ShmBufStorageVector;
-
-void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; }
-
-void *CartesianCommunicator::ShmBuffer(int rank) {
-  return NULL;
-}
-void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { 
-  return NULL;
-}
-void CartesianCommunicator::ShmInitGeneric(void){
-  ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
-  ShmCommBuf=(void *)&ShmBufStorageVector[0];
-}
-
-#endif
-  
-}
-

lib/communicator/Communicator_base.h

@@ -1,4 +1,3 @@
-
     /*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
@@ -35,142 +34,77 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_COMMS_MPI
 #include <mpi.h>
 #endif
-#ifdef GRID_COMMS_MPI3
-#include <mpi.h>
-#endif
-#ifdef GRID_COMMS_MPI3L
-#include <mpi.h>
-#endif
 #ifdef GRID_COMMS_SHMEM
 #include <mpp/shmem.h>
 #endif
-
 namespace Grid {
-
 class CartesianCommunicator {
   public:    
 
-  // 65536 ranks per node adequate for now
-  // 128MB shared memory for comms enought for 48^4 local vol comms
-  // Give external control (command line override?) of this
-
-  static const int      MAXLOG2RANKSPERNODE = 16;            
-  static uint64_t MAX_MPI_SHM_BYTES;
-
   // Communicator should know nothing of the physics grid, only processor grid.
+
     int              _Nprocessors;     // How many in all
     std::vector<int> _processors;      // Which dimensions get relayed out over processors lanes.
     int              _processor;       // linear processor rank
     std::vector<int> _processor_coor;  // linear processor coordinate
     unsigned long _ndimension;
 
-#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPI3L)
-  static MPI_Comm communicator_world;
+#ifdef GRID_COMMS_MPI
     MPI_Comm communicator;
     typedef MPI_Request CommsRequest_t;
-  static char name[MPI_MAX_PROCESSOR_NAME]; // processing node physical name
-  static int length; 
 #else 
     typedef int CommsRequest_t;
 #endif
 
-  ////////////////////////////////////////////////////////////////////
-  // Helper functionality for SHM Windows common to all other impls
-  ////////////////////////////////////////////////////////////////////
-  // Longer term; drop this in favour of a master / slave model with 
-  // cartesian communicator on a subset of ranks, slave ranks controlled
-  // by group leader with data xfer via shared memory
-  ////////////////////////////////////////////////////////////////////
-#ifdef GRID_COMMS_MPI3
-
-  static int ShmRank;
-  static int ShmSize;
-  static int GroupRank;
-  static int GroupSize;
-  static int WorldRank;
-  static int WorldSize;
-
-  std::vector<int>  WorldDims;
-  std::vector<int>  GroupDims;
-  std::vector<int>  ShmDims;
-  
-  std::vector<int> GroupCoor;
-  std::vector<int> ShmCoor;
-  std::vector<int> WorldCoor;
-
-  static std::vector<int> GroupRanks; 
-  static std::vector<int> MyGroup;
-  static int ShmSetup;
-  static MPI_Win ShmWindow; 
-  static MPI_Comm ShmComm;
-  
-  std::vector<int>  LexicographicToWorldRank;
-  
-  static std::vector<void *> ShmCommBufs;
-
-#else 
-  static void ShmInitGeneric(void);
-  static commVector<uint8_t> ShmBufStorageVector;
-#endif 
-
-  /////////////////////////////////
-  // Grid information and queries
-  // Implemented in Communicator_base.C
-  /////////////////////////////////
-  static void * ShmCommBuf;
-  size_t heap_top;
-  size_t heap_bytes;
-
-  void *ShmBufferSelf(void);
-  void *ShmBuffer(int rank);
-  void *ShmBufferTranslate(int rank,void * local_p);
-  void *ShmBufferMalloc(size_t bytes);
-  void ShmBufferFreeAll(void) ;
-  
-  ////////////////////////////////////////////////
-  // Must call in Grid startup
-  ////////////////////////////////////////////////
     static void Init(int *argc, char ***argv);
 
-  ////////////////////////////////////////////////
-  // Constructor of any given grid
-  ////////////////////////////////////////////////
+    // Constructor
     CartesianCommunicator(const std::vector<int> &pdimensions_in);
 
-  ////////////////////////////////////////////////////////////////////////////////////////
-  // Wraps MPI_Cart routines, or implements equivalent on other impls
-  ////////////////////////////////////////////////////////////////////////////////////////
+    // Wraps MPI_Cart routines
     void ShiftedRanks(int dim,int shift,int & source, int & dest);
     int  RankFromProcessorCoor(std::vector<int> &coor);
     void ProcessorCoorFromRank(int rank,std::vector<int> &coor);
 
-  int                      IsBoss(void)            ;
-  int                      BossRank(void)          ;
-  int                      ThisRank(void)          ;
-  const std::vector<int> & ThisProcessorCoor(void) ;
-  const std::vector<int> & ProcessorGrid(void)     ;
-  int                      ProcessorCount(void)    ;
-
-  void        		 PrintRankInfo(void)     ;
-  ////////////////////////////////////////////////////////////////////////////////
-  // very VERY rarely (Log, serial RNG) we need world without a grid
-  ////////////////////////////////////////////////////////////////////////////////
-  static int  RankWorld(void) ;
-  static void BroadcastWorld(int root,void* data, int bytes);
+    /////////////////////////////////
+    // Grid information queries
+    /////////////////////////////////
+    int                      IsBoss(void)            { return _processor==0; };
+    int                      BossRank(void)          { return 0; };
+    int                      ThisRank(void)          { return _processor; };
+    const std::vector<int> & ThisProcessorCoor(void) { return _processor_coor; };
+    const std::vector<int> & ProcessorGrid(void)     { return _processors; };
+    int                      ProcessorCount(void)    { return _Nprocessors; };
 
     ////////////////////////////////////////////////////////////
     // Reduction
     ////////////////////////////////////////////////////////////
     void GlobalSum(RealF &);
     void GlobalSumVector(RealF *,int N);
+
     void GlobalSum(RealD &);
     void GlobalSumVector(RealD *,int N);
+
     void GlobalSum(uint32_t &);
     void GlobalSum(uint64_t &);
-  void GlobalSum(ComplexF &c);
-  void GlobalSumVector(ComplexF *c,int N);
-  void GlobalSum(ComplexD &c);
-  void GlobalSumVector(ComplexD *c,int N);
+
+    void GlobalSum(ComplexF &c)
+    {
+      GlobalSumVector((float *)&c,2);
+    }
+    void GlobalSumVector(ComplexF *c,int N)
+    {
+      GlobalSumVector((float *)c,2*N);
+    }
+
+    void GlobalSum(ComplexD &c)
+    {
+      GlobalSumVector((double *)&c,2);
+    }
+    void GlobalSumVector(ComplexD *c,int N)
+    {
+      GlobalSumVector((double *)c,2*N);
+    }
     
     template<class obj> void GlobalSum(obj &o){
       typedef typename obj::scalar_type scalar_type;
@@ -178,7 +112,6 @@ class CartesianCommunicator {
       scalar_type * ptr = (scalar_type *)& o;
       GlobalSumVector(ptr,words);
     }
-  
     ////////////////////////////////////////////////////////////
     // Face exchange, buffer swap in translational invariant way
     ////////////////////////////////////////////////////////////
@@ -200,19 +133,8 @@ class CartesianCommunicator {
 			 void *recv,
 			 int recv_from_rank,
 			 int bytes);
-  
     void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
 
-  void StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-				  void *xmit,
-				  int xmit_to_rank,
-				  void *recv,
-				  int recv_from_rank,
-				  int bytes);
-  
-  void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
-  void StencilBarrier(void);
-
     ////////////////////////////////////////////////////////////
     // Barrier
     ////////////////////////////////////////////////////////////
@@ -222,12 +144,13 @@ class CartesianCommunicator {
     // Broadcast a buffer and composite larger
     ////////////////////////////////////////////////////////////
     void Broadcast(int root,void* data, int bytes);
-  
     template<class obj> void Broadcast(int root,obj &data)
     {
       Broadcast(root,(void *)&data,sizeof(data));
     };
 
+    static void BroadcastWorld(int root,void* data, int bytes);
+
 }; 
 }
 

lib/communicator/Communicator_mpi.cc

@@ -30,27 +30,21 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////////////////////////////////////////
-MPI_Comm CartesianCommunicator::communicator_world;
-char CartesianCommunicator::name[MPI_MAX_PROCESSOR_NAME]; // processing node physical name
-int CartesianCommunicator::length; 
-
-// Should error check all MPI calls.
+  // Should error check all MPI calls.
 void CartesianCommunicator::Init(int *argc, char ***argv) {
   int flag;
   MPI_Initialized(&flag); // needed to coexist with other libs apparently
   if ( !flag ) {
     MPI_Init(argc,argv);
   }
-  MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
-
-  MPI_Get_processor_name(name, &length);
-  ShmInitGeneric();
 }
 
+  int Rank(void) {
+    int pe;
+    MPI_Comm_rank(MPI_COMM_WORLD,&pe);
+    return pe;
+  }
+
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
   _ndimension = processors.size();
@@ -60,7 +54,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   _processors = processors;
   _processor_coor.resize(_ndimension);
   
-  MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator);
+  MPI_Cart_create(MPI_COMM_WORLD, _ndimension,&_processors[0],&periodic[0],1,&communicator);
   MPI_Comm_rank(communicator,&_processor);
   MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
 
@@ -73,6 +67,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   
   assert(Size==_Nprocessors);
 }
+
 void CartesianCommunicator::GlobalSum(uint32_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
   assert(ierr==0);
@@ -173,6 +168,7 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &
   int nreq=list.size();
   std::vector<MPI_Status> status(nreq);
   int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
+
   assert(ierr==0);
 }
 
@@ -191,29 +187,16 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
 		     communicator);
   assert(ierr==0);
 }
-  ///////////////////////////////////////////////////////
-  // Should only be used prior to Grid Init finished.
-  // Check for this?
-  ///////////////////////////////////////////////////////
-int CartesianCommunicator::RankWorld(void){ 
-  int r; 
-  MPI_Comm_rank(communicator_world,&r);
-  return r;
-}
+
 void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
 {
   int ierr= MPI_Bcast(data,
 		      bytes,
 		      MPI_BYTE,
 		      root,
-		      communicator_world);
+		      MPI_COMM_WORLD);
   assert(ierr==0);
 }
 
-void CartesianCommunicator::PrintRankInfo(){
-    std::cout << "Grid: Rank "<< _processor << "  -  Physical node name: " << name << std::endl;
 }
 
-
-}// end of namespace
-

lib/communicator/Communicator_mpi3.cc

@@ -1,585 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/communicator/Communicator_mpi.cc
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#include "Grid.h"
-#include <mpi.h>
-
-namespace Grid {
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////////////////////////////////////////
-int CartesianCommunicator::ShmSetup = 0;
-
-int CartesianCommunicator::ShmRank;
-int CartesianCommunicator::ShmSize;
-int CartesianCommunicator::GroupRank;
-int CartesianCommunicator::GroupSize;
-int CartesianCommunicator::WorldRank;
-int CartesianCommunicator::WorldSize;
-
-MPI_Comm CartesianCommunicator::communicator_world;
-MPI_Comm CartesianCommunicator::ShmComm;
-MPI_Win  CartesianCommunicator::ShmWindow;
-
-std::vector<int> CartesianCommunicator::GroupRanks;  
-std::vector<int> CartesianCommunicator::MyGroup;
-std::vector<void *> CartesianCommunicator::ShmCommBufs;
-
-void *CartesianCommunicator::ShmBufferSelf(void)
-{
-  return ShmCommBufs[ShmRank];
-}
-void *CartesianCommunicator::ShmBuffer(int rank)
-{
-  int gpeer = GroupRanks[rank];
-  if (gpeer == MPI_UNDEFINED){
-    return NULL;
-  } else { 
-    return ShmCommBufs[gpeer];
-  }
-}
-void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p)
-{
-  int gpeer = GroupRanks[rank];
-  if (gpeer == MPI_UNDEFINED){
-    return NULL;
-  } else { 
-    uint64_t offset = (uint64_t)local_p - (uint64_t)ShmCommBufs[ShmRank];
-    uint64_t remote = (uint64_t)ShmCommBufs[gpeer]+offset;
-    return (void *) remote;
-  }
-}
-
-void CartesianCommunicator::Init(int *argc, char ***argv) {
-  int flag;
-  MPI_Initialized(&flag); // needed to coexist with other libs apparently
-  if ( !flag ) {
-    MPI_Init(argc,argv);
-  }
-
-  MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
-  MPI_Comm_rank(communicator_world,&WorldRank);
-  MPI_Comm_size(communicator_world,&WorldSize);
-
-  /////////////////////////////////////////////////////////////////////
-  // Split into groups that can share memory
-  /////////////////////////////////////////////////////////////////////
-  MPI_Comm_split_type(communicator_world, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm);
-  MPI_Comm_rank(ShmComm     ,&ShmRank);
-  MPI_Comm_size(ShmComm     ,&ShmSize);
-  GroupSize = WorldSize/ShmSize;
-
-  /////////////////////////////////////////////////////////////////////
-  // find world ranks in our SHM group (i.e. which ranks are on our node)
-  /////////////////////////////////////////////////////////////////////
-  MPI_Group WorldGroup, ShmGroup;
-  MPI_Comm_group (communicator_world, &WorldGroup); 
-  MPI_Comm_group (ShmComm, &ShmGroup);
-  
-  std::vector<int> world_ranks(WorldSize); 
-  GroupRanks.resize(WorldSize); 
-  for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
-  
-  MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &GroupRanks[0]); 
-
-  ///////////////////////////////////////////////////////////////////
-  // Identify who is in my group and noninate the leader
-  ///////////////////////////////////////////////////////////////////
-  int g=0;
-  MyGroup.resize(ShmSize);
-  for(int rank=0;rank<WorldSize;rank++){
-    if(GroupRanks[rank]!=MPI_UNDEFINED){
-      assert(g<ShmSize);
-      MyGroup[g++] = rank;
-    }
-  }
-  
-  std::sort(MyGroup.begin(),MyGroup.end(),std::less<int>());
-  int myleader = MyGroup[0];
-  
-  std::vector<int> leaders_1hot(WorldSize,0);
-  std::vector<int> leaders_group(GroupSize,0);
-  leaders_1hot [ myleader ] = 1;
-    
-  ///////////////////////////////////////////////////////////////////
-  // global sum leaders over comm world
-  ///////////////////////////////////////////////////////////////////
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,communicator_world);
-  assert(ierr==0);
-  
-  ///////////////////////////////////////////////////////////////////
-  // find the group leaders world rank
-  ///////////////////////////////////////////////////////////////////
-  int group=0;
-  for(int l=0;l<WorldSize;l++){
-    if(leaders_1hot[l]){
-      leaders_group[group++] = l;
-    }
-  }
-  
-  ///////////////////////////////////////////////////////////////////
-  // Identify the rank of the group in which I (and my leader) live
-  ///////////////////////////////////////////////////////////////////
-  GroupRank=-1;
-  for(int g=0;g<GroupSize;g++){
-    if (myleader == leaders_group[g]){
-      GroupRank=g;
-    }
-  }
-  assert(GroupRank!=-1);
-  
-  //////////////////////////////////////////////////////////////////////////////////////////////////////////
-  // allocate the shared window for our group
-  //////////////////////////////////////////////////////////////////////////////////////////////////////////
-  
-  ShmCommBuf = 0;
-  ierr = MPI_Win_allocate_shared(MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,ShmComm,&ShmCommBuf,&ShmWindow);
-  assert(ierr==0);
-  // KNL hack -- force to numa-domain 1 in flat
-#if 0
-  //#include <numaif.h>
-  for(uint64_t page=0;page<MAX_MPI_SHM_BYTES;page+=4096){
-    void *pages = (void *) ( page + ShmCommBuf );
-    int status;
-    int flags=MPOL_MF_MOVE_ALL;
-    int nodes=1; // numa domain == MCDRAM
-    unsigned long count=1;
-    ierr= move_pages(0,count, &pages,&nodes,&status,flags);
-    if (ierr && (page==0)) perror("numa relocate command failed");
-  }
-#endif
-  MPI_Win_lock_all (MPI_MODE_NOCHECK, ShmWindow);
-  
-  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Plan: allocate a fixed SHM region. Scratch that is just used via some scheme during stencil comms, with no allocate free.
-  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-  ShmCommBufs.resize(ShmSize);
-  for(int r=0;r<ShmSize;r++){
-    MPI_Aint sz;
-    int dsp_unit;
-    MPI_Win_shared_query (ShmWindow, r, &sz, &dsp_unit, &ShmCommBufs[r]);
-  }
-
-  //////////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Verbose for now
-  //////////////////////////////////////////////////////////////////////////////////////////////////////////
-  if (WorldRank == 0){
-    std::cout<<GridLogMessage<< "Grid MPI-3 configuration: detected ";
-    std::cout<< WorldSize << " Ranks " ;
-    std::cout<< GroupSize << " Nodes " ;
-    std::cout<<  ShmSize  << " with ranks-per-node "<<std::endl;
-    
-    std::cout<<GridLogMessage     <<"Grid MPI-3 configuration: allocated shared memory region of size ";
-    std::cout<<std::hex << MAX_MPI_SHM_BYTES <<" ShmCommBuf address = "<<ShmCommBuf << std::dec<<std::endl;
-
-    for(int g=0;g<GroupSize;g++){
-      std::cout<<GridLogMessage<<" Node "<<g<<" led by MPI rank "<<leaders_group[g]<<std::endl;
-    }
-
-    std::cout<<GridLogMessage<<" Boss Node Shm Pointers are {";
-    for(int g=0;g<ShmSize;g++){
-      std::cout<<std::hex<<ShmCommBufs[g]<<std::dec;
-      if(g!=ShmSize-1) std::cout<<",";
-      else std::cout<<"}"<<std::endl;
-    }
-
-  }
-  
-  for(int g=0;g<GroupSize;g++){
-    if ( (ShmRank == 0) && (GroupRank==g) )  std::cout<<GridLogMessage<<"["<<g<<"] Node Group "<<g<<" is ranks {";
-    for(int r=0;r<ShmSize;r++){
-      if ( (ShmRank == 0) && (GroupRank==g) ) {
-	std::cout<<MyGroup[r];
-	if(r<ShmSize-1) std::cout<<",";
-	else std::cout<<"}"<<std::endl;
-      }
-      MPI_Barrier(communicator_world);
-    }
-  }
-  
-  assert(ShmSetup==0);  ShmSetup=1;
-}
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////
-// Want to implement some magic ... Group sub-cubes into those on same node
-////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
-{
-  std::vector<int> coor = _processor_coor;
-
-  assert(std::abs(shift) <_processors[dim]);
-
-  coor[dim] = (_processor_coor[dim] + shift + _processors[dim])%_processors[dim];
-  Lexicographic::IndexFromCoor(coor,source,_processors);
-  source = LexicographicToWorldRank[source];
-
-  coor[dim] = (_processor_coor[dim] - shift + _processors[dim])%_processors[dim];
-  Lexicographic::IndexFromCoor(coor,dest,_processors);
-  dest = LexicographicToWorldRank[dest];
-}
-int CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor)
-{
-  int rank;
-  Lexicographic::IndexFromCoor(coor,rank,_processors);
-  rank = LexicographicToWorldRank[rank];
-  return rank;
-}
-void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor)
-{
-  Lexicographic::CoorFromIndex(coor,rank,_processors);
-  rank = LexicographicToWorldRank[rank];
-}
-
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
-{ 
-  int ierr;
-
-  communicator=communicator_world;
-
-  _ndimension = processors.size();
-  
-  ////////////////////////////////////////////////////////////////
-  // Assert power of two shm_size.
-  ////////////////////////////////////////////////////////////////
-  int log2size = -1;
-  for(int i=0;i<=MAXLOG2RANKSPERNODE;i++){  
-    if ( (0x1<<i) == ShmSize ) {
-      log2size = i;
-      break;
-    }
-  }
-  assert(log2size != -1);
-  
-  ////////////////////////////////////////////////////////////////
-  // Identify subblock of ranks on node spreading across dims
-  // in a maximally symmetrical way
-  ////////////////////////////////////////////////////////////////
-  int dim = 0;
-  
-  std::vector<int> WorldDims = processors;
-
-  ShmDims.resize(_ndimension,1);
-  GroupDims.resize(_ndimension);
-    
-  ShmCoor.resize(_ndimension);
-  GroupCoor.resize(_ndimension);
-  WorldCoor.resize(_ndimension);
-
-  for(int l2=0;l2<log2size;l2++){
-    while ( WorldDims[dim] / ShmDims[dim] <= 1 ) dim=(dim+1)%_ndimension;
-    ShmDims[dim]*=2;
-    dim=(dim+1)%_ndimension;
-  }
-
-  ////////////////////////////////////////////////////////////////
-  // Establish torus of processes and nodes with sub-blockings
-  ////////////////////////////////////////////////////////////////
-  for(int d=0;d<_ndimension;d++){
-    GroupDims[d] = WorldDims[d]/ShmDims[d];
-  }
-
-  ////////////////////////////////////////////////////////////////
-  // Check processor counts match
-  ////////////////////////////////////////////////////////////////
-  _Nprocessors=1;
-  _processors = processors;
-  _processor_coor.resize(_ndimension);
-  for(int i=0;i<_ndimension;i++){
-    _Nprocessors*=_processors[i];
-  }
-  assert(WorldSize==_Nprocessors);
-      
-  ////////////////////////////////////////////////////////////////
-  // Establish mapping between lexico physics coord and WorldRank
-  // 
-  ////////////////////////////////////////////////////////////////
-  LexicographicToWorldRank.resize(WorldSize,0);
-  Lexicographic::CoorFromIndex(GroupCoor,GroupRank,GroupDims);
-  Lexicographic::CoorFromIndex(ShmCoor,ShmRank,ShmDims);
-  for(int d=0;d<_ndimension;d++){
-    WorldCoor[d] = GroupCoor[d]*ShmDims[d]+ShmCoor[d];
-  }
-  _processor_coor = WorldCoor;
-
-  int lexico;
-  Lexicographic::IndexFromCoor(WorldCoor,lexico,WorldDims);
-  LexicographicToWorldRank[lexico]=WorldRank;
-  _processor = lexico;
-
-  ///////////////////////////////////////////////////////////////////
-  // global sum Lexico to World mapping
-  ///////////////////////////////////////////////////////////////////
-  ierr=MPI_Allreduce(MPI_IN_PLACE,&LexicographicToWorldRank[0],WorldSize,MPI_INT,MPI_SUM,communicator);
-  assert(ierr==0);
-  
-};
-
-void CartesianCommunicator::GlobalSum(uint32_t &u){
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSum(uint64_t &u){
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSum(float &f){
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSumVector(float *f,int N)
-{
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSum(double &d)
-{
-  int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSumVector(double *d,int N)
-{
-  int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-
-
-// Basic Halo comms primitive
-void CartesianCommunicator::SendToRecvFrom(void *xmit,
-					   int dest,
-					   void *recv,
-					   int from,
-					   int bytes)
-{
-  std::vector<CommsRequest_t> reqs(0);
-  SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes);
-  SendToRecvFromComplete(reqs);
-}
-
-void CartesianCommunicator::SendRecvPacket(void *xmit,
-					   void *recv,
-					   int sender,
-					   int receiver,
-					   int bytes)
-{
-  MPI_Status stat;
-  assert(sender != receiver);
-  int tag = sender;
-  if ( _processor == sender ) {
-    MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator);
-  }
-  if ( _processor == receiver ) { 
-    MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat);
-  }
-}
-
-// Basic Halo comms primitive
-void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						void *xmit,
-						int dest,
-						void *recv,
-						int from,
-						int bytes)
-{
-#if 0
-  this->StencilBarrier();
-
-  MPI_Request xrq;
-  MPI_Request rrq;
-  
-  static int sequence;
-
-  int ierr;
-  int tag;
-  int check;
-
-  assert(dest != _processor);
-  assert(from != _processor);
-  
-  int gdest = GroupRanks[dest];
-  int gfrom = GroupRanks[from];
-  int gme   = GroupRanks[_processor];
-
-  sequence++;
-  
-  char *from_ptr = (char *)ShmCommBufs[ShmRank];
-
-  int small = (bytes<MAX_MPI_SHM_BYTES);
-
-  typedef uint64_t T;
-  int words = bytes/sizeof(T);
-
-  assert(((size_t)bytes &(sizeof(T)-1))==0);
-  assert(gme == ShmRank);
-
-  if ( small && (gdest !=MPI_UNDEFINED) ) {
-
-    char *to_ptr   = (char *)ShmCommBufs[gdest];
-
-    assert(gme != gdest);
-
-    T *ip = (T *)xmit;
-    T *op = (T *)to_ptr;
-PARALLEL_FOR_LOOP 
-    for(int w=0;w<words;w++) {
-      op[w]=ip[w];
-    }
-
-    bcopy(&_processor,&to_ptr[bytes],sizeof(_processor));
-    bcopy(&  sequence,&to_ptr[bytes+4],sizeof(sequence));
-  } else { 
-    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
-    assert(ierr==0);
-    list.push_back(xrq);
-  }
-
-  this->StencilBarrier();
-  
-  if (small && (gfrom !=MPI_UNDEFINED) ) {
-    T *ip = (T *)from_ptr;
-    T *op = (T *)recv;
-PARALLEL_FOR_LOOP 
-    for(int w=0;w<words;w++) {
-      op[w]=ip[w];
-    }
-    bcopy(&from_ptr[bytes]  ,&tag  ,sizeof(tag));
-    bcopy(&from_ptr[bytes+4],&check,sizeof(check));
-    assert(check==sequence);
-    assert(tag==from);
-  } else { 
-    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
-    assert(ierr==0);
-    list.push_back(rrq);
-  }
-
-  this->StencilBarrier();
-
-#else
-  MPI_Request xrq;
-  MPI_Request rrq;
-  int rank = _processor;
-  int ierr;
-  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
-  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
-  
-  assert(ierr==0);
-
-  list.push_back(xrq);
-  list.push_back(rrq);
-#endif
-}
-
-void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						       void *xmit,
-						       int dest,
-						       void *recv,
-						       int from,
-						       int bytes)
-{
-  MPI_Request xrq;
-  MPI_Request rrq;
-
-  int ierr;
-
-  assert(dest != _processor);
-  assert(from != _processor);
-  
-  int gdest = GroupRanks[dest];
-  int gfrom = GroupRanks[from];
-  int gme   = GroupRanks[_processor];
-
-  assert(gme == ShmRank);
-
-  if ( gdest == MPI_UNDEFINED ) {
-    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
-    assert(ierr==0);
-    list.push_back(xrq);
-  }
-  
-  if ( gfrom ==MPI_UNDEFINED) {
-    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
-    assert(ierr==0);
-    list.push_back(rrq);
-  }
-
-}
-
-
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list)
-{
-  SendToRecvFromComplete(list);
-}
-
-void CartesianCommunicator::StencilBarrier(void)
-{
-  MPI_Win_sync (ShmWindow);   
-  MPI_Barrier  (ShmComm);
-  MPI_Win_sync (ShmWindow);   
-}
-
-void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
-{
-  int nreq=list.size();
-  std::vector<MPI_Status> status(nreq);
-  int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
-  assert(ierr==0);
-}
-
-void CartesianCommunicator::Barrier(void)
-{
-  int ierr = MPI_Barrier(communicator);
-  assert(ierr==0);
-}
-
-void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
-{
-  int ierr=MPI_Bcast(data,
-		     bytes,
-		     MPI_BYTE,
-		     root,
-		     communicator);
-  assert(ierr==0);
-}
-
-void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
-{
-  int ierr= MPI_Bcast(data,
-		      bytes,
-		      MPI_BYTE,
-		      root,
-		      communicator_world);
-  assert(ierr==0);
-}
-
-  void CartesianCommunicator::PrintRankInfo(){
-    std::cout << "Grid: Rank "<< _processor << "  -  Physical node name: " << name << std::endl;
-  }
-
-  
-}
-

lib/communicator/Communicator_mpi3_leader.cc

@@ -1,877 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/communicator/Communicator_mpi.cc
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#include "Grid.h"
-#include <mpi.h>
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-/// Workarounds:
-/// i) bloody mac os doesn't implement unnamed semaphores since it is "optional" posix.
-///    darwin dispatch semaphores don't seem to be multiprocess.
-///
-/// ii) openmpi under --mca shmem posix works with two squadrons per node; 
-///     openmpi under default mca settings (I think --mca shmem mmap) on MacOS makes two squadrons map the SAME
-///     memory as each other, despite their living on different communicators. This appears to be a bug in OpenMPI.
-///
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-#include <semaphore.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-
-typedef sem_t *Grid_semaphore;
-
-#define SEM_INIT(S)      S = sem_open(sem_name,0,0600,0); assert ( S != SEM_FAILED );
-#define SEM_INIT_EXCL(S) sem_unlink(sem_name); S = sem_open(sem_name,O_CREAT|O_EXCL,0600,0); assert ( S != SEM_FAILED );
-#define SEM_POST(S) assert ( sem_post(S) == 0 ); 
-#define SEM_WAIT(S) assert ( sem_wait(S) == 0 );
-
-#include <sys/mman.h>
-
-namespace Grid {
-
-enum { COMMAND_ISEND, COMMAND_IRECV, COMMAND_WAITALL };
-
-struct Descriptor {
-  uint64_t buf;
-  size_t bytes;
-  int rank;
-  int tag;
-  int command;
-  MPI_Request request;
-};
-
-const int pool = 48;
-
-class SlaveState {
-public:
-  volatile int head;
-  volatile int start;
-  volatile int tail;
-  volatile Descriptor Descrs[pool];
-};
-
-class Slave {
-public:
-  Grid_semaphore  sem_head;
-  Grid_semaphore  sem_tail;
-  SlaveState *state;
-  MPI_Comm squadron;
-  uint64_t     base;
-  int universe_rank;
-  int vertical_rank;
-  char sem_name [NAME_MAX];
-  ////////////////////////////////////////////////////////////
-  // Descriptor circular pointers
-  ////////////////////////////////////////////////////////////
-  Slave() {};
-
-  void Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank);
-
-  void SemInit(void) {
-    sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank);
-    //    printf("SEM_NAME: %s \n",sem_name);
-    SEM_INIT(sem_head);
-    sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank);
-    //    printf("SEM_NAME: %s \n",sem_name);
-    SEM_INIT(sem_tail);
-  }  
-  void SemInitExcl(void) {
-    sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank);
-    //    printf("SEM_INIT_EXCL: %s \n",sem_name);
-    SEM_INIT_EXCL(sem_head);
-    sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank);
-    //    printf("SEM_INIT_EXCL: %s \n",sem_name);
-    SEM_INIT_EXCL(sem_tail);
-  }  
-  void WakeUpDMA(void) { 
-    SEM_POST(sem_head);
-  };
-  void WakeUpCompute(void) { 
-    SEM_POST(sem_tail);
-  };
-  void WaitForCommand(void) { 
-    SEM_WAIT(sem_head);
-  };
-  void WaitForComplete(void) { 
-    SEM_WAIT(sem_tail);
-  };
-  void EventLoop (void) {
-    //    std::cout<< " Entering event loop "<<std::endl;
-    while(1){
-      WaitForCommand();
-      //      std::cout << "Getting command "<<std::endl;
-      Event();
-    }
-  }
-
-  int Event (void) ;
-
-  uint64_t QueueCommand(int command,void *buf, int bytes, int hashtag, MPI_Comm comm,int u_rank) ;
-
-  void WaitAll() {
-    //    std::cout << "Queueing WAIT command  "<<std::endl;
-    QueueCommand(COMMAND_WAITALL,0,0,0,squadron,0);
-    //    std::cout << "Waking up DMA "<<std::endl;
-    WakeUpDMA();
-    //    std::cout << "Waiting from semaphore "<<std::endl;
-    WaitForComplete();
-    //    std::cout << "Checking FIFO is empty "<<std::endl;
-    assert ( state->tail == state->head );
-  }
-};
-
-////////////////////////////////////////////////////////////////////////
-// One instance of a data mover.
-// Master and Slave must agree on location in shared memory
-////////////////////////////////////////////////////////////////////////
-
-class MPIoffloadEngine { 
-public:
-
-  static std::vector<Slave> Slaves;
-
-  static int ShmSetup;
-  
-  static int UniverseRank;
-  static int UniverseSize;
-  
-  static MPI_Comm communicator_universe;
-  static MPI_Comm communicator_cached;
-
-  static MPI_Comm HorizontalComm;
-  static int HorizontalRank;
-  static int HorizontalSize;
-  
-  static MPI_Comm VerticalComm;
-  static MPI_Win  VerticalWindow; 
-  static int VerticalSize;
-  static int VerticalRank;
-  
-  static std::vector<void *> VerticalShmBufs;
-  static std::vector<std::vector<int> > UniverseRanks;
-  static std::vector<int> UserCommunicatorToWorldRanks; 
-  
-  static MPI_Group WorldGroup, CachedGroup;
-  
-  static void CommunicatorInit (MPI_Comm &communicator_world,
-				MPI_Comm &ShmComm,
-				void * &ShmCommBuf);
-
-  static void MapCommRankToWorldRank(int &hashtag, int & comm_world_peer,int tag, MPI_Comm comm,int commrank);
-
-  /////////////////////////////////////////////////////////
-  // routines for master proc must handle any communicator
-  /////////////////////////////////////////////////////////
-
-  static void QueueSend(int slave,void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
-     //    std::cout<< " Queueing send  "<< bytes<< " slave "<< slave << " to comm "<<rank  <<std::endl;
-    Slaves[slave].QueueCommand(COMMAND_ISEND,buf,bytes,tag,comm,rank);
-    //    std::cout << "Queued send command to rank "<< rank<< " via "<<slave <<std::endl;
-    Slaves[slave].WakeUpDMA();
-    //    std::cout << "Waking up DMA "<< slave<<std::endl;
-  };
-
-  static void QueueRecv(int slave, void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
-    //    std::cout<< " Queueing recv "<< bytes<< " slave "<< slave << " from comm "<<rank  <<std::endl;
-    Slaves[slave].QueueCommand(COMMAND_IRECV,buf,bytes,tag,comm,rank);
-    //    std::cout << "Queued recv command from rank "<< rank<< " via "<<slave <<std::endl;
-    Slaves[slave].WakeUpDMA();
-    //    std::cout << "Waking up DMA "<< slave<<std::endl;
-  };
-
-  static void WaitAll() {
-    for(int s=1;s<VerticalSize;s++) {
-      //      std::cout << "Waiting for slave "<< s<<std::endl;
-      Slaves[s].WaitAll();
-    }
-    //    std::cout << " Wait all Complete "<<std::endl;
-  };
-
-  static void GetWork(int nwork, int me, int & mywork, int & myoff,int units){
-    int basework = nwork/units;
-    int backfill = units-(nwork%units);
-    if ( me >= units ) { 
-      mywork = myoff = 0;
-    } else { 
-      mywork = (nwork+me)/units;
-      myoff  = basework * me;
-      if ( me > backfill ) 
-	myoff+= (me-backfill);
-    }
-    return;
-  };
-
-  static void QueueMultiplexedSend(void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
-    uint8_t * cbuf = (uint8_t *) buf;
-    int mywork, myoff, procs;
-    procs = VerticalSize-1;
-    for(int s=0;s<procs;s++) {
-      GetWork(bytes,s,mywork,myoff,procs);
-      QueueSend(s+1,&cbuf[myoff],mywork,tag,comm,rank);
-    }
-  };
-
-  static void QueueMultiplexedRecv(void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
-    uint8_t * cbuf = (uint8_t *) buf;
-    int mywork, myoff, procs;
-    procs = VerticalSize-1;
-    for(int s=0;s<procs;s++) {
-      GetWork(bytes,s,mywork,myoff,procs);
-      QueueRecv(s+1,&cbuf[myoff],mywork,tag,comm,rank);
-    }
-  };
-
-};
-
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-std::vector<Slave> MPIoffloadEngine::Slaves;
-    
-int MPIoffloadEngine::UniverseRank;
-int MPIoffloadEngine::UniverseSize;
-
-MPI_Comm  MPIoffloadEngine::communicator_universe;
-MPI_Comm  MPIoffloadEngine::communicator_cached;
-MPI_Group MPIoffloadEngine::WorldGroup;
-MPI_Group MPIoffloadEngine::CachedGroup;
-
-MPI_Comm MPIoffloadEngine::HorizontalComm;
-int      MPIoffloadEngine::HorizontalRank;
-int      MPIoffloadEngine::HorizontalSize;
-
-MPI_Comm MPIoffloadEngine::VerticalComm;
-int      MPIoffloadEngine::VerticalSize;
-int      MPIoffloadEngine::VerticalRank;
-MPI_Win  MPIoffloadEngine::VerticalWindow; 
-std::vector<void *>            MPIoffloadEngine::VerticalShmBufs;
-std::vector<std::vector<int> > MPIoffloadEngine::UniverseRanks;
-std::vector<int>               MPIoffloadEngine::UserCommunicatorToWorldRanks; 
-
-int MPIoffloadEngine::ShmSetup = 0;
-
-void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
-					 MPI_Comm &ShmComm,
-					 void * &ShmCommBuf)
-{      
-  int flag;
-  assert(ShmSetup==0);  
-  
-  //////////////////////////////////////////////////////////////////////
-  // Universe is all nodes prior to squadron grouping
-  //////////////////////////////////////////////////////////////////////
-  MPI_Comm_dup (MPI_COMM_WORLD,&communicator_universe);
-  MPI_Comm_rank(communicator_universe,&UniverseRank);
-  MPI_Comm_size(communicator_universe,&UniverseSize);
-  
-  /////////////////////////////////////////////////////////////////////
-  // Split into groups that can share memory (Verticals)
-  /////////////////////////////////////////////////////////////////////
-#undef MPI_SHARED_MEM_DEBUG
-#ifdef  MPI_SHARED_MEM_DEBUG
-  MPI_Comm_split(communicator_universe,(UniverseRank/4),UniverseRank,&VerticalComm);
-#else 
-  MPI_Comm_split_type(communicator_universe, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&VerticalComm);
-#endif
-  MPI_Comm_rank(VerticalComm     ,&VerticalRank);
-  MPI_Comm_size(VerticalComm     ,&VerticalSize);
-  
-  //////////////////////////////////////////////////////////////////////
-  // Split into horizontal groups by rank in squadron
-  //////////////////////////////////////////////////////////////////////
-  MPI_Comm_split(communicator_universe,VerticalRank,UniverseRank,&HorizontalComm);
-  MPI_Comm_rank(HorizontalComm,&HorizontalRank);
-  MPI_Comm_size(HorizontalComm,&HorizontalSize);
-  assert(HorizontalSize*VerticalSize==UniverseSize);
-  
-  ////////////////////////////////////////////////////////////////////////////////
-  // What is my place in the world
-  ////////////////////////////////////////////////////////////////////////////////
-  int WorldRank=0;
-  if(VerticalRank==0) WorldRank = HorizontalRank;
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&WorldRank,1,MPI_INT,MPI_SUM,VerticalComm);
-  assert(ierr==0);
-  
-  ////////////////////////////////////////////////////////////////////////////////
-  // Where is the world in the universe?
-  ////////////////////////////////////////////////////////////////////////////////
-  UniverseRanks = std::vector<std::vector<int> >(HorizontalSize,std::vector<int>(VerticalSize,0));
-  UniverseRanks[WorldRank][VerticalRank] = UniverseRank;
-  for(int w=0;w<HorizontalSize;w++){
-    ierr=MPI_Allreduce(MPI_IN_PLACE,&UniverseRanks[w][0],VerticalSize,MPI_INT,MPI_SUM,communicator_universe);
-    assert(ierr==0);
-  }
-  
-  //////////////////////////////////////////////////////////////////////////////////////////////////////////
-  // allocate the shared window for our group, pass back Shm info to CartesianCommunicator
-  //////////////////////////////////////////////////////////////////////////////////////////////////////////
-  VerticalShmBufs.resize(VerticalSize);
-
-#undef MPI_SHARED_MEM
-#ifdef MPI_SHARED_MEM
-  ierr = MPI_Win_allocate_shared(CartesianCommunicator::MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,VerticalComm,&ShmCommBuf,&VerticalWindow);
-  ierr|= MPI_Win_lock_all (MPI_MODE_NOCHECK, VerticalWindow);
-  assert(ierr==0);
-  //  std::cout<<"SHM "<<ShmCommBuf<<std::endl;
-
-  for(int r=0;r<VerticalSize;r++){
-    MPI_Aint sz;
-    int dsp_unit;
-    MPI_Win_shared_query (VerticalWindow, r, &sz, &dsp_unit, &VerticalShmBufs[r]);
-    //    std::cout<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
-  }
-#else 
-  char shm_name [NAME_MAX];
-  MPI_Barrier(VerticalComm);
-
-  if ( VerticalRank == 0 ) {
-    for(int r=0;r<VerticalSize;r++){
-
-      size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES;
-      if ( r>0 ) size = sizeof(SlaveState);
-
-      sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",WorldRank,r);
-      
-      shm_unlink(shm_name);
-
-      int fd=shm_open(shm_name,O_RDWR|O_CREAT,0600);
-      if ( fd < 0 ) {
-	perror("failed shm_open");
-	assert(0);
-      }
-
-      ftruncate(fd, size);
-
-      VerticalShmBufs[r] = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-
-      if ( VerticalShmBufs[r] == MAP_FAILED ) { 
-	perror("failed mmap");
-	assert(0);
-      }
-
-      uint64_t * check = (uint64_t *) VerticalShmBufs[r];
-      check[0] = WorldRank;
-      check[1] = r;
-
-      //      std::cout<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
-    }
-  }
-
-  MPI_Barrier(VerticalComm);
-
-  if ( VerticalRank != 0 ) { 
-  for(int r=0;r<VerticalSize;r++){
-
-    size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES ;
-    if ( r>0 ) size = sizeof(SlaveState);
-    
-    sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",WorldRank,r);
-    
-    int fd=shm_open(shm_name,O_RDWR|O_CREAT,0600);
-    if ( fd<0 ) {
-      perror("failed shm_open");
-      assert(0);
-    }
-    VerticalShmBufs[r] = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-
-    uint64_t * check = (uint64_t *) VerticalShmBufs[r];
-    assert(check[0]== WorldRank);
-    assert(check[1]== r);
-    std::cerr<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
-  }
-  }
-#endif
-  MPI_Barrier(VerticalComm);
-
-  //////////////////////////////////////////////////////////////////////
-  // Map rank of leader on node in their in new world, to the
-  // rank in this vertical plane's horizontal communicator
-  //////////////////////////////////////////////////////////////////////
-  communicator_world = HorizontalComm;
-  ShmComm            = VerticalComm;
-  ShmCommBuf         = VerticalShmBufs[0];
-  MPI_Comm_group (communicator_world, &WorldGroup); 
-  
-  ///////////////////////////////////////////////////////////
-  // Start the slave data movers
-  ///////////////////////////////////////////////////////////
-  if ( VerticalRank != 0 ) {
-    Slave indentured;
-    indentured.Init( (SlaveState *) VerticalShmBufs[VerticalRank], VerticalComm, UniverseRank,VerticalRank);
-    indentured.SemInitExcl();// init semaphore in shared memory
-    MPI_Barrier(VerticalComm);
-    MPI_Barrier(VerticalComm);
-    indentured.EventLoop();
-    assert(0);
-  } else {
-    Slaves.resize(VerticalSize);
-    for(int i=1;i<VerticalSize;i++){
-      Slaves[i].Init((SlaveState *)VerticalShmBufs[i],VerticalComm, UniverseRanks[HorizontalRank][i],i);
-    }
-    MPI_Barrier(VerticalComm);
-    for(int i=1;i<VerticalSize;i++){
-      Slaves[i].SemInit();// init semaphore in shared memory
-    }
-    MPI_Barrier(VerticalComm);
-  }
-  
-  ///////////////////////////////////////////////////////////
-  // Verbose for now
-  ///////////////////////////////////////////////////////////
-  
-  ShmSetup=1;
-  
-  if (UniverseRank == 0){
-      
-    std::cout<<GridLogMessage << "Grid MPI-3 configuration: detected ";
-    std::cout<<UniverseSize   << " Ranks " ;
-    std::cout<<HorizontalSize << " Nodes " ;
-    std::cout<<VerticalSize   << " with ranks-per-node "<<std::endl;
-    
-    std::cout<<GridLogMessage << "Grid MPI-3 configuration: using one lead process per node " << std::endl;
-    std::cout<<GridLogMessage << "Grid MPI-3 configuration: reduced communicator has size " << HorizontalSize << std::endl;
-    
-    for(int g=0;g<HorizontalSize;g++){
-      std::cout<<GridLogMessage<<" Node "<<g<<" led by MPI rank "<< UniverseRanks[g][0]<<std::endl;
-    }
-    
-    for(int g=0;g<HorizontalSize;g++){
-      std::cout<<GridLogMessage<<" { ";
-      for(int s=0;s<VerticalSize;s++){
-	std::cout<< UniverseRanks[g][s];
-	if ( s<VerticalSize-1 ) {
-	  std::cout<<",";
-	}
-      }
-      std::cout<<" } "<<std::endl;
-    }
-  }
-};
-
-  ///////////////////////////////////////////////////////////////////////////////////////////////
-  // Map the communicator into communicator_world, and find the neighbour.
-  // Cache the mappings; cache size is 1.
-  ///////////////////////////////////////////////////////////////////////////////////////////////
-void MPIoffloadEngine::MapCommRankToWorldRank(int &hashtag, int & comm_world_peer,int tag, MPI_Comm comm,int rank) {
-
-  if ( comm == HorizontalComm ) {
-    comm_world_peer = rank;
-    //    std::cout << " MapCommRankToWorldRank  horiz " <<rank<<"->"<<comm_world_peer<<std::endl;
-  } else if ( comm == communicator_cached ) {
-    comm_world_peer = UserCommunicatorToWorldRanks[rank];
-    //    std::cout << " MapCommRankToWorldRank  cached " <<rank<<"->"<<comm_world_peer<<std::endl;
-  } else { 
-    
-    int size;
-
-    MPI_Comm_size(comm,&size);
-
-    UserCommunicatorToWorldRanks.resize(size);
-
-    std::vector<int> cached_ranks(size); 
-
-    for(int r=0;r<size;r++) {
-      cached_ranks[r]=r;
-    }
-
-    communicator_cached=comm;
-    
-    MPI_Comm_group(communicator_cached, &CachedGroup);
-    
-    MPI_Group_translate_ranks(CachedGroup,size,&cached_ranks[0],WorldGroup, &UserCommunicatorToWorldRanks[0]); 
-    
-    comm_world_peer = UserCommunicatorToWorldRanks[rank];
-    //    std::cout << " MapCommRankToWorldRank  cache miss " <<rank<<"->"<<comm_world_peer<<std::endl;
-    
-    assert(comm_world_peer != MPI_UNDEFINED);
-  }
-
-  assert( (tag & (~0xFFFFL)) ==0); 
-  
-  uint64_t icomm = (uint64_t)comm;
-  int comm_hash = ((icomm>>0 )&0xFFFF)^((icomm>>16)&0xFFFF)
-                ^ ((icomm>>32)&0xFFFF)^((icomm>>48)&0xFFFF);
-  
-  //  hashtag = (comm_hash<<15) | tag;      
-  hashtag = tag;      
-
-};
-
-void Slave::Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank)
-{
-  squadron=_squadron;
-  universe_rank=_universe_rank;
-  vertical_rank=_vertical_rank;
-  state   =_state;
-  //  std::cout << "state "<<_state<<" comm "<<_squadron<<" universe_rank"<<universe_rank <<std::endl;
-  state->head = state->tail = state->start = 0;
-  base = (uint64_t)MPIoffloadEngine::VerticalShmBufs[0];
-  int rank; MPI_Comm_rank(_squadron,&rank);
-}
-#define PERI_PLUS(A) ( (A+1)%pool )
-int Slave::Event (void) {
-
-  static int tail_last;
-  static int head_last;
-  static int start_last;
-  int ierr;
-
-  ////////////////////////////////////////////////////
-  // Try to advance the start pointers
-  ////////////////////////////////////////////////////
-  int s=state->start;
-  if ( s != state->head ) {
-    switch ( state->Descrs[s].command ) {
-    case COMMAND_ISEND:
-      /*
-            std::cout<< " Send "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
-      	       << " to " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
-       << " Comm " << MPIoffloadEngine::communicator_universe<< " me " <<universe_rank<< std::endl;
-      */
-      ierr = MPI_Isend((void *)(state->Descrs[s].buf+base), 
-		       state->Descrs[s].bytes, 
-		       MPI_CHAR,
-		       state->Descrs[s].rank,
-		       state->Descrs[s].tag,
-		       MPIoffloadEngine::communicator_universe,
-		       (MPI_Request *)&state->Descrs[s].request);
-      assert(ierr==0);
-      state->start = PERI_PLUS(s);
-      return 1;
-      break;
-
-    case COMMAND_IRECV:
-      /*
-      std::cout<< " Recv "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
-	       << " from " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
-	       << " Comm " << MPIoffloadEngine::communicator_universe<< " me "<< universe_rank<< std::endl;
-      */
-      ierr=MPI_Irecv((void *)(state->Descrs[s].buf+base), 
-		     state->Descrs[s].bytes, 
-		     MPI_CHAR,
-		     state->Descrs[s].rank,
-		     state->Descrs[s].tag,
-		     MPIoffloadEngine::communicator_universe,
-		     (MPI_Request *)&state->Descrs[s].request);
-
-      //      std::cout<< " Request is "<<state->Descrs[s].request<<std::endl;
-      //      std::cout<< " Request0 is "<<state->Descrs[0].request<<std::endl;
-      assert(ierr==0);
-      state->start = PERI_PLUS(s);
-      return 1;
-      break;
-
-    case COMMAND_WAITALL:
-
-      for(int t=state->tail;t!=s; t=PERI_PLUS(t) ){
-	MPI_Wait((MPI_Request *)&state->Descrs[t].request,MPI_STATUS_IGNORE);
-      };
-      s=PERI_PLUS(s);
-      state->start = s;
-      state->tail  = s;
-
-      WakeUpCompute();
-
-      return 1;
-      break;
-
-    default:
-      assert(0);
-      break;
-    }
-  }
-  return 0;
-}
-  //////////////////////////////////////////////////////////////////////////////
-  // External interaction with the queue
-  //////////////////////////////////////////////////////////////////////////////
-  
-uint64_t Slave::QueueCommand(int command,void *buf, int bytes, int tag, MPI_Comm comm,int commrank) 
-{
-  /////////////////////////////////////////
-  // Spin; if FIFO is full until not full
-  /////////////////////////////////////////
-  int head =state->head;
-  int next = PERI_PLUS(head);
-    
-  // Set up descriptor
-  int worldrank;
-  int hashtag;
-  MPI_Comm    communicator;
-  MPI_Request request;
-  
-  MPIoffloadEngine::MapCommRankToWorldRank(hashtag,worldrank,tag,comm,commrank);
-
-  uint64_t relative= (uint64_t)buf - base;
-  state->Descrs[head].buf    = relative;
-  state->Descrs[head].bytes  = bytes;
-  state->Descrs[head].rank   = MPIoffloadEngine::UniverseRanks[worldrank][vertical_rank];
-  state->Descrs[head].tag    = hashtag;
-  state->Descrs[head].command= command;
-
-  /*  
-  if ( command == COMMAND_ISEND ) { 
-  std::cout << "QueueSend from "<< universe_rank <<" to commrank " << commrank 
-            << " to worldrank " << worldrank <<std::endl;
-  std::cout << " via VerticalRank "<< vertical_rank <<" to universerank " << MPIoffloadEngine::UniverseRanks[worldrank][vertical_rank]<<std::endl;
-  std::cout << " QueueCommand "<<buf<<"["<<bytes<<"]" << std::endl;
-  } 
-  if ( command == COMMAND_IRECV ) { 
-  std::cout << "QueueRecv on "<< universe_rank <<" from commrank " << commrank 
-            << " from worldrank " << worldrank <<std::endl;
-  std::cout << " via VerticalRank "<< vertical_rank <<" from universerank " << MPIoffloadEngine::UniverseRanks[worldrank][vertical_rank]<<std::endl;
-  std::cout << " QueueSend "<<buf<<"["<<bytes<<"]" << std::endl;
-  } 
-  */
-  // Block until FIFO has space
-  while( state->tail==next );
-
-  // Msync on weak order architectures
-  // Advance pointer
-  state->head = next;
-
-  return 0;
-}
-  
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-MPI_Comm CartesianCommunicator::communicator_world;
-
-void CartesianCommunicator::Init(int *argc, char ***argv) 
-{
-  int flag;
-  MPI_Initialized(&flag); // needed to coexist with other libs apparently
-  if ( !flag ) {
-    MPI_Init(argc,argv);
-  }
-  communicator_world = MPI_COMM_WORLD;
-  MPI_Comm ShmComm;
-  MPIoffloadEngine::CommunicatorInit (communicator_world,ShmComm,ShmCommBuf);
-}
-void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
-{
-  int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest);
-  assert(ierr==0);
-}
-int CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor)
-{
-  int rank;
-  int ierr=MPI_Cart_rank  (communicator, &coor[0], &rank);
-  assert(ierr==0);
-  return rank;
-}
-void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor)
-{
-  coor.resize(_ndimension);
-  int ierr=MPI_Cart_coords  (communicator, rank, _ndimension,&coor[0]);
-  assert(ierr==0);
-}
-
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
-{ 
-  _ndimension = processors.size();
-  std::vector<int> periodic(_ndimension,1);
-
-  _Nprocessors=1;
-  _processors = processors;
-
-  for(int i=0;i<_ndimension;i++){
-    _Nprocessors*=_processors[i];
-  }
-
-  int Size; 
-  MPI_Comm_size(communicator_world,&Size);
-  assert(Size==_Nprocessors);
-
-  _processor_coor.resize(_ndimension);
-  MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator);
-  MPI_Comm_rank  (communicator,&_processor);
-  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
-};
-
-void CartesianCommunicator::GlobalSum(uint32_t &u){
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSum(uint64_t &u){
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSum(float &f){
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSumVector(float *f,int N)
-{
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSum(double &d)
-{
-  int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-void CartesianCommunicator::GlobalSumVector(double *d,int N)
-{
-  int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
-  assert(ierr==0);
-}
-
-// Basic Halo comms primitive
-void CartesianCommunicator::SendToRecvFrom(void *xmit,
-					   int dest,
-					   void *recv,
-					   int from,
-					   int bytes)
-{
-  std::vector<CommsRequest_t> reqs(0);
-  SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes);
-  SendToRecvFromComplete(reqs);
-}
-
-void CartesianCommunicator::SendRecvPacket(void *xmit,
-					   void *recv,
-					   int sender,
-					   int receiver,
-					   int bytes)
-{
-  MPI_Status stat;
-  assert(sender != receiver);
-  int tag = sender;
-  if ( _processor == sender ) {
-    MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator);
-  }
-  if ( _processor == receiver ) { 
-    MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat);
-  }
-}
-
-// Basic Halo comms primitive
-void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						void *xmit,
-						int dest,
-						void *recv,
-						int from,
-						int bytes)
-{
-  MPI_Request xrq;
-  MPI_Request rrq;
-  int rank = _processor;
-  int ierr;
-  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
-  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
-  
-  assert(ierr==0);
-
-  list.push_back(xrq);
-  list.push_back(rrq);
-}
-
-void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						       void *xmit,
-						       int dest,
-						       void *recv,
-						       int from,
-						       int bytes)
-{
-  uint64_t xmit_i = (uint64_t) xmit;
-  uint64_t recv_i = (uint64_t) recv;
-  uint64_t shm    = (uint64_t) ShmCommBuf;
-  // assert xmit and recv lie in shared memory region
-  assert( (xmit_i >= shm) && (xmit_i+bytes <= shm+MAX_MPI_SHM_BYTES) );
-  assert( (recv_i >= shm) && (recv_i+bytes <= shm+MAX_MPI_SHM_BYTES) );
-  assert(from!=_processor);
-  assert(dest!=_processor);
-  MPIoffloadEngine::QueueMultiplexedSend(xmit,bytes,_processor,communicator,dest);
-  MPIoffloadEngine::QueueMultiplexedRecv(recv,bytes,from,communicator,from);
-}
-
-
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list)
-{
-  MPIoffloadEngine::WaitAll();
-}
-
-void CartesianCommunicator::StencilBarrier(void)
-{
-}
-
-void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
-{
-  int nreq=list.size();
-  std::vector<MPI_Status> status(nreq);
-  int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
-  assert(ierr==0);
-}
-
-void CartesianCommunicator::Barrier(void)
-{
-  int ierr = MPI_Barrier(communicator);
-  assert(ierr==0);
-}
-
-void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
-{
-  int ierr=MPI_Bcast(data,
-		     bytes,
-		     MPI_BYTE,
-		     root,
-		     communicator);
-  assert(ierr==0);
-}
-
-void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
-{
-  int ierr= MPI_Bcast(data,
-		      bytes,
-		      MPI_BYTE,
-		      root,
-		      communicator_world);
-  assert(ierr==0);
-}
-
-void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; }
-
-void *CartesianCommunicator::ShmBuffer(int rank) {
-  return NULL;
-}
-void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { 
-  return NULL;
-}
-
-  void CartesianCommunicator::PrintRankInfo(){
-    std::cout << "Grid: Rank "<< _processor << "  -  Physical node name: " << name << std::endl;
-  }
-
-};
-

lib/communicator/Communicator_none.cc

@@ -28,19 +28,11 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include "Grid.h"
 namespace Grid {
 
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
 void CartesianCommunicator::Init(int *argc, char *** arv)
 {
-  ShmInitGeneric();
-}
-
-void CartesianCommunicator::PrintRankInfo(){
-    std::cout << GridLogMessage << "No Rank Info available" << std::endl;
 }
 
+int Rank(void ){ return 0; };
 
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
@@ -97,17 +89,30 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &
   assert(0);
 }
 
-int  CartesianCommunicator::RankWorld(void){return 0;}
-void CartesianCommunicator::Barrier(void){}
-void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {}
-void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { }
-int  CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor) {  return 0;}
-void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor){ coor = _processor_coor ;}
+void CartesianCommunicator::Barrier(void)
+{
+}
+
+void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
+{
+}
+void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
+{
+}
+
+
 void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
 {
   source =0;
   dest=0;
 }
+int CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor)
+{
+  return 0;
+}
+void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor)
+{
+}
 
 
 }

lib/communicator/Communicator_shmem.cc

@@ -39,24 +39,14 @@ namespace Grid {
     BACKTRACEFILE();		   \
   }\
 }
-
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
+int Rank(void) {
+  return shmem_my_pe();
+}
 typedef struct HandShake_t { 
   uint64_t seq_local;
   uint64_t seq_remote;
 } HandShake;
 
-std::array<long,_SHMEM_REDUCE_SYNC_SIZE> make_psync_init(void) {
-  array<long,_SHMEM_REDUCE_SYNC_SIZE> ret;
-  ret.fill(SHMEM_SYNC_VALUE);
-  return ret;
-}
-static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync_init = make_psync_init();
-
 static Vector< HandShake > XConnections;
 static Vector< HandShake > RConnections;
 
@@ -71,9 +61,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     RConnections[pe].seq_remote= 0;
   }
   shmem_barrier_all();
-  ShmInitGeneric();
 }
-
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
   _ndimension = processors.size();
@@ -101,7 +89,7 @@ void CartesianCommunicator::GlobalSum(uint32_t &u){
   static long long source ;
   static long long dest   ;
   static long long llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+  static long      psync[_SHMEM_REDUCE_SYNC_SIZE];
 
   //  int nreduce=1;
   //  int pestart=0;
@@ -117,7 +105,7 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){
   static long long source ;
   static long long dest   ;
   static long long llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+  static long      psync[_SHMEM_REDUCE_SYNC_SIZE];
 
   //  int nreduce=1;
   //  int pestart=0;
@@ -133,7 +121,7 @@ void CartesianCommunicator::GlobalSum(float &f){
   static float source ;
   static float dest   ;
   static float llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
 
   source = f;
   dest   =0.0;
@@ -145,7 +133,7 @@ void CartesianCommunicator::GlobalSumVector(float *f,int N)
   static float source ;
   static float dest   = 0 ;
   static float llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
 
   if ( shmem_addr_accessible(f,_processor)  ){
     shmem_float_sum_to_all(f,f,N,0,0,_Nprocessors,llwrk,psync);
@@ -164,7 +152,7 @@ void CartesianCommunicator::GlobalSum(double &d)
   static double source;
   static double dest  ;
   static double llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
 
   source = d;
   dest   = 0;
@@ -176,8 +164,7 @@ void CartesianCommunicator::GlobalSumVector(double *d,int N)
   static double source ;
   static double dest   ;
   static double llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
-
+  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
 
   if ( shmem_addr_accessible(d,_processor)  ){
     shmem_double_sum_to_all(d,d,N,0,0,_Nprocessors,llwrk,psync);
@@ -243,9 +230,12 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
 
   if ( _processor == sender ) {
 
+    printf("Sender SHMEM pt2pt %d -> %d\n",sender,receiver);
     // Check he has posted a receive
     while(SendSeq->seq_remote == SendSeq->seq_local);
 
+    printf("Sender receive %d posted\n",sender,receiver);
+
     // Advance our send count
     seq = ++(SendSeq->seq_local);
     
@@ -254,19 +244,26 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
     shmem_putmem(recv,xmit,bytes,receiver);
     shmem_fence();
 
+    printf("Sender sent payload %d\n",seq);
     //Notify him we're done
     shmem_putmem((void *)&(RecvSeq->seq_remote),&seq,sizeof(seq),receiver);
     shmem_fence();
+    printf("Sender ringing door bell  %d\n",seq);
   }
   if ( _processor == receiver ) {
 
+    printf("Receiver SHMEM pt2pt %d->%d\n",sender,receiver);
     // Post a receive
     seq = ++(RecvSeq->seq_local);
     shmem_putmem((void *)&(SendSeq->seq_remote),&seq,sizeof(seq),sender);
 
+    printf("Receiver Opening letter box %d\n",seq);
+
+    
     // Now wait until he has advanced our reception counter
     while(RecvSeq->seq_remote != RecvSeq->seq_local);
 
+    printf("Receiver Got the mail %d\n",seq);
   }
 }
 
@@ -294,7 +291,7 @@ void CartesianCommunicator::Barrier(void)
 }
 void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
 {
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
   static uint32_t word;
   uint32_t *array = (uint32_t *) data;
   assert( (bytes % 4)==0);
@@ -317,7 +314,7 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
 }
 void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
 {
-  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
   static uint32_t word;
   uint32_t *array = (uint32_t *) data;
   assert( (bytes % 4)==0);
@@ -333,9 +330,5 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
   }
 }
 
-  void CartesianCommunicator::PrintRankInfo(){
-    std::cout << GridLogMessage << "SHMEM: Rank Info not implemented yet" << std::endl;
-  }
-
 }
 

lib/cshift/Cshift_common.h

@@ -45,7 +45,7 @@ public:
 // Gather for when there is no need to SIMD split with compression
 ///////////////////////////////////////////////////////////////////
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0)
+Gather_plane_simple (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 
@@ -114,7 +114,6 @@ PARALLEL_NESTED_LOOP2
 	int o      =   n*n1;
 	int offset = b+n*n2;
 	cobj temp =compress(rhs._odata[so+o+b]);
-
 	extract<cobj>(temp,pointers,offset);
 
       }
@@ -122,7 +121,6 @@ PARALLEL_NESTED_LOOP2
   } else { 
 
     assert(0); //Fixme think this is buggy
-
     for(int n=0;n<e1;n++){
       for(int b=0;b<e2;b++){
 	int o=n*rhs._grid->_slice_stride[dimension];
@@ -141,7 +139,7 @@ PARALLEL_NESTED_LOOP2
 //////////////////////////////////////////////////////
 // Gather for when there is no need to SIMD split
 //////////////////////////////////////////////////////
-template<class vobj> void Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
+template<class vobj> void Gather_plane_simple (const Lattice<vobj> &rhs,std::vector<vobj,alignedAllocator<vobj> > &buffer,             int dimension,int plane,int cbmask)
 {
   SimpleCompressor<vobj> dontcompress;
   Gather_plane_simple (rhs,buffer,dimension,plane,cbmask,dontcompress);
@@ -159,7 +157,7 @@ template<class vobj> void Gather_plane_extract(const Lattice<vobj> &rhs,std::vec
 //////////////////////////////////////////////////////
 // Scatter for when there is no need to SIMD split
 //////////////////////////////////////////////////////
-template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
+template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,std::vector<vobj,alignedAllocator<vobj> > &buffer, int dimension,int plane,int cbmask)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 

lib/cshift/Cshift_mpi.h

@@ -119,8 +119,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
   assert(shift<fd);
   
   int buffer_size = rhs._grid->_slice_nblock[dimension]*rhs._grid->_slice_block[dimension];
-  commVector<vobj> send_buf(buffer_size);
-  commVector<vobj> recv_buf(buffer_size);
+  std::vector<vobj,alignedAllocator<vobj> > send_buf(buffer_size);
+  std::vector<vobj,alignedAllocator<vobj> > recv_buf(buffer_size);
 
   int cb= (cbmask==0x2)? Odd : Even;
   int sshift= rhs._grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb);
@@ -191,8 +191,8 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
   int words = sizeof(vobj)/sizeof(vector_type);
 
-  std::vector<commVector<scalar_object> >   send_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
-  std::vector<commVector<scalar_object> >   recv_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
+  std::vector<Vector<scalar_object> >   send_buf_extract(Nsimd,Vector<scalar_object>(buffer_size) );
+  std::vector<Vector<scalar_object> >   recv_buf_extract(Nsimd,Vector<scalar_object>(buffer_size) );
 
   int bytes = buffer_size*sizeof(scalar_object);
 

lib/fftw/fftw3.h

@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2003, 2007-14 Matteo Frigo
+ * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
+ *
+ * The following statement of license applies *only* to this header file,
+ * and *not* to the other files distributed with FFTW or derived therefrom:
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/***************************** NOTE TO USERS *********************************
+ *
+ *                 THIS IS A HEADER FILE, NOT A MANUAL
+ *
+ *    If you want to know how to use FFTW, please read the manual,
+ *    online at http://www.fftw.org/doc/ and also included with FFTW.
+ *    For a quick start, see the manual's tutorial section.
+ *
+ *   (Reading header files to learn how to use a library is a habit
+ *    stemming from code lacking a proper manual.  Arguably, it's a
+ *    *bad* habit in most cases, because header files can contain
+ *    interfaces that are not part of the public, stable API.)
+ *
+ ****************************************************************************/
+
+#ifndef FFTW3_H
+#define FFTW3_H
+
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif /* __cplusplus */
+
+/* If <complex.h> is included, use the C99 complex type.  Otherwise
+   define a type bit-compatible with C99 complex */
+#if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I)
+#  define FFTW_DEFINE_COMPLEX(R, C) typedef R _Complex C
+#else
+#  define FFTW_DEFINE_COMPLEX(R, C) typedef R C[2]
+#endif
+
+#define FFTW_CONCAT(prefix, name) prefix ## name
+#define FFTW_MANGLE_DOUBLE(name) FFTW_CONCAT(fftw_, name)
+#define FFTW_MANGLE_FLOAT(name) FFTW_CONCAT(fftwf_, name)
+#define FFTW_MANGLE_LONG_DOUBLE(name) FFTW_CONCAT(fftwl_, name)
+#define FFTW_MANGLE_QUAD(name) FFTW_CONCAT(fftwq_, name)
+
+/* IMPORTANT: for Windows compilers, you should add a line
+        #define FFTW_DLL
+   here and in kernel/ifftw.h if you are compiling/using FFTW as a
+   DLL, in order to do the proper importing/exporting, or
+   alternatively compile with -DFFTW_DLL or the equivalent
+   command-line flag.  This is not necessary under MinGW/Cygwin, where
+   libtool does the imports/exports automatically. */
+#if defined(FFTW_DLL) && (defined(_WIN32) || defined(__WIN32__))
+   /* annoying Windows syntax for shared-library declarations */
+#  if defined(COMPILING_FFTW) /* defined in api.h when compiling FFTW */
+#    define FFTW_EXTERN extern __declspec(dllexport) 
+#  else /* user is calling FFTW; import symbol */
+#    define FFTW_EXTERN extern __declspec(dllimport) 
+#  endif
+#else
+#  define FFTW_EXTERN extern
+#endif
+
+enum fftw_r2r_kind_do_not_use_me {
+     FFTW_R2HC=0, FFTW_HC2R=1, FFTW_DHT=2,
+     FFTW_REDFT00=3, FFTW_REDFT01=4, FFTW_REDFT10=5, FFTW_REDFT11=6,
+     FFTW_RODFT00=7, FFTW_RODFT01=8, FFTW_RODFT10=9, FFTW_RODFT11=10
+};
+
+struct fftw_iodim_do_not_use_me {
+     int n;                     /* dimension size */
+     int is;			/* input stride */
+     int os;			/* output stride */
+};
+
+#include <stddef.h> /* for ptrdiff_t */
+struct fftw_iodim64_do_not_use_me {
+     ptrdiff_t n;                     /* dimension size */
+     ptrdiff_t is;			/* input stride */
+     ptrdiff_t os;			/* output stride */
+};
+
+typedef void (*fftw_write_char_func_do_not_use_me)(char c, void *);
+typedef int (*fftw_read_char_func_do_not_use_me)(void *);
+
+/*
+  huge second-order macro that defines prototypes for all API
+  functions.  We expand this macro for each supported precision
+ 
+  X: name-mangling macro
+  R: real data type
+  C: complex data type
+*/
+
+#define FFTW_DEFINE_API(X, R, C)					   \
+									   \
+FFTW_DEFINE_COMPLEX(R, C);						   \
+									   \
+typedef struct X(plan_s) *X(plan);					   \
+									   \
+typedef struct fftw_iodim_do_not_use_me X(iodim);			   \
+typedef struct fftw_iodim64_do_not_use_me X(iodim64);			   \
+									   \
+typedef enum fftw_r2r_kind_do_not_use_me X(r2r_kind);			   \
+									   \
+typedef fftw_write_char_func_do_not_use_me X(write_char_func);		   \
+typedef fftw_read_char_func_do_not_use_me X(read_char_func);		   \
+									   \
+FFTW_EXTERN void X(execute)(const X(plan) p);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft)(int rank, const int *n,			   \
+		    C *in, C *out, int sign, unsigned flags);		   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_1d)(int n, C *in, C *out, int sign,	   \
+		       unsigned flags);					   \
+FFTW_EXTERN X(plan) X(plan_dft_2d)(int n0, int n1,			   \
+		       C *in, C *out, int sign, unsigned flags);	   \
+FFTW_EXTERN X(plan) X(plan_dft_3d)(int n0, int n1, int n2,		   \
+		       C *in, C *out, int sign, unsigned flags);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_dft)(int rank, const int *n,		   \
+                         int howmany,					   \
+                         C *in, const int *inembed,			   \
+                         int istride, int idist,			   \
+                         C *out, const int *onembed,			   \
+                         int ostride, int odist,			   \
+                         int sign, unsigned flags);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_dft)(int rank, const X(iodim) *dims,	   \
+			 int howmany_rank,				   \
+			 const X(iodim) *howmany_dims,			   \
+			 C *in, C *out,					   \
+			 int sign, unsigned flags);			   \
+FFTW_EXTERN X(plan) X(plan_guru_split_dft)(int rank, const X(iodim) *dims, \
+			 int howmany_rank,				   \
+			 const X(iodim) *howmany_dims,			   \
+			 R *ri, R *ii, R *ro, R *io,			   \
+			 unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_dft)(int rank,			   \
+                         const X(iodim64) *dims,			   \
+			 int howmany_rank,				   \
+			 const X(iodim64) *howmany_dims,		   \
+			 C *in, C *out,					   \
+			 int sign, unsigned flags);			   \
+FFTW_EXTERN X(plan) X(plan_guru64_split_dft)(int rank,			   \
+                         const X(iodim64) *dims,			   \
+			 int howmany_rank,				   \
+			 const X(iodim64) *howmany_dims,		   \
+			 R *ri, R *ii, R *ro, R *io,			   \
+			 unsigned flags);				   \
+									   \
+FFTW_EXTERN void X(execute_dft)(const X(plan) p, C *in, C *out);	   \
+FFTW_EXTERN void X(execute_split_dft)(const X(plan) p, R *ri, R *ii,	   \
+                                      R *ro, R *io);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_dft_r2c)(int rank, const int *n,	   \
+                             int howmany,				   \
+                             R *in, const int *inembed,			   \
+                             int istride, int idist,			   \
+                             C *out, const int *onembed,		   \
+                             int ostride, int odist,			   \
+                             unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_r2c)(int rank, const int *n,		   \
+                        R *in, C *out, unsigned flags);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_r2c_1d)(int n,R *in,C *out,unsigned flags); \
+FFTW_EXTERN X(plan) X(plan_dft_r2c_2d)(int n0, int n1,			   \
+			   R *in, C *out, unsigned flags);		   \
+FFTW_EXTERN X(plan) X(plan_dft_r2c_3d)(int n0, int n1,			   \
+			   int n2,					   \
+			   R *in, C *out, unsigned flags);		   \
+									   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_dft_c2r)(int rank, const int *n,	   \
+			     int howmany,				   \
+			     C *in, const int *inembed,			   \
+			     int istride, int idist,			   \
+			     R *out, const int *onembed,		   \
+			     int ostride, int odist,			   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_c2r)(int rank, const int *n,		   \
+                        C *in, R *out, unsigned flags);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_c2r_1d)(int n,C *in,R *out,unsigned flags); \
+FFTW_EXTERN X(plan) X(plan_dft_c2r_2d)(int n0, int n1,			   \
+			   C *in, R *out, unsigned flags);		   \
+FFTW_EXTERN X(plan) X(plan_dft_c2r_3d)(int n0, int n1,			   \
+			   int n2,					   \
+			   C *in, R *out, unsigned flags);		   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_dft_r2c)(int rank, const X(iodim) *dims,   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     R *in, C *out,				   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru_dft_c2r)(int rank, const X(iodim) *dims,   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     C *in, R *out,				   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_split_dft_r2c)(				   \
+                             int rank, const X(iodim) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     R *in, R *ro, R *io,			   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru_split_dft_c2r)(				   \
+                             int rank, const X(iodim) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     R *ri, R *ii, R *out,			   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_dft_r2c)(int rank,			   \
+                             const X(iodim64) *dims,			   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     R *in, C *out,				   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru64_dft_c2r)(int rank,			   \
+                             const X(iodim64) *dims,			   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     C *in, R *out,				   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_split_dft_r2c)(			   \
+                             int rank, const X(iodim64) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     R *in, R *ro, R *io,			   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru64_split_dft_c2r)(			   \
+                             int rank, const X(iodim64) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     R *ri, R *ii, R *out,			   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN void X(execute_dft_r2c)(const X(plan) p, R *in, C *out);	   \
+FFTW_EXTERN void X(execute_dft_c2r)(const X(plan) p, C *in, R *out);	   \
+									   \
+FFTW_EXTERN void X(execute_split_dft_r2c)(const X(plan) p,		   \
+                                          R *in, R *ro, R *io);		   \
+FFTW_EXTERN void X(execute_split_dft_c2r)(const X(plan) p,		   \
+                                          R *ri, R *ii, R *out);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_r2r)(int rank, const int *n,		   \
+                         int howmany,					   \
+                         R *in, const int *inembed,			   \
+                         int istride, int idist,			   \
+                         R *out, const int *onembed,			   \
+                         int ostride, int odist,			   \
+                         const X(r2r_kind) *kind, unsigned flags);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_r2r)(int rank, const int *n, R *in, R *out,	   \
+                    const X(r2r_kind) *kind, unsigned flags);		   \
+									   \
+FFTW_EXTERN X(plan) X(plan_r2r_1d)(int n, R *in, R *out,		   \
+                       X(r2r_kind) kind, unsigned flags);		   \
+FFTW_EXTERN X(plan) X(plan_r2r_2d)(int n0, int n1, R *in, R *out,	   \
+                       X(r2r_kind) kind0, X(r2r_kind) kind1,		   \
+                       unsigned flags);					   \
+FFTW_EXTERN X(plan) X(plan_r2r_3d)(int n0, int n1, int n2,		   \
+                       R *in, R *out, X(r2r_kind) kind0,		   \
+                       X(r2r_kind) kind1, X(r2r_kind) kind2,		   \
+                       unsigned flags);					   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_r2r)(int rank, const X(iodim) *dims,	   \
+                         int howmany_rank,				   \
+                         const X(iodim) *howmany_dims,			   \
+                         R *in, R *out,					   \
+                         const X(r2r_kind) *kind, unsigned flags);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_r2r)(int rank, const X(iodim64) *dims,   \
+                         int howmany_rank,				   \
+                         const X(iodim64) *howmany_dims,		   \
+                         R *in, R *out,					   \
+                         const X(r2r_kind) *kind, unsigned flags);	   \
+									   \
+FFTW_EXTERN void X(execute_r2r)(const X(plan) p, R *in, R *out);	   \
+									   \
+FFTW_EXTERN void X(destroy_plan)(X(plan) p);				   \
+FFTW_EXTERN void X(forget_wisdom)(void);				   \
+FFTW_EXTERN void X(cleanup)(void);					   \
+									   \
+FFTW_EXTERN void X(set_timelimit)(double t);				   \
+									   \
+FFTW_EXTERN void X(plan_with_nthreads)(int nthreads);			   \
+FFTW_EXTERN int X(init_threads)(void);					   \
+FFTW_EXTERN void X(cleanup_threads)(void);				   \
+									   \
+FFTW_EXTERN int X(export_wisdom_to_filename)(const char *filename);	   \
+FFTW_EXTERN void X(export_wisdom_to_file)(FILE *output_file);		   \
+FFTW_EXTERN char *X(export_wisdom_to_string)(void);			   \
+FFTW_EXTERN void X(export_wisdom)(X(write_char_func) write_char,   	   \
+                                  void *data);				   \
+FFTW_EXTERN int X(import_system_wisdom)(void);				   \
+FFTW_EXTERN int X(import_wisdom_from_filename)(const char *filename);	   \
+FFTW_EXTERN int X(import_wisdom_from_file)(FILE *input_file);		   \
+FFTW_EXTERN int X(import_wisdom_from_string)(const char *input_string);	   \
+FFTW_EXTERN int X(import_wisdom)(X(read_char_func) read_char, void *data); \
+									   \
+FFTW_EXTERN void X(fprint_plan)(const X(plan) p, FILE *output_file);	   \
+FFTW_EXTERN void X(print_plan)(const X(plan) p);			   \
+FFTW_EXTERN char *X(sprint_plan)(const X(plan) p);			   \
+									   \
+FFTW_EXTERN void *X(malloc)(size_t n);					   \
+FFTW_EXTERN R *X(alloc_real)(size_t n);					   \
+FFTW_EXTERN C *X(alloc_complex)(size_t n);				   \
+FFTW_EXTERN void X(free)(void *p);					   \
+									   \
+FFTW_EXTERN void X(flops)(const X(plan) p,				   \
+                          double *add, double *mul, double *fmas);	   \
+FFTW_EXTERN double X(estimate_cost)(const X(plan) p);			   \
+FFTW_EXTERN double X(cost)(const X(plan) p);				   \
+									   \
+FFTW_EXTERN int X(alignment_of)(R *p);                                     \
+FFTW_EXTERN const char X(version)[];                                       \
+FFTW_EXTERN const char X(cc)[];						   \
+FFTW_EXTERN const char X(codelet_optim)[];
+
+
+/* end of FFTW_DEFINE_API macro */
+
+FFTW_DEFINE_API(FFTW_MANGLE_DOUBLE, double, fftw_complex)
+FFTW_DEFINE_API(FFTW_MANGLE_FLOAT, float, fftwf_complex)
+FFTW_DEFINE_API(FFTW_MANGLE_LONG_DOUBLE, long double, fftwl_complex)
+
+/* __float128 (quad precision) is a gcc extension on i386, x86_64, and ia64
+   for gcc >= 4.6 (compiled in FFTW with --enable-quad-precision) */
+#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) \
+ && !(defined(__ICC) || defined(__INTEL_COMPILER)) \
+ && (defined(__i386__) || defined(__x86_64__) || defined(__ia64__))
+#  if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I)
+/* note: __float128 is a typedef, which is not supported with the _Complex
+         keyword in gcc, so instead we use this ugly __attribute__ version.
+         However, we can't simply pass the __attribute__ version to
+         FFTW_DEFINE_API because the __attribute__ confuses gcc in pointer
+         types.  Hence redefining FFTW_DEFINE_COMPLEX.  Ugh. */
+#    undef FFTW_DEFINE_COMPLEX
+#    define FFTW_DEFINE_COMPLEX(R, C) typedef _Complex float __attribute__((mode(TC))) C
+#  endif
+FFTW_DEFINE_API(FFTW_MANGLE_QUAD, __float128, fftwq_complex)
+#endif
+
+#define FFTW_FORWARD (-1)
+#define FFTW_BACKWARD (+1)
+
+#define FFTW_NO_TIMELIMIT (-1.0)
+
+/* documented flags */
+#define FFTW_MEASURE (0U)
+#define FFTW_DESTROY_INPUT (1U << 0)
+#define FFTW_UNALIGNED (1U << 1)
+#define FFTW_CONSERVE_MEMORY (1U << 2)
+#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */
+#define FFTW_PRESERVE_INPUT (1U << 4) /* cancels FFTW_DESTROY_INPUT */
+#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */
+#define FFTW_ESTIMATE (1U << 6)
+#define FFTW_WISDOM_ONLY (1U << 21)
+
+/* undocumented beyond-guru flags */
+#define FFTW_ESTIMATE_PATIENT (1U << 7)
+#define FFTW_BELIEVE_PCOST (1U << 8)
+#define FFTW_NO_DFT_R2HC (1U << 9)
+#define FFTW_NO_NONTHREADED (1U << 10)
+#define FFTW_NO_BUFFERING (1U << 11)
+#define FFTW_NO_INDIRECT_OP (1U << 12)
+#define FFTW_ALLOW_LARGE_GENERIC (1U << 13) /* NO_LARGE_GENERIC is default */
+#define FFTW_NO_RANK_SPLITS (1U << 14)
+#define FFTW_NO_VRANK_SPLITS (1U << 15)
+#define FFTW_NO_VRECURSE (1U << 16)
+#define FFTW_NO_SIMD (1U << 17)
+#define FFTW_NO_SLOW (1U << 18)
+#define FFTW_NO_FIXED_RADIX_LARGE_N (1U << 19)
+#define FFTW_ALLOW_PRUNING (1U << 20)
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif /* __cplusplus */
+
+#endif /* FFTW3_H */

lib/lattice/Lattice_ET.h

@@ -261,7 +261,6 @@ GridUnopClass(UnaryExp, exp(a));
 GridBinOpClass(BinaryAdd, lhs + rhs);
 GridBinOpClass(BinarySub, lhs - rhs);
 GridBinOpClass(BinaryMul, lhs *rhs);
-GridBinOpClass(BinaryDiv, lhs /rhs);
 
 GridBinOpClass(BinaryAnd, lhs &rhs);
 GridBinOpClass(BinaryOr, lhs | rhs);
@@ -386,7 +385,6 @@ GRID_DEF_UNOP(exp, UnaryExp);
 GRID_DEF_BINOP(operator+, BinaryAdd);
 GRID_DEF_BINOP(operator-, BinarySub);
 GRID_DEF_BINOP(operator*, BinaryMul);
-GRID_DEF_BINOP(operator/, BinaryDiv);
 
 GRID_DEF_BINOP(operator&, BinaryAnd);
 GRID_DEF_BINOP(operator|, BinaryOr);

lib/lattice/Lattice_base.h

@@ -65,6 +65,9 @@ public:
     
 class LatticeExpressionBase {};
 
+template<class T> using Vector = std::vector<T,alignedAllocator<T> >;               // Aligned allocator??
+template<class T> using Matrix = std::vector<std::vector<T,alignedAllocator<T> > >; // Aligned allocator??
+
 template <typename Op, typename T1>                           
 class LatticeUnaryExpression  : public std::pair<Op,std::tuple<T1> > , public LatticeExpressionBase {
  public:
@@ -300,6 +303,17 @@ PARALLEL_FOR_LOOP
         *this = (*this)+r;
         return *this;
     }
+    
+    strong_inline friend Lattice<vobj> operator / (const Lattice<vobj> &lhs,const Lattice<vobj> &rhs){
+        conformable(lhs,rhs);
+        Lattice<vobj> ret(lhs._grid);
+PARALLEL_FOR_LOOP
+        for(int ss=0;ss<lhs._grid->oSites();ss++){
+	  ret._odata[ss] = lhs._odata[ss]*pow(rhs._odata[ss],-1.0);
+        }
+        return ret;
+    };
+
  }; // class Lattice
 
   template<class vobj> std::ostream& operator<< (std::ostream& stream, const Lattice<vobj> &o){

lib/lattice/Lattice_peekpoke.h

@@ -154,7 +154,7 @@ PARALLEL_FOR_LOOP
     template<class vobj,class sobj>
     void peekLocalSite(sobj &s,const Lattice<vobj> &l,std::vector<int> &site){
         
-      GridBase *grid = l._grid;
+      GridBase *grid=l._grid;
 
       typedef typename vobj::scalar_type scalar_type;
       typedef typename vobj::vector_type vector_type;
@@ -164,17 +164,15 @@ PARALLEL_FOR_LOOP
       assert( l.checkerboard== l._grid->CheckerBoard(site));
       assert( sizeof(sobj)*Nsimd == sizeof(vobj));
 
-      static const int words=sizeof(vobj)/sizeof(vector_type);
       int odx,idx;
       idx= grid->iIndex(site);
       odx= grid->oIndex(site);
 
-      scalar_type * vp = (scalar_type *)&l._odata[odx];
-      scalar_type * pt = (scalar_type *)&s;
+      std::vector<sobj> buf(Nsimd);
 
-      for(int w=0;w<words;w++){
-        pt[w] = vp[idx+w*Nsimd];
-      }
+      extract(l._odata[odx],buf);
+      
+      s = buf[idx];
 
       return;
     };
@@ -192,17 +190,18 @@ PARALLEL_FOR_LOOP
       assert( l.checkerboard== l._grid->CheckerBoard(site));
       assert( sizeof(sobj)*Nsimd == sizeof(vobj));
 
-      static const int words=sizeof(vobj)/sizeof(vector_type);
       int odx,idx;
       idx= grid->iIndex(site);
       odx= grid->oIndex(site);
 
-      scalar_type * vp = (scalar_type *)&l._odata[odx];
-      scalar_type * pt = (scalar_type *)&s;
+      std::vector<sobj> buf(Nsimd);
 
-      for(int w=0;w<words;w++){
-        vp[idx+w*Nsimd] = pt[w];
-      }
+      // extract-modify-merge cycle is easiest way and this is not perf critical
+      extract(l._odata[odx],buf);
+      
+      buf[idx] = s;
+
+      merge(l._odata[odx],buf);
 
       return;
     };

lib/lattice/Lattice_reduction.h

@@ -31,87 +31,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define GRID_LATTICE_REDUCTION_H
 
 namespace Grid {
-
-template <class T>
-class ReproducibilityState {
- public:
-  typedef typename T::vector_type vector_type;
-  typedef std::vector<vector_type, alignedAllocator<vector_type> >  sum_type;
-  unsigned int  n_call;
-  bool          do_check;
-  bool          enable_reprocheck;
-  bool          success;
-  unsigned int  interval; 
-  std::vector<sum_type> th_states;
-
-  void reset_counter() { n_call = 0; }
-  void reset() {
-    th_states.clear();
-    do_check          = false;
-    enable_reprocheck = false;
-    success           = true;
-    n_call            = 0;
-  };
-
-  ReproducibilityState():interval(1) { 
-    reset(); 
-  }
-
-  void check(GridBase* grid, sum_type &sumarray){
-    ///////////////////////  Reproducibility section, not threaded on purpouse
-    if (enable_reprocheck) {
-      if (do_check && (n_call % interval) == 0) {
-        for (int thread = 0; thread < sumarray.size(); thread++) {
-          int words = sizeof(sumarray[thread])/sizeof(unsigned char);
-          unsigned char xors[words];
-          bitwise_xor(sumarray[thread], th_states[n_call][thread],xors);
-          // OR all words
-          unsigned char res = 0;
-          for (int w = 0; w < words; w++) res = res | xors[w];
-
-          Grid_unquiesce_nodes();
-          int rank = 0;
-          while (rank < grid->_Nprocessors){
-            if (rank == grid->ThisRank() ){
-              if ( res ) {
-                std::cout << "Reproducibility failure report" << std::endl;
-                grid->PrintRankInfo();
-                std::cout << "Call: "<< n_call << " Thread: " << thread << std::endl;
-                std::cout << "Size of states: " << th_states.size() << std::endl;
-                std::cout << std::setprecision(GRID_REAL_DIGITS+1) << std::scientific;
-                std::cout << "Saved partial sum  : " << th_states[n_call][thread] << std::endl;
-                std::cout << "Current partial sum: " << sumarray[thread] << std::endl;
-                std::cout << "Saved state "  << std::endl; show_binaryrep(th_states[n_call][thread]);
-                std::cout << "Current state" << std::endl; show_binaryrep(sumarray[thread]);
-                std::cout << "XOR result"    << std::endl; show_binaryrep(xors, words);
-              		//std::cout << std::defaultfloat;  //not supported by some compilers
-                std::cout << std::setprecision(6);
-                success = false; 
-              }
-            }
-            rank++;
-            grid->Barrier();
-          }
-          Grid_quiesce_nodes();                         
-        }
-
-      } else if (!do_check)
-      {
-        std::cout << GridLogDebug << "Saving thread state for inner product. Call n. " 
-        << n_call << std::endl;
-        th_states.resize(n_call+1);
-        th_states[n_call].resize(grid->SumArraySize());
-          th_states[n_call] = sumarray;  // save threads state
-          //n_call++;
-        }
-        n_call++;
-      }
-    }
-
-};
-
-
-
 #ifdef GRID_WARN_SUBOPTIMAL
 #warning "Optimisation alert all these reduction loops are NOT threaded "
 #endif     
@@ -120,25 +39,12 @@ class ReproducibilityState {
     // Deterministic Reduction operations
     ////////////////////////////////////////////////////////////////////////////////////////////////////
   template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
-        ReproducibilityState<vobj> repr;
-        ComplexD nrm = innerProduct(arg, arg, repr);
-        return std::real(nrm); 
-    }
-
-    template <class vobj>
-    inline RealD norm2(const Lattice<vobj> &arg, ReproducibilityState<vobj>& rep) {
-      ComplexD nrm = innerProduct(arg, arg, rep);
+    ComplexD nrm = innerProduct(arg,arg);
     return std::real(nrm); 
   }
 
     template<class vobj>
-    inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right){
-      ReproducibilityState<vobj> repr;
-      return innerProduct(left, right, repr);
-    } 
-    
-    template<class vobj>
-    inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right, ReproducibilityState<vobj>& repr) 
+    inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right) 
     {
       typedef typename vobj::scalar_type scalar_type;
       typedef typename vobj::vector_type vector_type;
@@ -151,22 +57,18 @@ class ReproducibilityState {
 	sumarray[i]=zero;
       }
 
-      PARALLEL_FOR_LOOP_STATIC //request statically scheduled threads for reproducibility
+PARALLEL_FOR_LOOP
       for(int thr=0;thr<grid->SumArraySize();thr++){
 	int nwork, mywork, myoff;
 	GridThread::GetWork(left._grid->oSites(),thr,mywork,myoff);
 	
-        decltype(innerProduct(left._odata[0],right._odata[0])) vnrm = zero; // private to thread; sub summation
+	decltype(innerProduct(left._odata[0],right._odata[0])) vnrm=zero; // private to thread; sub summation
         for(int ss=myoff;ss<mywork+myoff; ss++){
 	  vnrm = vnrm + innerProduct(left._odata[ss],right._odata[ss]);
 	}
 	sumarray[thr]=TensorRemove(vnrm) ;
       }
     
-      /////////// Reproducibility
-      repr.check(grid, sumarray);
-      ///////////////////////////
-
       vector_type vvnrm; vvnrm=zero;  // sum across threads
       for(int i=0;i<grid->SumArraySize();i++){
 	vvnrm = vvnrm+sumarray[i];
@@ -212,7 +114,7 @@ class ReproducibilityState {
 	sumarray[i]=zero;
       }
 
-      PARALLEL_FOR_LOOP
+PARALLEL_FOR_LOOP
       for(int thr=0;thr<grid->SumArraySize();thr++){
 	int nwork, mywork, myoff;
 	GridThread::GetWork(grid->oSites(),thr,mywork,myoff);
@@ -244,7 +146,7 @@ class ReproducibilityState {
 
 
 template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<typename vobj::scalar_object> &result,int orthogdim)
-    {
+{
   typedef typename vobj::scalar_object sobj;
   GridBase  *grid = Data._grid;
   assert(grid!=NULL);

lib/lattice/Lattice_rng.h

@@ -31,6 +31,14 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #include <random>
 
+// Have not enable RNG_SPRNG_SHA256 by default yet.
+// Uncomment the following line to see the effect of the new RNG.
+// #define RNG_SPRNG_SHA256
+
+#ifdef RNG_SPRNG_SHA256
+#include "rng/sprng-sha256.h"
+#endif
+
 namespace Grid {
 
 
@@ -110,7 +118,11 @@ namespace Grid {
     int _seeded;
     // One generator per site.
     // Uniform and Gaussian distributions from these generators.
-#ifdef RNG_RANLUX
+#ifdef RNG_SPRNG_SHA256
+    typedef uint32_t      RngStateType;
+    typedef SprngSha256 RngEngine;
+    static const int RngStateCount = 22;
+#elif defined RNG_RANLUX
     typedef uint64_t      RngStateType;
     typedef std::ranlux48 RngEngine;
     static const int RngStateCount = 15;
@@ -273,6 +285,34 @@ namespace Grid {
     }
 
 
+#ifdef RNG_SPRNG_SHA256
+    template<class source> void Seed(source &src)
+    {
+      std::vector<int> gcoor;
+
+      long gsites = _grid->_gsites;
+
+      RngState rs;
+      for (int i = 0; i < 8; ++i) {
+        splitRngState(rs, rs, src());
+      }
+
+      for(long gidx=0;gidx<gsites;gidx++){
+
+        int rank,o_idx,i_idx;
+        _grid->GlobalIndexToGlobalCoor(gidx,gcoor);
+        _grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor);
+
+        int l_idx=generator_idx(o_idx,i_idx);
+
+        if( rank == _grid->ThisRank() ){
+          splitRngState(_generators[l_idx].rs, rs, gidx);
+        }
+      }
+      _seeded=1;
+
+    }
+#else
     // This loop could be made faster to avoid the Ahmdahl by
     // i)  seed generators on each timeslice, for x=y=z=0;
     // ii) seed generators on each z for x=y=0
@@ -297,9 +337,8 @@ namespace Grid {
 
 	int l_idx=generator_idx(o_idx,i_idx);
 	
-	const int num_rand_seed=16;
-	std::vector<int> site_seeds(num_rand_seed);
-	for(int i=0;i<site_seeds.size();i++){
+	std::vector<int> site_seeds(4);
+	for(int i=0;i<4;i++){
 	  site_seeds[i]= ui(pseeder);
 	}
 
@@ -313,6 +352,7 @@ namespace Grid {
       }
       _seeded=1;
     }    
+#endif
 
     //FIXME implement generic IO and create state save/restore
     //void SaveState(const std::string<char> &file);

lib/lattice/rng/rng-state.h

@@ -0,0 +1,353 @@
+// vim: set ts=2 sw=2 expandtab:
+
+// Copyright (c) 2016 Luchang Jin
+// All rights reserved.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#pragma once
+
+#ifndef INCLUDE_RNG_STATE_H
+#define INCLUDE_RNG_STATE_H
+
+#include "show.h"
+
+#ifndef USE_OPENSSL
+#include "sha256.h"
+#else
+#include <openssl/sha.h>
+#endif
+
+#include <stdint.h>
+#include <endian.h>
+#include <cstring>
+#include <cmath>
+#include <cassert>
+#include <string>
+#include <ostream>
+#include <istream>
+#include <vector>
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+namespace CURRENT_DEFAULT_NAMESPACE_NAME {
+#endif
+
+struct RngState;
+
+inline void reset(RngState& rs);
+
+inline void reset(RngState& rs, const std::string& seed);
+
+inline void reset(RngState& rs, const long seed)
+{
+  reset(rs, show(seed));
+}
+
+inline void splitRngState(RngState& rs, const RngState& rs0, const std::string& sindex);
+
+inline void splitRngState(RngState& rs, const RngState& rs0, const long sindex = 0)
+{
+  splitRngState(rs, rs0, show(sindex));
+}
+
+inline uint64_t randGen(RngState& rs);
+
+inline double uRandGen(RngState& rs, const double upper = 1.0, const double lower = 0.0);
+
+inline double gRandGen(RngState& rs, const double sigma = 1.0, const double center = 0.0);
+
+inline void computeHashWithInput(uint32_t hash[8], const RngState& rs, const std::string& input);
+
+struct RngState
+{
+  uint64_t numBytes;
+  uint32_t hash[8];
+  unsigned long index;
+  //
+  uint64_t cache[3];
+  double gaussian;
+  int cacheAvail;
+  bool gaussianAvail;
+  //
+  inline void init()
+  {
+    reset(*this);
+  }
+  //
+  RngState()
+  {
+    init();
+  }
+  RngState(const std::string& seed)
+  {
+    reset(*this, seed);
+  }
+  RngState(const long seed)
+  {
+    reset(*this, seed);
+  }
+  RngState(const RngState& rs0, const std::string& sindex)
+  {
+    splitRngState(*this, rs0, sindex);
+  }
+  RngState(const RngState& rs0, const long sindex)
+  {
+    splitRngState(*this, rs0, sindex);
+  }
+  //
+  RngState split(const std::string& sindex)
+  {
+    RngState rs(*this, sindex);
+    return rs;
+  }
+  RngState split(const long sindex)
+  {
+    RngState rs(*this, sindex);
+    return rs;
+  }
+};
+
+const size_t RNG_STATE_NUM_OF_INT32 = 2 + 8 + 2 + 3 * 2 + 2 + 1 + 1;
+
+inline uint64_t patchTwoUint32(const uint32_t a, const uint32_t b)
+{
+  return (uint64_t)a << 32 | (uint64_t)b;
+}
+
+inline void splitTwoUint32(uint32_t& a, uint32_t& b, const uint64_t x)
+{
+  b = (uint32_t)x;
+  a = (uint32_t)(x >> 32);
+  assert(x == patchTwoUint32(a, b));
+}
+
+inline void exportRngState(uint32_t* v, const RngState& rs)
+{
+  assert(22 == RNG_STATE_NUM_OF_INT32);
+  splitTwoUint32(v[0], v[1], rs.numBytes);
+  for (int i = 0; i < 8; ++i) {
+    v[2 + i] = rs.hash[i];
+  }
+  splitTwoUint32(v[10], v[11], rs.index);
+  for (int i = 0; i < 3; ++i) {
+    splitTwoUint32(v[12 + i * 2], v[12 + i * 2 + 1], rs.cache[i]);
+  }
+  union {
+    double d;
+    uint64_t l;
+  } g;
+  g.d = rs.gaussian;
+  splitTwoUint32(v[18], v[19], g.l);
+  v[20] = rs.cacheAvail;
+  v[21] = rs.gaussianAvail;
+}
+
+inline void importRngState(RngState& rs, const uint32_t* v)
+{
+  assert(22 == RNG_STATE_NUM_OF_INT32);
+  rs.numBytes = patchTwoUint32(v[0], v[1]);
+  for (int i = 0; i < 8; ++i) {
+    rs.hash[i] = v[2 + i];
+  }
+  rs.index = patchTwoUint32(v[10], v[11]);
+  for (int i = 0; i < 3; ++i) {
+    rs.cache[i] = patchTwoUint32(v[12 + i * 2], v[12 + i * 2 + 1]);
+  }
+  union {
+    double d;
+    uint64_t l;
+  } g;
+  g.l = patchTwoUint32(v[18], v[19]);
+  rs.gaussian = g.d;
+  rs.cacheAvail = v[20];
+  rs.gaussianAvail = v[21];
+}
+
+inline void exportRngState(std::vector<uint32_t>& v, const RngState& rs)
+{
+  v.resize(RNG_STATE_NUM_OF_INT32);
+  exportRngState(v.data(), rs);
+}
+
+inline void importRngState(RngState& rs, const std::vector<uint32_t>& v)
+{
+  assert(RNG_STATE_NUM_OF_INT32 == v.size());
+  importRngState(rs, v.data());
+}
+
+inline std::ostream& operator<<(std::ostream& os, const RngState& rs)
+{
+  std::vector<uint32_t> v(RNG_STATE_NUM_OF_INT32);
+  exportRngState(v, rs);
+  for (size_t i = 0; i < v.size() - 1; ++i) {
+    os << v[i] << " ";
+  }
+  os << v.back();
+  return os;
+}
+
+inline std::istream& operator>>(std::istream& is, RngState& rs)
+{
+  std::vector<uint32_t> v(RNG_STATE_NUM_OF_INT32);
+  for (size_t i = 0; i < v.size(); ++i) {
+    is >> v[i];
+  }
+  importRngState(rs, v);
+  return is;
+}
+
+inline std::string show(const RngState& rs)
+{
+  return shows(rs);
+}
+
+inline bool operator==(const RngState& rs1, const RngState& rs2)
+{
+  return 0 == memcmp(&rs1, &rs2, sizeof(RngState));
+}
+
+inline void reset(RngState& rs)
+{
+  std::memset(&rs, 0, sizeof(RngState));
+  rs.numBytes = 0;
+  rs.hash[0] = 0;
+  rs.hash[1] = 0;
+  rs.hash[2] = 0;
+  rs.hash[3] = 0;
+  rs.hash[4] = 0;
+  rs.hash[5] = 0;
+  rs.hash[6] = 0;
+  rs.hash[7] = 0;
+  rs.index = 0;
+  rs.cache[0] = 0;
+  rs.cache[1] = 0;
+  rs.cache[2] = 0;
+  rs.gaussian = 0.0;
+  rs.cacheAvail = 0;
+  rs.gaussianAvail = false;
+}
+
+inline void reset(RngState& rs, const std::string& seed)
+{
+  reset(rs);
+  splitRngState(rs, rs, seed);
+}
+
+inline void computeHashWithInput(uint32_t hash[8], const RngState& rs, const std::string& input)
+{
+  std::string data(32, ' ');
+  for (int i = 0; i < 8; ++i) {
+    data[i*4 + 0] = (rs.hash[i] >> 24) & 0xFF;
+    data[i*4 + 1] = (rs.hash[i] >> 16) & 0xFF;
+    data[i*4 + 2] = (rs.hash[i] >>  8) & 0xFF;
+    data[i*4 + 3] =  rs.hash[i]        & 0xFF;
+  }
+  data += input;
+#ifndef USE_OPENSSL
+  sha256::computeHash(hash, (const uint8_t*)data.c_str(), data.length());
+#else
+  {
+    uint8_t rawHash[32];
+    SHA256((unsigned char*)data.c_str(), data.length(), rawHash);
+    for (int i = 0; i < 8; ++i) {
+      hash[i] = (((uint32_t)rawHash[i*4 + 0]) << 24)
+              + (((uint32_t)rawHash[i*4 + 1]) << 16)
+              + (((uint32_t)rawHash[i*4 + 2]) <<  8)
+              + ( (uint32_t)rawHash[i*4 + 3]);
+    }
+  }
+#endif
+}
+
+inline void splitRngState(RngState& rs, const RngState& rs0, const std::string& sindex)
+  // produce a new rng ``rs'' uniquely identified by ``rs0'' and ``sindex''
+  // will not affect old rng ``rs0''
+  // the function should behave correctly even if ``rs'' is actually ``rs0''
+{
+  std::string input = ssprintf("[%lu] {%s}", rs0.index, sindex.c_str());
+  rs.numBytes = rs0.numBytes + 64 * ((32 + input.length() + 1 + 8 - 1) / 64 + 1);
+  computeHashWithInput(rs.hash, rs0, input);
+  rs.index = 0;
+  rs.cache[0] = 0;
+  rs.cache[1] = 0;
+  rs.cache[2] = 0;
+  rs.gaussian = 0.0;
+  rs.cacheAvail = 0;
+  rs.gaussianAvail = false;
+}
+
+inline uint64_t randGen(RngState& rs)
+{
+  assert(0 <= rs.cacheAvail && rs.cacheAvail <= 3);
+  rs.index += 1;
+  if (rs.cacheAvail > 0) {
+    rs.cacheAvail -= 1;
+    uint64_t r = rs.cache[rs.cacheAvail];
+    rs.cache[rs.cacheAvail] = 0;
+    return r;
+  } else {
+    uint32_t hash[8];
+    computeHashWithInput(hash, rs, ssprintf("[%lu]", rs.index));
+    rs.cache[0] = patchTwoUint32(hash[0], hash[1]);
+    rs.cache[1] = patchTwoUint32(hash[2], hash[3]);
+    rs.cache[2] = patchTwoUint32(hash[4], hash[5]);
+    rs.cacheAvail = 3;
+    return patchTwoUint32(hash[6], hash[7]);
+  }
+}
+
+inline double uRandGen(RngState& rs, const double upper, const double lower)
+{
+  uint64_t u = randGen(rs);
+  const double fac = 1.0 / (256.0 * 256.0 * 256.0 * 256.0) / (256.0 * 256.0 * 256.0 * 256.0);
+  return u * fac * (upper - lower) + lower;
+}
+
+inline double gRandGen(RngState& rs, const double sigma, const double center)
+{
+  rs.index += 1;
+  if (rs.gaussianAvail) {
+    rs.gaussianAvail = false;
+    return rs.gaussian * sigma + center;
+  } else {
+    // pick 2 uniform numbers in the square extending from
+    // -1 to 1 in each direction, see if they are in the
+    // unit circle, and try again if they are not.
+    int num_try = 1;
+    double v1, v2, rsq;
+    do {
+      v1 = uRandGen(rs, 1.0, -1.0);
+      v2 = uRandGen(rs, 1.0, -1.0);
+      if ((num_try % 1000)==0) {
+        printf("gRandGen : WARNING num_try=%d v1=%e v2=%e\n",num_try,v1,v2);
+      }
+      rsq = v1*v1 + v2*v2;
+      num_try++;
+    } while ((num_try < 10000) && (rsq >= 1.0 || rsq == 0));
+    if (num_try > 9999) {
+      printf("gRandGen : WARNING failed after 10000 tries (corrupted RNG?), returning ridiculous numbers (1e+10)\n");
+      return 1e+10;
+    }
+    double fac = std::sqrt(-2.0 * std::log(rsq)/rsq);
+    rs.gaussian = v1 * fac;
+    rs.gaussianAvail = true;
+    return v2 * fac * sigma + center;
+  }
+}
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+}
+#endif
+
+#endif

lib/lattice/rng/sha256.h

@@ -0,0 +1,348 @@
+// vim: set ts=2 sw=2 expandtab:
+
+// Copyright (c) 2016 Luchang Jin
+// All rights reserved.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Code within namespace sha256 are originally from Stephan Brumme.
+// see http://create.stephan-brumme.com/disclaimer.html
+
+#pragma once
+
+#include <stdint.h>
+#include <endian.h>
+#include <cstring>
+#include <cmath>
+#include <cassert>
+#include <string>
+#include <ostream>
+#include <istream>
+#include <vector>
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+namespace CURRENT_DEFAULT_NAMESPACE_NAME {
+#endif
+
+namespace sha256 {
+
+  const size_t BlockSize = 512 / 8;
+
+  const size_t HashBytes = 32;
+
+  const size_t HashValues = HashBytes / 4;
+
+  inline uint32_t rotate(uint32_t a, uint32_t c)
+  {
+    return (a >> c) | (a << (32 - c));
+  }
+
+  inline uint32_t swap(uint32_t x)
+  {
+    return (x >> 24) |
+      ((x >>  8) & 0x0000FF00) |
+      ((x <<  8) & 0x00FF0000) |
+      (x << 24);
+  }
+
+  inline uint32_t f1(uint32_t e, uint32_t f, uint32_t g)
+  // mix functions for processBlock()
+  {
+    uint32_t term1 = rotate(e, 6) ^ rotate(e, 11) ^ rotate(e, 25);
+    uint32_t term2 = (e & f) ^ (~e & g); //(g ^ (e & (f ^ g)))
+    return term1 + term2;
+  }
+
+  inline uint32_t f2(uint32_t a, uint32_t b, uint32_t c)
+  // mix functions for processBlock()
+  {
+    uint32_t term1 = rotate(a, 2) ^ rotate(a, 13) ^ rotate(a, 22);
+    uint32_t term2 = ((a | b) & c) | (a & b); //(a & (b ^ c)) ^ (b & c);
+    return term1 + term2;
+  }
+
+  inline void processBlock(uint32_t newHash[8], const uint32_t oldHash[8], const uint8_t data[64])
+    // process 64 bytes of data
+    // newHash and oldHash and be the same
+  {
+    // get last hash
+    uint32_t a = oldHash[0];
+    uint32_t b = oldHash[1];
+    uint32_t c = oldHash[2];
+    uint32_t d = oldHash[3];
+    uint32_t e = oldHash[4];
+    uint32_t f = oldHash[5];
+    uint32_t g = oldHash[6];
+    uint32_t h = oldHash[7];
+    // data represented as 16x 32-bit words
+    const uint32_t* input = (uint32_t*) data;
+    // convert to big endian
+    uint32_t words[64];
+    int i;
+    for (i = 0; i < 16; i++) {
+#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN)
+      words[i] =      input[i];
+#else
+      words[i] = swap(input[i]);
+#endif
+    }
+    uint32_t x,y; // temporaries
+    // first round
+    x = h + f1(e,f,g) + 0x428a2f98 + words[ 0]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0x71374491 + words[ 1]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0xb5c0fbcf + words[ 2]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0xe9b5dba5 + words[ 3]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0x3956c25b + words[ 4]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0x59f111f1 + words[ 5]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0x923f82a4 + words[ 6]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0xab1c5ed5 + words[ 7]; y = f2(b,c,d); e += x; a = x + y;
+    // secound round
+    x = h + f1(e,f,g) + 0xd807aa98 + words[ 8]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0x12835b01 + words[ 9]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0x243185be + words[10]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0x550c7dc3 + words[11]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0x72be5d74 + words[12]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0x80deb1fe + words[13]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0x9bdc06a7 + words[14]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0xc19bf174 + words[15]; y = f2(b,c,d); e += x; a = x + y;
+    // extend to 24 words
+    for (; i < 24; i++)
+      words[i] = words[i-16] +
+        (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+        words[i-7] +
+        (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+    // third round
+    x = h + f1(e,f,g) + 0xe49b69c1 + words[16]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0xefbe4786 + words[17]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0x0fc19dc6 + words[18]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0x240ca1cc + words[19]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0x2de92c6f + words[20]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0x4a7484aa + words[21]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0x5cb0a9dc + words[22]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0x76f988da + words[23]; y = f2(b,c,d); e += x; a = x + y;
+    // extend to 32 words
+    for (; i < 32; i++)
+      words[i] = words[i-16] +
+        (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+        words[i-7] +
+        (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+    // fourth round
+    x = h + f1(e,f,g) + 0x983e5152 + words[24]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0xa831c66d + words[25]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0xb00327c8 + words[26]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0xbf597fc7 + words[27]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0xc6e00bf3 + words[28]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0xd5a79147 + words[29]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0x06ca6351 + words[30]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0x14292967 + words[31]; y = f2(b,c,d); e += x; a = x + y;
+    // extend to 40 words
+    for (; i < 40; i++)
+      words[i] = words[i-16] +
+        (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+        words[i-7] +
+        (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+    // fifth round
+    x = h + f1(e,f,g) + 0x27b70a85 + words[32]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0x2e1b2138 + words[33]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0x4d2c6dfc + words[34]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0x53380d13 + words[35]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0x650a7354 + words[36]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0x766a0abb + words[37]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0x81c2c92e + words[38]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0x92722c85 + words[39]; y = f2(b,c,d); e += x; a = x + y;
+    // extend to 48 words
+    for (; i < 48; i++)
+      words[i] = words[i-16] +
+        (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+        words[i-7] +
+        (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+    // sixth round
+    x = h + f1(e,f,g) + 0xa2bfe8a1 + words[40]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0xa81a664b + words[41]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0xc24b8b70 + words[42]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0xc76c51a3 + words[43]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0xd192e819 + words[44]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0xd6990624 + words[45]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0xf40e3585 + words[46]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0x106aa070 + words[47]; y = f2(b,c,d); e += x; a = x + y;
+    // extend to 56 words
+    for (; i < 56; i++)
+      words[i] = words[i-16] +
+        (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+        words[i-7] +
+        (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+    // seventh round
+    x = h + f1(e,f,g) + 0x19a4c116 + words[48]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0x1e376c08 + words[49]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0x2748774c + words[50]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0x34b0bcb5 + words[51]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0x391c0cb3 + words[52]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0x4ed8aa4a + words[53]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0x5b9cca4f + words[54]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0x682e6ff3 + words[55]; y = f2(b,c,d); e += x; a = x + y;
+    // extend to 64 words
+    for (; i < 64; i++)
+      words[i] = words[i-16] +
+        (rotate(words[i-15],  7) ^ rotate(words[i-15], 18) ^ (words[i-15] >>  3)) +
+        words[i-7] +
+        (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10));
+    // eigth round
+    x = h + f1(e,f,g) + 0x748f82ee + words[56]; y = f2(a,b,c); d += x; h = x + y;
+    x = g + f1(d,e,f) + 0x78a5636f + words[57]; y = f2(h,a,b); c += x; g = x + y;
+    x = f + f1(c,d,e) + 0x84c87814 + words[58]; y = f2(g,h,a); b += x; f = x + y;
+    x = e + f1(b,c,d) + 0x8cc70208 + words[59]; y = f2(f,g,h); a += x; e = x + y;
+    x = d + f1(a,b,c) + 0x90befffa + words[60]; y = f2(e,f,g); h += x; d = x + y;
+    x = c + f1(h,a,b) + 0xa4506ceb + words[61]; y = f2(d,e,f); g += x; c = x + y;
+    x = b + f1(g,h,a) + 0xbef9a3f7 + words[62]; y = f2(c,d,e); f += x; b = x + y;
+    x = a + f1(f,g,h) + 0xc67178f2 + words[63]; y = f2(b,c,d); e += x; a = x + y;
+    // update hash
+    newHash[0] = a + oldHash[0];
+    newHash[1] = b + oldHash[1];
+    newHash[2] = c + oldHash[2];
+    newHash[3] = d + oldHash[3];
+    newHash[4] = e + oldHash[4];
+    newHash[5] = f + oldHash[5];
+    newHash[6] = g + oldHash[6];
+    newHash[7] = h + oldHash[7];
+  }
+
+  inline void processInput(
+      uint32_t hash[8],
+      const uint32_t oldHash[8], const uint64_t numBytes,
+      const uint8_t* input, const size_t inputSize)
+    // process final block, less than 64 bytes
+    // newHash and oldHash and be the same
+  {
+    // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte
+    // - append "1" bit to message
+    // - append "0" bits until message length in bit mod 512 is 448
+    // - append length as 64 bit integer
+    // process initial parts of input
+    std::memmove(hash, oldHash, 32);
+    const int nBlocks = inputSize / 64;
+    for (int i = 0; i < nBlocks; ++i) {
+      processBlock(hash, hash, input + i * 64);
+    }
+    // initialize buffer from input
+    const size_t bufferSize = inputSize - nBlocks * 64;
+    unsigned char buffer[BlockSize];
+    std::memcpy(buffer, input + nBlocks * 64, bufferSize);
+    // number of bits
+    size_t paddedLength = bufferSize * 8;
+    // plus one bit set to 1 (always appended)
+    paddedLength++;
+    // number of bits must be (numBits % 512) = 448
+    size_t lower11Bits = paddedLength & 511;
+    if (lower11Bits <= 448) {
+      paddedLength +=       448 - lower11Bits;
+    } else {
+      paddedLength += 512 + 448 - lower11Bits;
+    }
+    // convert from bits to bytes
+    paddedLength /= 8;
+    // only needed if additional data flows over into a second block
+    unsigned char extra[BlockSize];
+    // append a "1" bit, 128 => binary 10000000
+    if (bufferSize < BlockSize) {
+      buffer[bufferSize] = 128;
+    } else {
+      extra[0] = 128;
+    }
+    size_t i;
+    for (i = bufferSize + 1; i < BlockSize; i++) {
+      buffer[i] = 0;
+    }
+    for (; i < paddedLength; i++) {
+      extra[i - BlockSize] = 0;
+    }
+    // add message length in bits as 64 bit number
+    uint64_t msgBits = 8 * (numBytes + inputSize);
+    // find right position
+    unsigned char* addLength;
+    if (paddedLength < BlockSize) {
+      addLength = buffer + paddedLength;
+    } else {
+      addLength = extra + paddedLength - BlockSize;
+    }
+    // must be big endian
+    *addLength++ = (unsigned char)((msgBits >> 56) & 0xFF);
+    *addLength++ = (unsigned char)((msgBits >> 48) & 0xFF);
+    *addLength++ = (unsigned char)((msgBits >> 40) & 0xFF);
+    *addLength++ = (unsigned char)((msgBits >> 32) & 0xFF);
+    *addLength++ = (unsigned char)((msgBits >> 24) & 0xFF);
+    *addLength++ = (unsigned char)((msgBits >> 16) & 0xFF);
+    *addLength++ = (unsigned char)((msgBits >>  8) & 0xFF);
+    *addLength   = (unsigned char)( msgBits        & 0xFF);
+    // process blocks
+    processBlock(hash, hash, buffer);
+    // flowed over into a second block ?
+    if (paddedLength > BlockSize) {
+      processBlock(hash, hash, extra);
+    }
+  }
+
+  inline void setInitialHash(uint32_t hash[8])
+  {
+    hash[0] = 0x6a09e667;
+    hash[1] = 0xbb67ae85;
+    hash[2] = 0x3c6ef372;
+    hash[3] = 0xa54ff53a;
+    hash[4] = 0x510e527f;
+    hash[5] = 0x9b05688c;
+    hash[6] = 0x1f83d9ab;
+    hash[7] = 0x5be0cd19;
+  }
+
+  inline void computeHash(uint32_t hash[8], const void* data, const size_t size)
+  {
+    uint32_t initHash[8];
+    setInitialHash(initHash);
+    processInput(hash, initHash, 0, (const uint8_t*)data, size);
+  }
+
+  inline void rawHashFromHash(uint8_t rawHash[HashBytes], const uint32_t hash[HashValues])
+  {
+    uint8_t* current = rawHash;
+    for (size_t i = 0; i < HashValues; i++) {
+      *current++ = (hash[i] >> 24) & 0xFF;
+      *current++ = (hash[i] >> 16) & 0xFF;
+      *current++ = (hash[i] >>  8) & 0xFF;
+      *current++ =  hash[i]        & 0xFF;
+    }
+  }
+
+  inline std::string showRawHash(const uint8_t rawHash[HashBytes])
+  {
+    std::string result;
+    result.reserve(2 * HashBytes);
+    for (size_t i = 0; i < HashBytes; i++) {
+      static const char dec2hex[16+1] = "0123456789abcdef";
+      result += dec2hex[(rawHash[i] >> 4) & 15];
+      result += dec2hex[ rawHash[i]       & 15];
+    }
+    return result;
+  }
+
+  inline std::string showHash(const uint32_t hash[8])
+  {
+    unsigned char rawHash[HashBytes];
+    rawHashFromHash(rawHash, hash);
+    return showRawHash(rawHash);
+  }
+
+}
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+}
+#endif

lib/lattice/rng/show.h

@@ -0,0 +1,125 @@
+// vim: set ts=2 sw=2 expandtab:
+
+// Copyright (c) 2014 Luchang Jin
+// All rights reserved.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#pragma once
+
+#ifndef INCLUDE_SHOW_H
+#define INCLUDE_SHOW_H
+
+#include <sstream>
+#include <string>
+#include <cstdarg>
+#include <cstring>
+#include <cstdlib>
+#include <cstdio>
+#include <sstream>
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+namespace CURRENT_DEFAULT_NAMESPACE_NAME {
+#endif
+
+inline std::string vssprintf(const char* fmt, va_list args)
+{
+  std::string str;
+  char* cstr;
+  vasprintf(&cstr, fmt, args);
+  str += std::string(cstr);
+  std::free(cstr);
+  return str;
+}
+
+inline std::string ssprintf(const char* fmt, ...)
+{
+  va_list args;
+  va_start(args, fmt);
+  return vssprintf(fmt, args);
+}
+
+inline std::string show()
+{
+  return "";
+}
+
+inline std::string show(const int& x)
+{
+  return ssprintf("%d", x);
+}
+
+inline std::string show(const unsigned int& x)
+{
+  return ssprintf("%u", x);
+}
+
+inline std::string show(const long& x)
+{
+  return ssprintf("%ld", x);
+}
+
+inline std::string show(const unsigned long& x)
+{
+  return ssprintf("%lu", x);
+}
+
+inline std::string show(const double& x)
+{
+  return ssprintf("%24.17E", x);
+}
+
+inline std::string show(const bool& x)
+{
+  return x ? "true" : "false";
+}
+
+inline std::string show(const std::string& x)
+{
+  std::ostringstream out;
+  out << x;
+  return out.str();
+}
+
+template <class T>
+std::string shows(const T& x)
+{
+  std::ostringstream out;
+  out << x;
+  return out.str();
+}
+
+template <class T>
+T& reads(T& x, const std::string& str)
+{
+  std::istringstream in(str);
+  in >> x;
+  return x;
+}
+
+inline void fdisplay(FILE* fp, const std::string& str)
+{
+  fprintf(fp, "%s", str.c_str());
+}
+
+inline void fdisplayln(FILE* fp, const std::string& str)
+{
+  fprintf(fp, "%s\n", str.c_str());
+}
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+}
+#endif
+
+#endif

lib/lattice/rng/sprng-sha256.h

@@ -0,0 +1,115 @@
+// vim: set ts=2 sw=2 expandtab:
+
+// Copyright (c) 2016 Luchang Jin
+// All rights reserved.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#pragma once
+
+#ifndef INCLUDE_SPRNG_SHA256_H
+#define INCLUDE_SPRNG_SHA256_H
+
+#include "rng-state.h"
+
+#include <array>
+#include <cstring>
+#include <ostream>
+#include <istream>
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+namespace CURRENT_DEFAULT_NAMESPACE_NAME {
+#endif
+
+struct SprngSha256
+{
+  RngState rs;
+  //
+  using result_type = uint64_t;
+  //
+  static constexpr result_type default_seed = 0;
+  //
+  explicit SprngSha256(result_type val = default_seed)
+  {
+    seed(val);
+  }
+  template<typename Sseq, typename = typename
+    std::enable_if<!std::is_same<Sseq, SprngSha256>::value>
+    ::type>
+  explicit SprngSha256(Sseq& q)
+  {
+    seed(q);
+  }
+  //
+  static constexpr result_type min()
+  {
+    return 0;
+  }
+  //
+  static constexpr result_type max()
+  {
+    return UINT64_MAX;
+  }
+  //
+  void seed(result_type val = default_seed)
+  {
+    reset(rs, (long)val);
+  }
+  template <class Sseq>
+  typename std::enable_if<std::is_class<Sseq>::value>::type
+  seed(Sseq& q)
+  {
+    std::array<uint32_t, 8> seq;
+    q.generate(seq.begin(), seq.end());
+    reset(rs);
+    for (size_t i = 0; i < seq.size(); ++i) {
+      splitRngState(rs, rs, seq[i]);
+    }
+  }
+  //
+  result_type operator()()
+  {
+    return randGen(rs);
+  }
+  //
+  void discard(unsigned long long z)
+  {
+    for (unsigned long long i = 0; i < z; ++i) {
+      randGen(rs);
+    }
+  }
+};
+
+inline std::ostream& operator<<(std::ostream& os, const SprngSha256& ss)
+{
+  os << ss.rs;
+  return os;
+}
+
+inline std::istream& operator>>(std::istream& is, SprngSha256& ss)
+{
+  is >> ss.rs;
+  return is;
+}
+
+inline bool operator==(const SprngSha256& ss1, const SprngSha256& ss2)
+{
+  return ss1.rs == ss2.rs;
+}
+
+#ifdef CURRENT_DEFAULT_NAMESPACE_NAME
+}
+#endif
+
+#endif

lib/parallelIO/BinaryIO.h

@@ -457,7 +457,7 @@ class BinaryIO {
     // available (how short sighted is that?)
     //////////////////////////////////////////////////////////
     Umu = zero;
-    static uint32_t csum; csum=0;
+    static uint32_t csum=0;
     fobj fileObj;
     static sobj siteObj; // Static to place in symmetric region for SHMEM
 

lib/qcd/action/Actions.h

@@ -195,7 +195,6 @@ typedef WilsonTMFermion<WilsonImplD> WilsonTMFermionD;
 typedef DomainWallFermion<WilsonImplR> DomainWallFermionR;
 typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;
 typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
-
 typedef MobiusFermion<WilsonImplR> MobiusFermionR;
 typedef MobiusFermion<WilsonImplF> MobiusFermionF;
 typedef MobiusFermion<WilsonImplD> MobiusFermionD;
@@ -204,20 +203,6 @@ typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
 typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
 typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
 
-// Ls vectorised 
-typedef DomainWallFermion<DomainWallVec5dImplR> DomainWallFermionVec5dR;
-typedef DomainWallFermion<DomainWallVec5dImplF> DomainWallFermionVec5dF;
-typedef DomainWallFermion<DomainWallVec5dImplD> DomainWallFermionVec5dD;
-
-typedef MobiusFermion<DomainWallVec5dImplR> MobiusFermionVec5dR;
-typedef MobiusFermion<DomainWallVec5dImplF> MobiusFermionVec5dF;
-typedef MobiusFermion<DomainWallVec5dImplD> MobiusFermionVec5dD;
-
-typedef ZMobiusFermion<ZDomainWallVec5dImplR> ZMobiusFermionVec5dR;
-typedef ZMobiusFermion<ZDomainWallVec5dImplF> ZMobiusFermionVec5dF;
-typedef ZMobiusFermion<ZDomainWallVec5dImplD> ZMobiusFermionVec5dD;
-
-
 typedef ScaledShamirFermion<WilsonImplR> ScaledShamirFermionR;
 typedef ScaledShamirFermion<WilsonImplF> ScaledShamirFermionF;
 typedef ScaledShamirFermion<WilsonImplD> ScaledShamirFermionD;
@@ -269,7 +254,6 @@ typedef MobiusFermion<GparityWilsonImplF> GparityMobiusFermionF;
 typedef MobiusFermion<GparityWilsonImplD> GparityMobiusFermionD;
 
 
-
   }}
 ///////////////////////////////////////////////////////////////////////////////
 // G5 herm -- this has to live in QCD since dirac matrix is not in the broader sector of code

lib/qcd/action/fermion/CayleyFermion5D.cc

@@ -50,74 +50,6 @@ namespace QCD {
    mass(_mass)
  { }
 
-template<class Impl>  
-void CayleyFermion5D<Impl>::Dminus(const FermionField &psi, FermionField &chi)
-{
-  int Ls=this->Ls;
-  FermionField tmp(psi._grid);
-
-  this->DW(psi,tmp,DaggerNo);
-
-  for(int s=0;s<Ls;s++){
-    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp,s,s);// chi = (1-c[s] D_W) psi
-  }
-}
-
-
-template<class Impl> void CayleyFermion5D<Impl>::CayleyReport(void)
-{
-  this->Report();
-  std::vector<int> latt = GridDefaultLatt();          
-  RealD volume = this->Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt[mu];
-  RealD NP     = this->_FourDimGrid->_Nprocessors;
-  if ( M5Dcalls > 0 ) {
-    std::cout << GridLogMessage << "#### M5D calls report " << std::endl;
-    std::cout << GridLogMessage << "CayleyFermion5D Number of M5D Calls     : " << M5Dcalls   << std::endl;
-    std::cout << GridLogMessage << "CayleyFermion5D ComputeTime/Calls       : " << M5Dtime / M5Dcalls << " us" << std::endl;
-
-    // Flops = 6.0*(Nc*Ns) *Ls*vol
-    RealD mflops = 6.0*12*volume*M5Dcalls/M5Dtime/2; // 2 for red black counting
-    std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per rank       : " << mflops/NP << std::endl;
-  }
-
-  if ( MooeeInvCalls > 0 ) {
-
-    std::cout << GridLogMessage << "#### MooeeInv calls report " << std::endl;
-    std::cout << GridLogMessage << "CayleyFermion5D Number of MooeeInv Calls     : " << MooeeInvCalls   << std::endl;
-    std::cout << GridLogMessage << "CayleyFermion5D ComputeTime/Calls            : " << MooeeInvTime / MooeeInvCalls << " us" << std::endl;
-
-    // Flops = 9*12*Ls*vol/2
-    RealD mflops = 9.0*12*volume*MooeeInvCalls/MooeeInvTime/2; // 2 for red black counting
-    std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per rank       : " << mflops/NP << std::endl;
-  }
-
-}
-template<class Impl> void CayleyFermion5D<Impl>::CayleyZeroCounters(void)
-{
-  this->ZeroCounters();
-  M5Dflops=0;
-  M5Dcalls=0;
-  M5Dtime=0;
-  MooeeInvFlops=0;
-  MooeeInvCalls=0;
-  MooeeInvTime=0;
-}
-
-
-template<class Impl>  
-void CayleyFermion5D<Impl>::DminusDag(const FermionField &psi, FermionField &chi)
-{
-  int Ls=this->Ls;
-  FermionField tmp(psi._grid);
-
-  this->DW(psi,tmp,DaggerYes);
-
-  for(int s=0;s<Ls;s++){
-    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp,s,s);// chi = (1-c[s] D_W) psi
-  }
-}
 template<class Impl>  
 void CayleyFermion5D<Impl>::M5D   (const FermionField &psi, FermionField &chi)
 {

lib/qcd/action/fermion/CayleyFermion5D.h

@@ -56,9 +56,6 @@ namespace Grid {
       virtual void   M5D   (const FermionField &psi, FermionField &chi);
       virtual void   M5Ddag(const FermionField &psi, FermionField &chi);
 
-      virtual void   Dminus(const FermionField &psi, FermionField &chi);
-      virtual void   DminusDag(const FermionField &psi, FermionField &chi);
-
       /////////////////////////////////////////////////////
       // Instantiate different versions depending on Impl
       /////////////////////////////////////////////////////
@@ -120,19 +117,6 @@ namespace Grid {
 		      GridRedBlackCartesian &FourDimRedBlackGrid,
 		      RealD _mass,RealD _M5,const ImplParams &p= ImplParams());
 
-      
-
-     void CayleyReport(void);
-     void CayleyZeroCounters(void);
-
-     double M5Dflops;
-     double M5Dcalls;
-     double M5Dtime;
-
-     double MooeeInvFlops;
-     double MooeeInvCalls;
-     double MooeeInvTime;
-
     protected:
       void SetCoefficientsZolotarev(RealD zolohi,Approx::zolotarev_data *zdata,RealD b,RealD c);
       void SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c);

lib/qcd/action/fermion/CayleyFermion5Dcache.cc

@@ -51,9 +51,6 @@ void CayleyFermion5D<Impl>::M5D(const FermionField &psi,
   GridBase *grid=psi._grid;
   assert(phi.checkerboard == psi.checkerboard);
   chi.checkerboard=psi.checkerboard;
-  // Flops = 6.0*(Nc*Ns) *Ls*vol
-  M5Dcalls++;
-  M5Dtime-=usecond();
 PARALLEL_FOR_LOOP
   for(int ss=0;ss<grid->oSites();ss+=Ls){ // adds Ls
     for(int s=0;s<Ls;s++){
@@ -79,7 +76,6 @@ PARALLEL_FOR_LOOP
       }
     }
   }
-  M5Dtime+=usecond();
 }
 
 template<class Impl>  
@@ -95,9 +91,6 @@ void CayleyFermion5D<Impl>::M5Ddag(const FermionField &psi,
   assert(phi.checkerboard == psi.checkerboard);
   chi.checkerboard=psi.checkerboard;
 
-  // Flops = 6.0*(Nc*Ns) *Ls*vol
-  M5Dcalls++;
-  M5Dtime-=usecond();
 PARALLEL_FOR_LOOP
   for(int ss=0;ss<grid->oSites();ss+=Ls){ // adds Ls
     auto tmp = psi._odata[0];
@@ -123,7 +116,6 @@ PARALLEL_FOR_LOOP
       }
     }
   }
-  M5Dtime+=usecond();
 }
 
 template<class Impl>
@@ -134,14 +126,10 @@ void CayleyFermion5D<Impl>::MooeeInv    (const FermionField &psi, FermionField &
 
   chi.checkerboard=psi.checkerboard;
 
-  MooeeInvCalls++;
-  MooeeInvTime-=usecond();
-
 PARALLEL_FOR_LOOP
   for(int ss=0;ss<grid->oSites();ss+=Ls){ // adds Ls
     auto tmp = psi._odata[0];
 
-    // flops = 12*2*Ls + 12*2*Ls + 3*12*Ls + 12*2*Ls  = 12*Ls * (9) = 108*Ls flops
     // Apply (L^{\prime})^{-1}
     chi[ss]=psi[ss]; // chi[0]=psi[0]
     for(int s=1;s<Ls;s++){
@@ -167,9 +155,6 @@ PARALLEL_FOR_LOOP
       chi[ss+s] = chi[ss+s] - uee[s]*tmp;
     }
   }
-
-  MooeeInvTime+=usecond();
-
 }
 
 template<class Impl>
@@ -181,8 +166,6 @@ void CayleyFermion5D<Impl>::MooeeInvDag (const FermionField &psi, FermionField &
   assert(psi.checkerboard == psi.checkerboard);
   chi.checkerboard=psi.checkerboard;
 
-  MooeeInvCalls++;
-  MooeeInvTime-=usecond();
 
 PARALLEL_FOR_LOOP
   for(int ss=0;ss<grid->oSites();ss+=Ls){ // adds Ls
@@ -214,9 +197,6 @@ PARALLEL_FOR_LOOP
       chi[ss+s] = chi[ss+s] - lee[s]*tmp;
     }
   }
-
-  MooeeInvTime+=usecond();
-
 }
 
 #ifdef CAYLEY_DPERP_CACHE

lib/qcd/action/fermion/CayleyFermion5Dvec.cc

@@ -60,7 +60,7 @@ void CayleyFermion5D<Impl>::M5D(const FermionField &psi,
   GridBase *grid=psi._grid;
   int Ls   = this->Ls;
   int LLs  = grid->_rdimensions[0];
-  const int nsimd= Simd::Nsimd();
+  int nsimd= Simd::Nsimd();
 
   Vector<iSinglet<Simd> > u(LLs);
   Vector<iSinglet<Simd> > l(LLs);
@@ -86,15 +86,9 @@ void CayleyFermion5D<Impl>::M5D(const FermionField &psi,
     d_p[ss] = diag[s];
   }}
 
-
-  M5Dcalls++;
-  M5Dtime-=usecond();
-
-  assert(Nc==3);
-
 PARALLEL_FOR_LOOP
   for(int ss=0;ss<grid->oSites();ss+=LLs){ // adds LLs
-#if 0
+
     alignas(64) SiteHalfSpinor hp;
     alignas(64) SiteHalfSpinor hm;
     alignas(64) SiteSpinor fp;
@@ -111,113 +105,16 @@ PARALLEL_FOR_LOOP
       if ( vp<=v ) rotate(hp,hp,1);
       if ( vm>=v ) rotate(hm,hm,nsimd-1);
 
-	hp=0.5*hp;
-        hm=0.5*hm;
-
+      hp=hp*0.5;
+      hm=hm*0.5;
       spRecon5m(fp,hp);
       spRecon5p(fm,hm);
 
-	chi[ss+v] = d[v]*phi[ss+v];
-	chi[ss+v] = chi[ss+v]     +u[v]*fp;
+      chi[ss+v] = d[v]*phi[ss+v]+u[v]*fp;
       chi[ss+v] = chi[ss+v]     +l[v]*fm;
 
     }
-#else
-      for(int v=0;v<LLs;v++){
-
-	vprefetch(psi[ss+v+LLs]);
-	//	vprefetch(phi[ss+v+LLs]);
-
-	int vp= (v==LLs-1) ? 0     : v+1;
-	int vm= (v==0    ) ? LLs-1 : v-1;
-	
-	Simd hp_00 = psi[ss+vp]()(2)(0); 
-	Simd hp_01 = psi[ss+vp]()(2)(1); 
-	Simd hp_02 = psi[ss+vp]()(2)(2); 
-	Simd hp_10 = psi[ss+vp]()(3)(0); 
-	Simd hp_11 = psi[ss+vp]()(3)(1); 
-	Simd hp_12 = psi[ss+vp]()(3)(2); 
-	
-	Simd hm_00 = psi[ss+vm]()(0)(0); 
-	Simd hm_01 = psi[ss+vm]()(0)(1); 
-	Simd hm_02 = psi[ss+vm]()(0)(2); 
-	Simd hm_10 = psi[ss+vm]()(1)(0); 
-	Simd hm_11 = psi[ss+vm]()(1)(1); 
-	Simd hm_12 = psi[ss+vm]()(1)(2); 
-
-	//	if ( ss==0) std::cout << " hp_00 " <<hp_00<<std::endl;
-	//	if ( ss==0) std::cout << " hm_00 " <<hm_00<<std::endl;
-
-	if ( vp<=v ) {
-	  hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
-	  hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
-	  hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
-	  hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
-	  hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
-	  hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
   }
-	if ( vm>=v ) {
-	  hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
-	  hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
-	  hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
-	  hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
-	  hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
-	  hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
-	}
-
-	/*
-	if ( ss==0) std::cout << " dphi_00 " <<d[v]()()() * phi[ss+v]()(0)(0) <<std::endl;
-	if ( ss==0) std::cout << " dphi_10 " <<d[v]()()() * phi[ss+v]()(1)(0) <<std::endl;
-	if ( ss==0) std::cout << " dphi_20 " <<d[v]()()() * phi[ss+v]()(2)(0) <<std::endl;
-	if ( ss==0) std::cout << " dphi_30 " <<d[v]()()() * phi[ss+v]()(3)(0) <<std::endl;
-	*/	
-	Simd p_00  = d[v]()()() * phi[ss+v]()(0)(0)  + l[v]()()()*hm_00; 
-	Simd p_01  = d[v]()()() * phi[ss+v]()(0)(1)  + l[v]()()()*hm_01; 
-	Simd p_02  = d[v]()()() * phi[ss+v]()(0)(2)  + l[v]()()()*hm_02; 
-	Simd p_10  = d[v]()()() * phi[ss+v]()(1)(0)  + l[v]()()()*hm_10; 
-	Simd p_11  = d[v]()()() * phi[ss+v]()(1)(1)  + l[v]()()()*hm_11; 
-	Simd p_12  = d[v]()()() * phi[ss+v]()(1)(2)  + l[v]()()()*hm_12; 
-	Simd p_20  = d[v]()()() * phi[ss+v]()(2)(0)  + u[v]()()()*hp_00; 
-	Simd p_21  = d[v]()()() * phi[ss+v]()(2)(1)  + u[v]()()()*hp_01; 
-	Simd p_22  = d[v]()()() * phi[ss+v]()(2)(2)  + u[v]()()()*hp_02;  
-	Simd p_30  = d[v]()()() * phi[ss+v]()(3)(0)  + u[v]()()()*hp_10; 
-	Simd p_31  = d[v]()()() * phi[ss+v]()(3)(1)  + u[v]()()()*hp_11; 
-	Simd p_32  = d[v]()()() * phi[ss+v]()(3)(2)  + u[v]()()()*hp_12; 
-
-	
-	//	if ( ss==0){
-	/*
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(0)(0) << " bad "<<p_00<<" diff "<<chi[ss+v]()(0)(0)-p_00<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(0)(1) << " bad "<<p_01<<" diff "<<chi[ss+v]()(0)(1)-p_01<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(0)(2) << " bad "<<p_02<<" diff "<<chi[ss+v]()(0)(2)-p_02<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(1)(0) << " bad "<<p_10<<" diff "<<chi[ss+v]()(1)(0)-p_10<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(1)(1) << " bad "<<p_11<<" diff "<<chi[ss+v]()(1)(1)-p_11<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(1)(2) << " bad "<<p_12<<" diff "<<chi[ss+v]()(1)(2)-p_12<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(2)(0) << " bad "<<p_20<<" diff "<<chi[ss+v]()(2)(0)-p_20<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(2)(1) << " bad "<<p_21<<" diff "<<chi[ss+v]()(2)(1)-p_21<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(2)(2) << " bad "<<p_22<<" diff "<<chi[ss+v]()(2)(2)-p_22<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(3)(0) << " bad "<<p_30<<" diff "<<chi[ss+v]()(3)(0)-p_30<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(3)(1) << " bad "<<p_31<<" diff "<<chi[ss+v]()(3)(1)-p_31<<std::endl;
-	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(3)(2) << " bad "<<p_32<<" diff "<<chi[ss+v]()(3)(2)-p_32<<std::endl;
-	}
-	*/
-	vstream(chi[ss+v]()(0)(0),p_00);
-	vstream(chi[ss+v]()(0)(1),p_01);
-	vstream(chi[ss+v]()(0)(2),p_02);
-	vstream(chi[ss+v]()(1)(0),p_10);
-	vstream(chi[ss+v]()(1)(1),p_11);
-	vstream(chi[ss+v]()(1)(2),p_12);
-	vstream(chi[ss+v]()(2)(0),p_20);
-	vstream(chi[ss+v]()(2)(1),p_21);
-	vstream(chi[ss+v]()(2)(2),p_22);
-	vstream(chi[ss+v]()(3)(0),p_30);
-	vstream(chi[ss+v]()(3)(1),p_31);
-	vstream(chi[ss+v]()(3)(2),p_32);
-
-      }
-#endif
-  }
-  M5Dtime+=usecond();
 }
 
 template<class Impl>  
@@ -257,8 +154,6 @@ void CayleyFermion5D<Impl>::M5Ddag(const FermionField &psi,
     d_p[ss] = diag[s];
   }}
 
-  M5Dcalls++;
-  M5Dtime-=usecond();
 PARALLEL_FOR_LOOP
   for(int ss=0;ss<grid->oSites();ss+=LLs){ // adds LLs
 
@@ -288,8 +183,8 @@ PARALLEL_FOR_LOOP
 
     }
   }
-  M5Dtime+=usecond();
 }
+
 template<class Impl>
 void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv)
 {
@@ -355,11 +250,13 @@ void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField
     }
   }
   
-  MooeeInvCalls++;
-  MooeeInvTime-=usecond();
   // Dynamic allocate on stack to get per thread without serialised heap acces
-#pragma omp parallel  
-  {
+PARALLEL_FOR_LOOP
+  for(auto site=0;site<vol;site++){
+    
+    //    SiteHalfSpinor *SitePplus =(SiteHalfSpinor *) alloca(LLs*sizeof(SiteHalfSpinor));
+    //    SiteHalfSpinor *SitePminus=(SiteHalfSpinor *) alloca(LLs*sizeof(SiteHalfSpinor));
+    //    SiteSpinor     *SiteChi   =(SiteSpinor *)     alloca(LLs*sizeof(SiteSpinor));
 
     Vector<SiteHalfSpinor> SitePplus(LLs);
     Vector<SiteHalfSpinor> SitePminus(LLs);
@@ -370,9 +267,6 @@ void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField
     SiteHalfSpinor BcastP;
     SiteHalfSpinor BcastM;
 
-#pragma omp for 
-  for(auto site=0;site<vol;site++){
-
     for(int s=0;s<LLs;s++){
       int lex = s+LLs*site;
       spProj5p(SitePplus[s] ,psi[lex]);
@@ -400,8 +294,6 @@ void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField
       chi[lex] = SiteChi[s]*0.5;
     }
   }
-  }
-  MooeeInvTime+=usecond();
 }
 
 INSTANTIATE_DPERP(DomainWallVec5dImplD);

lib/qcd/action/fermion/DomainWallFermion.h

@@ -42,10 +42,6 @@ namespace Grid {
      INHERIT_IMPL_TYPES(Impl);
     public:
 
-      void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { 
-	this->MomentumSpacePropagatorHt(out,in,_m);
-      };
-
       virtual void   Instantiatable(void) {};
       // Constructors
       DomainWallFermion(GaugeField &_Umu,
@@ -55,7 +51,6 @@ namespace Grid {
 			GridRedBlackCartesian &FourDimRedBlackGrid,
 			RealD _mass,RealD _M5,const ImplParams &p= ImplParams()) : 
 
-
       CayleyFermion5D<Impl>(_Umu,
 			    FiveDimGrid,
 			    FiveDimRedBlackGrid,

lib/qcd/action/fermion/FermionOperator.h

@@ -91,20 +91,6 @@ namespace Grid {
       virtual void  Mdiag  (const FermionField &in, FermionField &out) { Mooee(in,out);};   // Same as Mooee applied to both CB's
       virtual void  Mdir   (const FermionField &in, FermionField &out,int dir,int disp)=0;   // case by case Wilson, Clover, Cayley, ContFrac, PartFrac
 
-
-      virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { assert(0);};
-
-      virtual void  FreePropagator(const FermionField &in,FermionField &out,RealD mass) { 
-	FFT theFFT((GridCartesian *) in._grid);
-
-	FermionField in_k(in._grid);
-	FermionField prop_k(in._grid);
-
-	theFFT.FFT_all_dim(in_k,in,FFT::forward);
-        this->MomentumSpacePropagator(prop_k,in_k,mass);
-	theFFT.FFT_all_dim(out,prop_k,FFT::backward);
-      };
-
       ///////////////////////////////////////////////
       // Updates gauge field during HMC
       ///////////////////////////////////////////////

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -33,7 +33,8 @@ directory
 #define GRID_QCD_FERMION_OPERATOR_IMPL_H
 
 namespace Grid {
-namespace QCD {
+
+  namespace QCD {
 
 
     //////////////////////////////////////////////
@@ -107,14 +108,13 @@ namespace QCD {
     INHERIT_GIMPL_TYPES(Base)	 \
     INHERIT_FIMPL_TYPES(Base)
     
-  /////////////////////////////////////////////////////////////////////////////
+    ///////
     // Single flavour four spinors with colour index
-  /////////////////////////////////////////////////////////////////////////////
+    ///////
     template <class S, class Representation = FundamentalRepresentation,class _Coeff_t = RealD >
-  class WilsonImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
-
+    class WilsonImpl
+      : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
     public:
-
       static const int Dimension = Representation::Dimension;
       typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
       
@@ -124,6 +124,7 @@ namespace QCD {
       const bool LsVectorised=false;
       typedef _Coeff_t Coeff_t;
 
+
       INHERIT_GIMPL_TYPES(Gimpl);
       
       template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
@@ -157,7 +158,8 @@ namespace QCD {
       }
       
       template <class ref>
-    inline void loadLinkElement(Simd &reg, ref &memory) {
+      inline void loadLinkElement(Simd &reg,
+				  ref &memory) {
 	reg = memory;
       }
       
@@ -200,12 +202,11 @@ namespace QCD {
       }
     };
 
-  ////////////////////////////////////////////////////////////////////////////////////
+    ///////
     // Single flavour four spinors with colour index, 5d redblack
-  ////////////////////////////////////////////////////////////////////////////////////
-
-template<class S,int Nrepresentation=Nc,class _Coeff_t = RealD>
-class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { 
+    ///////
+    template<class S,int Nrepresentation=Nc,class _Coeff_t = RealD>
+    class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { 
     public:
       
       static const int Dimension = Nrepresentation;
@@ -226,9 +227,12 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
       typedef Lattice<SiteSpinor> FermionField;
       
       // Make the doubled gauge field a *scalar*
-  typedef iImplDoubledGaugeField<typename Simd::scalar_type>  SiteDoubledGaugeField;  // This is a scalar
-  typedef iImplGaugeField<typename Simd::scalar_type>         SiteScalarGaugeField;  // scalar
-  typedef iImplGaugeLink<typename Simd::scalar_type>          SiteScalarGaugeLink;  // scalar
+      typedef iImplDoubledGaugeField<typename Simd::scalar_type>
+      SiteDoubledGaugeField;  // This is a scalar
+      typedef iImplGaugeField<typename Simd::scalar_type>
+      SiteScalarGaugeField;  // scalar
+      typedef iImplGaugeLink<typename Simd::scalar_type>
+      SiteScalarGaugeLink;  // scalar
       
       typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
       
@@ -246,7 +250,6 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
       inline void loadLinkElement(Simd &reg, ref &memory) {
 	vsplat(reg, memory);
       }
-
       inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
 			   const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
 			   StencilImpl &St) {
@@ -259,8 +262,8 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
 	mult(&phi(), &UU(), &chi());
       }
       
-  inline void DoubleStore(GridBase *GaugeGrid, DoubledGaugeField &Uds,const GaugeField &Umu) 
-  {
+      inline void DoubleStore(GridBase *GaugeGrid, DoubledGaugeField &Uds,
+			      const GaugeField &Umu) {
 	SiteScalarGaugeField ScalarUmu;
 	SiteDoubledGaugeField ScalarUds;
 	
@@ -286,25 +289,25 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
 	}
       }
       
-  inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,FermionField &A, int mu) 
-  {
+      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,
+				FermionField &A, int mu) {
 	assert(0);
       }
       
-  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,FermionField &Atilde, int mu) 
-  {
+      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,
+				FermionField &Atilde, int mu) {
 	assert(0);
       }
-};
+    };
     
     ////////////////////////////////////////////////////////////////////////////////////////
     // Flavour doubled spinors; is Gparity the only? what about C*?
     ////////////////////////////////////////////////////////////////////////////////////////
     
-template <class S, int Nrepresentation,class _Coeff_t = RealD>
-class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
+    template <class S, int Nrepresentation,class _Coeff_t = RealD>
+    class GparityWilsonImpl
+      : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
     public:
-
       static const int Dimension = Nrepresentation;
 
       const bool LsVectorised=false;
@@ -314,9 +317,15 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
       
       INHERIT_GIMPL_TYPES(Gimpl);
       
- template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
- template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
- template <typename vtype> using iImplDoubledGaugeField = iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
+      template <typename vtype>
+      using iImplSpinor =
+      iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
+      template <typename vtype>
+      using iImplHalfSpinor =
+	iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
+      template <typename vtype>
+      using iImplDoubledGaugeField =
+	iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
       
       typedef iImplSpinor<Simd> SiteSpinor;
       typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
@@ -332,6 +341,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
       
       ImplParams Params;
 
+
       GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
 
       bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
@@ -341,7 +351,6 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
       inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
 			   const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
 			   StencilImpl &St) {
-
 	typedef SiteHalfSpinor vobj;
 	typedef typename SiteHalfSpinor::scalar_object sobj;
 	
@@ -410,6 +419,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 
       inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
       {
+	
 	conformable(Uds._grid,GaugeGrid);
 	conformable(Umu._grid,GaugeGrid);
 	
@@ -419,6 +429,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 	
 	Lattice<iScalar<vInteger> > coor(GaugeGrid);
 	
+	
 	for(int mu=0;mu<Nd;mu++){
 	  
 	  LatticeCoordinate(coor,mu);
@@ -432,7 +443,8 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 	    Uconj = where(coor==neglink,-Uconj,Uconj);
 	  }
 	  
-PARALLEL_FOR_LOOP
+	  
+	  PARALLEL_FOR_LOOP
 	    for(auto ss=U.begin();ss<U.end();ss++){
 	      Uds[ss](0)(mu) = U[ss]();
 	      Uds[ss](1)(mu) = Uconj[ss]();
@@ -446,7 +458,7 @@ PARALLEL_FOR_LOOP
 	    Utmp = where(coor==0,Uconj,Utmp);
 	  }
 	  
-PARALLEL_FOR_LOOP
+	  PARALLEL_FOR_LOOP
 	    for(auto ss=U.begin();ss<U.end();ss++){
 	      Uds[ss](0)(mu+4) = Utmp[ss]();
 	    }
@@ -456,7 +468,7 @@ PARALLEL_FOR_LOOP
 	    Utmp = where(coor==0,U,Utmp);
 	  }
 	  
-PARALLEL_FOR_LOOP
+	  PARALLEL_FOR_LOOP
 	    for(auto ss=U.begin();ss<U.end();ss++){
 	      Uds[ss](1)(mu+4) = Utmp[ss]();
 	    }
@@ -465,13 +477,13 @@ PARALLEL_FOR_LOOP
       }
       
       
- inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A, int mu) {
-
+      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,
+				FermionField &A, int mu) {
 	// DhopDir provides U or Uconj depending on coor/flavour.
 	GaugeLinkField link(mat._grid);
 	// use lorentz for flavour as hack.
 	auto tmp = TraceIndex<SpinIndex>(outerProduct(Btilde, A));
-PARALLEL_FOR_LOOP
+	PARALLEL_FOR_LOOP
 	  for (auto ss = tmp.begin(); ss < tmp.end(); ss++) {
 	    link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1));
 	  }
@@ -479,13 +491,13 @@ PARALLEL_FOR_LOOP
 	return;
       }
       
- inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde, int mu) {
-
+      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,
+				FermionField &Atilde, int mu) {
 	int Ls = Btilde._grid->_fdimensions[0];
 	
 	GaugeLinkField tmp(mat._grid);
 	tmp = zero;
-PARALLEL_FOR_LOOP
+	PARALLEL_FOR_LOOP
 	  for (int ss = 0; ss < tmp._grid->oSites(); ss++) {
 	    for (int s = 0; s < Ls; s++) {
 	      int sF = s + Ls * ss;
@@ -496,13 +508,13 @@ PARALLEL_FOR_LOOP
 	PokeIndex<LorentzIndex>(mat, tmp, mu);
 	return;
       }
-
-};
+    };
 
     typedef WilsonImpl<vComplex,  FundamentalRepresentation > WilsonImplR;   // Real.. whichever prec
     typedef WilsonImpl<vComplexF, FundamentalRepresentation > WilsonImplF;  // Float
     typedef WilsonImpl<vComplexD, FundamentalRepresentation > WilsonImplD;  // Double
 
+
     typedef WilsonImpl<vComplex,  FundamentalRepresentation, ComplexD > ZWilsonImplR; // Real.. whichever prec
     typedef WilsonImpl<vComplexF, FundamentalRepresentation, ComplexD > ZWilsonImplF; // Float
     typedef WilsonImpl<vComplexD, FundamentalRepresentation, ComplexD > ZWilsonImplD; // Double
@@ -523,10 +535,9 @@ PARALLEL_FOR_LOOP
     typedef DomainWallVec5dImpl<vComplexF,Nc,ComplexD> ZDomainWallVec5dImplF; // Float
     typedef DomainWallVec5dImpl<vComplexD,Nc,ComplexD> ZDomainWallVec5dImplD; // Double
 
- typedef GparityWilsonImpl<vComplex , Nc> GparityWilsonImplR;  // Real.. whichever prec
+    typedef GparityWilsonImpl<vComplex, Nc>  GparityWilsonImplR;  // Real.. whichever prec
     typedef GparityWilsonImpl<vComplexF, Nc> GparityWilsonImplF;  // Float
     typedef GparityWilsonImpl<vComplexD, Nc> GparityWilsonImplD;  // Double
-
-}}
-
+}
+}
 #endif

lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h

@@ -42,10 +42,6 @@ namespace Grid {
      INHERIT_IMPL_TYPES(Impl);
     public:
 
-     void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { 
-       this->MomentumSpacePropagatorHw(out,in,_m);
-     };
-
       // Constructors
     OverlapWilsonCayleyTanhFermion(GaugeField &_Umu,
 				   GridCartesian         &FiveDimGrid,

lib/qcd/action/fermion/WilsonFermion.cc

@@ -101,7 +101,6 @@ void WilsonFermion<Impl>::Meooe(const FermionField &in, FermionField &out) {
     DhopOE(in, out, DaggerNo);
   }
 }
-
 template <class Impl>
 void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out) {
   if (in.checkerboard == Odd) {
@@ -111,86 +110,31 @@ void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out) {
   }
 }
 
-  template <class Impl>
-  void WilsonFermion<Impl>::Mooee(const FermionField &in, FermionField &out) {
+template <class Impl>
+void WilsonFermion<Impl>::Mooee(const FermionField &in, FermionField &out) {
   out.checkerboard = in.checkerboard;
   typename FermionField::scalar_type scal(4.0 + mass);
   out = scal * in;
-  }
+}
 
-  template <class Impl>
-  void WilsonFermion<Impl>::MooeeDag(const FermionField &in, FermionField &out) {
+template <class Impl>
+void WilsonFermion<Impl>::MooeeDag(const FermionField &in, FermionField &out) {
   out.checkerboard = in.checkerboard;
   Mooee(in, out);
-  }
+}
 
-  template<class Impl>
-  void WilsonFermion<Impl>::MooeeInv(const FermionField &in, FermionField &out) {
+template <class Impl>
+void WilsonFermion<Impl>::MooeeInv(const FermionField &in, FermionField &out) {
   out.checkerboard = in.checkerboard;
-    out = (1.0/(4.0+mass))*in;
-  }
+  out = (1.0 / (4.0 + mass)) * in;
+}
 
-  template<class Impl>
-  void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in, FermionField &out) {
+template <class Impl>
+void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in,
+                                      FermionField &out) {
   out.checkerboard = in.checkerboard;
-    MooeeInv(in,out);
-  }
-
-  template<class Impl>
-  void WilsonFermion<Impl>::MomentumSpacePropagator(FermionField &out, const FermionField &in,RealD _m) {
-
-    // what type LatticeComplex 
-    conformable(_grid,out._grid);
-
-    typedef typename FermionField::vector_type vector_type;
-    typedef typename FermionField::scalar_type ScalComplex;
-
-    typedef Lattice<iSinglet<vector_type> > LatComplex;
-
-    Gamma::GammaMatrix Gmu [] = {
-      Gamma::GammaX,
-      Gamma::GammaY,
-      Gamma::GammaZ,
-      Gamma::GammaT
-    };
-
-    std::vector<int> latt_size   = _grid->_fdimensions;
-
-    FermionField   num  (_grid); num  = zero;
-    LatComplex    wilson(_grid); wilson= zero;
-    LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
-
-    LatComplex denom(_grid); denom= zero;
-    LatComplex kmu(_grid); 
-    ScalComplex ci(0.0,1.0);
-    // momphase = n * 2pi / L
-    for(int mu=0;mu<Nd;mu++) {
-
-      LatticeCoordinate(kmu,mu);
-
-      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
-
-      kmu = TwoPiL * kmu;
-
-      wilson = wilson + 2.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
-
-      num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);    // derivative term
-
-      denom=denom + sin(kmu)*sin(kmu);
-    }
-
-    wilson = wilson + _m;     // 2 sin^2 k/2 + m
-
-    num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
-
-    denom= denom+wilson*wilson; // sin^2 k + (2 sin^2 k/2 + m)^2
-
-    denom= one/denom;
-
-    out = num*denom; // [ -i gmu sin k + 2 sin^2 k/2 + m] / [ sin^2 k + (2 sin^2 k/2 + m)^2 ]
-
-  }
- 
+  MooeeInv(in, out);
+}
 
 ///////////////////////////////////
 // Internal
@@ -222,7 +166,7 @@ void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
     ////////////////////////
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < B._grid->oSites(); sss++) {
-      Kernels::DiracOptDhopDir(st, U, st.CommBuf(), sss, sss, B, Btilde, mu,
+      Kernels::DiracOptDhopDir(st, U, st.comm_buf, sss, sss, B, Btilde, mu,
                                gamma);
     }
 
@@ -333,7 +277,7 @@ void WilsonFermion<Impl>::DhopDirDisp(const FermionField &in, FermionField &out,
 
   PARALLEL_FOR_LOOP
   for (int sss = 0; sss < in._grid->oSites(); sss++) {
-    Kernels::DiracOptDhopDir(Stencil, Umu, Stencil.CommBuf(), sss, sss, in, out,
+    Kernels::DiracOptDhopDir(Stencil, Umu, Stencil.comm_buf, sss, sss, in, out,
                              dirdisp, gamma);
   }
 };
@@ -351,13 +295,13 @@ void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
   if (dag == DaggerYes) {
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < in._grid->oSites(); sss++) {
-      Kernels::DiracOptDhopSiteDag(st, lo, U, st.CommBuf(), sss, sss, 1, 1, in,
+      Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sss, sss, 1, 1, in,
                                    out);
     }
   } else {
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < in._grid->oSites(); sss++) {
-      Kernels::DiracOptDhopSite(st, lo, U, st.CommBuf(), sss, sss, 1, 1, in,
+      Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sss, sss, 1, 1, in,
                                 out);
     }
   }

lib/qcd/action/fermion/WilsonFermion.h

@@ -78,15 +78,16 @@ class WilsonFermion : public WilsonKernels<Impl>, public WilsonFermionStatic {
   virtual void MooeeInv(const FermionField &in, FermionField &out);
   virtual void MooeeInvDag(const FermionField &in, FermionField &out);
 
-  virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _mass) ;
-
   ////////////////////////
   // Derivative interface
   ////////////////////////
   // Interface calls an internal routine
-  void DhopDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-  void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-  void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
+  void DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V,
+                 int dag);
+  void DhopDerivOE(GaugeField &mat, const FermionField &U,
+                   const FermionField &V, int dag);
+  void DhopDerivEO(GaugeField &mat, const FermionField &U,
+                   const FermionField &V, int dag);
 
   ///////////////////////////////////////////////////////////////
   // non-hermitian hopping term; half cb or both

lib/qcd/action/fermion/WilsonFermion5D.cc

@@ -185,19 +185,18 @@ void WilsonFermion5D<Impl>::Report(void)
   if ( DhopCalls > 0 ) {
     std::cout << GridLogMessage << "#### Dhop calls report " << std::endl;
     std::cout << GridLogMessage << "WilsonFermion5D Number of Dhop Calls     : " << DhopCalls  << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " << DhopCommTime<< " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : " << DhopCommTime / DhopCalls << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : " << DhopComputeTime << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : " << DhopComputeTime / DhopCalls << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " << DhopCommTime
+              << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : "
+              << DhopCommTime / DhopCalls << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : "
+              << DhopComputeTime << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : "
+              << DhopComputeTime / DhopCalls << " us" << std::endl;
 
-    RealD mflops = 1344*volume*DhopCalls/DhopComputeTime/2; // 2 for red black counting
+    RealD mflops = 1344*volume*DhopCalls/DhopComputeTime;
     std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per rank       : " << mflops/NP << std::endl;
-
-    RealD Fullmflops = 1344*volume*DhopCalls/(DhopComputeTime+DhopCommTime)/2; // 2 for red black counting
-    std::cout << GridLogMessage << "Average mflops/s per call (full)         : " << Fullmflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per rank (full): " << Fullmflops/NP << std::endl;
-
+    std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
 
    }
 
@@ -211,18 +210,18 @@ void WilsonFermion5D<Impl>::Report(void)
   std::cout << GridLogMessage << "WilsonFermion5D Total Dhop Compute time  : " <<DerivDhopComputeTime <<" us"<<std::endl;
   std::cout << GridLogMessage << "WilsonFermion5D Dhop ComputeTime/Calls   : " <<DerivDhopComputeTime/DerivCalls<<" us" <<std::endl;
 
+
+
   RealD mflops = 144*volume*DerivCalls/DerivDhopComputeTime;
   std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
   std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
 
-    RealD Fullmflops = 144*volume*DerivCalls/(DerivDhopComputeTime+DerivCommTime)/2; // 2 for red black counting
-    std::cout << GridLogMessage << "Average mflops/s per call (full)         : " << Fullmflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per node (full): " << Fullmflops/NP << std::endl;  }
+  }
 
   if (DerivCalls > 0 || DhopCalls > 0){
-    std::cout << GridLogMessage << "WilsonFermion5D Stencil"    <<std::endl;  Stencil.Report();
+  std::cout << GridLogMessage << "WilsonFermion5D Stencil"<<std::endl;  Stencil.Report();
   std::cout << GridLogMessage << "WilsonFermion5D StencilEven"<<std::endl;  StencilEven.Report();
-    std::cout << GridLogMessage << "WilsonFermion5D StencilOdd" <<std::endl;  StencilOdd.Report();
+  std::cout << GridLogMessage << "WilsonFermion5D StencilOdd"<<std::endl;  StencilOdd.Report();
   }
 }
 
@@ -276,7 +275,7 @@ PARALLEL_FOR_LOOP
     for(int s=0;s<Ls;s++){
       int sU=ss;
       int sF = s+Ls*sU; 
-      Kernels::DiracOptDhopDir(Stencil,Umu,Stencil.CommBuf(),sF,sU,in,out,dirdisp,gamma);
+      Kernels::DiracOptDhopDir(Stencil,Umu,Stencil.comm_buf,sF,sU,in,out,dirdisp,gamma);
     }
   }
 };
@@ -328,7 +327,8 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
         assert(sF < B._grid->oSites());
         assert(sU < U._grid->oSites());
 
-        Kernels::DiracOptDhopDir(st, U, st.CommBuf(), sF, sU, B, Btilde, mu, gamma);
+        Kernels::DiracOptDhopDir(st, U, st.comm_buf, sF, sU, B, Btilde, mu,
+                                 gamma);
 
         ////////////////////////////
         // spin trace outer product
@@ -342,7 +342,7 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
 }
 
 template<class Impl>
-void WilsonFermion5D<Impl>::DhopDeriv(GaugeField &mat,
+void WilsonFermion5D<Impl>::DhopDeriv(      GaugeField &mat,
               const FermionField &A,
               const FermionField &B,
               int dag)
@@ -396,6 +396,7 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
            DoubledGaugeField & U,
            const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls++;
   //  assert((dag==DaggerNo) ||(dag==DaggerYes));
   Compressor compressor(dag);
 
@@ -412,24 +413,26 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
     for (int ss = 0; ss < U._grid->oSites(); ss++) {
       int sU = ss;
       int sF = LLs * sU;
-      Kernels::DiracOptDhopSiteDag(st, lo, U, st.CommBuf(), sF, sU, LLs, 1, in, out);
+      Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in,
+                                   out);
     }
 #ifdef AVX512
   } else if (stat.is_init() ) {
 
     int nthreads;
     stat.start();
-#pragma omp parallel
+    #pragma omp parallel
     {
-#pragma omp master
+    #pragma omp master
     nthreads = omp_get_num_threads();
     int mythread = omp_get_thread_num();
     stat.enter(mythread);
-#pragma omp for nowait
-    for(int ss=0;ss<U._grid->oSites();ss++) {
+    #pragma omp for nowait
+   for(int ss=0;ss<U._grid->oSites();ss++)
+    {
        int sU=ss;
        int sF=LLs*sU;
-      Kernels::DiracOptDhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out);
+       Kernels::DiracOptDhopSite(st,lo,U,st.comm_buf,sF,sU,LLs,1,in,out);
      }
     stat.exit(mythread);
     }
@@ -440,7 +443,8 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
     for (int ss = 0; ss < U._grid->oSites(); ss++) {
       int sU = ss;
       int sF = LLs * sU;
-      Kernels::DiracOptDhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out);
+      Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in,
+                                out);
     }
   }
   DhopComputeTime+=usecond();
@@ -450,7 +454,6 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopOE(const FermionField &in, FermionField &out,int dag)
 {
-  DhopCalls++;
   conformable(in._grid,FermionRedBlackGrid());    // verifies half grid
   conformable(in._grid,out._grid); // drops the cb check
 
@@ -462,7 +465,6 @@ void WilsonFermion5D<Impl>::DhopOE(const FermionField &in, FermionField &out,int
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
 {
-  DhopCalls++;
   conformable(in._grid,FermionRedBlackGrid());    // verifies half grid
   conformable(in._grid,out._grid); // drops the cb check
 
@@ -474,7 +476,6 @@ void WilsonFermion5D<Impl>::DhopEO(const FermionField &in, FermionField &out,int
 template<class Impl>
 void WilsonFermion5D<Impl>::Dhop(const FermionField &in, FermionField &out,int dag)
 {
-  DhopCalls+=2;
   conformable(in._grid,FermionGrid()); // verifies full grid
   conformable(in._grid,out._grid);
 
@@ -490,148 +491,6 @@ void WilsonFermion5D<Impl>::DW(const FermionField &in, FermionField &out,int dag
   axpy(out,4.0-M5,in,out);
 }
 
-template<class Impl>
-void WilsonFermion5D<Impl>::MomentumSpacePropagatorHt(FermionField &out,const FermionField &in, RealD mass) 
-{
-  // what type LatticeComplex 
-  GridBase *_grid = _FourDimGrid;
-  conformable(_grid,out._grid);
-  
-  typedef typename FermionField::vector_type vector_type;
-  typedef typename FermionField::scalar_type ScalComplex;
-  typedef iSinglet<ScalComplex> Tcomplex;
-  typedef Lattice<iSinglet<vector_type> > LatComplex;
-  
-  Gamma::GammaMatrix Gmu [] = {
-    Gamma::GammaX,
-    Gamma::GammaY,
-    Gamma::GammaZ,
-    Gamma::GammaT
-  };
-
-  std::vector<int> latt_size   = _grid->_fdimensions;
-
-  
-  FermionField   num  (_grid); num  = zero;
-
-  LatComplex    sk(_grid);  sk = zero;
-  LatComplex    sk2(_grid); sk2= zero;
-  LatComplex    W(_grid); W= zero;
-  LatComplex    a(_grid); a= zero;
-  LatComplex    one  (_grid); one = ScalComplex(1.0,0.0);
-  LatComplex denom(_grid); denom= zero;
-  LatComplex cosha(_grid); 
-  LatComplex kmu(_grid); 
-  LatComplex Wea(_grid); 
-  LatComplex Wema(_grid); 
-
-  ScalComplex ci(0.0,1.0);
-  
-  for(int mu=0;mu<Nd;mu++) {
-    
-    LatticeCoordinate(kmu,mu);
-    
-    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
-    
-    kmu = TwoPiL * kmu;
-    
-    sk2 = sk2 + 2.0*sin(kmu*0.5)*sin(kmu*0.5);
-    sk  = sk  +     sin(kmu)    *sin(kmu); 
-    
-    num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);
-    
-  }
-  
-  W = one - M5 + sk2;
-
-  ////////////////////////////////////////////
-  // Cosh alpha -> alpha
-  ////////////////////////////////////////////
-  cosha =  (one + W*W + sk) / (W*2.0);
-
-  // FIXME Need a Lattice acosh
-  for(int idx=0;idx<_grid->lSites();idx++){
-    std::vector<int> lcoor(Nd);
-    Tcomplex cc;
-    RealD sgn;
-    _grid->LocalIndexToLocalCoor(idx,lcoor);
-    peekLocalSite(cc,cosha,lcoor);
-    assert((double)real(cc)>=1.0);
-    assert(fabs((double)imag(cc))<=1.0e-15);
-    cc = ScalComplex(::acosh(real(cc)),0.0);
-    pokeLocalSite(cc,a,lcoor);
-  }
-  
-  Wea = ( exp( a) * W  ); 
-  Wema= ( exp(-a) * W  ); 
-  
-  num   = num + ( one - Wema ) * mass * in;
-  denom= ( Wea - one ) + mass*mass * (one - Wema); 
-  out = num/denom;
-}
-
-template<class Impl>
-void WilsonFermion5D<Impl>::MomentumSpacePropagatorHw(FermionField &out,const FermionField &in,RealD mass) 
-{
-    Gamma::GammaMatrix Gmu [] = {
-      Gamma::GammaX,
-      Gamma::GammaY,
-      Gamma::GammaZ,
-      Gamma::GammaT
-    };
-
-    GridBase *_grid = _FourDimGrid;
-    conformable(_grid,out._grid);
-
-    typedef typename FermionField::vector_type vector_type;
-    typedef typename FermionField::scalar_type ScalComplex;
-
-    typedef Lattice<iSinglet<vector_type> > LatComplex;
-
-
-    std::vector<int> latt_size   = _grid->_fdimensions;
-
-    LatComplex    sk(_grid);  sk = zero;
-    LatComplex    sk2(_grid); sk2= zero;
-
-    LatComplex    w_k(_grid); w_k= zero;
-    LatComplex    b_k(_grid); b_k= zero;
-
-    LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
-
-    FermionField   num  (_grid); num  = zero;
-    LatComplex denom(_grid); denom= zero;
-    LatComplex kmu(_grid); 
-    ScalComplex ci(0.0,1.0);
-
-    for(int mu=0;mu<Nd;mu++) {
-
-      LatticeCoordinate(kmu,mu);
-
-      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
-
-      kmu = TwoPiL * kmu;
-
-      sk2 = sk2 + 2.0*sin(kmu*0.5)*sin(kmu*0.5);
-      sk  = sk  + sin(kmu)*sin(kmu); 
-
-      num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);
-
-    }
-    num = num + mass * in ;
-
-    b_k = sk2 - M5;
-     
-    w_k = sqrt(sk + b_k*b_k);
-
-    denom= ( w_k + b_k + mass*mass) ;
-
-    denom= one/denom;
-    out = num*denom;
-
-}
-
-
 FermOpTemplateInstantiate(WilsonFermion5D);
 GparityFermOpTemplateInstantiate(WilsonFermion5D);
   

lib/qcd/action/fermion/WilsonFermion5D.h

@@ -34,18 +34,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/Stat.h>
 
 namespace Grid {
-namespace QCD {
 
-  ////////////////////////////////////////////////////////////////////////////////
-  // This is the 4d red black case appropriate to support
-  //
-  // parity = (x+y+z+t)|2;
-  // generalised five dim fermions like mobius, zolotarev etc..	
-  //
-  // i.e. even even contains fifth dim hopping term.
-  //
-  // [DIFFERS from original CPS red black implementation parity = (x+y+z+t+s)|2 ]
-  ////////////////////////////////////////////////////////////////////////////////
+  namespace QCD {
 
     ////////////////////////////////////////////////////////////////////////////////
     // This is the 4d red black case appropriate to support
@@ -112,9 +102,6 @@ namespace QCD {
       virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
       virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
 
-      void MomentumSpacePropagatorHt(FermionField &out,const FermionField &in,RealD mass) ;
-      void MomentumSpacePropagatorHw(FermionField &out,const FermionField &in,RealD mass) ;
-
       // Implement hopping term non-hermitian hopping term; half cb or both
       // Implement s-diagonal DW
       void DW    (const FermionField &in, FermionField &out,int dag);
@@ -195,7 +182,7 @@ namespace QCD {
       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  comm_buf;
       
     };
-
-}}
+  }
+}
 
 #endif

lib/qcd/action/fermion/WilsonKernels.cc

@@ -32,7 +32,8 @@ directory
 namespace Grid {
 namespace QCD {
 
-int WilsonKernelsStatic::Opt;
+int WilsonKernelsStatic::HandOpt;
+int WilsonKernelsStatic::AsmOpt;
 
 template <class Impl>
 WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
@@ -42,8 +43,9 @@ WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
 ////////////////////////////////////////////
 
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-						     SiteHalfSpinor *buf, int sF,
+void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(
+    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
     int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
@@ -218,8 +220,9 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(StencilImpl &st, LebesgueOr
 
 // Need controls to do interior, exterior, or both
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptGenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-						  SiteHalfSpinor *buf, int sF,
+void WilsonKernels<Impl>::DiracOptGenericDhopSite(
+    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
     int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
@@ -393,9 +396,10 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSite(StencilImpl &st, LebesgueOrder
 };
 
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptDhopDir( StencilImpl &st, DoubledGaugeField &U,SiteHalfSpinor *buf, int sF,
+void WilsonKernels<Impl>::DiracOptDhopDir(
+    StencilImpl &st, DoubledGaugeField &U,
+    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
     int sU, const FermionField &in, FermionField &out, int dir, int gamma) {
-
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteSpinor result;

lib/qcd/action/fermion/WilsonKernels.h

@@ -32,69 +32,70 @@ directory
 #define GRID_QCD_DHOP_H
 
 namespace Grid {
-namespace QCD {
+
+  namespace QCD {
 
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
     // Helper routines that implement Wilson stencil for a single site.
     // Common to both the WilsonFermion and WilsonFermion5D
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class WilsonKernelsStatic { 
+    class WilsonKernelsStatic { 
     public:
-  enum { OptGeneric, OptHandUnroll, OptInlineAsm };
       // S-direction is INNERMOST and takes no part in the parity.
-  static int Opt;  // these are a temporary hack
-};
+      static int AsmOpt;  // these are a temporary hack
+      static int HandOpt; // these are a temporary hack
+    };
 
-template<class Impl> class WilsonKernels : public FermionOperator<Impl> , public WilsonKernelsStatic { 
+    template<class Impl> class WilsonKernels : public FermionOperator<Impl> , public WilsonKernelsStatic { 
     public:
 
       INHERIT_IMPL_TYPES(Impl);
       typedef FermionOperator<Impl> Base;
      
-public:
+    public:
 
       template <bool EnableBool = true>
       typename std::enable_if<Impl::Dimension == 3 && Nc == 3 &&EnableBool, void>::type
-  DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) 
-  {
-    switch(Opt) {
+	DiracOptDhopSite(
+			 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+			 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+			 int sF, int sU, int Ls, int Ns, const FermionField &in,
+			 FermionField &out) {
 #ifdef AVX512
-    case OptInlineAsm:
-       WilsonKernels<Impl>::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
-       break;
+	if (AsmOpt) {
+	  WilsonKernels<Impl>::DiracOptAsmDhopSite(st, lo, U, buf, sF, sU, Ls, Ns,
+						   in, out);
+
+	} else {
+#else
+	  {
 #endif
-    case OptHandUnroll:
 	    for (int site = 0; site < Ns; site++) {
 	      for (int s = 0; s < Ls; s++) {
-	  WilsonKernels<Impl>::DiracOptHandDhopSite(st,lo,U,buf,sF,sU,in,out);
+		if (HandOpt)
+		  WilsonKernels<Impl>::DiracOptHandDhopSite(st, lo, U, buf, sF, sU,
+							    in, out);
+		else
+		  WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU,
+							       in, out);
 		sF++;
 	      }
 	      sU++;
 	    }
-      break;
-    case OptGeneric:
-      for (int site = 0; site < Ns; site++) {
-	for (int s = 0; s < Ls; s++) {
-	  WilsonKernels<Impl>::DiracOptGenericDhopSite(st,lo,U,buf,sF,sU,in,out);
-	  sF++;
-	}
-	sU++;
-      }
-      break;
-    default:
-      assert(0);
 	  }
 	}
 
 	template <bool EnableBool = true>
 	  typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type
-  DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
-    // no kernel choice  
+	  DiracOptDhopSite(
+			   StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+			   std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+			   int sF, int sU, int Ls, int Ns, const FermionField &in,
+			   FermionField &out) {
 	  for (int site = 0; site < Ns; site++) {
 	    for (int s = 0; s < Ls; s++) {
-	WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in, out);
+	      WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in,
+							   out);
 	      sF++;
 	    }
 	    sU++;
@@ -102,82 +103,104 @@ public:
 	}
 
 	template <bool EnableBool = true>
-  typename std::enable_if<Impl::Dimension == 3 && Nc == 3 && EnableBool,void>::type
-  DiracOptDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-		      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
-
-    switch(Opt) {
+	  typename std::enable_if<Impl::Dimension == 3 && Nc == 3 && EnableBool,
+				  void>::type
+	  DiracOptDhopSiteDag(
+			      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+			      std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+			      int sF, int sU, int Ls, int Ns, const FermionField &in,
+			      FermionField &out) {
 #ifdef AVX512
-    case OptInlineAsm:
-      WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
-      break;
+				    if (AsmOpt) {
+				      WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st, lo, U, buf, sF, sU, Ls,
+										  Ns, in, out);
+				    } else {
+#else
+				      {
 #endif
-    case OptHandUnroll:
 					for (int site = 0; site < Ns; site++) {
 					  for (int s = 0; s < Ls; s++) {
-	  WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+					    if (HandOpt)
+					      WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st, lo, U, buf, sF, sU,
+											   in, out);
+					    else
+					      WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF,
+											      sU, in, out);
 					    sF++;
 					  }
 					  sU++;
 					}
-      break;
-    case OptGeneric:
-      for (int site = 0; site < Ns; site++) {
-	for (int s = 0; s < Ls; s++) {
-	  WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
-	  sF++;
-	}
-	sU++;
-      }
-      break;
-    default:
-      assert(0);
 				      }
 				    }
 
 				    template <bool EnableBool = true>
-  typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool,void>::type
-  DiracOptDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,SiteHalfSpinor * buf,
-		      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
-
+				      typename std::enable_if<
+				      (Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool,
+				      void>::type
+				      DiracOptDhopSiteDag(
+							  StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+							  std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							  int sF, int sU, int Ls, int Ns, const FermionField &in,
+							  FermionField &out) {
 					for (int site = 0; site < Ns; site++) {
 					  for (int s = 0; s < Ls; s++) {
-	WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+					    WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF, sU,
+											    in, out);
 					    sF++;
 					  }
 					  sU++;
 					}
 				      }
 
-  void DiracOptDhopDir(StencilImpl &st, DoubledGaugeField &U,SiteHalfSpinor * buf,
-		       int sF, int sU, const FermionField &in, FermionField &out, int dirdisp, int gamma);
+				    void DiracOptDhopDir(
+							 StencilImpl &st, DoubledGaugeField &U,
+							 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							 int sF, int sU, const FermionField &in, FermionField &out, int dirdisp,
+							 int gamma);
 
-private:
+	private:
 				    // Specialised variants
-  void DiracOptGenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+				    void DiracOptGenericDhopSite(
+								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+								 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 								 int sF, int sU, const FermionField &in, FermionField &out);
 
-  void DiracOptGenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+				    void DiracOptGenericDhopSiteDag(
+								    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+								    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 								    int sF, int sU, const FermionField &in, FermionField &out);
 
-  void DiracOptAsmDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			   int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
+				    void DiracOptAsmDhopSite(
+							     StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+							     std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							     int sF, int sU, int Ls, int Ns, const FermionField &in,
+							     FermionField &out);
 
-  void DiracOptAsmDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
+				    void DiracOptAsmDhopSiteDag(
+								StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+								std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+								int sF, int sU, int Ls, int Ns, const FermionField &in,
+								FermionField &out);
 
-  void DiracOptHandDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+				    void DiracOptHandDhopSite(
+							      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+							      std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 							      int sF, int sU, const FermionField &in, FermionField &out);
 
-  void DiracOptHandDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+				    void DiracOptHandDhopSiteDag(
+								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+								 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 								 int sF, int sU, const FermionField &in, FermionField &out);
 
-public:
-
+	public:
 				    WilsonKernels(const ImplParams &p = ImplParams());
+				  };
+    
+      }
+    }
+
+
 
-};
 
-}}
 
 #endif

lib/qcd/action/fermion/WilsonKernelsAsm.cc

@@ -10,7 +10,6 @@
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -34,46 +33,48 @@ Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 
 namespace Grid {
-namespace QCD {
+  namespace QCD {
     
-///////////////////////////////////////////////////////////
-// Default to no assembler implementation
-///////////////////////////////////////////////////////////
-template<class Impl> void 
-WilsonKernels<Impl >::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+    ///////////////////////////////////////////////////////////
+    // Default to no assembler implementation
+    ///////////////////////////////////////////////////////////
+    template<class Impl>
+      void WilsonKernels<Impl >::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+                             std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
                              int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-{
+    {
       assert(0);
-}
+    }
+    template<class Impl>
+      void WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+                                std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+                                int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+    {
+      assert(0);
+    }
+
 
-template<class Impl> void 
-WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-					     int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-{
-  assert(0);
-}
 
 #if defined(AVX512) 
-#include <simd/Intel512wilson.h>
+    
     
     ///////////////////////////////////////////////////////////
     // If we are AVX512 specialise the single precision routine
     ///////////////////////////////////////////////////////////
     
+#include <simd/Intel512wilson.h>
 #include <simd/Intel512single.h>
     
-static Vector<vComplexF> signsF;
+    static Vector<vComplexF> signs;
     
-  template<typename vtype>    
-  int setupSigns(Vector<vtype>& signs ){
-    Vector<vtype> bother(2);
+    int setupSigns(void ){
+      Vector<vComplexF> bother(2);
       signs = bother;
       vrsign(signs[0]);
       visign(signs[1]);
       return 1;
     }
-
-  static int signInitF = setupSigns(signsF);
+    static int signInit = setupSigns();
   
 #define label(A)  ilabel(A)
 #define ilabel(A) ".globl\n"  #A ":\n" 
@@ -81,18 +82,18 @@ static Vector<vComplexF> signsF;
 #define MAYBEPERM(A,perm) if (perm) { A ; }
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
 #define FX(A) WILSONASM_ ##A
-#define COMPLEX_TYPE vComplexF
-#define signs signsF
   
 #undef KERNEL_DAG
-template<> void 
-WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+    template<>
+    void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+							 std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
       
 #define KERNEL_DAG
-template<> void 
-WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+    template<>
+    void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+							    std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
@@ -103,94 +104,36 @@ WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder
 #undef FX 
 #define FX(A) DWFASM_ ## A
 #define MAYBEPERM(A,B) 
-//#define VMOVIDUP(A,B,C)                                  VBCASTIDUPf(A,B,C)
-//#define VMOVRDUP(A,B,C)                                  VBCASTRDUPf(A,B,C)
+#define VMOVIDUP(A,B,C)                                  VBCASTIDUPf(A,B,C)
+#define VMOVRDUP(A,B,C)                                  VBCASTRDUPf(A,B,C)
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
 				    
 #undef KERNEL_DAG
-template<> void 
-WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+    template<>
+    void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 								  int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
 #define KERNEL_DAG
-template<> void 
-WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
-#undef COMPLEX_TYPE
-#undef signs
-#undef VMOVRDUP
-#undef MAYBEPERM
-#undef MULT_2SPIN
-#undef FX 
-	
-///////////////////////////////////////////////////////////
-// If we are AVX512 specialise the double precision routine
-///////////////////////////////////////////////////////////
-
-#include <simd/Intel512double.h>
-    
-static Vector<vComplexD> signsD;
-#define signs signsD
-static int signInitD = setupSigns(signsD);
-    
-#define MAYBEPERM(A,perm) if (perm) { A ; }
-#define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
-#define FX(A) WILSONASM_ ##A
-#define COMPLEX_TYPE vComplexD
-  
-#undef KERNEL_DAG
-template<> void 
-WilsonKernels<WilsonImplD>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+    template<>
+    void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+								     std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 								     int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
-#define KERNEL_DAG
-template<> void 
-WilsonKernels<WilsonImplD>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-						   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+#endif
 
-#undef VMOVIDUP
-#undef VMOVRDUP
-#undef MAYBEPERM
-#undef MULT_2SPIN
-#undef FX 
-#define FX(A) DWFASM_ ## A
-#define MAYBEPERM(A,B) 
-//#define VMOVIDUP(A,B,C)                                  VBCASTIDUPd(A,B,C)
-//#define VMOVRDUP(A,B,C)                                  VBCASTRDUPd(A,B,C)
-#define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
-				    
-#undef KERNEL_DAG
-template<> void 
-WilsonKernels<DomainWallVec5dImplD>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
-							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
-				    
-#define KERNEL_DAG
-template<> void 
-WilsonKernels<DomainWallVec5dImplD>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
-	
-#undef COMPLEX_TYPE
-#undef signs
-#undef VMOVRDUP
-#undef MAYBEPERM
-#undef MULT_2SPIN
-#undef FX 
-
-#endif //AVX512
 
 #define INSTANTIATE_ASM(A)\
-template void WilsonKernels<A>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,\
+template void WilsonKernels<A>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
+                                   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
                                   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
- \
-template void WilsonKernels<A>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,\
+template void WilsonKernels<A>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
+                                   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
                                   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
 
+
 INSTANTIATE_ASM(WilsonImplF);
 INSTANTIATE_ASM(WilsonImplD);
 INSTANTIATE_ASM(ZWilsonImplF);
@@ -201,6 +144,6 @@ INSTANTIATE_ASM(DomainWallVec5dImplF);
 INSTANTIATE_ASM(DomainWallVec5dImplD);
 INSTANTIATE_ASM(ZDomainWallVec5dImplF);
 INSTANTIATE_ASM(ZDomainWallVec5dImplD);
-
-}}
+  }
+}
 

lib/qcd/action/fermion/WilsonKernelsAsmBody.h

@@ -5,9 +5,7 @@
   const uint64_t plocal =(uint64_t) & in._odata[0];
 
   //  vComplexF isigns[2] = { signs[0], signs[1] };
-  //COMPLEX_TYPE is vComplexF of vComplexD depending 
-  //on the chosen precision
-  COMPLEX_TYPE *isigns = &signs[0];
+  vComplexF *isigns = &signs[0];
 
   MASK_REGS;
   int nmax=U._grid->oSites();

lib/qcd/action/fermion/WilsonKernelsHand.cc

@@ -311,8 +311,9 @@ namespace Grid {
 namespace QCD {
 
 
-template<class Impl> void 
-WilsonKernels<Impl>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
+  template<class Impl>
+  void WilsonKernels<Impl>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+					       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 					       int ss,int sU,const FermionField &in, FermionField &out)
 {
   typedef typename Simd::scalar_type S;
@@ -553,8 +554,9 @@ WilsonKernels<Impl>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,Doub
   }
 }
 
-template<class Impl>
-void WilsonKernels<Impl>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+  template<class Impl>
+  void WilsonKernels<Impl>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+					       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 					       int ss,int sU,const FermionField &in, FermionField &out)
 {
   //  std::cout << "Hand op Dhop "<<std::endl;
@@ -796,34 +798,37 @@ void WilsonKernels<Impl>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder
   }
 }
 
+
   ////////////////////////////////////////////////
   // Specialise Gparity to simple implementation
   ////////////////////////////////////////////////
-template<> void 
-WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							SiteHalfSpinor *buf,
+template<>
+void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+							     std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 							     int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
 }
 
-template<> void 
-WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							   SiteHalfSpinor *buf,
+template<>
+void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+								std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 								int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
 }
 
-template<> void 
-WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+template<>
+void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+							     std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 							     int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
 }
 
-template<> void 
-WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+template<>
+void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+								std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 								int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
@@ -835,9 +840,11 @@ WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,Lebes
 // Need Nc=3 though //
 
 #define INSTANTIATE_THEM(A) \
-template void WilsonKernels<A>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
-						     int ss,int sU,const FermionField &in, FermionField &out); \
-template void WilsonKernels<A>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
+template void WilsonKernels<A>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
+							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+							       int ss,int sU,const FermionField &in, FermionField &out);\
+template void WilsonKernels<A>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
+								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
 								  int ss,int sU,const FermionField &in, FermionField &out);
 
 INSTANTIATE_THEM(WilsonImplF);

lib/qcd/hmc/HmcRunner.h

@@ -116,7 +116,7 @@ class NerscHmcRunnerTemplate {
     NoSmearing<Gimpl> SmearingPolicy;
     typedef MinimumNorm2<GaugeField, NoSmearing<Gimpl>, RepresentationsPolicy >
         IntegratorType;  // change here to change the algorithm
-    IntegratorParameters MDpar(40, 1.0);
+    IntegratorParameters MDpar(20, 1.0);
     IntegratorType MDynamics(UGrid, MDpar, TheAction, SmearingPolicy);
 
     // Checkpoint strategy

lib/qcd/utils/LinalgUtils.h

@@ -39,8 +39,8 @@ namespace QCD{
 //on the 5d (rb4d) checkerboarded lattices
 ////////////////////////////////////////////////////////////////////////
 
-template<class vobj,class Coeff>
-void axpibg5x(Lattice<vobj> &z,const Lattice<vobj> &x,Coeff a,Coeff b)
+template<class vobj> 
+void axpibg5x(Lattice<vobj> &z,const Lattice<vobj> &x,RealD a,RealD b)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,z);
@@ -57,8 +57,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj,class Coeff> 
-void axpby_ssp(Lattice<vobj> &z, Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj> 
+void axpby_ssp(Lattice<vobj> &z, RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -72,8 +72,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj,class Coeff> 
-void ag5xpby_ssp(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj> 
+void ag5xpby_ssp(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -90,8 +90,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj,class Coeff> 
-void axpbg5y_ssp(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj> 
+void axpbg5y_ssp(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -108,8 +108,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj,class Coeff> 
-void ag5xpbg5y_ssp(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj> 
+void ag5xpbg5y_ssp(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -127,8 +127,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj,class Coeff> 
-void axpby_ssp_pminus(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj> 
+void axpby_ssp_pminus(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -144,8 +144,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj,class Coeff> 
-void axpby_ssp_pplus(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj> 
+void axpby_ssp_pplus(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);

lib/qcd/utils/SUn.h

@@ -674,37 +674,6 @@ class SU {
       out += la;
     }
   }
-/*
- add GaugeTrans
-*/
-
-template<typename GaugeField,typename GaugeMat>
-  static void GaugeTransform( GaugeField &Umu, GaugeMat &g){
-    GridBase *grid = Umu._grid;
-    conformable(grid,g._grid);
-
-    GaugeMat U(grid);
-    GaugeMat ag(grid); ag = adj(g);
-
-    for(int mu=0;mu<Nd;mu++){
-      U= PeekIndex<LorentzIndex>(Umu,mu);
-      U = g*U*Cshift(ag, mu, 1);
-      PokeIndex<LorentzIndex>(Umu,U,mu);
-    }
-  }
-  template<typename GaugeMat>
-    static void GaugeTransform( std::vector<GaugeMat> &U, GaugeMat &g){
-    GridBase *grid = g._grid;
-    GaugeMat ag(grid); ag = adj(g);
-    for(int mu=0;mu<Nd;mu++){
-      U[mu] = g*U[mu]*Cshift(ag, mu, 1);
-    }
-  }
-  template<typename GaugeField,typename GaugeMat>
-  static void RandomGaugeTransform(GridParallelRNG &pRNG, GaugeField &Umu, GaugeMat &g){
-    LieRandomize(pRNG,g,1.0);
-    GaugeTransform(Umu,g);
-  }
 
   // Projects the algebra components a lattice matrix (of dimension ncol*ncol -1 )
   // inverse operation: FundamentalLieAlgebraMatrix
@@ -733,33 +702,23 @@ template<typename GaugeField,typename GaugeMat>
       PokeIndex<LorentzIndex>(out, Umu, mu);
     }
   }
-  template<typename GaugeField>
-  static void TepidConfiguration(GridParallelRNG &pRNG,GaugeField &out){
-    typedef typename GaugeField::vector_type vector_type;
-    typedef iSUnMatrix<vector_type> vMatrixType;
-    typedef Lattice<vMatrixType> LatticeMatrixType;
-
-    LatticeMatrixType Umu(out._grid);
-    for(int mu=0;mu<Nd;mu++){
-      LieRandomize(pRNG,Umu,0.01);
-      PokeIndex<LorentzIndex>(out,Umu,mu);
+  static void TepidConfiguration(GridParallelRNG &pRNG,
+                                 LatticeGaugeField &out) {
+    LatticeMatrix Umu(out._grid);
+    for (int mu = 0; mu < Nd; mu++) {
+      LieRandomize(pRNG, Umu, 0.01);
+      PokeIndex<LorentzIndex>(out, Umu, mu);
     }
   }
-  template<typename GaugeField>
-  static void ColdConfiguration(GridParallelRNG &pRNG,GaugeField &out){
-    typedef typename GaugeField::vector_type vector_type;
-    typedef iSUnMatrix<vector_type> vMatrixType;
-    typedef Lattice<vMatrixType> LatticeMatrixType;
-
-    LatticeMatrixType Umu(out._grid);
-    Umu=1.0;
-    for(int mu=0;mu<Nd;mu++){
-      PokeIndex<LorentzIndex>(out,Umu,mu);
+  static void ColdConfiguration(GridParallelRNG &pRNG, LatticeGaugeField &out) {
+    LatticeMatrix Umu(out._grid);
+    Umu = 1.0;
+    for (int mu = 0; mu < Nd; mu++) {
+      PokeIndex<LorentzIndex>(out, Umu, mu);
     }
   }
 
-  template<typename LatticeMatrixType>
-  static void taProj( const LatticeMatrixType &in,  LatticeMatrixType &out){
+  static void taProj(const LatticeMatrix &in, LatticeMatrix &out) {
     out = Ta(in);
   }
   template <typename LatticeMatrixType>

lib/simd/Grid_avx.h

@@ -66,9 +66,6 @@ namespace Optimization {
     double f[4];
   };
 
-
-
-
   struct Vsplat{
     //Complex float
     inline __m256 operator()(float a, float b){
@@ -170,7 +167,7 @@ namespace Optimization {
     }
     //Integer
     inline __m256i operator()(__m256i a, __m256i b){
-#if defined (AVX1) || defined (AVXFMA) || defined (AVXFMA4)
+#if defined (AVX1) || defined (AVXFMA4)
           __m128i a0,a1;
           __m128i b0,b1;
           a0 = _mm256_extractf128_si256(a,0);
@@ -198,7 +195,7 @@ namespace Optimization {
     }
     //Integer
     inline __m256i operator()(__m256i a, __m256i b){
-#if defined (AVX1) || defined (AVXFMA) || defined (AVXFMA4)
+#if defined (AVX1) || defined (AVXFMA4)
           __m128i a0,a1;
           __m128i b0,b1;
           a0 = _mm256_extractf128_si256(a,0);
@@ -236,7 +233,7 @@ namespace Optimization {
       a_imag = _mm256_mul_ps( a_imag,tmp  );  // (Ai, Ai) * (Bi, Br) = Ai Bi, Ai Br
       return _mm256_maddsub_ps( a_real, b, a_imag ); // Ar Br , Ar Bi   +- Ai Bi             = ArBr-AiBi , ArBi+AiBr
 #endif
-#if defined (AVX2)  || defined (AVXFMA)
+#if defined (AVX2)
       __m256 a_real = _mm256_moveldup_ps( a ); // Ar Ar
       __m256 a_imag = _mm256_movehdup_ps( a ); // Ai Ai
       a_imag = _mm256_mul_ps( a_imag, _mm256_shuffle_ps( b,b, _MM_SELECT_FOUR_FOUR(2,3,0,1) ));  // (Ai, Ai) * (Bi, Br) = Ai Bi, Ai Br
@@ -282,7 +279,7 @@ namespace Optimization {
       a_imag = _mm256_mul_pd( a_imag, _mm256_permute_pd( b, 0x5 ) );  // (Ai, Ai) * (Bi, Br) = Ai Bi, Ai Br
       return _mm256_maddsub_pd( a_real, b, a_imag ); // Ar Br , Ar Bi   +- Ai Bi             = ArBr-AiBi , ArBi+AiBr
 #endif
-#if defined (AVX2) || defined (AVXFMA)
+#if defined (AVX2)
       __m256d a_real = _mm256_movedup_pd( a ); // Ar Ar
       __m256d a_imag = _mm256_shuffle_pd(a,a,0xF);//aiai
       a_imag = _mm256_mul_pd( a_imag, _mm256_permute_pd( b, 0x5 ) );  // (Ai, Ai) * (Bi, Br) = Ai Bi, Ai Br
@@ -323,7 +320,7 @@ namespace Optimization {
 #if defined (AVXFMA4)
       a= _mm256_macc_ps(b,c,a);
 #endif
-#if defined (AVX2) || defined (AVXFMA)
+#if defined (AVX2)
       a= _mm256_fmadd_ps( b, c, a);
 #endif
     }
@@ -335,7 +332,7 @@ namespace Optimization {
 #if defined (AVXFMA4)
       a= _mm256_macc_pd(b,c,a);
 #endif
-#if defined (AVX2) || defined (AVXFMA)
+#if defined (AVX2)
       a= _mm256_fmadd_pd( b, c, a);
 #endif
     }
@@ -350,7 +347,7 @@ namespace Optimization {
     }
     // Integer
     inline __m256i operator()(__m256i a, __m256i b){
-#if defined (AVX1) || defined (AVXFMA)
+#if defined (AVX1) 
       __m128i a0,a1;
       __m128i b0,b1;
       a0 = _mm256_extractf128_si256(a,0);
@@ -368,18 +365,6 @@ namespace Optimization {
     }
   };
 
-  struct Div{
-    // Real float
-    inline __m256 operator()(__m256 a, __m256 b){
-      return _mm256_div_ps(a,b);
-    }
-    // Real double
-    inline __m256d operator()(__m256d a, __m256d b){
-      return _mm256_div_pd(a,b);
-    }
-  };
-
-
   struct Conj{
     // Complex single
     inline __m256 operator()(__m256 in){
@@ -452,13 +437,14 @@ namespace Optimization {
 
   };
 
-#if defined (AVX2)
-#define _mm256_alignr_epi32_grid(ret,a,b,n) ret=(__m256)  _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*4)%16)
-#define _mm256_alignr_epi64_grid(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16)
+#if defined (AVX2) || defined (AVXFMA4) 
+#define _mm256_alignr_epi32(ret,a,b,n) ret=(__m256) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*4)%16)
+#define _mm256_alignr_epi64(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16)
 #endif
 
 #if defined (AVX1) || defined (AVXFMA)
-#define _mm256_alignr_epi32_grid(ret,a,b,n) {	\
+
+#define _mm256_alignr_epi32(ret,a,b,n) {	\
     __m128 aa, bb;				\
 						\
     aa  = _mm256_extractf128_ps(a,1);		\
@@ -472,7 +458,7 @@ namespace Optimization {
     ret = _mm256_insertf128_ps(ret,aa,0);	\
   }
 
-#define _mm256_alignr_epi64_grid(ret,a,b,n) {	\
+#define _mm256_alignr_epi64(ret,a,b,n) {	\
     __m128d aa, bb;				\
 						\
     aa  = _mm256_extractf128_pd(a,1);		\
@@ -488,6 +474,19 @@ namespace Optimization {
 
 #endif
 
+    inline std::ostream & operator << (std::ostream& stream, const __m256 a)
+    {
+      const float *p=(const float *)&a;
+      stream<< "{"<<p[0]<<","<<p[1]<<","<<p[2]<<","<<p[3]<<","<<p[4]<<","<<p[5]<<","<<p[6]<<","<<p[7]<<"}";
+      return stream;
+    };
+    inline std::ostream & operator<< (std::ostream& stream, const __m256d a)
+    {
+      const double *p=(const double *)&a;
+      stream<< "{"<<p[0]<<","<<p[1]<<","<<p[2]<<","<<p[3]<<"}";
+      return stream;
+    };
+
   struct Rotate{
 
     static inline __m256 rotate(__m256 in,int n){ 
@@ -519,10 +518,11 @@ namespace Optimization {
       __m256 tmp = Permute::Permute0(in);
       __m256 ret;
       if ( n > 3 ) { 
-	_mm256_alignr_epi32_grid(ret,in,tmp,n);  
+	_mm256_alignr_epi32(ret,in,tmp,n);  
       } else {
-        _mm256_alignr_epi32_grid(ret,tmp,in,n);          
+        _mm256_alignr_epi32(ret,tmp,in,n);          
       }
+      //      std::cout << " align epi32 n=" <<n<<" in "<<tmp<<in<<" -> "<< ret <<std::endl;
       return ret;
     };
 
@@ -531,15 +531,18 @@ namespace Optimization {
       __m256d tmp = Permute::Permute0(in);
       __m256d ret;
       if ( n > 1 ) {
-	_mm256_alignr_epi64_grid(ret,in,tmp,n);          
+	_mm256_alignr_epi64(ret,in,tmp,n);          
       } else {
-        _mm256_alignr_epi64_grid(ret,tmp,in,n);          
+        _mm256_alignr_epi64(ret,tmp,in,n);          
       }
+      //      std::cout << " align epi64 n=" <<n<<" in "<<tmp<<in<<" -> "<< ret <<std::endl;
       return ret;
     };
 
   };
 
+
+
   //Complex float Reduce
   template<>
     inline Grid::ComplexF Reduce<Grid::ComplexF, __m256>::operator()(__m256 in){
@@ -628,7 +631,6 @@ namespace Optimization {
   // Arithmetic operations
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
-  typedef Optimization::Div         DivSIMD;
   typedef Optimization::Mult        MultSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;

lib/simd/Grid_avx512.h

@@ -27,22 +27,21 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
+//----------------------------------------------------------------------
+/*! @file Grid_knc.h
+  @brief Optimization libraries for AVX512 instructions set for KNC
+
+  Using intrinsics
+*/
+// Time-stamp: <2015-06-09 14:27:28 neo>
+//----------------------------------------------------------------------
+
 #include <immintrin.h>
 
 
 namespace Grid{
 namespace Optimization {
   
-  union u512f {
-    __m512 v;
-    float f[16];
-  };
-
-  union u512d {
-    __m512d v;
-    double f[8];
-  };
-  
   struct Vsplat{
     //Complex float
     inline __m512 operator()(float a, float b){
@@ -86,13 +85,13 @@ namespace Optimization {
   struct Vstream{
     //Float
     inline void operator()(float * a, __m512 b){
-      _mm512_stream_ps(a,b);
-      //      _mm512_store_ps(a,b);
+      //_mm512_stream_ps(a,b);
+      _mm512_store_ps(a,b);
     }
     //Double
     inline void operator()(double * a, __m512d b){
-      _mm512_stream_pd(a,b);
-      //      _mm512_store_pd(a,b);
+      //_mm512_stream_pd(a,b);
+      _mm512_store_pd(a,b);
     }
 
   };
@@ -231,17 +230,6 @@ namespace Optimization {
     }
   };
 
-  struct Div{
-    // Real float
-    inline __m512 operator()(__m512 a, __m512 b){
-      return _mm512_div_ps(a,b);
-    }
-    // Real double
-    inline __m512d operator()(__m512d a, __m512d b){
-      return _mm512_div_pd(a,b);
-    }
-  };
-
 
   struct Conj{
     // Complex single
@@ -373,8 +361,8 @@ namespace Optimization {
   // Some Template specialization
 
   // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases
-#ifndef __INTEL_COMPILER
-#warning "Slow reduction due to incomplete reduce intrinsics"
+#undef GNU_CLANG_COMPILER 
+#ifdef GNU_CLANG_COMPILER
   //Complex float Reduce
   template<>
     inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){
@@ -499,7 +487,6 @@ namespace Optimization {
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
   typedef Optimization::Mult        MultSIMD;
-  typedef Optimization::Div         DivSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;
   typedef Optimization::TimesMinusI TimesMinusISIMD;

lib/simd/Grid_generic.h

@@ -6,7 +6,8 @@
 
     Copyright (C) 2015
 
-Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -26,352 +27,133 @@ Author: Antonin Portelli <antonin.portelli@me.com>
     *************************************************************************************/
     /*  END LEGAL */
 
-static_assert(GEN_SIMD_WIDTH % 16u == 0, "SIMD vector size is not an integer multiple of 16 bytes");
-
-//#define VECTOR_LOOPS
-
-// playing with compiler pragmas
-#ifdef VECTOR_LOOPS
-#ifdef __clang__
-#define VECTOR_FOR(i, w, inc)\
-_Pragma("clang loop unroll(full) vectorize(enable) interleave(enable) vectorize_width(w)")\
-for (unsigned int i = 0; i < w; i += inc)
-#elif defined __INTEL_COMPILER
-#define VECTOR_FOR(i, w, inc)\
-_Pragma("simd vectorlength(w*8)")\
-for (unsigned int i = 0; i < w; i += inc)
-#else
-#define VECTOR_FOR(i, w, inc)\
-for (unsigned int i = 0; i < w; i += inc)
-#endif
-#else
-#define VECTOR_FOR(i, w, inc)\
-for (unsigned int i = 0; i < w; i += inc)
-#endif
-
 namespace Grid {
 namespace Optimization {
 
-  // type traits giving the number of elements for each vector type
-  template <typename T> struct W;
-  template <> struct W<double> {
-    constexpr static unsigned int c = GEN_SIMD_WIDTH/16u;
-    constexpr static unsigned int r = GEN_SIMD_WIDTH/8u;
-  };
-  template <> struct W<float> {
-    constexpr static unsigned int c = GEN_SIMD_WIDTH/8u;
-    constexpr static unsigned int r = GEN_SIMD_WIDTH/4u;
+  template<class vtype>
+  union uconv {
+    float f;
+    vtype v;
   };
 
-  // SIMD vector types
-  template <typename T>
-  struct vec {
-    alignas(GEN_SIMD_WIDTH) T v[W<T>::r];
+  union u128f {
+    float v;
+    float f[4];
+  };
+  union u128d {
+    double v;
+    double f[2];
   };
-  
-  typedef vec<float>   vecf;
-  typedef vec<double>  vecd;
   
   struct Vsplat{
-    // Complex
-    template <typename T>
-    inline vec<T> operator()(T a, T b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::r, 2)
-      {
-        out.v[i]   = a;
-        out.v[i+1] = b;
-      }
-
+    //Complex float
+    inline u128f operator()(float a, float b){
+      u128f out; 
+      out.f[0] = a;
+      out.f[1] = b;
+      out.f[2] = a;
+      out.f[3] = b;
       return out;
     }
-    
-    // Real
-    template <typename T>
-    inline vec<T> operator()(T a){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::r, 1)
-      {
-        out.v[i] = a;
-      }
-      
+    // Real float
+    inline u128f operator()(float a){
+      u128f out; 
+      out.f[0] = a;
+      out.f[1] = a;
+      out.f[2] = a;
+      out.f[3] = a;
       return out;
     }
-    
-    // Integer
+    //Complex double
+    inline u128d operator()(double a, double b){
+      u128d out; 
+      out.f[0] = a;
+      out.f[1] = b;
+      return out;
+    }
+    //Real double
+    inline u128d operator()(double a){
+      u128d out; 
+      out.f[0] = a;
+      out.f[1] = a;
+      return out;
+    }
+    //Integer
     inline int operator()(Integer a){
       return a;
     }
   };
 
   struct Vstore{
-    // Real
-    template <typename T>
-    inline void operator()(vec<T> a, T *D){
-      *((vec<T> *)D) = a;
+    //Float 
+    inline void operator()(u128f a, float* F){
+      memcpy(F,a.f,4*sizeof(float));
+    }
+    //Double
+    inline void operator()(u128d a, double* D){
+      memcpy(D,a.f,2*sizeof(double));
     }
     //Integer
-    inline void operator()(int a, Integer *I){
-      *I = a;
+    inline void operator()(int a, Integer* I){
+      I[0] = a;
     }
 
   };
 
   struct Vstream{
-    // Real
-    template <typename T>
-    inline void operator()(T * a, vec<T> b){
-      *((vec<T> *)a) = b;
+    //Float
+    inline void operator()(float * a, u128f b){
+      memcpy(a,b.f,4*sizeof(float));
     }
+    //Double
+    inline void operator()(double * a, u128d b){
+      memcpy(a,b.f,2*sizeof(double));
+    }
+
+
   };
 
   struct Vset{
-    // Complex
-    template <typename T>
-    inline vec<T> operator()(std::complex<T> *a){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::c, 1)
-      {
-        out.v[2*i]   = a[i].real();
-        out.v[2*i+1] = a[i].imag();
-      }
-      
+    // Complex float 
+    inline u128f operator()(Grid::ComplexF *a){
+      u128f out; 
+      out.f[0] = a[0].real();
+      out.f[1] = a[0].imag();
+      out.f[2] = a[1].real();
+      out.f[3] = a[1].imag();
       return out;
     }
-    
-    // Real
-    template <typename T>
-    inline vec<T> operator()(T *a){
-      vec<T> out;
-      
-      out = *((vec<T> *)a);
-      
+    // Complex double 
+    inline u128d operator()(Grid::ComplexD *a){
+      u128d out; 
+      out.f[0] = a[0].real();
+      out.f[1] = a[0].imag();
+      return out;
+    }
+    // Real float 
+    inline u128f operator()(float *a){
+      u128f out; 
+      out.f[0] = a[0];
+      out.f[1] = a[1];
+      out.f[2] = a[2];
+      out.f[3] = a[3];
+      return out;
+    }
+    // Real double
+    inline u128d operator()(double *a){
+      u128d out; 
+      out.f[0] = a[0];
+      out.f[1] = a[1];
       return out;
     }
-
     // Integer
     inline int operator()(Integer *a){
-      return *a;
+      return a[0];
     }
+
+
   };
 
-  /////////////////////////////////////////////////////
-  // Arithmetic operations
-  /////////////////////////////////////////////////////
-  struct Sum{
-    // Complex/Real
-    template <typename T>
-    inline vec<T> operator()(vec<T> a, vec<T> b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::r, 1)
-      {
-        out.v[i] = a.v[i] + b.v[i];
-      }
-      
-      return out;
-    }
-    
-    //I nteger
-    inline int operator()(int a, int b){
-      return a + b;
-    }
-  };
-
-  struct Sub{
-    // Complex/Real
-    template <typename T>
-    inline vec<T> operator()(vec<T> a, vec<T> b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::r, 1)
-      {
-        out.v[i] = a.v[i] - b.v[i];
-      }
-      
-      return out;
-    }
-    
-    //Integer
-    inline int operator()(int a, int b){
-      return a-b;
-    }
-  };
-
-  struct Mult{
-    // Real
-    template <typename T>
-    inline vec<T> operator()(vec<T> a, vec<T> b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::r, 1)
-      {
-        out.v[i] = a.v[i]*b.v[i];
-      }
-      
-      return out;
-    }
-    
-    // Integer
-    inline int operator()(int a, int b){
-      return a*b;
-    }
-  };
-  
-  #define cmul(a, b, c, i)\
-  c[i]   = a[i]*b[i]   - a[i+1]*b[i+1];\
-  c[i+1] = a[i]*b[i+1] + a[i+1]*b[i];
-  
-  struct MultComplex{
-    // Complex
-    template <typename T>
-    inline vec<T> operator()(vec<T> a, vec<T> b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::c, 1)
-      {
-        cmul(a.v, b.v, out.v, 2*i);
-      }      
-      
-      return out;
-    }
-  };
-  
-  #undef cmul
-
-  struct Div{
-    // Real
-    template <typename T>
-    inline vec<T> operator()(vec<T> a, vec<T> b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::r, 1)
-      {
-        out.v[i] = a.v[i]/b.v[i];
-      }
-      
-      return out;
-    }
-  };
-  
-  #define conj(a, b, i)\
-  b[i]   = a[i];\
-  b[i+1] = -a[i+1];
-  
-  struct Conj{
-    // Complex
-    template <typename T>
-    inline vec<T> operator()(vec<T> a){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::c, 1)
-      {
-        conj(a.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-  };
-  
-  #undef conj
-
-  #define timesmi(a, b, i)\
-  b[i]   = a[i+1];\
-  b[i+1] = -a[i];
-  
-  struct TimesMinusI{
-    // Complex
-    template <typename T>
-    inline vec<T> operator()(vec<T> a, vec<T> b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::c, 1)
-      {
-        timesmi(a.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-  };
-
-  #undef timesmi
-  
-  #define timesi(a, b, i)\
-  b[i]   = -a[i+1];\
-  b[i+1] = a[i];
-  
-  struct TimesI{
-    // Complex
-    template <typename T>
-    inline vec<T> operator()(vec<T> a, vec<T> b){
-      vec<T> out;
-      
-      VECTOR_FOR(i, W<T>::c, 1)
-      {
-        timesi(a.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-  };
-  
-  #undef timesi
-
-  //////////////////////////////////////////////
-  // Some Template specialization
-  #define perm(a, b, n, w)\
-  unsigned int _mask = w >> (n + 1);\
-  VECTOR_FOR(i, w, 1)\
-  {\
-    b[i] = a[i^_mask];\
-  }
-  
-  #define DECL_PERMUTE_N(n)\
-  template <typename T>\
-  static inline vec<T> Permute##n(vec<T> in) {\
-    vec<T> out;\
-    perm(in.v, out.v, n, W<T>::r);\
-    return out;\
-  }
-  
-  struct Permute{
-    DECL_PERMUTE_N(0);
-    DECL_PERMUTE_N(1);
-    DECL_PERMUTE_N(2);
-    DECL_PERMUTE_N(3);
-  };
-  
-  #undef perm
-  #undef DECL_PERMUTE_N
-  
-  #define rot(a, b, n, w)\
-  VECTOR_FOR(i, w, 1)\
-  {\
-    b[i] = a[(i + n)%w];\
-  }
-  
-  struct Rotate{
-    template <typename T>
-    static inline vec<T> rotate(vec<T> in, int n){
-      vec<T> out;
-      
-      rot(in.v, out.v, n, W<T>::r);
-      
-      return out;
-    }
-  };
-
-  #undef rot
-  
-  #define acc(v, a, off, step, n)\
-  for (unsigned int i = off; i < n; i += step)\
-  {\
-    a += v[i];\
-  }
-  
   template <typename Out_type, typename In_type>
   struct Reduce{
     //Need templated class to overload output type
@@ -383,66 +165,315 @@ namespace Optimization {
     }
   };
 
-  //Complex float Reduce
-  template <>
-  inline Grid::ComplexF Reduce<Grid::ComplexF, vecf>::operator()(vecf in){
-    float a = 0.f, b = 0.f;
-    
-    acc(in.v, a, 0, 2, W<float>::r);
-    acc(in.v, b, 1, 2, W<float>::r);
-    
-    return Grid::ComplexF(a, b);
+  /////////////////////////////////////////////////////
+  // Arithmetic operations
+  /////////////////////////////////////////////////////
+  struct Sum{
+    //Complex/Real float
+    inline u128f operator()(u128f a, u128f b){
+      u128f out;
+      out.f[0] = a.f[0] + b.f[0];
+      out.f[1] = a.f[1] + b.f[1];
+      out.f[2] = a.f[2] + b.f[2];
+      out.f[3] = a.f[3] + b.f[3];
+      return out;
     }
+    //Complex/Real double
+    inline u128d operator()(u128d a, u128d b){
+      u128d out;
+      out.f[0] = a.f[0] + b.f[0];
+      out.f[1] = a.f[1] + b.f[1];
+      return out;
+    }
+    //Integer
+    inline int operator()(int a, int b){
+      return a + b;
+    }
+  };
 
+  struct Sub{
+    //Complex/Real float
+    inline u128f operator()(u128f a, u128f b){
+      u128f out;
+      out.f[0] = a.f[0] - b.f[0];
+      out.f[1] = a.f[1] - b.f[1];
+      out.f[2] = a.f[2] - b.f[2];
+      out.f[3] = a.f[3] - b.f[3];
+      return out;
+    }
+    //Complex/Real double
+    inline u128d operator()(u128d a, u128d b){
+      u128d out;
+      out.f[0] = a.f[0] - b.f[0];
+      out.f[1] = a.f[1] - b.f[1];
+      return out;
+    }
+    //Integer
+    inline int operator()(int a, int b){
+      return a-b;
+    }
+  };
+
+  struct MultComplex{
+    // Complex float
+    inline u128f operator()(u128f a, u128f b){
+      u128f out;
+      out.f[0] = a.f[0]*b.f[0] - a.f[1]*b.f[1];
+      out.f[1] = a.f[0]*b.f[1] + a.f[1]*b.f[0];
+      out.f[2] = a.f[2]*b.f[2] - a.f[3]*b.f[3];
+      out.f[3] = a.f[2]*b.f[3] + a.f[3]*b.f[2];
+      return out;
+    }
+    // Complex double
+    inline u128d operator()(u128d a, u128d b){
+      u128d out;
+      out.f[0] = a.f[0]*b.f[0] - a.f[1]*b.f[1];
+      out.f[1] = a.f[0]*b.f[1] + a.f[1]*b.f[0];
+      return out;
+    }
+  };
+
+  struct Mult{
+    //CK: Appear unneeded
+    // inline float  mac(float a, float b,double c){
+    //   return 0;
+    // }
+    // inline double mac(double a, double b,double c){
+    //   return 0;
+    // }
+
+    // Real float
+    inline u128f operator()(u128f a, u128f b){
+      u128f out;
+      out.f[0] = a.f[0]*b.f[0];
+      out.f[1] = a.f[1]*b.f[1];
+      out.f[2] = a.f[2]*b.f[2];
+      out.f[3] = a.f[3]*b.f[3];
+      return out;
+    }
+    // Real double
+    inline u128d operator()(u128d a, u128d b){
+      u128d out;
+      out.f[0] = a.f[0]*b.f[0];
+      out.f[1] = a.f[1]*b.f[1];
+      return out;
+    }
+    // Integer
+    inline int operator()(int a, int b){
+      return a*b;
+    }
+  };
+
+  struct Conj{
+    // Complex single
+    inline u128f operator()(u128f in){
+      u128f out;
+      out.f[0] = in.f[0];
+      out.f[1] = -in.f[1];
+      out.f[2] = in.f[2];
+      out.f[3] = -in.f[3];
+      return out;
+    }
+    // Complex double
+    inline u128d operator()(u128d in){
+      u128d out;
+      out.f[0] = in.f[0];
+      out.f[1] = -in.f[1];
+      return out;
+    }
+    // do not define for integer input
+  };
+
+  struct TimesMinusI{
+    //Complex single
+    inline u128f operator()(u128f in, u128f ret){ //note ret is ignored
+      u128f out;
+      out.f[0] = in.f[1];
+      out.f[1] = -in.f[0];
+      out.f[2] = in.f[3];
+      out.f[3] = -in.f[2];
+      return out;
+    }
+    //Complex double
+    inline u128d operator()(u128d in, u128d ret){
+      u128d out;
+      out.f[0] = in.f[1];
+      out.f[1] = -in.f[0];
+      return out;
+    }
+  };
+
+  struct TimesI{
+    //Complex single
+    inline u128f operator()(u128f in, u128f ret){ //note ret is ignored
+      u128f out;
+      out.f[0] = -in.f[1];
+      out.f[1] = in.f[0];
+      out.f[2] = -in.f[3];
+      out.f[3] = in.f[2];
+      return out;
+    }
+    //Complex double
+    inline u128d operator()(u128d in, u128d ret){
+      u128d out;
+      out.f[0] = -in.f[1];
+      out.f[1] = in.f[0];
+      return out;
+    }
+  };
+
+  //////////////////////////////////////////////
+  // Some Template specialization
+  struct Permute{
+    //We just have to mirror the permutes of Grid_sse4.h
+    static inline u128f Permute0(u128f in){ //AB CD -> CD AB
+      u128f out;
+      out.f[0] = in.f[2];
+      out.f[1] = in.f[3];
+      out.f[2] = in.f[0];
+      out.f[3] = in.f[1];
+      return out;
+    };
+    static inline u128f Permute1(u128f in){ //AB CD -> BA DC
+      u128f out;
+      out.f[0] = in.f[1];
+      out.f[1] = in.f[0];
+      out.f[2] = in.f[3];
+      out.f[3] = in.f[2];
+      return out;
+    };
+    static inline u128f Permute2(u128f in){
+      return in;
+    };
+    static inline u128f Permute3(u128f in){
+      return in;
+    };
+
+    static inline u128d Permute0(u128d in){ //AB -> BA
+      u128d out;
+      out.f[0] = in.f[1];
+      out.f[1] = in.f[0];
+      return out;      
+    };
+    static inline u128d Permute1(u128d in){
+      return in;
+    };
+    static inline u128d Permute2(u128d in){
+      return in;
+    };
+    static inline u128d Permute3(u128d in){
+      return in;
+    };
+
+  };
+  
+  template < typename vtype > 
+    void permute(vtype &a, vtype b, int perm) {
+   };
+    
+  struct Rotate{
+
+    static inline u128f rotate(u128f in,int n){
+      u128f out;
+      switch(n){
+      case 0:
+        out.f[0] = in.f[0];
+        out.f[1] = in.f[1];
+        out.f[2] = in.f[2];
+        out.f[3] = in.f[3];
+        break;
+      case 1:
+        out.f[0] = in.f[1];
+        out.f[1] = in.f[2];
+        out.f[2] = in.f[3];
+        out.f[3] = in.f[0];
+        break;
+      case 2:
+        out.f[0] = in.f[2];
+        out.f[1] = in.f[3];
+        out.f[2] = in.f[0];
+        out.f[3] = in.f[1];
+        break;
+      case 3:
+        out.f[0] = in.f[3];
+        out.f[1] = in.f[0];
+        out.f[2] = in.f[1];
+        out.f[3] = in.f[2];
+        break;
+      default: assert(0);
+      }
+      return out;
+    }
+    static inline u128d rotate(u128d in,int n){
+      u128d out;
+      switch(n){
+      case 0:
+        out.f[0] = in.f[0];
+        out.f[1] = in.f[1];
+        break;
+      case 1:
+        out.f[0] = in.f[1];
+        out.f[1] = in.f[0];
+        break;
+      default: assert(0);
+      }
+      return out;
+    }
+  };
+
+  //Complex float Reduce
+  template<>
+  inline Grid::ComplexF Reduce<Grid::ComplexF, u128f>::operator()(u128f in){ //2 complex
+    return Grid::ComplexF(in.f[0] + in.f[2], in.f[1] + in.f[3]);
+  }
   //Real float Reduce
   template<>
-  inline Grid::RealF Reduce<Grid::RealF, vecf>::operator()(vecf in){
-    float a = 0.;
-    
-    acc(in.v, a, 0, 1, W<float>::r);
-    
-    return a;
+  inline Grid::RealF Reduce<Grid::RealF, u128f>::operator()(u128f in){ //4 floats
+    return in.f[0] + in.f[1] + in.f[2] + in.f[3];
   }
   
+  
   //Complex double Reduce
   template<>
-  inline Grid::ComplexD Reduce<Grid::ComplexD, vecd>::operator()(vecd in){
-    double a = 0., b = 0.;
-    
-    acc(in.v, a, 0, 2, W<double>::r);
-    acc(in.v, b, 1, 2, W<double>::r);
-    
-    return Grid::ComplexD(a, b);
+  inline Grid::ComplexD Reduce<Grid::ComplexD, u128d>::operator()(u128d in){ //1 complex
+    return Grid::ComplexD(in.f[0],in.f[1]);
   }
   
   //Real double Reduce
   template<>
-  inline Grid::RealD Reduce<Grid::RealD, vecd>::operator()(vecd in){
-    double a = 0.f;
-    
-    acc(in.v, a, 0, 1, W<double>::r);
-    
-    return a;
+  inline Grid::RealD Reduce<Grid::RealD, u128d>::operator()(u128d in){ //2 doubles
+    return in.f[0] + in.f[1];
   }
 
   //Integer Reduce
   template<>
   inline Integer Reduce<Integer, int>::operator()(int in){
-    return in;
+    // FIXME unimplemented
+   printf("Reduce : Missing integer implementation -> FIX\n");
+    assert(0);
   }
 }
 
 //////////////////////////////////////////////////////////////////////////////////////
 // Here assign types 
 
-  typedef Optimization::vecf SIMD_Ftype; // Single precision type
-  typedef Optimization::vecd SIMD_Dtype; // Double precision type
+  typedef Optimization::u128f SIMD_Ftype;  // Single precision type
+  typedef Optimization::u128d SIMD_Dtype; // Double precision type
   typedef int SIMD_Itype; // Integer type
 
   // prefetch utilities
   inline void v_prefetch0(int size, const char *ptr){};
   inline void prefetch_HINT_T0(const char *ptr){};
 
+
+
+  // Gpermute function
+  template < typename VectorSIMD > 
+    inline void Gpermute(VectorSIMD &y,const VectorSIMD &b, int perm ) {
+    Optimization::permute(y.v,b.v,perm);
+  }
+
+
   // Function name aliases
   typedef Optimization::Vsplat   VsplatSIMD;
   typedef Optimization::Vstore   VstoreSIMD;
@@ -450,13 +481,16 @@ namespace Optimization {
   typedef Optimization::Vstream  VstreamSIMD;
   template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
 
+ 
+
+
   // Arithmetic operations
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
-  typedef Optimization::Div         DivSIMD;
   typedef Optimization::Mult        MultSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;
   typedef Optimization::TimesMinusI TimesMinusISIMD;
   typedef Optimization::TimesI      TimesISIMD;
+
 }

lib/simd/Grid_imci.h

@@ -26,6 +26,14 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
+//----------------------------------------------------------------------
+/*! @file Grid_knc.h
+  @brief Optimization libraries for AVX512 instructions set for KNC
+
+  Using intrinsics
+*/
+// Time-stamp: <2015-06-09 14:27:28 neo>
+//----------------------------------------------------------------------
 
 #include <immintrin.h>
 #include <zmmintrin.h>
@@ -236,17 +244,6 @@ namespace Optimization {
     }
   };
 
-  struct Div{
-    // Real float
-    inline __m512 operator()(__m512 a, __m512 b){
-      return _mm512_div_ps(a,b);
-    }
-    // Real double
-    inline __m512d operator()(__m512d a, __m512d b){
-      return _mm512_div_pd(a,b);
-    }
-  };
-
 
   struct Conj{
     // Complex single
@@ -440,7 +437,6 @@ namespace Optimization {
   // Arithmetic operations
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
-  typedef Optimization::Div         DivSIMD;
   typedef Optimization::Mult        MultSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;

lib/simd/Grid_qpx.h

@@ -245,21 +245,6 @@ namespace Optimization {
     }
   };
   
-  struct Div{
-    // Real double
-    inline vector4double operator()(vector4double a, vector4double b){
-      return vec_swdiv(a, b);
-    }
-
-    // Real float
-    FLOAT_WRAP_2(operator(), inline)
-
-    // Integer
-    inline int operator()(int a, int b){
-      return a/b;
-    }
-  };
-
   struct Conj{
     // Complex double
     inline vector4double operator()(vector4double v){
@@ -428,7 +413,6 @@ template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
 typedef Optimization::Sum         SumSIMD;
 typedef Optimization::Sub         SubSIMD;
 typedef Optimization::Mult        MultSIMD;
-typedef Optimization::Div         DivSIMD;
 typedef Optimization::MultComplex MultComplexSIMD;
 typedef Optimization::Conj        ConjSIMD;
 typedef Optimization::TimesMinusI TimesMinusISIMD;

lib/simd/Grid_sse4.h

@@ -224,18 +224,6 @@ namespace Optimization {
     }
   };
 
-  struct Div{
-    // Real float
-    inline __m128 operator()(__m128 a, __m128 b){
-      return _mm_div_ps(a,b);
-    }
-    // Real double
-    inline __m128d operator()(__m128d a, __m128d b){
-      return _mm_div_pd(a,b);
-    }
-  };
-
-
   struct Conj{
     // Complex single
     inline __m128 operator()(__m128 in){
@@ -384,8 +372,6 @@ namespace Optimization {
   }
 }
 
-
-
 //////////////////////////////////////////////////////////////////////////////////////
 // Here assign types 
 
@@ -412,7 +398,6 @@ namespace Optimization {
   // Arithmetic operations
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
-  typedef Optimization::Div         DivSIMD;
   typedef Optimization::Mult        MultSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;

lib/simd/Grid_vector_types.h

@@ -38,13 +38,13 @@ directory
 #ifndef GRID_VECTOR_TYPES
 #define GRID_VECTOR_TYPES
 
-#ifdef GEN
+#ifdef GENERIC_VEC
 #include "Grid_generic.h"
 #endif
 #ifdef SSE4
 #include "Grid_sse4.h"
 #endif
-#if defined(AVX1) || defined (AVXFMA) || defined(AVX2) || defined(AVXFMA4)
+#if defined(AVX1) || defined(AVX2) || defined(AVXFMA4)
 #include "Grid_avx.h"
 #endif
 #if defined AVX512
@@ -60,13 +60,6 @@ directory
 #include "Grid_neon.h"
 #endif
 
-#ifdef GRID_DEFAULT_PRECISION_SINGLE
-#define GRID_REAL_DIGITS FLT_DIG
-#endif
-#ifdef GRID_DEFAULT_PRECISION_DOUBLE
-#define GRID_REAL_DIGITS DBL_DIG
-#endif
-
 namespace Grid {
 
 //////////////////////////////////////
@@ -84,24 +77,38 @@ struct RealPart<std::complex<T> > {
 //////////////////////////////////////
 // demote a vector to real type
 //////////////////////////////////////
+
 // type alias used to simplify the syntax of std::enable_if
-template <typename T> using Invoke = typename T::type;
-template <typename Condition, typename ReturnType> using EnableIf = Invoke<std::enable_if<Condition::value, ReturnType> >;
-template <typename Condition, typename ReturnType> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
+template <typename T>
+using Invoke = typename T::type;
+template <typename Condition, typename ReturnType>
+using EnableIf = Invoke<std::enable_if<Condition::value, ReturnType> >;
+template <typename Condition, typename ReturnType>
+using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
 
 ////////////////////////////////////////////////////////
 // Check for complexity with type traits
-template <typename T> struct is_complex : public std::false_type {};
-template <> struct is_complex<std::complex<double> > : public std::true_type {};
-template <> struct is_complex<std::complex<float> > : public std::true_type {};
+template <typename T>
+struct is_complex : public std::false_type {};
+template <>
+struct is_complex<std::complex<double> > : public std::true_type {};
+template <>
+struct is_complex<std::complex<float> > : public std::true_type {};
 
-template <typename T> using IfReal       = Invoke<std::enable_if<std::is_floating_point<T>::value, int> >;
-template <typename T> using IfComplex    = Invoke<std::enable_if<is_complex<T>::value, int> >;
-template <typename T> using IfInteger    = Invoke<std::enable_if<std::is_integral<T>::value, int> >;
+template <typename T>
+using IfReal = Invoke<std::enable_if<std::is_floating_point<T>::value, int> >;
+template <typename T>
+using IfComplex = Invoke<std::enable_if<is_complex<T>::value, int> >;
+template <typename T>
+using IfInteger = Invoke<std::enable_if<std::is_integral<T>::value, int> >;
 
-template <typename T> using IfNotReal    = Invoke<std::enable_if<!std::is_floating_point<T>::value, int> >;
-template <typename T> using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
-template <typename T> using IfNotInteger = Invoke<std::enable_if<!std::is_integral<T>::value, int> >;
+template <typename T>
+using IfNotReal =
+    Invoke<std::enable_if<!std::is_floating_point<T>::value, int> >;
+template <typename T>
+using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
+template <typename T>
+using IfNotInteger = Invoke<std::enable_if<!std::is_integral<T>::value, int> >;
 
 ////////////////////////////////////////////////////////
 // Define the operation templates functors
@@ -137,7 +144,7 @@ class Grid_simd {
 
   Vector_type v;
 
-  static inline constexpr int Nsimd(void) {
+  static inline int Nsimd(void) {
     return sizeof(Vector_type) / sizeof(Scalar_type);
   }
 
@@ -153,23 +160,6 @@ class Grid_simd {
   Grid_simd(const Grid_simd &rhs) : v(rhs.v){};  // compiles in movaps
   Grid_simd(const Grid_simd &&rhs) : v(rhs.v){};
 
-  /////////////////////////////
-  // Comparisons
-  /////////////////////////////
-  inline bool operator==(const Grid_simd& lhs){ 
-    Grid_simd::conv_t conv1;
-    Grid_simd::conv_t conv2;
-    conv1.v = v;
-    conv2.v = lhs.v;
-    return std::equal(std::begin(conv1.s), std::end(conv1.s), std::begin(conv2.s));
-  }
-
-  inline bool operator!=(const Grid_simd& lhs){ 
-    return !(*this == lhs);
-  }
-
-
-
   /////////////////////////////
   // Constructors
   /////////////////////////////
@@ -295,20 +285,6 @@ class Grid_simd {
     return a * b;
   }
 
-  //////////////////////////////////
-  // Divides
-  //////////////////////////////////
-  friend inline Grid_simd operator/(const Scalar_type &a, Grid_simd b) {
-    Grid_simd va;
-    vsplat(va, a);
-    return va / b;
-  }
-  friend inline Grid_simd operator/(Grid_simd b, const Scalar_type &a) {
-    Grid_simd va;
-    vsplat(va, a);
-    return b / a;
-  }
-
   ///////////////////////
   // Unary negation
   ///////////////////////
@@ -452,6 +428,7 @@ inline void rotate(Grid_simd<S,V> &ret,Grid_simd<S,V> b,int nrot)
   ret.v = Optimization::Rotate::rotate(b.v,2*nrot);
 }
 
+
 template <class S, class V> 
 inline void vbroadcast(Grid_simd<S,V> &ret,const Grid_simd<S,V> &src,int lane){
   S* typepun =(S*) &src;
@@ -535,6 +512,7 @@ template <class S, class V, IfInteger<S> = 0>
 inline void vfalse(Grid_simd<S, V> &ret) {
   vsplat(ret, 0);
 }
+
 template <class S, class V>
 inline void zeroit(Grid_simd<S, V> &z) {
   vzero(z);
@@ -552,6 +530,7 @@ inline void vstream(Grid_simd<S, V> &out, const Grid_simd<S, V> &in) {
   typedef typename S::value_type T;
   binary<void>((T *)&out.v, in.v, VstreamSIMD());
 }
+
 template <class S, class V, IfInteger<S> = 0>
 inline void vstream(Grid_simd<S, V> &out, const Grid_simd<S, V> &in) {
   out = in;
@@ -590,34 +569,6 @@ inline Grid_simd<S, V> operator*(Grid_simd<S, V> a, Grid_simd<S, V> b) {
   return ret;
 };
 
-// Distinguish between complex types and others
-template <class S, class V, IfComplex<S> = 0>
-inline Grid_simd<S, V> operator/(Grid_simd<S, V> a, Grid_simd<S, V> b) {
-  typedef Grid_simd<S, V> simd;
-
-  simd ret;
-  simd den;
-  typename simd::conv_t conv;
-
-  ret = a * conjugate(b) ;
-  den = b * conjugate(b) ;
-
-  
-  auto real_den = toReal(den);
-
-  ret.v=binary<V>(ret.v, real_den.v, DivSIMD());
-
-  return ret;
-};
-
-// Real/Integer types
-template <class S, class V, IfNotComplex<S> = 0>
-inline Grid_simd<S, V> operator/(Grid_simd<S, V> a, Grid_simd<S, V> b) {
-  Grid_simd<S, V> ret;
-  ret.v = binary<V>(a.v, b.v, DivSIMD());
-  return ret;
-};
-
 ///////////////////////
 // Conjugate
 ///////////////////////
@@ -631,6 +582,7 @@ template <class S, class V, IfNotComplex<S> = 0>
 inline Grid_simd<S, V> conjugate(const Grid_simd<S, V> &in) {
   return in;  // for real objects
 }
+
 // Suppress adj for integer types... // odd; why conjugate above but not adj??
 template <class S, class V, IfNotInteger<S> = 0>
 inline Grid_simd<S, V> adj(const Grid_simd<S, V> &in) {
@@ -644,12 +596,14 @@ template <class S, class V, IfComplex<S> = 0>
 inline void timesMinusI(Grid_simd<S, V> &ret, const Grid_simd<S, V> &in) {
   ret.v = binary<V>(in.v, ret.v, TimesMinusISIMD());
 }
+
 template <class S, class V, IfComplex<S> = 0>
 inline Grid_simd<S, V> timesMinusI(const Grid_simd<S, V> &in) {
   Grid_simd<S, V> ret;
   timesMinusI(ret, in);
   return ret;
 }
+
 template <class S, class V, IfNotComplex<S> = 0>
 inline Grid_simd<S, V> timesMinusI(const Grid_simd<S, V> &in) {
   return in;
@@ -662,12 +616,14 @@ template <class S, class V, IfComplex<S> = 0>
 inline void timesI(Grid_simd<S, V> &ret, const Grid_simd<S, V> &in) {
   ret.v = binary<V>(in.v, ret.v, TimesISIMD());
 }
+
 template <class S, class V, IfComplex<S> = 0>
 inline Grid_simd<S, V> timesI(const Grid_simd<S, V> &in) {
   Grid_simd<S, V> ret;
   timesI(ret, in);
   return ret;
 }
+
 template <class S, class V, IfNotComplex<S> = 0>
 inline Grid_simd<S, V> timesI(const Grid_simd<S, V> &in) {
   return in;

lib/stencil/Lebesgue.cc

@@ -32,7 +32,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 
 int LebesgueOrder::UseLebesgueOrder;
-std::vector<int> LebesgueOrder::Block({8,2,2,2});
+std::vector<int> LebesgueOrder::Block({2,2,2,2});
 
 LebesgueOrder::IndexInteger LebesgueOrder::alignup(IndexInteger n){
   n--;           // 1000 0011 --> 1000 0010

lib/tensors/Tensor_arith_mul.h

@@ -127,36 +127,6 @@ iVector<rtype,N> operator * (const iVector<mtype,N>& lhs,const iScalar<vtype>& r
     return ret;
 }
     
-//////////////////////////////////////////////////////////////////
-// Divide by scalar
-//////////////////////////////////////////////////////////////////
-template<class rtype,class vtype> strong_inline
-iScalar<rtype> operator / (const iScalar<rtype>& lhs,const iScalar<vtype>& rhs)
-{
-    iScalar<rtype> ret;
-    ret._internal = lhs._internal/rhs._internal;
-    return ret;
-}
-template<class rtype,class vtype,int N> strong_inline
-iVector<rtype,N> operator / (const iVector<rtype,N>& lhs,const iScalar<vtype>& rhs)
-{
-    iVector<rtype,N> ret;
-    for(int i=0;i<N;i++){
-      ret._internal[i] = lhs._internal[i]/rhs._internal;
-    }
-    return ret;
-}
-template<class rtype,class vtype,int N> strong_inline
-iMatrix<rtype,N> operator / (const iMatrix<rtype,N>& lhs,const iScalar<vtype>& rhs)
-{
-    iMatrix<rtype,N> ret;
-    for(int i=0;i<N;i++){
-    for(int j=0;j<N;j++){
-      ret._internal[i][j] = lhs._internal[i][j]/rhs._internal;
-    }}
-    return ret;
-}
-    
     //////////////////////////////////////////////////////////////////
     // Glue operators to mult routines. Must resolve return type cleverly from typeof(internal)
     // since nesting matrix<scalar> x matrix<matrix>-> matrix<matrix>

lib/tensors/Tensor_extract_merge.h

@@ -250,7 +250,8 @@ void merge(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int
   }
  }
 
-template<class vobj> inline void merge1(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
+template<class vobj> inline 
+void merge1(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
 {
   typedef typename vobj::scalar_type scalar_type ;
   typedef typename vobj::vector_type vector_type ;
@@ -268,7 +269,8 @@ template<class vobj> inline void merge1(vobj &vec,std::vector<typename vobj::sca
   }}
 }
 
-template<class vobj> inline void merge2(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
+template<class vobj> inline 
+void merge2(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
 {
   typedef typename vobj::scalar_type scalar_type ;
   typedef typename vobj::vector_type vector_type ;

m4/ac_prog_doxygen.m4

@@ -0,0 +1,54 @@
+dnl Check for doxygen to create API docs
+dnl
+AC_DEFUN([AC_PROG_DOXYGEN],
+[
+AC_ARG_ENABLE(doxygen,
+AS_HELP_STRING([--enable-doxygen],[enable documentation generation with doxygen (auto)]))
+
+AC_ARG_ENABLE(dot,
+AS_HELP_STRING([--enable-dot],[use 'dot' to generate graphs in doxygen (auto)]))              
+AC_ARG_ENABLE(html-docs,
+AS_HELP_STRING([--enable-html-docs],[enable HTML generation with doxygen (yes)]),
+[],[ enable_html_docs=yes])              
+AC_ARG_ENABLE(latex-docs,
+AS_HELP_STRING([--enable-latex-docs],
+[enable LaTeX documentation generation with doxygen (no)]),[],[enable_latex_docs=no])              
+
+if test "x$enable_doxygen" = xno; then
+ enable_doc=no	
+else 
+ AC_CHECK_PROG(DOXYGEN, doxygen, doxygen)
+ if test x$DOXYGEN = x; then
+  if test "x$enable_doxygen" = xyes; then
+   AC_MSG_ERROR([could not find doxygen])
+  fi
+  enable_doc=no
+ else
+  doxy_ver=`doxygen --version`
+  doxy_major=`expr "$doxy_ver" : '\(@<:@0-9@:>@\)\..*'`
+  doxy_minor=`expr "$doxy_ver" : '@<:@0-9@:>@\.\(@<:@0-9@:>@\).*'`
+  if test $doxy_major -eq "1" -a $doxy_minor -ge "3" ; then
+   enable_doc=yes
+   AC_CHECK_PROG(DOT, dot, dot)
+  else
+   AC_MSG_WARN([doxygen version $doxy_ver too old, doxygen will not be used.])
+   enable_doc=no
+  fi
+ fi
+fi
+
+AM_CONDITIONAL(DOXYGEN_DOC, test x$enable_doc = xyes)
+
+if test x$DOT = x; then
+ if test "x$enable_dot" = xyes; then
+  AC_MSG_ERROR([could not find dot])
+ fi
+ enable_dot=no
+else
+ enable_dot=yes
+fi
+
+AC_SUBST(enable_dot)
+AC_SUBST(enable_html_docs)
+AC_SUBST(enable_latex_docs)
+])

scripts/arm_configure.experimental

@@ -0,0 +1 @@
+./configure --host=arm-linux-gnueabihf  CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target arm-linux-gnueabihf -I/usr/arm-linux-gnueabihf/include/ -I/home/neo/Codes/gmp6.0/gmp-arm/include/ -I/usr/arm-linux-gnueabihf/include/c++/4.8.2/arm-linux-gnueabihf/ -L/home/neo/Codes/gmp6.0/gmp-arm/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-arm/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-arm/lib/ -static -mcpu=cortex-a7' --enable-simd=NEONv7

scripts/arm_configure.experimental_cortex57

@@ -0,0 +1,3 @@
+#./configure --host=arm-linux-gnueabihf  CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target arm-linux-gnueabihf -I/usr/arm-linux-gnueabihf/include/ -I/home/neo/Codes/gmp6.0/gmp-arm/include/ -I/usr/lib/llvm-3.5/lib/clang/3.5.0/include/ -L/home/neo/Codes/gmp6.0/gmp-arm/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-arm/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-arm/lib/ -static -mcpu=cortex-a57' --enable-simd=NEONv7
+
+./configure --host=aarch64-linux-gnu  CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target aarch64-linux-gnu -static -I/home/neo/Codes/gmp6.0/gmp-armv8/include/ -L/home/neo/Codes/gmp6.0/gmp-armv8/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-armv8/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-armv8/lib/ -I/usr/aarch64-linux-gnu/include/ -I/usr/aarch64-linux-gnu/include/c++/4.8.2/aarch64-linux-gnu/' --enable-simd=NEONv7

scripts/bench_wilson.sh

@@ -0,0 +1,9 @@
+for omp in 1 2 4
+do
+echo > wilson.t$omp
+for vol in 4.4.4.4 4.4.4.8 4.4.8.8  4.8.8.8  8.8.8.8   8.8.8.16 8.8.16.16  8.16.16.16
+do   
+perf=` ./benchmarks/Grid_wilson --grid $vol --omp $omp  | grep mflop | awk '{print $3}'`
+echo $vol $perf >> wilson.t$omp
+done
+done

scripts/build-all

@@ -0,0 +1,46 @@
+#!/bin/bash  -e
+
+DIRS="clang-avx clang-avx-openmp clang-avx-openmp-mpi clang-avx-mpi clang-avx2 clang-avx2-openmp clang-avx2-openmp-mpi clang-avx2-mpi clang-sse"
+EXTRADIRS="g++-avx g++-sse4 icpc-avx icpc-avx2 icpc-avx512"
+BLACK="\033[30m"
+RED="\033[31m"
+GREEN="\033[32m"
+YELLOW="\033[33m"
+BLUE="\033[34m"
+PINK="\033[35m"
+CYAN="\033[36m"
+WHITE="\033[37m"
+NORMAL="\033[0;39m"
+
+for D in $DIRS
+do
+
+echo
+echo -e $RED ==============================
+echo -e $GREEN $D
+echo -e $RED ==============================
+echo -e $BLUE
+
+  cd builds/$D
+  make clean all -j 8
+  cd ../../
+echo -e $NORMAL
+done
+
+if [ "X$1" == "Xextra" ]
+then
+for D in $EXTRADIRS
+do
+
+echo
+echo -e $RED ==============================
+echo -e $RED $D
+echo -e $RED ==============================
+echo -e $BLUE
+
+  cd builds/$D
+  make clean all -j 8
+  cd ../../
+echo -e $NORMAL
+done
+fi