From 62febd2823e63d88cc9c2041120a0d240cd71c8e Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:23:09 +0100
Subject: [PATCH 001/149] Wilson prop test

---
 tests/core/Test_fft.cc | 86 +++++++++++++++++++++++++++++++-----------
 1 file changed, 64 insertions(+), 22 deletions(-)

diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index 2fdaeed2..c076695d 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -46,16 +46,19 @@ int main (int argc, char ** argv)
   for(int d=0;d<latt_size.size();d++){
     vol = vol * latt_size[d];
   }
-  GridCartesian        Fine(latt_size,simd_layout,mpi_layout);
+  GridCartesian         GRID(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian RBGRID(latt_size,simd_layout,mpi_layout);
 
-  LatticeComplexD     one(&Fine);
-  LatticeComplexD      zz(&Fine);
-  LatticeComplexD       C(&Fine);
-  LatticeComplexD  Ctilde(&Fine);
-  LatticeComplexD    coor(&Fine);
+  LatticeComplexD     one(&GRID);
+  LatticeComplexD      zz(&GRID);
+  LatticeComplexD       C(&GRID);
+  LatticeComplexD  Ctilde(&GRID);
+  LatticeComplexD  Cref  (&GRID);
+  LatticeComplexD  Csav  (&GRID);
+  LatticeComplexD    coor(&GRID);
 
-  LatticeSpinMatrixD    S(&Fine);
-  LatticeSpinMatrixD    Stilde(&Fine);
+  LatticeSpinMatrixD    S(&GRID);
+  LatticeSpinMatrixD    Stilde(&GRID);
   
   std::vector<int> p({1,2,3,2});
 
@@ -72,31 +75,41 @@ int main (int argc, char ** argv)
   }
 
   C = exp(C*ci);
-
+  Csav = C;
   S=zero;
   S = S+C;
 
-  FFT theFFT(&Fine);
+  FFT theFFT(&GRID);
+
+  theFFT.FFT_dim(Ctilde,C,0,FFT::forward);  C=Ctilde;
+  theFFT.FFT_dim(Ctilde,C,1,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<" Mflops "<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,2,FFT::forward);  C=Ctilde;
+  theFFT.FFT_dim(Ctilde,C,3,FFT::forward);  
 
-  theFFT.FFT_dim(Ctilde,C,0,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Ctilde,C,1,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Ctilde,C,2,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Ctilde,C,3,FFT::forward);  std::cout << theFFT.MFlops()<<std::endl;
 
   //  C=zero;
   //  Ctilde = where(abs(Ctilde)<1.0e-10,C,Ctilde);
   TComplexD cVol;
   cVol()()() = vol;
 
-  C=zero;
-  pokeSite(cVol,C,p);
-  C=C-Ctilde;
-  std::cout << "diff scalar "<<norm2(C) << std::endl;
+  Cref=zero;
+  pokeSite(cVol,Cref,p);
+  Cref=Cref-Ctilde;
+  std::cout << "diff scalar "<<norm2(Cref) << std::endl;
 
-  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,3,FFT::forward);std::cout << theFFT.MFlops()<<std::endl;
+  C=Csav;
+  theFFT.FFT_all_dim(Ctilde,C,FFT::forward);
+  theFFT.FFT_all_dim(Cref,Ctilde,FFT::backward); 
+
+  std::cout << norm2(C) << " " << norm2(Ctilde) << " " << norm2(Cref)<< " vol " << vol<< std::endl;
+
+  Cref= Cref - C;
+  std::cout << " invertible check " << norm2(Cref)<<std::endl;
+
+  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde;
+  theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<" mflops "<<std::endl;
+  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;
+  theFFT.FFT_dim(Stilde,S,3,FFT::forward);
 
   SpinMatrixD Sp; 
   Sp = zero; Sp = Sp+cVol;
@@ -107,5 +120,34 @@ int main (int argc, char ** argv)
   S= S-Stilde;
   std::cout << "diff FT[SpinMat] "<<norm2(S) << std::endl;
 
+  /*
+   */
+  std::vector<int> seeds({1,2,3,4});
+  GridParallelRNG          pRNG(&GRID);
+  pRNG.SeedFixedIntegers(seeds);
+
+  LatticeGaugeFieldD Umu(&GRID);
+  LatticeFermionD    src(&GRID); gaussian(pRNG,src);
+  LatticeFermionD    tmp(&GRID);
+  LatticeFermionD    ref(&GRID);
+
+  SU3::ColdConfiguration(pRNG,Umu); // Unit gauge
+  
+  RealD mass=0.1;
+  WilsonFermionD Dw(Umu,GRID,RBGRID,mass);
+
+  Dw.M(src,tmp);
+
+  std::cout << " src = " <<norm2(src)<<std::endl;
+  std::cout << " tmp = " <<norm2(tmp)<<std::endl;
+
+  Dw.FreePropagator(tmp,ref);
+
+  std::cout << " ref = " <<norm2(ref)<<std::endl;
+  
+  ref = ref - src;
+  
+  std::cout << " ref-src = " <<norm2(ref)<<std::endl;
+
   Grid_finalize();
 }

From 0744f38866e711ffa894d83dc371f1c28c54c131 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:23:28 +0100
Subject: [PATCH 002/149] Demangle support is useful

---
 lib/Init.cc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/lib/Init.cc b/lib/Init.cc
index 34e503a4..4cb4bb50 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -44,9 +44,26 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid.h>
 #include <algorithm>
 #include <iterator>
+#include <cstdlib>
+#include <memory>
+#include <cxxabi.h>
+
 
 namespace Grid {
 
+    std::string demangle(const char* name) {
+
+      int status = -4; // some arbitrary value to eliminate the compiler warning
+
+      // enable c++11 by passing the flag -std=c++11 to g++
+      std::unique_ptr<char, void(*)(void*)> res {
+	abi::__cxa_demangle(name, NULL, NULL, &status),
+	  std::free
+	  };
+
+      return (status==0) ? res.get() : name ;
+    }
+
 //////////////////////////////////////////////////////
 // Convenience functions to access stadard command line arg
 // driven parallelism controls

From 3475f45ce7feb00a2921c9a846312473ee52d986 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:23:48 +0100
Subject: [PATCH 003/149] Demangle support for typeid stuff

---
 lib/Init.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/Init.h b/lib/Init.h
index 25fda569..6eb07a49 100644
--- a/lib/Init.h
+++ b/lib/Init.h
@@ -52,6 +52,8 @@ namespace Grid {
   void GridCmdOptionCSL(std::string str,std::vector<std::string> & vec);
   void GridCmdOptionIntVector(std::string &str,std::vector<int> & vec);
 
+  std::string demangle(const char* name) ;
+
   void GridParseLayout(char **argv,int argc,
 		       std::vector<int> &latt,
 		       std::vector<int> &simd,

From 9005b82c6d869c22ff8fd326d02b9398483f96c0 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:24:52 +0100
Subject: [PATCH 004/149] Multi dim FFT, and normalisation fix

---
 lib/FFT.h | 42 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index 4cda6483..53a86767 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -32,6 +32,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef HAVE_FFTW	
 #include <fftw3.h>
 #endif
+
+
 namespace Grid {
 
   template<class scalar> struct FFTW { };
@@ -115,7 +117,7 @@ namespace Grid {
 
   public:
 
-    static const int forward=FFTW_FORWARD;
+    static const int forward =FFTW_FORWARD;
     static const int backward=FFTW_BACKWARD;
 
     double Flops(void) {return flops;}
@@ -139,7 +141,29 @@ namespace Grid {
     }
     
     template<class vobj>
-    void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int inverse){
+    void FFT_dim_mask(Lattice<vobj> &result,const Lattice<vobj> &source,std::vector<int> mask,int sign){
+
+      conformable(result._grid,vgrid);
+      conformable(source._grid,vgrid);
+      Lattice<vobj> tmp(vgrid);
+      tmp = source;
+      for(int d=0;d<Nd;d++){
+	if( mask[d] ) {
+	  FFT_dim(result,tmp,d,sign);
+	  tmp=result;
+	}
+      }
+    }
+
+    template<class vobj>
+    void FFT_all_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int sign){
+      std::vector<int> mask(Nd,1);
+      FFT_dim_mask(result,source,mask,sign);
+    }
+
+
+    template<class vobj>
+    void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int sign){
 
       conformable(result._grid,vgrid);
       conformable(source._grid,vgrid);
@@ -158,6 +182,12 @@ namespace Grid {
       // Construct pencils
       typedef typename vobj::scalar_object sobj;
       typedef typename sobj::scalar_type   scalar;
+      
+      /*
+      std::cout << "FFT : vobj   "<<demangle(typeid(vobj).name()) <<std::endl;
+      std::cout << "FFT : sobj   "<<demangle(typeid(sobj).name()) <<std::endl;
+      std::cout << "FFT : scalar "<<demangle(typeid(scalar).name()) <<std::endl;
+      */
 
       Lattice<vobj> ssource(vgrid); ssource =source;
       Lattice<sobj> pgsource(&pencil_g);
@@ -184,9 +214,10 @@ namespace Grid {
 	istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */
 	int *inembed = n, *onembed = n;
 
-	
-	int sign = FFTW_FORWARD;
-	if (inverse) sign = FFTW_BACKWARD;
+	scalar div;
+	if ( sign == backward ) div = 1.0/G;
+	else if ( sign == forward ) div = 1.0;
+	else assert(0);
 
 	FFTW_plan p;
 	{
@@ -258,6 +289,7 @@ PARALLEL_FOR_LOOP
 	  sobj s;
 	  gcoor[dim] = lcoor[dim]+L*pc;
 	  peekLocalSite(s,pgresult,gcoor);
+	  s = s * div;
 	  pokeLocalSite(s,result,lcoor);
 	}
       	  

From d15ab66aae0909685bb163c1738884e5e856f370 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:25:22 +0100
Subject: [PATCH 005/149] FFT moves higher in include order

---
 lib/Grid.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/Grid.h b/lib/Grid.h
index 486ee4d3..0c5983f3 100644
--- a/lib/Grid.h
+++ b/lib/Grid.h
@@ -77,11 +77,10 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/Stencil.h>      
 #include <Grid/Algorithms.h>   
 #include <Grid/parallelIO/BinaryIO.h>
-#include <Grid/qcd/QCD.h>
-#include <Grid/parallelIO/NerscIO.h>
-
 #include <Grid/FFT.h>
 
+#include <Grid/qcd/QCD.h>
+#include <Grid/parallelIO/NerscIO.h>
 #include <Grid/qcd/hmc/NerscCheckpointer.h>
 #include <Grid/qcd/hmc/HmcRunner.h>
 

From 02e983a0cd5b13a34238598824867e50f47fece2 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:26:02 +0100
Subject: [PATCH 006/149] Momentum space prop and free prop convolution

---
 lib/qcd/action/fermion/FermionOperator.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/lib/qcd/action/fermion/FermionOperator.h b/lib/qcd/action/fermion/FermionOperator.h
index ea5583eb..daee2e5b 100644
--- a/lib/qcd/action/fermion/FermionOperator.h
+++ b/lib/qcd/action/fermion/FermionOperator.h
@@ -91,6 +91,19 @@ namespace Grid {
       virtual void  Mdiag  (const FermionField &in, FermionField &out) { Mooee(in,out);};   // Same as Mooee applied to both CB's
       virtual void  Mdir   (const FermionField &in, FermionField &out,int dir,int disp)=0;   // case by case Wilson, Clover, Cayley, ContFrac, PartFrac
 
+
+      virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in) { assert(0);};
+      virtual void  FreePropagator(const FermionField &in,FermionField &out) { 
+	FFT theFFT((GridCartesian *) in._grid);
+
+	FermionField in_k(in._grid);
+	FermionField prop_k(in._grid);
+
+	theFFT.FFT_all_dim(in_k,in,FFT::forward);
+        this->MomentumSpacePropagator(prop_k,in_k);
+	theFFT.FFT_all_dim(out,prop_k,FFT::backward);
+      };
+
       ///////////////////////////////////////////////
       // Updates gauge field during HMC
       ///////////////////////////////////////////////

From 0c1d7e4daf4ad2a7e41a6e5f689bb07682619f58 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:26:36 +0100
Subject: [PATCH 007/149] Mom space prop for Wilson action

---
 lib/qcd/action/fermion/WilsonFermion.cc | 52 +++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/lib/qcd/action/fermion/WilsonFermion.cc b/lib/qcd/action/fermion/WilsonFermion.cc
index 59632409..caf1913d 100644
--- a/lib/qcd/action/fermion/WilsonFermion.cc
+++ b/lib/qcd/action/fermion/WilsonFermion.cc
@@ -136,6 +136,58 @@ namespace QCD {
     out.checkerboard = in.checkerboard;
     MooeeInv(in,out);
   }
+  template<class Impl>
+  void WilsonFermion<Impl>:: MomentumSpacePropagator(FermionField &out, const FermionField &in) {
+
+    // what type LatticeComplex 
+    conformable(_grid,out._grid);
+
+    typedef typename FermionField::vector_type vector_type;
+    typedef typename FermionField::scalar_type ScalComplex;
+
+    typedef Lattice<iSinglet<vector_type> > LatComplex;
+
+    Gamma::GammaMatrix Gmu [] = {
+      Gamma::GammaX,
+      Gamma::GammaY,
+      Gamma::GammaZ,
+      Gamma::GammaT
+    };
+
+    std::vector<int> latt_size   = _grid->_fdimensions;
+
+    FermionField   num  (_grid); num  = zero;
+    LatComplex    wilson(_grid); wilson= zero;
+    LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
+
+    LatComplex denom(_grid); denom= zero;
+    LatComplex kmu(_grid); 
+    ScalComplex ci(0.0,1.0);
+    // momphase = n * 2pi / L
+    for(int mu=0;mu<Nd;mu++) {
+
+      LatticeCoordinate(kmu,mu);
+
+      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+
+      kmu = TwoPiL * kmu;
+
+      wilson = wilson + 2.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
+
+      num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);    // derivative term
+
+      denom=denom + sin(kmu)*sin(kmu);
+    }
+
+    wilson = wilson + mass;     // 2 sin^2 k/2 + m
+
+    num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
+
+    denom= denom+wilson*wilson; // sin^2 k + (2 sin^2 k/2 + m)^2
+    denom= one/denom;
+    out = num*denom; // [ -i gmu sin k + 2 sin^2 k/2 + m] / [ sin^2 k + (2 sin^2 k/2 + m)^2 ]
+
+  }
   
   ///////////////////////////////////
   // Internal

From b573d1f35a7da5a0084b7844ae1339c7fe9c9d36 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:27:04 +0100
Subject: [PATCH 008/149] Wilson tree level added

---
 lib/qcd/action/fermion/WilsonFermion.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/qcd/action/fermion/WilsonFermion.h b/lib/qcd/action/fermion/WilsonFermion.h
index 3de2cac4..c050c0eb 100644
--- a/lib/qcd/action/fermion/WilsonFermion.h
+++ b/lib/qcd/action/fermion/WilsonFermion.h
@@ -78,6 +78,8 @@ namespace Grid {
       virtual void MooeeInv(const FermionField &in, FermionField &out) ;
       virtual void MooeeInvDag(const FermionField &in, FermionField &out) ;
 
+      virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in) ;
+
       ////////////////////////
       // Derivative interface
       ////////////////////////

From 8535d433a745cb50e2fd0dc5f46cf01ee3a61446 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:27:53 +0100
Subject: [PATCH 009/149] Cold or hot must support any precisoin

---
 lib/qcd/utils/SUn.h | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/lib/qcd/utils/SUn.h b/lib/qcd/utils/SUn.h
index e9403836..67da9787 100644
--- a/lib/qcd/utils/SUn.h
+++ b/lib/qcd/utils/SUn.h
@@ -617,22 +617,33 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
       PokeIndex<LorentzIndex>(out,Umu,mu);
     }
   }
-  static void TepidConfiguration(GridParallelRNG &pRNG,LatticeGaugeField &out){
-    LatticeMatrix Umu(out._grid);
+  template<typename GaugeField>
+  static void TepidConfiguration(GridParallelRNG &pRNG,GaugeField &out){
+    typedef typename GaugeField::vector_type vector_type;
+    typedef iSUnMatrix<vector_type> vMatrixType;
+    typedef Lattice<vMatrixType> LatticeMatrixType;
+
+    LatticeMatrixType Umu(out._grid);
     for(int mu=0;mu<Nd;mu++){
       LieRandomize(pRNG,Umu,0.01);
       PokeIndex<LorentzIndex>(out,Umu,mu);
     }
   }
-  static void ColdConfiguration(GridParallelRNG &pRNG,LatticeGaugeField &out){
-    LatticeMatrix Umu(out._grid);
+  template<typename GaugeField>
+  static void ColdConfiguration(GridParallelRNG &pRNG,GaugeField &out){
+    typedef typename GaugeField::vector_type vector_type;
+    typedef iSUnMatrix<vector_type> vMatrixType;
+    typedef Lattice<vMatrixType> LatticeMatrixType;
+
+    LatticeMatrixType Umu(out._grid);
     Umu=1.0;
     for(int mu=0;mu<Nd;mu++){
       PokeIndex<LorentzIndex>(out,Umu,mu);
     }
   }
 
-  static void taProj( const LatticeMatrix &in,  LatticeMatrix &out){
+  template<typename LatticeMatrixType>
+  static void taProj( const LatticeMatrixType &in,  LatticeMatrixType &out){
     out = Ta(in);
   }
   template<typename LatticeMatrixType>

From 7422953e36f895f1a7499452461e0e2c4ef4716c Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Aug 2016 00:42:47 +0100
Subject: [PATCH 010/149] Poisson solver example

---
 tests/core/Test_poisson_fft.cc | 138 +++++++++++++++++++++++++++++++++
 1 file changed, 138 insertions(+)
 create mode 100644 tests/core/Test_poisson_fft.cc

diff --git a/tests/core/Test_poisson_fft.cc b/tests/core/Test_poisson_fft.cc
new file mode 100644
index 00000000..2252ec7a
--- /dev/null
+++ b/tests/core/Test_poisson_fft.cc
@@ -0,0 +1,138 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_poisson_fft.cc
+
+    Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  int N=128;
+  int N2=64;
+  int W=16;
+  int D=8;
+  std::vector<int> latt_size  ({N,N});
+  std::vector<int> simd_layout({vComplexD::Nsimd(),1});
+  std::vector<int> mpi_layout ({1,1});
+
+  int vol = 1;
+  int nd  = latt_size.size();
+  for(int d=0;d<nd;d++){
+    vol = vol * latt_size[d];
+  }
+
+  GridCartesian         GRID(latt_size,simd_layout,mpi_layout);
+
+  LatticeComplexD     pos(&GRID);
+  LatticeComplexD      zz(&GRID);
+  LatticeComplexD     neg(&GRID);
+  LatticeInteger     coor(&GRID);
+  LatticeComplexD  Charge(&GRID);
+  LatticeComplexD  ChargeTilde(&GRID);
+  LatticeComplexD       V(&GRID);
+  LatticeComplexD       Vtilde(&GRID);
+
+  pos = ComplexD(1.0,0.0);
+  neg = -pos;
+  zz  = ComplexD(0.0,0.0);
+
+  Charge=zero;
+  
+  // Parallel plate capacitor
+  {
+    int mu=0;
+    LatticeCoordinate(coor,mu);
+
+    Charge=where(coor==Integer(N2-D),pos,zz);
+    Charge=where(coor==Integer(N2+D),neg,Charge);
+  }
+
+  {
+    int mu=1;
+    LatticeCoordinate(coor,mu);
+    Charge=where(coor<Integer(N2-W),zz,Charge);
+    Charge=where(coor>Integer(N2+W),zz,Charge);
+  }
+
+  //  std::cout << Charge <<std::endl;
+
+  std::vector<LatticeComplexD> k(4,&GRID);
+  LatticeComplexD ksq(&GRID);
+
+  ksq=zero;
+  for(int mu=0;mu<nd;mu++) {
+    
+    Integer L=latt_size[mu];
+    
+    LatticeCoordinate(coor,mu);
+    LatticeCoordinate(k[mu],mu);
+
+    k[mu] = where ( coor > (L/2), k[mu]-L, k[mu]);
+
+    //    std::cout << k[mu]<<std::endl;
+
+    RealD TwoPiL =  M_PI * 2.0/ L;
+
+    k[mu] = TwoPiL * k[mu];      
+
+    ksq = ksq + k[mu]*k[mu];
+
+  }
+  
+  // D^2 V   = - rho
+  // ksq Vtilde = rhoTilde
+  // Vtilde = rhoTilde/Ksq
+  // Fix zero of potential : Vtilde(0) = 0;
+  std::vector<int> zero_mode(nd,0);
+  TComplexD Tone = ComplexD(1.0,0.0);
+  pokeSite(Tone,ksq,zero_mode); 
+
+  //  std::cout << "Charge\n" << Charge <<std::endl;
+
+  FFT theFFT(&GRID);
+  theFFT.FFT_all_dim(ChargeTilde,Charge,FFT::forward); 
+  //  std::cout << "Rhotilde\n" << ChargeTilde <<std::endl;
+
+  Vtilde = ChargeTilde / ksq;
+  //  std::cout << "Vtilde\n" << Vtilde <<std::endl;
+
+  TComplexD Tzero = ComplexD(0.0,0.0);
+  pokeSite(Tzero,Vtilde,zero_mode);
+
+  theFFT.FFT_all_dim(V,Vtilde,FFT::backward); 
+
+  std::cout << "V\n" << V <<std::endl;
+
+  Grid_finalize();
+}

From 296396646d1198fc09709bde98d498aaf70302ed Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:36:14 +0100
Subject: [PATCH 011/149] FPE's on macos set up

---
 lib/Init.cc | 39 +++++++++++++++++++++++----------------
 lib/Init.h  |  1 -
 2 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/lib/Init.cc b/lib/Init.cc
index 4cb4bb50..cff43185 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -46,23 +46,10 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <iterator>
 #include <cstdlib>
 #include <memory>
-#include <cxxabi.h>
 
 
 namespace Grid {
 
-    std::string demangle(const char* name) {
-
-      int status = -4; // some arbitrary value to eliminate the compiler warning
-
-      // enable c++11 by passing the flag -std=c++11 to g++
-      std::unique_ptr<char, void(*)(void*)> res {
-	abi::__cxa_demangle(name, NULL, NULL, &status),
-	  std::free
-	  };
-
-      return (status==0) ? res.get() : name ;
-    }
 
 //////////////////////////////////////////////////////
 // Convenience functions to access stadard command line arg
@@ -343,10 +330,30 @@ void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr)
   exit(0);
   return;
 };
-#ifdef GRID_FPE
+
+
 #define _GNU_SOURCE
 #include <fenv.h>
+
+#ifdef __APPLE__
+static int
+feenableexcept (unsigned int excepts)
+{
+  static fenv_t fenv;
+  unsigned int new_excepts = excepts & FE_ALL_EXCEPT,
+    old_excepts;  // previous masks
+
+  if ( fegetenv (&fenv) ) return -1;
+  old_excepts = fenv.__control & FE_ALL_EXCEPT;
+
+  // unmask
+  fenv.__control &= ~new_excepts;
+  fenv.__mxcsr   &= ~(new_excepts << 7);
+
+  return ( fesetenv (&fenv) ? -1 : old_excepts );
+}
 #endif
+
 void Grid_debug_handler_init(void)
 {
   struct sigaction sa,osa;
@@ -355,9 +362,9 @@ void Grid_debug_handler_init(void)
   sa.sa_flags    = SA_SIGINFO;
   sigaction(SIGSEGV,&sa,NULL);
   sigaction(SIGTRAP,&sa,NULL);
-#ifdef GRID_FPE
+
   feenableexcept( FE_INVALID|FE_OVERFLOW|FE_DIVBYZERO);
+
   sigaction(SIGFPE,&sa,NULL);
-#endif
 }
 }
diff --git a/lib/Init.h b/lib/Init.h
index 6eb07a49..8986d24b 100644
--- a/lib/Init.h
+++ b/lib/Init.h
@@ -52,7 +52,6 @@ namespace Grid {
   void GridCmdOptionCSL(std::string str,std::vector<std::string> & vec);
   void GridCmdOptionIntVector(std::string &str,std::vector<int> & vec);
 
-  std::string demangle(const char* name) ;
 
   void GridParseLayout(char **argv,int argc,
 		       std::vector<int> &latt,

From 567b6cf23f5b33bb101965fc7613b969373db20a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:36:51 +0100
Subject: [PATCH 012/149] demangle moves to logging

---
 lib/Log.cc | 15 +++++++++++++++
 lib/Log.h  |  3 ++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/lib/Log.cc b/lib/Log.cc
index 2082570d..5152bd77 100644
--- a/lib/Log.cc
+++ b/lib/Log.cc
@@ -31,8 +31,23 @@ directory
 /*  END LEGAL */
 #include <Grid.h>
 
+#include <cxxabi.h>
+
 namespace Grid {
 
+  std::string demangle(const char* name) {
+    
+    int status = -4; // some arbitrary value to eliminate the compiler warning
+    
+    // enable c++11 by passing the flag -std=c++11 to g++
+    std::unique_ptr<char, void(*)(void*)> res {
+      abi::__cxa_demangle(name, NULL, NULL, &status),
+	std::free
+	};
+    
+    return (status==0) ? res.get() : name ;
+  }
+  
 GridStopWatch Logger::StopWatch;
 std::ostream Logger::devnull(0);
 
diff --git a/lib/Log.h b/lib/Log.h
index 156f52ee..9f9e19a7 100644
--- a/lib/Log.h
+++ b/lib/Log.h
@@ -143,6 +143,7 @@ extern GridLogger GridLogIterative  ;
 extern GridLogger GridLogIntegrator  ;
 extern Colours    GridLogColours;
 
+ std::string demangle(const char* name) ;
 
 #define _NBACKTRACE (256)
 extern void * Grid_backtrace_buffer[_NBACKTRACE];
@@ -161,7 +162,7 @@ std::fclose(fp);	    \
 int symbols    = backtrace        (Grid_backtrace_buffer,_NBACKTRACE);\
 char **strings = backtrace_symbols(Grid_backtrace_buffer,symbols);\
 for (int i = 0; i < symbols; i++){\
-  std::fprintf (fp,"BackTrace Strings: %d %s\n",i, strings[i]); std::fflush(fp); \
+  std::fprintf (fp,"BackTrace Strings: %d %s\n",i, demangle(strings[i]).c_str()); std::fflush(fp); \
 }\
 }
 #else 

From 16b37b956cb8018ba9acfab1424305854dde750a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:37:59 +0100
Subject: [PATCH 013/149] divide goes to ET

---
 lib/lattice/Lattice_base.h | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/lib/lattice/Lattice_base.h b/lib/lattice/Lattice_base.h
index 3bfa5613..b97e1359 100644
--- a/lib/lattice/Lattice_base.h
+++ b/lib/lattice/Lattice_base.h
@@ -303,17 +303,6 @@ PARALLEL_FOR_LOOP
         *this = (*this)+r;
         return *this;
     }
-    
-    strong_inline friend Lattice<vobj> operator / (const Lattice<vobj> &lhs,const Lattice<vobj> &rhs){
-        conformable(lhs,rhs);
-        Lattice<vobj> ret(lhs._grid);
-PARALLEL_FOR_LOOP
-        for(int ss=0;ss<lhs._grid->oSites();ss++){
-	  ret._odata[ss] = lhs._odata[ss]*pow(rhs._odata[ss],-1.0);
-        }
-        return ret;
-    };
-
  }; // class Lattice
 
   template<class vobj> std::ostream& operator<< (std::ostream& stream, const Lattice<vobj> &o){

From 81a7a0307698246638776d1fad7a6610b6116e83 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:38:17 +0100
Subject: [PATCH 014/149] Integer <<

---
 lib/Simd.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/Simd.h b/lib/Simd.h
index 9568c555..adc2849d 100644
--- a/lib/Simd.h
+++ b/lib/Simd.h
@@ -237,6 +237,18 @@ namespace Grid {
     stream<<">";
     return stream;
   }
+  inline std::ostream& operator<< (std::ostream& stream, const vInteger &o){
+    int nn=vInteger::Nsimd();
+    std::vector<Integer,alignedAllocator<Integer> > buf(nn);
+    vstore(o,&buf[0]);
+    stream<<"<";
+    for(int i=0;i<nn;i++){
+      stream<<buf[i];
+      if(i<nn-1) stream<<",";
+    }
+    stream<<">";
+    return stream;
+  }
 
 
 }

From 52a39f0fcdcdf152b495a86630f377e92277985b Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:38:38 +0100
Subject: [PATCH 015/149] Divide in ET

---
 lib/lattice/Lattice_ET.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/lattice/Lattice_ET.h b/lib/lattice/Lattice_ET.h
index 7ebac99d..1bb83901 100644
--- a/lib/lattice/Lattice_ET.h
+++ b/lib/lattice/Lattice_ET.h
@@ -261,6 +261,7 @@ GridUnopClass(UnaryExp, exp(a));
 GridBinOpClass(BinaryAdd, lhs + rhs);
 GridBinOpClass(BinarySub, lhs - rhs);
 GridBinOpClass(BinaryMul, lhs *rhs);
+GridBinOpClass(BinaryDiv, lhs /rhs);
 
 GridBinOpClass(BinaryAnd, lhs &rhs);
 GridBinOpClass(BinaryOr, lhs | rhs);
@@ -385,6 +386,7 @@ GRID_DEF_UNOP(exp, UnaryExp);
 GRID_DEF_BINOP(operator+, BinaryAdd);
 GRID_DEF_BINOP(operator-, BinarySub);
 GRID_DEF_BINOP(operator*, BinaryMul);
+GRID_DEF_BINOP(operator/, BinaryDiv);
 
 GRID_DEF_BINOP(operator&, BinaryAnd);
 GRID_DEF_BINOP(operator|, BinaryOr);

From b6713ecb60c2a0095ae4af44911b4bd6600d03a8 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:39:09 +0100
Subject: [PATCH 016/149] Momentum space rules for Overlap, DWF untested to
 date

---
 lib/qcd/action/fermion/WilsonFermion.cc   |   2 +
 lib/qcd/action/fermion/WilsonFermion5D.cc | 164 ++++++++++++++++++++++
 lib/qcd/action/fermion/WilsonFermion5D.h  |   3 +
 3 files changed, 169 insertions(+)

diff --git a/lib/qcd/action/fermion/WilsonFermion.cc b/lib/qcd/action/fermion/WilsonFermion.cc
index caf1913d..55fbfa01 100644
--- a/lib/qcd/action/fermion/WilsonFermion.cc
+++ b/lib/qcd/action/fermion/WilsonFermion.cc
@@ -184,7 +184,9 @@ namespace QCD {
     num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
 
     denom= denom+wilson*wilson; // sin^2 k + (2 sin^2 k/2 + m)^2
+
     denom= one/denom;
+
     out = num*denom; // [ -i gmu sin k + 2 sin^2 k/2 + m] / [ sin^2 k + (2 sin^2 k/2 + m)^2 ]
 
   }
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index ef7e84ab..28f02b67 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -392,6 +392,170 @@ void WilsonFermion5D<Impl>::DW(const FermionField &in, FermionField &out,int dag
   axpy(out,4.0-M5,in,out);
 }
 
+template<class Impl>
+void WilsonFermion5D<Impl>::MomentumSpacePropagatorHt(FermionField &out,const FermionField &in) 
+{
+    // what type LatticeComplex 
+    GridBase *_grid = _FourDimGrid;
+    conformable(_grid,out._grid);
+
+    typedef typename FermionField::vector_type vector_type;
+    typedef typename FermionField::scalar_type ScalComplex;
+    typedef iSinglet<ScalComplex> Tcomplex;
+    typedef Lattice<iSinglet<vector_type> > LatComplex;
+
+    Gamma::GammaMatrix Gmu [] = {
+      Gamma::GammaX,
+      Gamma::GammaY,
+      Gamma::GammaZ,
+      Gamma::GammaT
+    };
+
+    std::vector<int> latt_size   = _grid->_fdimensions;
+
+
+    LatComplex    sk(_grid);  sk = zero;
+    LatComplex    sk2(_grid); sk2= zero;
+
+    LatComplex    W(_grid); W= zero;
+    LatComplex    a(_grid); a= zero;
+    LatComplex    cosha(_grid); a= zero;
+
+    LatComplex     one  (_grid);     one = ScalComplex(1.0,0.0);
+
+    FermionField   num  (_grid); num  = zero;
+    LatComplex denom(_grid); denom= zero;
+    LatComplex kmu(_grid); 
+    ScalComplex ci(0.0,1.0);
+
+    for(int mu=0;mu<Nd;mu++) {
+
+      LatticeCoordinate(kmu,mu);
+
+      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+
+      kmu = TwoPiL * kmu;
+
+      sk2 = sk2 + 2.0*sin(kmu*0.5)*sin(kmu*0.5);
+      sk  = sk  + sin(kmu)*sin(kmu); 
+
+      num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);
+
+    }
+
+    W = one - M5 - sk2;
+    cosha =  (one + W*W + sk) / (W*2.0);
+    
+    for(int idx=0;idx<_grid->lSites();idx++){
+      Tcomplex cc;
+      RealD sgn;
+      std::vector<int> lcoor(Nd);
+      _grid->LocalIndexToLocalCoor(idx,lcoor);
+      peekLocalSite(cc,cosha,lcoor);
+      sgn= ::fabs(real(cc));
+      cc = ScalComplex(::acosh(sgn),0.0);
+      pokeLocalSite(cc,a,lcoor);
+    }
+
+  std::cout << "Ht mom space num" << norm2(num)<<std::endl;// num ok
+  std::cout << "Ht mom space W  " << norm2(W)<<std::endl;// w ok
+  std::cout << "Ht mom space a  " << norm2(a)<<std::endl;// a ok
+  denom= ( exp(a) ); 
+  std::cout << "Ht mom space den" << norm2(denom)<<std::endl;
+  denom= ( exp(a) * W  ); 
+  std::cout << "Ht mom space den W" << norm2(denom)<<std::endl;
+  denom= ( exp(a) * W - one ); 
+  std::cout << "Ht mom space den W- 1" << norm2(denom)<<std::endl;
+  denom=where(sk==ScalComplex(0,0),one,denom);
+  std::cout << "Ht mom space den W- 1" << denom<<std::endl;
+  std::cout << "Ht mom space one " << norm2(one)<<std::endl;
+  denom= one/denom;
+  std::cout << "Ht mom space div " << norm2(denom)<<std::endl;
+  out = num*denom;
+  std::cout << "Ht mom space out" << norm2(out)<<std::endl;
+}
+
+template<class Impl>
+void WilsonFermion5D<Impl>::MomentumSpacePropagatorHw(FermionField &out,const FermionField &in) 
+{
+    // what type LatticeComplex 
+    GridBase *_grid = _FourDimGrid;
+    conformable(_grid,out._grid);
+
+    typedef typename FermionField::vector_type vector_type;
+    typedef typename FermionField::scalar_type ScalComplex;
+
+    typedef Lattice<iSinglet<vector_type> > LatComplex;
+
+    Gamma::GammaMatrix Gmu [] = {
+      Gamma::GammaX,
+      Gamma::GammaY,
+      Gamma::GammaZ,
+      Gamma::GammaT
+    };
+
+    std::vector<int> latt_size   = _grid->_fdimensions;
+
+
+    LatComplex    sk(_grid);  sk = zero;
+    LatComplex    sk2(_grid); sk2= zero;
+
+    LatComplex    w_k(_grid); w_k= zero;
+    LatComplex    b_k(_grid); b_k= zero;
+
+    LatComplex     one  (_grid); one = ScalComplex(1.0,0.0);
+
+    FermionField   num  (_grid); num  = zero;
+    LatComplex denom(_grid); denom= zero;
+    LatComplex kmu(_grid); 
+    ScalComplex ci(0.0,1.0);
+
+    for(int mu=0;mu<Nd;mu++) {
+
+      LatticeCoordinate(kmu,mu);
+
+      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+
+      kmu = TwoPiL * kmu;
+
+      sk2 = sk2 + 2.0*sin(kmu*0.5)*sin(kmu*0.5);
+      sk  = sk  + sin(kmu)*sin(kmu); 
+
+      num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);
+
+    }
+
+    b_k = sk2 - M5;
+     
+    w_k = sqrt(sk + b_k*b_k);
+
+    std::cout << "Hw M5 " << M5<<std::endl;
+    std::cout << "Hw mom space NUM" << norm2(num)<<std::endl;
+    std::cout << "Hw mom space BK" << norm2(b_k)<<std::endl;
+    std::cout << "Hw mom space BK" << b_k<<std::endl;
+    std::cout << "Hw mom space WK" << norm2(w_k)<<std::endl;
+    std::cout << "Hw mom space WK" << w_k<<std::endl;
+
+    denom= ( w_k + b_k ) * (2.0 * M5);
+    denom=where(sk==ScalComplex(0,0),one,denom);
+
+    std::cout << "Hw mom space DEMON" << norm2(denom)<<std::endl;
+
+    std::cout << "Hw mom space DEMON" << denom<<std::endl;
+    
+    std::cout << "Hw mom space one" << norm2(one)<<std::endl;
+
+    denom= one/denom;
+
+    std::cout << "Hw mom space div " << norm2(denom)<<std::endl;
+
+    std::cout << "denom " << denom<<std::endl;
+
+    out = num*denom;
+    std::cout << "Hw mom space OUT" << norm2(out)<<std::endl;
+}
+
+
 FermOpTemplateInstantiate(WilsonFermion5D);
 GparityFermOpTemplateInstantiate(WilsonFermion5D);
   
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.h b/lib/qcd/action/fermion/WilsonFermion5D.h
index 4ff37007..a56bee11 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.h
+++ b/lib/qcd/action/fermion/WilsonFermion5D.h
@@ -88,6 +88,9 @@ namespace Grid {
       virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
       virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
 
+      void MomentumSpacePropagatorHt(FermionField &out,const FermionField &in) ;
+      void MomentumSpacePropagatorHw(FermionField &out,const FermionField &in) ;
+
       // Implement hopping term non-hermitian hopping term; half cb or both
       // Implement s-diagonal DW
       void DW    (const FermionField &in, FermionField &out,int dag);

From 836e9295652de09e5e25512f11e2692e1f8ae1f7 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:42:22 +0100
Subject: [PATCH 017/149] Divide handling improved

---
 lib/simd/Grid_avx.h          | 17 ++++++-
 lib/simd/Grid_avx512.h       | 12 +++++
 lib/simd/Grid_imci.h         | 12 +++++
 lib/simd/Grid_sse4.h         | 15 ++++++
 lib/simd/Grid_vector_types.h | 90 ++++++++++++++++++++++--------------
 5 files changed, 110 insertions(+), 36 deletions(-)

diff --git a/lib/simd/Grid_avx.h b/lib/simd/Grid_avx.h
index 03faabee..33496e07 100644
--- a/lib/simd/Grid_avx.h
+++ b/lib/simd/Grid_avx.h
@@ -365,6 +365,18 @@ namespace Optimization {
     }
   };
 
+  struct Div{
+    // Real float
+    inline __m256 operator()(__m256 a, __m256 b){
+      return _mm256_div_ps(a,b);
+    }
+    // Real double
+    inline __m256d operator()(__m256d a, __m256d b){
+      return _mm256_div_pd(a,b);
+    }
+  };
+
+
   struct Conj{
     // Complex single
     inline __m256 operator()(__m256 in){
@@ -473,7 +485,7 @@ namespace Optimization {
   }
 
 #endif
-
+  /*
     inline std::ostream & operator << (std::ostream& stream, const __m256 a)
     {
       const float *p=(const float *)&a;
@@ -486,7 +498,7 @@ namespace Optimization {
       stream<< "{"<<p[0]<<","<<p[1]<<","<<p[2]<<","<<p[3]<<"}";
       return stream;
     };
-
+  */
   struct Rotate{
 
     static inline __m256 rotate(__m256 in,int n){ 
@@ -631,6 +643,7 @@ namespace Optimization {
   // Arithmetic operations
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
+  typedef Optimization::Div         DivSIMD;
   typedef Optimization::Mult        MultSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;
diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h
index b057a61b..394b75d6 100644
--- a/lib/simd/Grid_avx512.h
+++ b/lib/simd/Grid_avx512.h
@@ -230,6 +230,17 @@ namespace Optimization {
     }
   };
 
+  struct Div{
+    // Real float
+    inline __m512 operator()(__m512 a, __m512 b){
+      return _mm512_div_ps(a,b);
+    }
+    // Real double
+    inline __m512d operator()(__m512d a, __m512d b){
+      return _mm512_div_pd(a,b);
+    }
+  };
+
 
   struct Conj{
     // Complex single
@@ -427,6 +438,7 @@ namespace Optimization {
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
   typedef Optimization::Mult        MultSIMD;
+  typedef Optimization::Div         DivSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;
   typedef Optimization::TimesMinusI TimesMinusISIMD;
diff --git a/lib/simd/Grid_imci.h b/lib/simd/Grid_imci.h
index 119a31a3..48cff96d 100644
--- a/lib/simd/Grid_imci.h
+++ b/lib/simd/Grid_imci.h
@@ -244,6 +244,17 @@ namespace Optimization {
     }
   };
 
+  struct Div{
+    // Real float
+    inline __m512 operator()(__m512 a, __m512 b){
+      return _mm512_div_ps(a,b);
+    }
+    // Real double
+    inline __m512d operator()(__m512d a, __m512d b){
+      return _mm512_div_pd(a,b);
+    }
+  };
+
 
   struct Conj{
     // Complex single
@@ -437,6 +448,7 @@ namespace Optimization {
   // Arithmetic operations
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
+  typedef Optimization::Div         DivSIMD;
   typedef Optimization::Mult        MultSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;
diff --git a/lib/simd/Grid_sse4.h b/lib/simd/Grid_sse4.h
index a4002373..560eda11 100644
--- a/lib/simd/Grid_sse4.h
+++ b/lib/simd/Grid_sse4.h
@@ -224,6 +224,18 @@ namespace Optimization {
     }
   };
 
+  struct Div{
+    // Real float
+    inline __m128 operator()(__m128 a, __m128 b){
+      return _mm_div_ps(a,b);
+    }
+    // Real double
+    inline __m128d operator()(__m128d a, __m128d b){
+      return _mm_div_pd(a,b);
+    }
+  };
+
+
   struct Conj{
     // Complex single
     inline __m128 operator()(__m128 in){
@@ -372,6 +384,8 @@ namespace Optimization {
   }
 }
 
+
+
 //////////////////////////////////////////////////////////////////////////////////////
 // Here assign types 
 
@@ -398,6 +412,7 @@ namespace Optimization {
   // Arithmetic operations
   typedef Optimization::Sum         SumSIMD;
   typedef Optimization::Sub         SubSIMD;
+  typedef Optimization::Div         DivSIMD;
   typedef Optimization::Mult        MultSIMD;
   typedef Optimization::MultComplex MultComplexSIMD;
   typedef Optimization::Conj        ConjSIMD;
diff --git a/lib/simd/Grid_vector_types.h b/lib/simd/Grid_vector_types.h
index 30576fb6..4c3e903d 100644
--- a/lib/simd/Grid_vector_types.h
+++ b/lib/simd/Grid_vector_types.h
@@ -77,38 +77,24 @@ struct RealPart<std::complex<T> > {
 //////////////////////////////////////
 // demote a vector to real type
 //////////////////////////////////////
-
 // type alias used to simplify the syntax of std::enable_if
-template <typename T>
-using Invoke = typename T::type;
-template <typename Condition, typename ReturnType>
-using EnableIf = Invoke<std::enable_if<Condition::value, ReturnType> >;
-template <typename Condition, typename ReturnType>
-using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
+template <typename T> using Invoke = typename T::type;
+template <typename Condition, typename ReturnType> using EnableIf = Invoke<std::enable_if<Condition::value, ReturnType> >;
+template <typename Condition, typename ReturnType> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
 
 ////////////////////////////////////////////////////////
 // Check for complexity with type traits
-template <typename T>
-struct is_complex : public std::false_type {};
-template <>
-struct is_complex<std::complex<double> > : public std::true_type {};
-template <>
-struct is_complex<std::complex<float> > : public std::true_type {};
+template <typename T> struct is_complex : public std::false_type {};
+template <> struct is_complex<std::complex<double> > : public std::true_type {};
+template <> struct is_complex<std::complex<float> > : public std::true_type {};
 
-template <typename T>
-using IfReal = Invoke<std::enable_if<std::is_floating_point<T>::value, int> >;
-template <typename T>
-using IfComplex = Invoke<std::enable_if<is_complex<T>::value, int> >;
-template <typename T>
-using IfInteger = Invoke<std::enable_if<std::is_integral<T>::value, int> >;
+template <typename T> using IfReal       = Invoke<std::enable_if<std::is_floating_point<T>::value, int> >;
+template <typename T> using IfComplex    = Invoke<std::enable_if<is_complex<T>::value, int> >;
+template <typename T> using IfInteger    = Invoke<std::enable_if<std::is_integral<T>::value, int> >;
 
-template <typename T>
-using IfNotReal =
-    Invoke<std::enable_if<!std::is_floating_point<T>::value, int> >;
-template <typename T>
-using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
-template <typename T>
-using IfNotInteger = Invoke<std::enable_if<!std::is_integral<T>::value, int> >;
+template <typename T> using IfNotReal    = Invoke<std::enable_if<!std::is_floating_point<T>::value, int> >;
+template <typename T> using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
+template <typename T> using IfNotInteger = Invoke<std::enable_if<!std::is_integral<T>::value, int> >;
 
 ////////////////////////////////////////////////////////
 // Define the operation templates functors
@@ -285,6 +271,20 @@ class Grid_simd {
     return a * b;
   }
 
+  //////////////////////////////////
+  // Divides
+  //////////////////////////////////
+  friend inline Grid_simd operator/(const Scalar_type &a, Grid_simd b) {
+    Grid_simd va;
+    vsplat(va, a);
+    return va / b;
+  }
+  friend inline Grid_simd operator/(Grid_simd b, const Scalar_type &a) {
+    Grid_simd va;
+    vsplat(va, a);
+    return b / a;
+  }
+
   ///////////////////////
   // Unary negation
   ///////////////////////
@@ -428,7 +428,6 @@ inline void rotate(Grid_simd<S,V> &ret,Grid_simd<S,V> b,int nrot)
   ret.v = Optimization::Rotate::rotate(b.v,2*nrot);
 }
 
-
 template <class S, class V> 
 inline void vbroadcast(Grid_simd<S,V> &ret,const Grid_simd<S,V> &src,int lane){
   S* typepun =(S*) &src;
@@ -512,7 +511,6 @@ template <class S, class V, IfInteger<S> = 0>
 inline void vfalse(Grid_simd<S, V> &ret) {
   vsplat(ret, 0);
 }
-
 template <class S, class V>
 inline void zeroit(Grid_simd<S, V> &z) {
   vzero(z);
@@ -530,7 +528,6 @@ inline void vstream(Grid_simd<S, V> &out, const Grid_simd<S, V> &in) {
   typedef typename S::value_type T;
   binary<void>((T *)&out.v, in.v, VstreamSIMD());
 }
-
 template <class S, class V, IfInteger<S> = 0>
 inline void vstream(Grid_simd<S, V> &out, const Grid_simd<S, V> &in) {
   out = in;
@@ -569,6 +566,36 @@ inline Grid_simd<S, V> operator*(Grid_simd<S, V> a, Grid_simd<S, V> b) {
   return ret;
 };
 
+// Distinguish between complex types and others
+template <class S, class V, IfComplex<S> = 0>
+inline Grid_simd<S, V> operator/(Grid_simd<S, V> a, Grid_simd<S, V> b) {
+  typedef Grid_simd<S, V> simd;
+
+  simd ret;
+  simd den;
+  typename simd::conv_t conv;
+
+  ret = a * conjugate(b) ;
+  den = b * conjugate(b) ;
+
+  
+  auto real_den = toReal(den);
+
+  ret.v=binary<V>(ret.v, real_den.v, DivSIMD());
+
+  std::cout << " Div call from complex "<<a<<" / " << b<< " = " <<ret<< std::endl;
+
+  return ret;
+};
+
+// Real/Integer types
+template <class S, class V, IfNotComplex<S> = 0>
+inline Grid_simd<S, V> operator/(Grid_simd<S, V> a, Grid_simd<S, V> b) {
+  Grid_simd<S, V> ret;
+  ret.v = binary<V>(a.v, b.v, DivSIMD());
+  return ret;
+};
+
 ///////////////////////
 // Conjugate
 ///////////////////////
@@ -582,7 +609,6 @@ template <class S, class V, IfNotComplex<S> = 0>
 inline Grid_simd<S, V> conjugate(const Grid_simd<S, V> &in) {
   return in;  // for real objects
 }
-
 // Suppress adj for integer types... // odd; why conjugate above but not adj??
 template <class S, class V, IfNotInteger<S> = 0>
 inline Grid_simd<S, V> adj(const Grid_simd<S, V> &in) {
@@ -596,14 +622,12 @@ template <class S, class V, IfComplex<S> = 0>
 inline void timesMinusI(Grid_simd<S, V> &ret, const Grid_simd<S, V> &in) {
   ret.v = binary<V>(in.v, ret.v, TimesMinusISIMD());
 }
-
 template <class S, class V, IfComplex<S> = 0>
 inline Grid_simd<S, V> timesMinusI(const Grid_simd<S, V> &in) {
   Grid_simd<S, V> ret;
   timesMinusI(ret, in);
   return ret;
 }
-
 template <class S, class V, IfNotComplex<S> = 0>
 inline Grid_simd<S, V> timesMinusI(const Grid_simd<S, V> &in) {
   return in;
@@ -616,14 +640,12 @@ template <class S, class V, IfComplex<S> = 0>
 inline void timesI(Grid_simd<S, V> &ret, const Grid_simd<S, V> &in) {
   ret.v = binary<V>(in.v, ret.v, TimesISIMD());
 }
-
 template <class S, class V, IfComplex<S> = 0>
 inline Grid_simd<S, V> timesI(const Grid_simd<S, V> &in) {
   Grid_simd<S, V> ret;
   timesI(ret, in);
   return ret;
 }
-
 template <class S, class V, IfNotComplex<S> = 0>
 inline Grid_simd<S, V> timesI(const Grid_simd<S, V> &in) {
   return in;

From 2ba7d43dddd40e010c30716aa74200d2ae84a62a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:43:14 +0100
Subject: [PATCH 018/149] Divide handling

---
 lib/tensors/Tensor_arith_mul.h | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/lib/tensors/Tensor_arith_mul.h b/lib/tensors/Tensor_arith_mul.h
index 917f7373..c24853b7 100644
--- a/lib/tensors/Tensor_arith_mul.h
+++ b/lib/tensors/Tensor_arith_mul.h
@@ -126,6 +126,36 @@ iVector<rtype,N> operator * (const iVector<mtype,N>& lhs,const iScalar<vtype>& r
     mult(&ret,&lhs,&rhs);
     return ret;
 }
+
+//////////////////////////////////////////////////////////////////
+// Divide by scalar
+//////////////////////////////////////////////////////////////////
+template<class rtype,class vtype> strong_inline
+iScalar<rtype> operator / (const iScalar<rtype>& lhs,const iScalar<vtype>& rhs)
+{
+    iScalar<rtype> ret;
+    ret._internal = lhs._internal/rhs._internal;
+    return ret;
+}
+template<class rtype,class vtype,int N> strong_inline
+iVector<rtype,N> operator / (const iVector<rtype,N>& lhs,const iScalar<vtype>& rhs)
+{
+    iVector<rtype,N> ret;
+    for(int i=0;i<N;i++){
+      ret._internal[i] = lhs._internal[i]/rhs._internal;
+    }
+    return ret;
+}
+template<class rtype,class vtype,int N> strong_inline
+iMatrix<rtype,N> operator / (const iMatrix<rtype,N>& lhs,const iScalar<vtype>& rhs)
+{
+    iMatrix<rtype,N> ret;
+    for(int i=0;i<N;i++){
+    for(int j=0;j<N;j++){
+      ret._internal[i][j] = lhs._internal[i][j]/rhs._internal;
+    }}
+    return ret;
+}
     
     //////////////////////////////////////////////////////////////////
     // Glue operators to mult routines. Must resolve return type cleverly from typeof(internal)

From 7089b6d5a58912d7a18bf07c3bdc7a0848f82690 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:43:40 +0100
Subject: [PATCH 019/149] Setting up but not implemented some QED rules

---
 lib/qcd/action/gauge/Photon.h | 126 ++++++++++++++++++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100644 lib/qcd/action/gauge/Photon.h

diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
new file mode 100644
index 00000000..a7556b19
--- /dev/null
+++ b/lib/qcd/action/gauge/Photon.h
@@ -0,0 +1,126 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/gauge/Photon.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef QCD_PHOTON_ACTION_H
+#define QCD_PHOTON_ACTION_H
+
+namespace Grid{
+  namespace QCD{
+
+    template<class Gimpl>
+    class Photon  {
+      
+    public:
+
+      INHERIT_GIMPL_TYPES(Gimpl);
+
+      enum PhotonType { FEYNMAN_L, FEYNMAN_TL };
+
+      PhotonType GaugeBC;
+      
+      Photon(PhotonType _GaugeBC) : GaugeBC(_GaugeBC){};
+
+      void  FreePropagator (const GaugeField &in,GaugeField &out) {
+	FFT theFFT((GridCartesian *) in._grid);
+
+	GaugeField in_k(in._grid);
+	GaugeField prop_k(in._grid);
+
+	theFFT.FFT_all_dim(in_k,in,FFT::forward);
+        MomentumSpacePropagator(prop_k,in_k);
+	theFFT.FFT_all_dim(out,prop_k,FFT::backward);
+      }
+      void  MomentumSpacePropagator(GaugeField &out,const GaugeField &in) {
+	if ( GaugeBC == FEYNMAN_L ) { 
+	  FeynmanGaugeMomentumSpacePropagator_L(out,in);
+	} else if ( GaugeBC == FEYNMAN_TL ) { 
+	  FeynmanGaugeMomentumSpacePropagator_TL(out,in);
+	} else {  // No coulomb yet
+	  assert(0);
+	}
+      } 
+      void  FeynmanGaugeMomentumSpacePropagator_L(GaugeField &out, const GaugeField &in) { 
+
+	FeynmanGaugeMomentumSpacePropagator_TL(out,in);
+
+	GridBase *grid = out._grid;
+	LatticeInteger     coor(grid);
+	GaugeField zz(grid); zz=zero;
+
+	// xyzt
+	for(int d = 0; d < grid->_ndimension-1;d++){
+	  LatticeCoordinate(coor,d);
+	  out = where(coor==Integer(0),zz,out);
+	}
+      }
+      void  FeynmanGaugeMomentumSpacePropagator_TL(GaugeField &out, const GaugeField &in) { 
+
+
+	// what type LatticeComplex 
+	GridBase *grid = out._grid;
+	int nd = grid->_ndimension;
+
+	typedef typename GaugeField::vector_type vector_type;
+	typedef typename GaugeField::scalar_type ScalComplex;
+	typedef Lattice<iSinglet<vector_type> > LatComplex;
+    
+	std::vector<int> latt_size   = grid->_fdimensions;
+
+	LatComplex denom(grid); denom= zero;
+	LatComplex   one(grid); one = ScalComplex(1.0,0.0);
+
+	LatComplex kmu(grid); 
+	ScalComplex ci(0.0,1.0);
+	// momphase = n * 2pi / L
+	for(int mu=0;mu<Nd;mu++) {
+	  
+	  LatticeCoordinate(kmu,mu);
+	  
+	  RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+
+	  kmu = TwoPiL * kmu ;
+	  
+	  denom = denom + 4.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
+	}
+	std::vector<int> zero_mode(nd,0);
+	TComplexD Tone = ComplexD(1.0,0.0);
+	TComplexD Tzero= ComplexD(0.0,0.0);
+
+	pokeSite(Tone,denom,zero_mode); 
+
+	denom= one/denom;
+	
+	pokeSite(Tzero,denom,zero_mode); 
+
+	out = zero;
+	out = in*denom;
+      };
+
+    };
+
+}}
+#endif

From 34f887ca1c35234717568ee80c9edbab6df7eef4 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:44:36 +0100
Subject: [PATCH 020/149] Test_fft not complete; preparing for tests of
 momentum space DWF and Overlap feynman rules but not there yet.

---
 tests/core/Test_fft.cc | 98 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 84 insertions(+), 14 deletions(-)

diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index c076695d..a20791fb 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -27,6 +27,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 #include <Grid/Grid.h>
+#include <Grid/qcd/action/gauge/Photon.h>
 
 using namespace Grid;
 using namespace Grid::QCD;
@@ -127,27 +128,96 @@ int main (int argc, char ** argv)
   pRNG.SeedFixedIntegers(seeds);
 
   LatticeGaugeFieldD Umu(&GRID);
-  LatticeFermionD    src(&GRID); gaussian(pRNG,src);
-  LatticeFermionD    tmp(&GRID);
-  LatticeFermionD    ref(&GRID);
 
   SU3::ColdConfiguration(pRNG,Umu); // Unit gauge
   
-  RealD mass=0.1;
-  WilsonFermionD Dw(Umu,GRID,RBGRID,mass);
+  {
+    LatticeFermionD    src(&GRID); gaussian(pRNG,src);
+    LatticeFermionD    tmp(&GRID);
+    LatticeFermionD    ref(&GRID);
+    
+    RealD mass=0.1;
+    WilsonFermionD Dw(Umu,GRID,RBGRID,mass);
+    
+    Dw.M(src,tmp);
 
-  Dw.M(src,tmp);
+    std::cout << " src = " <<norm2(src)<<std::endl;
+    std::cout << " tmp = " <<norm2(tmp)<<std::endl;
+    
+    Dw.FreePropagator(tmp,ref);
 
-  std::cout << " src = " <<norm2(src)<<std::endl;
-  std::cout << " tmp = " <<norm2(tmp)<<std::endl;
+    std::cout << " ref = " <<norm2(ref)<<std::endl;
+    
+    ref = ref - src;
+    
+    std::cout << " ref-src = " <<norm2(ref)<<std::endl;
+  }
 
-  Dw.FreePropagator(tmp,ref);
+  {
+    LatticeFermionD    src(&GRID); gaussian(pRNG,src);
+    LatticeFermionD    tmp(&GRID);
+    LatticeFermionD    ref(&GRID);
 
-  std::cout << " ref = " <<norm2(ref)<<std::endl;
-  
-  ref = ref - src;
-  
-  std::cout << " ref-src = " <<norm2(ref)<<std::endl;
+    const int Ls=8;
+    GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,&GRID);
+    GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,&GRID);
 
+    RealD mass=0.1;
+    RealD M5  =0.9;
+    DomainWallFermionD Ddwf(Umu,*FGrid,*FrbGrid,GRID,RBGRID,mass,M5);
+
+    // Need to solve and project 4d. New test required.
+
+    Ddwf.MomentumSpacePropagatorHw(ref,src) ;
+    std::cout << " Hw Mom space \n";
+
+
+    Ddwf.MomentumSpacePropagatorHt(ref,src) ;
+    std::cout << " Ht Mom space \n";
+
+      
+    {   
+      Gamma G5(Gamma::Gamma5);
+
+      LatticeFermionD    src5(FGrid); src5=zero;
+      LatticeFermionD    result5(FGrid); result5=zero;
+      LatticeFermionD    result4(&GRID); 
+
+      const int sdir=0;
+      
+      tmp =   (src + G5*src)*0.5;
+      InsertSlice(tmp,src5,Ls-1,sdir);
+      
+      tmp =   (src - G5*src)*0.5;
+      InsertSlice(tmp,src5,0,sdir);
+      
+      MdagMLinearOperator<DomainWallFermionD,LatticeFermionD> HermOp(Ddwf);
+      ConjugateGradient<LatticeFermionD> CG(1.0e-4,1000);
+      CG(HermOp,src5,result5);
+      result5 = zero;
+
+      ExtractSlice(tmp,result5,0,sdir);
+      result4 = (tmp+G5*tmp)*0.5;
+      
+      ExtractSlice(tmp,result5,Ls-1,sdir);
+      result4 = result4+(tmp-G5*tmp)*0.5;
+
+      std::cout << "src     "<<norm2(src)<<std::endl;
+      std::cout << "src5    "<<norm2(src5)<<std::endl;
+      std::cout << "result4 "<<norm2(result4)<<std::endl;
+      std::cout << "ref     "<<norm2(ref)<<std::endl;
+    }
+  }
+
+  {
+    typedef GaugeImplTypes<vComplexD, 1> QEDGimplTypesD;
+    typedef Photon<QEDGimplTypesD>       QEDGaction;
+    QEDGaction Maxwell(QEDGaction::FEYNMAN_L);
+    QEDGaction::GaugeField Prop(&GRID);Prop=zero;
+    QEDGaction::GaugeField Source(&GRID);Source=zero;
+
+    Maxwell.FreePropagator (Source,Prop);
+    std::cout << " MaxwellFree propagator\n";
+  }
   Grid_finalize();
 }

From 167cc2650ea8ad87f9920326513feea559ffc523 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 09:58:09 +0100
Subject: [PATCH 021/149] GNU SOURCE problem on travis

---
 lib/Init.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/Init.cc b/lib/Init.cc
index cff43185..2f9619a9 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -331,8 +331,6 @@ void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr)
   return;
 };
 
-
-#define _GNU_SOURCE
 #include <fenv.h>
 
 #ifdef __APPLE__

From 9353b6edfe7fd0567f40fad0f0dfb497a5d91c22 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 10:09:13 +0100
Subject: [PATCH 022/149] Fenv out of grid namespace

---
 lib/Init.cc | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/lib/Init.cc b/lib/Init.cc
index 2f9619a9..221c056d 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -48,6 +48,26 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <memory>
 
 
+#include <fenv.h>
+#ifdef __APPLE__
+static int
+feenableexcept (unsigned int excepts)
+{
+  static fenv_t fenv;
+  unsigned int new_excepts = excepts & FE_ALL_EXCEPT,
+    old_excepts;  // previous masks
+
+  if ( fegetenv (&fenv) ) return -1;
+  old_excepts = fenv.__control & FE_ALL_EXCEPT;
+
+  // unmask
+  fenv.__control &= ~new_excepts;
+  fenv.__mxcsr   &= ~(new_excepts << 7);
+
+  return ( fesetenv (&fenv) ? -1 : old_excepts );
+}
+#endif
+
 namespace Grid {
 
 
@@ -331,27 +351,6 @@ void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr)
   return;
 };
 
-#include <fenv.h>
-
-#ifdef __APPLE__
-static int
-feenableexcept (unsigned int excepts)
-{
-  static fenv_t fenv;
-  unsigned int new_excepts = excepts & FE_ALL_EXCEPT,
-    old_excepts;  // previous masks
-
-  if ( fegetenv (&fenv) ) return -1;
-  old_excepts = fenv.__control & FE_ALL_EXCEPT;
-
-  // unmask
-  fenv.__control &= ~new_excepts;
-  fenv.__mxcsr   &= ~(new_excepts << 7);
-
-  return ( fesetenv (&fenv) ? -1 : old_excepts );
-}
-#endif
-
 void Grid_debug_handler_init(void)
 {
   struct sigaction sa,osa;

From 87acd06990090acae4f19ae147ca794d243764f5 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 26 Sep 2016 10:11:34 +0100
Subject: [PATCH 023/149] Use streaming stores

---
 lib/simd/Intel512common.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/simd/Intel512common.h b/lib/simd/Intel512common.h
index dabbf6d8..74793638 100644
--- a/lib/simd/Intel512common.h
+++ b/lib/simd/Intel512common.h
@@ -139,8 +139,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define ZLOADd(OFF,PTR,ri,ir)  VLOADd(OFF,PTR,ir)  VSHUFd(ir,ri)
 
 
-#define VSTOREf(OFF,PTR,SRC)   "vmovaps " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
-#define VSTOREd(OFF,PTR,SRC)   "vmovapd " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
+#define VSTOREf(OFF,PTR,SRC)   "vmovntps " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
+#define VSTOREd(OFF,PTR,SRC)   "vmovntpd " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
 
 // Swaps Re/Im ; could unify this with IMCI
 #define VSHUFd(A,DEST)         "vpshufd  $0x4e," #A "," #DEST  ";\n"    

From b56c9ffa5272d975ae7ff9294a7faa6764081566 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Mon, 10 Oct 2016 14:43:37 +0100
Subject: [PATCH 024/149] Fix for AVXFMA

---
 configure.ac                                 | 8 +++++++-
 lib/qcd/action/fermion/FermionOperatorImpl.h | 6 +++---
 tests/hmc/Test_hmc_EODWFRatio.cc             | 4 +++-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/configure.ac b/configure.ac
index 827a15a9..b7205b4a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -117,7 +117,7 @@ CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
 ############### SIMD instruction selection
-AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=SSE4|AVX|AVXFMA4|AVX2|AVX512|AVX512MIC|IMCI|KNL|KNC],\
+AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=SSE4|AVX|AVXFMA4|AVXFMA|AVX2|AVX512|AVX512MIC|IMCI|KNL|KNC],\
 	[Select instructions to be SSE4.0, AVX 1.0, AVX 2.0+FMA, AVX 512, IMCI])],\
 	[ac_SIMD=${enable_simd}],[ac_SIMD=GEN])
 
@@ -133,6 +133,9 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVXFMA4)
         AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
         SIMD_FLAGS='-mavx -mfma4';;
+      AVXFMA)
+        AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA3])
+        SIMD_FLAGS='-mavx -mfma';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
         SIMD_FLAGS='-mavx2 -mfma';;
@@ -162,6 +165,9 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVXFMA4)
         AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
         SIMD_FLAGS='-mavx -mfma';;
+      AVXFMA)
+        AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA4])
+        SIMD_FLAGS='-mavx -mfma';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
         SIMD_FLAGS='-march=core-avx2 -xcore-avx2';;
diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h
index 910bf488..4248ec4c 100644
--- a/lib/qcd/action/fermion/FermionOperatorImpl.h
+++ b/lib/qcd/action/fermion/FermionOperatorImpl.h
@@ -186,10 +186,10 @@ namespace Grid {
       inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde,int mu){
 	
 	int Ls=Btilde._grid->_fdimensions[0];
-	
 	GaugeLinkField tmp(mat._grid);
 	tmp = zero;
-	PARALLEL_FOR_LOOP
+
+        PARALLEL_FOR_LOOP
 	  for(int sss=0;sss<tmp._grid->oSites();sss++){
 	    int sU=sss;
 	    for(int s=0;s<Ls;s++){
@@ -198,7 +198,7 @@ namespace Grid {
 	    }
 	  }
 	PokeIndex<LorentzIndex>(mat,tmp,mu);
-	
+
       }
     };
 
diff --git a/tests/hmc/Test_hmc_EODWFRatio.cc b/tests/hmc/Test_hmc_EODWFRatio.cc
index 20ef7db6..02b312b8 100644
--- a/tests/hmc/Test_hmc_EODWFRatio.cc
+++ b/tests/hmc/Test_hmc_EODWFRatio.cc
@@ -75,8 +75,10 @@ public:
     Level1.push_back(&Waction);
     TheAction.push_back(Level1);
 
+    NumOp.ZeroCounters();
+    DenOp.ZeroCounters();
     Run(argc,argv);
-
+    
     std::cout << GridLogMessage << "Numerator report, Pauli-Villars term         : " << std::endl;
     NumOp.Report();
     std::cout << GridLogMessage << "Denominator report, Dw(m) term (includes CG) : " << std::endl;

From 611b5d74baebba6215e48a2a15c4642309c8a839 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Mon, 10 Oct 2016 15:26:17 +0100
Subject: [PATCH 025/149] Fix for AVX+FMA3 compilation

---
 lib/simd/Grid_avx.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/simd/Grid_avx.h b/lib/simd/Grid_avx.h
index 03faabee..05f2f1ab 100644
--- a/lib/simd/Grid_avx.h
+++ b/lib/simd/Grid_avx.h
@@ -442,7 +442,7 @@ namespace Optimization {
 #define _mm256_alignr_epi64(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16)
 #endif
 
-#if defined (AVX1) 
+#if defined (AVX1) || defined (AVXFMA)
 
 #define _mm256_alignr_epi32(ret,a,b,n) {	\
     __m128 aa, bb;				\

From 082ae350c66a0f514bb5f4bd009a28c5ba9bbb35 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:42:30 +0100
Subject: [PATCH 026/149] static schedule by default

---
 lib/Threads.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Threads.h b/lib/Threads.h
index 9d1295e5..5abe2ca4 100644
--- a/lib/Threads.h
+++ b/lib/Threads.h
@@ -37,7 +37,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #ifdef GRID_OMP
 #include <omp.h>
-#define PARALLEL_FOR_LOOP _Pragma("omp parallel for ")
+#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(static)")
 #define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)")
 #else
 #define PARALLEL_FOR_LOOP 

From 09ca32d678461ad1da759e286d7007d629584bd4 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:42:55 +0100
Subject: [PATCH 027/149] Dminus added for Cayley

---
 lib/qcd/action/fermion/CayleyFermion5D.cc | 24 +++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/lib/qcd/action/fermion/CayleyFermion5D.cc b/lib/qcd/action/fermion/CayleyFermion5D.cc
index c4196043..57b047d4 100644
--- a/lib/qcd/action/fermion/CayleyFermion5D.cc
+++ b/lib/qcd/action/fermion/CayleyFermion5D.cc
@@ -50,6 +50,30 @@ namespace QCD {
    mass(_mass)
  { }
 
+template<class Impl>  
+void CayleyFermion5D<Impl>::Dminus(const FermionField &psi, FermionField &chi)
+{
+  int Ls=this->Ls;
+  FermionField tmp(psi._grid);
+
+  this->DW(psi,tmp,DaggerNo);
+
+  for(int s=0;s<Ls;s++){
+    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp,s,s);// chi = (1-c[s] D_W) psi
+  }
+}
+template<class Impl>  
+void CayleyFermion5D<Impl>::DminusDag(const FermionField &psi, FermionField &chi)
+{
+  int Ls=this->Ls;
+  FermionField tmp(psi._grid);
+
+  this->DW(psi,tmp,DaggerYes);
+
+  for(int s=0;s<Ls;s++){
+    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp,s,s);// chi = (1-c[s] D_W) psi
+  }
+}
 template<class Impl>  
 void CayleyFermion5D<Impl>::M5D   (const FermionField &psi, FermionField &chi)
 {

From c0d5b99016d6ac9914c5b9eeee8a5fc6c760f54b Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:43:19 +0100
Subject: [PATCH 028/149] Dminus

---
 lib/qcd/action/fermion/CayleyFermion5D.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/qcd/action/fermion/CayleyFermion5D.h b/lib/qcd/action/fermion/CayleyFermion5D.h
index 53a93a05..1d8c2b95 100644
--- a/lib/qcd/action/fermion/CayleyFermion5D.h
+++ b/lib/qcd/action/fermion/CayleyFermion5D.h
@@ -56,6 +56,9 @@ namespace Grid {
       virtual void   M5D   (const FermionField &psi, FermionField &chi);
       virtual void   M5Ddag(const FermionField &psi, FermionField &chi);
 
+      virtual void   Dminus(const FermionField &psi, FermionField &chi);
+      virtual void   DminusDag(const FermionField &psi, FermionField &chi);
+
       /////////////////////////////////////////////////////
       // Instantiate different versions depending on Impl
       /////////////////////////////////////////////////////
@@ -117,6 +120,7 @@ namespace Grid {
 		      GridRedBlackCartesian &FourDimRedBlackGrid,
 		      RealD _mass,RealD _M5,const ImplParams &p= ImplParams());
 
+
     protected:
       void SetCoefficientsZolotarev(RealD zolohi,Approx::zolotarev_data *zdata,RealD b,RealD c);
       void SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c);

From d7ce164e6e25ef2fd2047583d5b74753b6a64769 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:43:36 +0100
Subject: [PATCH 029/149] Feynman rule for DWF

---
 lib/qcd/action/fermion/DomainWallFermion.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/qcd/action/fermion/DomainWallFermion.h b/lib/qcd/action/fermion/DomainWallFermion.h
index b72ca8ec..c0b6b6aa 100644
--- a/lib/qcd/action/fermion/DomainWallFermion.h
+++ b/lib/qcd/action/fermion/DomainWallFermion.h
@@ -42,6 +42,10 @@ namespace Grid {
      INHERIT_IMPL_TYPES(Impl);
     public:
 
+      void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { 
+	this->MomentumSpacePropagatorHt(out,in,_m);
+      };
+
       virtual void   Instantiatable(void) {};
       // Constructors
       DomainWallFermion(GaugeField &_Umu,
@@ -51,6 +55,7 @@ namespace Grid {
 			GridRedBlackCartesian &FourDimRedBlackGrid,
 			RealD _mass,RealD _M5,const ImplParams &p= ImplParams()) : 
 
+
       CayleyFermion5D<Impl>(_Umu,
 			    FiveDimGrid,
 			    FiveDimRedBlackGrid,

From c0145745047b2500f19b57cfa4c28d2aa7ac3ddc Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:44:00 +0100
Subject: [PATCH 030/149] A "please implement me" feynman rule. If this were
 abstract virtual it would require/force implementation

---
 lib/qcd/action/fermion/FermionOperator.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/qcd/action/fermion/FermionOperator.h b/lib/qcd/action/fermion/FermionOperator.h
index daee2e5b..742c6e08 100644
--- a/lib/qcd/action/fermion/FermionOperator.h
+++ b/lib/qcd/action/fermion/FermionOperator.h
@@ -92,15 +92,16 @@ namespace Grid {
       virtual void  Mdir   (const FermionField &in, FermionField &out,int dir,int disp)=0;   // case by case Wilson, Clover, Cayley, ContFrac, PartFrac
 
 
-      virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in) { assert(0);};
-      virtual void  FreePropagator(const FermionField &in,FermionField &out) { 
+      virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { assert(0);};
+
+      virtual void  FreePropagator(const FermionField &in,FermionField &out,RealD mass) { 
 	FFT theFFT((GridCartesian *) in._grid);
 
 	FermionField in_k(in._grid);
 	FermionField prop_k(in._grid);
 
 	theFFT.FFT_all_dim(in_k,in,FFT::forward);
-        this->MomentumSpacePropagator(prop_k,in_k);
+        this->MomentumSpacePropagator(prop_k,in_k,mass);
 	theFFT.FFT_all_dim(out,prop_k,FFT::backward);
       };
 

From 6f26d2e8d4133ce88a1ca5056b083ecdf7d545b6 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:45:18 +0100
Subject: [PATCH 031/149] Overlap tree level feynman rule

---
 lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h b/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h
index cd23ddd5..9cab0e22 100644
--- a/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h
+++ b/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h
@@ -42,7 +42,11 @@ namespace Grid {
      INHERIT_IMPL_TYPES(Impl);
     public:
 
-      // Constructors
+     void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { 
+       this->MomentumSpacePropagatorHw(out,in,_m);
+     };
+
+     // Constructors
     OverlapWilsonCayleyTanhFermion(GaugeField &_Umu,
 				   GridCartesian         &FiveDimGrid,
 				   GridRedBlackCartesian &FiveDimRedBlackGrid,

From 616e7cd83eb4c45b3102c6a232c5f26b61517a01 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:45:48 +0100
Subject: [PATCH 032/149] Mass parameter

---
 lib/qcd/action/fermion/WilsonFermion.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonFermion.cc b/lib/qcd/action/fermion/WilsonFermion.cc
index 55fbfa01..36c1870c 100644
--- a/lib/qcd/action/fermion/WilsonFermion.cc
+++ b/lib/qcd/action/fermion/WilsonFermion.cc
@@ -137,7 +137,7 @@ namespace QCD {
     MooeeInv(in,out);
   }
   template<class Impl>
-  void WilsonFermion<Impl>:: MomentumSpacePropagator(FermionField &out, const FermionField &in) {
+  void WilsonFermion<Impl>:: MomentumSpacePropagator(FermionField &out, const FermionField &in,RealD _m) {
 
     // what type LatticeComplex 
     conformable(_grid,out._grid);
@@ -179,7 +179,7 @@ namespace QCD {
       denom=denom + sin(kmu)*sin(kmu);
     }
 
-    wilson = wilson + mass;     // 2 sin^2 k/2 + m
+    wilson = wilson + _m;     // 2 sin^2 k/2 + m
 
     num   = num + wilson*in;     // -i gmu sin k + 2 sin^2 k/2 + m
 

From 657e0a8f4d457f0350826e2a12eff6643f031ccd Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:46:10 +0100
Subject: [PATCH 033/149] Mass parameter

---
 lib/qcd/action/fermion/WilsonFermion.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/qcd/action/fermion/WilsonFermion.h b/lib/qcd/action/fermion/WilsonFermion.h
index c050c0eb..3a1ba432 100644
--- a/lib/qcd/action/fermion/WilsonFermion.h
+++ b/lib/qcd/action/fermion/WilsonFermion.h
@@ -78,7 +78,7 @@ namespace Grid {
       virtual void MooeeInv(const FermionField &in, FermionField &out) ;
       virtual void MooeeInvDag(const FermionField &in, FermionField &out) ;
 
-      virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in) ;
+      virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _mass) ;
 
       ////////////////////////
       // Derivative interface

From 96f1d1b828f613a7d2278a441cc86ee59cfd2526 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:46:45 +0100
Subject: [PATCH 034/149] Debugged Domain wall and Overlap feynman rules
 (infinite Ls, finite mass).

---
 lib/qcd/action/fermion/WilsonFermion5D.cc | 184 ++++++++++------------
 1 file changed, 81 insertions(+), 103 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index 28f02b67..ea3e8535 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -393,92 +393,95 @@ void WilsonFermion5D<Impl>::DW(const FermionField &in, FermionField &out,int dag
 }
 
 template<class Impl>
-void WilsonFermion5D<Impl>::MomentumSpacePropagatorHt(FermionField &out,const FermionField &in) 
+void WilsonFermion5D<Impl>::MomentumSpacePropagatorHt(FermionField &out,const FermionField &in, RealD mass) 
 {
-    // what type LatticeComplex 
-    GridBase *_grid = _FourDimGrid;
-    conformable(_grid,out._grid);
+  // what type LatticeComplex 
+  GridBase *_grid = _FourDimGrid;
+  conformable(_grid,out._grid);
+  
+  typedef typename FermionField::vector_type vector_type;
+  typedef typename FermionField::scalar_type ScalComplex;
+  typedef iSinglet<ScalComplex> Tcomplex;
+  typedef Lattice<iSinglet<vector_type> > LatComplex;
+  
+  Gamma::GammaMatrix Gmu [] = {
+    Gamma::GammaX,
+    Gamma::GammaY,
+    Gamma::GammaZ,
+    Gamma::GammaT
+  };
 
-    typedef typename FermionField::vector_type vector_type;
-    typedef typename FermionField::scalar_type ScalComplex;
-    typedef iSinglet<ScalComplex> Tcomplex;
-    typedef Lattice<iSinglet<vector_type> > LatComplex;
+  std::vector<int> latt_size   = _grid->_fdimensions;
 
-    Gamma::GammaMatrix Gmu [] = {
-      Gamma::GammaX,
-      Gamma::GammaY,
-      Gamma::GammaZ,
-      Gamma::GammaT
-    };
+  
+  FermionField   num  (_grid); num  = zero;
 
-    std::vector<int> latt_size   = _grid->_fdimensions;
+  LatComplex    sk(_grid);  sk = zero;
+  LatComplex    sk2(_grid); sk2= zero;
+  LatComplex    W(_grid); W= zero;
+  LatComplex    a(_grid); a= zero;
+  LatComplex    one  (_grid); one = ScalComplex(1.0,0.0);
+  LatComplex denom(_grid); denom= zero;
+  LatComplex cosha(_grid); 
+  LatComplex kmu(_grid); 
+  LatComplex Wea(_grid); 
+  LatComplex Wema(_grid); 
 
-
-    LatComplex    sk(_grid);  sk = zero;
-    LatComplex    sk2(_grid); sk2= zero;
-
-    LatComplex    W(_grid); W= zero;
-    LatComplex    a(_grid); a= zero;
-    LatComplex    cosha(_grid); a= zero;
-
-    LatComplex     one  (_grid);     one = ScalComplex(1.0,0.0);
-
-    FermionField   num  (_grid); num  = zero;
-    LatComplex denom(_grid); denom= zero;
-    LatComplex kmu(_grid); 
-    ScalComplex ci(0.0,1.0);
-
-    for(int mu=0;mu<Nd;mu++) {
-
-      LatticeCoordinate(kmu,mu);
-
-      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
-
-      kmu = TwoPiL * kmu;
-
-      sk2 = sk2 + 2.0*sin(kmu*0.5)*sin(kmu*0.5);
-      sk  = sk  + sin(kmu)*sin(kmu); 
-
-      num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);
-
-    }
-
-    W = one - M5 - sk2;
-    cosha =  (one + W*W + sk) / (W*2.0);
+  ScalComplex ci(0.0,1.0);
+  
+  for(int mu=0;mu<Nd;mu++) {
     
-    for(int idx=0;idx<_grid->lSites();idx++){
-      Tcomplex cc;
-      RealD sgn;
-      std::vector<int> lcoor(Nd);
-      _grid->LocalIndexToLocalCoor(idx,lcoor);
-      peekLocalSite(cc,cosha,lcoor);
-      sgn= ::fabs(real(cc));
-      cc = ScalComplex(::acosh(sgn),0.0);
-      pokeLocalSite(cc,a,lcoor);
-    }
+    LatticeCoordinate(kmu,mu);
+    
+    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+    
+    kmu = TwoPiL * kmu;
+    
+    sk2 = sk2 + 2.0*sin(kmu*0.5)*sin(kmu*0.5);
+    sk  = sk  +     sin(kmu)    *sin(kmu); 
+    
+    num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);
+    
+  }
+  
+  W = one - M5 + sk2;
 
-  std::cout << "Ht mom space num" << norm2(num)<<std::endl;// num ok
-  std::cout << "Ht mom space W  " << norm2(W)<<std::endl;// w ok
-  std::cout << "Ht mom space a  " << norm2(a)<<std::endl;// a ok
-  denom= ( exp(a) ); 
-  std::cout << "Ht mom space den" << norm2(denom)<<std::endl;
-  denom= ( exp(a) * W  ); 
-  std::cout << "Ht mom space den W" << norm2(denom)<<std::endl;
-  denom= ( exp(a) * W - one ); 
-  std::cout << "Ht mom space den W- 1" << norm2(denom)<<std::endl;
-  denom=where(sk==ScalComplex(0,0),one,denom);
-  std::cout << "Ht mom space den W- 1" << denom<<std::endl;
-  std::cout << "Ht mom space one " << norm2(one)<<std::endl;
-  denom= one/denom;
-  std::cout << "Ht mom space div " << norm2(denom)<<std::endl;
-  out = num*denom;
-  std::cout << "Ht mom space out" << norm2(out)<<std::endl;
+  ////////////////////////////////////////////
+  // Cosh alpha -> alpha
+  ////////////////////////////////////////////
+  cosha =  (one + W*W + sk) / (W*2.0);
+
+  // FIXME Need a Lattice acosh
+  for(int idx=0;idx<_grid->lSites();idx++){
+    std::vector<int> lcoor(Nd);
+    Tcomplex cc;
+    RealD sgn;
+    _grid->LocalIndexToLocalCoor(idx,lcoor);
+    peekLocalSite(cc,cosha,lcoor);
+    assert((double)real(cc)>=1.0);
+    assert(fabs((double)imag(cc))<=1.0e-15);
+    cc = ScalComplex(::acosh(real(cc)),0.0);
+    pokeLocalSite(cc,a,lcoor);
+  }
+  
+  Wea = ( exp( a) * W  ); 
+  Wema= ( exp(-a) * W  ); 
+  
+  num   = num + ( one - Wema ) * mass * in;
+  denom= ( Wea - one ) + mass*mass * (one - Wema); 
+  out = num/denom;
 }
 
 template<class Impl>
-void WilsonFermion5D<Impl>::MomentumSpacePropagatorHw(FermionField &out,const FermionField &in) 
+void WilsonFermion5D<Impl>::MomentumSpacePropagatorHw(FermionField &out,const FermionField &in,RealD mass) 
 {
-    // what type LatticeComplex 
+    Gamma::GammaMatrix Gmu [] = {
+      Gamma::GammaX,
+      Gamma::GammaY,
+      Gamma::GammaZ,
+      Gamma::GammaT
+    };
+
     GridBase *_grid = _FourDimGrid;
     conformable(_grid,out._grid);
 
@@ -487,16 +490,9 @@ void WilsonFermion5D<Impl>::MomentumSpacePropagatorHw(FermionField &out,const Fe
 
     typedef Lattice<iSinglet<vector_type> > LatComplex;
 
-    Gamma::GammaMatrix Gmu [] = {
-      Gamma::GammaX,
-      Gamma::GammaY,
-      Gamma::GammaZ,
-      Gamma::GammaT
-    };
 
     std::vector<int> latt_size   = _grid->_fdimensions;
 
-
     LatComplex    sk(_grid);  sk = zero;
     LatComplex    sk2(_grid); sk2= zero;
 
@@ -524,35 +520,17 @@ void WilsonFermion5D<Impl>::MomentumSpacePropagatorHw(FermionField &out,const Fe
       num = num - sin(kmu)*ci*(Gamma(Gmu[mu])*in);
 
     }
+    num = num + mass * in ;
 
     b_k = sk2 - M5;
      
     w_k = sqrt(sk + b_k*b_k);
 
-    std::cout << "Hw M5 " << M5<<std::endl;
-    std::cout << "Hw mom space NUM" << norm2(num)<<std::endl;
-    std::cout << "Hw mom space BK" << norm2(b_k)<<std::endl;
-    std::cout << "Hw mom space BK" << b_k<<std::endl;
-    std::cout << "Hw mom space WK" << norm2(w_k)<<std::endl;
-    std::cout << "Hw mom space WK" << w_k<<std::endl;
-
-    denom= ( w_k + b_k ) * (2.0 * M5);
-    denom=where(sk==ScalComplex(0,0),one,denom);
-
-    std::cout << "Hw mom space DEMON" << norm2(denom)<<std::endl;
-
-    std::cout << "Hw mom space DEMON" << denom<<std::endl;
-    
-    std::cout << "Hw mom space one" << norm2(one)<<std::endl;
+    denom= ( w_k + b_k + mass*mass) ;
 
     denom= one/denom;
-
-    std::cout << "Hw mom space div " << norm2(denom)<<std::endl;
-
-    std::cout << "denom " << denom<<std::endl;
-
     out = num*denom;
-    std::cout << "Hw mom space OUT" << norm2(out)<<std::endl;
+
 }
 
 

From 3619167d6246222b675aa9e1e9fb9541525c5671 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:47:33 +0100
Subject: [PATCH 035/149] Mass parameter

---
 lib/qcd/action/fermion/WilsonFermion5D.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonFermion5D.h b/lib/qcd/action/fermion/WilsonFermion5D.h
index a56bee11..fc8b5d41 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.h
+++ b/lib/qcd/action/fermion/WilsonFermion5D.h
@@ -88,8 +88,8 @@ namespace Grid {
       virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
       virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
 
-      void MomentumSpacePropagatorHt(FermionField &out,const FermionField &in) ;
-      void MomentumSpacePropagatorHw(FermionField &out,const FermionField &in) ;
+      void MomentumSpacePropagatorHt(FermionField &out,const FermionField &in,RealD mass) ;
+      void MomentumSpacePropagatorHw(FermionField &out,const FermionField &in,RealD mass) ;
 
       // Implement hopping term non-hermitian hopping term; half cb or both
       // Implement s-diagonal DW

From dc389e467c833c13ed1a4a882009351297ff1b12 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:48:05 +0100
Subject: [PATCH 036/149] axpy_ssp for any coeff type via template

---
 lib/qcd/utils/LinalgUtils.h | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/lib/qcd/utils/LinalgUtils.h b/lib/qcd/utils/LinalgUtils.h
index c2cb95ee..e7e6a794 100644
--- a/lib/qcd/utils/LinalgUtils.h
+++ b/lib/qcd/utils/LinalgUtils.h
@@ -39,8 +39,8 @@ namespace QCD{
 //on the 5d (rb4d) checkerboarded lattices
 ////////////////////////////////////////////////////////////////////////
 
-template<class vobj> 
-void axpibg5x(Lattice<vobj> &z,const Lattice<vobj> &x,RealD a,RealD b)
+template<class vobj,class Coeff>
+void axpibg5x(Lattice<vobj> &z,const Lattice<vobj> &x,Coeff a,Coeff b)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,z);
@@ -57,8 +57,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj> 
-void axpby_ssp(Lattice<vobj> &z, RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj,class Coeff> 
+void axpby_ssp(Lattice<vobj> &z, Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -72,8 +72,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj> 
-void ag5xpby_ssp(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj,class Coeff> 
+void ag5xpby_ssp(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -90,8 +90,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj> 
-void axpbg5y_ssp(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj,class Coeff> 
+void axpbg5y_ssp(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -108,8 +108,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj> 
-void ag5xpbg5y_ssp(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj,class Coeff> 
+void ag5xpbg5y_ssp(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -127,8 +127,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj> 
-void axpby_ssp_pminus(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj,class Coeff> 
+void axpby_ssp_pminus(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);
@@ -144,8 +144,8 @@ PARALLEL_FOR_LOOP
   }
 }
 
-template<class vobj> 
-void axpby_ssp_pplus(Lattice<vobj> &z,RealD a,const Lattice<vobj> &x,RealD b,const Lattice<vobj> &y,int s,int sp)
+template<class vobj,class Coeff> 
+void axpby_ssp_pplus(Lattice<vobj> &z,Coeff a,const Lattice<vobj> &x,Coeff b,const Lattice<vobj> &y,int s,int sp)
 {
   z.checkerboard = x.checkerboard;
   conformable(x,y);

From db749f103f86fbf3977cccd66b7b06f3d9687a9a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:48:35 +0100
Subject: [PATCH 037/149] Add Wilson, DWF, Overlap feynman rule tests

---
 tests/core/Test_fft.cc | 346 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 289 insertions(+), 57 deletions(-)

diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index a20791fb..562a53a1 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -1,6 +1,6 @@
     /*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+    grid` physics library, www.github.com/paboyle/Grid 
 
     Source file: ./tests/Test_cshift.cc
 
@@ -61,18 +61,22 @@ int main (int argc, char ** argv)
   LatticeSpinMatrixD    S(&GRID);
   LatticeSpinMatrixD    Stilde(&GRID);
   
-  std::vector<int> p({1,2,3,2});
+  std::vector<int> p({1,3,2,3});
 
   one = ComplexD(1.0,0.0);
   zz  = ComplexD(0.0,0.0);
 
   ComplexD ci(0.0,1.0);
 
+
+  std::cout<<"*************************************************"<<std::endl;
+  std::cout<<"Testing Fourier xfrom of known plane wave        "<<std::endl;
+  std::cout<<"*************************************************"<<std::endl;
   C=zero;
   for(int mu=0;mu<4;mu++){
     RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
     LatticeCoordinate(coor,mu);
-    C = C - (TwoPiL * p[mu]) * coor;
+    C = C + (TwoPiL * p[mu]) * coor;
   }
 
   C = exp(C*ci);
@@ -82,11 +86,11 @@ int main (int argc, char ** argv)
 
   FFT theFFT(&GRID);
 
-  theFFT.FFT_dim(Ctilde,C,0,FFT::forward);  C=Ctilde;
+  std::cout<<" Benchmarking FFT of LatticeComplex  "<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,0,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<" Mflops "<<std::endl;
   theFFT.FFT_dim(Ctilde,C,1,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<" Mflops "<<std::endl;
-  theFFT.FFT_dim(Ctilde,C,2,FFT::forward);  C=Ctilde;
-  theFFT.FFT_dim(Ctilde,C,3,FFT::forward);  
-
+  theFFT.FFT_dim(Ctilde,C,2,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<" Mflops "<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,3,FFT::forward);            std::cout << theFFT.MFlops()<<" Mflops "<<std::endl;
 
   //  C=zero;
   //  Ctilde = where(abs(Ctilde)<1.0e-10,C,Ctilde);
@@ -95,9 +99,11 @@ int main (int argc, char ** argv)
 
   Cref=zero;
   pokeSite(cVol,Cref,p);
+  //  std::cout <<"Ctilde "<< Ctilde <<std::endl;
+  //  std::cout <<"Cref   "<< Cref <<std::endl;
+
   Cref=Cref-Ctilde;
   std::cout << "diff scalar "<<norm2(Cref) << std::endl;
-
   C=Csav;
   theFFT.FFT_all_dim(Ctilde,C,FFT::forward);
   theFFT.FFT_all_dim(Cref,Ctilde,FFT::backward); 
@@ -107,10 +113,11 @@ int main (int argc, char ** argv)
   Cref= Cref - C;
   std::cout << " invertible check " << norm2(Cref)<<std::endl;
 
-  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde;
+  std::cout<<" Benchmarking FFT of LatticeSpinMatrix  "<<std::endl;
+  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<" mflops "<<std::endl;
   theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<" mflops "<<std::endl;
-  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;
-  theFFT.FFT_dim(Stilde,S,3,FFT::forward);
+  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<" mflops "<<std::endl;
+  theFFT.FFT_dim(Stilde,S,3,FFT::forward);std::cout << theFFT.MFlops()<<" mflops "<<std::endl;
 
   SpinMatrixD Sp; 
   Sp = zero; Sp = Sp+cVol;
@@ -124,94 +131,319 @@ int main (int argc, char ** argv)
   /*
    */
   std::vector<int> seeds({1,2,3,4});
+  GridSerialRNG          sRNG;  sRNG.SeedFixedIntegers(seeds); // naughty seeding
   GridParallelRNG          pRNG(&GRID);
   pRNG.SeedFixedIntegers(seeds);
 
   LatticeGaugeFieldD Umu(&GRID);
 
   SU3::ColdConfiguration(pRNG,Umu); // Unit gauge
-  
+  //  Umu=zero;
+  ////////////////////////////////////////////////////
+  // Wilson test
+  ////////////////////////////////////////////////////
   {
     LatticeFermionD    src(&GRID); gaussian(pRNG,src);
     LatticeFermionD    tmp(&GRID);
     LatticeFermionD    ref(&GRID);
     
-    RealD mass=0.1;
+    RealD mass=0.01;
     WilsonFermionD Dw(Umu,GRID,RBGRID,mass);
     
     Dw.M(src,tmp);
 
-    std::cout << " src = " <<norm2(src)<<std::endl;
-    std::cout << " tmp = " <<norm2(tmp)<<std::endl;
+    std::cout << "Dw src = " <<norm2(src)<<std::endl;
+    std::cout << "Dw tmp = " <<norm2(tmp)<<std::endl;
     
-    Dw.FreePropagator(tmp,ref);
+    Dw.FreePropagator(tmp,ref,mass);
 
-    std::cout << " ref = " <<norm2(ref)<<std::endl;
+    std::cout << "Dw ref = " <<norm2(ref)<<std::endl;
     
     ref = ref - src;
     
-    std::cout << " ref-src = " <<norm2(ref)<<std::endl;
+    std::cout << "Dw ref-src = " <<norm2(ref)<<std::endl;
   }
 
+  ////////////////////////////////////////////////////
+  // Dwf matrix
+  ////////////////////////////////////////////////////
   {
-    LatticeFermionD    src(&GRID); gaussian(pRNG,src);
-    LatticeFermionD    tmp(&GRID);
-    LatticeFermionD    ref(&GRID);
+    std::cout<<"****************************************"<<std::endl;
+    std::cout<<"Testing Fourier representation of Ddwf"<<std::endl;
+    std::cout<<"****************************************"<<std::endl;
+    
+    const int Ls=16;
+    const int sdir=0;
+    RealD mass=0.01;
+    RealD M5  =1.0;
+    Gamma G5(Gamma::Gamma5);
 
-    const int Ls=8;
     GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,&GRID);
     GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,&GRID);
 
-    RealD mass=0.1;
-    RealD M5  =0.9;
+    std::cout<<"Making Ddwf"<<std::endl;
     DomainWallFermionD Ddwf(Umu,*FGrid,*FrbGrid,GRID,RBGRID,mass,M5);
 
-    // Need to solve and project 4d. New test required.
-
-    Ddwf.MomentumSpacePropagatorHw(ref,src) ;
-    std::cout << " Hw Mom space \n";
-
-
-    Ddwf.MomentumSpacePropagatorHt(ref,src) ;
-    std::cout << " Ht Mom space \n";
+    GridParallelRNG          RNG5(FGrid);      RNG5.SeedFixedIntegers(seeds);
+    LatticeFermionD    src5(FGrid); gaussian(RNG5,src5);
+    LatticeFermionD    src5_p(FGrid); 
+    LatticeFermionD    result5(FGrid); 
+    LatticeFermionD    ref5(FGrid); 
+    LatticeFermionD    tmp5(FGrid); 
 
+    /////////////////////////////////////////////////////////////////
+    // result5 is the non pert operator in 4d mom space
+    /////////////////////////////////////////////////////////////////
+    Ddwf.M(src5,tmp5); 
+    ref5 = tmp5;
+    
+    FFT theFFT5(FGrid);
       
-    {   
-      Gamma G5(Gamma::Gamma5);
+    theFFT5.FFT_dim(result5,tmp5,1,FFT::forward); tmp5 = result5;
+    theFFT5.FFT_dim(result5,tmp5,2,FFT::forward); tmp5 = result5;
+    theFFT5.FFT_dim(result5,tmp5,3,FFT::forward); tmp5 = result5;
+    theFFT5.FFT_dim(result5,tmp5,4,FFT::forward); result5 = result5*ComplexD(::sqrt(1.0/vol),0.0);
+    
+    std::cout<<"Fourier xformed Ddwf"<<std::endl;
+    
+    tmp5 = src5;
+    theFFT5.FFT_dim(src5_p,tmp5,1,FFT::forward); tmp5 = src5_p;
+    theFFT5.FFT_dim(src5_p,tmp5,2,FFT::forward); tmp5 = src5_p;
+    theFFT5.FFT_dim(src5_p,tmp5,3,FFT::forward); tmp5 = src5_p;
+    theFFT5.FFT_dim(src5_p,tmp5,4,FFT::forward); src5_p = src5_p*ComplexD(::sqrt(1.0/vol),0.0);
 
-      LatticeFermionD    src5(FGrid); src5=zero;
-      LatticeFermionD    result5(FGrid); result5=zero;
-      LatticeFermionD    result4(&GRID); 
-
-      const int sdir=0;
+    std::cout<<"Fourier xformed src5"<<std::endl;
       
-      tmp =   (src + G5*src)*0.5;
-      InsertSlice(tmp,src5,Ls-1,sdir);
+    /////////////////////////////////////////////////////////////////
+    // work out the predicted from Fourier
+    /////////////////////////////////////////////////////////////////
+    Gamma::GammaMatrix Gmu [] = {
+      Gamma::GammaX,
+      Gamma::GammaY,
+      Gamma::GammaZ,
+      Gamma::GammaT,
+      Gamma::Gamma5
+    };
+    LatticeFermionD    Kinetic(FGrid); Kinetic = zero;
+    LatticeComplexD    kmu(FGrid); 
+    LatticeInteger     scoor(FGrid); 
+    LatticeComplexD    sk (FGrid); sk = zero;
+    LatticeComplexD    sk2(FGrid); sk2= zero;
+    LatticeComplexD    W(FGrid); W= zero;
+    //      LatticeComplexD    a(FGrid); a= zero;
+    LatticeComplexD    one(FGrid); one =ComplexD(1.0,0.0);
+    ComplexD ci(0.0,1.0);
+    
+    for(int mu=0;mu<Nd;mu++) {
       
-      tmp =   (src - G5*src)*0.5;
-      InsertSlice(tmp,src5,0,sdir);
+      LatticeCoordinate(kmu,mu+1);
       
-      MdagMLinearOperator<DomainWallFermionD,LatticeFermionD> HermOp(Ddwf);
-      ConjugateGradient<LatticeFermionD> CG(1.0e-4,1000);
-      CG(HermOp,src5,result5);
-      result5 = zero;
-
-      ExtractSlice(tmp,result5,0,sdir);
-      result4 = (tmp+G5*tmp)*0.5;
+      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+      
+      kmu = TwoPiL * kmu;
+      
+      sk2 = sk2 + 2.0*sin(kmu*0.5)*sin(kmu*0.5);
+      sk  = sk  +     sin(kmu)    *sin(kmu); 
+      
+      // -1/2 Dw ->  1/2 gmu (eip - emip) = i sinp gmu
+      Kinetic = Kinetic + sin(kmu)*ci*(Gamma(Gmu[mu])*src5_p);
       
-      ExtractSlice(tmp,result5,Ls-1,sdir);
-      result4 = result4+(tmp-G5*tmp)*0.5;
-
-      std::cout << "src     "<<norm2(src)<<std::endl;
-      std::cout << "src5    "<<norm2(src5)<<std::endl;
-      std::cout << "result4 "<<norm2(result4)<<std::endl;
-      std::cout << "ref     "<<norm2(ref)<<std::endl;
     }
+    
+    // NB implicit sum over mu
+    //
+    // 1-1/2 Dw = 1 - 1/2 ( eip+emip)
+    //          = - 1/2 (ei - 2 +  emi) 
+    //          = - 1/4 2 (eih - eimh)(eih - eimh) 
+    //          = 2 sink/2 ink/2 = sk2
+    
+    W = one - M5 + sk2; 
+    Kinetic = Kinetic + W * src5_p;
+    
+    LatticeCoordinate(scoor,sdir);
+    
+    tmp5 = Cshift(src5_p,sdir,+1);
+    tmp5 = (tmp5 - G5*tmp5)*0.5;
+    tmp5 = where(scoor==Integer(Ls-1),mass*tmp5,-tmp5);
+    Kinetic = Kinetic + tmp5;
+    
+    tmp5 = Cshift(src5_p,sdir,-1);
+    tmp5 = (tmp5 + G5*tmp5)*0.5;
+    tmp5 = where(scoor==Integer(0),mass*tmp5,-tmp5);
+    Kinetic = Kinetic + tmp5;
+    
+    std::cout<<"Momentum space Ddwf  "<< norm2(Kinetic)<<std::endl;
+    std::cout<<"Stencil Ddwf         "<< norm2(result5)<<std::endl;
+    
+    result5 = result5 - Kinetic;
+    std::cout<<"diff "<< norm2(result5)<<std::endl;
+    
+  }
+
+  ////////////////////////////////////////////////////
+  // Dwf prop
+  ////////////////////////////////////////////////////
+  {
+    std::cout<<"****************************************"<<std::endl;
+    std::cout << "Testing Ddwf Ht Mom space 4d propagator \n";
+    std::cout<<"****************************************"<<std::endl;
+
+    LatticeFermionD    src(&GRID); gaussian(pRNG,src);
+    LatticeFermionD    tmp(&GRID);
+    LatticeFermionD    ref(&GRID);
+    LatticeFermionD    diff(&GRID);
+
+    std::vector<int> point(4,0);
+    src=zero;
+    SpinColourVectorD ferm; gaussian(sRNG,ferm);
+    pokeSite(ferm,src,point);
+
+    const int Ls=32;
+    GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,&GRID);
+    GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,&GRID);
+
+    RealD mass=0.01;
+    RealD M5  =0.8;
+    DomainWallFermionD Ddwf(Umu,*FGrid,*FrbGrid,GRID,RBGRID,mass,M5);
+
+    // Momentum space prop
+    std::cout << " Solving by FFT and Feynman rules" <<std::endl;
+    Ddwf.FreePropagator(src,ref,mass) ;
+
+    Gamma G5(Gamma::Gamma5);
+
+    LatticeFermionD    src5(FGrid); src5=zero;
+    LatticeFermionD    tmp5(FGrid); 
+    LatticeFermionD    result5(FGrid); result5=zero;
+    LatticeFermionD    result4(&GRID); 
+    const int sdir=0;
+
+    ////////////////////////////////////////////////////////////////////////
+    // Domain wall physical field source
+    ////////////////////////////////////////////////////////////////////////
+    /*
+	chi_5[0]   = chiralProjectPlus(chi);
+	chi_5[Ls-1]= chiralProjectMinus(chi);
+    */      
+    tmp =   (src + G5*src)*0.5;      InsertSlice(tmp,src5,   0,sdir);
+    tmp =   (src - G5*src)*0.5;      InsertSlice(tmp,src5,Ls-1,sdir);
+    
+    ////////////////////////////////////////////////////////////////////////
+    // Conjugate gradient on normal equations system
+    ////////////////////////////////////////////////////////////////////////
+    std::cout << " Solving by Conjugate Gradient (CGNE)" <<std::endl;
+    Ddwf.Mdag(src5,tmp5);
+    src5=tmp5;
+    MdagMLinearOperator<DomainWallFermionD,LatticeFermionD> HermOp(Ddwf);
+    ConjugateGradient<LatticeFermionD> CG(1.0e-16,10000);
+    CG(HermOp,src5,result5);
+    
+    ////////////////////////////////////////////////////////////////////////
+    // Domain wall physical field propagator
+    ////////////////////////////////////////////////////////////////////////
+    /*
+      psi  = chiralProjectMinus(psi_5[0]);
+      psi += chiralProjectPlus(psi_5[Ls-1]);
+    */
+    ExtractSlice(tmp,result5,0   ,sdir);   result4 =         (tmp-G5*tmp)*0.5;
+    ExtractSlice(tmp,result5,Ls-1,sdir);   result4 = result4+(tmp+G5*tmp)*0.5;
+    
+    std::cout << " Taking difference" <<std::endl;
+    std::cout << "Ddwf result4 "<<norm2(result4)<<std::endl;
+    std::cout << "Ddwf ref     "<<norm2(ref)<<std::endl;
+    
+    diff = ref - result4;
+    std::cout << "result - ref     "<<norm2(diff)<<std::endl;
+
+  }
+
+
+
+  ////////////////////////////////////////////////////
+  // Dwf prop
+  ////////////////////////////////////////////////////
+  {
+    std::cout<<"****************************************"<<std::endl;
+    std::cout << "Testing Dov Ht Mom space 4d propagator \n";
+    std::cout<<"****************************************"<<std::endl;
+
+    LatticeFermionD    src(&GRID); gaussian(pRNG,src);
+    LatticeFermionD    tmp(&GRID);
+    LatticeFermionD    ref(&GRID);
+    LatticeFermionD    diff(&GRID);
+
+    std::vector<int> point(4,0);
+    src=zero;
+    SpinColourVectorD ferm; gaussian(sRNG,ferm);
+    pokeSite(ferm,src,point);
+
+    const int Ls=48;
+    GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,&GRID);
+    GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,&GRID);
+
+    RealD mass=0.01;
+    RealD M5  =0.8;
+
+    OverlapWilsonCayleyTanhFermionD Dov(Umu,*FGrid,*FrbGrid,GRID,RBGRID,mass,M5,1.0);
+
+    // Momentum space prop
+    std::cout << " Solving by FFT and Feynman rules" <<std::endl;
+    Dov.FreePropagator(src,ref,mass) ;
+
+    Gamma G5(Gamma::Gamma5);
+
+    LatticeFermionD    src5(FGrid); src5=zero;
+    LatticeFermionD    tmp5(FGrid); 
+    LatticeFermionD    result5(FGrid); result5=zero;
+    LatticeFermionD    result4(&GRID); 
+    const int sdir=0;
+
+    ////////////////////////////////////////////////////////////////////////
+    // Domain wall physical field source; need D_minus
+    ////////////////////////////////////////////////////////////////////////
+    /*
+	chi_5[0]   = chiralProjectPlus(chi);
+	chi_5[Ls-1]= chiralProjectMinus(chi);
+    */      
+    tmp =   (src + G5*src)*0.5;      InsertSlice(tmp,src5,   0,sdir);
+    tmp =   (src - G5*src)*0.5;      InsertSlice(tmp,src5,Ls-1,sdir);
+
+    
+    ////////////////////////////////////////////////////////////////////////
+    // Conjugate gradient on normal equations system
+    ////////////////////////////////////////////////////////////////////////
+    std::cout << " Solving by Conjugate Gradient (CGNE)" <<std::endl;
+    Dov.Dminus(src5,tmp5);
+    src5=tmp5;
+    Dov.Mdag(src5,tmp5);
+    src5=tmp5;
+    MdagMLinearOperator<OverlapWilsonCayleyTanhFermionD,LatticeFermionD> HermOp(Dov);
+    ConjugateGradient<LatticeFermionD> CG(1.0e-16,10000);
+    CG(HermOp,src5,result5);
+    
+    ////////////////////////////////////////////////////////////////////////
+    // Domain wall physical field propagator
+    ////////////////////////////////////////////////////////////////////////
+    /*
+      psi  = chiralProjectMinus(psi_5[0]);
+      psi += chiralProjectPlus(psi_5[Ls-1]);
+    */
+    ExtractSlice(tmp,result5,0   ,sdir);   result4 =         (tmp-G5*tmp)*0.5;
+    ExtractSlice(tmp,result5,Ls-1,sdir);   result4 = result4+(tmp+G5*tmp)*0.5;
+    
+    std::cout << " Taking difference" <<std::endl;
+    std::cout << "Dov result4 "<<norm2(result4)<<std::endl;
+    std::cout << "Dov ref     "<<norm2(ref)<<std::endl;
+    
+    diff = ref - result4;
+    std::cout << "result - ref     "<<norm2(diff)<<std::endl;
+
   }
 
   {
     typedef GaugeImplTypes<vComplexD, 1> QEDGimplTypesD;
     typedef Photon<QEDGimplTypesD>       QEDGaction;
+
     QEDGaction Maxwell(QEDGaction::FEYNMAN_L);
     QEDGaction::GaugeField Prop(&GRID);Prop=zero;
     QEDGaction::GaugeField Source(&GRID);Source=zero;

From 3d5c9a1ee98146322cb48eb6674b47f1fcfc6063 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:50:13 +0100
Subject: [PATCH 038/149] No compile fix on clang++ 3.9

---
 lib/simd/Grid_avx.h | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/simd/Grid_avx.h b/lib/simd/Grid_avx.h
index 33496e07..902142d2 100644
--- a/lib/simd/Grid_avx.h
+++ b/lib/simd/Grid_avx.h
@@ -450,13 +450,12 @@ namespace Optimization {
   };
 
 #if defined (AVX2) || defined (AVXFMA4) 
-#define _mm256_alignr_epi32(ret,a,b,n) ret=(__m256) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*4)%16)
-#define _mm256_alignr_epi64(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16)
+#define _mm256_alignr_epi32_grid(ret,a,b,n) ret=(__m256)  _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*4)%16)
+#define _mm256_alignr_epi64_grid(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16)
 #endif
 
 #if defined (AVX1) 
-
-#define _mm256_alignr_epi32(ret,a,b,n) {	\
+#define _mm256_alignr_epi32_grid(ret,a,b,n) {	\
     __m128 aa, bb;				\
 						\
     aa  = _mm256_extractf128_ps(a,1);		\
@@ -470,7 +469,7 @@ namespace Optimization {
     ret = _mm256_insertf128_ps(ret,aa,0);	\
   }
 
-#define _mm256_alignr_epi64(ret,a,b,n) {	\
+#define _mm256_alignr_epi64_grid(ret,a,b,n) {	\
     __m128d aa, bb;				\
 						\
     aa  = _mm256_extractf128_pd(a,1);		\
@@ -530,9 +529,9 @@ namespace Optimization {
       __m256 tmp = Permute::Permute0(in);
       __m256 ret;
       if ( n > 3 ) { 
-	_mm256_alignr_epi32(ret,in,tmp,n);  
+	_mm256_alignr_epi32_grid(ret,in,tmp,n);  
       } else {
-        _mm256_alignr_epi32(ret,tmp,in,n);          
+        _mm256_alignr_epi32_grid(ret,tmp,in,n);          
       }
       //      std::cout << " align epi32 n=" <<n<<" in "<<tmp<<in<<" -> "<< ret <<std::endl;
       return ret;
@@ -543,9 +542,9 @@ namespace Optimization {
       __m256d tmp = Permute::Permute0(in);
       __m256d ret;
       if ( n > 1 ) {
-	_mm256_alignr_epi64(ret,in,tmp,n);          
+	_mm256_alignr_epi64_grid(ret,in,tmp,n);          
       } else {
-        _mm256_alignr_epi64(ret,tmp,in,n);          
+        _mm256_alignr_epi64_grid(ret,tmp,in,n);          
       }
       //      std::cout << " align epi64 n=" <<n<<" in "<<tmp<<in<<" -> "<< ret <<std::endl;
       return ret;

From 8d11681aac62ac368a195f1c0f36c8b4335514fa Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 10 Oct 2016 23:50:42 +0100
Subject: [PATCH 039/149] verbose remove

---
 lib/simd/Grid_vector_types.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/simd/Grid_vector_types.h b/lib/simd/Grid_vector_types.h
index 4c3e903d..e1592eef 100644
--- a/lib/simd/Grid_vector_types.h
+++ b/lib/simd/Grid_vector_types.h
@@ -583,8 +583,6 @@ inline Grid_simd<S, V> operator/(Grid_simd<S, V> a, Grid_simd<S, V> b) {
 
   ret.v=binary<V>(ret.v, real_den.v, DivSIMD());
 
-  std::cout << " Div call from complex "<<a<<" / " << b<< " = " <<ret<< std::endl;
-
   return ret;
 };
 

From 6f408256bc346cdbfe95c2ff40e3133f8ad3ff06 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 11 Oct 2016 10:03:01 +0100
Subject: [PATCH 040/149] FMA4 option moved on the align

---
 lib/simd/Grid_avx.h | 23 +++--------------------
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/lib/simd/Grid_avx.h b/lib/simd/Grid_avx.h
index 902142d2..3268fee3 100644
--- a/lib/simd/Grid_avx.h
+++ b/lib/simd/Grid_avx.h
@@ -449,12 +449,12 @@ namespace Optimization {
 
   };
 
-#if defined (AVX2) || defined (AVXFMA4) 
+#if defined (AVX2)
 #define _mm256_alignr_epi32_grid(ret,a,b,n) ret=(__m256)  _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*4)%16)
 #define _mm256_alignr_epi64_grid(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16)
 #endif
 
-#if defined (AVX1) 
+#if defined (AVX1) || defined (AVXFMA4)  
 #define _mm256_alignr_epi32_grid(ret,a,b,n) {	\
     __m128 aa, bb;				\
 						\
@@ -484,20 +484,7 @@ namespace Optimization {
   }
 
 #endif
-  /*
-    inline std::ostream & operator << (std::ostream& stream, const __m256 a)
-    {
-      const float *p=(const float *)&a;
-      stream<< "{"<<p[0]<<","<<p[1]<<","<<p[2]<<","<<p[3]<<","<<p[4]<<","<<p[5]<<","<<p[6]<<","<<p[7]<<"}";
-      return stream;
-    };
-    inline std::ostream & operator<< (std::ostream& stream, const __m256d a)
-    {
-      const double *p=(const double *)&a;
-      stream<< "{"<<p[0]<<","<<p[1]<<","<<p[2]<<","<<p[3]<<"}";
-      return stream;
-    };
-  */
+
   struct Rotate{
 
     static inline __m256 rotate(__m256 in,int n){ 
@@ -533,7 +520,6 @@ namespace Optimization {
       } else {
         _mm256_alignr_epi32_grid(ret,tmp,in,n);          
       }
-      //      std::cout << " align epi32 n=" <<n<<" in "<<tmp<<in<<" -> "<< ret <<std::endl;
       return ret;
     };
 
@@ -546,14 +532,11 @@ namespace Optimization {
       } else {
         _mm256_alignr_epi64_grid(ret,tmp,in,n);          
       }
-      //      std::cout << " align epi64 n=" <<n<<" in "<<tmp<<in<<" -> "<< ret <<std::endl;
       return ret;
     };
 
   };
 
-
-
   //Complex float Reduce
   template<>
     inline Grid::ComplexF Reduce<Grid::ComplexF, __m256>::operator()(__m256 in){

From 6e01264bb7f5bafa2bc5b2043a93d77d8393f378 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 11 Oct 2016 10:03:39 +0100
Subject: [PATCH 041/149] don't use static by default

---
 lib/Threads.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Threads.h b/lib/Threads.h
index 5abe2ca4..9d1295e5 100644
--- a/lib/Threads.h
+++ b/lib/Threads.h
@@ -37,7 +37,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #ifdef GRID_OMP
 #include <omp.h>
-#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(static)")
+#define PARALLEL_FOR_LOOP _Pragma("omp parallel for ")
 #define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)")
 #else
 #define PARALLEL_FOR_LOOP 

From 42cd148f5e450e1f22de3d53001256883716980e Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 11 Oct 2016 16:06:06 +0100
Subject: [PATCH 042/149] Base pointer for comms buffer under AVX512 assembly

---
 lib/Stencil.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index f5056485..061accc3 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -265,16 +265,18 @@
        }
        inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
 	 //_mm_prefetch((char *)&_entries[ent+1],_MM_HINT_T0);
+	 uint64_t cbase = & comm_buf[0];
 	 local = _entries[ent]._is_local;
 	 perm  = _entries[ent]._permute;
 	 if (perm)  ptype = _permute_type[point]; 
-	 if (local) return base + _entries[ent]._byte_offset;
-	 else       return _entries[ent]._byte_offset;
+	 if (local) return  base + _entries[ent]._byte_offset;
+	 else       return cbase +_entries[ent]._byte_offset;
        }
        inline uint64_t GetPFInfo(int ent,uint64_t base) {
+	 uint64_t cbase = & comm_buf[0];
 	 int local = _entries[ent]._is_local;
-	 if (local) return base + _entries[ent]._byte_offset;
-	 else       return        _entries[ent]._byte_offset;
+	 if (local) return  base + _entries[ent]._byte_offset;
+	 else       return cbase + _entries[ent]._byte_offset;
        }
 
        // Comms buffers

From 7240d731843c2f86e75ecfda5de1eb680bf9baca Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 11 Oct 2016 22:21:07 +0100
Subject: [PATCH 043/149] Parallelise the x faces; fix the segv on KNL with
 comms

---
 lib/cartesian/Cartesian_base.h      |  7 ++-----
 lib/cartesian/Cartesian_full.h      |  7 +++++++
 lib/cartesian/Cartesian_red_black.h | 25 ++++++++++++++++++++++++-
 lib/cshift/Cshift_common.h          | 11 +++++++++--
 4 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h
index b69c3435..8a24c87b 100644
--- a/lib/cartesian/Cartesian_base.h
+++ b/lib/cartesian/Cartesian_base.h
@@ -81,11 +81,8 @@ public:
     virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0;
     virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0;
     virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int cb)=0;
-    int  CheckerBoardFromOindex (int Oindex){
-      std::vector<int> ocoor;
-      oCoorFromOindex(ocoor,Oindex); 
-      return CheckerBoard(ocoor);
-    }
+    virtual int CheckerBoardFromOindex (int Oindex)=0;
+    virtual int CheckerBoardFromOindexTable (int Oindex)=0;
 
     //////////////////////////////////////////////////////////////////////////////////////////////
     // Local layout calculations
diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h
index 1f8f7514..14ab8b55 100644
--- a/lib/cartesian/Cartesian_full.h
+++ b/lib/cartesian/Cartesian_full.h
@@ -39,6 +39,13 @@ class GridCartesian: public GridBase {
 
 public:
 
+    virtual int  CheckerBoardFromOindexTable (int Oindex) {
+      return 0;
+    }
+    virtual int  CheckerBoardFromOindex (int Oindex)
+    {
+      return 0;
+    }
     virtual int CheckerBoarded(int dim){
       return 0;
     }
diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h
index 9f5a1103..3c10403f 100644
--- a/lib/cartesian/Cartesian_red_black.h
+++ b/lib/cartesian/Cartesian_red_black.h
@@ -43,6 +43,7 @@ class GridRedBlackCartesian : public GridBase
 public:
     std::vector<int> _checker_dim_mask;
     int              _checker_dim;
+    std::vector<int> _checker_board;
 
     virtual int CheckerBoarded(int dim){
       if( dim==_checker_dim) return 1;
@@ -72,12 +73,20 @@ public:
       // or by looping over x,y,z and multiply rather than computing checkerboard.
 	  
       if ( (source_cb+ocb)&1 ) {
-
 	return (shift)/2;
       } else {
 	return (shift+1)/2;
       }
     }
+    virtual int  CheckerBoardFromOindexTable (int Oindex) {
+      return _checker_board[Oindex];
+    }
+    virtual int  CheckerBoardFromOindex (int Oindex)
+    {
+      std::vector<int> ocoor;
+      oCoorFromOindex(ocoor,Oindex);
+      return CheckerBoard(ocoor);
+    }
     virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite){
 
       if(dim != _checker_dim) return shift;
@@ -185,6 +194,8 @@ public:
 	  _ostride[d] = _ostride[d-1]*_rdimensions[d-1];
 	  _istride[d] = _istride[d-1]*_simd_layout[d-1];
 	}
+
+
       }
             
       ////////////////////////////////////////////////////////////////////////////////////////////
@@ -205,6 +216,18 @@ public:
 	_slice_nblock[d]=nblock;
 	block = block*_rdimensions[d];
       }
+
+      ////////////////////////////////////////////////
+      // Create a checkerboard lookup table
+      ////////////////////////////////////////////////
+      int rvol = 1;
+      for(int d=0;d<_ndimension;d++){
+	rvol=rvol * _rdimensions[d];
+      }
+      _checker_board.resize(rvol);
+      for(int osite=0;osite<_osites;osite++){
+	_checker_board[osite] = CheckerBoardFromOindex (osite);
+      }
       
     };
 protected:
diff --git a/lib/cshift/Cshift_common.h b/lib/cshift/Cshift_common.h
index b8e1284a..b0e9b798 100644
--- a/lib/cshift/Cshift_common.h
+++ b/lib/cshift/Cshift_common.h
@@ -1,3 +1,4 @@
+
     /*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
@@ -56,6 +57,7 @@ Gather_plane_simple (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<
   
   int e1=rhs._grid->_slice_nblock[dimension];
   int e2=rhs._grid->_slice_block[dimension];
+
   int stride=rhs._grid->_slice_stride[dimension];
   if ( cbmask == 0x3 ) { 
 PARALLEL_NESTED_LOOP2
@@ -68,15 +70,20 @@ PARALLEL_NESTED_LOOP2
     }
   } else { 
      int bo=0;
+     std::vector<std::pair<int,int> > table;
      for(int n=0;n<e1;n++){
        for(int b=0;b<e2;b++){
 	 int o  = n*stride;
-	 int ocb=1<<rhs._grid->CheckerBoardFromOindex(o+b);// Could easily be a table lookup
+	 int ocb=1<<rhs._grid->CheckerBoardFromOindexTable(o+b);
 	 if ( ocb &cbmask ) {
-	   buffer[off+bo++]=compress(rhs._odata[so+o+b]);
+	   table.push_back(std::pair<int,int> (bo++,o+b));
 	 }
        }
      }
+PARALLEL_FOR_LOOP     
+     for(int i=0;i<table.size();i++){
+       buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
+     }
   }
 }
 

From 0f182f033b3421041ccea8f2dae35d9036ac2b4d Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 11 Oct 2016 22:29:06 +0100
Subject: [PATCH 044/149] Drop macos with gcc

---
 .travis.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index de9c54d3..ae3efda8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,10 +9,6 @@ matrix:
     - os:        osx
       osx_image: xcode7.2
       compiler: clang
-    - os:        osx
-      osx_image: xcode7.2
-      compiler: gcc
-      env: VERSION=-5
     - compiler: gcc
       addons:
         apt:

From f7c2aa3ba54db5edec66a7cc3af521dddd5b8dc9 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 12 Oct 2016 00:29:13 +0100
Subject: [PATCH 045/149] runtime by default

---
 lib/Threads.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Threads.h b/lib/Threads.h
index 9d1295e5..b8915d74 100644
--- a/lib/Threads.h
+++ b/lib/Threads.h
@@ -37,7 +37,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #ifdef GRID_OMP
 #include <omp.h>
-#define PARALLEL_FOR_LOOP _Pragma("omp parallel for ")
+#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(runtime)")
 #define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)")
 #else
 #define PARALLEL_FOR_LOOP 

From 6b27c42dfe1d5c010e83804843a9b89bbf5480fd Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 12 Oct 2016 00:29:39 +0100
Subject: [PATCH 046/149] Cosmetic

---
 lib/qcd/action/gauge/Photon.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
index a7556b19..29d40f8b 100644
--- a/lib/qcd/action/gauge/Photon.h
+++ b/lib/qcd/action/gauge/Photon.h
@@ -77,8 +77,8 @@ namespace Grid{
 	  out = where(coor==Integer(0),zz,out);
 	}
       }
-      void  FeynmanGaugeMomentumSpacePropagator_TL(GaugeField &out, const GaugeField &in) { 
 
+      void  FeynmanGaugeMomentumSpacePropagator_TL(GaugeField &out, const GaugeField &in) { 
 
 	// what type LatticeComplex 
 	GridBase *grid = out._grid;
@@ -92,8 +92,8 @@ namespace Grid{
 
 	LatComplex denom(grid); denom= zero;
 	LatComplex   one(grid); one = ScalComplex(1.0,0.0);
+	LatComplex   kmu(grid); 
 
-	LatComplex kmu(grid); 
 	ScalComplex ci(0.0,1.0);
 	// momphase = n * 2pi / L
 	for(int mu=0;mu<Nd;mu++) {

From a123dcd7e9e54daad79522fa41dc7a899983835c Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 12 Oct 2016 00:29:57 +0100
Subject: [PATCH 047/149] Static required for shmem. Reading same object twice
 requires csum reset

---
 lib/parallelIO/BinaryIO.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h
index 184209dc..acd96a39 100644
--- a/lib/parallelIO/BinaryIO.h
+++ b/lib/parallelIO/BinaryIO.h
@@ -457,7 +457,7 @@ class BinaryIO {
     // available (how short sighted is that?)
     //////////////////////////////////////////////////////////
     Umu = zero;
-    static uint32_t csum=0;
+    static uint32_t csum; csum=0;
     fobj fileObj;
     static sobj siteObj; // Static to place in symmetric region for SHMEM
 

From 2d4a45c75849431d3654af6fdf6a86c4f670db32 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 12 Oct 2016 09:14:15 +0100
Subject: [PATCH 048/149] Typecast pointer

---
 lib/Stencil.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index 061accc3..83aace54 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -265,15 +265,15 @@
        }
        inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
 	 //_mm_prefetch((char *)&_entries[ent+1],_MM_HINT_T0);
-	 uint64_t cbase = & comm_buf[0];
+	 uint64_t cbase = (uint64_t) & comm_buf[0];
 	 local = _entries[ent]._is_local;
 	 perm  = _entries[ent]._permute;
 	 if (perm)  ptype = _permute_type[point]; 
 	 if (local) return  base + _entries[ent]._byte_offset;
-	 else       return cbase +_entries[ent]._byte_offset;
+	 else       return cbase + _entries[ent]._byte_offset;
        }
        inline uint64_t GetPFInfo(int ent,uint64_t base) {
-	 uint64_t cbase = & comm_buf[0];
+	 uint64_t cbase =  (uint64_t)& comm_buf[0];
 	 int local = _entries[ent]._is_local;
 	 if (local) return  base + _entries[ent]._byte_offset;
 	 else       return cbase + _entries[ent]._byte_offset;

From 81f2aeaece16b3ac4ab15ba92e0390095d96ab3c Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 12 Oct 2016 11:45:22 +0100
Subject: [PATCH 049/149] KNL streaming stores, and KNL performance coutners

---
 benchmarks/Benchmark_dwf.cc                   |   2 +
 benchmarks/Benchmark_dwf_sweep.cc             |   2 +-
 configure.ac                                  |  14 ++
 lib/AlignedAllocator.h                        |   5 +-
 lib/Stat.cc                                   | 233 ++++++++++++++++++
 lib/Stat.h                                    | 100 ++++++++
 lib/Threads.h                                 |   6 +-
 lib/cartesian/Cartesian_red_black.h           |   2 +-
 lib/qcd/action/fermion/WilsonFermion5D.cc     |  22 ++
 lib/qcd/action/fermion/WilsonFermion5D.h      |   3 +
 lib/qcd/action/fermion/WilsonKernelsAsmBody.h |   4 +
 lib/simd/Intel512common.h                     |   7 +-
 12 files changed, 393 insertions(+), 7 deletions(-)
 create mode 100644 lib/Stat.cc
 create mode 100644 lib/Stat.h

diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index bc9ab708..6a283085 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -251,11 +251,13 @@ int main (int argc, char ** argv)
       sr_o = zero;
 
       sDw.ZeroCounters();
+      sDw.stat.init("DhopEO");
       double t0=usecond();
       for (int i = 0; i < ncall; i++) {
         sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
       }
       double t1=usecond();
+      sDw.stat.print();
 
       double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
       double flops=(1344.0*volume*ncall)/2;
diff --git a/benchmarks/Benchmark_dwf_sweep.cc b/benchmarks/Benchmark_dwf_sweep.cc
index 18d2f389..ee78a33d 100644
--- a/benchmarks/Benchmark_dwf_sweep.cc
+++ b/benchmarks/Benchmark_dwf_sweep.cc
@@ -51,7 +51,7 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
-  const int Ls=16;
+  const int Ls=8;
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
diff --git a/configure.ac b/configure.ac
index b7205b4a..7bcdc49f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -70,6 +70,20 @@ case ${ac_LAPACK} in
         AC_DEFINE([USE_LAPACK],[1],[use LAPACK])
 esac
 
+################## first-touch ####################
+AC_ARG_ENABLE([numa],
+    [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
+    [ac_NUMA=${enable_NUMA}],[ac_NUMA=no])
+
+case ${ac_NUMA} in
+    no)
+        ;;
+    yes)
+        AC_DEFINE([GRID_NUMA],[1],[First touch numa locality]);;
+    *)
+        AC_DEFINE([GRID_NUMA],[1],[First touch numa locality]);;
+esac
+
 ################## FFTW3 ####################
 AC_ARG_WITH([fftw],    
             [AS_HELP_STRING([--with-fftw=prefix],
diff --git a/lib/AlignedAllocator.h b/lib/AlignedAllocator.h
index 2cd8263d..8f69971d 100644
--- a/lib/AlignedAllocator.h
+++ b/lib/AlignedAllocator.h
@@ -113,9 +113,8 @@ public:
 
 #endif
     _Tp tmp;
-#undef FIRST_TOUCH_OPTIMISE
-#ifdef FIRST_TOUCH_OPTIMISE
-#pragma omp parallel for 
+#ifdef GRID_NUMA
+#pragma omp parallel for schedule(static)
   for(int i=0;i<__n;i++){
     ptr[i]=tmp;
   }
diff --git a/lib/Stat.cc b/lib/Stat.cc
new file mode 100644
index 00000000..9a025225
--- /dev/null
+++ b/lib/Stat.cc
@@ -0,0 +1,233 @@
+#include <Grid.h>
+#include <PerfCount.h>
+#include <Stat.h>
+
+
+namespace Grid { 
+
+
+bool PmuStat::pmu_initialized=false;
+
+
+void PmuStat::init(const char *regname)
+{
+  name = regname;
+  if (!pmu_initialized)
+    {
+      std::cout<<"initialising pmu"<<std::endl;
+      pmu_initialized = true;
+      pmu_init();
+    }
+  clear();
+}
+void PmuStat::clear(void)
+{
+  count = 0;
+  tregion = 0;
+  pmc0 = 0;
+  pmc1 = 0;
+  inst = 0;
+  cyc = 0;
+  ref = 0;
+  tcycles = 0;
+  reads = 0;
+  writes = 0;
+}
+void PmuStat::print(void)
+{
+  std::cout <<"Reg "<<std::string(name)<<":\n";
+  std::cout <<"  region "<<tregion<<std::endl;
+  std::cout <<"  cycles "<<tcycles<<std::endl;
+  std::cout <<"  inst   "<<inst   <<std::endl;
+  std::cout <<"  cyc    "<<cyc    <<std::endl;
+  std::cout <<"  ref    "<<ref    <<std::endl;
+  std::cout <<"  pmc0   "<<pmc0   <<std::endl;
+  std::cout <<"  pmc1   "<<pmc1   <<std::endl;
+  std::cout <<"  count  "<<count  <<std::endl;
+  std::cout <<"  reads  "<<reads  <<std::endl;
+  std::cout <<"  writes "<<writes <<std::endl;
+}
+void PmuStat::start(void)
+{
+  pmu_start();
+  ++count;
+  xmemctrs(&mrstart, &mwstart);
+  tstart = _rdtsc();
+}
+void PmuStat::enter(int t)
+{
+  counters[0][t] = _rdpmc(0);
+  counters[1][t] = _rdpmc(1);
+  counters[2][t] = _rdpmc((1<<30)|0);
+  counters[3][t] = _rdpmc((1<<30)|1);
+  counters[4][t] = _rdpmc((1<<30)|2);
+  counters[5][t] = _rdtsc();
+}
+void PmuStat::exit(int t)
+{
+  counters[0][t] = _rdpmc(0) - counters[0][t];
+  counters[1][t] = _rdpmc(1) - counters[1][t];
+  counters[2][t] = _rdpmc((1<<30)|0) - counters[2][t];
+  counters[3][t] = _rdpmc((1<<30)|1) - counters[3][t];
+  counters[4][t] = _rdpmc((1<<30)|2) - counters[4][t];
+  counters[5][t] = _rdtsc() - counters[5][t];
+}
+void PmuStat::accum(int nthreads)
+{
+  tend = _rdtsc();
+  xmemctrs(&mrend, &mwend);
+  pmu_stop();
+  for (int t = 0; t < nthreads; ++t) {
+    pmc0 += counters[0][t];
+    pmc1 += counters[1][t];
+    inst += counters[2][t];
+    cyc += counters[3][t];
+    ref += counters[4][t];
+    tcycles += counters[5][t];
+  }
+  uint64_t region = tend - tstart;
+  tregion += region;
+  uint64_t mreads = mrend - mrstart;
+  reads += mreads;
+  uint64_t mwrites = mwend - mwstart;
+  writes += mwrites;
+}
+
+
+void PmuStat::pmu_fini(void) {}
+void PmuStat::pmu_start(void) {};
+void PmuStat::pmu_stop(void) {};
+void PmuStat::pmu_init(void)
+{
+#ifdef _KNIGHTS_LANDING_
+  KNLsetup();
+#endif
+}
+void PmuStat::xmemctrs(uint64_t *mr, uint64_t *mw)
+{
+#ifdef _KNIGHTS_LANDING_
+  ctrs c;
+  KNLreadctrs(c);
+  uint64_t emr = 0, emw = 0;
+  for (int i = 0; i < NEDC; ++i)
+    {
+      emr += c.edcrd[i];
+      emw += c.edcwr[i];
+    }
+  *mr = emr;
+  *mw = emw;
+#else
+  *mr = *mw = 0;
+#endif
+}
+
+#ifdef _KNIGHTS_LANDING_
+
+struct knl_gbl_ PmuStat::gbl;
+
+#define PMU_MEM
+
+void PmuStat::KNLevsetup(const char *ename, int &fd, int event, int umask)
+{
+  char fname[1024];
+  snprintf(fname, sizeof(fname), "%s/type", ename);
+  FILE *fp = fopen(fname, "r");
+  if (fp == 0) {
+    ::printf("open %s", fname);
+    ::exit(0);
+  }
+  int type;
+  int ret = fscanf(fp, "%d", &type);
+  assert(ret == 1);
+  fclose(fp);
+  //  std::cout << "Using PMU type "<<type<<" from " << std::string(ename) <<std::endl;
+
+  struct perf_event_attr hw = {};
+  hw.size = sizeof(hw);
+  hw.type = type;
+  // see /sys/devices/uncore_*/format/*
+  // All of the events we are interested in are configured the same way, but
+  // that isn't always true. Proper code would parse the format files
+  hw.config = event | (umask << 8);
+  //hw.read_format = PERF_FORMAT_GROUP;
+  // unfortunately the above only works within a single PMU; might
+  // as well just read them one at a time
+  int cpu = 0;
+  fd = perf_event_open(&hw, -1, cpu, -1, 0);
+  if (fd == -1) {
+    ::printf("CPU %d, box %s, event 0x%lx", cpu, ename, hw.config);
+    ::exit(0);
+  } else { 
+    //    std::cout << "event "<<std::string(ename)<<" set up for fd "<<fd<<" hw.config "<<hw.config <<std::endl;
+  }
+}
+
+
+ void PmuStat::KNLsetup(void){
+
+   int ret;
+   char fname[1024];
+
+   // MC RPQ inserts and WPQ inserts (reads & writes)
+   for (int mc = 0; mc < NMC; ++mc)
+     {
+       ::snprintf(fname, sizeof(fname), "/sys/devices/uncore_imc_%d",mc);
+       // RPQ Inserts
+       KNLevsetup(fname, gbl.mc_rd[mc], 0x1, 0x1);
+       // WPQ Inserts
+       KNLevsetup(fname, gbl.mc_wr[mc], 0x2, 0x1);
+     }
+   // EDC RPQ inserts and WPQ inserts
+   for (int edc=0; edc < NEDC; ++edc)
+     {
+       ::snprintf(fname, sizeof(fname), "/sys/devices/uncore_edc_eclk_%d",edc);
+       // RPQ inserts
+       KNLevsetup(fname, gbl.edc_rd[edc], 0x1, 0x1);
+       // WPQ inserts
+       KNLevsetup(fname, gbl.edc_wr[edc], 0x2, 0x1);
+     }
+   // EDC HitE, HitM, MissE, MissM
+   for (int edc=0; edc < NEDC; ++edc)
+     {
+       ::snprintf(fname, sizeof(fname), "/sys/devices/uncore_edc_uclk_%d", edc);
+       KNLevsetup(fname, gbl.edc_hite[edc], 0x2, 0x1);
+       KNLevsetup(fname, gbl.edc_hitm[edc], 0x2, 0x2);
+       KNLevsetup(fname, gbl.edc_misse[edc], 0x2, 0x4);
+       KNLevsetup(fname, gbl.edc_missm[edc], 0x2, 0x8);
+     }
+ }
+
+uint64_t PmuStat::KNLreadctr(int fd)
+{
+  uint64_t data;
+  size_t s = ::read(fd, &data, sizeof(data));
+  if (s != sizeof(uint64_t)){
+    ::printf("read counter %lu", s);
+    ::exit(0);
+  }
+  return data;
+}
+
+void PmuStat::KNLreadctrs(ctrs &c)
+{
+  for (int i = 0; i < NMC; ++i)
+    {
+      c.mcrd[i] = KNLreadctr(gbl.mc_rd[i]);
+      c.mcwr[i] = KNLreadctr(gbl.mc_wr[i]);
+    }
+  for (int i = 0; i < NEDC; ++i)
+    {
+      c.edcrd[i] = KNLreadctr(gbl.edc_rd[i]);
+      c.edcwr[i] = KNLreadctr(gbl.edc_wr[i]);
+    }
+  for (int i = 0; i < NEDC; ++i)
+    {
+      c.edchite[i] = KNLreadctr(gbl.edc_hite[i]);
+      c.edchitm[i] = KNLreadctr(gbl.edc_hitm[i]);
+      c.edcmisse[i] = KNLreadctr(gbl.edc_misse[i]);
+      c.edcmissm[i] = KNLreadctr(gbl.edc_missm[i]);
+    }
+}
+
+#endif
+}
diff --git a/lib/Stat.h b/lib/Stat.h
new file mode 100644
index 00000000..e93e1538
--- /dev/null
+++ b/lib/Stat.h
@@ -0,0 +1,100 @@
+#ifndef _GRID_STAT_H
+#define _GRID_STAT_H
+
+#ifdef AVX512
+#define _KNIGHTS_LANDING_
+#endif
+
+#ifdef _KNIGHTS_LANDING_
+
+#define NMC 6
+#define NEDC 8
+namespace Grid { 
+struct ctrs
+{
+    uint64_t mcrd[NMC];
+    uint64_t mcwr[NMC];
+    uint64_t edcrd[NEDC]; 
+    uint64_t edcwr[NEDC];
+    uint64_t edchite[NEDC];
+    uint64_t edchitm[NEDC];
+    uint64_t edcmisse[NEDC];
+    uint64_t edcmissm[NEDC];
+};
+// Peter/Azusa:
+// Our modification of a code provided by Larry Meadows from Intel
+// Verified by email exchange non-NDA, ok for github. Should be as uses /sys/devices/ FS
+// so is already public and in the linux kernel for KNL.
+struct knl_gbl_
+{
+  int mc_rd[NMC];
+  int mc_wr[NMC];
+  int edc_rd[NEDC];
+  int edc_wr[NEDC];
+  int edc_hite[NEDC];
+  int edc_hitm[NEDC];
+  int edc_misse[NEDC];
+  int edc_missm[NEDC];
+};
+#endif
+
+class PmuStat
+{
+    const char *name;
+    __declspec(align(64)) uint64_t counters[8][256];
+#ifdef _KNIGHTS_LANDING_
+    static struct knl_gbl_ gbl;
+#endif
+
+    uint64_t reads;     // memory reads
+    uint64_t writes;    // memory writes
+    uint64_t mrstart;   // memory read counter at start of parallel region
+    uint64_t mrend;     // memory read counter at end of parallel region
+    uint64_t mwstart;   // memory write counter at start of parallel region
+    uint64_t mwend;     // memory write counter at end of parallel region
+
+    // cumulative counters
+    uint64_t count;     // number of invocations
+    uint64_t tregion;   // total time in parallel region (from thread 0)
+    uint64_t tcycles;   // total cycles inside parallel region
+    uint64_t inst, ref, cyc;   // fixed counters
+    uint64_t pmc0, pmc1;// pmu
+    // add memory counters here
+    // temp variables
+    uint64_t tstart;    // tsc at start of parallel region
+    uint64_t tend;      // tsc at end of parallel region
+    // map for ctrs values
+    // 0 pmc0 start
+    // 1 pmc0 end
+    // 2 pmc1 start
+    // 3 pmc1 end
+    // 4 tsc start
+    // 5 tsc end
+    static bool pmu_initialized;
+public:
+    static bool is_init(void){ return pmu_initialized;}
+    static void pmu_init(void);
+    static void pmu_fini(void);
+    static void pmu_start(void);
+    static void pmu_stop(void);
+    void accum(int nthreads);
+    static void xmemctrs(uint64_t *mr, uint64_t *mw);
+    void start(void);
+    void enter(int t);
+    void exit(int t);
+    void print(void);
+    void init(const char *regname);
+    void clear(void);
+#ifdef _KNIGHTS_LANDING_
+    static void     KNLsetup(void);
+    static uint64_t KNLreadctr(int fd);
+    static void     KNLreadctrs(ctrs &c);
+    static void     KNLevsetup(const char *ename, int &fd, int event, int umask);
+#endif
+    
+  };
+
+}
+
+#endif
+
diff --git a/lib/Threads.h b/lib/Threads.h
index 9d1295e5..d502dd82 100644
--- a/lib/Threads.h
+++ b/lib/Threads.h
@@ -37,7 +37,11 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #ifdef GRID_OMP
 #include <omp.h>
-#define PARALLEL_FOR_LOOP _Pragma("omp parallel for ")
+#ifdef GRID_NUMA
+#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(static)")
+#else
+#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(runtime)")
+#endif
 #define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)")
 #else
 #define PARALLEL_FOR_LOOP 
diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h
index 3c10403f..db0508d5 100644
--- a/lib/cartesian/Cartesian_red_black.h
+++ b/lib/cartesian/Cartesian_red_black.h
@@ -178,7 +178,7 @@ public:
 	// all elements of a simd vector must have same checkerboard.
 	// If Ls vectorised, this must still be the case; e.g. dwf rb5d
 	if ( _simd_layout[d]>1 ) {
-	  if ( d != _checker_dim ) { 
+	  if ( checker_dim_mask[d] ) { 
 	    assert( (_rdimensions[d]&0x1) == 0 );
 	  }
 	}
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index 3ced3443..a96b6fca 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -416,6 +416,28 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
       Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in,
                                    out);
     }
+#ifdef AVX512
+  } else if (stat.is_init() ) {
+
+    int nthreads;
+    stat.start();
+    #pragma omp parallel
+    {
+    #pragma omp master
+    nthreads = omp_get_num_threads();
+    int mythread = omp_get_thread_num();
+    stat.enter(mythread);
+    #pragma omp for nowait
+   for(int ss=0;ss<U._grid->oSites();ss++)
+    {
+       int sU=ss;
+       int sF=LLs*sU;
+       Kernels::DiracOptDhopSite(st,lo,U,st.comm_buf,sF,sU,LLs,1,in,out);
+     }
+    stat.exit(mythread);
+    }
+    stat.accum(nthreads);
+#endif
   } else {
     PARALLEL_FOR_LOOP
     for (int ss = 0; ss < U._grid->oSites(); ss++) {
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.h b/lib/qcd/action/fermion/WilsonFermion5D.h
index b9c35b7c..bc7ec543 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.h
+++ b/lib/qcd/action/fermion/WilsonFermion5D.h
@@ -31,6 +31,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifndef  GRID_QCD_WILSON_FERMION_5D_H
 #define  GRID_QCD_WILSON_FERMION_5D_H
 
+#include <Grid/Stat.h>
+
 namespace Grid {
 
   namespace QCD {
@@ -60,6 +62,7 @@ namespace Grid {
     public:
      INHERIT_IMPL_TYPES(Impl);
      typedef WilsonKernels<Impl> Kernels;
+     PmuStat stat;
 
      void Report(void);
      void ZeroCounters(void);
diff --git a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h
index d236a774..12579d8c 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h
+++ b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h
@@ -134,7 +134,9 @@
   ////////////////////////////////
   // Xm
   ////////////////////////////////
+#ifndef STREAM_STORE
   basep= (uint64_t) &out._odata[ss];
+#endif
   //  basep= st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
@@ -229,7 +231,9 @@
     LOAD_CHI(base);
   }
   base= (uint64_t) &out._odata[ss];
+#ifndef STREAM_STORE
   PREFETCH_CHIMU(base);
+#endif
   {
     MULT_2SPIN_DIR_PFTM(Tm,basep);
   }
diff --git a/lib/simd/Intel512common.h b/lib/simd/Intel512common.h
index dabbf6d8..dbfb30c2 100644
--- a/lib/simd/Intel512common.h
+++ b/lib/simd/Intel512common.h
@@ -138,9 +138,14 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define ZLOADf(OFF,PTR,ri,ir)  VLOADf(OFF,PTR,ir)  VSHUFf(ir,ri)
 #define ZLOADd(OFF,PTR,ri,ir)  VLOADd(OFF,PTR,ir)  VSHUFd(ir,ri)
 
-
+#define STREAM_STORE
+#ifdef STREAM_STORE
+#define VSTOREf(OFF,PTR,SRC)   "vmovntps " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
+#define VSTOREd(OFF,PTR,SRC)   "vmovntpd " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
+#else
 #define VSTOREf(OFF,PTR,SRC)   "vmovaps " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
 #define VSTOREd(OFF,PTR,SRC)   "vmovapd " #SRC "," #OFF "*64(" #PTR ")"  ";\n"
+#endif
 
 // Swaps Re/Im ; could unify this with IMCI
 #define VSHUFd(A,DEST)         "vpshufd  $0x4e," #A "," #DEST  ";\n"    

From 9b63e97108f6716795a1f63cc43b5a813dbb3e3e Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 12 Oct 2016 11:51:21 +0100
Subject: [PATCH 050/149] align not absolutely required and confuses clang++

---
 lib/Stat.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Stat.h b/lib/Stat.h
index e93e1538..f3c0722e 100644
--- a/lib/Stat.h
+++ b/lib/Stat.h
@@ -40,11 +40,11 @@ struct knl_gbl_
 
 class PmuStat
 {
-    const char *name;
-    __declspec(align(64)) uint64_t counters[8][256];
+    uint64_t counters[8][256];
 #ifdef _KNIGHTS_LANDING_
     static struct knl_gbl_ gbl;
 #endif
+    const char *name;
 
     uint64_t reads;     // memory reads
     uint64_t writes;    // memory writes

From 496beffa883ca419e5b20c9a04ea40c85655a5be Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 12 Oct 2016 12:06:08 +0100
Subject: [PATCH 051/149] Fix non-KNL build

---
 lib/Stat.h | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/Stat.h b/lib/Stat.h
index f3c0722e..553c4f8a 100644
--- a/lib/Stat.h
+++ b/lib/Stat.h
@@ -5,11 +5,14 @@
 #define _KNIGHTS_LANDING_
 #endif
 
-#ifdef _KNIGHTS_LANDING_
+namespace Grid { 
 
+///////////////////////////////////////////////////////////////////////////////
+// Extra KNL counters from MCDRAM
+///////////////////////////////////////////////////////////////////////////////
+#ifdef _KNIGHTS_LANDING_
 #define NMC 6
 #define NEDC 8
-namespace Grid { 
 struct ctrs
 {
     uint64_t mcrd[NMC];
@@ -37,6 +40,7 @@ struct knl_gbl_
   int edc_missm[NEDC];
 };
 #endif
+///////////////////////////////////////////////////////////////////////////////
 
 class PmuStat
 {
@@ -95,6 +99,6 @@ public:
   };
 
 }
-
 #endif
 
+

From bd205a3293535f7be117ce04ca61a6e80223f174 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 12 Oct 2016 12:09:15 +0100
Subject: [PATCH 052/149] Fixing for non x86 and non KNL

---
 lib/Stat.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/lib/Stat.cc b/lib/Stat.cc
index 9a025225..ed568517 100644
--- a/lib/Stat.cc
+++ b/lib/Stat.cc
@@ -11,6 +11,7 @@ bool PmuStat::pmu_initialized=false;
 
 void PmuStat::init(const char *regname)
 {
+#ifdef __x86_64__
   name = regname;
   if (!pmu_initialized)
     {
@@ -19,9 +20,11 @@ void PmuStat::init(const char *regname)
       pmu_init();
     }
   clear();
+#endif
 }
 void PmuStat::clear(void)
 {
+#ifdef __x86_64__
   count = 0;
   tregion = 0;
   pmc0 = 0;
@@ -32,9 +35,11 @@ void PmuStat::clear(void)
   tcycles = 0;
   reads = 0;
   writes = 0;
+#endif
 }
 void PmuStat::print(void)
 {
+#ifdef __x86_64__
   std::cout <<"Reg "<<std::string(name)<<":\n";
   std::cout <<"  region "<<tregion<<std::endl;
   std::cout <<"  cycles "<<tcycles<<std::endl;
@@ -46,34 +51,42 @@ void PmuStat::print(void)
   std::cout <<"  count  "<<count  <<std::endl;
   std::cout <<"  reads  "<<reads  <<std::endl;
   std::cout <<"  writes "<<writes <<std::endl;
+#endif
 }
 void PmuStat::start(void)
 {
+#ifdef __x86_64__
   pmu_start();
   ++count;
   xmemctrs(&mrstart, &mwstart);
   tstart = _rdtsc();
+#endif
 }
 void PmuStat::enter(int t)
 {
+#ifdef __x86_64__
   counters[0][t] = _rdpmc(0);
   counters[1][t] = _rdpmc(1);
   counters[2][t] = _rdpmc((1<<30)|0);
   counters[3][t] = _rdpmc((1<<30)|1);
   counters[4][t] = _rdpmc((1<<30)|2);
   counters[5][t] = _rdtsc();
+#endif
 }
 void PmuStat::exit(int t)
 {
+#ifdef __x86_64__
   counters[0][t] = _rdpmc(0) - counters[0][t];
   counters[1][t] = _rdpmc(1) - counters[1][t];
   counters[2][t] = _rdpmc((1<<30)|0) - counters[2][t];
   counters[3][t] = _rdpmc((1<<30)|1) - counters[3][t];
   counters[4][t] = _rdpmc((1<<30)|2) - counters[4][t];
   counters[5][t] = _rdtsc() - counters[5][t];
+#endif
 }
 void PmuStat::accum(int nthreads)
 {
+#ifdef __x86_64__
   tend = _rdtsc();
   xmemctrs(&mrend, &mwend);
   pmu_stop();
@@ -91,6 +104,7 @@ void PmuStat::accum(int nthreads)
   reads += mreads;
   uint64_t mwrites = mwend - mwstart;
   writes += mwrites;
+#endif
 }
 
 

From 6472b431f06079e3da8f50ea09d74e808d3fa661 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 12 Oct 2016 12:29:08 +0100
Subject: [PATCH 053/149] __rdpmc needed for gcc, clang++

---
 lib/Stat.cc | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/lib/Stat.cc b/lib/Stat.cc
index ed568517..7f2e4086 100644
--- a/lib/Stat.cc
+++ b/lib/Stat.cc
@@ -59,35 +59,35 @@ void PmuStat::start(void)
   pmu_start();
   ++count;
   xmemctrs(&mrstart, &mwstart);
-  tstart = _rdtsc();
+  tstart = __rdtsc();
 #endif
 }
 void PmuStat::enter(int t)
 {
 #ifdef __x86_64__
-  counters[0][t] = _rdpmc(0);
-  counters[1][t] = _rdpmc(1);
-  counters[2][t] = _rdpmc((1<<30)|0);
-  counters[3][t] = _rdpmc((1<<30)|1);
-  counters[4][t] = _rdpmc((1<<30)|2);
-  counters[5][t] = _rdtsc();
+  counters[0][t] = __rdpmc(0);
+  counters[1][t] = __rdpmc(1);
+  counters[2][t] = __rdpmc((1<<30)|0);
+  counters[3][t] = __rdpmc((1<<30)|1);
+  counters[4][t] = __rdpmc((1<<30)|2);
+  counters[5][t] = __rdtsc();
 #endif
 }
 void PmuStat::exit(int t)
 {
 #ifdef __x86_64__
-  counters[0][t] = _rdpmc(0) - counters[0][t];
-  counters[1][t] = _rdpmc(1) - counters[1][t];
-  counters[2][t] = _rdpmc((1<<30)|0) - counters[2][t];
-  counters[3][t] = _rdpmc((1<<30)|1) - counters[3][t];
-  counters[4][t] = _rdpmc((1<<30)|2) - counters[4][t];
-  counters[5][t] = _rdtsc() - counters[5][t];
+  counters[0][t] = __rdpmc(0) - counters[0][t];
+  counters[1][t] = __rdpmc(1) - counters[1][t];
+  counters[2][t] = __rdpmc((1<<30)|0) - counters[2][t];
+  counters[3][t] = __rdpmc((1<<30)|1) - counters[3][t];
+  counters[4][t] = __rdpmc((1<<30)|2) - counters[4][t];
+  counters[5][t] = __rdtsc() - counters[5][t];
 #endif
 }
 void PmuStat::accum(int nthreads)
 {
 #ifdef __x86_64__
-  tend = _rdtsc();
+  tend = __rdtsc();
   xmemctrs(&mrend, &mwend);
   pmu_stop();
   for (int t = 0; t < nthreads; ++t) {

From 8bbd9ebc270acb0aa277187d743644a0a5d818b0 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 12 Oct 2016 13:47:20 +0100
Subject: [PATCH 054/149] Reversing changes to Stencil class

---
 lib/Stat.h    | 2 +-
 lib/Stencil.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/Stat.h b/lib/Stat.h
index 553c4f8a..96bd594a 100644
--- a/lib/Stat.h
+++ b/lib/Stat.h
@@ -2,7 +2,7 @@
 #define _GRID_STAT_H
 
 #ifdef AVX512
-#define _KNIGHTS_LANDING_
+#define _KNIGHTS_LANDING_ROOTONLY
 #endif
 
 namespace Grid { 
diff --git a/lib/Stencil.h b/lib/Stencil.h
index 83aace54..72c55d0a 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -265,7 +265,7 @@
        }
        inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
 	 //_mm_prefetch((char *)&_entries[ent+1],_MM_HINT_T0);
-	 uint64_t cbase = (uint64_t) & comm_buf[0];
+	 uint64_t cbase = (uint64_t) 0;
 	 local = _entries[ent]._is_local;
 	 perm  = _entries[ent]._permute;
 	 if (perm)  ptype = _permute_type[point]; 
@@ -273,7 +273,7 @@
 	 else       return cbase + _entries[ent]._byte_offset;
        }
        inline uint64_t GetPFInfo(int ent,uint64_t base) {
-	 uint64_t cbase =  (uint64_t)& comm_buf[0];
+	 uint64_t cbase =  (uint64_t) 0;
 	 int local = _entries[ent]._is_local;
 	 if (local) return  base + _entries[ent]._byte_offset;
 	 else       return cbase + _entries[ent]._byte_offset;

From 8b0d171c9af6e0569c7df8003f445b4b00dc9fdf Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 12 Oct 2016 17:49:32 +0100
Subject: [PATCH 055/149] 32bit issue on the KNL code variant where byte
 offsets were stored

---
 lib/Init.cc   |  6 +++---
 lib/Stencil.h | 19 +++++++++++--------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/lib/Init.cc b/lib/Init.cc
index 34e503a4..bd9638c3 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -246,15 +246,15 @@ void Grid_init(int *argc,char ***argv)
   std::cout <<std::endl;
   std::cout <<COL_RED  << "__|__|__|__|__"<<             "|__|__|_"<<COL_PURPLE<<"_|__|__|"<<                "__|__|__|__|__"<<std::endl; 
   std::cout <<COL_RED  << "__|__|__|__|__"<<             "|__|__|_"<<COL_PURPLE<<"_|__|__|"<<                "__|__|__|__|__"<<std::endl; 
-  std::cout <<COL_RED  << "__|__|  |  |  "<<             "|  |  | "<<COL_PURPLE<<" |  |  |"<<                "  |  |  | _|__"<<std::endl; 
-  std::cout <<COL_RED  << "__|__         "<<             "        "<<COL_PURPLE<<"        "<<                "          _|__"<<std::endl; 
+  std::cout <<COL_RED  << "__|_ |  |  |  "<<             "|  |  | "<<COL_PURPLE<<" |  |  |"<<                "  |  |  | _|__"<<std::endl; 
+  std::cout <<COL_RED  << "__|_          "<<             "        "<<COL_PURPLE<<"        "<<                "          _|__"<<std::endl; 
   std::cout <<COL_RED  << "__|_  "<<COL_GREEN<<" GGGG   "<<COL_RED<<" RRRR   "<<COL_BLUE  <<" III    "<<COL_PURPLE<<"DDDD  "<<COL_PURPLE<<"    _|__"<<std::endl;
   std::cout <<COL_RED  << "__|_  "<<COL_GREEN<<"G       "<<COL_RED<<" R   R  "<<COL_BLUE  <<"  I     "<<COL_PURPLE<<"D   D "<<COL_PURPLE<<"    _|__"<<std::endl;
   std::cout <<COL_RED  << "__|_  "<<COL_GREEN<<"G       "<<COL_RED<<" R   R  "<<COL_BLUE  <<"  I     "<<COL_PURPLE<<"D    D"<<COL_PURPLE<<"    _|__"<<std::endl;
   std::cout <<COL_BLUE << "__|_  "<<COL_GREEN<<"G  GG   "<<COL_RED<<" RRRR   "<<COL_BLUE  <<"  I     "<<COL_PURPLE<<"D    D"<<COL_GREEN <<"    _|__"<<std::endl;
   std::cout <<COL_BLUE << "__|_  "<<COL_GREEN<<"G   G   "<<COL_RED<<" R  R   "<<COL_BLUE  <<"  I     "<<COL_PURPLE<<"D   D "<<COL_GREEN <<"    _|__"<<std::endl;
   std::cout <<COL_BLUE << "__|_  "<<COL_GREEN<<" GGGG   "<<COL_RED<<" R   R  "<<COL_BLUE  <<" III    "<<COL_PURPLE<<"DDDD  "<<COL_GREEN <<"    _|__"<<std::endl;
-  std::cout <<COL_BLUE << "__|__         "<<             "        "<<COL_GREEN <<"        "<<                "          _|__"<<std::endl; 
+  std::cout <<COL_BLUE << "__|_          "<<             "        "<<COL_GREEN <<"        "<<                "          _|__"<<std::endl; 
   std::cout <<COL_BLUE << "__|__|__|__|__"<<             "|__|__|_"<<COL_GREEN <<"_|__|__|"<<                "__|__|__|__|__"<<std::endl; 
   std::cout <<COL_BLUE << "__|__|__|__|__"<<             "|__|__|_"<<COL_GREEN <<"_|__|__|"<<                "__|__|__|__|__"<<std::endl; 
   std::cout <<COL_BLUE << "  |  |  |  |  "<<             "|  |  | "<<COL_GREEN <<" |  |  |"<<                "  |  |  |  |  "<<std::endl; 
diff --git a/lib/Stencil.h b/lib/Stencil.h
index 72c55d0a..8282ba90 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -71,8 +71,8 @@
  namespace Grid {
 
    struct StencilEntry { 
-     uint32_t _offset;
-     uint32_t _byte_offset;
+     uint64_t _offset;
+     uint64_t _byte_offset;
      uint16_t _is_local;
      uint16_t _permute;
      uint32_t _around_the_world; //256 bits, 32 bytes, 1/2 cacheline
@@ -255,7 +255,8 @@
 	   if( _entries[i]._is_local ) {
 	     _entries[i]._byte_offset = _entries[i]._offset*sizeof(vobj);
 	   } else { 
-	     _entries[i]._byte_offset =(uint64_t)&comm_buf[0]+ _entries[i]._offset*sizeof(cobj);
+	     // PrecomputeByteOffsets [5] 16384/32768 140735768678528 140735781261056 2581581952
+	     _entries[i]._byte_offset = _entries[i]._offset*sizeof(cobj);
 	   }
 	 }
        };
@@ -264,16 +265,18 @@
 	 //	 _mm_prefetch((char *)&_entries[ent],_MM_HINT_T0);
        }
        inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
-	 //_mm_prefetch((char *)&_entries[ent+1],_MM_HINT_T0);
-	 uint64_t cbase = (uint64_t) 0;
+	 uint64_t cbase = (uint64_t)&comm_buf[0];
 	 local = _entries[ent]._is_local;
 	 perm  = _entries[ent]._permute;
 	 if (perm)  ptype = _permute_type[point]; 
-	 if (local) return  base + _entries[ent]._byte_offset;
-	 else       return cbase + _entries[ent]._byte_offset;
+	 if (local) {
+	   return  base + _entries[ent]._byte_offset;
+	 } else {
+	   return cbase + _entries[ent]._byte_offset;
+	 }
        }
        inline uint64_t GetPFInfo(int ent,uint64_t base) {
-	 uint64_t cbase =  (uint64_t) 0;
+	 uint64_t cbase = (uint64_t)&comm_buf[0];
 	 int local = _entries[ent]._is_local;
 	 if (local) return  base + _entries[ent]._byte_offset;
 	 else       return cbase + _entries[ent]._byte_offset;

From c8079e6621bbdb048e45910dbd9d44faf324c886 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 13 Oct 2016 22:28:50 +0100
Subject: [PATCH 056/149] Time the face gateher in x-dir more carefully

---
 lib/Stencil.h | 66 ++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 63 insertions(+), 3 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index 8282ba90..a47aa3ef 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -70,6 +70,60 @@
 
  namespace Grid {
 
+template<class vobj,class cobj,class compressor> void 
+Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0,
+			     double &t_table ,double & t_data )
+{
+  int rd = rhs._grid->_rdimensions[dimension];
+
+  if ( !rhs._grid->CheckerBoarded(dimension) ) {
+    cbmask = 0x3;
+  }
+  
+  int so  = plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
+  
+  int e1=rhs._grid->_slice_nblock[dimension];
+  int e2=rhs._grid->_slice_block[dimension];
+
+  t_table = 0.0;
+  t_data  = 0.0;
+
+  int stride=rhs._grid->_slice_stride[dimension];
+  if ( cbmask == 0x3 ) { 
+  t_data-=usecond();
+PARALLEL_NESTED_LOOP2
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int o  = n*stride;
+	int bo = n*e2;
+	buffer[off+bo+b]=compress(rhs._odata[so+o+b]);
+      }
+    }
+  t_data+=usecond();
+  } else { 
+     int bo=0;
+     t_table-=usecond();
+     std::vector<std::pair<int,int> > table;
+     for(int n=0;n<e1;n++){
+       for(int b=0;b<e2;b++){
+	 int o  = n*stride;
+	 int ocb=1<<rhs._grid->CheckerBoardFromOindexTable(o+b);
+	 if ( ocb &cbmask ) {
+	   table.push_back(std::pair<int,int> (bo++,o+b));
+	 }
+       }
+     }
+     t_table+=usecond();
+     t_data-=usecond();
+PARALLEL_FOR_LOOP     
+     for(int i=0;i<table.size();i++){
+       buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
+     }
+     t_data+=usecond();
+  }
+}
+
+
    struct StencilEntry { 
      uint64_t _offset;
      uint64_t _byte_offset;
@@ -102,8 +156,8 @@
 
        std::vector<Packet> Packets;
        
- #define SEND_IMMEDIATE
- #define SERIAL_SENDS
+#define SEND_IMMEDIATE
+#define SERIAL_SENDS
 
        void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){
  #ifdef SEND_IMMEDIATE
@@ -305,6 +359,8 @@
        double gathermtime;
        double splicetime;
        double nosplicetime;
+       double t_data;
+       double t_table;
        double calls;
 
        void ZeroCounters(void) {
@@ -317,6 +373,8 @@
          gathermtime = 0.;
          splicetime = 0.;
          nosplicetime = 0.;
+	 t_data = 0.0;
+         t_table= 0.0;
          comms_bytes = 0.;
          calls = 0.;
        };
@@ -336,6 +394,8 @@
        PRINTIT(gathermtime);
        PRINTIT(splicetime);
        PRINTIT(nosplicetime);
+       PRINTIT(t_table);
+       PRINTIT(t_data);
 	 }
        };
  #endif
@@ -783,7 +843,7 @@
 	       int bytes = words * sizeof(cobj);
 
 	       gathertime-=usecond();
-	       Gather_plane_simple (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset);
+	       Gather_plane_simple_stencil (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,t_table,t_data);
 	       gathertime+=usecond();
 
 	       int rank           = _grid->_processor;

From bc1a4d40ba71b9f4c474127d79e72e169fc0762a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 17 Oct 2016 16:16:44 +0100
Subject: [PATCH 057/149] Faster integer handling avoid push_back

---
 lib/Stencil.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index a47aa3ef..910d16c5 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -71,7 +71,7 @@
  namespace Grid {
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0,
+Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
 			     double &t_table ,double & t_data )
 {
   int rd = rhs._grid->_rdimensions[dimension];
@@ -103,20 +103,20 @@ PARALLEL_NESTED_LOOP2
   } else { 
      int bo=0;
      t_table-=usecond();
-     std::vector<std::pair<int,int> > table;
+     std::vector<std::pair<int,int> > table(e1*e2);
      for(int n=0;n<e1;n++){
        for(int b=0;b<e2;b++){
 	 int o  = n*stride;
 	 int ocb=1<<rhs._grid->CheckerBoardFromOindexTable(o+b);
 	 if ( ocb &cbmask ) {
-	   table.push_back(std::pair<int,int> (bo++,o+b));
+	   table[bo]=std::pair<int,int>(bo,o+b); bo++;
 	 }
        }
      }
      t_table+=usecond();
      t_data-=usecond();
 PARALLEL_FOR_LOOP     
-     for(int i=0;i<table.size();i++){
+     for(int i=0;i<bo;i++){
        buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
      }
      t_data+=usecond();

From 811ca45473f4fb9a4942057df7bc1e350df6f75c Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 17 Oct 2016 16:23:21 +0100
Subject: [PATCH 058/149] GNU clang hack for AVX512 since there are missing
 reduce intrinsics in Clang 3.9 and GCC-6 AVX512 support

---
 lib/simd/Grid_avx512.h | 62 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 61 insertions(+), 1 deletion(-)

diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h
index b057a61b..821898d9 100644
--- a/lib/simd/Grid_avx512.h
+++ b/lib/simd/Grid_avx512.h
@@ -359,7 +359,67 @@ namespace Optimization {
 
   //////////////////////////////////////////////
   // Some Template specialization
+
+  // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases
+#undef GNU_CLANG_COMPILER 
+#ifdef GNU_CLANG_COMPILER
+  //Complex float Reduce
+  template<>
+    inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){
+    __m512 v1,v2;
+    v1=Optimization::Permute::Permute0(in); // avx 512; quad complex single
+    v1= _mm512_add_ps(v1,in);
+    v2=Optimization::Permute::Permute1(v1); 
+    v1 = _mm512_add_ps(v1,v2);
+    v2=Optimization::Permute::Permute2(v1); 
+    v1 = _mm512_add_ps(v1,v2);
+    u512f conv; conv.v = v1;
+    return Grid::ComplexF(conv.f[0],conv.f[1]);
+  }
   
+  //Real float Reduce
+  template<>
+    inline Grid::RealF Reduce<Grid::RealF, __m512>::operator()(__m512 in){
+    __m512 v1,v2;
+    v1 = Optimization::Permute::Permute0(in); // avx 512; octo-double
+    v1 = _mm512_add_ps(v1,in);
+    v2 = Optimization::Permute::Permute1(v1); 
+    v1 = _mm512_add_ps(v1,v2);
+    v2 = Optimization::Permute::Permute2(v1); 
+    v1 = _mm512_add_ps(v1,v2);
+    v2 = Optimization::Permute::Permute3(v1); 
+    v1 = _mm512_add_ps(v1,v2);
+    u512f conv; conv.v=v1;
+    return conv.f[0];
+  }
+  
+  
+  //Complex double Reduce
+  template<>
+    inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){
+    __m512d v1;
+    v1 = Optimization::Permute::Permute0(in); // sse 128; paired complex single
+    v1 = _mm512_add_pd(v1,in);
+    v1 = Optimization::Permute::Permute1(in); // sse 128; paired complex single
+    v1 = _mm512_add_pd(v1,in);
+    u512d conv; conv.v = v1;
+    return Grid::ComplexD(conv.f[0],conv.f[1]);
+  }
+  
+  //Real double Reduce
+  template<>
+    inline Grid::RealD Reduce<Grid::RealD, __m512d>::operator()(__m512d in){
+    __m512d v1,v2;
+    v1 = Optimization::Permute::Permute0(in); // avx 512; quad double
+    v1 = _mm512_add_pd(v1,in);
+      v2 = Optimization::Permute::Permute1(v1); 
+      v1 = _mm512_add_pd(v1,v2);
+      v2 = Optimization::Permute::Permute2(v1); 
+      v1 = _mm512_add_pd(v1,v2);
+     u512d conv; conv.v = v1;
+     return conv.f[0];
+  }
+#else
   //Complex float Reduce
   template<>
   inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){
@@ -371,7 +431,6 @@ namespace Optimization {
     return _mm512_reduce_add_ps(in);
   }
   
-  
   //Complex double Reduce
   template<>
   inline Grid::ComplexD Reduce<Grid::ComplexD, __m512d>::operator()(__m512d in){
@@ -391,6 +450,7 @@ namespace Optimization {
     printf("Reduce : Missing integer implementation -> FIX\n");
     assert(0);
   }
+#endif
   
   
 }

From 7af9b8731847667eaf3b2e33a2457b977a7254ae Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 18 Oct 2016 09:51:37 +0100
Subject: [PATCH 059/149] Cache face tables to improve performance. Extract
 merge now looking poor.

---
 lib/Stencil.h | 118 +++++++++++++++++++++++++++++++-------------------
 1 file changed, 73 insertions(+), 45 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index 910d16c5..db608fe7 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -71,39 +71,31 @@
  namespace Grid {
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
-			     double &t_table ,double & t_data )
+Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,std::vector<std::pair<int,int> >& table)
 {
+  table.resize(0);
   int rd = rhs._grid->_rdimensions[dimension];
 
   if ( !rhs._grid->CheckerBoarded(dimension) ) {
     cbmask = 0x3;
   }
-  
-  int so  = plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
-  
+  int so= plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
   int e1=rhs._grid->_slice_nblock[dimension];
   int e2=rhs._grid->_slice_block[dimension];
 
-  t_table = 0.0;
-  t_data  = 0.0;
-
   int stride=rhs._grid->_slice_stride[dimension];
   if ( cbmask == 0x3 ) { 
-  t_data-=usecond();
-PARALLEL_NESTED_LOOP2
+    table.resize(e1*e2);
     for(int n=0;n<e1;n++){
       for(int b=0;b<e2;b++){
 	int o  = n*stride;
 	int bo = n*e2;
-	buffer[off+bo+b]=compress(rhs._odata[so+o+b]);
+	table[bo+b]=std::pair<int,int>(bo+b,o+b);
       }
     }
-  t_data+=usecond();
   } else { 
      int bo=0;
-     t_table-=usecond();
-     std::vector<std::pair<int,int> > table(e1*e2);
+     table.resize(e1*e2/2);
      for(int n=0;n<e1;n++){
        for(int b=0;b<e2;b++){
 	 int o  = n*stride;
@@ -113,16 +105,31 @@ PARALLEL_NESTED_LOOP2
 	 }
        }
      }
-     t_table+=usecond();
-     t_data-=usecond();
-PARALLEL_FOR_LOOP     
-     for(int i=0;i<bo;i++){
-       buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
-     }
-     t_data+=usecond();
   }
 }
 
+template<class vobj,class cobj,class compressor> void 
+Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,
+			   compressor &compress, int off,int so)
+{
+PARALLEL_FOR_LOOP     
+     for(int i=0;i<table.size();i++){
+       buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
+     }
+}
+
+template<class vobj,class cobj,class compressor> void 
+Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
+			     double &t_table ,double & t_data )
+{
+  std::vector<std::pair<int,int> > table;
+  Gather_plane_simple_table_compute (rhs, buffer,dimension,plane,cbmask,compress,off,table);
+  int so  = plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
+  Gather_plane_simple_table         (table,rhs,buffer,compress,off,so);
+}
+
+
+
 
    struct StencilEntry { 
      uint64_t _offset;
@@ -155,6 +162,9 @@ PARALLEL_FOR_LOOP
        };
 
        std::vector<Packet> Packets;
+
+       int face_table_computed;
+       std::vector<std::vector<std::pair<int,int> > > face_table ;
        
 #define SEND_IMMEDIATE
 #define SERIAL_SENDS
@@ -383,19 +393,22 @@ PARALLEL_FOR_LOOP
 #define PRINTIT(A)	\
  std::cout << GridLogMessage << " Stencil " << #A << " "<< A/calls<<std::endl;
 	 if ( calls > 0. ) {
- std::cout << GridLogMessage << " Stencil calls "<<calls<<std::endl;
-       PRINTIT(jointime);
-       PRINTIT(gathertime);
-       PRINTIT(commtime);
-       PRINTIT(halogtime);
-       PRINTIT(mergetime);
-       PRINTIT(spintime);
-       PRINTIT(comms_bytes);
-       PRINTIT(gathermtime);
-       PRINTIT(splicetime);
-       PRINTIT(nosplicetime);
-       PRINTIT(t_table);
-       PRINTIT(t_data);
+	   std::cout << GridLogMessage << " Stencil calls "<<calls<<std::endl;
+	   PRINTIT(halogtime);
+	   PRINTIT(gathertime);
+	   PRINTIT(gathermtime);
+	   PRINTIT(mergetime);
+	   if(comms_bytes>1.0){
+	     PRINTIT(comms_bytes);
+	     PRINTIT(commtime);
+	     std::cout << GridLogMessage << " Stencil " << comms_bytes/commtime/1000. << " GB/s "<<std::endl;
+	   }
+	   PRINTIT(jointime);
+	   PRINTIT(spintime);
+	   PRINTIT(splicetime);
+	   PRINTIT(nosplicetime);
+	   PRINTIT(t_table);
+	   PRINTIT(t_data);
 	 }
        };
  #endif
@@ -407,6 +420,7 @@ PARALLEL_FOR_LOOP
 				      const std::vector<int> &distances) 
      :   _permute_type(npoints), _comm_buf_size(npoints)
      {
+       face_table_computed=0;
        _npoints = npoints;
        _grid    = grid;
        _directions = directions;
@@ -734,7 +748,7 @@ PARALLEL_FOR_LOOP
        }
 #endif
        template<class compressor>
-       void HaloGatherDir(const Lattice<vobj> &source,compressor &compress,int point)
+       void HaloGatherDir(const Lattice<vobj> &source,compressor &compress,int point,int & face_idx)
        {
 	   int dimension    = _directions[point];
 	   int displacement = _distances[point];
@@ -762,23 +776,23 @@ PARALLEL_FOR_LOOP
 	     if ( sshift[0] == sshift[1] ) {
 	       if (splice_dim) {
 		 splicetime-=usecond();
-		 GatherSimd(source,dimension,shift,0x3,compress);
+		 GatherSimd(source,dimension,shift,0x3,compress,face_idx);
 		 splicetime+=usecond();
 	       } else { 
 		 nosplicetime-=usecond();
-		 Gather(source,dimension,shift,0x3,compress);
+		 Gather(source,dimension,shift,0x3,compress,face_idx);
 		 nosplicetime+=usecond();
 	       }
 	     } else {
 	       if(splice_dim){
 		 splicetime-=usecond();
-		 GatherSimd(source,dimension,shift,0x1,compress);// if checkerboard is unfavourable take two passes
-		 GatherSimd(source,dimension,shift,0x2,compress);// both with block stride loop iteration
+		 GatherSimd(source,dimension,shift,0x1,compress,face_idx);// if checkerboard is unfavourable take two passes
+		 GatherSimd(source,dimension,shift,0x2,compress,face_idx);// both with block stride loop iteration
 		 splicetime+=usecond();
 	       } else {
 		 nosplicetime-=usecond();
-		 Gather(source,dimension,shift,0x1,compress);
-		 Gather(source,dimension,shift,0x2,compress);
+		 Gather(source,dimension,shift,0x1,compress,face_idx);
+		 Gather(source,dimension,shift,0x2,compress,face_idx);
 		 nosplicetime+=usecond();
 	       }
 	     }
@@ -796,17 +810,19 @@ PARALLEL_FOR_LOOP
 	 u_comm_offset=0;
 
 	 // Gather all comms buffers
+	 int face_idx=0;
 	 for(int point = 0 ; point < _npoints; point++) {
 	   compress.Point(point);
-	   HaloGatherDir(source,compress,point);
+	   HaloGatherDir(source,compress,point,face_idx);
 	 }
+	 face_table_computed=1;
 
 	 assert(u_comm_offset==_unified_buffer_size);
 	 halogtime+=usecond();
        }
 
        template<class compressor>
-	 void Gather(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor & compress)
+	 void Gather(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor & compress,int &face_idx)
 	 {
 	   typedef typename cobj::vector_type vector_type;
 	   typedef typename cobj::scalar_type scalar_type;
@@ -843,8 +859,20 @@ PARALLEL_FOR_LOOP
 	       int bytes = words * sizeof(cobj);
 
 	       gathertime-=usecond();
-	       Gather_plane_simple_stencil (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,t_table,t_data);
+	       int so  = sx*rhs._grid->_ostride[dimension]; // base offset for start of plane 
+	       if ( !face_table_computed ) {
+		 t_table-=usecond();
+		 face_table.resize(face_idx+1);
+		 Gather_plane_simple_table_compute (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,face_table[face_idx]);
+		 t_table+=usecond();
+	       }
+	       t_data-=usecond();
+	       Gather_plane_simple_table         (face_table[face_idx],rhs,u_send_buf,compress,u_comm_offset,so);
+	       face_idx++;
+	       t_data+=usecond();
 	       gathertime+=usecond();
+	       
+	       //	       Gather_plane_simple_stencil (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,t_table,t_data);
 
 	       int rank           = _grid->_processor;
 	       int recv_from_rank;
@@ -867,7 +895,7 @@ PARALLEL_FOR_LOOP
 
 
        template<class compressor>
-	 void  GatherSimd(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor &compress)
+	 void  GatherSimd(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor &compress,int & face_idx)
 	 {
 	   const int Nsimd = _grid->Nsimd();
 

From 3cbe974eb48eb6ee97cb420b08475beeda5cc71b Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:55:21 +0100
Subject: [PATCH 060/149] Layout

---
 lib/tensors/Tensor_extract_merge.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/tensors/Tensor_extract_merge.h b/lib/tensors/Tensor_extract_merge.h
index 41a431ad..a32d3785 100644
--- a/lib/tensors/Tensor_extract_merge.h
+++ b/lib/tensors/Tensor_extract_merge.h
@@ -250,8 +250,7 @@ void merge(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int
   }
  }
 
-template<class vobj> inline 
-void merge1(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
+template<class vobj> inline void merge1(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
 {
   typedef typename vobj::scalar_type scalar_type ;
   typedef typename vobj::vector_type vector_type ;
@@ -269,8 +268,7 @@ void merge1(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int
   }}
 }
 
-template<class vobj> inline 
-void merge2(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
+template<class vobj> inline void merge2(vobj &vec,std::vector<typename vobj::scalar_object *> &extracted,int offset)
 {
   typedef typename vobj::scalar_type scalar_type ;
   typedef typename vobj::vector_type vector_type ;

From 8c043da5b7dd583c79e0b540277060e6311d27cc Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:56:05 +0100
Subject: [PATCH 061/149] SHMEM and comms allocator made different

---
 lib/AlignedAllocator.h | 130 ++++++++++++++++++++++++++++-------------
 1 file changed, 88 insertions(+), 42 deletions(-)

diff --git a/lib/AlignedAllocator.h b/lib/AlignedAllocator.h
index 8f69971d..d0de1ec3 100644
--- a/lib/AlignedAllocator.h
+++ b/lib/AlignedAllocator.h
@@ -40,14 +40,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <mm_malloc.h>
 #endif
 
-#ifdef GRID_COMMS_SHMEM
-extern "C" { 
-#include <mpp/shmem.h>
-extern void * shmem_align(size_t, size_t);
-extern void  shmem_free(void *);
-}
-#endif
-
 namespace Grid {
 
 ////////////////////////////////////////////////////////////////////
@@ -65,28 +57,81 @@ public:
   typedef _Tp        value_type;
 
   template<typename _Tp1>  struct rebind { typedef alignedAllocator<_Tp1> other; };
-
   alignedAllocator() throw() { }
-
   alignedAllocator(const alignedAllocator&) throw() { }
-
   template<typename _Tp1> alignedAllocator(const alignedAllocator<_Tp1>&) throw() { }
-
   ~alignedAllocator() throw() { }
-
   pointer       address(reference __x)       const { return &__x; }
-  //  const_pointer address(const_reference __x) const { return &__x; }
-
   size_type  max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
 
   pointer allocate(size_type __n, const void* _p= 0)
   { 
+#ifdef HAVE_MM_MALLOC_H
+    _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128);
+#else
+    _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp));
+#endif
+
+    _Tp tmp;
+#ifdef GRID_NUMA
+#pragma omp parallel for schedule(static)
+  for(int i=0;i<__n;i++){
+    ptr[i]=tmp;
+  }
+#endif 
+    return ptr;
+  }
+
+  void deallocate(pointer __p, size_type) { 
+#ifdef HAVE_MM_MALLOC_H
+    _mm_free((void *)__p); 
+#else
+    free((void *)__p);
+#endif
+  }
+  void construct(pointer __p, const _Tp& __val) { };
+  void construct(pointer __p) { };
+  void destroy(pointer __p) { };
+};
+template<typename _Tp>  inline bool operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
+template<typename _Tp>  inline bool operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// MPI3 : comms must use shm region
+// SHMEM: comms must use symmetric heap
+//////////////////////////////////////////////////////////////////////////////////////////
 #ifdef GRID_COMMS_SHMEM
-
-    _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64);
-
-
+extern "C" { 
+#include <mpp/shmem.h>
+extern void * shmem_align(size_t, size_t);
+extern void  shmem_free(void *);
+}
 #define PARANOID_SYMMETRIC_HEAP
+#endif
+
+template<typename _Tp>
+class commAllocator {
+public: 
+  typedef std::size_t     size_type;
+  typedef std::ptrdiff_t  difference_type;
+  typedef _Tp*       pointer;
+  typedef const _Tp* const_pointer;
+  typedef _Tp&       reference;
+  typedef const _Tp& const_reference;
+  typedef _Tp        value_type;
+
+  template<typename _Tp1>  struct rebind { typedef commAllocator<_Tp1> other; };
+  commAllocator() throw() { }
+  commAllocator(const commAllocator&) throw() { }
+  template<typename _Tp1> commAllocator(const commAllocator<_Tp1>&) throw() { }
+  ~commAllocator() throw() { }
+  pointer       address(reference __x)       const { return &__x; }
+  size_type  max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
+
+#ifdef GRID_COMMS_SHMEM
+  pointer allocate(size_type __n, const void* _p= 0)
+  { 
+    _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64);
 #ifdef PARANOID_SYMMETRIC_HEAP
     static void * bcast;
     static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
@@ -99,51 +144,52 @@ public:
       BACKTRACEFILE();
       exit(0);
     }
-
     assert( bcast == (void *) ptr);
-
 #endif 
+    return ptr;
+  }
+  void deallocate(pointer __p, size_type) { 
+    shmem_free((void *)__p);
+  }
+#elif defined(GRID_COMMS_MPI3)
+  pointer allocate(size_type __n, const void* _p= 0)
+  { 
+#error "implement MPI3 windowed allocate"
+    
+  }
+  void deallocate(pointer __p, size_type) { 
+#error "implement MPI3 windowed allocate"
+  }
 #else
-
+  pointer allocate(size_type __n, const void* _p= 0)
 #ifdef HAVE_MM_MALLOC_H
     _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128);
 #else
     _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp));
 #endif
-
-#endif
-    _Tp tmp;
-#ifdef GRID_NUMA
-#pragma omp parallel for schedule(static)
-  for(int i=0;i<__n;i++){
-    ptr[i]=tmp;
-  }
-#endif 
     return ptr;
   }
-
   void deallocate(pointer __p, size_type) { 
-#ifdef GRID_COMMS_SHMEM
-    shmem_free((void *)__p);
-#else
 #ifdef HAVE_MM_MALLOC_H
     _mm_free((void *)__p); 
 #else
     free((void *)__p);
-#endif
 #endif
   }
+#endif
   void construct(pointer __p, const _Tp& __val) { };
   void construct(pointer __p) { };
-
   void destroy(pointer __p) { };
 };
+template<typename _Tp>  inline bool operator==(const commAllocator<_Tp>&, const commAllocator<_Tp>&){ return true; }
+template<typename _Tp>  inline bool operator!=(const commAllocator<_Tp>&, const commAllocator<_Tp>&){ return false; }
 
-template<typename _Tp>  inline bool
-operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
-
-template<typename _Tp>  inline bool
-operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
+////////////////////////////////////////////////////////////////////////////////
+// Template typedefs
+////////////////////////////////////////////////////////////////////////////////
+template<class T> using Vector    = std::vector<T,alignedAllocator<T> >;           
+template<class T> using commVector = std::vector<T,commAllocator<T> >;              
+template<class T> using Matrix = std::vector<std::vector<T,alignedAllocator<T> > >;
     
 }; // namespace Grid
 #endif

From 39f1c880b8172b0354064e4971a27e3cf3aef3d6 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:56:40 +0100
Subject: [PATCH 062/149] mpi3

---
 configure.ac | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/configure.ac b/configure.ac
index 7bcdc49f..d6340a71 100644
--- a/configure.ac
+++ b/configure.ac
@@ -244,6 +244,9 @@ case ${ac_COMMS} in
      mpi)
        AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
      ;;
+     mpi3)
+       AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
+     ;;
      shmem)
        AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
      ;;
@@ -253,6 +256,7 @@ case ${ac_COMMS} in
 esac
 AM_CONDITIONAL(BUILD_COMMS_SHMEM,[ test "X${ac_COMMS}X" == "XshmemX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI,[ test "X${ac_COMMS}X" == "XmpiX" || test "X${ac_COMMS}X" == "Xmpi-autoX" ])
+AM_CONDITIONAL(BUILD_COMMS_MPI3,[ test "X${ac_COMMS}X" == "Xmpi3X"] )
 AM_CONDITIONAL(BUILD_COMMS_NONE,[ test "X${ac_COMMS}X" == "XnoneX" ])
 
 ############### RNG selection

From 4955672fc369baa88c83708838b8aef5502f8399 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:57:00 +0100
Subject: [PATCH 063/149] MPI3

---
 lib/Cshift.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/Cshift.h b/lib/Cshift.h
index bbc56cae..4eeb6cea 100644
--- a/lib/Cshift.h
+++ b/lib/Cshift.h
@@ -38,6 +38,10 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/cshift/Cshift_mpi.h>
 #endif 
 
+#ifdef GRID_COMMS_MPI3
+#include <Grid/cshift/Cshift_mpi.h>
+#endif 
+
 #ifdef GRID_COMMS_SHMEM
 #include <Grid/cshift/Cshift_mpi.h> // uses same implementation of communicator
 #endif 

From cbcfea466f75f185a171ffe529b9a14b7938aff3 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:57:14 +0100
Subject: [PATCH 064/149] MPI3

---
 lib/Makefile.am | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/Makefile.am b/lib/Makefile.am
index bc752af5..35e76cc8 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,6 +3,10 @@ if BUILD_COMMS_MPI
   extra_sources+=communicator/Communicator_mpi.cc
 endif
 
+if BUILD_COMMS_MPI3
+  extra_sources+=communicator/Communicator_mpi3.cc
+endif
+
 if BUILD_COMMS_SHMEM
   extra_sources+=communicator/Communicator_shmem.cc
 endif

From c7cccaaa69e7888db33fe8c8c82612da43dee15c Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:57:31 +0100
Subject: [PATCH 065/149] Comm vector for shmem

---
 lib/Stencil.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index db608fe7..65de9945 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -71,7 +71,7 @@
  namespace Grid {
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,std::vector<std::pair<int,int> >& table)
+Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,std::vector<std::pair<int,int> >& table)
 {
   table.resize(0);
   int rd = rhs._grid->_rdimensions[dimension];
@@ -109,7 +109,7 @@ Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,std::vector<cobj,ali
 }
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,
+Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice<vobj> &rhs,commVector<cobj> &buffer,
 			   compressor &compress, int off,int so)
 {
 PARALLEL_FOR_LOOP     
@@ -119,7 +119,7 @@ PARALLEL_FOR_LOOP
 }
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
+Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
 			     double &t_table ,double & t_data )
 {
   std::vector<std::pair<int,int> > table;
@@ -347,10 +347,10 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,std::vector<cobj,alignedAl
        }
 
        // Comms buffers
-       std::vector<Vector<scalar_object> > u_simd_send_buf;
-       std::vector<Vector<scalar_object> > u_simd_recv_buf;
-       Vector<cobj>          u_send_buf;
-       Vector<cobj>          comm_buf;
+       std::vector<commVector<scalar_object> > u_simd_send_buf;
+       std::vector<commVector<scalar_object> > u_simd_recv_buf;
+       commVector<cobj>          u_send_buf;
+       commVector<cobj>          comm_buf;
        int u_comm_offset;
        int _unified_buffer_size;
 

From 5fe2b85cbd951e2a0137e8ce291f0556bbb091ea Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:58:01 +0100
Subject: [PATCH 066/149] MPI3 and shared memory support

---
 lib/communicator/Communicator_base.h | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 94d277e9..9b5ae8cb 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -34,6 +34,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_COMMS_MPI
 #include <mpi.h>
 #endif
+#ifdef GRID_COMMS_MPI3
+#include <mpi.h>
+#endif
 #ifdef GRID_COMMS_SHMEM
 #include <mpp/shmem.h>
 #endif
@@ -52,6 +55,29 @@ class CartesianCommunicator {
 #ifdef GRID_COMMS_MPI
     MPI_Comm communicator;
     typedef MPI_Request CommsRequest_t;
+#elif  GRID_COMMS_MPI3
+    MPI_Comm communicator;
+    typedef MPI_Request CommsRequest_t;
+
+    const int MAXLOG2RANKSPERNODE = 16; // 65536 ranks per node adequate for now
+
+    std::vector<int>  WorldDims;
+    std::vector<int>  GroupDims;
+    std::vector<int>  ShmDims;
+
+    std::vector<int> GroupCoor;
+    std::vector<int> ShmCoor;
+    std::vector<int> WorldCoor;
+
+    int GroupRank;
+    int ShmRank;
+    int WorldRank;
+
+    int GroupSize;
+    int ShmSize;
+    int WorldSize;
+
+    std::vector<int>  LexicographicToWorldRank;
 #else 
     typedef int CommsRequest_t;
 #endif

From 9b39f35ae6c3942f21541745cb50b79ec8077981 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:58:53 +0100
Subject: [PATCH 067/149] commVector different for SHMEM compat

---
 lib/cshift/Cshift_common.h | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/cshift/Cshift_common.h b/lib/cshift/Cshift_common.h
index b0e9b798..2b146daa 100644
--- a/lib/cshift/Cshift_common.h
+++ b/lib/cshift/Cshift_common.h
@@ -45,7 +45,7 @@ public:
 // Gather for when there is no need to SIMD split with compression
 ///////////////////////////////////////////////////////////////////
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple (const Lattice<vobj> &rhs,std::vector<cobj,alignedAllocator<cobj> > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0)
+Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 
@@ -114,6 +114,7 @@ PARALLEL_NESTED_LOOP2
 	int o      =   n*n1;
 	int offset = b+n*n2;
 	cobj temp =compress(rhs._odata[so+o+b]);
+
 	extract<cobj>(temp,pointers,offset);
 
       }
@@ -121,6 +122,7 @@ PARALLEL_NESTED_LOOP2
   } else { 
 
     assert(0); //Fixme think this is buggy
+
     for(int n=0;n<e1;n++){
       for(int b=0;b<e2;b++){
 	int o=n*rhs._grid->_slice_stride[dimension];
@@ -139,7 +141,7 @@ PARALLEL_NESTED_LOOP2
 //////////////////////////////////////////////////////
 // Gather for when there is no need to SIMD split
 //////////////////////////////////////////////////////
-template<class vobj> void Gather_plane_simple (const Lattice<vobj> &rhs,std::vector<vobj,alignedAllocator<vobj> > &buffer,             int dimension,int plane,int cbmask)
+template<class vobj> void Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
 {
   SimpleCompressor<vobj> dontcompress;
   Gather_plane_simple (rhs,buffer,dimension,plane,cbmask,dontcompress);
@@ -157,7 +159,7 @@ template<class vobj> void Gather_plane_extract(const Lattice<vobj> &rhs,std::vec
 //////////////////////////////////////////////////////
 // Scatter for when there is no need to SIMD split
 //////////////////////////////////////////////////////
-template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,std::vector<vobj,alignedAllocator<vobj> > &buffer, int dimension,int plane,int cbmask)
+template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 

From 4f8e636a438007a14ff95b2e59fe6297a6c38283 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:59:16 +0100
Subject: [PATCH 068/149] commVector

---
 lib/cshift/Cshift_mpi.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/cshift/Cshift_mpi.h b/lib/cshift/Cshift_mpi.h
index 704fda34..b3c07cd6 100644
--- a/lib/cshift/Cshift_mpi.h
+++ b/lib/cshift/Cshift_mpi.h
@@ -119,8 +119,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
   assert(shift<fd);
   
   int buffer_size = rhs._grid->_slice_nblock[dimension]*rhs._grid->_slice_block[dimension];
-  std::vector<vobj,alignedAllocator<vobj> > send_buf(buffer_size);
-  std::vector<vobj,alignedAllocator<vobj> > recv_buf(buffer_size);
+  commVector<vobj> send_buf(buffer_size);
+  commVector<vobj> recv_buf(buffer_size);
 
   int cb= (cbmask==0x2)? Odd : Even;
   int sshift= rhs._grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb);
@@ -191,8 +191,8 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
   int words = sizeof(vobj)/sizeof(vector_type);
 
-  std::vector<Vector<scalar_object> >   send_buf_extract(Nsimd,Vector<scalar_object>(buffer_size) );
-  std::vector<Vector<scalar_object> >   recv_buf_extract(Nsimd,Vector<scalar_object>(buffer_size) );
+  std::vector<commVector<scalar_object> >   send_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
+  std::vector<commVector<scalar_object> >   recv_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
 
   int bytes = buffer_size*sizeof(scalar_object);
 

From f9d5e95d72291c54a5be219e9a3333e9296a45da Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 16:59:39 +0100
Subject: [PATCH 069/149] allocator template typedefs moved to AlignedAllocator

---
 lib/lattice/Lattice_base.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/lattice/Lattice_base.h b/lib/lattice/Lattice_base.h
index 3bfa5613..e04c1443 100644
--- a/lib/lattice/Lattice_base.h
+++ b/lib/lattice/Lattice_base.h
@@ -65,9 +65,6 @@ public:
     
 class LatticeExpressionBase {};
 
-template<class T> using Vector = std::vector<T,alignedAllocator<T> >;               // Aligned allocator??
-template<class T> using Matrix = std::vector<std::vector<T,alignedAllocator<T> > >; // Aligned allocator??
-
 template <typename Op, typename T1>                           
 class LatticeUnaryExpression  : public std::pair<Op,std::tuple<T1> > , public LatticeExpressionBase {
  public:

From b58adc6a4bbfa461acb8cdc3d90133ea04ecc35f Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 17:00:15 +0100
Subject: [PATCH 070/149] commVector

---
 lib/communicator/Communicator_mpi3.cc       | 358 ++++++++++++++++++++
 lib/qcd/action/fermion/WilsonKernels.cc     |   6 +-
 lib/qcd/action/fermion/WilsonKernels.h      |  22 +-
 lib/qcd/action/fermion/WilsonKernelsAsm.cc  |  16 +-
 lib/qcd/action/fermion/WilsonKernelsHand.cc |  16 +-
 5 files changed, 388 insertions(+), 30 deletions(-)
 create mode 100644 lib/communicator/Communicator_mpi3.cc

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
new file mode 100644
index 00000000..11734fd9
--- /dev/null
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -0,0 +1,358 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/communicator/Communicator_mpi.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include "Grid.h"
+#include <mpi.h>
+
+namespace Grid {
+
+// Global used by Init and nowhere else. How to hide?
+int Rank(void) {
+  int pe;
+  MPI_Comm_rank(MPI_COMM_WORLD,&pe);
+  return pe;
+}
+  // Should error check all MPI calls.
+void CartesianCommunicator::Init(int *argc, char ***argv) {
+  int flag;
+  MPI_Initialized(&flag); // needed to coexist with other libs apparently
+  if ( !flag ) {
+    MPI_Init(argc,argv);
+  }
+}
+  ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Want to implement some magic ... Group sub-cubes into those on same node
+  //
+  ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
+{
+  std::vector<int> coor = _processor_coor;
+
+  assert(std::abs(shift) <_processors[dim]);
+
+  coor[dim] = (_processor_coor[dim] + shift + _processors[dim])%_processors[dim];
+  Lexicographic::IndexFromCoor(coor,source,_processors);
+  source = LexicographicToWorldRank[source];
+
+  coor[dim] = (_processor_coor[dim] - shift + _processors[dim])%_processors[dim];
+  Lexicographic::IndexFromCoor(coor,dest,_processors);
+  dest = LexicographicToWorldRank[dest];
+}
+int CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor)
+{
+  int rank;
+  Lexicographic::IndexFromCoor(coor,rank,_processors);
+  rank = LexicographicToWorldRank[rank];
+  return rank;
+}
+void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor)
+{
+  Lexicographic::CoorFromIndex(coor,rank,_processors);
+  rank = LexicographicToWorldRank[rank];
+}
+
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
+{ 
+  _ndimension = processors.size();
+  std::cout << "Creating "<< _ndimension << " dim communicator "<<std::endl;
+  for(int d =0;d<_ndimension;d++){
+    std::cout << processors[d]<<" ";
+  };
+  std::cout << std::endl;
+
+  WorldDims = processors;
+
+  communicator = MPI_COMM_WORLD;
+  MPI_Comm shmcomm;
+  MPI_Comm_split_type(communicator, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&shmcomm);
+  MPI_Comm_rank(communicator,&WorldRank);
+  MPI_Comm_size(communicator,&WorldSize);
+  MPI_Comm_rank(shmcomm     ,&ShmRank);
+  MPI_Comm_size(shmcomm     ,&ShmSize);
+  GroupSize = WorldSize/ShmSize;
+
+  std::cout<< "Ranks per node "<< ShmSize << std::endl;
+  std::cout<< "Nodes          "<< GroupSize << std::endl;
+  std::cout<< "Ranks          "<< WorldSize << std::endl;
+  
+  ////////////////////////////////////////////////////////////////
+  // Assert power of two shm_size.
+  ////////////////////////////////////////////////////////////////
+  int log2size = -1;
+  for(int i=0;i<=MAXLOG2RANKSPERNODE;i++){  
+    if ( (0x1<<i) == ShmSize ) {
+      log2size = i;
+      break;
+    }
+  }
+  assert(log2size != -1);
+  
+  ////////////////////////////////////////////////////////////////
+  // Identify subblock of ranks on node spreading across dims
+  // in a maximally symmetrical way
+  ////////////////////////////////////////////////////////////////
+  int dim = 0;
+  
+  ShmDims.resize(_ndimension,1);
+  GroupDims.resize(_ndimension);
+  
+  ShmCoor.resize(_ndimension);
+  GroupCoor.resize(_ndimension);
+  WorldCoor.resize(_ndimension);
+  for(int l2=0;l2<log2size;l2++){
+    while ( WorldDims[dim] / ShmDims[dim] <= 1 ) dim=(dim+1)%_ndimension;
+    ShmDims[dim]*=2;
+    dim=(dim+1)%_ndimension;
+  }
+  
+  std::cout << "Shm group dims "<<std::endl;
+  for(int d =0;d<_ndimension;d++){
+    std::cout << ShmDims[d]<<" ";
+  };
+  std::cout << std::endl;
+
+  ////////////////////////////////////////////////////////////////
+  // Establish torus of processes and nodes with sub-blockings
+  ////////////////////////////////////////////////////////////////
+  for(int d=0;d<_ndimension;d++){
+    GroupDims[d] = WorldDims[d]/ShmDims[d];
+  }
+  std::cout << "Group dims "<<std::endl;
+  for(int d =0;d<_ndimension;d++){
+    std::cout << GroupDims[d]<<" ";
+  };
+  std::cout << std::endl;
+  
+  MPI_Group WorldGroup, ShmGroup;
+  MPI_Comm_group (communicator, &WorldGroup); 
+  MPI_Comm_group (shmcomm, &ShmGroup);
+  
+  std::vector<int> world_ranks(WorldSize); 
+  std::vector<int> group_ranks(WorldSize); 
+  std::vector<int> mygroup(GroupSize);
+  for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
+  
+  MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &group_ranks[0]); 
+
+  ////////////////////////////////////////////////////////////////
+  // Check processor counts match
+  ////////////////////////////////////////////////////////////////
+  _Nprocessors=1;
+  _processors = processors;
+  _processor_coor.resize(_ndimension);
+  for(int i=0;i<_ndimension;i++){
+    std::cout << " p " << _processors[i]<<std::endl;
+    _Nprocessors*=_processors[i];
+  }
+  std::cout << " World " <<WorldSize <<" Nproc "<<_Nprocessors<<std::endl;
+  assert(WorldSize==_Nprocessors);
+  
+  ///////////////////////////////////////////////////////////////////
+  // Identify who is in my group and noninate the leader
+  ///////////////////////////////////////////////////////////////////
+  int g=0;
+  for(int rank=0;rank<WorldSize;rank++){
+    if(group_ranks[rank]!=MPI_UNDEFINED){
+	  mygroup[g] = rank;
+    }
+  }
+  
+  std::sort(mygroup.begin(),mygroup.end(),std::greater<int>());
+  int myleader = mygroup[0];
+  
+  std::vector<int> leaders_1hot(WorldSize,0);
+  std::vector<int> leaders_group(GroupSize,0);
+  leaders_1hot [ myleader ] = 1;
+      
+  ///////////////////////////////////////////////////////////////////
+  // global sum leaders over comm world
+  ///////////////////////////////////////////////////////////////////
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,communicator);
+  assert(ierr==0);
+  
+  ///////////////////////////////////////////////////////////////////
+  // find the group leaders world rank
+  ///////////////////////////////////////////////////////////////////
+  int group=0;
+  for(int l=0;l<WorldSize;l++){
+    if(leaders_1hot[l]){
+      leaders_group[group++] = l;
+    }
+  }
+  
+  ///////////////////////////////////////////////////////////////////
+  // Identify the rank of the group in which I (and my leader) live
+  ///////////////////////////////////////////////////////////////////
+  GroupRank=-1;
+  for(int g=0;g<GroupSize;g++){
+    if (myleader == leaders_group[g]){
+      GroupRank=g;
+    }
+  }
+  assert(GroupRank!=-1);
+      
+  ////////////////////////////////////////////////////////////////
+  // Establish mapping between lexico physics coord and WorldRank
+  // 
+  ////////////////////////////////////////////////////////////////
+  LexicographicToWorldRank.resize(WorldSize,0);
+  Lexicographic::CoorFromIndex(GroupCoor,GroupRank,GroupDims);
+  Lexicographic::CoorFromIndex(ShmCoor,ShmRank,ShmDims);
+  for(int d=0;d<_ndimension;d++){
+    WorldCoor[d] = GroupCoor[d]*ShmDims[d]+ShmCoor[d];
+  }
+  _processor_coor = WorldCoor;
+
+  int lexico;
+  Lexicographic::IndexFromCoor(WorldCoor,lexico,WorldDims);
+  LexicographicToWorldRank[lexico]=WorldRank;
+  _processor = lexico;
+
+  ///////////////////////////////////////////////////////////////////
+  // global sum Lexico to World mapping
+  ///////////////////////////////////////////////////////////////////
+  ierr=MPI_Allreduce(MPI_IN_PLACE,&LexicographicToWorldRank[0],WorldSize,MPI_INT,MPI_SUM,communicator);
+  assert(ierr==0);
+  
+};
+
+void CartesianCommunicator::GlobalSum(uint32_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(uint64_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(float &f){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSumVector(float *f,int N)
+{
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(double &d)
+{
+  int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSumVector(double *d,int N)
+{
+  int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+
+
+// Basic Halo comms primitive
+void CartesianCommunicator::SendToRecvFrom(void *xmit,
+					   int dest,
+					   void *recv,
+					   int from,
+					   int bytes)
+{
+  std::vector<CommsRequest_t> reqs(0);
+  SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes);
+  SendToRecvFromComplete(reqs);
+}
+
+void CartesianCommunicator::SendRecvPacket(void *xmit,
+					   void *recv,
+					   int sender,
+					   int receiver,
+					   int bytes)
+{
+  MPI_Status stat;
+  assert(sender != receiver);
+  int tag = sender;
+  if ( _processor == sender ) {
+    MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator);
+  }
+  if ( _processor == receiver ) { 
+    MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat);
+  }
+}
+
+// Basic Halo comms primitive
+void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						void *xmit,
+						int dest,
+						void *recv,
+						int from,
+						int bytes)
+{
+  MPI_Request xrq;
+  MPI_Request rrq;
+  int rank = _processor;
+  int ierr;
+  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+  
+  assert(ierr==0);
+
+  list.push_back(xrq);
+  list.push_back(rrq);
+}
+void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
+{
+  int nreq=list.size();
+  std::vector<MPI_Status> status(nreq);
+  int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
+
+  assert(ierr==0);
+}
+
+void CartesianCommunicator::Barrier(void)
+{
+  int ierr = MPI_Barrier(communicator);
+  assert(ierr==0);
+}
+
+void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
+{
+  int ierr=MPI_Bcast(data,
+		     bytes,
+		     MPI_BYTE,
+		     root,
+		     communicator);
+  assert(ierr==0);
+}
+
+void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
+{
+  int ierr= MPI_Bcast(data,
+		      bytes,
+		      MPI_BYTE,
+		      root,
+		      MPI_COMM_WORLD);
+  assert(ierr==0);
+}
+
+}
+
diff --git a/lib/qcd/action/fermion/WilsonKernels.cc b/lib/qcd/action/fermion/WilsonKernels.cc
index 49bd98c5..58e62cc8 100644
--- a/lib/qcd/action/fermion/WilsonKernels.cc
+++ b/lib/qcd/action/fermion/WilsonKernels.cc
@@ -45,7 +45,7 @@ WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
 template <class Impl>
 void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(
     StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
+    commVector<SiteHalfSpinor> &buf, int sF,
     int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
@@ -222,7 +222,7 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(
 template <class Impl>
 void WilsonKernels<Impl>::DiracOptGenericDhopSite(
     StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
+    commVector<SiteHalfSpinor> &buf, int sF,
     int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
@@ -398,7 +398,7 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSite(
 template <class Impl>
 void WilsonKernels<Impl>::DiracOptDhopDir(
     StencilImpl &st, DoubledGaugeField &U,
-    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
+    commVector<SiteHalfSpinor> &buf, int sF,
     int sU, const FermionField &in, FermionField &out, int dir, int gamma) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
diff --git a/lib/qcd/action/fermion/WilsonKernels.h b/lib/qcd/action/fermion/WilsonKernels.h
index 23c145de..66770c3c 100644
--- a/lib/qcd/action/fermion/WilsonKernels.h
+++ b/lib/qcd/action/fermion/WilsonKernels.h
@@ -58,7 +58,7 @@ namespace Grid {
       typename std::enable_if<Impl::Dimension == 3 && Nc == 3 &&EnableBool, void>::type
 	DiracOptDhopSite(
 			 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-			 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+			 commVector<SiteHalfSpinor> &buf,
 			 int sF, int sU, int Ls, int Ns, const FermionField &in,
 			 FermionField &out) {
 #ifdef AVX512
@@ -89,7 +89,7 @@ namespace Grid {
 	  typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type
 	  DiracOptDhopSite(
 			   StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-			   std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+			   commVector<SiteHalfSpinor> &buf,
 			   int sF, int sU, int Ls, int Ns, const FermionField &in,
 			   FermionField &out) {
 	  for (int site = 0; site < Ns; site++) {
@@ -107,7 +107,7 @@ namespace Grid {
 				  void>::type
 	  DiracOptDhopSiteDag(
 			      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-			      std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+			      commVector<SiteHalfSpinor> &buf,
 			      int sF, int sU, int Ls, int Ns, const FermionField &in,
 			      FermionField &out) {
 #ifdef AVX512
@@ -139,7 +139,7 @@ namespace Grid {
 				      void>::type
 				      DiracOptDhopSiteDag(
 							  StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-							  std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							  commVector<SiteHalfSpinor> &buf,
 							  int sF, int sU, int Ls, int Ns, const FermionField &in,
 							  FermionField &out) {
 					for (int site = 0; site < Ns; site++) {
@@ -154,7 +154,7 @@ namespace Grid {
 
 				    void DiracOptDhopDir(
 							 StencilImpl &st, DoubledGaugeField &U,
-							 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							 commVector<SiteHalfSpinor> &buf,
 							 int sF, int sU, const FermionField &in, FermionField &out, int dirdisp,
 							 int gamma);
 
@@ -162,34 +162,34 @@ namespace Grid {
 				    // Specialised variants
 				    void DiracOptGenericDhopSite(
 								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+								 commVector<SiteHalfSpinor> &buf,
 								 int sF, int sU, const FermionField &in, FermionField &out);
 
 				    void DiracOptGenericDhopSiteDag(
 								    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+								    commVector<SiteHalfSpinor> &buf,
 								    int sF, int sU, const FermionField &in, FermionField &out);
 
 				    void DiracOptAsmDhopSite(
 							     StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-							     std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							     commVector<SiteHalfSpinor> &buf,
 							     int sF, int sU, int Ls, int Ns, const FermionField &in,
 							     FermionField &out);
 
 				    void DiracOptAsmDhopSiteDag(
 								StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+								commVector<SiteHalfSpinor> &buf,
 								int sF, int sU, int Ls, int Ns, const FermionField &in,
 								FermionField &out);
 
 				    void DiracOptHandDhopSite(
 							      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-							      std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							      commVector<SiteHalfSpinor> &buf,
 							      int sF, int sU, const FermionField &in, FermionField &out);
 
 				    void DiracOptHandDhopSiteDag(
 								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+								 commVector<SiteHalfSpinor> &buf,
 								 int sF, int sU, const FermionField &in, FermionField &out);
 
 	public:
diff --git a/lib/qcd/action/fermion/WilsonKernelsAsm.cc b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
index b09699ef..7857e89a 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsm.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
@@ -40,14 +40,14 @@ namespace Grid {
     ///////////////////////////////////////////////////////////
     template<class Impl>
       void WilsonKernels<Impl >::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-                             std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+                             commVector<SiteHalfSpinor>  &buf,
                              int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
     {
       assert(0);
     }
     template<class Impl>
       void WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-                                std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+                                commVector<SiteHalfSpinor>  &buf,
                                 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
     {
       assert(0);
@@ -86,14 +86,14 @@ namespace Grid {
 #undef KERNEL_DAG
     template<>
     void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-							 std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+							 commVector<SiteHalfSpinor>  &buf,
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
       
 #define KERNEL_DAG
     template<>
     void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-							    std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+							    commVector<SiteHalfSpinor>  &buf,
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
@@ -111,14 +111,14 @@ namespace Grid {
 #undef KERNEL_DAG
     template<>
     void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+								  commVector<SiteHalfSpinor>  &buf,
 								  int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
 #define KERNEL_DAG
     template<>
     void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-								     std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+								     commVector<SiteHalfSpinor>  &buf,
 								     int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
@@ -127,10 +127,10 @@ namespace Grid {
 
 #define INSTANTIATE_ASM(A)\
 template void WilsonKernels<A>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
-                                   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+                                   commVector<SiteHalfSpinor>  &buf,\
                                   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
 template void WilsonKernels<A>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
-                                   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+                                   commVector<SiteHalfSpinor>  &buf,\
                                   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
 
 
diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index 15c8ab56..9d7eac23 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -313,7 +313,7 @@ namespace QCD {
 
   template<class Impl>
   void WilsonKernels<Impl>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-					       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+					       commVector<SiteHalfSpinor>  &buf,
 					       int ss,int sU,const FermionField &in, FermionField &out)
 {
   typedef typename Simd::scalar_type S;
@@ -556,7 +556,7 @@ namespace QCD {
 
   template<class Impl>
   void WilsonKernels<Impl>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-					       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+					       commVector<SiteHalfSpinor>  &buf,
 					       int ss,int sU,const FermionField &in, FermionField &out)
 {
   //  std::cout << "Hand op Dhop "<<std::endl;
@@ -804,7 +804,7 @@ namespace QCD {
   ////////////////////////////////////////////////
 template<>
 void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							     std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+							     commVector<SiteHalfSpinor>  &buf,
 							     int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
@@ -812,7 +812,7 @@ void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,Leb
 
 template<>
 void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+								commVector<SiteHalfSpinor>  &buf,
 								int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
@@ -820,7 +820,7 @@ void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,
 
 template<>
 void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							     std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+							     commVector<SiteHalfSpinor>  &buf,
 							     int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
@@ -828,7 +828,7 @@ void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,Leb
 
 template<>
 void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+								commVector<SiteHalfSpinor>  &buf,
 								int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
@@ -841,10 +841,10 @@ void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,
 
 #define INSTANTIATE_THEM(A) \
 template void WilsonKernels<A>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+							       commVector<SiteHalfSpinor>  &buf,\
 							       int ss,int sU,const FermionField &in, FermionField &out);\
 template void WilsonKernels<A>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
-								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+								  commVector<SiteHalfSpinor>  &buf,\
 								  int ss,int sU,const FermionField &in, FermionField &out);
 
 INSTANTIATE_THEM(WilsonImplF);

From 5b5925b8e5aa454467f9b53d35dcd9f779a302b2 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 20 Oct 2016 17:09:40 +0100
Subject: [PATCH 071/149] Forgot to add

---
 lib/AlignedAllocator.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/AlignedAllocator.h b/lib/AlignedAllocator.h
index d0de1ec3..89731246 100644
--- a/lib/AlignedAllocator.h
+++ b/lib/AlignedAllocator.h
@@ -161,7 +161,8 @@ public:
 #error "implement MPI3 windowed allocate"
   }
 #else
-  pointer allocate(size_type __n, const void* _p= 0)
+  pointer allocate(size_type __n, const void* _p= 0) 
+  {
 #ifdef HAVE_MM_MALLOC_H
     _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128);
 #else
@@ -187,9 +188,9 @@ template<typename _Tp>  inline bool operator!=(const commAllocator<_Tp>&, const
 ////////////////////////////////////////////////////////////////////////////////
 // Template typedefs
 ////////////////////////////////////////////////////////////////////////////////
-template<class T> using Vector    = std::vector<T,alignedAllocator<T> >;           
+template<class T> using Vector     = std::vector<T,alignedAllocator<T> >;           
 template<class T> using commVector = std::vector<T,commAllocator<T> >;              
-template<class T> using Matrix = std::vector<std::vector<T,alignedAllocator<T> > >;
+template<class T> using Matrix     = std::vector<std::vector<T,alignedAllocator<T> > >;
     
 }; // namespace Grid
 #endif

From a762b1fb712fa620e5662e4e7d41a94925ecff00 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 21 Oct 2016 09:03:26 +0100
Subject: [PATCH 072/149] MPI3 working with a bounce through shared memory on
 my laptop. Longer term plan: make the "u_comm_buf" in Stencil point to the
 shared region and avoid the send between ranks on same node.

---
 benchmarks/Benchmark_dwf.cc           |   7 +-
 lib/AlignedAllocator.h                |   9 -
 lib/Init.cc                           |   2 +-
 lib/Log.cc                            |   2 +-
 lib/Threads.h                         |  17 ++
 lib/communicator/Communicator_base.h  |  26 ++-
 lib/communicator/Communicator_mpi.cc  |  11 +-
 lib/communicator/Communicator_mpi3.cc | 250 +++++++++++++++++---------
 8 files changed, 208 insertions(+), 116 deletions(-)

diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index 6a283085..a32d7535 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -208,7 +208,6 @@ int main (int argc, char ** argv)
 
     std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
 
-
     RealF sum=0;
     for(int x=0;x<latt4[0];x++){
     for(int y=0;y<latt4[1];y++){
@@ -221,9 +220,9 @@ int main (int argc, char ** argv)
       peekSite(simd,sresult,site);
       sum=sum+norm2(normal-simd);
       if (norm2(normal-simd) > 1.0e-6 ) {
-  std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" "<<norm2(normal-simd)<<std::endl;
-  std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" normal "<<normal<<std::endl;
-  std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" simd   "<<simd<<std::endl;
+	std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" "<<norm2(normal-simd)<<std::endl;
+	std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" normal "<<normal<<std::endl;
+	std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" simd   "<<simd<<std::endl;
       }
     }}}}}
     std::cout<<GridLogMessage<<" difference between normal and simd is "<<sum<<std::endl;
diff --git a/lib/AlignedAllocator.h b/lib/AlignedAllocator.h
index 89731246..44929ca8 100644
--- a/lib/AlignedAllocator.h
+++ b/lib/AlignedAllocator.h
@@ -151,15 +151,6 @@ public:
   void deallocate(pointer __p, size_type) { 
     shmem_free((void *)__p);
   }
-#elif defined(GRID_COMMS_MPI3)
-  pointer allocate(size_type __n, const void* _p= 0)
-  { 
-#error "implement MPI3 windowed allocate"
-    
-  }
-  void deallocate(pointer __p, size_type) { 
-#error "implement MPI3 windowed allocate"
-  }
 #else
   pointer allocate(size_type __n, const void* _p= 0) 
   {
diff --git a/lib/Init.cc b/lib/Init.cc
index bd9638c3..a21cbce0 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -279,7 +279,7 @@ void Grid_init(int *argc,char ***argv)
   
 void Grid_finalize(void)
 {
-#ifdef GRID_COMMS_MPI
+#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3)
   MPI_Finalize();
   Grid_unquiesce_nodes();
 #endif
diff --git a/lib/Log.cc b/lib/Log.cc
index 2082570d..a55a1c9e 100644
--- a/lib/Log.cc
+++ b/lib/Log.cc
@@ -73,7 +73,7 @@ void GridLogConfigure(std::vector<std::string> &logstreams) {
 ////////////////////////////////////////////////////////////
 void Grid_quiesce_nodes(void) {
   int me = 0;
-#ifdef GRID_COMMS_MPI
+#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3)
   MPI_Comm_rank(MPI_COMM_WORLD, &me);
 #endif
 #ifdef GRID_COMMS_SHMEM
diff --git a/lib/Threads.h b/lib/Threads.h
index d502dd82..70350685 100644
--- a/lib/Threads.h
+++ b/lib/Threads.h
@@ -31,6 +31,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #ifdef _OPENMP
 #define GRID_OMP
+#warning "OpenMP"
 #endif
 
 #define UNROLL  _Pragma("unroll")
@@ -127,6 +128,22 @@ class GridThread {
     ThreadBarrier();
   };
 
+  static void bcopy(const void *src, void *dst, size_t len) {
+#ifdef GRID_OMP
+#pragma omp parallel 
+    {
+      const char *c_src =(char *) src;
+      char *c_dest=(char *) dst;
+      int me,mywork,myoff;
+      GridThread::GetWorkBarrier(len,me, mywork,myoff);
+      bcopy(&c_src[myoff],&c_dest[myoff],mywork);
+    }
+#else 
+    bcopy(src,dst,len);
+#endif
+  }
+
+
 };
 
 }
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 9b5ae8cb..6d6602be 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -45,7 +45,7 @@ class CartesianCommunicator {
   public:    
 
   // Communicator should know nothing of the physics grid, only processor grid.
-
+  
     int              _Nprocessors;     // How many in all
     std::vector<int> _processors;      // Which dimensions get relayed out over processors lanes.
     int              _processor;       // linear processor rank
@@ -56,10 +56,13 @@ class CartesianCommunicator {
     MPI_Comm communicator;
     typedef MPI_Request CommsRequest_t;
 #elif  GRID_COMMS_MPI3
+    int shm_mode;
+
     MPI_Comm communicator;
     typedef MPI_Request CommsRequest_t;
 
-    const int MAXLOG2RANKSPERNODE = 16; // 65536 ranks per node adequate for now
+    const int MAXLOG2RANKSPERNODE = 16;     // 65536 ranks per node adequate for now
+    const uint64_t MAX_MPI_SHM_BYTES = 256*1024*1024; // 256MB shared memory for comms enought for 48^4 local vol comms
 
     std::vector<int>  WorldDims;
     std::vector<int>  GroupDims;
@@ -69,14 +72,23 @@ class CartesianCommunicator {
     std::vector<int> ShmCoor;
     std::vector<int> WorldCoor;
 
-    int GroupRank;
-    int ShmRank;
-    int WorldRank;
+    static std::vector<int> GroupRanks; 
+    static std::vector<int> MyGroup;
+    static int ShmSetup;
+    static MPI_Win ShmWindow; 
+    static MPI_Comm ShmComm;
 
-    int GroupSize;
-    int ShmSize;
+    void * ShmCommBuf;
+    std::vector<void *> ShmCommBufs;
+
+    int WorldRank;
     int WorldSize;
 
+    static int ShmRank;
+    static int ShmSize;
+    static int GroupSize;
+    static int GroupRank;
+
     std::vector<int>  LexicographicToWorldRank;
 #else 
     typedef int CommsRequest_t;
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index dff9811a..9c66202f 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -39,11 +39,11 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   }
 }
 
-  int Rank(void) {
-    int pe;
-    MPI_Comm_rank(MPI_COMM_WORLD,&pe);
-    return pe;
-  }
+int Rank(void) {
+  int pe;
+  MPI_Comm_rank(MPI_COMM_WORLD,&pe);
+  return pe;
+}
 
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
@@ -168,7 +168,6 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &
   int nreq=list.size();
   std::vector<MPI_Status> status(nreq);
   int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
-
   assert(ierr==0);
 }
 
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 11734fd9..1a36e610 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -30,6 +30,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
+
+
 // Global used by Init and nowhere else. How to hide?
 int Rank(void) {
   int pe;
@@ -76,29 +78,129 @@ void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &c
   rank = LexicographicToWorldRank[rank];
 }
 
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+int CartesianCommunicator::ShmSetup = 0;
+int CartesianCommunicator::ShmRank;
+int CartesianCommunicator::ShmSize;
+int CartesianCommunicator::GroupRank;
+int CartesianCommunicator::GroupSize;
+MPI_Comm CartesianCommunicator::ShmComm;
+MPI_Win  CartesianCommunicator::ShmWindow;
+std::vector<int> CartesianCommunicator::GroupRanks; 
+std::vector<int> CartesianCommunicator::MyGroup;
+
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 { 
+
   _ndimension = processors.size();
-  std::cout << "Creating "<< _ndimension << " dim communicator "<<std::endl;
-  for(int d =0;d<_ndimension;d++){
-    std::cout << processors[d]<<" ";
-  };
-  std::cout << std::endl;
 
   WorldDims = processors;
 
   communicator = MPI_COMM_WORLD;
-  MPI_Comm shmcomm;
-  MPI_Comm_split_type(communicator, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&shmcomm);
   MPI_Comm_rank(communicator,&WorldRank);
   MPI_Comm_size(communicator,&WorldSize);
-  MPI_Comm_rank(shmcomm     ,&ShmRank);
-  MPI_Comm_size(shmcomm     ,&ShmSize);
-  GroupSize = WorldSize/ShmSize;
 
-  std::cout<< "Ranks per node "<< ShmSize << std::endl;
-  std::cout<< "Nodes          "<< GroupSize << std::endl;
-  std::cout<< "Ranks          "<< WorldSize << std::endl;
+  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Plan: allocate a fixed SHM region. Scratch that is just used via some scheme during stencil comms, with no allocate free.
+  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Does every grid need one, or could we share across all grids via a singleton/guard?
+  int ierr;
+
+  if ( !ShmSetup ) { 
+
+    MPI_Comm_split_type(communicator, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm);
+    MPI_Comm_rank(ShmComm     ,&ShmRank);
+    MPI_Comm_size(ShmComm     ,&ShmSize);
+    GroupSize = WorldSize/ShmSize;
+
+    /////////////////////////////////////////////////////////////////////
+    // find world ranks in our SHM group (i.e. which ranks are on our node)
+    /////////////////////////////////////////////////////////////////////
+    MPI_Group WorldGroup, ShmGroup;
+    MPI_Comm_group (communicator, &WorldGroup); 
+    MPI_Comm_group (ShmComm, &ShmGroup);
+
+    std::vector<int> world_ranks(WorldSize); 
+    GroupRanks.resize(WorldSize); 
+    MyGroup.resize(ShmSize);
+    for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
+  
+    MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &GroupRanks[0]); 
+
+    ///////////////////////////////////////////////////////////////////
+    // Identify who is in my group and noninate the leader
+    ///////////////////////////////////////////////////////////////////
+    int g=0;
+    for(int rank=0;rank<WorldSize;rank++){
+      if(GroupRanks[rank]!=MPI_UNDEFINED){
+	assert(g<ShmSize);
+	MyGroup[g++] = rank;
+      }
+    }
+  
+    std::sort(MyGroup.begin(),MyGroup.end(),std::greater<int>());
+    int myleader = MyGroup[0];
+    
+    std::vector<int> leaders_1hot(WorldSize,0);
+    std::vector<int> leaders_group(GroupSize,0);
+    leaders_1hot [ myleader ] = 1;
+    
+    ///////////////////////////////////////////////////////////////////
+    // global sum leaders over comm world
+    ///////////////////////////////////////////////////////////////////
+    ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,communicator);
+    assert(ierr==0);
+  
+    ///////////////////////////////////////////////////////////////////
+    // find the group leaders world rank
+    ///////////////////////////////////////////////////////////////////
+    int group=0;
+    for(int l=0;l<WorldSize;l++){
+      if(leaders_1hot[l]){
+	leaders_group[group++] = l;
+      }
+    }
+  
+    ///////////////////////////////////////////////////////////////////
+    // Identify the rank of the group in which I (and my leader) live
+    ///////////////////////////////////////////////////////////////////
+    GroupRank=-1;
+    for(int g=0;g<GroupSize;g++){
+      if (myleader == leaders_group[g]){
+	GroupRank=g;
+      }
+    }
+    assert(GroupRank!=-1);
+    
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    // allocate the shared window for our group
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    
+    ShmCommBuf = 0;
+    ierr = MPI_Win_allocate_shared(MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,ShmComm,&ShmCommBuf,&ShmWindow);
+    assert(ierr==0);
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    // Verbose for now
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    std::cout<< "Ranks per node "<< ShmSize << std::endl;
+    std::cout<< "Nodes          "<< GroupSize << std::endl;
+    std::cout<< "Ranks          "<< WorldSize << std::endl;
+    std::cout<< "Shm CommBuf "<< ShmCommBuf << std::endl;
+
+    // Done
+    ShmSetup=1;
+
+  }
+
+  ShmCommBufs.resize(ShmSize);
+  for(int r=0;r<ShmSize;r++){
+    MPI_Aint sz;
+    int dsp_unit;
+    MPI_Win_shared_query (ShmWindow, r, &sz, &dsp_unit, &ShmCommBufs[r]);
+  }
   
   ////////////////////////////////////////////////////////////////
   // Assert power of two shm_size.
@@ -120,7 +222,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   
   ShmDims.resize(_ndimension,1);
   GroupDims.resize(_ndimension);
-  
+    
   ShmCoor.resize(_ndimension);
   GroupCoor.resize(_ndimension);
   WorldCoor.resize(_ndimension);
@@ -129,12 +231,6 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
     ShmDims[dim]*=2;
     dim=(dim+1)%_ndimension;
   }
-  
-  std::cout << "Shm group dims "<<std::endl;
-  for(int d =0;d<_ndimension;d++){
-    std::cout << ShmDims[d]<<" ";
-  };
-  std::cout << std::endl;
 
   ////////////////////////////////////////////////////////////////
   // Establish torus of processes and nodes with sub-blockings
@@ -142,22 +238,6 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   for(int d=0;d<_ndimension;d++){
     GroupDims[d] = WorldDims[d]/ShmDims[d];
   }
-  std::cout << "Group dims "<<std::endl;
-  for(int d =0;d<_ndimension;d++){
-    std::cout << GroupDims[d]<<" ";
-  };
-  std::cout << std::endl;
-  
-  MPI_Group WorldGroup, ShmGroup;
-  MPI_Comm_group (communicator, &WorldGroup); 
-  MPI_Comm_group (shmcomm, &ShmGroup);
-  
-  std::vector<int> world_ranks(WorldSize); 
-  std::vector<int> group_ranks(WorldSize); 
-  std::vector<int> mygroup(GroupSize);
-  for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
-  
-  MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &group_ranks[0]); 
 
   ////////////////////////////////////////////////////////////////
   // Check processor counts match
@@ -166,55 +246,9 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   _processors = processors;
   _processor_coor.resize(_ndimension);
   for(int i=0;i<_ndimension;i++){
-    std::cout << " p " << _processors[i]<<std::endl;
     _Nprocessors*=_processors[i];
   }
-  std::cout << " World " <<WorldSize <<" Nproc "<<_Nprocessors<<std::endl;
   assert(WorldSize==_Nprocessors);
-  
-  ///////////////////////////////////////////////////////////////////
-  // Identify who is in my group and noninate the leader
-  ///////////////////////////////////////////////////////////////////
-  int g=0;
-  for(int rank=0;rank<WorldSize;rank++){
-    if(group_ranks[rank]!=MPI_UNDEFINED){
-	  mygroup[g] = rank;
-    }
-  }
-  
-  std::sort(mygroup.begin(),mygroup.end(),std::greater<int>());
-  int myleader = mygroup[0];
-  
-  std::vector<int> leaders_1hot(WorldSize,0);
-  std::vector<int> leaders_group(GroupSize,0);
-  leaders_1hot [ myleader ] = 1;
-      
-  ///////////////////////////////////////////////////////////////////
-  // global sum leaders over comm world
-  ///////////////////////////////////////////////////////////////////
-  int ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,communicator);
-  assert(ierr==0);
-  
-  ///////////////////////////////////////////////////////////////////
-  // find the group leaders world rank
-  ///////////////////////////////////////////////////////////////////
-  int group=0;
-  for(int l=0;l<WorldSize;l++){
-    if(leaders_1hot[l]){
-      leaders_group[group++] = l;
-    }
-  }
-  
-  ///////////////////////////////////////////////////////////////////
-  // Identify the rank of the group in which I (and my leader) live
-  ///////////////////////////////////////////////////////////////////
-  GroupRank=-1;
-  for(int g=0;g<GroupSize;g++){
-    if (myleader == leaders_group[g]){
-      GroupRank=g;
-    }
-  }
-  assert(GroupRank!=-1);
       
   ////////////////////////////////////////////////////////////////
   // Establish mapping between lexico physics coord and WorldRank
@@ -309,16 +343,56 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 {
   MPI_Request xrq;
   MPI_Request rrq;
+  
   int rank = _processor;
   int ierr;
-  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
-  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+  int tag;
+  int small = (bytes<MAX_MPI_SHM_BYTES) || (shm_mode==0);
+  static int sequence;
+  int check;
+  assert(dest != _processor);
+  assert(from != _processor);
   
-  assert(ierr==0);
-
-  list.push_back(xrq);
-  list.push_back(rrq);
+  int gdest = GroupRanks[dest];
+  int gme   = GroupRanks[_processor];
+  sequence++;
+  
+  assert(gme == ShmRank);
+  
+  if ( small && (dest !=MPI_UNDEFINED) ) {
+    char *ptr = (char *)ShmCommBufs[gdest];
+    assert(gme != gdest);
+    GridThread::bcopy(xmit,ptr,bytes);
+    bcopy(&_processor,&ptr[bytes],sizeof(_processor));
+    bcopy(&  sequence,&ptr[bytes+4],sizeof(sequence));
+  } else { 
+    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+    assert(ierr==0);
+    list.push_back(xrq);
+  }
+  
+  MPI_Win_sync (ShmWindow);   
+  MPI_Barrier  (ShmComm);
+  MPI_Win_sync (ShmWindow);   
+  
+  if (small && (from !=MPI_UNDEFINED) ) {
+    char *ptr = (char *)ShmCommBufs[ShmRank];
+    GridThread::bcopy(ptr,recv,bytes);
+    bcopy(&ptr[bytes]  ,&tag  ,sizeof(tag));
+    bcopy(&ptr[bytes+4],&check,sizeof(check));
+    assert(check==sequence);
+    assert(tag==from);
+  } else { 
+    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+    assert(ierr==0);
+    list.push_back(rrq);
+  }
+  
+  MPI_Win_sync (ShmWindow);   
+  MPI_Barrier  (ShmComm);
+  MPI_Win_sync (ShmWindow);   
 }
+
 void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
 {
   int nreq=list.size();

From 202078eb1b9d02aa5eee05f1ffe0528801a3ef05 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 09:07:20 +0100
Subject: [PATCH 073/149] Cray / OpenSHMEM ordering differs

---
 lib/AlignedAllocator.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/AlignedAllocator.h b/lib/AlignedAllocator.h
index 89731246..8301d811 100644
--- a/lib/AlignedAllocator.h
+++ b/lib/AlignedAllocator.h
@@ -130,8 +130,12 @@ public:
 
 #ifdef GRID_COMMS_SHMEM
   pointer allocate(size_type __n, const void* _p= 0)
-  { 
+  {
+#ifdef CRAY
     _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64);
+#else
+    _Tp *ptr = (_Tp *) shmem_align(64,__n*sizeof(_Tp));
+#endif
 #ifdef PARANOID_SYMMETRIC_HEAP
     static void * bcast;
     static long  psync[_SHMEM_REDUCE_SYNC_SIZE];

From 20a091c3eddfdb67a82ece6413740a93650a2f98 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 09:08:36 +0100
Subject: [PATCH 074/149] Intel vs. Clang intrinsics differences absorbed

---
 lib/simd/Grid_avx512.h | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h
index 821898d9..d4fd446c 100644
--- a/lib/simd/Grid_avx512.h
+++ b/lib/simd/Grid_avx512.h
@@ -41,6 +41,22 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid{
 namespace Optimization {
+
+  template<class vtype>
+  union uconv {
+    __m512 f;
+    vtype v;
+  };
+
+  union u512f {
+    __m512 v;
+    float f[8];
+  };
+
+  union u512d {
+    __m512 v;
+    double f[4];
+  };
   
   struct Vsplat{
     //Complex float
@@ -361,8 +377,9 @@ namespace Optimization {
   // Some Template specialization
 
   // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases
-#undef GNU_CLANG_COMPILER 
-#ifdef GNU_CLANG_COMPILER
+
+#ifndef __INTEL_COMPILER
+#warning "Slow reduction due to incomplete reduce intrinsics"
   //Complex float Reduce
   template<>
     inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){

From 306160ad9ad7675774931164e48154806c0ce6aa Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 21 Oct 2016 12:07:28 +0100
Subject: [PATCH 075/149] bcopy threaded

---
 lib/communicator/Communicator_mpi3.cc | 54 ++++++++++++++++++++++-----
 1 file changed, 44 insertions(+), 10 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 1a36e610..240aa09e 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -196,10 +196,12 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   }
 
   ShmCommBufs.resize(ShmSize);
+  ShmStencilBufs.resize(ShmSize);
   for(int r=0;r<ShmSize;r++){
     MPI_Aint sz;
     int dsp_unit;
     MPI_Win_shared_query (ShmWindow, r, &sz, &dsp_unit, &ShmCommBufs[r]);
+    ShmStencilBufs[r] = (void *) ((uint64_t)ShmCommBufs[r]+MAX_MPI_SHM_BYTES/4);
   }
   
   ////////////////////////////////////////////////////////////////
@@ -341,30 +343,54 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 						int from,
 						int bytes)
 {
+#undef SHM_USE_BCOPY
   MPI_Request xrq;
   MPI_Request rrq;
   
+  static int sequence;
+
   int rank = _processor;
   int ierr;
   int tag;
-  int small = (bytes<MAX_MPI_SHM_BYTES) || (shm_mode==0);
-  static int sequence;
   int check;
+
   assert(dest != _processor);
   assert(from != _processor);
   
   int gdest = GroupRanks[dest];
   int gme   = GroupRanks[_processor];
+
   sequence++;
   
+  char *to_ptr   = (char *)ShmCommBufs[gdest];
+  char *from_ptr = (char *)ShmCommBufs[ShmRank];
+
+  int small = (bytes<MAX_MPI_SHM_BYTES) || (shm_mode==0);
+
+#ifndef SHM_USE_BCOPY
+  int words = bytes>>2;
+  assert(((size_t)bytes &0x3)==0);
+  assert(((size_t)xmit  &0x3)==0);
+  assert(((size_t)recv  &0x3)==0);
+#endif
+
   assert(gme == ShmRank);
   
   if ( small && (dest !=MPI_UNDEFINED) ) {
-    char *ptr = (char *)ShmCommBufs[gdest];
     assert(gme != gdest);
-    GridThread::bcopy(xmit,ptr,bytes);
-    bcopy(&_processor,&ptr[bytes],sizeof(_processor));
-    bcopy(&  sequence,&ptr[bytes+4],sizeof(sequence));
+    float *ip = (float *)xmit;
+    float *op = (float *)to_ptr;
+
+#ifdef SHM_USE_BCOPY
+    bcopy(ip,op,bytes);
+#else
+    PARALLEL_FOR_LOOP 
+    for(int w=0;w<words;w++) {
+      op[w]=ip[w];
+    }
+#endif
+    bcopy(&_processor,&to_ptr[bytes],sizeof(_processor));
+    bcopy(&  sequence,&to_ptr[bytes+4],sizeof(sequence));
   } else { 
     ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
     assert(ierr==0);
@@ -376,10 +402,18 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
   MPI_Win_sync (ShmWindow);   
   
   if (small && (from !=MPI_UNDEFINED) ) {
-    char *ptr = (char *)ShmCommBufs[ShmRank];
-    GridThread::bcopy(ptr,recv,bytes);
-    bcopy(&ptr[bytes]  ,&tag  ,sizeof(tag));
-    bcopy(&ptr[bytes+4],&check,sizeof(check));
+    float *ip = (float *)from_ptr;
+    float *op = (float *)recv;
+#ifdef SHM_USE_BCOPY
+    bcopy(ip,op,bytes);
+#else
+    PARALLEL_FOR_LOOP 
+    for(int w=0;w<words;w++) {
+      op[w]=ip[w];
+    }
+#endif
+    bcopy(&from_ptr[bytes]  ,&tag  ,sizeof(tag));
+    bcopy(&from_ptr[bytes+4],&check,sizeof(check));
     assert(check==sequence);
     assert(tag==from);
   } else { 

From 2c54a53d0a3b473c62b21f0ccd500e43ba197dea Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 21 Oct 2016 12:12:14 +0100
Subject: [PATCH 076/149] Compile verbose reduce

---
 lib/Threads.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/Threads.h b/lib/Threads.h
index 70350685..08e5d545 100644
--- a/lib/Threads.h
+++ b/lib/Threads.h
@@ -31,7 +31,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #ifdef _OPENMP
 #define GRID_OMP
-#warning "OpenMP"
 #endif
 
 #define UNROLL  _Pragma("unroll")

From 63d219498ba4a036d74c8520e7455c94543cff49 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 21 Oct 2016 13:10:13 +0100
Subject: [PATCH 077/149] first (dirty) implementation of Feynman stoctachtic
 EM field

---
 lib/qcd/action/gauge/Photon.h | 344 ++++++++++++++++++++++------------
 1 file changed, 226 insertions(+), 118 deletions(-)

diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
index 29d40f8b..ea6f21b9 100644
--- a/lib/qcd/action/gauge/Photon.h
+++ b/lib/qcd/action/gauge/Photon.h
@@ -1,126 +1,234 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/qcd/action/gauge/Photon.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
+/*************************************************************************************
+ 
+ Grid physics library, www.github.com/paboyle/Grid
+ 
+ Source file: ./lib/qcd/action/gauge/Photon.h
+ 
+ Copyright (C) 2015
+ 
+ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+ 
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ 
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ 
+ See the full license in the file "LICENSE" in the top level distribution directory
+ *************************************************************************************/
+/*  END LEGAL */
 #ifndef QCD_PHOTON_ACTION_H
 #define QCD_PHOTON_ACTION_H
 
 namespace Grid{
-  namespace QCD{
+namespace QCD{
+  
+  template<class Gimpl>
+  class Photon
+  {
+  public:
+    INHERIT_GIMPL_TYPES(Gimpl);
+    enum class Gauge    {Feynman, Coulomb, Landau};
+    enum class ZmScheme {QedL, QedTL};
+  public:
+    Photon(Gauge gauge, ZmScheme zmScheme);
+    virtual ~Photon(void) = default;
+    void FreePropagator(const GaugeField &in, GaugeField &out);
+    void MomentumSpacePropagator(const GaugeField &in, GaugeField &out);
+    void StochasticField(GaugeField &out, GridParallelRNG &rng);
+  private:
+    void kHatSquared(GaugeLinkField &out);
+    void zmSub(GaugeLinkField &out);
+  private:
+    Gauge    gauge_;
+    ZmScheme zmScheme_;
+  };
 
-    template<class Gimpl>
-    class Photon  {
-      
-    public:
-
-      INHERIT_GIMPL_TYPES(Gimpl);
-
-      enum PhotonType { FEYNMAN_L, FEYNMAN_TL };
-
-      PhotonType GaugeBC;
-      
-      Photon(PhotonType _GaugeBC) : GaugeBC(_GaugeBC){};
-
-      void  FreePropagator (const GaugeField &in,GaugeField &out) {
-	FFT theFFT((GridCartesian *) in._grid);
-
-	GaugeField in_k(in._grid);
-	GaugeField prop_k(in._grid);
-
-	theFFT.FFT_all_dim(in_k,in,FFT::forward);
-        MomentumSpacePropagator(prop_k,in_k);
-	theFFT.FFT_all_dim(out,prop_k,FFT::backward);
-      }
-      void  MomentumSpacePropagator(GaugeField &out,const GaugeField &in) {
-	if ( GaugeBC == FEYNMAN_L ) { 
-	  FeynmanGaugeMomentumSpacePropagator_L(out,in);
-	} else if ( GaugeBC == FEYNMAN_TL ) { 
-	  FeynmanGaugeMomentumSpacePropagator_TL(out,in);
-	} else {  // No coulomb yet
-	  assert(0);
-	}
-      } 
-      void  FeynmanGaugeMomentumSpacePropagator_L(GaugeField &out, const GaugeField &in) { 
-
-	FeynmanGaugeMomentumSpacePropagator_TL(out,in);
-
-	GridBase *grid = out._grid;
-	LatticeInteger     coor(grid);
-	GaugeField zz(grid); zz=zero;
-
-	// xyzt
-	for(int d = 0; d < grid->_ndimension-1;d++){
-	  LatticeCoordinate(coor,d);
-	  out = where(coor==Integer(0),zz,out);
-	}
-      }
-
-      void  FeynmanGaugeMomentumSpacePropagator_TL(GaugeField &out, const GaugeField &in) { 
-
-	// what type LatticeComplex 
-	GridBase *grid = out._grid;
-	int nd = grid->_ndimension;
-
-	typedef typename GaugeField::vector_type vector_type;
-	typedef typename GaugeField::scalar_type ScalComplex;
-	typedef Lattice<iSinglet<vector_type> > LatComplex;
+  template<class Gimpl>
+  Photon<Gimpl>::Photon(Gauge gauge, ZmScheme zmScheme)
+  : gauge_(gauge), zmScheme_(zmScheme)
+  {}
+  
+  template<class Gimpl>
+  void Photon<Gimpl>::FreePropagator (const GaugeField &in,GaugeField &out)
+  {
+    FFT theFFT(in._grid);
     
-	std::vector<int> latt_size   = grid->_fdimensions;
-
-	LatComplex denom(grid); denom= zero;
-	LatComplex   one(grid); one = ScalComplex(1.0,0.0);
-	LatComplex   kmu(grid); 
-
-	ScalComplex ci(0.0,1.0);
-	// momphase = n * 2pi / L
-	for(int mu=0;mu<Nd;mu++) {
-	  
-	  LatticeCoordinate(kmu,mu);
-	  
-	  RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
-
-	  kmu = TwoPiL * kmu ;
-	  
-	  denom = denom + 4.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
-	}
-	std::vector<int> zero_mode(nd,0);
-	TComplexD Tone = ComplexD(1.0,0.0);
-	TComplexD Tzero= ComplexD(0.0,0.0);
-
-	pokeSite(Tone,denom,zero_mode); 
-
-	denom= one/denom;
-	
-	pokeSite(Tzero,denom,zero_mode); 
-
-	out = zero;
-	out = in*denom;
-      };
-
-    };
-
+    GaugeField in_k(in._grid);
+    GaugeField prop_k(in._grid);
+    
+    theFFT.FFT_all_dim(in_k,in,FFT::forward);
+    MomentumSpacePropagator(prop_k,in_k);
+    theFFT.FFT_all_dim(out,prop_k,FFT::backward);
+  }
+  
+  template<class Gimpl>
+  void Photon<Gimpl>::kHatSquared(GaugeLinkField &out)
+  {
+    GridBase           *grid = out._grid;
+    GaugeLinkField     kmu(grid);
+    const unsigned int nd    = grid->_ndimension;
+    std::vector<int>   &l    = grid->_fdimensions;
+    
+    out = zero;
+    for(int mu = 0; mu < nd; mu++)
+    {
+      RealD twoPiL = M_PI*2./l[mu];
+      
+      LatticeCoordinate(kmu,mu);
+      kmu = 2.*sin(.5*twoPiL*kmu);
+      out = out + kmu*kmu;
+    }
+  }
+  
+  template<class Gimpl>
+  void Photon<Gimpl>::zmSub(GaugeLinkField &out)
+  {
+    GridBase           *grid = out._grid;
+    const unsigned int nd    = grid->_ndimension;
+    
+    switch (zmScheme_)
+    {
+      case ZmScheme::QedTL:
+      {
+        std::vector<int> zm(nd,0);
+        TComplexD        Tzero = ComplexD(0.0,0.0);
+        
+        pokeSite(Tzero, out, zm);
+        
+        break;
+      }
+      case ZmScheme::QedL:
+      {
+        LatticeInteger spNrm(grid), coor(grid);
+        GaugeLinkField z(grid);
+        
+        spNrm = zero;
+        for(int d = 0; d < grid->_ndimension - 1; d++)
+        {
+          LatticeCoordinate(coor,d);
+          spNrm = spNrm + coor*coor;
+        }
+        out = where(spNrm == 0, 0.*out, out);
+        
+        break;
+      }
+      default:
+        break;
+    }
+  }
+  
+  template<class Gimpl>
+  void Photon<Gimpl>::MomentumSpacePropagator(const GaugeField &in,
+                                               GaugeField &out)
+  {
+    GridBase           *grid = out._grid;
+    const unsigned int nd    = grid->_ndimension;
+    LatticeComplex     k2Inv(grid), one(grid);
+    
+    one = ComplexD(1.0,0.0);
+    kHatSquared(k2Inv);
+    
+    std::vector<int> zm(nd,0);
+    TComplexD Tone = ComplexD(1.0,0.0);
+    TComplexD Tzero= ComplexD(0.0,0.0);
+    
+    pokeSite(Tone, k2Inv, zm);
+    k2Inv = one/k2Inv;
+    pokeSite(Tzero, k2Inv,zm);
+    
+    out = in*k2Inv;
+  }
+  
+  template<class Gimpl>
+  void Photon<Gimpl>::StochasticField(GaugeField &out, GridParallelRNG &rng)
+  {
+    auto               *grid = out._grid;
+    const unsigned int nd = grid->_ndimension;
+    GaugeLinkField     k2(grid), r(grid);
+    GaugeField         aTilde(grid);
+    FFT                fft(grid);
+    
+    kHatSquared(k2);
+    zmSub(k2);
+    for(int mu = 0; mu < nd; mu++)
+    {
+      gaussian(rng, r);
+      r = k2*r;
+      pokeLorentz(aTilde, r, mu);
+    }
+    fft.FFT_all_dim(out, aTilde, FFT::backward);
+  }
+//  template<class Gimpl>
+//  void Photon<Gimpl>::FeynmanGaugeMomentumSpacePropagator_L(GaugeField &out,
+//                                                            const GaugeField &in)
+//  {
+//    
+//    FeynmanGaugeMomentumSpacePropagator_TL(out,in);
+//    
+//    GridBase *grid = out._grid;
+//    LatticeInteger     coor(grid);
+//    GaugeField zz(grid); zz=zero;
+//    
+//    // xyzt
+//    for(int d = 0; d < grid->_ndimension-1;d++){
+//      LatticeCoordinate(coor,d);
+//      out = where(coor==Integer(0),zz,out);
+//    }
+//  }
+//  
+//  template<class Gimpl>
+//  void Photon<Gimpl>::FeynmanGaugeMomentumSpacePropagator_TL(GaugeField &out,
+//                                                             const GaugeField &in)
+//  {
+//    
+//    // what type LatticeComplex
+//    GridBase *grid = out._grid;
+//    int nd = grid->_ndimension;
+//    
+//    typedef typename GaugeField::vector_type vector_type;
+//    typedef typename GaugeField::scalar_type ScalComplex;
+//    typedef Lattice<iSinglet<vector_type> > LatComplex;
+//    
+//    std::vector<int> latt_size   = grid->_fdimensions;
+//    
+//    LatComplex denom(grid); denom= zero;
+//    LatComplex   one(grid); one = ScalComplex(1.0,0.0);
+//    LatComplex   kmu(grid);
+//    
+//    ScalComplex ci(0.0,1.0);
+//    // momphase = n * 2pi / L
+//    for(int mu=0;mu<Nd;mu++) {
+//      
+//      LatticeCoordinate(kmu,mu);
+//      
+//      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+//      
+//      kmu = TwoPiL * kmu ;
+//      
+//      denom = denom + 4.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
+//    }
+//    std::vector<int> zero_mode(nd,0);
+//    TComplexD Tone = ComplexD(1.0,0.0);
+//    TComplexD Tzero= ComplexD(0.0,0.0);
+//    
+//    pokeSite(Tone,denom,zero_mode);
+//    
+//    denom= one/denom;
+//    
+//    pokeSite(Tzero,denom,zero_mode);
+//    
+//    out = zero;
+//    out = in*denom;
+//  };
+  
 }}
 #endif

From f331809c2715cd4dc1b0e486c77d8552010a79de Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 13:35:37 +0100
Subject: [PATCH 078/149] Use variable type for loop

---
 lib/communicator/Communicator_mpi3.cc | 45 ++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 240aa09e..7d5ffff3 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -1,3 +1,4 @@
+
     /*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
@@ -181,6 +182,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
     ShmCommBuf = 0;
     ierr = MPI_Win_allocate_shared(MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,ShmComm,&ShmCommBuf,&ShmWindow);
     assert(ierr==0);
+    MPI_Win_lock_all (MPI_MODE_NOCHECK, ShmWindow);
 
     /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
     // Verbose for now
@@ -368,22 +370,24 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
   int small = (bytes<MAX_MPI_SHM_BYTES) || (shm_mode==0);
 
 #ifndef SHM_USE_BCOPY
-  int words = bytes>>2;
-  assert(((size_t)bytes &0x3)==0);
-  assert(((size_t)xmit  &0x3)==0);
-  assert(((size_t)recv  &0x3)==0);
+  typedef double T;
+  int words = bytes/sizeof(T);
+  assert(((size_t)bytes &(sizeof(T)-1))==0);
+  //  assert(((size_t)xmit  &(sizeof(T)-1))==0);
+  //  assert(((size_t)recv  &(sizeof(T)-1))==0);
 #endif
 
   assert(gme == ShmRank);
-  
+
+  //  std::cerr << "proc dest from gme  gdest "<<_processor<<" "<<dest <<" "<< from <<" "<<gme<<" "<< gdest<<std::endl; Barrier();
   if ( small && (dest !=MPI_UNDEFINED) ) {
     assert(gme != gdest);
-    float *ip = (float *)xmit;
-    float *op = (float *)to_ptr;
 
 #ifdef SHM_USE_BCOPY
-    bcopy(ip,op,bytes);
+    bcopy(xmit,to_ptr,bytes);
 #else
+    T *ip = (T *)xmit;
+    T *op = (T *)to_ptr;
     PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
       op[w]=ip[w];
@@ -397,16 +401,19 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
     list.push_back(xrq);
   }
   
+  //  std::cout << "Syncing "<<std::endl; Barrier();
   MPI_Win_sync (ShmWindow);   
   MPI_Barrier  (ShmComm);
   MPI_Win_sync (ShmWindow);   
+
+  //  std::cout << "Receiving "<<std::endl; Barrier();
   
   if (small && (from !=MPI_UNDEFINED) ) {
-    float *ip = (float *)from_ptr;
-    float *op = (float *)recv;
 #ifdef SHM_USE_BCOPY
-    bcopy(ip,op,bytes);
+    bcopy(from_ptr,recv,bytes);
 #else
+    T *ip = (T *)from_ptr;
+    T *op = (T *)recv;
     PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
       op[w]=ip[w];
@@ -422,9 +429,25 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
     list.push_back(rrq);
   }
   
+  //  std::cout << "Syncing"<<std::endl; Barrier();
+
   MPI_Win_sync (ShmWindow);   
   MPI_Barrier  (ShmComm);
   MPI_Win_sync (ShmWindow);   
+
+#if 0
+  MPI_Request xrq;
+  MPI_Request rrq;
+  int rank = _processor;
+  int ierr;
+  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+  
+  assert(ierr==0);
+
+  list.push_back(xrq);
+  list.push_back(rrq);
+#endif
 }
 
 void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)

From fad96cf2507a5a36bfe3fdd7f9dd9bb19be15543 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 13:36:00 +0100
Subject: [PATCH 079/149] StencilBufs

---
 lib/communicator/Communicator_base.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 6d6602be..f2a6bf00 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -80,6 +80,7 @@ class CartesianCommunicator {
 
     void * ShmCommBuf;
     std::vector<void *> ShmCommBufs;
+    std::vector<void *> ShmStencilBufs;
 
     int WorldRank;
     int WorldSize;

From 6a9eae6b6b5253fa6372f0a742110517e92b209a Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 13:36:18 +0100
Subject: [PATCH 080/149] Reporting improvements

---
 lib/qcd/action/fermion/WilsonFermion5D.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index a96b6fca..7c4fafff 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -194,7 +194,7 @@ void WilsonFermion5D<Impl>::Report(void)
     std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : "
               << DhopComputeTime / DhopCalls << " us" << std::endl;
 
-    RealD mflops = 1344*volume*DhopCalls/DhopComputeTime;
+    RealD mflops = 1344*volume*DhopCalls/DhopComputeTime/2; // 2 for red black counting
     std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
     std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
 
@@ -396,7 +396,6 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
            DoubledGaugeField & U,
            const FermionField &in, FermionField &out,int dag)
 {
-  DhopCalls++;
   //  assert((dag==DaggerNo) ||(dag==DaggerYes));
   Compressor compressor(dag);
 
@@ -454,6 +453,7 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopOE(const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls++;
   conformable(in._grid,FermionRedBlackGrid());    // verifies half grid
   conformable(in._grid,out._grid); // drops the cb check
 
@@ -465,6 +465,7 @@ void WilsonFermion5D<Impl>::DhopOE(const FermionField &in, FermionField &out,int
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls++;
   conformable(in._grid,FermionRedBlackGrid());    // verifies half grid
   conformable(in._grid,out._grid); // drops the cb check
 
@@ -476,6 +477,7 @@ void WilsonFermion5D<Impl>::DhopEO(const FermionField &in, FermionField &out,int
 template<class Impl>
 void WilsonFermion5D<Impl>::Dhop(const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls+=2;
   conformable(in._grid,FermionGrid()); // verifies full grid
   conformable(in._grid,out._grid);
 

From 462921e5491fbef99c93cfc460819890b0a63b66 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 21 Oct 2016 14:41:08 +0100
Subject: [PATCH 081/149] QED: fix stochastic field

---
 lib/qcd/action/gauge/Photon.h | 37 +++++++++++++++++------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
index ea6f21b9..aac72898 100644
--- a/lib/qcd/action/gauge/Photon.h
+++ b/lib/qcd/action/gauge/Photon.h
@@ -45,7 +45,7 @@ namespace QCD{
     void MomentumSpacePropagator(const GaugeField &in, GaugeField &out);
     void StochasticField(GaugeField &out, GridParallelRNG &rng);
   private:
-    void kHatSquared(GaugeLinkField &out);
+    void invKHatSquared(GaugeLinkField &out);
     void zmSub(GaugeLinkField &out);
   private:
     Gauge    gauge_;
@@ -71,13 +71,17 @@ namespace QCD{
   }
   
   template<class Gimpl>
-  void Photon<Gimpl>::kHatSquared(GaugeLinkField &out)
+  void Photon<Gimpl>::invKHatSquared(GaugeLinkField &out)
   {
     GridBase           *grid = out._grid;
-    GaugeLinkField     kmu(grid);
+    GaugeLinkField     kmu(grid), one(grid);
     const unsigned int nd    = grid->_ndimension;
     std::vector<int>   &l    = grid->_fdimensions;
+    std::vector<int>   zm(nd,0);
+    TComplexD          Tone = ComplexD(1.0,0.0);
+    TComplexD          Tzero= ComplexD(0.0,0.0);
     
+    one = ComplexD(1.0,0.0);
     out = zero;
     for(int mu = 0; mu < nd; mu++)
     {
@@ -87,6 +91,9 @@ namespace QCD{
       kmu = 2.*sin(.5*twoPiL*kmu);
       out = out + kmu*kmu;
     }
+    pokeSite(Tone, out, zm);
+    out = one/out;
+    pokeSite(Tzero, out,zm);
   }
   
   template<class Gimpl>
@@ -131,19 +138,10 @@ namespace QCD{
                                                GaugeField &out)
   {
     GridBase           *grid = out._grid;
-    const unsigned int nd    = grid->_ndimension;
-    LatticeComplex     k2Inv(grid), one(grid);
+    LatticeComplex     k2Inv(grid);
     
-    one = ComplexD(1.0,0.0);
-    kHatSquared(k2Inv);
-    
-    std::vector<int> zm(nd,0);
-    TComplexD Tone = ComplexD(1.0,0.0);
-    TComplexD Tzero= ComplexD(0.0,0.0);
-    
-    pokeSite(Tone, k2Inv, zm);
-    k2Inv = one/k2Inv;
-    pokeSite(Tzero, k2Inv,zm);
+    invKHatSquared(k2Inv);
+    zmSub(k2Inv);
     
     out = in*k2Inv;
   }
@@ -153,16 +151,17 @@ namespace QCD{
   {
     auto               *grid = out._grid;
     const unsigned int nd = grid->_ndimension;
-    GaugeLinkField     k2(grid), r(grid);
+    GaugeLinkField     sqrtK2Inv(grid), r(grid);
     GaugeField         aTilde(grid);
     FFT                fft(grid);
     
-    kHatSquared(k2);
-    zmSub(k2);
+    invKHatSquared(sqrtK2Inv);
+    sqrtK2Inv = sqrt(real(sqrtK2Inv));
+    zmSub(sqrtK2Inv);
     for(int mu = 0; mu < nd; mu++)
     {
       gaussian(rng, r);
-      r = k2*r;
+      r = sqrtK2Inv*r;
       pokeLorentz(aTilde, r, mu);
     }
     fft.FFT_all_dim(out, aTilde, FFT::backward);

From 09fd5c43a711900e101dc1e3d2a449b8a4f1a93c Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 15:17:39 +0100
Subject: [PATCH 082/149] Reasonably fast version

---
 lib/communicator/Communicator_mpi3.cc | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 7d5ffff3..5c7bb09b 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -28,6 +28,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #include "Grid.h"
 #include <mpi.h>
+#include <numaif.h>
 
 namespace Grid {
 
@@ -182,6 +183,15 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
     ShmCommBuf = 0;
     ierr = MPI_Win_allocate_shared(MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,ShmComm,&ShmCommBuf,&ShmWindow);
     assert(ierr==0);
+    for(uint64_t page=0;page<MAX_MPI_SHM_BYTES;page+=4096){
+      void *pages = (void *) ( page + ShmCommBuf );
+      int status;
+      int flags=MPOL_MF_MOVE_ALL;
+      int nodes=1; // numa domain == MCDRAM
+      unsigned long count=1;
+      ierr= move_pages(0,count, &pages,&nodes,&status,flags);
+      if (ierr && (page==0)) perror("numa relocate command failed");
+    }
     MPI_Win_lock_all (MPI_MODE_NOCHECK, ShmWindow);
 
     /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -367,10 +377,10 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
   char *to_ptr   = (char *)ShmCommBufs[gdest];
   char *from_ptr = (char *)ShmCommBufs[ShmRank];
 
-  int small = (bytes<MAX_MPI_SHM_BYTES) || (shm_mode==0);
+  int small = (bytes<MAX_MPI_SHM_BYTES);
 
 #ifndef SHM_USE_BCOPY
-  typedef double T;
+  typedef long long T __attribute__ ((__vector_size__ (16)));
   int words = bytes/sizeof(T);
   assert(((size_t)bytes &(sizeof(T)-1))==0);
   //  assert(((size_t)xmit  &(sizeof(T)-1))==0);

From 7c8f79b1479e9f15d1c82c27579f8741757a41f8 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 21 Oct 2016 15:20:12 +0100
Subject: [PATCH 083/149] more stochastic QED fixes

---
 lib/qcd/action/Actions.h      | 1 +
 lib/qcd/action/gauge/Photon.h | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/qcd/action/Actions.h b/lib/qcd/action/Actions.h
index ba6e577d..a23c6495 100644
--- a/lib/qcd/action/Actions.h
+++ b/lib/qcd/action/Actions.h
@@ -57,6 +57,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 ////////////////////////////////////////////
 // Gauge Actions
 ////////////////////////////////////////////
+#include <Grid/qcd/action/gauge/Photon.h>
 #include <Grid/qcd/action/gauge/WilsonGaugeAction.h>
 #include <Grid/qcd/action/gauge/PlaqPlusRectangleAction.h>
 
diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
index aac72898..0dd5a9dc 100644
--- a/lib/qcd/action/gauge/Photon.h
+++ b/lib/qcd/action/gauge/Photon.h
@@ -124,7 +124,7 @@ namespace QCD{
           LatticeCoordinate(coor,d);
           spNrm = spNrm + coor*coor;
         }
-        out = where(spNrm == 0, 0.*out, out);
+        out = where(spNrm == Integer(0), 0.*out, out);
         
         break;
       }
@@ -149,7 +149,7 @@ namespace QCD{
   template<class Gimpl>
   void Photon<Gimpl>::StochasticField(GaugeField &out, GridParallelRNG &rng)
   {
-    auto               *grid = out._grid;
+    auto               *grid = dynamic_cast<GridCartesian *>(out._grid);
     const unsigned int nd = grid->_ndimension;
     GaugeLinkField     sqrtK2Inv(grid), r(grid);
     GaugeField         aTilde(grid);

From 75ebd3a0d135fb58095362c7f6432c779f4745b2 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 22:34:29 +0100
Subject: [PATCH 084/149] Typo fixes and rotate for CLANG

---
 lib/simd/Grid_avx512.h    | 12 +++++++++++-
 lib/simd/Intel512avx.h    |  2 +-
 lib/simd/Intel512common.h |  2 +-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h
index 821898d9..45d6d988 100644
--- a/lib/simd/Grid_avx512.h
+++ b/lib/simd/Grid_avx512.h
@@ -41,6 +41,16 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid{
 namespace Optimization {
+
+  union u512f {
+    __m512 v;
+    float f[16];
+  };
+
+  union u512d {
+    __m512d v;
+    double f[8];
+  };
   
   struct Vsplat{
     //Complex float
@@ -361,7 +371,7 @@ namespace Optimization {
   // Some Template specialization
 
   // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases
-#undef GNU_CLANG_COMPILER 
+#define GNU_CLANG_COMPILER 
 #ifdef GNU_CLANG_COMPILER
   //Complex float Reduce
   template<>
diff --git a/lib/simd/Intel512avx.h b/lib/simd/Intel512avx.h
index cdd54a33..19157db4 100644
--- a/lib/simd/Intel512avx.h
+++ b/lib/simd/Intel512avx.h
@@ -53,7 +53,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #define ZMULMEM2SPd(O,P,tmp,B,C,Briir,Biirr,Criir,Ciirr)\
   VSHUFMEMd(O,P,tmp)  \
-  VMULMEMd(O,P,B,Biirr)  \ 
+  VMULMEMd(O,P,B,Biirr)  \
   VMULMEMd(O,P,C,Ciirr)  \
   VMULd(tmp,B,Briir)  \
   VMULd(tmp,C,Criir) 
diff --git a/lib/simd/Intel512common.h b/lib/simd/Intel512common.h
index dbfb30c2..cfa20c26 100644
--- a/lib/simd/Intel512common.h
+++ b/lib/simd/Intel512common.h
@@ -37,7 +37,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 // Opcodes common 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 #define MASK_REGS \
-  __asm__ ("mov     $0xAAAA, %%eax \n"\ 
+  __asm__ ("mov     $0xAAAA, %%eax \n"\
            "kmovw    %%eax, %%k6 \n"\
            "mov     $0x5555, %%eax \n"\
            "kmovw    %%eax, %%k7 \n" : : : "%eax");

From 910b8dd6a1d89443ed42306439161db09d05d81b Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 22:35:29 +0100
Subject: [PATCH 085/149] use simd type

---
 lib/communicator/Communicator_mpi3.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 5c7bb09b..be4e66a9 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -380,7 +380,7 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
   int small = (bytes<MAX_MPI_SHM_BYTES);
 
 #ifndef SHM_USE_BCOPY
-  typedef long long T __attribute__ ((__vector_size__ (16)));
+  typedef vRealD T;
   int words = bytes/sizeof(T);
   assert(((size_t)bytes &(sizeof(T)-1))==0);
   //  assert(((size_t)xmit  &(sizeof(T)-1))==0);
@@ -400,7 +400,7 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
     T *op = (T *)to_ptr;
     PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
-      op[w]=ip[w];
+      vstream(op[w],ip[w]);
     }
 #endif
     bcopy(&_processor,&to_ptr[bytes],sizeof(_processor));
@@ -426,7 +426,7 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
     T *op = (T *)recv;
     PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
-      op[w]=ip[w];
+      vstream(op[w],ip[w]);
     }
 #endif
     bcopy(&from_ptr[bytes]  ,&tag  ,sizeof(tag));

From 0fcd2e71882e555966a4d38ce9252128756cef13 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 21 Oct 2016 22:44:10 +0100
Subject: [PATCH 086/149] Simplify the comms structure prior to implementing
 Shared memory direct bouncs

---
 lib/Stencil.h | 168 +++++++++++++++-----------------------------------
 1 file changed, 49 insertions(+), 119 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index 65de9945..4850425b 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -70,20 +70,20 @@
 
  namespace Grid {
 
-template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,std::vector<std::pair<int,int> >& table)
+inline void Gather_plane_simple_table_compute (GridBase *grid,int dimension,int plane,int cbmask,
+					       int off,std::vector<std::pair<int,int> > & table)
 {
   table.resize(0);
-  int rd = rhs._grid->_rdimensions[dimension];
+  int rd = grid->_rdimensions[dimension];
 
-  if ( !rhs._grid->CheckerBoarded(dimension) ) {
+  if ( !grid->CheckerBoarded(dimension) ) {
     cbmask = 0x3;
   }
-  int so= plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
-  int e1=rhs._grid->_slice_nblock[dimension];
-  int e2=rhs._grid->_slice_block[dimension];
+  int so= plane*grid->_ostride[dimension]; // base offset for start of plane 
+  int e1=grid->_slice_nblock[dimension];
+  int e2=grid->_slice_block[dimension];
 
-  int stride=rhs._grid->_slice_stride[dimension];
+  int stride=grid->_slice_stride[dimension];
   if ( cbmask == 0x3 ) { 
     table.resize(e1*e2);
     for(int n=0;n<e1;n++){
@@ -99,7 +99,7 @@ Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,commVector<cobj> &bu
      for(int n=0;n<e1;n++){
        for(int b=0;b<e2;b++){
 	 int o  = n*stride;
-	 int ocb=1<<rhs._grid->CheckerBoardFromOindexTable(o+b);
+	 int ocb=1<<grid->CheckerBoardFromOindexTable(o+b);
 	 if ( ocb &cbmask ) {
 	   table[bo]=std::pair<int,int>(bo,o+b); bo++;
 	 }
@@ -109,8 +109,7 @@ Gather_plane_simple_table_compute (const Lattice<vobj> &rhs,commVector<cobj> &bu
 }
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice<vobj> &rhs,commVector<cobj> &buffer,
-			   compressor &compress, int off,int so)
+Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice<vobj> &rhs,cobj *buffer,compressor &compress, int off,int so)
 {
 PARALLEL_FOR_LOOP     
      for(int i=0;i<table.size();i++){
@@ -119,11 +118,11 @@ PARALLEL_FOR_LOOP
 }
 
 template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
+Gather_plane_simple_stencil (const Lattice<vobj> &rhs,cobj *buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
 			     double &t_table ,double & t_data )
 {
   std::vector<std::pair<int,int> > table;
-  Gather_plane_simple_table_compute (rhs, buffer,dimension,plane,cbmask,compress,off,table);
+  Gather_plane_simple_table_compute (rhs._grid,dimension,plane,cbmask,off,table);
   int so  = plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
   Gather_plane_simple_table         (table,rhs,buffer,compress,off,so);
 }
@@ -143,10 +142,11 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
    class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal fill in.
    public:
 
-       typedef uint32_t StencilInteger;
-       typedef typename cobj::vector_type vector_type;
-       typedef typename cobj::scalar_type scalar_type;
-       typedef typename cobj::scalar_object scalar_object;
+     typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
+     typedef uint32_t StencilInteger;
+     typedef typename cobj::vector_type vector_type;
+     typedef typename cobj::scalar_type scalar_type;
+     typedef typename cobj::scalar_object scalar_object;
 
        //////////////////////////////////////////
        // Comms packet queue for asynch thread
@@ -158,7 +158,6 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
 	 Integer to_rank;
 	 Integer from_rank;
 	 Integer bytes;
-	 volatile Integer done;
        };
 
        std::vector<Packet> Packets;
@@ -166,81 +165,39 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
        int face_table_computed;
        std::vector<std::vector<std::pair<int,int> > > face_table ;
        
-#define SEND_IMMEDIATE
-#define SERIAL_SENDS
-
        void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){
- #ifdef SEND_IMMEDIATE
-	 commtime-=usecond();
-	 _grid->SendToRecvFrom(xmit,to,rcv,from,bytes);
-	 commtime+=usecond();
- #endif
 	 Packet p;
 	 p.send_buf = xmit;
 	 p.recv_buf = rcv;
 	 p.to_rank  = to;
 	 p.from_rank= from;
 	 p.bytes    = bytes;
-	 p.done     = 0;
 	 comms_bytes+=2.0*bytes;
 	 Packets.push_back(p);
-
        }
 
- #ifdef SERIAL_SENDS
-       void Communicate(void ) { 
-	 commtime-=usecond();
+       void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
+       {
+	 reqs.resize(Packets.size());
+         commtime-=usecond();
 	 for(int i=0;i<Packets.size();i++){
- #ifndef SEND_IMMEDIATE
-	   _grid->SendToRecvFrom(
-				 Packets[i].send_buf,
-				 Packets[i].to_rank,
-				 Packets[i].recv_buf,
-				 Packets[i].from_rank,
-				 Packets[i].bytes);
- #endif
-	   Packets[i].done = 1;
+	   _grid->SendToRecvFromBegin(reqs[i],
+				      Packets[i].send_buf,
+				      Packets[i].to_rank,
+				      Packets[i].recv_buf,
+				      Packets[i].from_rank,
+				      Packets[i].bytes);
 	 }
 	 commtime+=usecond();
        }
- #else
-       void Communicate(void ) { 
-	 typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
-	 std::vector<std::vector<CommsRequest_t> > reqs(Packets.size());
-	 commtime-=usecond();
-	 const int concurrency=2;
-	 for(int i=0;i<Packets.size();i+=concurrency){
-	   for(int ii=0;ii<concurrency;ii++){
-	     int j = i+ii;
-	     if ( j<Packets.size() ) {
- #ifndef SEND_IMMEDIATE
-	       _grid->SendToRecvFromBegin(reqs[j],
-					  Packets[j].send_buf,
-					  Packets[j].to_rank,
-					  Packets[j].recv_buf,
-					  Packets[j].from_rank,
-					  Packets[j].bytes);
- #endif
-	     }
-	   }
-	   for(int ii=0;ii<concurrency;ii++){
-	     int j = i+ii;
-	     if ( j<Packets.size() ) {
- #ifndef SEND_IMMEDIATE
-	       _grid->SendToRecvFromComplete(reqs[i]);
- #endif
-	     }
-	   }
-	   for(int ii=0;ii<concurrency;ii++){
-	     int j = i+ii;
-	     if ( j<Packets.size() ) {
-	       Packets[j].done = 1;
-	     }
-	   }
+       void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs)
+       {
+         commtime-=usecond();
+	 for(int i=0;i<Packets.size();i++){
+	   _grid->SendToRecvFromComplete(reqs[i]);
 	 }
 	 commtime+=usecond();
        }
- #endif
 
        ///////////////////////////////////////////
        // Simd merge queue for asynch comms
@@ -260,36 +217,19 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
 	 m.rpointers= rpointers;
 	 m.buffer_size = buffer_size;
 	 m.packet_id   = packet_id;
- #ifdef SEND_IMMEDIATE
-	 mergetime-=usecond();
- PARALLEL_FOR_LOOP
-	 for(int o=0;o<m.buffer_size;o++){
-	   merge1(m.mpointer[o],m.rpointers,o);
-	 }
-	 mergetime+=usecond();
- #else
 	 Mergers.push_back(m);
- #endif
-
        }
 
        void CommsMerge(void ) { 
-	 //PARALLEL_NESTED_LOOP2 
+
 	 for(int i=0;i<Mergers.size();i++){	
 
-	 spintime-=usecond();
-	 int packet_id = Mergers[i].packet_id;
-	 while(! Packets[packet_id].done ); // spin for completion
-	 spintime+=usecond();
-
- #ifndef SEND_IMMEDIATE
 	 mergetime-=usecond();
- PARALLEL_FOR_LOOP
+PARALLEL_FOR_LOOP
 	   for(int o=0;o<Mergers[i].buffer_size;o++){
 	     merge1(Mergers[i].mpointer[o],Mergers[i].rpointers,o);
 	   }
 	 mergetime+=usecond();
- #endif
 
 	 }
        }
@@ -346,11 +286,14 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
 	 else       return cbase + _entries[ent]._byte_offset;
        }
 
+       ///////////////////////////////////////////////////////////
        // Comms buffers
+       ///////////////////////////////////////////////////////////
        std::vector<commVector<scalar_object> > u_simd_send_buf;
        std::vector<commVector<scalar_object> > u_simd_recv_buf;
        commVector<cobj>          u_send_buf;
        commVector<cobj>          comm_buf;
+
        int u_comm_offset;
        int _unified_buffer_size;
 
@@ -483,7 +426,7 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
 	 }
        }
        u_send_buf.resize(_unified_buffer_size);
-       comm_buf.resize(_unified_buffer_size);
+         comm_buf.resize(_unified_buffer_size);
 
        PrecomputeByteOffsets();
 
@@ -722,31 +665,16 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
        template<class compressor>
        void HaloExchange(const Lattice<vobj> &source,compressor &compress) 
        {
+	 std::vector<std::vector<CommsRequest_t> > reqs;
 	 calls++;
 	 Mergers.resize(0);
          Packets.resize(0);
          HaloGather(source,compress);
-	 this->Communicate();
+	 this->CommunicateBegin(reqs);
+	 this->CommunicateComplete(reqs);
 	 CommsMerge(); // spins
        }
-#if 0
-       // Overlapping comms and compute typically slows down compute and  is useless
-       // unless memory bandwidth greatly exceeds network
-       template<class compressor>
-       std::thread HaloExchangeBegin(const Lattice<vobj> &source,compressor &compress) {
-	 Mergers.resize(0); 
-	 Packets.resize(0);
-	 HaloGather(source,compress);
-	 return std::thread([&] { this->Communicate(); });
-       }
-       void HaloExchangeComplete(std::thread &thr) 
-       {
-	 CommsMerge(); // spins
-	 jointime-=usecond();
-	 thr.join();
-	 jointime+=usecond();
-       }
-#endif
+
        template<class compressor>
        void HaloGatherDir(const Lattice<vobj> &source,compressor &compress,int point,int & face_idx)
        {
@@ -851,6 +779,9 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
 	     int sx        = (x+sshift)%rd;
 	     int comm_proc = ((x+sshift)/rd)%pd;
 
+	     cobj *u_send_buf_p;
+	     cobj   *comm_buf_p;
+
 	     if (comm_proc) {
 
 	       int words = buffer_size;
@@ -863,16 +794,15 @@ Gather_plane_simple_stencil (const Lattice<vobj> &rhs,commVector<cobj> &buffer,i
 	       if ( !face_table_computed ) {
 		 t_table-=usecond();
 		 face_table.resize(face_idx+1);
-		 Gather_plane_simple_table_compute (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,face_table[face_idx]);
+		 cobj *ptr; ptr = &u_send_buf[0];
+		 Gather_plane_simple_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,
+						    face_table[face_idx]);
 		 t_table+=usecond();
 	       }
 	       t_data-=usecond();
-	       Gather_plane_simple_table         (face_table[face_idx],rhs,u_send_buf,compress,u_comm_offset,so);
-	       face_idx++;
+	       Gather_plane_simple_table         (face_table[face_idx],rhs,&u_send_buf[0],compress,u_comm_offset,so);  face_idx++;
 	       t_data+=usecond();
 	       gathertime+=usecond();
-	       
-	       //	       Gather_plane_simple_stencil (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset,t_table,t_data);
 
 	       int rank           = _grid->_processor;
 	       int recv_from_rank;

From c190221fd35eba678ec9810aa3b44816085d8283 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Sat, 22 Oct 2016 18:14:27 +0100
Subject: [PATCH 087/149] Internal SHM comms in non-simd directions working
 Need to fix simd directions

---
 benchmarks/Benchmark_dwf.cc                  |    8 +-
 lib/Stencil.h                                | 1609 +++++++++---------
 lib/communicator/Communicator_base.h         |   15 +-
 lib/communicator/Communicator_mpi.cc         |    8 +
 lib/communicator/Communicator_mpi3.cc        |  105 +-
 lib/communicator/Communicator_none.cc        |    8 +
 lib/communicator/Communicator_shmem.cc       |    8 +
 lib/qcd/action/fermion/FermionOperatorImpl.h |  857 +++++-----
 lib/qcd/action/fermion/WilsonFermion.cc      |    8 +-
 lib/qcd/action/fermion/WilsonFermion5D.cc    |  101 +-
 lib/qcd/action/fermion/WilsonFermion5D.h     |  289 ++--
 lib/qcd/action/fermion/WilsonKernels.cc      |   21 +-
 lib/qcd/action/fermion/WilsonKernels.h       |  275 ++-
 lib/qcd/action/fermion/WilsonKernelsAsm.cc   |   95 +-
 lib/qcd/action/fermion/WilsonKernelsHand.cc  |   55 +-
 tests/Test_stencil.cc                        |    6 +-
 16 files changed, 1729 insertions(+), 1739 deletions(-)

diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index a32d7535..f75f0385 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -153,7 +153,7 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
     std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
-    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
     Dw.Report();
@@ -192,7 +192,7 @@ int main (int argc, char ** argv)
 
     std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
-    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
     sDw.Report();
   
     if(0){
@@ -262,7 +262,7 @@ int main (int argc, char ** argv)
       double flops=(1344.0*volume*ncall)/2;
 
       std::cout<<GridLogMessage << "sDeo mflop/s =   "<< flops/(t1-t0)<<std::endl;
-      std::cout<<GridLogMessage << "sDeo mflop/s per node   "<< flops/(t1-t0)/NP<<std::endl;
+      std::cout<<GridLogMessage << "sDeo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
       sDw.Report();
 
       sDw.DhopEO(ssrc_o,sr_e,DaggerNo);
@@ -333,7 +333,7 @@ int main (int argc, char ** argv)
     double flops=(1344.0*volume*ncall)/2;
 
     std::cout<<GridLogMessage << "Deo mflop/s =   "<< flops/(t1-t0)<<std::endl;
-    std::cout<<GridLogMessage << "Deo mflop/s per node   "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "Deo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
     Dw.Report();
   }
   Dw.DhopEO(src_o,r_e,DaggerNo);
diff --git a/lib/Stencil.h b/lib/Stencil.h
index 4850425b..6b942539 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -32,6 +32,8 @@
 
  #include <Grid/stencil/Lebesgue.h>   // subdir aggregate
 
+const int ShmDirectCopy = 1;
+
  //////////////////////////////////////////////////////////////////////////////////////////
  // Must not lose sight that goal is to be able to construct really efficient
  // gather to a point stencil code. CSHIFT is not the best way, so need
@@ -68,7 +70,7 @@
  //
  //////////////////////////////////////////////////////////////////////////////////////////
 
- namespace Grid {
+namespace Grid {
 
 inline void Gather_plane_simple_table_compute (GridBase *grid,int dimension,int plane,int cbmask,
 					       int off,std::vector<std::pair<int,int> > & table)
@@ -117,822 +119,837 @@ PARALLEL_FOR_LOOP
      }
 }
 
-template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple_stencil (const Lattice<vobj> &rhs,cobj *buffer,int dimension,int plane,int cbmask,compressor &compress, int off,
-			     double &t_table ,double & t_data )
-{
-  std::vector<std::pair<int,int> > table;
-  Gather_plane_simple_table_compute (rhs._grid,dimension,plane,cbmask,off,table);
-  int so  = plane*rhs._grid->_ostride[dimension]; // base offset for start of plane 
-  Gather_plane_simple_table         (table,rhs,buffer,compress,off,so);
-}
+struct StencilEntry { 
+  uint64_t _offset;
+  uint64_t _byte_offset;
+  uint16_t _is_local;
+  uint16_t _permute;
+  uint32_t _around_the_world; //256 bits, 32 bytes, 1/2 cacheline
+};
 
+template<class vobj,class cobj>
+class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal fill in.
+ public:
 
+  typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
+  typedef uint32_t StencilInteger;
+  typedef typename cobj::vector_type vector_type;
+  typedef typename cobj::scalar_type scalar_type;
+  typedef typename cobj::scalar_object scalar_object;
+  
+  //////////////////////////////////////////
+  // Comms packet queue for asynch thread
+  //////////////////////////////////////////
 
+  struct Packet {
+    void * send_buf;
+    void * recv_buf;
+    Integer to_rank;
+    Integer from_rank;
+    Integer bytes;
+  };
 
-   struct StencilEntry { 
-     uint64_t _offset;
-     uint64_t _byte_offset;
-     uint16_t _is_local;
-     uint16_t _permute;
-     uint32_t _around_the_world; //256 bits, 32 bytes, 1/2 cacheline
-   };
+  std::vector<Packet> Packets;
 
-   template<class vobj,class cobj>
-   class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal fill in.
-   public:
+  int face_table_computed;
+  std::vector<std::vector<std::pair<int,int> > > face_table ;
+  
+  void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){
+    Packet p;
+    p.send_buf = xmit;
+    p.recv_buf = rcv;
+    p.to_rank  = to;
+    p.from_rank= from;
+    p.bytes    = bytes;
+    comms_bytes+=2.0*bytes;
+    Packets.push_back(p);
+  }
 
-     typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
-     typedef uint32_t StencilInteger;
-     typedef typename cobj::vector_type vector_type;
-     typedef typename cobj::scalar_type scalar_type;
-     typedef typename cobj::scalar_object scalar_object;
+  void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
+  {
+    reqs.resize(Packets.size());
+    commtime-=usecond();
+    for(int i=0;i<Packets.size();i++){
+      if( ShmDirectCopy ) {
+	_grid->StencilSendToRecvFromBegin(reqs[i],
+					  Packets[i].send_buf,
+					  Packets[i].to_rank,
+					  Packets[i].recv_buf,
+					  Packets[i].from_rank,
+					  Packets[i].bytes);
+      }else{
+	_grid->SendToRecvFromBegin(reqs[i],
+				   Packets[i].send_buf,
+				   Packets[i].to_rank,
+				   Packets[i].recv_buf,
+				   Packets[i].from_rank,
+				   Packets[i].bytes);
+      }
+    }
+    commtime+=usecond();
+  }
+  void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs)
+  {
+    commtime-=usecond();
+    for(int i=0;i<Packets.size();i++){
+      if( ShmDirectCopy ) 
+	_grid->StencilSendToRecvFromComplete(reqs[i]);
+      else 
+	_grid->SendToRecvFromComplete(reqs[i]);
+    }
+    commtime+=usecond();
+  }
 
-       //////////////////////////////////////////
-       // Comms packet queue for asynch thread
-       //////////////////////////////////////////
+  ///////////////////////////////////////////
+  // Simd merge queue for asynch comms
+  ///////////////////////////////////////////
+  struct Merge {
+    cobj * mpointer;
+    std::vector<scalar_object *> rpointers;
+    Integer buffer_size;
+    Integer packet_id;
+  };
+  
+  std::vector<Merge> Mergers;
 
-       struct Packet {
-	 void * send_buf;
-	 void * recv_buf;
-	 Integer to_rank;
-	 Integer from_rank;
-	 Integer bytes;
-       };
+  void AddMerge(cobj *merge_p,std::vector<scalar_object *> &rpointers,Integer buffer_size,Integer packet_id) {
+    Merge m;
+    m.mpointer = merge_p;
+    m.rpointers= rpointers;
+    m.buffer_size = buffer_size;
+    m.packet_id   = packet_id;
+    Mergers.push_back(m);
+  }
 
-       std::vector<Packet> Packets;
+  void CommsMerge(void ) { 
 
-       int face_table_computed;
-       std::vector<std::vector<std::pair<int,int> > > face_table ;
-       
-       void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){
-	 Packet p;
-	 p.send_buf = xmit;
-	 p.recv_buf = rcv;
-	 p.to_rank  = to;
-	 p.from_rank= from;
-	 p.bytes    = bytes;
-	 comms_bytes+=2.0*bytes;
-	 Packets.push_back(p);
-       }
-
-       void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
-       {
-	 reqs.resize(Packets.size());
-         commtime-=usecond();
-	 for(int i=0;i<Packets.size();i++){
-	   _grid->SendToRecvFromBegin(reqs[i],
-				      Packets[i].send_buf,
-				      Packets[i].to_rank,
-				      Packets[i].recv_buf,
-				      Packets[i].from_rank,
-				      Packets[i].bytes);
-	 }
-	 commtime+=usecond();
-       }
-       void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs)
-       {
-         commtime-=usecond();
-	 for(int i=0;i<Packets.size();i++){
-	   _grid->SendToRecvFromComplete(reqs[i]);
-	 }
-	 commtime+=usecond();
-       }
-
-       ///////////////////////////////////////////
-       // Simd merge queue for asynch comms
-       ///////////////////////////////////////////
-       struct Merge {
-	 cobj * mpointer;
-	 std::vector<scalar_object *> rpointers;
-	 Integer buffer_size;
-	 Integer packet_id;
-       };
-
-       std::vector<Merge> Mergers;
-
-       void AddMerge(cobj *merge_p,std::vector<scalar_object *> &rpointers,Integer buffer_size,Integer packet_id) {
-	 Merge m;
-	 m.mpointer = merge_p;
-	 m.rpointers= rpointers;
-	 m.buffer_size = buffer_size;
-	 m.packet_id   = packet_id;
-	 Mergers.push_back(m);
-       }
-
-       void CommsMerge(void ) { 
-
-	 for(int i=0;i<Mergers.size();i++){	
-
-	 mergetime-=usecond();
+    for(int i=0;i<Mergers.size();i++){	
+      
+      mergetime-=usecond();
 PARALLEL_FOR_LOOP
-	   for(int o=0;o<Mergers[i].buffer_size;o++){
-	     merge1(Mergers[i].mpointer[o],Mergers[i].rpointers,o);
-	   }
-	 mergetime+=usecond();
+      for(int o=0;o<Mergers[i].buffer_size;o++){
+	merge1(Mergers[i].mpointer[o],Mergers[i].rpointers,o);
+      }
+      mergetime+=usecond();
 
-	 }
-       }
+    }
+  }
 
-       ////////////////////////////////////////
-       // Basic Grid and stencil info
-       ////////////////////////////////////////
+  ////////////////////////////////////////
+  // Basic Grid and stencil info
+  ////////////////////////////////////////
+  
+  int                               _checkerboard;
+  int                               _npoints; // Move to template param?
+  GridBase *                        _grid;
+  
+  // npoints of these
+  std::vector<int>                  _directions;
+  std::vector<int>                  _distances;
+  std::vector<int>                  _comm_buf_size;
+  std::vector<int>                  _permute_type;
+  
+  // npoints x Osites() of these
+  // Flat vector, change layout for cache friendly.
+  Vector<StencilEntry>  _entries;
+  
+  inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; }
+  
+  void PrecomputeByteOffsets(void){
+    for(int i=0;i<_entries.size();i++){
+      if( _entries[i]._is_local ) {
+	_entries[i]._byte_offset = _entries[i]._offset*sizeof(vobj);
+      } else { 
+	_entries[i]._byte_offset = _entries[i]._offset*sizeof(cobj);
+      }
+    }
+  };
 
-       int                               _checkerboard;
-       int                               _npoints; // Move to template param?
-       GridBase *                        _grid;
+  inline uint64_t Touch(int ent) {
+    //	 _mm_prefetch((char *)&_entries[ent],_MM_HINT_T0);
+  }
+  inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
+    uint64_t cbase = (uint64_t)&u_recv_buf_p[0];
+    local = _entries[ent]._is_local;
+    perm  = _entries[ent]._permute;
+    if (perm)  ptype = _permute_type[point]; 
+    if (local) {
+      return  base + _entries[ent]._byte_offset;
+    } else {
+      return cbase + _entries[ent]._byte_offset;
+    }
+  }
+  inline uint64_t GetPFInfo(int ent,uint64_t base) {
+    uint64_t cbase = (uint64_t)&u_recv_buf_p[0];
+    int local = _entries[ent]._is_local;
+    if (local) return  base + _entries[ent]._byte_offset;
+    else       return cbase + _entries[ent]._byte_offset;
+  }
+  
+  ///////////////////////////////////////////////////////////
+  // Unified Comms buffers for all directions
+  ///////////////////////////////////////////////////////////
+  std::vector<commVector<scalar_object> > u_simd_send_buf;
+  std::vector<commVector<scalar_object> > u_simd_recv_buf;
+  commVector<cobj>          u_send_buf;
+  commVector<cobj>          u_recv_buf_hide;
+  cobj* u_recv_buf_p;
 
-       // npoints of these
-       std::vector<int>                  _directions;
-       std::vector<int>                  _distances;
-       std::vector<int>                  _comm_buf_size;
-       std::vector<int>                  _permute_type;
+  int u_comm_offset;
+  int _unified_buffer_size;
+  
+  cobj *CommBuf(void) { return u_recv_buf_p; }
 
-       // npoints x Osites() of these
-       // Flat vector, change layout for cache friendly.
-       Vector<StencilEntry>  _entries;
-
-       inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; }
-
-       void PrecomputeByteOffsets(void){
-	 for(int i=0;i<_entries.size();i++){
-	   if( _entries[i]._is_local ) {
-	     _entries[i]._byte_offset = _entries[i]._offset*sizeof(vobj);
-	   } else { 
-	     // PrecomputeByteOffsets [5] 16384/32768 140735768678528 140735781261056 2581581952
-	     _entries[i]._byte_offset = _entries[i]._offset*sizeof(cobj);
-	   }
-	 }
-       };
-
-       inline uint64_t Touch(int ent) {
-	 //	 _mm_prefetch((char *)&_entries[ent],_MM_HINT_T0);
-       }
-       inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
-	 uint64_t cbase = (uint64_t)&comm_buf[0];
-	 local = _entries[ent]._is_local;
-	 perm  = _entries[ent]._permute;
-	 if (perm)  ptype = _permute_type[point]; 
-	 if (local) {
-	   return  base + _entries[ent]._byte_offset;
-	 } else {
-	   return cbase + _entries[ent]._byte_offset;
-	 }
-       }
-       inline uint64_t GetPFInfo(int ent,uint64_t base) {
-	 uint64_t cbase = (uint64_t)&comm_buf[0];
-	 int local = _entries[ent]._is_local;
-	 if (local) return  base + _entries[ent]._byte_offset;
-	 else       return cbase + _entries[ent]._byte_offset;
-       }
-
-       ///////////////////////////////////////////////////////////
-       // Comms buffers
-       ///////////////////////////////////////////////////////////
-       std::vector<commVector<scalar_object> > u_simd_send_buf;
-       std::vector<commVector<scalar_object> > u_simd_recv_buf;
-       commVector<cobj>          u_send_buf;
-       commVector<cobj>          comm_buf;
-
-       int u_comm_offset;
-       int _unified_buffer_size;
-
-       /////////////////////////////////////////
-       // Timing info; ugly; possibly temporary
-       /////////////////////////////////////////
+  /////////////////////////////////////////
+  // Timing info; ugly; possibly temporary
+  /////////////////////////////////////////
  #define TIMING_HACK
  #ifdef TIMING_HACK
-       double jointime;
-       double gathertime;
-       double commtime;
-       double halogtime;
-       double mergetime;
-       double spintime;
-       double comms_bytes;
-       double gathermtime;
-       double splicetime;
-       double nosplicetime;
-       double t_data;
-       double t_table;
-       double calls;
-
-       void ZeroCounters(void) {
-         gathertime = 0.;
-         jointime = 0.;
-         commtime = 0.;
-         halogtime = 0.;
-         mergetime = 0.;
-         spintime = 0.;
-         gathermtime = 0.;
-         splicetime = 0.;
-         nosplicetime = 0.;
-	 t_data = 0.0;
-         t_table= 0.0;
-         comms_bytes = 0.;
-         calls = 0.;
-       };
-
-       void Report(void) {
+  double jointime;
+  double gathertime;
+  double commtime;
+  double halogtime;
+  double mergetime;
+  double spintime;
+  double comms_bytes;
+  double gathermtime;
+  double splicetime;
+  double nosplicetime;
+  double t_data;
+  double t_table;
+  double calls;
+  
+  void ZeroCounters(void) {
+    gathertime = 0.;
+    jointime = 0.;
+    commtime = 0.;
+    halogtime = 0.;
+    mergetime = 0.;
+    spintime = 0.;
+    gathermtime = 0.;
+    splicetime = 0.;
+    nosplicetime = 0.;
+    t_data = 0.0;
+    t_table= 0.0;
+    comms_bytes = 0.;
+    calls = 0.;
+  };
+  
+  void Report(void) {
 #define PRINTIT(A)	\
  std::cout << GridLogMessage << " Stencil " << #A << " "<< A/calls<<std::endl;
-	 if ( calls > 0. ) {
-	   std::cout << GridLogMessage << " Stencil calls "<<calls<<std::endl;
-	   PRINTIT(halogtime);
-	   PRINTIT(gathertime);
-	   PRINTIT(gathermtime);
-	   PRINTIT(mergetime);
-	   if(comms_bytes>1.0){
-	     PRINTIT(comms_bytes);
-	     PRINTIT(commtime);
-	     std::cout << GridLogMessage << " Stencil " << comms_bytes/commtime/1000. << " GB/s "<<std::endl;
-	   }
-	   PRINTIT(jointime);
-	   PRINTIT(spintime);
-	   PRINTIT(splicetime);
-	   PRINTIT(nosplicetime);
-	   PRINTIT(t_table);
+    if ( calls > 0. ) {
+      std::cout << GridLogMessage << " Stencil calls "<<calls<<std::endl;
+      PRINTIT(halogtime);
+      PRINTIT(gathertime);
+      PRINTIT(gathermtime);
+      PRINTIT(mergetime);
+      if(comms_bytes>1.0){
+	PRINTIT(comms_bytes);
+	PRINTIT(commtime);
+	std::cout << GridLogMessage << " Stencil " << comms_bytes/commtime/1000. << " GB/s "<<std::endl;
+      }
+      PRINTIT(jointime);
+      PRINTIT(spintime);
+      PRINTIT(splicetime);
+      PRINTIT(nosplicetime);
+      PRINTIT(t_table);
 	   PRINTIT(t_data);
-	 }
-       };
+    }
+  };
  #endif
 
-   CartesianStencil(GridBase *grid,
-				      int npoints,
-				      int checkerboard,
-				      const std::vector<int> &directions,
-				      const std::vector<int> &distances) 
-     :   _permute_type(npoints), _comm_buf_size(npoints)
-     {
-       face_table_computed=0;
-       _npoints = npoints;
-       _grid    = grid;
-       _directions = directions;
-       _distances  = distances;
-       _unified_buffer_size=0;
-
-       int osites  = _grid->oSites();
-
-       _entries.resize(_npoints* osites);
-       for(int ii=0;ii<npoints;ii++){
-
-	 int i = ii; // reverse direction to get SIMD comms done first
-	 int point = i;
-
-
-	 int dimension    = directions[i];
-	 int displacement = distances[i];
-	 int shift = displacement;
-
-	 int fd = _grid->_fdimensions[dimension];
-	 int rd = _grid->_rdimensions[dimension];
-	 _permute_type[point]=_grid->PermuteType(dimension);
-
-	 _checkerboard = checkerboard;
-
-	 // the permute type
-	 int simd_layout     = _grid->_simd_layout[dimension];
-	 int comm_dim        = _grid->_processors[dimension] >1 ;
-	 int splice_dim      = _grid->_simd_layout[dimension]>1 && (comm_dim);
-	 int rotate_dim      = _grid->_simd_layout[dimension]>2;
-
-	 assert ( (rotate_dim && comm_dim) == false) ; // Do not think spread out is supported
-
-	 int sshift[2];
-
-	 // Underlying approach. For each local site build
-	 // up a table containing the npoint "neighbours" and whether they 
-	 // live in lattice or a comms buffer.
-	 if ( !comm_dim ) {
-	   sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even);
-	   sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd);
-
-	   if ( sshift[0] == sshift[1] ) {
-	     Local(point,dimension,shift,0x3);
-	   } else {
-	     Local(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
-	     Local(point,dimension,shift,0x2);// both with block stride loop iteration
-	   }
-	 } else { // All permute extract done in comms phase prior to Stencil application
-	   //        So tables are the same whether comm_dim or splice_dim
-	   sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even);
-	   sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd);
-
-	   if ( sshift[0] == sshift[1] ) {
-	     Comms(point,dimension,shift,0x3);
-	   } else {
-	     Comms(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
-	     Comms(point,dimension,shift,0x2);// both with block stride loop iteration
-	   }
-	 }
-       }
-       u_send_buf.resize(_unified_buffer_size);
-         comm_buf.resize(_unified_buffer_size);
-
-       PrecomputeByteOffsets();
-
-       const int Nsimd = grid->Nsimd();
-       u_simd_send_buf.resize(Nsimd);
-       u_simd_recv_buf.resize(Nsimd);
-       for(int l=0;l<Nsimd;l++){
-	 u_simd_send_buf[l].resize(_unified_buffer_size);
-	 u_simd_recv_buf[l].resize(_unified_buffer_size);
-       }
-     }
-
-     void Local     (int point, int dimension,int shiftpm,int cbmask)
-     {
-       int fd = _grid->_fdimensions[dimension];
-       int rd = _grid->_rdimensions[dimension];
-       int ld = _grid->_ldimensions[dimension];
-       int gd = _grid->_gdimensions[dimension];
-       int ly = _grid->_simd_layout[dimension];
-
-       // Map to always positive shift modulo global full dimension.
-       int shift = (shiftpm+fd)%fd;
-
-       // the permute type
-       int permute_dim =_grid->PermuteDim(dimension);
-
-       for(int x=0;x<rd;x++){       
-
-	 int o   = 0;
-	 int bo  = x * _grid->_ostride[dimension];
-
-	 int cb= (cbmask==0x2)? Odd : Even;
-
-	 int sshift = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb);
-	 int sx     = (x+sshift)%rd;
-
-	 int wraparound=0;
-	 if ( (shiftpm==-1) && (sx>x)  ) {
-	   wraparound = 1;
-	 }
-	 if ( (shiftpm== 1) && (sx<x)  ) {
-	   wraparound = 1;
-	 }
-
-	 int permute_slice=0;
-	 if(permute_dim){
-	   int wrap = sshift/rd;
-	   int  num = sshift%rd;
-	   if ( x< rd-num ) permute_slice=wrap;
-	   else permute_slice = (wrap+1)%ly;
-	 }
-
-	 CopyPlane(point,dimension,x,sx,cbmask,permute_slice,wraparound);
-
-       }
-     }
-
-     void Comms     (int point,int dimension,int shiftpm,int cbmask)
-     {
-       GridBase *grid=_grid;
-       const int Nsimd = grid->Nsimd();
-
-       int fd              = _grid->_fdimensions[dimension];
-       int ld              = _grid->_ldimensions[dimension];
-       int rd              = _grid->_rdimensions[dimension];
-       int pd              = _grid->_processors[dimension];
-       int simd_layout     = _grid->_simd_layout[dimension];
-       int comm_dim        = _grid->_processors[dimension] >1 ;
-
-       assert(comm_dim==1);
-       int shift = (shiftpm + fd) %fd;
-       assert(shift>=0);
-       assert(shift<fd);
-
-       int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension]; // done in reduced dims, so SIMD factored
-
-       _comm_buf_size[point] = buffer_size; // Size of _one_ plane. Multiple planes may be gathered and
-					    // send to one or more remote nodes.
-
-       int cb= (cbmask==0x2)? Odd : Even;
-       int sshift= _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb);
-
-       for(int x=0;x<rd;x++){       
-
-	 int permute_type=grid->PermuteType(dimension);
-
-	 int sx        =  (x+sshift)%rd;
-
-	 int offnode = 0;
-	 if ( simd_layout > 1 ) {
-
-	   for(int i=0;i<Nsimd;i++){
-
-	     int inner_bit = (Nsimd>>(permute_type+1));
-	     int ic= (i&inner_bit)? 1:0;
-	     int my_coor          = rd*ic + x;
-	     int nbr_coor         = my_coor+sshift;
-	     int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
-
-	     if ( nbr_proc ) { 
-	       offnode =1;
-	     }
-	   }
-
-	 } else { 
-	   int comm_proc = ((x+sshift)/rd)%pd;
-	   offnode = (comm_proc!= 0);
-	 }
-
-
-	 int wraparound=0;
-	 if ( (shiftpm==-1) && (sx>x) && (grid->_processor_coor[dimension]==0) ) {
-	   wraparound = 1;
-	 }
-	 if ( (shiftpm== 1) && (sx<x) && (grid->_processor_coor[dimension]==grid->_processors[dimension]-1) ) {
-	   wraparound = 1;
-	 }
-	 if (!offnode) {
-
-	   int permute_slice=0;
-	   CopyPlane(point,dimension,x,sx,cbmask,permute_slice,wraparound); 
-
-	 } else {
-
-	   int words = buffer_size;
-	   if (cbmask != 0x3) words=words>>1;
-
-	   int rank           = grid->_processor;
-	   int recv_from_rank;
-	   int xmit_to_rank;
-
-	   int unified_buffer_offset = _unified_buffer_size;
-	   _unified_buffer_size    += words;
-
-	   ScatterPlane(point,dimension,x,cbmask,unified_buffer_offset,wraparound); // permute/extract/merge is done in comms phase
-
-	 }
-       }
-     }
-   // Routine builds up integer table for each site in _offsets, _is_local, _permute
-   void CopyPlane(int point, int dimension,int lplane,int rplane,int cbmask,int permute,int wrap)
-     {
-       int rd = _grid->_rdimensions[dimension];
-
-       if ( !_grid->CheckerBoarded(dimension) ) {
-
-	 int o   = 0;                                     // relative offset to base within plane
-	 int ro  = rplane*_grid->_ostride[dimension]; // base offset for start of plane 
-	 int lo  = lplane*_grid->_ostride[dimension]; // offset in buffer
-
-	 // Simple block stride gather of SIMD objects
-	 for(int n=0;n<_grid->_slice_nblock[dimension];n++){
-	   for(int b=0;b<_grid->_slice_block[dimension];b++){
-	     int idx=point+(lo+o+b)*_npoints;
-	     _entries[idx]._offset  =ro+o+b;
-	     _entries[idx]._permute=permute;
-	     _entries[idx]._is_local=1;
-	     _entries[idx]._around_the_world=wrap;
-	   }
-	   o +=_grid->_slice_stride[dimension];
-	 }
-
-       } else {
-
-	 int ro  = rplane*_grid->_ostride[dimension]; // base offset for start of plane 
-	 int lo  = lplane*_grid->_ostride[dimension]; // base offset for start of plane 
-	 int o   = 0;                                     // relative offset to base within plane
-
-	 for(int n=0;n<_grid->_slice_nblock[dimension];n++){
-	   for(int b=0;b<_grid->_slice_block[dimension];b++){
-
-	     int ocb=1<<_grid->CheckerBoardFromOindex(o+b);
-
-	     if ( ocb&cbmask ) {
-	       int idx = point+(lo+o+b)*_npoints;
-	       _entries[idx]._offset =ro+o+b;
-	       _entries[idx]._is_local=1;
-	       _entries[idx]._permute=permute;
-	       _entries[idx]._around_the_world=wrap;
-	     }
-
-	   }
-	   o +=_grid->_slice_stride[dimension];
-	 }
-
-       }
-     }
-   // Routine builds up integer table for each site in _offsets, _is_local, _permute
-    void ScatterPlane (int point,int dimension,int plane,int cbmask,int offset, int wrap)
-     {
-       int rd = _grid->_rdimensions[dimension];
-
-       if ( !_grid->CheckerBoarded(dimension) ) {
-
-	 int so  = plane*_grid->_ostride[dimension]; // base offset for start of plane 
-	 int o   = 0;                                    // relative offset to base within plane
-	 int bo  = 0;                                    // offset in buffer
-
-	 // Simple block stride gather of SIMD objects
-	 for(int n=0;n<_grid->_slice_nblock[dimension];n++){
-	   for(int b=0;b<_grid->_slice_block[dimension];b++){
-	     int idx=point+(so+o+b)*_npoints;
-	     _entries[idx]._offset  =offset+(bo++);
-	     _entries[idx]._is_local=0;
-	     _entries[idx]._permute=0;
-	     _entries[idx]._around_the_world=wrap;
-	   }
-	   o +=_grid->_slice_stride[dimension];
-	 }
-
-       } else { 
-
-	 int so  = plane*_grid->_ostride[dimension]; // base offset for start of plane 
-	 int o   = 0;                                      // relative offset to base within plane
-	 int bo  = 0;                                      // offset in buffer
-
-	 for(int n=0;n<_grid->_slice_nblock[dimension];n++){
-	   for(int b=0;b<_grid->_slice_block[dimension];b++){
-
-	     int ocb=1<<_grid->CheckerBoardFromOindex(o+b);// Could easily be a table lookup
-	     if ( ocb & cbmask ) {
-	       int idx = point+(so+o+b)*_npoints;
-	       _entries[idx]._offset  =offset+(bo++);
-	       _entries[idx]._is_local=0;
-	       _entries[idx]._permute =0;
-	       _entries[idx]._around_the_world=wrap;
-	     }
-	   }
-	   o +=_grid->_slice_stride[dimension];
-	 }
-       }
-     }
-
-
-
-       template<class compressor>
-       void HaloExchange(const Lattice<vobj> &source,compressor &compress) 
-       {
-	 std::vector<std::vector<CommsRequest_t> > reqs;
-	 calls++;
-	 Mergers.resize(0);
-         Packets.resize(0);
-         HaloGather(source,compress);
-	 this->CommunicateBegin(reqs);
-	 this->CommunicateComplete(reqs);
-	 CommsMerge(); // spins
-       }
-
-       template<class compressor>
-       void HaloGatherDir(const Lattice<vobj> &source,compressor &compress,int point,int & face_idx)
-       {
-	   int dimension    = _directions[point];
-	   int displacement = _distances[point];
-
-	   int fd = _grid->_fdimensions[dimension];
-	   int rd = _grid->_rdimensions[dimension];
-
-
-	   // Map to always positive shift modulo global full dimension.
-	   int shift = (displacement+fd)%fd;
-
-	   //     	  int checkerboard = _grid->CheckerBoardDestination(source.checkerboard,shift);
-	   assert (source.checkerboard== _checkerboard);
-
-	   // the permute type
-	   int simd_layout     = _grid->_simd_layout[dimension];
-	   int comm_dim        = _grid->_processors[dimension] >1 ;
-	   int splice_dim      = _grid->_simd_layout[dimension]>1 && (comm_dim);
-
-	   // Gather phase
-	   int sshift [2];
-	   if ( comm_dim ) {
-	     sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even);
-	     sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd);
-	     if ( sshift[0] == sshift[1] ) {
-	       if (splice_dim) {
-		 splicetime-=usecond();
-		 GatherSimd(source,dimension,shift,0x3,compress,face_idx);
-		 splicetime+=usecond();
-	       } else { 
-		 nosplicetime-=usecond();
-		 Gather(source,dimension,shift,0x3,compress,face_idx);
-		 nosplicetime+=usecond();
-	       }
-	     } else {
-	       if(splice_dim){
-		 splicetime-=usecond();
-		 GatherSimd(source,dimension,shift,0x1,compress,face_idx);// if checkerboard is unfavourable take two passes
-		 GatherSimd(source,dimension,shift,0x2,compress,face_idx);// both with block stride loop iteration
-		 splicetime+=usecond();
-	       } else {
-		 nosplicetime-=usecond();
-		 Gather(source,dimension,shift,0x1,compress,face_idx);
-		 Gather(source,dimension,shift,0x2,compress,face_idx);
-		 nosplicetime+=usecond();
-	       }
-	     }
-	   }
-       }
-
-       template<class compressor>
-       void HaloGather(const Lattice<vobj> &source,compressor &compress)
-       {
-	 // conformable(source._grid,_grid);
-	 assert(source._grid==_grid);
-	 halogtime-=usecond();
-
-	 assert (comm_buf.size() == _unified_buffer_size );
-	 u_comm_offset=0;
-
-	 // Gather all comms buffers
-	 int face_idx=0;
-	 for(int point = 0 ; point < _npoints; point++) {
-	   compress.Point(point);
-	   HaloGatherDir(source,compress,point,face_idx);
-	 }
-	 face_table_computed=1;
-
-	 assert(u_comm_offset==_unified_buffer_size);
-	 halogtime+=usecond();
-       }
-
-       template<class compressor>
-	 void Gather(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor & compress,int &face_idx)
-	 {
-	   typedef typename cobj::vector_type vector_type;
-	   typedef typename cobj::scalar_type scalar_type;
-
-	   GridBase *grid=_grid;
-	   assert(rhs._grid==_grid);
-	   //	  conformable(_grid,rhs._grid);
-
-	   int fd              = _grid->_fdimensions[dimension];
-	   int rd              = _grid->_rdimensions[dimension];
-	   int pd              = _grid->_processors[dimension];
-	   int simd_layout     = _grid->_simd_layout[dimension];
-	   int comm_dim        = _grid->_processors[dimension] >1 ;
-	   assert(simd_layout==1);
-	   assert(comm_dim==1);
-	   assert(shift>=0);
-	   assert(shift<fd);
-
-	   int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension];
-
-	   int cb= (cbmask==0x2)? Odd : Even;
-	   int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb);
-
-	   for(int x=0;x<rd;x++){       
-
-	     int sx        = (x+sshift)%rd;
-	     int comm_proc = ((x+sshift)/rd)%pd;
-
-	     cobj *u_send_buf_p;
-	     cobj   *comm_buf_p;
-
-	     if (comm_proc) {
-
-	       int words = buffer_size;
-	       if (cbmask != 0x3) words=words>>1;
-
-	       int bytes = words * sizeof(cobj);
-
-	       gathertime-=usecond();
-	       int so  = sx*rhs._grid->_ostride[dimension]; // base offset for start of plane 
-	       if ( !face_table_computed ) {
-		 t_table-=usecond();
-		 face_table.resize(face_idx+1);
-		 cobj *ptr; ptr = &u_send_buf[0];
-		 Gather_plane_simple_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,
-						    face_table[face_idx]);
-		 t_table+=usecond();
-	       }
-	       t_data-=usecond();
-	       Gather_plane_simple_table         (face_table[face_idx],rhs,&u_send_buf[0],compress,u_comm_offset,so);  face_idx++;
-	       t_data+=usecond();
-	       gathertime+=usecond();
-
-	       int rank           = _grid->_processor;
-	       int recv_from_rank;
-	       int xmit_to_rank;
-	       _grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
-	       assert (xmit_to_rank   != _grid->ThisRank());
-	       assert (recv_from_rank != _grid->ThisRank());
-
-	       //      FIXME Implement asynchronous send & also avoid buffer copy
-	       AddPacket((void *)&u_send_buf[u_comm_offset],
-			 (void *)  &comm_buf[u_comm_offset],
-			 xmit_to_rank,
-			 recv_from_rank,
-			 bytes);
-
-	       u_comm_offset+=words;
-	     }
-	   }
-	 }
-
-
-       template<class compressor>
-	 void  GatherSimd(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor &compress,int & face_idx)
-	 {
-	   const int Nsimd = _grid->Nsimd();
-
-	   int fd = _grid->_fdimensions[dimension];
-	   int rd = _grid->_rdimensions[dimension];
-	   int ld = _grid->_ldimensions[dimension];
-	   int pd              = _grid->_processors[dimension];
-	   int simd_layout     = _grid->_simd_layout[dimension];
-	   int comm_dim        = _grid->_processors[dimension] >1 ;
-
-	   assert(comm_dim==1);
-	   // This will not work with a rotate dim
-	   assert(simd_layout==2);
-	   assert(shift>=0);
-	   assert(shift<fd);
-
-	   int permute_type=_grid->PermuteType(dimension);
-
-	   ///////////////////////////////////////////////
-	   // Simd direction uses an extract/merge pair
-	   ///////////////////////////////////////////////
-	   int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension];
-	   int words = sizeof(cobj)/sizeof(vector_type);
-
-	   assert(cbmask==0x3); // Fixme think there is a latent bug if not true
-
-	   int bytes = buffer_size*sizeof(scalar_object);
-
-	   std::vector<scalar_object *> rpointers(Nsimd);
-	   std::vector<scalar_object *> spointers(Nsimd);
-
-	   ///////////////////////////////////////////
-	   // Work out what to send where
-	   ///////////////////////////////////////////
-
-	   int cb    = (cbmask==0x2)? Odd : Even;
-	   int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb);
-
-	   // loop over outer coord planes orthog to dim
-	   for(int x=0;x<rd;x++){       
-
-	     int any_offnode = ( ((x+sshift)%fd) >= rd );
-
-	     if ( any_offnode ) {
-
-	       for(int i=0;i<Nsimd;i++){       
-		 spointers[i] = &u_simd_send_buf[i][u_comm_offset];
-	       }
-
-	       int sx   = (x+sshift)%rd;
-
-	       gathermtime-=usecond();
-	       Gather_plane_extract<cobj>(rhs,spointers,dimension,sx,cbmask,compress);
-	       gathermtime+=usecond();
-
-	       for(int i=0;i<Nsimd;i++){
-
-		 // FIXME 
-		 // This logic is hard coded to simd_layout ==2 and not allowing >2
-		 //		std::cout << "GatherSimd : lane 1st elem " << i << u_simd_send_buf[i ][u_comm_offset]<<std::endl;
-
-		 int inner_bit = (Nsimd>>(permute_type+1));
-		 int ic= (i&inner_bit)? 1:0;
-
-		 int my_coor          = rd*ic + x;
-		 int nbr_coor         = my_coor+sshift;
-		 int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
-		 int nbr_lcoor= (nbr_coor%ld);
-		 int nbr_ic   = (nbr_lcoor)/rd;    // inner coord of peer
-		 int nbr_ox   = (nbr_lcoor%rd);    // outer coord of peer
-		 int nbr_lane = (i&(~inner_bit));
-
-		 if (nbr_ic) nbr_lane|=inner_bit;
-		 assert (sx == nbr_ox);
-
-		 auto rp = &u_simd_recv_buf[i       ][u_comm_offset];
-		 auto sp = &u_simd_send_buf[nbr_lane][u_comm_offset];
-
-		 void *vrp = (void *)rp;
-		 void *vsp = (void *)sp;
-
-
-		 if(nbr_proc){
-
-		   int recv_from_rank;
-		   int xmit_to_rank;
-
-		   _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
-
-		   AddPacket( vsp,vrp,xmit_to_rank,recv_from_rank,bytes);
-
-		   rpointers[i] = rp;
-
-		 } else { 
-
-		   rpointers[i] = sp;
-
-		 }
-	       }
-
-	       AddMerge(&comm_buf[u_comm_offset],rpointers,buffer_size,Packets.size()-1);
-
-	       u_comm_offset     +=buffer_size;
-	     }
-	   }
-	 }
-
-   };
- }
+ CartesianStencil(GridBase *grid,
+		  int npoints,
+		  int checkerboard,
+		  const std::vector<int> &directions,
+		  const std::vector<int> &distances) 
+   :   _permute_type(npoints), _comm_buf_size(npoints)
+  {
+    face_table_computed=0;
+    _npoints = npoints;
+    _grid    = grid;
+    _directions = directions;
+    _distances  = distances;
+    _unified_buffer_size=0;
+
+    int osites  = _grid->oSites();
+    
+    _entries.resize(_npoints* osites);
+    for(int ii=0;ii<npoints;ii++){
+      
+      int i = ii; // reverse direction to get SIMD comms done first
+      int point = i;
+      
+      int dimension    = directions[i];
+      int displacement = distances[i];
+      int shift = displacement;
+      
+      int fd = _grid->_fdimensions[dimension];
+      int rd = _grid->_rdimensions[dimension];
+      _permute_type[point]=_grid->PermuteType(dimension);
+      
+      _checkerboard = checkerboard;
+      
+      // the permute type
+      int simd_layout     = _grid->_simd_layout[dimension];
+      int comm_dim        = _grid->_processors[dimension] >1 ;
+      int splice_dim      = _grid->_simd_layout[dimension]>1 && (comm_dim);
+      int rotate_dim      = _grid->_simd_layout[dimension]>2;
+
+      assert ( (rotate_dim && comm_dim) == false) ; // Do not think spread out is supported
+      
+      int sshift[2];
+      
+      // Underlying approach. For each local site build
+      // up a table containing the npoint "neighbours" and whether they 
+      // live in lattice or a comms buffer.
+      if ( !comm_dim ) {
+	sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even);
+	sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd);
+	
+	if ( sshift[0] == sshift[1] ) {
+	  Local(point,dimension,shift,0x3);
+	} else {
+	  Local(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
+	  Local(point,dimension,shift,0x2);// both with block stride loop iteration
+	}
+      } else { // All permute extract done in comms phase prior to Stencil application
+	//        So tables are the same whether comm_dim or splice_dim
+	sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even);
+	sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd);
+	
+	if ( sshift[0] == sshift[1] ) {
+	  Comms(point,dimension,shift,0x3);
+	} else {
+	  Comms(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
+	  Comms(point,dimension,shift,0x2);// both with block stride loop iteration
+	}
+      }
+    }
+    u_send_buf.resize(_unified_buffer_size);
+
+    /////////////////////////////////////////////////////////////////////////////////
+    // Try to allocate for receiving in a shared memory region, fall back to buffer
+    /////////////////////////////////////////////////////////////////////////////////
+    if( ShmDirectCopy ) {
+
+      u_recv_buf_p=(cobj *)_grid->ShmBufferSelf();
+      if ( u_recv_buf_p == NULL ) {
+	u_recv_buf_hide.resize(_unified_buffer_size);
+	u_recv_buf_p=&u_recv_buf_hide[0];
+      }
+    } else {
+      u_recv_buf_hide.resize(_unified_buffer_size);
+      u_recv_buf_p=&u_recv_buf_hide[0];
+    }
+
+    PrecomputeByteOffsets();
+
+    const int Nsimd = grid->Nsimd();
+    u_simd_send_buf.resize(Nsimd);
+    u_simd_recv_buf.resize(Nsimd);
+    for(int l=0;l<Nsimd;l++){
+      u_simd_send_buf[l].resize(_unified_buffer_size);
+      u_simd_recv_buf[l].resize(_unified_buffer_size);
+    }
+  }
+
+  void Local     (int point, int dimension,int shiftpm,int cbmask)
+  {
+    int fd = _grid->_fdimensions[dimension];
+    int rd = _grid->_rdimensions[dimension];
+    int ld = _grid->_ldimensions[dimension];
+    int gd = _grid->_gdimensions[dimension];
+    int ly = _grid->_simd_layout[dimension];
+    
+    // Map to always positive shift modulo global full dimension.
+    int shift = (shiftpm+fd)%fd;
+
+    // the permute type
+    int permute_dim =_grid->PermuteDim(dimension);
+    
+    for(int x=0;x<rd;x++){       
+      
+      int o   = 0;
+      int bo  = x * _grid->_ostride[dimension];
+      
+      int cb= (cbmask==0x2)? Odd : Even;
+      
+      int sshift = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb);
+      int sx     = (x+sshift)%rd;
+      
+      int wraparound=0;
+      if ( (shiftpm==-1) && (sx>x)  ) {
+	wraparound = 1;
+      }
+      if ( (shiftpm== 1) && (sx<x)  ) {
+	wraparound = 1;
+      }
+      
+      int permute_slice=0;
+      if(permute_dim){
+	int wrap = sshift/rd;
+	int  num = sshift%rd;
+	if ( x< rd-num ) permute_slice=wrap;
+	else permute_slice = (wrap+1)%ly;
+      }
+      
+      CopyPlane(point,dimension,x,sx,cbmask,permute_slice,wraparound);
+      
+    }
+  }
+  
+  void Comms     (int point,int dimension,int shiftpm,int cbmask)
+  {
+    GridBase *grid=_grid;
+    const int Nsimd = grid->Nsimd();
+    
+    int fd              = _grid->_fdimensions[dimension];
+    int ld              = _grid->_ldimensions[dimension];
+    int rd              = _grid->_rdimensions[dimension];
+    int pd              = _grid->_processors[dimension];
+    int simd_layout     = _grid->_simd_layout[dimension];
+    int comm_dim        = _grid->_processors[dimension] >1 ;
+    
+    assert(comm_dim==1);
+    int shift = (shiftpm + fd) %fd;
+    assert(shift>=0);
+    assert(shift<fd);
+    
+    int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension]; // done in reduced dims, so SIMD factored
+    
+    _comm_buf_size[point] = buffer_size; // Size of _one_ plane. Multiple planes may be gathered and
+    // send to one or more remote nodes.
+    
+    int cb= (cbmask==0x2)? Odd : Even;
+    int sshift= _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb);
+    
+    for(int x=0;x<rd;x++){       
+      
+      int permute_type=grid->PermuteType(dimension);
+      
+      int sx        =  (x+sshift)%rd;
+      
+      int offnode = 0;
+      if ( simd_layout > 1 ) {
+	
+	for(int i=0;i<Nsimd;i++){
+	  
+	  int inner_bit = (Nsimd>>(permute_type+1));
+	  int ic= (i&inner_bit)? 1:0;
+	  int my_coor          = rd*ic + x;
+	  int nbr_coor         = my_coor+sshift;
+	  int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
+	  
+	  if ( nbr_proc ) { 
+	    offnode =1;
+	  }
+	}
+	
+      } else { 
+	int comm_proc = ((x+sshift)/rd)%pd;
+	offnode = (comm_proc!= 0);
+      }
+      
+      
+      int wraparound=0;
+      if ( (shiftpm==-1) && (sx>x) && (grid->_processor_coor[dimension]==0) ) {
+	wraparound = 1;
+      }
+      if ( (shiftpm== 1) && (sx<x) && (grid->_processor_coor[dimension]==grid->_processors[dimension]-1) ) {
+	wraparound = 1;
+      }
+      if (!offnode) {
+	
+	int permute_slice=0;
+	CopyPlane(point,dimension,x,sx,cbmask,permute_slice,wraparound); 
+	
+      } else {
+
+	int words = buffer_size;
+	if (cbmask != 0x3) words=words>>1;
+	
+	int rank           = grid->_processor;
+	int recv_from_rank;
+	int xmit_to_rank;
+	
+	int unified_buffer_offset = _unified_buffer_size;
+	_unified_buffer_size    += words;
+	
+	ScatterPlane(point,dimension,x,cbmask,unified_buffer_offset,wraparound); // permute/extract/merge is done in comms phase
+	
+      }
+    }
+  }
+  // Routine builds up integer table for each site in _offsets, _is_local, _permute
+  void CopyPlane(int point, int dimension,int lplane,int rplane,int cbmask,int permute,int wrap)
+  {
+    int rd = _grid->_rdimensions[dimension];
+    
+    if ( !_grid->CheckerBoarded(dimension) ) {
+      
+      int o   = 0;                                     // relative offset to base within plane
+      int ro  = rplane*_grid->_ostride[dimension]; // base offset for start of plane 
+      int lo  = lplane*_grid->_ostride[dimension]; // offset in buffer
+      
+      // Simple block stride gather of SIMD objects
+      for(int n=0;n<_grid->_slice_nblock[dimension];n++){
+	for(int b=0;b<_grid->_slice_block[dimension];b++){
+	  int idx=point+(lo+o+b)*_npoints;
+	  _entries[idx]._offset  =ro+o+b;
+	  _entries[idx]._permute=permute;
+	  _entries[idx]._is_local=1;
+	  _entries[idx]._around_the_world=wrap;
+	}
+	o +=_grid->_slice_stride[dimension];
+      }
+      
+    } else {
+      
+      int ro  = rplane*_grid->_ostride[dimension]; // base offset for start of plane 
+      int lo  = lplane*_grid->_ostride[dimension]; // base offset for start of plane 
+      int o   = 0;                                     // relative offset to base within plane
+      
+      for(int n=0;n<_grid->_slice_nblock[dimension];n++){
+	for(int b=0;b<_grid->_slice_block[dimension];b++){
+	  
+	  int ocb=1<<_grid->CheckerBoardFromOindex(o+b);
+	  
+	  if ( ocb&cbmask ) {
+	    int idx = point+(lo+o+b)*_npoints;
+	    _entries[idx]._offset =ro+o+b;
+	    _entries[idx]._is_local=1;
+	    _entries[idx]._permute=permute;
+	    _entries[idx]._around_the_world=wrap;
+	  }
+	  
+	}
+	o +=_grid->_slice_stride[dimension];
+      }
+      
+    }
+  }
+  // Routine builds up integer table for each site in _offsets, _is_local, _permute
+  void ScatterPlane (int point,int dimension,int plane,int cbmask,int offset, int wrap)
+  {
+    int rd = _grid->_rdimensions[dimension];
+    
+    if ( !_grid->CheckerBoarded(dimension) ) {
+      
+      int so  = plane*_grid->_ostride[dimension]; // base offset for start of plane 
+      int o   = 0;                                    // relative offset to base within plane
+      int bo  = 0;                                    // offset in buffer
+      
+      // Simple block stride gather of SIMD objects
+      for(int n=0;n<_grid->_slice_nblock[dimension];n++){
+	for(int b=0;b<_grid->_slice_block[dimension];b++){
+	  int idx=point+(so+o+b)*_npoints;
+	  _entries[idx]._offset  =offset+(bo++);
+	  _entries[idx]._is_local=0;
+	  _entries[idx]._permute=0;
+	  _entries[idx]._around_the_world=wrap;
+	}
+	o +=_grid->_slice_stride[dimension];
+      }
+      
+    } else { 
+      
+      int so  = plane*_grid->_ostride[dimension]; // base offset for start of plane 
+      int o   = 0;                                      // relative offset to base within plane
+      int bo  = 0;                                      // offset in buffer
+      
+      for(int n=0;n<_grid->_slice_nblock[dimension];n++){
+	for(int b=0;b<_grid->_slice_block[dimension];b++){
+	  
+	  int ocb=1<<_grid->CheckerBoardFromOindex(o+b);// Could easily be a table lookup
+	  if ( ocb & cbmask ) {
+	    int idx = point+(so+o+b)*_npoints;
+	    _entries[idx]._offset  =offset+(bo++);
+	    _entries[idx]._is_local=0;
+	    _entries[idx]._permute =0;
+	    _entries[idx]._around_the_world=wrap;
+	  }
+	}
+	o +=_grid->_slice_stride[dimension];
+      }
+    }
+  }
+  
+  template<class compressor> void HaloExchange(const Lattice<vobj> &source,compressor &compress) 
+  {
+    std::vector<std::vector<CommsRequest_t> > reqs;
+    calls++;
+    Mergers.resize(0);
+    Packets.resize(0);
+    HaloGather(source,compress);
+    this->CommunicateBegin(reqs);
+    this->CommunicateComplete(reqs);
+    CommsMerge(); // spins
+  }
+  
+  template<class compressor> void HaloGatherDir(const Lattice<vobj> &source,compressor &compress,int point,int & face_idx)
+  {
+    int dimension    = _directions[point];
+    int displacement = _distances[point];
+
+    int fd = _grid->_fdimensions[dimension];
+    int rd = _grid->_rdimensions[dimension];
+    
+    
+    // Map to always positive shift modulo global full dimension.
+    int shift = (displacement+fd)%fd;
+    
+    //     	  int checkerboard = _grid->CheckerBoardDestination(source.checkerboard,shift);
+    assert (source.checkerboard== _checkerboard);
+    
+    // the permute type
+    int simd_layout     = _grid->_simd_layout[dimension];
+    int comm_dim        = _grid->_processors[dimension] >1 ;
+    int splice_dim      = _grid->_simd_layout[dimension]>1 && (comm_dim);
+    
+    // Gather phase
+    int sshift [2];
+    if ( comm_dim ) {
+      sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even);
+      sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd);
+      if ( sshift[0] == sshift[1] ) {
+	if (splice_dim) {
+	  splicetime-=usecond();
+	  GatherSimd(source,dimension,shift,0x3,compress,face_idx);
+	  splicetime+=usecond();
+	} else { 
+	  nosplicetime-=usecond();
+	  Gather(source,dimension,shift,0x3,compress,face_idx);
+	  nosplicetime+=usecond();
+	}
+      } else {
+	if(splice_dim){
+	  splicetime-=usecond();
+	  GatherSimd(source,dimension,shift,0x1,compress,face_idx);// if checkerboard is unfavourable take two passes
+	  GatherSimd(source,dimension,shift,0x2,compress,face_idx);// both with block stride loop iteration
+	  splicetime+=usecond();
+	} else {
+	  nosplicetime-=usecond();
+	  Gather(source,dimension,shift,0x1,compress,face_idx);
+	  Gather(source,dimension,shift,0x2,compress,face_idx);
+	  nosplicetime+=usecond();
+	}
+      }
+    }
+  }
+  
+  template<class compressor>
+  void HaloGather(const Lattice<vobj> &source,compressor &compress)
+  {
+    // conformable(source._grid,_grid);
+    assert(source._grid==_grid);
+    halogtime-=usecond();
+    
+    u_comm_offset=0;
+    
+    // Gather all comms buffers
+    int face_idx=0;
+    for(int point = 0 ; point < _npoints; point++) {
+      compress.Point(point);
+      HaloGatherDir(source,compress,point,face_idx);
+    }
+    face_table_computed=1;
+    
+    assert(u_comm_offset==_unified_buffer_size);
+    halogtime+=usecond();
+  }
+  
+  template<class compressor>
+  void Gather(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor & compress,int &face_idx)
+  {
+    typedef typename cobj::vector_type vector_type;
+    typedef typename cobj::scalar_type scalar_type;
+    
+    GridBase *grid=_grid;
+    assert(rhs._grid==_grid);
+    //	  conformable(_grid,rhs._grid);
+    
+    int fd              = _grid->_fdimensions[dimension];
+    int rd              = _grid->_rdimensions[dimension];
+    int pd              = _grid->_processors[dimension];
+    int simd_layout     = _grid->_simd_layout[dimension];
+    int comm_dim        = _grid->_processors[dimension] >1 ;
+    assert(simd_layout==1);
+    assert(comm_dim==1);
+    assert(shift>=0);
+    assert(shift<fd);
+    
+    int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension];
+    
+    int cb= (cbmask==0x2)? Odd : Even;
+    int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb);
+    
+    for(int x=0;x<rd;x++){       
+      
+      int sx        = (x+sshift)%rd;
+      int comm_proc = ((x+sshift)/rd)%pd;
+      
+      if (comm_proc) {
+
+	int words = buffer_size;
+	if (cbmask != 0x3) words=words>>1;
+	
+	int bytes = words * sizeof(cobj);
+	
+	gathertime-=usecond();
+	int so  = sx*rhs._grid->_ostride[dimension]; // base offset for start of plane 
+	if ( !face_table_computed ) {
+	  t_table-=usecond();
+	  face_table.resize(face_idx+1);
+	  Gather_plane_simple_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,
+					     face_table[face_idx]);
+	  t_table+=usecond();
+	}
+	
+	
+	int rank           = _grid->_processor;
+	int recv_from_rank;
+	int xmit_to_rank;
+	_grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
+	
+	assert (xmit_to_rank   != _grid->ThisRank());
+	assert (recv_from_rank != _grid->ThisRank());
+	
+	/////////////////////////////////////////////////////////
+	// try the direct copy if possible
+	/////////////////////////////////////////////////////////
+
+	cobj *u_send_buf_p = &u_send_buf[0];
+	if (ShmDirectCopy) { 
+	  cobj *shm = (cobj *) _grid->ShmBuffer(xmit_to_rank);
+	  if ( shm!=NULL) { 
+	    u_send_buf_p = shm;
+	  }
+	}
+	
+	t_data-=usecond();
+	Gather_plane_simple_table         (face_table[face_idx],rhs,u_send_buf_p,compress,u_comm_offset,so);  face_idx++;
+	t_data+=usecond();
+	
+	AddPacket((void *)&u_send_buf_p[u_comm_offset],
+		  (void *)&u_recv_buf_p[u_comm_offset],
+		  xmit_to_rank,
+		  recv_from_rank,
+		  bytes);
+
+	gathertime+=usecond();
+	u_comm_offset+=words;
+      }
+    }
+  }
+  
+  template<class compressor>
+  void  GatherSimd(const Lattice<vobj> &rhs,int dimension,int shift,int cbmask,compressor &compress,int & face_idx)
+  {
+    const int Nsimd = _grid->Nsimd();
+
+    int fd = _grid->_fdimensions[dimension];
+    int rd = _grid->_rdimensions[dimension];
+    int ld = _grid->_ldimensions[dimension];
+    int pd              = _grid->_processors[dimension];
+    int simd_layout     = _grid->_simd_layout[dimension];
+    int comm_dim        = _grid->_processors[dimension] >1 ;
+    
+    assert(comm_dim==1);
+    // This will not work with a rotate dim
+    assert(simd_layout==2);
+    assert(shift>=0);
+    assert(shift<fd);
+    
+    int permute_type=_grid->PermuteType(dimension);
+    
+    ///////////////////////////////////////////////
+    // Simd direction uses an extract/merge pair
+    ///////////////////////////////////////////////
+    int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension];
+    int words = sizeof(cobj)/sizeof(vector_type);
+    
+    assert(cbmask==0x3); // Fixme think there is a latent bug if not true
+    
+    int bytes = buffer_size*sizeof(scalar_object);
+    
+    std::vector<scalar_object *> rpointers(Nsimd);
+    std::vector<scalar_object *> spointers(Nsimd);
+    
+    ///////////////////////////////////////////
+    // Work out what to send where
+    ///////////////////////////////////////////
+    
+    int cb    = (cbmask==0x2)? Odd : Even;
+    int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb);
+    
+    // loop over outer coord planes orthog to dim
+    for(int x=0;x<rd;x++){       
+      
+      int any_offnode = ( ((x+sshift)%fd) >= rd );
+      
+      if ( any_offnode ) {
+	
+	for(int i=0;i<Nsimd;i++){       
+	  spointers[i] = &u_simd_send_buf[i][u_comm_offset];
+	}
+	
+	int sx   = (x+sshift)%rd;
+	
+	gathermtime-=usecond();
+	Gather_plane_extract<cobj>(rhs,spointers,dimension,sx,cbmask,compress);
+	gathermtime+=usecond();
+
+	for(int i=0;i<Nsimd;i++){
+	  
+	  // FIXME 
+	  // This logic is hard coded to simd_layout ==2 and not allowing >2
+	  //		std::cout << "GatherSimd : lane 1st elem " << i << u_simd_send_buf[i ][u_comm_offset]<<std::endl;
+	  
+	  int inner_bit = (Nsimd>>(permute_type+1));
+	  int ic= (i&inner_bit)? 1:0;
+	  
+	  int my_coor          = rd*ic + x;
+	  int nbr_coor         = my_coor+sshift;
+	  int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
+	  int nbr_lcoor= (nbr_coor%ld);
+	  int nbr_ic   = (nbr_lcoor)/rd;    // inner coord of peer
+	  int nbr_ox   = (nbr_lcoor%rd);    // outer coord of peer
+	  int nbr_lane = (i&(~inner_bit));
+	  
+	  if (nbr_ic) nbr_lane|=inner_bit;
+	  assert (sx == nbr_ox);
+	  
+	  auto rp = &u_simd_recv_buf[i       ][u_comm_offset];
+	  auto sp = &u_simd_send_buf[nbr_lane][u_comm_offset];
+	  
+	  if(nbr_proc){
+	    
+	    int recv_from_rank;
+	    int xmit_to_rank;
+	    
+	    _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
+	    
+	    AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
+	    
+	    rpointers[i] = rp;
+	    
+	  } else { 
+	    
+	    rpointers[i] = sp;
+	    
+	  }
+	}
+
+	assert(0);
+	AddMerge(&u_recv_buf_p[u_comm_offset],rpointers,buffer_size,Packets.size()-1);
+	
+	u_comm_offset     +=buffer_size;
+      }
+    }
+  }
+  
+};
+}
 #endif
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index f2a6bf00..4139f72a 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -80,7 +80,6 @@ class CartesianCommunicator {
 
     void * ShmCommBuf;
     std::vector<void *> ShmCommBufs;
-    std::vector<void *> ShmStencilBufs;
 
     int WorldRank;
     int WorldSize;
@@ -105,6 +104,10 @@ class CartesianCommunicator {
     int  RankFromProcessorCoor(std::vector<int> &coor);
     void ProcessorCoorFromRank(int rank,std::vector<int> &coor);
 
+    // Helper function for SHM Windows in MPI3
+    void *ShmBufferSelf(void);
+    void *ShmBuffer(int rank);
+
     /////////////////////////////////
     // Grid information queries
     /////////////////////////////////
@@ -173,6 +176,16 @@ class CartesianCommunicator {
 			 int recv_from_rank,
 			 int bytes);
     void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
+    void StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+			 void *xmit,
+			 int xmit_to_rank,
+			 void *recv,
+			 int recv_from_rank,
+			 int bytes);
+    void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall)
+    {
+      SendToRecvFromComplete(waitall);
+    }
 
     ////////////////////////////////////////////////////////////
     // Barrier
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index 9c66202f..4291b319 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -67,6 +67,14 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   
   assert(Size==_Nprocessors);
 }
+void *CartesianCommunicator::ShmBufferSelf(void)
+{
+  return NULL;
+}
+void *CartesianCommunicator::ShmBuffer(int rank)
+{
+  return NULL;
+}
 
 void CartesianCommunicator::GlobalSum(uint32_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index be4e66a9..f5bbdbda 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -197,10 +197,10 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
     /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
     // Verbose for now
     /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    std::cout<< "Ranks per node "<< ShmSize << std::endl;
-    std::cout<< "Nodes          "<< GroupSize << std::endl;
-    std::cout<< "Ranks          "<< WorldSize << std::endl;
-    std::cout<< "Shm CommBuf "<< ShmCommBuf << std::endl;
+    std::cout<<GridLogMessage<< "MPI-3 configuration: Ranks per node "<< ShmSize ;
+    std::cout<< " Nodes "<< GroupSize;
+    std::cout<< " Ranks "<< WorldSize;
+    std::cout<< " Shm CommBuf address"<< std::hex <<ShmCommBuf << std::dec<<std::endl;
 
     // Done
     ShmSetup=1;
@@ -208,12 +208,10 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   }
 
   ShmCommBufs.resize(ShmSize);
-  ShmStencilBufs.resize(ShmSize);
   for(int r=0;r<ShmSize;r++){
     MPI_Aint sz;
     int dsp_unit;
     MPI_Win_shared_query (ShmWindow, r, &sz, &dsp_unit, &ShmCommBufs[r]);
-    ShmStencilBufs[r] = (void *) ((uint64_t)ShmCommBufs[r]+MAX_MPI_SHM_BYTES/4);
   }
   
   ////////////////////////////////////////////////////////////////
@@ -240,6 +238,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   ShmCoor.resize(_ndimension);
   GroupCoor.resize(_ndimension);
   WorldCoor.resize(_ndimension);
+
   for(int l2=0;l2<log2size;l2++){
     while ( WorldDims[dim] / ShmDims[dim] <= 1 ) dim=(dim+1)%_ndimension;
     ShmDims[dim]*=2;
@@ -347,6 +346,21 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
   }
 }
 
+
+void *CartesianCommunicator::ShmBufferSelf(void)
+{
+  return ShmCommBufs[ShmRank];
+}
+void *CartesianCommunicator::ShmBuffer(int rank)
+{
+  int gpeer = GroupRanks[rank];
+  if (gpeer == MPI_UNDEFINED){
+    return NULL;
+  } else { 
+    return ShmCommBufs[gpeer];
+  }
+}
+
 // Basic Halo comms primitive
 void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 						void *xmit,
@@ -355,13 +369,11 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 						int from,
 						int bytes)
 {
-#undef SHM_USE_BCOPY
   MPI_Request xrq;
   MPI_Request rrq;
   
   static int sequence;
 
-  int rank = _processor;
   int ierr;
   int tag;
   int check;
@@ -370,6 +382,7 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
   assert(from != _processor);
   
   int gdest = GroupRanks[dest];
+  int gfrom = GroupRanks[from];
   int gme   = GroupRanks[_processor];
 
   sequence++;
@@ -379,30 +392,23 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 
   int small = (bytes<MAX_MPI_SHM_BYTES);
 
-#ifndef SHM_USE_BCOPY
   typedef vRealD T;
   int words = bytes/sizeof(T);
-  assert(((size_t)bytes &(sizeof(T)-1))==0);
-  //  assert(((size_t)xmit  &(sizeof(T)-1))==0);
-  //  assert(((size_t)recv  &(sizeof(T)-1))==0);
-#endif
 
+  assert(((size_t)bytes &(sizeof(T)-1))==0);
   assert(gme == ShmRank);
 
-  //  std::cerr << "proc dest from gme  gdest "<<_processor<<" "<<dest <<" "<< from <<" "<<gme<<" "<< gdest<<std::endl; Barrier();
-  if ( small && (dest !=MPI_UNDEFINED) ) {
+  if ( small && (gdest !=MPI_UNDEFINED) ) {
+
     assert(gme != gdest);
 
-#ifdef SHM_USE_BCOPY
-    bcopy(xmit,to_ptr,bytes);
-#else
     T *ip = (T *)xmit;
     T *op = (T *)to_ptr;
-    PARALLEL_FOR_LOOP 
+PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
       vstream(op[w],ip[w]);
     }
-#endif
+
     bcopy(&_processor,&to_ptr[bytes],sizeof(_processor));
     bcopy(&  sequence,&to_ptr[bytes+4],sizeof(sequence));
   } else { 
@@ -411,24 +417,17 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
     list.push_back(xrq);
   }
   
-  //  std::cout << "Syncing "<<std::endl; Barrier();
   MPI_Win_sync (ShmWindow);   
   MPI_Barrier  (ShmComm);
   MPI_Win_sync (ShmWindow);   
 
-  //  std::cout << "Receiving "<<std::endl; Barrier();
-  
-  if (small && (from !=MPI_UNDEFINED) ) {
-#ifdef SHM_USE_BCOPY
-    bcopy(from_ptr,recv,bytes);
-#else
+  if (small && (gfrom !=MPI_UNDEFINED) ) {
     T *ip = (T *)from_ptr;
     T *op = (T *)recv;
-    PARALLEL_FOR_LOOP 
+PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
       vstream(op[w],ip[w]);
     }
-#endif
     bcopy(&from_ptr[bytes]  ,&tag  ,sizeof(tag));
     bcopy(&from_ptr[bytes+4],&check,sizeof(check));
     assert(check==sequence);
@@ -438,28 +437,52 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
     assert(ierr==0);
     list.push_back(rrq);
   }
-  
-  //  std::cout << "Syncing"<<std::endl; Barrier();
 
   MPI_Win_sync (ShmWindow);   
   MPI_Barrier  (ShmComm);
   MPI_Win_sync (ShmWindow);   
+}
 
-#if 0
+void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						       void *xmit,
+						       int dest,
+						       void *recv,
+						       int from,
+						       int bytes)
+{
   MPI_Request xrq;
   MPI_Request rrq;
-  int rank = _processor;
-  int ierr;
-  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
-  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
-  
-  assert(ierr==0);
 
-  list.push_back(xrq);
-  list.push_back(rrq);
-#endif
+  int ierr;
+
+  assert(dest != _processor);
+  assert(from != _processor);
+  
+  int gdest = GroupRanks[dest];
+  int gfrom = GroupRanks[from];
+  int gme   = GroupRanks[_processor];
+
+  assert(gme == ShmRank);
+
+  if ( gdest == MPI_UNDEFINED ) {
+    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+    assert(ierr==0);
+    list.push_back(xrq);
+  }
+  
+  if ( gfrom ==MPI_UNDEFINED) {
+    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+    assert(ierr==0);
+    list.push_back(rrq);
+  }
+
+  MPI_Win_sync (ShmWindow);   
+  MPI_Barrier  (ShmComm);
+  MPI_Win_sync (ShmWindow);   
+  
 }
 
+
 void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
 {
   int nreq=list.size();
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index 8601255a..80b8fb90 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -33,6 +33,14 @@ void CartesianCommunicator::Init(int *argc, char *** arv)
 }
 
 int Rank(void ){ return 0; };
+void *CartesianCommunicator::ShmBufferSelf(void)
+{
+  return NULL;
+}
+void *CartesianCommunicator::ShmBuffer(int rank)
+{
+  return NULL;
+}
 
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc
index 091e266e..4af719b0 100644
--- a/lib/communicator/Communicator_shmem.cc
+++ b/lib/communicator/Communicator_shmem.cc
@@ -50,6 +50,14 @@ typedef struct HandShake_t {
 static Vector< HandShake > XConnections;
 static Vector< HandShake > RConnections;
 
+void *CartesianCommunicator::ShmBufferSelf(void)
+{
+  return NULL;
+}
+void *CartesianCommunicator::ShmBuffer(int rank)
+{
+  return NULL;
+}
 void CartesianCommunicator::Init(int *argc, char ***argv) {
   shmem_init();
   XConnections.resize(shmem_n_pes());
diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h
index 4248ec4c..0800dea6 100644
--- a/lib/qcd/action/fermion/FermionOperatorImpl.h
+++ b/lib/qcd/action/fermion/FermionOperatorImpl.h
@@ -33,511 +33,500 @@ directory
 #define GRID_QCD_FERMION_OPERATOR_IMPL_H
 
 namespace Grid {
-
-  namespace QCD {
+namespace QCD {
 
 
-    //////////////////////////////////////////////
-    // Template parameter class constructs to package
-    // externally control Fermion implementations
-    // in orthogonal directions
-    //
-    // Ultimately need Impl to always define types where XXX is opaque
-    //
-    //    typedef typename XXX               Simd;
-    //    typedef typename XXX     GaugeLinkField;	
-    //    typedef typename XXX         GaugeField;
-    //    typedef typename XXX      GaugeActField;
-    //    typedef typename XXX       FermionField;
-    //    typedef typename XXX  DoubledGaugeField;
-    //    typedef typename XXX         SiteSpinor;
-    //    typedef typename XXX     SiteHalfSpinor;	
-    //    typedef typename XXX         Compressor;	
-    //
-    // and Methods:
-    //    void ImportGauge(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
-    //    void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
-    //    void multLink(SiteHalfSpinor &phi,const SiteDoubledGaugeField &U,const SiteHalfSpinor &chi,int mu,StencilEntry *SE,StencilImpl &St)
-    //    void InsertForce4D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu)
-    //    void InsertForce5D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu)
-    //
-    //
-    // To acquire the typedefs from "Base" (either a base class or template param) use:
-    //
-    // INHERIT_GIMPL_TYPES(Base)
-    // INHERIT_FIMPL_TYPES(Base)
-    // INHERIT_IMPL_TYPES(Base)
-    //
-    // The Fermion operators will do the following:
-    //
-    // struct MyOpParams { 
-    //   RealD mass;
-    // };
-    //
-    //
-    // template<class Impl>
-    // class MyOp : public<Impl> { 
-    // public:
-    //
-    //    INHERIT_ALL_IMPL_TYPES(Impl);
-    //
-    //    MyOp(MyOpParams Myparm, ImplParams &ImplParam) :  Impl(ImplParam)
-    //    {
-    //
-    //    };
-    //    
-    //  }
-    //////////////////////////////////////////////
-
-
-    ////////////////////////////////////////////////////////////////////////
-    // Implementation dependent fermion types
-    ////////////////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////
+  // Template parameter class constructs to package
+  // externally control Fermion implementations
+  // in orthogonal directions
+  //
+  // Ultimately need Impl to always define types where XXX is opaque
+  //
+  //    typedef typename XXX               Simd;
+  //    typedef typename XXX     GaugeLinkField;	
+  //    typedef typename XXX         GaugeField;
+  //    typedef typename XXX      GaugeActField;
+  //    typedef typename XXX       FermionField;
+  //    typedef typename XXX  DoubledGaugeField;
+  //    typedef typename XXX         SiteSpinor;
+  //    typedef typename XXX     SiteHalfSpinor;	
+  //    typedef typename XXX         Compressor;	
+  //
+  // and Methods:
+  //    void ImportGauge(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
+  //    void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
+  //    void multLink(SiteHalfSpinor &phi,const SiteDoubledGaugeField &U,const SiteHalfSpinor &chi,int mu,StencilEntry *SE,StencilImpl &St)
+  //    void InsertForce4D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu)
+  //    void InsertForce5D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu)
+  //
+  //
+  // To acquire the typedefs from "Base" (either a base class or template param) use:
+  //
+  // INHERIT_GIMPL_TYPES(Base)
+  // INHERIT_FIMPL_TYPES(Base)
+  // INHERIT_IMPL_TYPES(Base)
+  //
+  // The Fermion operators will do the following:
+  //
+  // struct MyOpParams { 
+  //   RealD mass;
+  // };
+  //
+  //
+  // template<class Impl>
+  // class MyOp : public<Impl> { 
+  // public:
+  //
+  //    INHERIT_ALL_IMPL_TYPES(Impl);
+  //
+  //    MyOp(MyOpParams Myparm, ImplParams &ImplParam) :  Impl(ImplParam)
+  //    {
+  //
+  //    };
+  //    
+  //  }
+  //////////////////////////////////////////////
+  
 
+  ////////////////////////////////////////////////////////////////////////
+  // Implementation dependent fermion types
+  ////////////////////////////////////////////////////////////////////////
+  
 #define INHERIT_FIMPL_TYPES(Impl)\
-    typedef typename Impl::FermionField           FermionField;		\
-    typedef typename Impl::DoubledGaugeField DoubledGaugeField;		\
-    typedef typename Impl::SiteSpinor               SiteSpinor;		\
-    typedef typename Impl::SiteHalfSpinor       SiteHalfSpinor;		\
-    typedef typename Impl::Compressor               Compressor;		\
-    typedef typename Impl::StencilImpl             StencilImpl;		\
-    typedef typename Impl::ImplParams ImplParams;			\
-    typedef typename Impl::Coeff_t       Coeff_t;
-
+  typedef typename Impl::FermionField           FermionField;		\
+  typedef typename Impl::DoubledGaugeField DoubledGaugeField;		\
+  typedef typename Impl::SiteSpinor               SiteSpinor;		\
+  typedef typename Impl::SiteHalfSpinor       SiteHalfSpinor;		\
+  typedef typename Impl::Compressor               Compressor;		\
+  typedef typename Impl::StencilImpl             StencilImpl;		\
+  typedef typename Impl::ImplParams ImplParams;				\
+  typedef typename Impl::Coeff_t       Coeff_t;
+  
 #define INHERIT_IMPL_TYPES(Base) \
-    INHERIT_GIMPL_TYPES(Base)	 \
-    INHERIT_FIMPL_TYPES(Base)
+  INHERIT_GIMPL_TYPES(Base)	 \
+  INHERIT_FIMPL_TYPES(Base)
+  
+  /////////////////////////////////////////////////////////////////////////////
+  // Single flavour four spinors with colour index
+  /////////////////////////////////////////////////////////////////////////////
+  template <class S, class Representation = FundamentalRepresentation,class _Coeff_t = RealD >
+  class WilsonImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
+
+    public:
+
+    static const int Dimension = Representation::Dimension;
+    typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
+      
+    //Necessary?
+    constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
     
-    ///////
-    // Single flavour four spinors with colour index
-    ///////
-    template <class S, class Representation = FundamentalRepresentation,class _Coeff_t = RealD >
-    class WilsonImpl
-      : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
-    public:
-      static const int Dimension = Representation::Dimension;
-      typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
-      
-      //Necessary?
-      constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
+    const bool LsVectorised=false;
+    typedef _Coeff_t Coeff_t;
 
-      const bool LsVectorised=false;
-      typedef _Coeff_t Coeff_t;
-
-
-      INHERIT_GIMPL_TYPES(Gimpl);
+    INHERIT_GIMPL_TYPES(Gimpl);
       
-      template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
-      template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
-      template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
+    template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
+    template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
+    template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
+    
+    typedef iImplSpinor<Simd>            SiteSpinor;
+    typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
+    typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
+    
+    typedef Lattice<SiteSpinor>            FermionField;
+    typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
+    
+    typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+    typedef WilsonImplParams ImplParams;
+    typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
+    
+    ImplParams Params;
+    
+    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
       
-      typedef iImplSpinor<Simd>            SiteSpinor;
-      typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
-      typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
+    bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
       
-      typedef Lattice<SiteSpinor>            FermionField;
-      typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
+    inline void multLink(SiteHalfSpinor &phi,
+			 const SiteDoubledGaugeField &U,
+			 const SiteHalfSpinor &chi,
+			 int mu,
+			 StencilEntry *SE,
+			 StencilImpl &St) {
+      mult(&phi(), &U(mu), &chi());
+    }
       
-      typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
-      typedef WilsonImplParams ImplParams;
-      typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
+    template <class ref>
+    inline void loadLinkElement(Simd &reg, ref &memory) {
+      reg = memory;
+    }
       
-      ImplParams Params;
-      
-      WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
-      
-      bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
-      
-      inline void multLink(SiteHalfSpinor &phi,
-			   const SiteDoubledGaugeField &U,
-			   const SiteHalfSpinor &chi,
-			   int mu,
-			   StencilEntry *SE,
-			   StencilImpl &St) {
-	mult(&phi(), &U(mu), &chi());
+    inline void DoubleStore(GridBase *GaugeGrid,
+			    DoubledGaugeField &Uds,
+			    const GaugeField &Umu) {
+      conformable(Uds._grid, GaugeGrid);
+      conformable(Umu._grid, GaugeGrid);
+      GaugeLinkField U(GaugeGrid);
+      for (int mu = 0; mu < Nd; mu++) {
+	U = PeekIndex<LorentzIndex>(Umu, mu);
+	PokeIndex<LorentzIndex>(Uds, U, mu);
+	U = adj(Cshift(U, mu, -1));
+	PokeIndex<LorentzIndex>(Uds, U, mu + 4);
       }
+    }
+
+    inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A,int mu){
+      GaugeLinkField link(mat._grid);
+      link = TraceIndex<SpinIndex>(outerProduct(Btilde,A)); 
+      PokeIndex<LorentzIndex>(mat,link,mu);
+    }   
       
-      template <class ref>
-      inline void loadLinkElement(Simd &reg,
-				  ref &memory) {
-	reg = memory;
-      }
+    inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde,int mu){
       
-      inline void DoubleStore(GridBase *GaugeGrid,
-			      DoubledGaugeField &Uds,
-			      const GaugeField &Umu) {
-	conformable(Uds._grid, GaugeGrid);
-	conformable(Umu._grid, GaugeGrid);
-	GaugeLinkField U(GaugeGrid);
-	for (int mu = 0; mu < Nd; mu++) {
-	  U = PeekIndex<LorentzIndex>(Umu, mu);
-	  PokeIndex<LorentzIndex>(Uds, U, mu);
-	  U = adj(Cshift(U, mu, -1));
-	  PokeIndex<LorentzIndex>(Uds, U, mu + 4);
+      int Ls=Btilde._grid->_fdimensions[0];
+      GaugeLinkField tmp(mat._grid);
+      tmp = zero;
+      
+      PARALLEL_FOR_LOOP
+      for(int sss=0;sss<tmp._grid->oSites();sss++){
+	int sU=sss;
+	for(int s=0;s<Ls;s++){
+	  int sF = s+Ls*sU;
+	  tmp[sU] = tmp[sU]+ traceIndex<SpinIndex>(outerProduct(Btilde[sF],Atilde[sF])); // ordering here
 	}
       }
-
-      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A,int mu){
-	GaugeLinkField link(mat._grid);
-	link = TraceIndex<SpinIndex>(outerProduct(Btilde,A)); 
-	PokeIndex<LorentzIndex>(mat,link,mu);
-      }   
+      PokeIndex<LorentzIndex>(mat,tmp,mu);
       
-      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde,int mu){
-	
-	int Ls=Btilde._grid->_fdimensions[0];
-	GaugeLinkField tmp(mat._grid);
-	tmp = zero;
+    }
+  };
 
-        PARALLEL_FOR_LOOP
-	  for(int sss=0;sss<tmp._grid->oSites();sss++){
-	    int sU=sss;
-	    for(int s=0;s<Ls;s++){
-	      int sF = s+Ls*sU;
-	      tmp[sU] = tmp[sU]+ traceIndex<SpinIndex>(outerProduct(Btilde[sF],Atilde[sF])); // ordering here
-	    }
-	  }
-	PokeIndex<LorentzIndex>(mat,tmp,mu);
+  ////////////////////////////////////////////////////////////////////////////////////
+  // Single flavour four spinors with colour index, 5d redblack
+  ////////////////////////////////////////////////////////////////////////////////////
 
+template<class S,int Nrepresentation=Nc,class _Coeff_t = RealD>
+class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { 
+  public:
+      
+  static const int Dimension = Nrepresentation;
+  const bool LsVectorised=true;
+  typedef _Coeff_t Coeff_t;      
+  typedef PeriodicGaugeImpl<GaugeImplTypes<S, Nrepresentation> > Gimpl;
+  
+  INHERIT_GIMPL_TYPES(Gimpl);
+  
+  template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Nrepresentation>, Ns> >;
+  template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhs> >;
+  template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>;
+  template <typename vtype> using iImplGaugeField        = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nd>;
+  template <typename vtype> using iImplGaugeLink         = iScalar<iScalar<iMatrix<vtype, Nrepresentation> > >;
+  
+  typedef iImplSpinor<Simd> SiteSpinor;
+  typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
+  typedef Lattice<SiteSpinor> FermionField;
+  
+  // Make the doubled gauge field a *scalar*
+  typedef iImplDoubledGaugeField<typename Simd::scalar_type>  SiteDoubledGaugeField;  // This is a scalar
+  typedef iImplGaugeField<typename Simd::scalar_type>         SiteScalarGaugeField;  // scalar
+  typedef iImplGaugeLink<typename Simd::scalar_type>          SiteScalarGaugeLink;  // scalar
+      
+  typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
+      
+  typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+  typedef WilsonImplParams ImplParams;
+  typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
+  
+  ImplParams Params;
+  
+  DomainWallVec5dImpl(const ImplParams &p = ImplParams()) : Params(p){};
+      
+  bool overlapCommsCompute(void) { return false; };
+      
+  template <class ref>
+  inline void loadLinkElement(Simd &reg, ref &memory) {
+    vsplat(reg, memory);
+  }
+
+  inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
+		       const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
+		       StencilImpl &St) {
+    SiteGaugeLink UU;
+    for (int i = 0; i < Nrepresentation; i++) {
+      for (int j = 0; j < Nrepresentation; j++) {
+	vsplat(UU()()(i, j), U(mu)()(i, j));
       }
-    };
-
-    ///////
-    // Single flavour four spinors with colour index, 5d redblack
-    ///////
-    template<class S,int Nrepresentation=Nc,class _Coeff_t = RealD>
-    class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { 
-    public:
+    }
+    mult(&phi(), &UU(), &chi());
+  }
       
-      static const int Dimension = Nrepresentation;
-      const bool LsVectorised=true;
-      typedef _Coeff_t Coeff_t;      
-      typedef PeriodicGaugeImpl<GaugeImplTypes<S, Nrepresentation> > Gimpl;
-
-      INHERIT_GIMPL_TYPES(Gimpl);
+  inline void DoubleStore(GridBase *GaugeGrid, DoubledGaugeField &Uds,const GaugeField &Umu) 
+  {
+    SiteScalarGaugeField ScalarUmu;
+    SiteDoubledGaugeField ScalarUds;
+    
+    GaugeLinkField U(Umu._grid);
+    GaugeField Uadj(Umu._grid);
+    for (int mu = 0; mu < Nd; mu++) {
+      U = PeekIndex<LorentzIndex>(Umu, mu);
+      U = adj(Cshift(U, mu, -1));
+      PokeIndex<LorentzIndex>(Uadj, U, mu);
+    }
+    
+    for (int lidx = 0; lidx < GaugeGrid->lSites(); lidx++) {
+      std::vector<int> lcoor;
+      GaugeGrid->LocalIndexToLocalCoor(lidx, lcoor);
       
-      template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Nrepresentation>, Ns> >;
-      template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhs> >;
-      template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>;
-      template <typename vtype> using iImplGaugeField        = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nd>;
-      template <typename vtype> using iImplGaugeLink         = iScalar<iScalar<iMatrix<vtype, Nrepresentation> > >;
+      peekLocalSite(ScalarUmu, Umu, lcoor);
+      for (int mu = 0; mu < 4; mu++) ScalarUds(mu) = ScalarUmu(mu);
       
-      typedef iImplSpinor<Simd> SiteSpinor;
-      typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
-      typedef Lattice<SiteSpinor> FermionField;
+      peekLocalSite(ScalarUmu, Uadj, lcoor);
+      for (int mu = 0; mu < 4; mu++) ScalarUds(mu + 4) = ScalarUmu(mu);
       
-      // Make the doubled gauge field a *scalar*
-      typedef iImplDoubledGaugeField<typename Simd::scalar_type>
-      SiteDoubledGaugeField;  // This is a scalar
-      typedef iImplGaugeField<typename Simd::scalar_type>
-      SiteScalarGaugeField;  // scalar
-      typedef iImplGaugeLink<typename Simd::scalar_type>
-      SiteScalarGaugeLink;  // scalar
+      pokeLocalSite(ScalarUds, Uds, lcoor);
+    }
+  }
       
-      typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
+  inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,FermionField &A, int mu) 
+  {
+    assert(0);
+  }
       
-      typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
-      typedef WilsonImplParams ImplParams;
-      typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
-      
-      ImplParams Params;
-      
-      DomainWallVec5dImpl(const ImplParams &p = ImplParams()) : Params(p){};
-      
-      bool overlapCommsCompute(void) { return false; };
-      
-      template <class ref>
-      inline void loadLinkElement(Simd &reg, ref &memory) {
-	vsplat(reg, memory);
-      }
-      inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
-			   const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
-			   StencilImpl &St) {
-	SiteGaugeLink UU;
-	for (int i = 0; i < Nrepresentation; i++) {
-	  for (int j = 0; j < Nrepresentation; j++) {
-	    vsplat(UU()()(i, j), U(mu)()(i, j));
-	  }
-	}
-	mult(&phi(), &UU(), &chi());
-      }
-      
-      inline void DoubleStore(GridBase *GaugeGrid, DoubledGaugeField &Uds,
-			      const GaugeField &Umu) {
-	SiteScalarGaugeField ScalarUmu;
-	SiteDoubledGaugeField ScalarUds;
-	
-	GaugeLinkField U(Umu._grid);
-	GaugeField Uadj(Umu._grid);
-	for (int mu = 0; mu < Nd; mu++) {
-	  U = PeekIndex<LorentzIndex>(Umu, mu);
-	  U = adj(Cshift(U, mu, -1));
-	  PokeIndex<LorentzIndex>(Uadj, U, mu);
-	}
-	
-	for (int lidx = 0; lidx < GaugeGrid->lSites(); lidx++) {
-	  std::vector<int> lcoor;
-	  GaugeGrid->LocalIndexToLocalCoor(lidx, lcoor);
-	  
-	  peekLocalSite(ScalarUmu, Umu, lcoor);
-	  for (int mu = 0; mu < 4; mu++) ScalarUds(mu) = ScalarUmu(mu);
-	  
-	  peekLocalSite(ScalarUmu, Uadj, lcoor);
-	  for (int mu = 0; mu < 4; mu++) ScalarUds(mu + 4) = ScalarUmu(mu);
-	  
-	  pokeLocalSite(ScalarUds, Uds, lcoor);
-	}
-      }
-      
-      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,
-				FermionField &A, int mu) {
+  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,FermionField &Atilde, int mu) 
+  {
 	assert(0);
-      }
-      
-      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,
-				FermionField &Atilde, int mu) {
-	assert(0);
-      }
-    };
+  }
+};
     
     ////////////////////////////////////////////////////////////////////////////////////////
     // Flavour doubled spinors; is Gparity the only? what about C*?
     ////////////////////////////////////////////////////////////////////////////////////////
     
-    template <class S, int Nrepresentation,class _Coeff_t = RealD>
-    class GparityWilsonImpl
-      : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
-    public:
-      static const int Dimension = Nrepresentation;
+template <class S, int Nrepresentation,class _Coeff_t = RealD>
+class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
+ public:
 
-      const bool LsVectorised=false;
+ static const int Dimension = Nrepresentation;
 
-      typedef _Coeff_t Coeff_t;
-      typedef ConjugateGaugeImpl< GaugeImplTypes<S,Nrepresentation> > Gimpl;
+ const bool LsVectorised=false;
+
+ typedef _Coeff_t Coeff_t;
+ typedef ConjugateGaugeImpl< GaugeImplTypes<S,Nrepresentation> > Gimpl;
+ 
+ INHERIT_GIMPL_TYPES(Gimpl);
       
-      INHERIT_GIMPL_TYPES(Gimpl);
+ template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
+ template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
+ template <typename vtype> using iImplDoubledGaugeField = iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
       
-      template <typename vtype>
-      using iImplSpinor =
-      iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
-      template <typename vtype>
-      using iImplHalfSpinor =
-	iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
-      template <typename vtype>
-      using iImplDoubledGaugeField =
-	iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
+ typedef iImplSpinor<Simd> SiteSpinor;
+ typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
+ typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
+ 
+ typedef Lattice<SiteSpinor> FermionField;
+ typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
+ 
+ typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+ typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
+ 
+ typedef GparityWilsonImplParams ImplParams;
       
-      typedef iImplSpinor<Simd> SiteSpinor;
-      typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
-      typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
-      
-      typedef Lattice<SiteSpinor> FermionField;
-      typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
-      
-      typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
-      typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
+ ImplParams Params;
 
-      typedef GparityWilsonImplParams ImplParams;
-      
-      ImplParams Params;
+ GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
 
+ bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
 
-      GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
+ // provide the multiply by link that is differentiated between Gparity (with
+ // flavour index) and non-Gparity
+ inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
+		      const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
+		      StencilImpl &St) {
 
-      bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
-
-      // provide the multiply by link that is differentiated between Gparity (with
-      // flavour index) and non-Gparity
-      inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
-			   const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
-			   StencilImpl &St) {
-	typedef SiteHalfSpinor vobj;
-	typedef typename SiteHalfSpinor::scalar_object sobj;
+  typedef SiteHalfSpinor vobj;
+   typedef typename SiteHalfSpinor::scalar_object sobj;
 	
-	vobj vtmp;
-	sobj stmp;
+   vobj vtmp;
+   sobj stmp;
 	
-	GridBase *grid = St._grid;
+   GridBase *grid = St._grid;
 	
-	const int Nsimd = grid->Nsimd();
+   const int Nsimd = grid->Nsimd();
 	
-	int direction = St._directions[mu];
-	int distance = St._distances[mu];
-	int ptype = St._permute_type[mu];
-	int sl = St._grid->_simd_layout[direction];
+   int direction = St._directions[mu];
+   int distance = St._distances[mu];
+   int ptype = St._permute_type[mu];
+   int sl = St._grid->_simd_layout[direction];
+   
+   // Fixme X.Y.Z.T hardcode in stencil
+   int mmu = mu % Nd;
 	
-	// Fixme X.Y.Z.T hardcode in stencil
-	int mmu = mu % Nd;
+   // assert our assumptions
+   assert((distance == 1) || (distance == -1));  // nearest neighbour stencil hard code
+   assert((sl == 1) || (sl == 2));
+   
+   std::vector<int> icoor;
 	
-	// assert our assumptions
-	assert((distance == 1) || (distance == -1));  // nearest neighbour stencil hard code
-	assert((sl == 1) || (sl == 2));
-	
-	std::vector<int> icoor;
-	
-	if ( SE->_around_the_world && Params.twists[mmu] ) {
+   if ( SE->_around_the_world && Params.twists[mmu] ) {
 
-	  if ( sl == 2 ) {
+     if ( sl == 2 ) {
+       
+       std::vector<sobj> vals(Nsimd);
 
-	    std::vector<sobj> vals(Nsimd);
+       extract(chi,vals);
+       for(int s=0;s<Nsimd;s++){
 
-	    extract(chi,vals);
-	    for(int s=0;s<Nsimd;s++){
-
-	      grid->iCoorFromIindex(icoor,s);
+	 grid->iCoorFromIindex(icoor,s);
 	      
-	      assert((icoor[direction]==0)||(icoor[direction]==1));
+	 assert((icoor[direction]==0)||(icoor[direction]==1));
 	      
-	      int permute_lane;
-	      if ( distance == 1) {
-		permute_lane = icoor[direction]?1:0;
-	      } else {
-		permute_lane = icoor[direction]?0:1;
+	 int permute_lane;
+	 if ( distance == 1) {
+	   permute_lane = icoor[direction]?1:0;
+	 } else {
+	   permute_lane = icoor[direction]?0:1;
+	 }
+	      
+	 if ( permute_lane ) { 
+	   stmp(0) = vals[s](1);
+	   stmp(1) = vals[s](0);
+	   vals[s] = stmp;
 	      }
-	      
-	      if ( permute_lane ) { 
-		stmp(0) = vals[s](1);
-		stmp(1) = vals[s](0);
-		vals[s] = stmp;
-	      }
-	    }
-	    merge(vtmp,vals);
+       }
+       merge(vtmp,vals);
+	    
+     } else { 
+       vtmp(0) = chi(1);
+       vtmp(1) = chi(0);
+     }
+     mult(&phi(0),&U(0)(mu),&vtmp(0));
+     mult(&phi(1),&U(1)(mu),&vtmp(1));
+     
+   } else { 
+     mult(&phi(0),&U(0)(mu),&chi(0));
+     mult(&phi(1),&U(1)(mu),&chi(1));
+   }
+   
+ }
 
-	  } else { 
-	    vtmp(0) = chi(1);
-	    vtmp(1) = chi(0);
-	  }
-	  mult(&phi(0),&U(0)(mu),&vtmp(0));
-	  mult(&phi(1),&U(1)(mu),&vtmp(1));
-	  
-	} else { 
-	  mult(&phi(0),&U(0)(mu),&chi(0));
-	  mult(&phi(1),&U(1)(mu),&chi(1));
-	}
+ inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
+ {
+   conformable(Uds._grid,GaugeGrid);
+   conformable(Umu._grid,GaugeGrid);
+   
+   GaugeLinkField Utmp (GaugeGrid);
+   GaugeLinkField U    (GaugeGrid);
+   GaugeLinkField Uconj(GaugeGrid);
+   
+   Lattice<iScalar<vInteger> > coor(GaugeGrid);
 	
-  }
-
-      inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
-      {
-	
-	conformable(Uds._grid,GaugeGrid);
-	conformable(Umu._grid,GaugeGrid);
-	
-	GaugeLinkField Utmp (GaugeGrid);
-	GaugeLinkField U    (GaugeGrid);
-	GaugeLinkField Uconj(GaugeGrid);
-	
-	Lattice<iScalar<vInteger> > coor(GaugeGrid);
-	
-	
-	for(int mu=0;mu<Nd;mu++){
+   for(int mu=0;mu<Nd;mu++){
 	  
-	  LatticeCoordinate(coor,mu);
+     LatticeCoordinate(coor,mu);
 	  
-	  U     = PeekIndex<LorentzIndex>(Umu,mu);
-	  Uconj = conjugate(U);
+     U     = PeekIndex<LorentzIndex>(Umu,mu);
+     Uconj = conjugate(U);
+     
+     // This phase could come from a simple bc 1,1,-1,1 ..
+     int neglink = GaugeGrid->GlobalDimensions()[mu]-1;
+     if ( Params.twists[mu] ) { 
+       Uconj = where(coor==neglink,-Uconj,Uconj);
+     }
 	  
-	  // This phase could come from a simple bc 1,1,-1,1 ..
-	  int neglink = GaugeGrid->GlobalDimensions()[mu]-1;
-	  if ( Params.twists[mu] ) { 
-	    Uconj = where(coor==neglink,-Uconj,Uconj);
-	  }
+PARALLEL_FOR_LOOP
+     for(auto ss=U.begin();ss<U.end();ss++){
+       Uds[ss](0)(mu) = U[ss]();
+       Uds[ss](1)(mu) = Uconj[ss]();
+     }
 	  
+     U     = adj(Cshift(U    ,mu,-1));      // correct except for spanning the boundary
+     Uconj = adj(Cshift(Uconj,mu,-1));
+ 
+     Utmp = U;
+     if ( Params.twists[mu] ) { 
+       Utmp = where(coor==0,Uconj,Utmp);
+     }
 	  
-	  PARALLEL_FOR_LOOP
-	    for(auto ss=U.begin();ss<U.end();ss++){
-	      Uds[ss](0)(mu) = U[ss]();
-	      Uds[ss](1)(mu) = Uconj[ss]();
-	    }
+PARALLEL_FOR_LOOP
+     for(auto ss=U.begin();ss<U.end();ss++){
+       Uds[ss](0)(mu+4) = Utmp[ss]();
+     }
 	  
-	  U     = adj(Cshift(U    ,mu,-1));      // correct except for spanning the boundary
-	  Uconj = adj(Cshift(Uconj,mu,-1));
+     Utmp = Uconj;
+     if ( Params.twists[mu] ) { 
+       Utmp = where(coor==0,U,Utmp);
+     }
 	  
-	  Utmp = U;
-	  if ( Params.twists[mu] ) { 
-	    Utmp = where(coor==0,Uconj,Utmp);
-	  }
+PARALLEL_FOR_LOOP
+     for(auto ss=U.begin();ss<U.end();ss++){
+       Uds[ss](1)(mu+4) = Utmp[ss]();
+     }
 	  
-	  PARALLEL_FOR_LOOP
-	    for(auto ss=U.begin();ss<U.end();ss++){
-	      Uds[ss](0)(mu+4) = Utmp[ss]();
-	    }
-	  
-	  Utmp = Uconj;
-	  if ( Params.twists[mu] ) { 
-	    Utmp = where(coor==0,U,Utmp);
-	  }
-	  
-	  PARALLEL_FOR_LOOP
-	    for(auto ss=U.begin();ss<U.end();ss++){
-	      Uds[ss](1)(mu+4) = Utmp[ss]();
-	    }
-	  
-	}
-      }
+   }
+ }
       
       
-      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,
-				FermionField &A, int mu) {
-	// DhopDir provides U or Uconj depending on coor/flavour.
-	GaugeLinkField link(mat._grid);
-	// use lorentz for flavour as hack.
-	auto tmp = TraceIndex<SpinIndex>(outerProduct(Btilde, A));
-	PARALLEL_FOR_LOOP
-	  for (auto ss = tmp.begin(); ss < tmp.end(); ss++) {
-	    link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1));
-	  }
-	PokeIndex<LorentzIndex>(mat, link, mu);
-	return;
-      }
+ inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A, int mu) {
+
+   // DhopDir provides U or Uconj depending on coor/flavour.
+   GaugeLinkField link(mat._grid);
+   // use lorentz for flavour as hack.
+   auto tmp = TraceIndex<SpinIndex>(outerProduct(Btilde, A));
+PARALLEL_FOR_LOOP
+   for (auto ss = tmp.begin(); ss < tmp.end(); ss++) {
+     link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1));
+   }
+   PokeIndex<LorentzIndex>(mat, link, mu);
+   return;
+ }
       
-      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,
-				FermionField &Atilde, int mu) {
-	int Ls = Btilde._grid->_fdimensions[0];
+ inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde, int mu) {
+
+   int Ls = Btilde._grid->_fdimensions[0];
 	
-	GaugeLinkField tmp(mat._grid);
-	tmp = zero;
-	PARALLEL_FOR_LOOP
-	  for (int ss = 0; ss < tmp._grid->oSites(); ss++) {
-	    for (int s = 0; s < Ls; s++) {
-	      int sF = s + Ls * ss;
-	      auto ttmp = traceIndex<SpinIndex>(outerProduct(Btilde[sF], Atilde[sF]));
-	      tmp[ss]() = tmp[ss]() + ttmp(0, 0) + conjugate(ttmp(1, 1));
-	    }
-	  }
-	PokeIndex<LorentzIndex>(mat, tmp, mu);
-	return;
-      }
-    };
+   GaugeLinkField tmp(mat._grid);
+   tmp = zero;
+PARALLEL_FOR_LOOP
+   for (int ss = 0; ss < tmp._grid->oSites(); ss++) {
+     for (int s = 0; s < Ls; s++) {
+       int sF = s + Ls * ss;
+       auto ttmp = traceIndex<SpinIndex>(outerProduct(Btilde[sF], Atilde[sF]));
+       tmp[ss]() = tmp[ss]() + ttmp(0, 0) + conjugate(ttmp(1, 1));
+     }
+   }
+   PokeIndex<LorentzIndex>(mat, tmp, mu);
+   return;
+ }
 
-    typedef WilsonImpl<vComplex,  FundamentalRepresentation > WilsonImplR;   // Real.. whichever prec
-    typedef WilsonImpl<vComplexF, FundamentalRepresentation > WilsonImplF;  // Float
-    typedef WilsonImpl<vComplexD, FundamentalRepresentation > WilsonImplD;  // Double
+};
 
+ typedef WilsonImpl<vComplex,  FundamentalRepresentation > WilsonImplR;   // Real.. whichever prec
+ typedef WilsonImpl<vComplexF, FundamentalRepresentation > WilsonImplF;  // Float
+ typedef WilsonImpl<vComplexD, FundamentalRepresentation > WilsonImplD;  // Double
 
-    typedef WilsonImpl<vComplex,  FundamentalRepresentation, ComplexD > ZWilsonImplR; // Real.. whichever prec
-    typedef WilsonImpl<vComplexF, FundamentalRepresentation, ComplexD > ZWilsonImplF; // Float
-    typedef WilsonImpl<vComplexD, FundamentalRepresentation, ComplexD > ZWilsonImplD; // Double
+ typedef WilsonImpl<vComplex,  FundamentalRepresentation, ComplexD > ZWilsonImplR; // Real.. whichever prec
+ typedef WilsonImpl<vComplexF, FundamentalRepresentation, ComplexD > ZWilsonImplF; // Float
+ typedef WilsonImpl<vComplexD, FundamentalRepresentation, ComplexD > ZWilsonImplD; // Double
+ 
+ typedef WilsonImpl<vComplex,  AdjointRepresentation > WilsonAdjImplR;   // Real.. whichever prec
+ typedef WilsonImpl<vComplexF, AdjointRepresentation > WilsonAdjImplF;  // Float
+ typedef WilsonImpl<vComplexD, AdjointRepresentation > WilsonAdjImplD;  // Double
+ 
+ typedef WilsonImpl<vComplex,  TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplR;   // Real.. whichever prec
+ typedef WilsonImpl<vComplexF, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplF;  // Float
+ typedef WilsonImpl<vComplexD, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplD;  // Double
+ 
+ typedef DomainWallVec5dImpl<vComplex ,Nc> DomainWallVec5dImplR; // Real.. whichever prec
+ typedef DomainWallVec5dImpl<vComplexF,Nc> DomainWallVec5dImplF; // Float
+ typedef DomainWallVec5dImpl<vComplexD,Nc> DomainWallVec5dImplD; // Double
+ 
+ typedef DomainWallVec5dImpl<vComplex ,Nc,ComplexD> ZDomainWallVec5dImplR; // Real.. whichever prec
+ typedef DomainWallVec5dImpl<vComplexF,Nc,ComplexD> ZDomainWallVec5dImplF; // Float
+ typedef DomainWallVec5dImpl<vComplexD,Nc,ComplexD> ZDomainWallVec5dImplD; // Double
+ 
+ typedef GparityWilsonImpl<vComplex , Nc> GparityWilsonImplR;  // Real.. whichever prec
+ typedef GparityWilsonImpl<vComplexF, Nc> GparityWilsonImplF;  // Float
+ typedef GparityWilsonImpl<vComplexD, Nc> GparityWilsonImplD;  // Double
 
-    typedef WilsonImpl<vComplex,  AdjointRepresentation > WilsonAdjImplR;   // Real.. whichever prec
-    typedef WilsonImpl<vComplexF, AdjointRepresentation > WilsonAdjImplF;  // Float
-    typedef WilsonImpl<vComplexD, AdjointRepresentation > WilsonAdjImplD;  // Double
+}}
 
-    typedef WilsonImpl<vComplex,  TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplR;   // Real.. whichever prec
-    typedef WilsonImpl<vComplexF, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplF;  // Float
-    typedef WilsonImpl<vComplexD, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplD;  // Double
-
-    typedef DomainWallVec5dImpl<vComplex ,Nc> DomainWallVec5dImplR; // Real.. whichever prec
-    typedef DomainWallVec5dImpl<vComplexF,Nc> DomainWallVec5dImplF; // Float
-    typedef DomainWallVec5dImpl<vComplexD,Nc> DomainWallVec5dImplD; // Double
-    
-    typedef DomainWallVec5dImpl<vComplex ,Nc,ComplexD> ZDomainWallVec5dImplR; // Real.. whichever prec
-    typedef DomainWallVec5dImpl<vComplexF,Nc,ComplexD> ZDomainWallVec5dImplF; // Float
-    typedef DomainWallVec5dImpl<vComplexD,Nc,ComplexD> ZDomainWallVec5dImplD; // Double
-
-    typedef GparityWilsonImpl<vComplex, Nc>  GparityWilsonImplR;  // Real.. whichever prec
-    typedef GparityWilsonImpl<vComplexF, Nc> GparityWilsonImplF;  // Float
-    typedef GparityWilsonImpl<vComplexD, Nc> GparityWilsonImplD;  // Double
-}
-}
 #endif
diff --git a/lib/qcd/action/fermion/WilsonFermion.cc b/lib/qcd/action/fermion/WilsonFermion.cc
index d887e05b..4bc28bc7 100644
--- a/lib/qcd/action/fermion/WilsonFermion.cc
+++ b/lib/qcd/action/fermion/WilsonFermion.cc
@@ -166,7 +166,7 @@ void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
     ////////////////////////
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < B._grid->oSites(); sss++) {
-      Kernels::DiracOptDhopDir(st, U, st.comm_buf, sss, sss, B, Btilde, mu,
+      Kernels::DiracOptDhopDir(st, U, st.CommBuf(), sss, sss, B, Btilde, mu,
                                gamma);
     }
 
@@ -277,7 +277,7 @@ void WilsonFermion<Impl>::DhopDirDisp(const FermionField &in, FermionField &out,
 
   PARALLEL_FOR_LOOP
   for (int sss = 0; sss < in._grid->oSites(); sss++) {
-    Kernels::DiracOptDhopDir(Stencil, Umu, Stencil.comm_buf, sss, sss, in, out,
+    Kernels::DiracOptDhopDir(Stencil, Umu, Stencil.CommBuf(), sss, sss, in, out,
                              dirdisp, gamma);
   }
 };
@@ -295,13 +295,13 @@ void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
   if (dag == DaggerYes) {
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < in._grid->oSites(); sss++) {
-      Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sss, sss, 1, 1, in,
+      Kernels::DiracOptDhopSiteDag(st, lo, U, st.CommBuf(), sss, sss, 1, 1, in,
                                    out);
     }
   } else {
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < in._grid->oSites(); sss++) {
-      Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sss, sss, 1, 1, in,
+      Kernels::DiracOptDhopSite(st, lo, U, st.CommBuf(), sss, sss, 1, 1, in,
                                 out);
     }
   }
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index 7c4fafff..8c059667 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -184,44 +184,37 @@ void WilsonFermion5D<Impl>::Report(void)
 
   if ( DhopCalls > 0 ) {
     std::cout << GridLogMessage << "#### Dhop calls report " << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D Number of Dhop Calls     : " << DhopCalls  << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " << DhopCommTime
-              << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : "
-              << DhopCommTime / DhopCalls << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : "
-              << DhopComputeTime << " us" << std::endl;
-    std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : "
-              << DhopComputeTime / DhopCalls << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Number of Dhop Calls     : " << DhopCalls   << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " << DhopCommTime<< " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : " << DhopCommTime / DhopCalls << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : " << DhopComputeTime << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : " << DhopComputeTime / DhopCalls << " us" << std::endl;
 
     RealD mflops = 1344*volume*DhopCalls/DhopComputeTime/2; // 2 for red black counting
     std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
-    std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
+    std::cout << GridLogMessage << "Average mflops/s per call per rank       : " << mflops/NP << std::endl;
 
    }
 
   if ( DerivCalls > 0 ) {
-  std::cout << GridLogMessage << "#### Deriv calls report "<< std::endl;
-  std::cout << GridLogMessage << "WilsonFermion5D Number of Deriv Calls    : " <<DerivCalls <<std::endl;
-  std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " <<DerivCommTime <<" us"<<std::endl;
-  std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : " <<DerivCommTime/DerivCalls<<" us" <<std::endl;
-  std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : " <<DerivComputeTime <<" us"<<std::endl;
-  std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : " <<DerivComputeTime/DerivCalls<<" us" <<std::endl;
-  std::cout << GridLogMessage << "WilsonFermion5D Total Dhop Compute time  : " <<DerivDhopComputeTime <<" us"<<std::endl;
-  std::cout << GridLogMessage << "WilsonFermion5D Dhop ComputeTime/Calls   : " <<DerivDhopComputeTime/DerivCalls<<" us" <<std::endl;
-
-
-
-  RealD mflops = 144*volume*DerivCalls/DerivDhopComputeTime;
-  std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
-  std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
-
+    std::cout << GridLogMessage << "#### Deriv calls report "<< std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Number of Deriv Calls    : " <<DerivCalls <<std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " <<DerivCommTime <<" us"<<std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : " <<DerivCommTime/DerivCalls<<" us" <<std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : " <<DerivComputeTime <<" us"<<std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : " <<DerivComputeTime/DerivCalls<<" us" <<std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Dhop Compute time  : " <<DerivDhopComputeTime <<" us"<<std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Dhop ComputeTime/Calls   : " <<DerivDhopComputeTime/DerivCalls<<" us" <<std::endl;
+    
+    RealD mflops = 144*volume*DerivCalls/DerivDhopComputeTime;
+    std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
+    std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
   }
 
   if (DerivCalls > 0 || DhopCalls > 0){
-  std::cout << GridLogMessage << "WilsonFermion5D Stencil"<<std::endl;  Stencil.Report();
-  std::cout << GridLogMessage << "WilsonFermion5D StencilEven"<<std::endl;  StencilEven.Report();
-  std::cout << GridLogMessage << "WilsonFermion5D StencilOdd"<<std::endl;  StencilOdd.Report();
+    std::cout << GridLogMessage << "WilsonFermion5D Stencil"<<std::endl;  Stencil.Report();
+    std::cout << GridLogMessage << "WilsonFermion5D StencilEven"<<std::endl;  StencilEven.Report();
+    std::cout << GridLogMessage << "WilsonFermion5D StencilOdd"<<std::endl;  StencilOdd.Report();
   }
 }
 
@@ -275,7 +268,7 @@ PARALLEL_FOR_LOOP
     for(int s=0;s<Ls;s++){
       int sU=ss;
       int sF = s+Ls*sU; 
-      Kernels::DiracOptDhopDir(Stencil,Umu,Stencil.comm_buf,sF,sU,in,out,dirdisp,gamma);
+      Kernels::DiracOptDhopDir(Stencil,Umu,Stencil.CommBuf(),sF,sU,in,out,dirdisp,gamma);
     }
   }
 };
@@ -327,8 +320,7 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
         assert(sF < B._grid->oSites());
         assert(sU < U._grid->oSites());
 
-        Kernels::DiracOptDhopDir(st, U, st.comm_buf, sF, sU, B, Btilde, mu,
-                                 gamma);
+        Kernels::DiracOptDhopDir(st, U, st.CommBuf(), sF, sU, B, Btilde, mu, gamma);
 
         ////////////////////////////
         // spin trace outer product
@@ -342,10 +334,10 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
 }
 
 template<class Impl>
-void WilsonFermion5D<Impl>::DhopDeriv(      GaugeField &mat,
-              const FermionField &A,
-              const FermionField &B,
-              int dag)
+void WilsonFermion5D<Impl>::DhopDeriv(GaugeField &mat,
+				      const FermionField &A,
+				      const FermionField &B,
+				      int dag)
 {
   conformable(A._grid,FermionGrid());  
   conformable(A._grid,B._grid);
@@ -358,9 +350,9 @@ void WilsonFermion5D<Impl>::DhopDeriv(      GaugeField &mat,
 
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopDerivEO(GaugeField &mat,
-          const FermionField &A,
-          const FermionField &B,
-          int dag)
+					const FermionField &A,
+					const FermionField &B,
+					int dag)
 {
   conformable(A._grid,FermionRedBlackGrid());
   conformable(GaugeRedBlackGrid(),mat._grid);
@@ -376,9 +368,9 @@ void WilsonFermion5D<Impl>::DhopDerivEO(GaugeField &mat,
 
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopDerivOE(GaugeField &mat,
-          const FermionField &A,
-          const FermionField &B,
-          int dag)
+					const FermionField &A,
+					const FermionField &B,
+					int dag)
 {
   conformable(A._grid,FermionRedBlackGrid());
   conformable(GaugeRedBlackGrid(),mat._grid);
@@ -393,8 +385,8 @@ void WilsonFermion5D<Impl>::DhopDerivOE(GaugeField &mat,
 
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
-           DoubledGaugeField & U,
-           const FermionField &in, FermionField &out,int dag)
+					 DoubledGaugeField & U,
+					 const FermionField &in, FermionField &out,int dag)
 {
   //  assert((dag==DaggerNo) ||(dag==DaggerYes));
   Compressor compressor(dag);
@@ -412,27 +404,25 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
     for (int ss = 0; ss < U._grid->oSites(); ss++) {
       int sU = ss;
       int sF = LLs * sU;
-      Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in,
-                                   out);
+      Kernels::DiracOptDhopSiteDag(st, lo, U, st.CommBuf(), sF, sU, LLs, 1, in, out);
     }
 #ifdef AVX512
   } else if (stat.is_init() ) {
 
     int nthreads;
     stat.start();
-    #pragma omp parallel
+#pragma omp parallel
     {
-    #pragma omp master
+#pragma omp master
     nthreads = omp_get_num_threads();
     int mythread = omp_get_thread_num();
     stat.enter(mythread);
-    #pragma omp for nowait
-   for(int ss=0;ss<U._grid->oSites();ss++)
-    {
-       int sU=ss;
-       int sF=LLs*sU;
-       Kernels::DiracOptDhopSite(st,lo,U,st.comm_buf,sF,sU,LLs,1,in,out);
-     }
+#pragma omp for nowait
+    for(int ss=0;ss<U._grid->oSites();ss++) {
+      int sU=ss;
+      int sF=LLs*sU;
+      Kernels::DiracOptDhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out);
+    }
     stat.exit(mythread);
     }
     stat.accum(nthreads);
@@ -442,8 +432,7 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
     for (int ss = 0; ss < U._grid->oSites(); ss++) {
       int sU = ss;
       int sF = LLs * sU;
-      Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in,
-                                out);
+      Kernels::DiracOptDhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out);
     }
   }
   DhopComputeTime+=usecond();
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.h b/lib/qcd/action/fermion/WilsonFermion5D.h
index bc7ec543..295e2997 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.h
+++ b/lib/qcd/action/fermion/WilsonFermion5D.h
@@ -34,155 +34,154 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/Stat.h>
 
 namespace Grid {
+namespace QCD {
 
-  namespace QCD {
+  ////////////////////////////////////////////////////////////////////////////////
+  // This is the 4d red black case appropriate to support
+  //
+  // parity = (x+y+z+t)|2;
+  // generalised five dim fermions like mobius, zolotarev etc..	
+  //
+  // i.e. even even contains fifth dim hopping term.
+  //
+  // [DIFFERS from original CPS red black implementation parity = (x+y+z+t+s)|2 ]
+  ////////////////////////////////////////////////////////////////////////////////
 
-    ////////////////////////////////////////////////////////////////////////////////
-    // This is the 4d red black case appropriate to support
-    //
-    // parity = (x+y+z+t)|2;
-    // generalised five dim fermions like mobius, zolotarev etc..	
-    //
-    // i.e. even even contains fifth dim hopping term.
-    //
-    // [DIFFERS from original CPS red black implementation parity = (x+y+z+t+s)|2 ]
-    ////////////////////////////////////////////////////////////////////////////////
-
-    class WilsonFermion5DStatic { 
-    public:
-      // S-direction is INNERMOST and takes no part in the parity.
-      static const std::vector<int> directions;
-      static const std::vector<int> displacements;
-      const int npoint = 8;
-    };
-
-    template<class Impl>
-    class WilsonFermion5D : public WilsonKernels<Impl>, public WilsonFermion5DStatic
-    {
-    public:
-     INHERIT_IMPL_TYPES(Impl);
-     typedef WilsonKernels<Impl> Kernels;
-     PmuStat stat;
-
-     void Report(void);
-     void ZeroCounters(void);
-     double DhopCalls;
-     double DhopCommTime;
-     double DhopComputeTime;
-
-     double DerivCalls;
-     double DerivCommTime;
-     double DerivComputeTime;
-     double DerivDhopComputeTime;
-
-      ///////////////////////////////////////////////////////////////
-      // Implement the abstract base
-      ///////////////////////////////////////////////////////////////
-      GridBase *GaugeGrid(void)              { return _FourDimGrid ;}
-      GridBase *GaugeRedBlackGrid(void)      { return _FourDimRedBlackGrid ;}
-      GridBase *FermionGrid(void)            { return _FiveDimGrid;}
-      GridBase *FermionRedBlackGrid(void)    { return _FiveDimRedBlackGrid;}
-
-      // full checkerboard operations; leave unimplemented as abstract for now
-      virtual RealD  M    (const FermionField &in, FermionField &out){assert(0); return 0.0;};
-      virtual RealD  Mdag (const FermionField &in, FermionField &out){assert(0); return 0.0;};
-
-      // half checkerboard operations; leave unimplemented as abstract for now
-      virtual void   Meooe       (const FermionField &in, FermionField &out){assert(0);};
-      virtual void   Mooee       (const FermionField &in, FermionField &out){assert(0);};
-      virtual void   MooeeInv    (const FermionField &in, FermionField &out){assert(0);};
-
-      virtual void   MeooeDag    (const FermionField &in, FermionField &out){assert(0);};
-      virtual void   MooeeDag    (const FermionField &in, FermionField &out){assert(0);};
-      virtual void   MooeeInvDag (const FermionField &in, FermionField &out){assert(0);};
-      virtual void   Mdir   (const FermionField &in, FermionField &out,int dir,int disp){assert(0);};   // case by case Wilson, Clover, Cayley, ContFrac, PartFrac
-
-      // These can be overridden by fancy 5d chiral action
-      virtual void DhopDeriv  (GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-      virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-      virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-
-      // Implement hopping term non-hermitian hopping term; half cb or both
-      // Implement s-diagonal DW
-      void DW    (const FermionField &in, FermionField &out,int dag);
-      void Dhop  (const FermionField &in, FermionField &out,int dag);
-      void DhopOE(const FermionField &in, FermionField &out,int dag);
-      void DhopEO(const FermionField &in, FermionField &out,int dag);
-
-      // add a DhopComm
+  class WilsonFermion5DStatic { 
+  public:
+    // S-direction is INNERMOST and takes no part in the parity.
+    static const std::vector<int> directions;
+    static const std::vector<int> displacements;
+    const int npoint = 8;
+  };
+  
+  template<class Impl>
+  class WilsonFermion5D : public WilsonKernels<Impl>, public WilsonFermion5DStatic
+  {
+  public:
+    INHERIT_IMPL_TYPES(Impl);
+    typedef WilsonKernels<Impl> Kernels;
+    PmuStat stat;
+    
+    void Report(void);
+    void ZeroCounters(void);
+    double DhopCalls;
+    double DhopCommTime;
+    double DhopComputeTime;
+    
+    double DerivCalls;
+    double DerivCommTime;
+    double DerivComputeTime;
+    double DerivDhopComputeTime;
+    
+    ///////////////////////////////////////////////////////////////
+    // Implement the abstract base
+    ///////////////////////////////////////////////////////////////
+    GridBase *GaugeGrid(void)              { return _FourDimGrid ;}
+    GridBase *GaugeRedBlackGrid(void)      { return _FourDimRedBlackGrid ;}
+    GridBase *FermionGrid(void)            { return _FiveDimGrid;}
+    GridBase *FermionRedBlackGrid(void)    { return _FiveDimRedBlackGrid;}
+    
+    // full checkerboard operations; leave unimplemented as abstract for now
+    virtual RealD  M    (const FermionField &in, FermionField &out){assert(0); return 0.0;};
+    virtual RealD  Mdag (const FermionField &in, FermionField &out){assert(0); return 0.0;};
+    
+    // half checkerboard operations; leave unimplemented as abstract for now
+    virtual void   Meooe       (const FermionField &in, FermionField &out){assert(0);};
+    virtual void   Mooee       (const FermionField &in, FermionField &out){assert(0);};
+    virtual void   MooeeInv    (const FermionField &in, FermionField &out){assert(0);};
+    
+    virtual void   MeooeDag    (const FermionField &in, FermionField &out){assert(0);};
+    virtual void   MooeeDag    (const FermionField &in, FermionField &out){assert(0);};
+    virtual void   MooeeInvDag (const FermionField &in, FermionField &out){assert(0);};
+    virtual void   Mdir   (const FermionField &in, FermionField &out,int dir,int disp){assert(0);};   // case by case Wilson, Clover, Cayley, ContFrac, PartFrac
+    
+    // These can be overridden by fancy 5d chiral action
+    virtual void DhopDeriv  (GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
+    virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
+    virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
+    
+    // Implement hopping term non-hermitian hopping term; half cb or both
+    // Implement s-diagonal DW
+    void DW    (const FermionField &in, FermionField &out,int dag);
+    void Dhop  (const FermionField &in, FermionField &out,int dag);
+    void DhopOE(const FermionField &in, FermionField &out,int dag);
+    void DhopEO(const FermionField &in, FermionField &out,int dag);
+    
+    // add a DhopComm
       // -- suboptimal interface will presently trigger multiple comms.
-      void DhopDir(const FermionField &in, FermionField &out,int dir,int disp);
-
-      ///////////////////////////////////////////////////////////////
-      // New methods added 
-      ///////////////////////////////////////////////////////////////
-      void DerivInternal(StencilImpl & st,
-			 DoubledGaugeField & U,
-			 GaugeField &mat,
-			 const FermionField &A,
-			 const FermionField &B,
-			 int dag);
-
-      void DhopInternal(StencilImpl & st,
-			LebesgueOrder &lo,
-			DoubledGaugeField &U,
-			const FermionField &in, 
-			FermionField &out,
-			int dag);
-
-      // Constructors
-      WilsonFermion5D(GaugeField &_Umu,
-		      GridCartesian         &FiveDimGrid,
-		      GridRedBlackCartesian &FiveDimRedBlackGrid,
-		      GridCartesian         &FourDimGrid,
-		      GridRedBlackCartesian &FourDimRedBlackGrid,
-		      double _M5,const ImplParams &p= ImplParams());
-
-      // Constructors
-      /*
+    void DhopDir(const FermionField &in, FermionField &out,int dir,int disp);
+    
+    ///////////////////////////////////////////////////////////////
+    // New methods added 
+    ///////////////////////////////////////////////////////////////
+    void DerivInternal(StencilImpl & st,
+		       DoubledGaugeField & U,
+		       GaugeField &mat,
+		       const FermionField &A,
+		       const FermionField &B,
+		       int dag);
+    
+    void DhopInternal(StencilImpl & st,
+		      LebesgueOrder &lo,
+		      DoubledGaugeField &U,
+		      const FermionField &in, 
+		      FermionField &out,
+		      int dag);
+    
+    // Constructors
+    WilsonFermion5D(GaugeField &_Umu,
+		    GridCartesian         &FiveDimGrid,
+		    GridRedBlackCartesian &FiveDimRedBlackGrid,
+		    GridCartesian         &FourDimGrid,
+		    GridRedBlackCartesian &FourDimRedBlackGrid,
+		    double _M5,const ImplParams &p= ImplParams());
+    
+    // Constructors
+    /*
       WilsonFermion5D(int simd, 
-		      GaugeField &_Umu,
-		      GridCartesian         &FiveDimGrid,
-		      GridRedBlackCartesian &FiveDimRedBlackGrid,
-		      GridCartesian         &FourDimGrid,
-		      double _M5,const ImplParams &p= ImplParams());
-      */
+      GaugeField &_Umu,
+      GridCartesian         &FiveDimGrid,
+      GridRedBlackCartesian &FiveDimRedBlackGrid,
+      GridCartesian         &FourDimGrid,
+      double _M5,const ImplParams &p= ImplParams());
+    */
+    
+    // DoubleStore
+    void ImportGauge(const GaugeField &_Umu);
+    
+    ///////////////////////////////////////////////////////////////
+    // Data members require to support the functionality
+    ///////////////////////////////////////////////////////////////
+  public:
+    
+    // Add these to the support from Wilson
+    GridBase *_FourDimGrid;
+    GridBase *_FourDimRedBlackGrid;
+    GridBase *_FiveDimGrid;
+    GridBase *_FiveDimRedBlackGrid;
+    
+    double                        M5;
+    int Ls;
+    
+    //Defines the stencils for even and odd
+    StencilImpl Stencil; 
+    StencilImpl StencilEven; 
+    StencilImpl StencilOdd; 
+    
+    // Copy of the gauge field , with even and odd subsets
+    DoubledGaugeField Umu;
+    DoubledGaugeField UmuEven;
+    DoubledGaugeField UmuOdd;
+    
+    LebesgueOrder Lebesgue;
+    LebesgueOrder LebesgueEvenOdd;
+    
+    // Comms buffer
+    std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  comm_buf;
+    
+  };
 
-      // DoubleStore
-      void ImportGauge(const GaugeField &_Umu);
-
-      ///////////////////////////////////////////////////////////////
-      // Data members require to support the functionality
-      ///////////////////////////////////////////////////////////////
-    public:
-
-      // Add these to the support from Wilson
-      GridBase *_FourDimGrid;
-      GridBase *_FourDimRedBlackGrid;
-      GridBase *_FiveDimGrid;
-      GridBase *_FiveDimRedBlackGrid;
-
-      double                        M5;
-      int Ls;
-
-      //Defines the stencils for even and odd
-      StencilImpl Stencil; 
-      StencilImpl StencilEven; 
-      StencilImpl StencilOdd; 
-
-      // Copy of the gauge field , with even and odd subsets
-      DoubledGaugeField Umu;
-      DoubledGaugeField UmuEven;
-      DoubledGaugeField UmuOdd;
-
-      LebesgueOrder Lebesgue;
-      LebesgueOrder LebesgueEvenOdd;
-
-      // Comms buffer
-      std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  comm_buf;
-      
-    };
-  }
-}
+}}
 
 #endif
diff --git a/lib/qcd/action/fermion/WilsonKernels.cc b/lib/qcd/action/fermion/WilsonKernels.cc
index 58e62cc8..52ee8704 100644
--- a/lib/qcd/action/fermion/WilsonKernels.cc
+++ b/lib/qcd/action/fermion/WilsonKernels.cc
@@ -43,10 +43,9 @@ WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
 ////////////////////////////////////////////
 
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(
-    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-    commVector<SiteHalfSpinor> &buf, int sF,
-    int sU, const FermionField &in, FermionField &out) {
+void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+						     SiteHalfSpinor *buf, int sF,
+						     int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteHalfSpinor *chi_p;
@@ -220,10 +219,9 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(
 
 // Need controls to do interior, exterior, or both
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptGenericDhopSite(
-    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-    commVector<SiteHalfSpinor> &buf, int sF,
-    int sU, const FermionField &in, FermionField &out) {
+void WilsonKernels<Impl>::DiracOptGenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+						  SiteHalfSpinor *buf, int sF,
+						  int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteHalfSpinor *chi_p;
@@ -396,10 +394,9 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSite(
 };
 
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptDhopDir(
-    StencilImpl &st, DoubledGaugeField &U,
-    commVector<SiteHalfSpinor> &buf, int sF,
-    int sU, const FermionField &in, FermionField &out, int dir, int gamma) {
+void WilsonKernels<Impl>::DiracOptDhopDir( StencilImpl &st, DoubledGaugeField &U,SiteHalfSpinor *buf, int sF,
+					   int sU, const FermionField &in, FermionField &out, int dir, int gamma) {
+
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteSpinor result;
diff --git a/lib/qcd/action/fermion/WilsonKernels.h b/lib/qcd/action/fermion/WilsonKernels.h
index 66770c3c..669ee4be 100644
--- a/lib/qcd/action/fermion/WilsonKernels.h
+++ b/lib/qcd/action/fermion/WilsonKernels.h
@@ -32,175 +32,132 @@ directory
 #define GRID_QCD_DHOP_H
 
 namespace Grid {
+namespace QCD {
 
-  namespace QCD {
-
-    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    // Helper routines that implement Wilson stencil for a single site.
-    // Common to both the WilsonFermion and WilsonFermion5D
-    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    class WilsonKernelsStatic { 
-    public:
-      // S-direction is INNERMOST and takes no part in the parity.
-      static int AsmOpt;  // these are a temporary hack
-      static int HandOpt; // these are a temporary hack
-    };
-
-    template<class Impl> class WilsonKernels : public FermionOperator<Impl> , public WilsonKernelsStatic { 
-    public:
-
-      INHERIT_IMPL_TYPES(Impl);
-      typedef FermionOperator<Impl> Base;
-     
-    public:
-
-      template <bool EnableBool = true>
-      typename std::enable_if<Impl::Dimension == 3 && Nc == 3 &&EnableBool, void>::type
-	DiracOptDhopSite(
-			 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-			 commVector<SiteHalfSpinor> &buf,
-			 int sF, int sU, int Ls, int Ns, const FermionField &in,
-			 FermionField &out) {
+  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Helper routines that implement Wilson stencil for a single site.
+  // Common to both the WilsonFermion and WilsonFermion5D
+  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+class WilsonKernelsStatic { 
+ public:
+  // S-direction is INNERMOST and takes no part in the parity.
+  static int AsmOpt;  // these are a temporary hack
+  static int HandOpt; // these are a temporary hack
+};
+ 
+template<class Impl> class WilsonKernels : public FermionOperator<Impl> , public WilsonKernelsStatic { 
+ public:
+   
+  INHERIT_IMPL_TYPES(Impl);
+  typedef FermionOperator<Impl> Base;
+   
+public:
+   
+  template <bool EnableBool = true>
+  typename std::enable_if<Impl::Dimension == 3 && Nc == 3 &&EnableBool, void>::type
+  DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
 #ifdef AVX512
-	if (AsmOpt) {
-	  WilsonKernels<Impl>::DiracOptAsmDhopSite(st, lo, U, buf, sF, sU, Ls, Ns,
-						   in, out);
-
-	} else {
+    if (AsmOpt) {
+      WilsonKernels<Impl>::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
+    } else {
 #else
-	  {
+    {
 #endif
-	    for (int site = 0; site < Ns; site++) {
-	      for (int s = 0; s < Ls; s++) {
-		if (HandOpt)
-		  WilsonKernels<Impl>::DiracOptHandDhopSite(st, lo, U, buf, sF, sU,
-							    in, out);
-		else
-		  WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU,
-							       in, out);
-		sF++;
-	      }
-	      sU++;
-	    }
-	  }
+      for (int site = 0; site < Ns; site++) {
+	for (int s = 0; s < Ls; s++) {
+	  if (HandOpt)
+	    WilsonKernels<Impl>::DiracOptHandDhopSite(st,lo,U,buf,sF,sU,in,out);
+	  else
+	    WilsonKernels<Impl>::DiracOptGenericDhopSite(st,lo,U,buf,sF,sU,in,out);
+	  sF++;
 	}
-
-	template <bool EnableBool = true>
-	  typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type
-	  DiracOptDhopSite(
-			   StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-			   commVector<SiteHalfSpinor> &buf,
-			   int sF, int sU, int Ls, int Ns, const FermionField &in,
-			   FermionField &out) {
-	  for (int site = 0; site < Ns; site++) {
-	    for (int s = 0; s < Ls; s++) {
-	      WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in,
-							   out);
-	      sF++;
-	    }
-	    sU++;
-	  }
-	}
-
-	template <bool EnableBool = true>
-	  typename std::enable_if<Impl::Dimension == 3 && Nc == 3 && EnableBool,
-				  void>::type
-	  DiracOptDhopSiteDag(
-			      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-			      commVector<SiteHalfSpinor> &buf,
-			      int sF, int sU, int Ls, int Ns, const FermionField &in,
-			      FermionField &out) {
-#ifdef AVX512
-				    if (AsmOpt) {
-				      WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st, lo, U, buf, sF, sU, Ls,
-										  Ns, in, out);
-				    } else {
-#else
-				      {
-#endif
-					for (int site = 0; site < Ns; site++) {
-					  for (int s = 0; s < Ls; s++) {
-					    if (HandOpt)
-					      WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st, lo, U, buf, sF, sU,
-											   in, out);
-					    else
-					      WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF,
-											      sU, in, out);
-					    sF++;
-					  }
-					  sU++;
-					}
-				      }
-				    }
-
-				    template <bool EnableBool = true>
-				      typename std::enable_if<
-				      (Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool,
-				      void>::type
-				      DiracOptDhopSiteDag(
-							  StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-							  commVector<SiteHalfSpinor> &buf,
-							  int sF, int sU, int Ls, int Ns, const FermionField &in,
-							  FermionField &out) {
-					for (int site = 0; site < Ns; site++) {
-					  for (int s = 0; s < Ls; s++) {
-					    WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF, sU,
-											    in, out);
-					    sF++;
-					  }
-					  sU++;
-					}
-				      }
-
-				    void DiracOptDhopDir(
-							 StencilImpl &st, DoubledGaugeField &U,
-							 commVector<SiteHalfSpinor> &buf,
-							 int sF, int sU, const FermionField &in, FermionField &out, int dirdisp,
-							 int gamma);
-
-	private:
-				    // Specialised variants
-				    void DiracOptGenericDhopSite(
-								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								 commVector<SiteHalfSpinor> &buf,
-								 int sF, int sU, const FermionField &in, FermionField &out);
-
-				    void DiracOptGenericDhopSiteDag(
-								    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								    commVector<SiteHalfSpinor> &buf,
-								    int sF, int sU, const FermionField &in, FermionField &out);
-
-				    void DiracOptAsmDhopSite(
-							     StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-							     commVector<SiteHalfSpinor> &buf,
-							     int sF, int sU, int Ls, int Ns, const FermionField &in,
-							     FermionField &out);
-
-				    void DiracOptAsmDhopSiteDag(
-								StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								commVector<SiteHalfSpinor> &buf,
-								int sF, int sU, int Ls, int Ns, const FermionField &in,
-								FermionField &out);
-
-				    void DiracOptHandDhopSite(
-							      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-							      commVector<SiteHalfSpinor> &buf,
-							      int sF, int sU, const FermionField &in, FermionField &out);
-
-				    void DiracOptHandDhopSiteDag(
-								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-								 commVector<SiteHalfSpinor> &buf,
-								 int sF, int sU, const FermionField &in, FermionField &out);
-
-	public:
-				    WilsonKernels(const ImplParams &p = ImplParams());
-				  };
-    
+	sU++;
       }
     }
+  }
+     
+  template <bool EnableBool = true>
+  typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type
+  DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
+     
+    for (int site = 0; site < Ns; site++) {
+      for (int s = 0; s < Ls; s++) {
+	WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in, out);
+	sF++;
+      }
+      sU++;
+    }
+  }
+     
+  template <bool EnableBool = true>
+  typename std::enable_if<Impl::Dimension == 3 && Nc == 3 && EnableBool,void>::type
+  DiracOptDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+		      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
+#ifdef AVX512
+    if (AsmOpt) {
+      WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
+    } else {
+#else
+    {
+#endif
+      for (int site = 0; site < Ns; site++) {
+	for (int s = 0; s < Ls; s++) {
+	  if (HandOpt)
+	    WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	  else
+	    WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	  sF++;
+	}
+	sU++;
+      }
+    }
+  }
 
+  template <bool EnableBool = true>
+  typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool,void>::type
+  DiracOptDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,SiteHalfSpinor * buf,
+		      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
 
+    for (int site = 0; site < Ns; site++) {
+      for (int s = 0; s < Ls; s++) {
+	WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	sF++;
+      }
+      sU++;
+    }
+  }
 
+  void DiracOptDhopDir(StencilImpl &st, DoubledGaugeField &U,SiteHalfSpinor * buf,
+		       int sF, int sU, const FermionField &in, FermionField &out, int dirdisp, int gamma);
+      
+private:
+     // Specialised variants
+  void DiracOptGenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			       int sF, int sU, const FermionField &in, FermionField &out);
+      
+  void DiracOptGenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+				  int sF, int sU, const FermionField &in, FermionField &out);
 
+  void DiracOptAsmDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			   int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
+
+  void DiracOptAsmDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
+
+  void DiracOptHandDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			    int sF, int sU, const FermionField &in, FermionField &out);
+
+  void DiracOptHandDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			       int sF, int sU, const FermionField &in, FermionField &out);
+      
+public:
+
+  WilsonKernels(const ImplParams &p = ImplParams());
+
+};
+    
+}}
 
 #endif
diff --git a/lib/qcd/action/fermion/WilsonKernelsAsm.cc b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
index 7857e89a..74862400 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsm.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
@@ -33,31 +33,27 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 
 namespace Grid {
-  namespace QCD {
+namespace QCD {
     
-    ///////////////////////////////////////////////////////////
-    // Default to no assembler implementation
-    ///////////////////////////////////////////////////////////
-    template<class Impl>
-      void WilsonKernels<Impl >::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-                             commVector<SiteHalfSpinor>  &buf,
-                             int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-    {
-      assert(0);
-    }
-    template<class Impl>
-      void WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-                                commVector<SiteHalfSpinor>  &buf,
-                                int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
-    {
-      assert(0);
-    }
-
+///////////////////////////////////////////////////////////
+// Default to no assembler implementation
+///////////////////////////////////////////////////////////
+template<class Impl> void 
+WilsonKernels<Impl >::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+					  int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+{
+  assert(0);
+}
 
+template<class Impl> void 
+WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+					     int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+{
+  assert(0);
+}
 
 #if defined(AVX512) 
     
-    
     ///////////////////////////////////////////////////////////
     // If we are AVX512 specialise the single precision routine
     ///////////////////////////////////////////////////////////
@@ -65,16 +61,16 @@ namespace Grid {
 #include <simd/Intel512wilson.h>
 #include <simd/Intel512single.h>
     
-    static Vector<vComplexF> signs;
+static Vector<vComplexF> signs;
     
-    int setupSigns(void ){
-      Vector<vComplexF> bother(2);
-      signs = bother;
-      vrsign(signs[0]);
-      visign(signs[1]);
-      return 1;
-    }
-    static int signInit = setupSigns();
+  int setupSigns(void ){
+    Vector<vComplexF> bother(2);
+    signs = bother;
+    vrsign(signs[0]);
+    visign(signs[1]);
+    return 1;
+  }
+  static int signInit = setupSigns();
   
 #define label(A)  ilabel(A)
 #define ilabel(A) ".globl\n"  #A ":\n" 
@@ -84,17 +80,15 @@ namespace Grid {
 #define FX(A) WILSONASM_ ##A
   
 #undef KERNEL_DAG
-    template<>
-    void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-							 commVector<SiteHalfSpinor>  &buf,
-							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
       
 #define KERNEL_DAG
-    template<>
-    void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-							    commVector<SiteHalfSpinor>  &buf,
-							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+						   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
 #undef VMOVIDUP
@@ -109,31 +103,26 @@ namespace Grid {
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
 				    
 #undef KERNEL_DAG
-    template<>
-    void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-								  commVector<SiteHalfSpinor>  &buf,
-								  int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
 #define KERNEL_DAG
-    template<>
-    void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-								     commVector<SiteHalfSpinor>  &buf,
-								     int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
 #endif
 
-
 #define INSTANTIATE_ASM(A)\
-template void WilsonKernels<A>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
-                                   commVector<SiteHalfSpinor>  &buf,\
+template void WilsonKernels<A>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,\
                                   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
-template void WilsonKernels<A>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
-                                   commVector<SiteHalfSpinor>  &buf,\
+ \
+template void WilsonKernels<A>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,\
                                   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
 
-
 INSTANTIATE_ASM(WilsonImplF);
 INSTANTIATE_ASM(WilsonImplD);
 INSTANTIATE_ASM(ZWilsonImplF);
@@ -144,6 +133,6 @@ INSTANTIATE_ASM(DomainWallVec5dImplF);
 INSTANTIATE_ASM(DomainWallVec5dImplD);
 INSTANTIATE_ASM(ZDomainWallVec5dImplF);
 INSTANTIATE_ASM(ZDomainWallVec5dImplD);
-  }
-}
+
+}}
 
diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index 9d7eac23..f5900832 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -311,10 +311,9 @@ namespace Grid {
 namespace QCD {
 
 
-  template<class Impl>
-  void WilsonKernels<Impl>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-					       commVector<SiteHalfSpinor>  &buf,
-					       int ss,int sU,const FermionField &in, FermionField &out)
+template<class Impl> void 
+WilsonKernels<Impl>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
+					  int ss,int sU,const FermionField &in, FermionField &out)
 {
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
@@ -554,10 +553,9 @@ namespace QCD {
   }
 }
 
-  template<class Impl>
-  void WilsonKernels<Impl>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-					       commVector<SiteHalfSpinor>  &buf,
-					       int ss,int sU,const FermionField &in, FermionField &out)
+template<class Impl>
+void WilsonKernels<Impl>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+						  int ss,int sU,const FermionField &in, FermionField &out)
 {
   //  std::cout << "Hand op Dhop "<<std::endl;
   typedef typename Simd::scalar_type S;
@@ -798,38 +796,35 @@ namespace QCD {
   }
 }
 
-
   ////////////////////////////////////////////////
   // Specialise Gparity to simple implementation
   ////////////////////////////////////////////////
-template<>
-void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							     commVector<SiteHalfSpinor>  &buf,
-							     int sF,int sU,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+							SiteHalfSpinor *buf,
+							int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
 }
 
-template<>
-void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								commVector<SiteHalfSpinor>  &buf,
-								int sF,int sU,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+							   SiteHalfSpinor *buf,
+							   int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
 }
 
-template<>
-void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							     commVector<SiteHalfSpinor>  &buf,
-							     int sF,int sU,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
 }
 
-template<>
-void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								commVector<SiteHalfSpinor>  &buf,
-								int sF,int sU,const FermionField &in, FermionField &out)
+template<> void 
+WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							   int sF,int sU,const FermionField &in, FermionField &out)
 {
   assert(0);
 }
@@ -840,12 +835,10 @@ void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,
 // Need Nc=3 though //
 
 #define INSTANTIATE_THEM(A) \
-template void WilsonKernels<A>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
-							       commVector<SiteHalfSpinor>  &buf,\
-							       int ss,int sU,const FermionField &in, FermionField &out);\
-template void WilsonKernels<A>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
-								  commVector<SiteHalfSpinor>  &buf,\
-								  int ss,int sU,const FermionField &in, FermionField &out);
+template void WilsonKernels<A>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
+						     int ss,int sU,const FermionField &in, FermionField &out); \
+template void WilsonKernels<A>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
+							int ss,int sU,const FermionField &in, FermionField &out);
 
 INSTANTIATE_THEM(WilsonImplF);
 INSTANTIATE_THEM(WilsonImplD);
diff --git a/tests/Test_stencil.cc b/tests/Test_stencil.cc
index 0e35a414..1b71b8a5 100644
--- a/tests/Test_stencil.cc
+++ b/tests/Test_stencil.cc
@@ -116,7 +116,7 @@ int main (int argc, char ** argv)
 	  else if (SE->_is_local)
 	    Check._odata[i] = Foo._odata[SE->_offset];
 	  else 
-	    Check._odata[i] = myStencil.comm_buf[SE->_offset];
+	    Check._odata[i] = myStencil.CommBuf()[SE->_offset];
 	}
 
 	Real nrmC = norm2(Check);
@@ -207,7 +207,7 @@ int main (int argc, char ** argv)
 	  else if (SE->_is_local)
 	    OCheck._odata[i] = EFoo._odata[SE->_offset];
 	  else 
-	    OCheck._odata[i] = EStencil.comm_buf[SE->_offset];
+	    OCheck._odata[i] = EStencil.CommBuf()[SE->_offset];
 	}
 	for(int i=0;i<ECheck._grid->oSites();i++){
 	  int permute_type;
@@ -220,7 +220,7 @@ int main (int argc, char ** argv)
 	  else if (SE->_is_local)
 	    ECheck._odata[i] = OFoo._odata[SE->_offset];
 	  else 
-	    ECheck._odata[i] = OStencil.comm_buf[SE->_offset];
+	    ECheck._odata[i] = OStencil.CommBuf()[SE->_offset];
 	}
 	
 	setCheckerboard(Check,ECheck);

From ea25a4d9ac9d07f223a0c362d9ed7ec5fff998a8 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Sun, 23 Oct 2016 06:10:05 +0100
Subject: [PATCH 088/149] Works

---
 lib/Stencil.h | 60 +++++++++++++++++++++++++++++++++++----------------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index 6b942539..f5fa1f8b 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -289,11 +289,17 @@ PARALLEL_FOR_LOOP
   ///////////////////////////////////////////////////////////
   // Unified Comms buffers for all directions
   ///////////////////////////////////////////////////////////
-  std::vector<commVector<scalar_object> > u_simd_send_buf;
-  std::vector<commVector<scalar_object> > u_simd_recv_buf;
+  // Vectors that live on the symmetric heap in case of SHMEM
+  std::vector<commVector<scalar_object> > u_simd_send_buf_hide;
+  std::vector<commVector<scalar_object> > u_simd_recv_buf_hide;
   commVector<cobj>          u_send_buf;
   commVector<cobj>          u_recv_buf_hide;
+  // These are used; either SHM objects or refs to the above symmetric heap vectors
+  // depending on comms target
   cobj* u_recv_buf_p;
+  cobj* u_send_buf_p;
+  std::vector<scalar_object *> u_simd_send_buf;
+  std::vector<scalar_object *> u_simd_recv_buf;
 
   int u_comm_offset;
   int _unified_buffer_size;
@@ -427,32 +433,42 @@ PARALLEL_FOR_LOOP
 	}
       }
     }
-    u_send_buf.resize(_unified_buffer_size);
 
     /////////////////////////////////////////////////////////////////////////////////
     // Try to allocate for receiving in a shared memory region, fall back to buffer
     /////////////////////////////////////////////////////////////////////////////////
-    if( ShmDirectCopy ) {
+    const int Nsimd = grid->Nsimd();
 
-      u_recv_buf_p=(cobj *)_grid->ShmBufferSelf();
-      if ( u_recv_buf_p == NULL ) {
-	u_recv_buf_hide.resize(_unified_buffer_size);
-	u_recv_buf_p=&u_recv_buf_hide[0];
+    uint8_t *shm_ptr   = (uint8_t *)_grid->ShmBufferSelf();
+
+    u_simd_send_buf.resize(Nsimd);
+    u_simd_recv_buf.resize(Nsimd);
+
+    u_send_buf.resize(_unified_buffer_size);
+
+    if( ShmDirectCopy && shm_ptr != NULL ) {
+
+      u_recv_buf_p=(cobj *)shm_ptr; shm_ptr+= _unified_buffer_size*sizeof(cobj);
+      for(int l=0;l<Nsimd;l++){
+	u_simd_send_buf[l] = (scalar_object *)shm_ptr; shm_ptr += _unified_buffer_size*sizeof(scalar_object);
+	u_simd_recv_buf[l] = (scalar_object *)shm_ptr; shm_ptr += _unified_buffer_size*sizeof(scalar_object);
       }
+
     } else {
+
       u_recv_buf_hide.resize(_unified_buffer_size);
+      u_simd_send_buf_hide.resize(Nsimd,commVector<scalar_object>(_unified_buffer_size));
+      u_simd_recv_buf_hide.resize(Nsimd,commVector<scalar_object>(_unified_buffer_size));
+
       u_recv_buf_p=&u_recv_buf_hide[0];
+      for(int l=0;l<Nsimd;l++){
+	u_simd_send_buf[l] = & u_simd_send_buf_hide[l][0];
+	u_simd_recv_buf[l] = & u_simd_recv_buf_hide[l][0];
+      }
     }
 
     PrecomputeByteOffsets();
 
-    const int Nsimd = grid->Nsimd();
-    u_simd_send_buf.resize(Nsimd);
-    u_simd_recv_buf.resize(Nsimd);
-    for(int l=0;l<Nsimd;l++){
-      u_simd_send_buf[l].resize(_unified_buffer_size);
-      u_simd_recv_buf[l].resize(_unified_buffer_size);
-    }
   }
 
   void Local     (int point, int dimension,int shiftpm,int cbmask)
@@ -930,10 +946,19 @@ PARALLEL_FOR_LOOP
 	    int xmit_to_rank;
 	    
 	    _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
-	    
+ 
+
 	    AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
+
+	    auto shm_or_rp = rp;
+	    if (ShmDirectCopy) { 
+	      scalar_object *shm = (scalar_object *) _grid->ShmBufferTranslate(xmit_to_rank,sp);
+	      if ( shm!=NULL) { 
+		shm_or_rp = shm;
+	      }
+	    }
 	    
-	    rpointers[i] = rp;
+	    rpointers[i] = shm_or_rp;
 	    
 	  } else { 
 	    
@@ -942,7 +967,6 @@ PARALLEL_FOR_LOOP
 	  }
 	}
 
-	assert(0);
 	AddMerge(&u_recv_buf_p[u_comm_offset],rpointers,buffer_size,Packets.size()-1);
 	
 	u_comm_offset     +=buffer_size;

From b6a65059a21f00c2d9f2a01f10f550996d6acb02 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Mon, 24 Oct 2016 17:30:43 +0100
Subject: [PATCH 089/149] Update to use shared memory to contain the stencil
 comms buffers Tested on 2.1.1.1 1.2.1.1 4.1.1.1 1.4.1.1 2.2.1.1 subnode
 decompositions

---
 benchmarks/Benchmark_dwf.cc            |  21 +-
 lib/Init.cc                            |  14 +-
 lib/Init.h                             |   2 +
 lib/Log.cc                             |   5 +
 lib/Log.h                              |  59 ++--
 lib/Makefile.am                        |   4 +
 lib/Stencil.h                          |  73 ++---
 lib/communicator/Communicator_base.cc  | 132 +++++++++
 lib/communicator/Communicator_base.h   | 313 ++++++++++----------
 lib/communicator/Communicator_mpi.cc   |  41 +--
 lib/communicator/Communicator_mpi3.cc  | 388 ++++++++++++++-----------
 lib/communicator/Communicator_none.cc  |  55 ++--
 lib/communicator/Communicator_shmem.cc |  57 ++--
 13 files changed, 706 insertions(+), 458 deletions(-)
 create mode 100644 lib/communicator/Communicator_base.cc

diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index f75f0385..c9e2fa22 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -156,6 +156,7 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+    assert (norm2(err)< 1.0e-5 );
     Dw.Report();
   }
 
@@ -208,7 +209,7 @@ int main (int argc, char ** argv)
 
     std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
 
-    RealF sum=0;
+    RealD sum=0;
     for(int x=0;x<latt4[0];x++){
     for(int y=0;y<latt4[1];y++){
     for(int z=0;z<latt4[2];z++){
@@ -226,12 +227,12 @@ int main (int argc, char ** argv)
       }
     }}}}}
     std::cout<<GridLogMessage<<" difference between normal and simd is "<<sum<<std::endl;
+    assert (sum< 1.0e-5 );
 
 
     if (1) {
 
       LatticeFermion sr_eo(sFGrid);
-      LatticeFermion serr(sFGrid);
 
       LatticeFermion ssrc_e (sFrbGrid);
       LatticeFermion ssrc_o (sFrbGrid);
@@ -243,8 +244,6 @@ int main (int argc, char ** argv)
 
       setCheckerboard(sr_eo,ssrc_o);
       setCheckerboard(sr_eo,ssrc_e);
-      serr = sr_eo-ssrc; 
-      std::cout<<GridLogMessage << "EO src norm diff   "<< norm2(serr)<<std::endl;
 
       sr_e = zero;
       sr_o = zero;
@@ -272,9 +271,18 @@ int main (int argc, char ** argv)
       pickCheckerboard(Even,ssrc_e,sresult);
       pickCheckerboard(Odd ,ssrc_o,sresult);
       ssrc_e = ssrc_e - sr_e;
+      RealD error = norm2(ssrc_e);
+
       std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm"<<norm2(sr_e) <<std::endl;
       ssrc_o = ssrc_o - sr_o;
+
+      error+= norm2(ssrc_o);
       std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm"<<norm2(sr_o) <<std::endl;
+      if(error>1.0e-5) { 
+	setCheckerboard(ssrc,ssrc_o);
+	setCheckerboard(ssrc,ssrc_e);
+	std::cout<< ssrc << std::endl;
+      }
     }
 
 
@@ -306,7 +314,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
   err = ref-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-
+  assert(norm2(err)<1.0e-5);
   LatticeFermion src_e (FrbGrid);
   LatticeFermion src_o (FrbGrid);
   LatticeFermion r_e   (FrbGrid);
@@ -349,11 +357,14 @@ int main (int argc, char ** argv)
 
   err = r_eo-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+  assert(norm2(err)<1.0e-5);
 
   pickCheckerboard(Even,src_e,err);
   pickCheckerboard(Odd,src_o,err);
   std::cout<<GridLogMessage << "norm diff even  "<< norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "norm diff odd   "<< norm2(src_o)<<std::endl;
+  assert(norm2(src_e)<1.0e-5);
+  assert(norm2(src_o)<1.0e-5);
 
 
   }
diff --git a/lib/Init.cc b/lib/Init.cc
index a21cbce0..a2a8526a 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -171,14 +171,17 @@ std::string GridCmdVectorIntToString(const std::vector<int> & vec){
 /////////////////////////////////////////////////////////
 //
 /////////////////////////////////////////////////////////
+static int Grid_is_initialised = 0;
+
+
 void Grid_init(int *argc,char ***argv)
 {
+  GridLogger::StopWatch.Start();
+
   CartesianCommunicator::Init(argc,argv);
 
   // Parse command line args.
 
-  GridLogger::StopWatch.Start();
-
   std::string arg;
   std::vector<std::string> logstreams;
   std::string defaultLog("Error,Warning,Message,Performance");
@@ -216,11 +219,14 @@ void Grid_init(int *argc,char ***argv)
   if( GridCmdOptionExists(*argv,*argv+*argc,"--lebesgue") ){
     LebesgueOrder::UseLebesgueOrder=1;
   }
-
   if( GridCmdOptionExists(*argv,*argv+*argc,"--cacheblocking") ){
     arg= GridCmdOptionPayload(*argv,*argv+*argc,"--cacheblocking");
     GridCmdOptionIntVector(arg,LebesgueOrder::Block);
   }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--timestamp") ){
+    GridLogTimestamp(1);
+  }
+
   GridParseLayout(*argv,*argc,
 		  Grid_default_latt,
 		  Grid_default_mpi);
@@ -274,6 +280,8 @@ void Grid_init(int *argc,char ***argv)
   std::cout << "GNU General Public License for more details."<<std::endl;
   std::cout << COL_BACKGROUND <<std::endl;
   std::cout << std::endl;
+
+  Grid_is_initialised = 1;
 }
 
   
diff --git a/lib/Init.h b/lib/Init.h
index 25fda569..d68cee37 100644
--- a/lib/Init.h
+++ b/lib/Init.h
@@ -33,6 +33,7 @@ namespace Grid {
 
   void Grid_init(int *argc,char ***argv);
   void Grid_finalize(void);
+
   // internal, controled with --handle
   void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr);
   void Grid_debug_handler_init(void);
@@ -44,6 +45,7 @@ namespace Grid {
   const std::vector<int> &GridDefaultMpi(void);
   const int              &GridThreads(void)  ;
   void                    GridSetThreads(int t) ;
+  void GridLogTimestamp(int);
 
   // Common parsing chores
   std::string GridCmdOptionPayload(char ** begin, char ** end, const std::string & option);
diff --git a/lib/Log.cc b/lib/Log.cc
index a55a1c9e..d4ac42ee 100644
--- a/lib/Log.cc
+++ b/lib/Log.cc
@@ -34,8 +34,13 @@ directory
 namespace Grid {
 
 GridStopWatch Logger::StopWatch;
+int Logger::timestamp;
 std::ostream Logger::devnull(0);
 
+void GridLogTimestamp(int on){
+  Logger::Timestamp(on);
+}
+
 Colours GridLogColours(0);
 GridLogger GridLogError(1, "Error", GridLogColours, "RED");
 GridLogger GridLogWarning(1, "Warning", GridLogColours, "YELLOW");
diff --git a/lib/Log.h b/lib/Log.h
index 156f52ee..dd3fe927 100644
--- a/lib/Log.h
+++ b/lib/Log.h
@@ -37,10 +37,11 @@
 #include <execinfo.h>
 #endif
 
-    namespace Grid {
+namespace Grid {
 
+//////////////////////////////////////////////////////////////////////////////////////////////////
 // Dress the output; use std::chrono for time stamping via the StopWatch class
-int Rank(void); // used for early stage debug before library init
+//////////////////////////////////////////////////////////////////////////////////////////////////
 
 
 class Colours{
@@ -55,7 +56,6 @@ public:
 
   void Active(bool activate){
     is_active=activate;
-
     if (is_active){
      colour["BLACK"]  ="\033[30m";
      colour["RED"]    ="\033[31m";
@@ -66,21 +66,18 @@ public:
      colour["CYAN"]   ="\033[36m";
      colour["WHITE"]  ="\033[37m";
      colour["NORMAL"] ="\033[0;39m";
-   } else {
-    colour["BLACK"] ="";
-    colour["RED"]   ="";
-    colour["GREEN"] ="";
-    colour["YELLOW"]="";
-    colour["BLUE"]  ="";
-    colour["PURPLE"]="";
-    colour["CYAN"]  ="";
-    colour["WHITE"] ="";
-    colour["NORMAL"]="";
-  }
-
-
-};
-
+    } else {
+      colour["BLACK"] ="";
+      colour["RED"]   ="";
+      colour["GREEN"] ="";
+      colour["YELLOW"]="";
+      colour["BLUE"]  ="";
+      colour["PURPLE"]="";
+      colour["CYAN"]  ="";
+      colour["WHITE"] ="";
+      colour["NORMAL"]="";
+    }
+  };
 };
 
 
@@ -88,6 +85,7 @@ class Logger {
 protected:
   Colours &Painter;
   int active;
+  static int timestamp;
   std::string name, topName;
   std::string COLOUR;
 
@@ -99,25 +97,28 @@ public:
   std::string evidence() {return Painter.colour["YELLOW"];}
   std::string colour() {return Painter.colour[COLOUR];}
 
-  Logger(std::string topNm, int on, std::string nm, Colours& col_class, std::string col)
-  : active(on),
-  name(nm),
-  topName(topNm),
-  Painter(col_class),
-  COLOUR(col){} ;
+  Logger(std::string topNm, int on, std::string nm, Colours& col_class, std::string col)  : active(on),
+    name(nm),
+    topName(topNm),
+    Painter(col_class),
+    COLOUR(col) {} ;
   
   void Active(int on) {active = on;};
   int  isActive(void) {return active;};
+  static void Timestamp(int on) {timestamp = on;};
   
   friend std::ostream& operator<< (std::ostream& stream, Logger& log){
 
     if ( log.active ) {
-      StopWatch.Stop();
-      GridTime now = StopWatch.Elapsed();
-      StopWatch.Start();
       stream << log.background()<< log.topName << log.background()<< " : ";
       stream << log.colour() <<std::setw(14) << std::left << log.name << log.background() << " : ";
-      stream << log.evidence()<< now << log.background() << " : " << log.colour();
+      if ( log.timestamp ) {
+	StopWatch.Stop();
+	GridTime now = StopWatch.Elapsed();
+	StopWatch.Start();
+	stream << log.evidence()<< now << log.background() << " : " ;
+      }
+      stream << log.colour();
       return stream;
     } else { 
       return devnull;
@@ -149,7 +150,7 @@ extern void * Grid_backtrace_buffer[_NBACKTRACE];
 
 #define BACKTRACEFILE() {\
 char string[20];					\
-std::sprintf(string,"backtrace.%d",Rank());				\
+std::sprintf(string,"backtrace.%d",CartesianCommunicator::RankWorld()); \
 std::FILE * fp = std::fopen(string,"w");				\
 BACKTRACEFP(fp)\
 std::fclose(fp);	    \
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 35e76cc8..3285a545 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -1,18 +1,22 @@
 extra_sources=
 if BUILD_COMMS_MPI
   extra_sources+=communicator/Communicator_mpi.cc
+  extra_sources+=communicator/Communicator_base.cc
 endif
 
 if BUILD_COMMS_MPI3
   extra_sources+=communicator/Communicator_mpi3.cc
+  extra_sources+=communicator/Communicator_base.cc
 endif
 
 if BUILD_COMMS_SHMEM
   extra_sources+=communicator/Communicator_shmem.cc
+  extra_sources+=communicator/Communicator_base.cc
 endif
 
 if BUILD_COMMS_NONE
   extra_sources+=communicator/Communicator_none.cc
+  extra_sources+=communicator/Communicator_base.cc
 endif
 
 #
diff --git a/lib/Stencil.h b/lib/Stencil.h
index f5fa1f8b..f22acb6f 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -290,10 +290,11 @@ PARALLEL_FOR_LOOP
   // Unified Comms buffers for all directions
   ///////////////////////////////////////////////////////////
   // Vectors that live on the symmetric heap in case of SHMEM
-  std::vector<commVector<scalar_object> > u_simd_send_buf_hide;
-  std::vector<commVector<scalar_object> > u_simd_recv_buf_hide;
-  commVector<cobj>          u_send_buf;
-  commVector<cobj>          u_recv_buf_hide;
+  //  std::vector<commVector<scalar_object> > u_simd_send_buf_hide;
+  //  std::vector<commVector<scalar_object> > u_simd_recv_buf_hide;
+  //  commVector<cobj>          u_send_buf_hide;
+  //  commVector<cobj>          u_recv_buf_hide;
+
   // These are used; either SHM objects or refs to the above symmetric heap vectors
   // depending on comms target
   cobj* u_recv_buf_p;
@@ -439,36 +440,19 @@ PARALLEL_FOR_LOOP
     /////////////////////////////////////////////////////////////////////////////////
     const int Nsimd = grid->Nsimd();
 
-    uint8_t *shm_ptr   = (uint8_t *)_grid->ShmBufferSelf();
+    _grid->ShmBufferFreeAll();
 
     u_simd_send_buf.resize(Nsimd);
     u_simd_recv_buf.resize(Nsimd);
 
-    u_send_buf.resize(_unified_buffer_size);
-
-    if( ShmDirectCopy && shm_ptr != NULL ) {
-
-      u_recv_buf_p=(cobj *)shm_ptr; shm_ptr+= _unified_buffer_size*sizeof(cobj);
-      for(int l=0;l<Nsimd;l++){
-	u_simd_send_buf[l] = (scalar_object *)shm_ptr; shm_ptr += _unified_buffer_size*sizeof(scalar_object);
-	u_simd_recv_buf[l] = (scalar_object *)shm_ptr; shm_ptr += _unified_buffer_size*sizeof(scalar_object);
-      }
-
-    } else {
-
-      u_recv_buf_hide.resize(_unified_buffer_size);
-      u_simd_send_buf_hide.resize(Nsimd,commVector<scalar_object>(_unified_buffer_size));
-      u_simd_recv_buf_hide.resize(Nsimd,commVector<scalar_object>(_unified_buffer_size));
-
-      u_recv_buf_p=&u_recv_buf_hide[0];
-      for(int l=0;l<Nsimd;l++){
-	u_simd_send_buf[l] = & u_simd_send_buf_hide[l][0];
-	u_simd_recv_buf[l] = & u_simd_recv_buf_hide[l][0];
-      }
+    u_send_buf_p=(cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj));
+    u_recv_buf_p=(cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj));
+    for(int l=0;l<Nsimd;l++){
+      u_simd_recv_buf[l] = (scalar_object *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(scalar_object));
+      u_simd_send_buf[l] = (scalar_object *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(scalar_object));
     }
 
     PrecomputeByteOffsets();
-
   }
 
   void Local     (int point, int dimension,int shiftpm,int cbmask)
@@ -698,6 +682,7 @@ PARALLEL_FOR_LOOP
     calls++;
     Mergers.resize(0);
     Packets.resize(0);
+    _grid->StencilBarrier();
     HaloGather(source,compress);
     this->CommunicateBegin(reqs);
     this->CommunicateComplete(reqs);
@@ -836,19 +821,17 @@ PARALLEL_FOR_LOOP
 	// try the direct copy if possible
 	/////////////////////////////////////////////////////////
 
-	cobj *u_send_buf_p = &u_send_buf[0];
-	if (ShmDirectCopy) { 
-	  cobj *shm = (cobj *) _grid->ShmBuffer(xmit_to_rank);
-	  if ( shm!=NULL) { 
-	    u_send_buf_p = shm;
-	  }
+
+	cobj *send_buf = (cobj *)_grid->ShmBufferTranslate(xmit_to_rank,u_recv_buf_p);
+	if ( (ShmDirectCopy==0)||send_buf==NULL ) { 
+	  cobj *send_buf = u_send_buf_p;
 	}
 	
 	t_data-=usecond();
-	Gather_plane_simple_table         (face_table[face_idx],rhs,u_send_buf_p,compress,u_comm_offset,so);  face_idx++;
+	Gather_plane_simple_table         (face_table[face_idx],rhs,send_buf,compress,u_comm_offset,so);  face_idx++;
 	t_data+=usecond();
 	
-	AddPacket((void *)&u_send_buf_p[u_comm_offset],
+	AddPacket((void *)&send_buf[u_comm_offset],
 		  (void *)&u_recv_buf_p[u_comm_offset],
 		  xmit_to_rank,
 		  recv_from_rank,
@@ -947,18 +930,16 @@ PARALLEL_FOR_LOOP
 	    
 	    _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
  
-
-	    AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
-
-	    auto shm_or_rp = rp;
-	    if (ShmDirectCopy) { 
-	      scalar_object *shm = (scalar_object *) _grid->ShmBufferTranslate(xmit_to_rank,sp);
-	      if ( shm!=NULL) { 
-		shm_or_rp = shm;
-	      }
-	    }
+	    scalar_object *shm = (scalar_object *) _grid->ShmBufferTranslate(recv_from_rank,sp);
+	    if ((ShmDirectCopy==0)||(shm==NULL)) { 
+	      shm = rp;
+	    } 
 	    
-	    rpointers[i] = shm_or_rp;
+	    // if Direct, StencilSendToRecvFrom will suppress copy to a peer on node
+	    // assuming above pointer flip
+	    AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
+	    
+	    rpointers[i] = shm;
 	    
 	  } else { 
 	    
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
new file mode 100644
index 00000000..1272b6a2
--- /dev/null
+++ b/lib/communicator/Communicator_base.cc
@@ -0,0 +1,132 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/communicator/Communicator_none.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include "Grid.h"
+namespace Grid {
+
+///////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////
+int CartesianCommunicator::ShmRank;
+int CartesianCommunicator::ShmSize;
+int CartesianCommunicator::GroupRank;
+int CartesianCommunicator::GroupSize;
+int CartesianCommunicator::WorldRank;
+int CartesianCommunicator::WorldSize;
+int CartesianCommunicator::Slave;
+void *              CartesianCommunicator::ShmCommBuf;
+
+/////////////////////////////////
+// Alloc, free shmem region
+/////////////////////////////////
+void *CartesianCommunicator::ShmBufferMalloc(size_t bytes){
+  //  bytes = (bytes+sizeof(vRealD))&(~(sizeof(vRealD)-1));// align up bytes
+  void *ptr = (void *)heap_top;
+  heap_top  += bytes;
+  heap_bytes+= bytes;
+  assert(heap_bytes < MAX_MPI_SHM_BYTES);
+  return ptr;
+}
+void *CartesianCommunicator::ShmBufferFreeAll(void) { 
+  heap_top  =(size_t)ShmBufferSelf();
+  heap_bytes=0;
+}
+
+/////////////////////////////////
+// Grid information queries
+/////////////////////////////////
+int                      CartesianCommunicator::IsBoss(void)            { return _processor==0; };
+int                      CartesianCommunicator::BossRank(void)          { return 0; };
+int                      CartesianCommunicator::ThisRank(void)          { return _processor; };
+const std::vector<int> & CartesianCommunicator::ThisProcessorCoor(void) { return _processor_coor; };
+const std::vector<int> & CartesianCommunicator::ProcessorGrid(void)     { return _processors; };
+int                      CartesianCommunicator::ProcessorCount(void)    { return _Nprocessors; };
+
+////////////////////////////////////////////////////////////////////////////////
+// very VERY rarely (Log, serial RNG) we need world without a grid
+////////////////////////////////////////////////////////////////////////////////
+int  CartesianCommunicator::RankWorld(void) { return WorldRank; };
+int CartesianCommunicator::Ranks    (void) { return WorldSize; };
+int CartesianCommunicator::Nodes    (void) { return GroupSize; };
+int CartesianCommunicator::Cores    (void) { return ShmSize;   };
+int CartesianCommunicator::NodeRank (void) { return GroupRank; };
+int CartesianCommunicator::CoreRank (void) { return ShmRank;   };
+
+void CartesianCommunicator::GlobalSum(ComplexF &c)
+{
+  GlobalSumVector((float *)&c,2);
+}
+void CartesianCommunicator::GlobalSumVector(ComplexF *c,int N)
+{
+  GlobalSumVector((float *)c,2*N);
+}
+void CartesianCommunicator::GlobalSum(ComplexD &c)
+{
+  GlobalSumVector((double *)&c,2);
+}
+void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
+{
+  GlobalSumVector((double *)c,2*N);
+}
+
+#ifndef GRID_COMMS_MPI3
+
+void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						       void *xmit,
+						       int xmit_to_rank,
+						       void *recv,
+						       int recv_from_rank,
+						       int bytes)
+{
+  SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
+}
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall)
+{
+  SendToRecvFromComplete(waitall);
+}
+void StencilBarrier(void){};
+
+commVector<uint8_t> CartesianCommunicator::ShmBufStorageVector;
+
+void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; }
+void *CartesianCommunicator::ShmBuffer(int rank) {
+  if (rank != ShmRank ) return NULL;
+  else                  return ShmCommBuf;
+}
+void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { 
+  if (rank != ShmRank ) return NULL;
+  else                  return local_p;
+}
+void CartesianCommunicator::ShmInitGeneric(void){
+  ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
+  ShmCommBuf=(void *)&ShmBufStorageVector[0];
+}
+
+#endif
+  
+}
+
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 4139f72a..576c749e 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -40,169 +40,188 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_COMMS_SHMEM
 #include <mpp/shmem.h>
 #endif
+
 namespace Grid {
+
 class CartesianCommunicator {
   public:    
 
+  // 65536 ranks per node adequate for now
+  // 128MB shared memory for comms enought for 48^4 local vol comms
+  // Give external control (command line override?) of this
+
+  static const int      MAXLOG2RANKSPERNODE = 16;            
+  static const uint64_t MAX_MPI_SHM_BYTES   = 128*1024*1024; 
+
   // Communicator should know nothing of the physics grid, only processor grid.
-  
-    int              _Nprocessors;     // How many in all
-    std::vector<int> _processors;      // Which dimensions get relayed out over processors lanes.
-    int              _processor;       // linear processor rank
-    std::vector<int> _processor_coor;  // linear processor coordinate
-    unsigned long _ndimension;
+  int              _Nprocessors;     // How many in all
+  std::vector<int> _processors;      // Which dimensions get relayed out over processors lanes.
+  int              _processor;       // linear processor rank
+  std::vector<int> _processor_coor;  // linear processor coordinate
+  unsigned long _ndimension;
 
-#ifdef GRID_COMMS_MPI
-    MPI_Comm communicator;
-    typedef MPI_Request CommsRequest_t;
-#elif  GRID_COMMS_MPI3
-    int shm_mode;
-
-    MPI_Comm communicator;
-    typedef MPI_Request CommsRequest_t;
-
-    const int MAXLOG2RANKSPERNODE = 16;     // 65536 ranks per node adequate for now
-    const uint64_t MAX_MPI_SHM_BYTES = 256*1024*1024; // 256MB shared memory for comms enought for 48^4 local vol comms
-
-    std::vector<int>  WorldDims;
-    std::vector<int>  GroupDims;
-    std::vector<int>  ShmDims;
-
-    std::vector<int> GroupCoor;
-    std::vector<int> ShmCoor;
-    std::vector<int> WorldCoor;
-
-    static std::vector<int> GroupRanks; 
-    static std::vector<int> MyGroup;
-    static int ShmSetup;
-    static MPI_Win ShmWindow; 
-    static MPI_Comm ShmComm;
-
-    void * ShmCommBuf;
-    std::vector<void *> ShmCommBufs;
-
-    int WorldRank;
-    int WorldSize;
-
-    static int ShmRank;
-    static int ShmSize;
-    static int GroupSize;
-    static int GroupRank;
-
-    std::vector<int>  LexicographicToWorldRank;
+#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3)
+  MPI_Comm communicator;
+  static MPI_Comm communicator_world;
+  typedef MPI_Request CommsRequest_t;
 #else 
-    typedef int CommsRequest_t;
+  typedef int CommsRequest_t;
 #endif
 
-    static void Init(int *argc, char ***argv);
+  ////////////////////////////////////////////////////////////////////
+  // Helper functionality for SHM Windows common to all other impls
+  ////////////////////////////////////////////////////////////////////
+  // Longer term; drop this in favour of a master / slave model with 
+  // cartesian communicator on a subset of ranks, slave ranks controlled
+  // by group leader with data xfer via shared memory
+  ////////////////////////////////////////////////////////////////////
+#ifdef  GRID_COMMS_MPI3
+  std::vector<int>  WorldDims;
+  std::vector<int>  GroupDims;
+  std::vector<int>  ShmDims;
+  
+  std::vector<int> GroupCoor;
+  std::vector<int> ShmCoor;
+  std::vector<int> WorldCoor;
+  
+  static std::vector<int> GroupRanks; 
+  static std::vector<int> MyGroup;
+  static int ShmSetup;
+  static MPI_Win ShmWindow; 
+  static MPI_Comm ShmComm;
+  
+  std::vector<int>  LexicographicToWorldRank;
+  
+  static std::vector<void *> ShmCommBufs;
+#else 
+  static void ShmInitGeneric(void);
+  static commVector<uint8_t> ShmBufStorageVector;
+#endif 
+  static void * ShmCommBuf;
+  size_t heap_top;
+  size_t heap_bytes;
+  void *ShmBufferSelf(void);
+  void *ShmBuffer(int rank);
+  void *ShmBufferTranslate(int rank,void * local_p);
+  void *ShmBufferMalloc(size_t bytes);
+  void *ShmBufferFreeAll(void) ;
+  
+  ////////////////////////////////////////////////
+  // Must call in Grid startup
+  ////////////////////////////////////////////////
+  static void Init(int *argc, char ***argv);
+  
+  ////////////////////////////////////////////////
+  // Constructor of any given grid
+  ////////////////////////////////////////////////
+  CartesianCommunicator(const std::vector<int> &pdimensions_in);
+  
+  ////////////////////////////////////////////////////////////////////////////////////////
+  // Wraps MPI_Cart routines, or implements equivalent on other impls
+  ////////////////////////////////////////////////////////////////////////////////////////
+  void ShiftedRanks(int dim,int shift,int & source, int & dest);
+  int  RankFromProcessorCoor(std::vector<int> &coor);
+  void ProcessorCoorFromRank(int rank,std::vector<int> &coor);
+  
+  /////////////////////////////////
+  // Grid information and queries
+  /////////////////////////////////
+  static int ShmRank;
+  static int ShmSize;
+  static int GroupSize;
+  static int GroupRank;
+  static int WorldRank;
+  static int WorldSize;
+  static int Slave;
+  
+  int                      IsBoss(void)            ;
+  int                      BossRank(void)          ;
+  int                      ThisRank(void)          ;
+  const std::vector<int> & ThisProcessorCoor(void) ;
+  const std::vector<int> & ProcessorGrid(void)     ;
+  int                      ProcessorCount(void)    ;
+  static int Ranks    (void);
+  static int Nodes    (void);
+  static int Cores    (void);
+  static int NodeRank (void);
+  static int CoreRank (void);
 
-    // Constructor
-    CartesianCommunicator(const std::vector<int> &pdimensions_in);
+  ////////////////////////////////////////////////////////////////////////////////
+  // very VERY rarely (Log, serial RNG) we need world without a grid
+  ////////////////////////////////////////////////////////////////////////////////
+  static int  RankWorld(void) ;
+  static void BroadcastWorld(int root,void* data, int bytes);
+  
+  ////////////////////////////////////////////////////////////
+  // Reduction
+  ////////////////////////////////////////////////////////////
+  void GlobalSum(RealF &);
+  void GlobalSumVector(RealF *,int N);
+  void GlobalSum(RealD &);
+  void GlobalSumVector(RealD *,int N);
+  void GlobalSum(uint32_t &);
+  void GlobalSum(uint64_t &);
+  void GlobalSum(ComplexF &c);
+  void GlobalSumVector(ComplexF *c,int N);
+  void GlobalSum(ComplexD &c);
+  void GlobalSumVector(ComplexD *c,int N);
+  
+  template<class obj> void GlobalSum(obj &o){
+    typedef typename obj::scalar_type scalar_type;
+    int words = sizeof(obj)/sizeof(scalar_type);
+    scalar_type * ptr = (scalar_type *)& o;
+    GlobalSumVector(ptr,words);
+  }
+  
+  ////////////////////////////////////////////////////////////
+  // Face exchange, buffer swap in translational invariant way
+  ////////////////////////////////////////////////////////////
+  void SendToRecvFrom(void *xmit,
+		      int xmit_to_rank,
+		      void *recv,
+		      int recv_from_rank,
+		      int bytes);
+  
+  void SendRecvPacket(void *xmit,
+		      void *recv,
+		      int xmit_to_rank,
+		      int recv_from_rank,
+		      int bytes);
+  
+  void SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+			   void *xmit,
+			   int xmit_to_rank,
+			   void *recv,
+			   int recv_from_rank,
+			   int bytes);
+  
+  void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
 
-    // Wraps MPI_Cart routines
-    void ShiftedRanks(int dim,int shift,int & source, int & dest);
-    int  RankFromProcessorCoor(std::vector<int> &coor);
-    void ProcessorCoorFromRank(int rank,std::vector<int> &coor);
+  void StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+				  void *xmit,
+				  int xmit_to_rank,
+				  void *recv,
+				  int recv_from_rank,
+				  int bytes);
+  
+  void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
+  void StencilBarrier(void);
 
-    // Helper function for SHM Windows in MPI3
-    void *ShmBufferSelf(void);
-    void *ShmBuffer(int rank);
-
-    /////////////////////////////////
-    // Grid information queries
-    /////////////////////////////////
-    int                      IsBoss(void)            { return _processor==0; };
-    int                      BossRank(void)          { return 0; };
-    int                      ThisRank(void)          { return _processor; };
-    const std::vector<int> & ThisProcessorCoor(void) { return _processor_coor; };
-    const std::vector<int> & ProcessorGrid(void)     { return _processors; };
-    int                      ProcessorCount(void)    { return _Nprocessors; };
-
-    ////////////////////////////////////////////////////////////
-    // Reduction
-    ////////////////////////////////////////////////////////////
-    void GlobalSum(RealF &);
-    void GlobalSumVector(RealF *,int N);
-
-    void GlobalSum(RealD &);
-    void GlobalSumVector(RealD *,int N);
-
-    void GlobalSum(uint32_t &);
-    void GlobalSum(uint64_t &);
-
-    void GlobalSum(ComplexF &c)
-    {
-      GlobalSumVector((float *)&c,2);
-    }
-    void GlobalSumVector(ComplexF *c,int N)
-    {
-      GlobalSumVector((float *)c,2*N);
-    }
-
-    void GlobalSum(ComplexD &c)
-    {
-      GlobalSumVector((double *)&c,2);
-    }
-    void GlobalSumVector(ComplexD *c,int N)
-    {
-      GlobalSumVector((double *)c,2*N);
-    }
-    
-    template<class obj> void GlobalSum(obj &o){
-      typedef typename obj::scalar_type scalar_type;
-      int words = sizeof(obj)/sizeof(scalar_type);
-      scalar_type * ptr = (scalar_type *)& o;
-      GlobalSumVector(ptr,words);
-    }
-    ////////////////////////////////////////////////////////////
-    // Face exchange, buffer swap in translational invariant way
-    ////////////////////////////////////////////////////////////
-    void SendToRecvFrom(void *xmit,
-			int xmit_to_rank,
-			void *recv,
-			int recv_from_rank,
-			int bytes);
-
-    void SendRecvPacket(void *xmit,
-			void *recv,
-			int xmit_to_rank,
-			int recv_from_rank,
-			int bytes);
-
-    void SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-			 void *xmit,
-			 int xmit_to_rank,
-			 void *recv,
-			 int recv_from_rank,
-			 int bytes);
-    void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
-    void StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-			 void *xmit,
-			 int xmit_to_rank,
-			 void *recv,
-			 int recv_from_rank,
-			 int bytes);
-    void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall)
-    {
-      SendToRecvFromComplete(waitall);
-    }
-
-    ////////////////////////////////////////////////////////////
-    // Barrier
-    ////////////////////////////////////////////////////////////
-    void Barrier(void);
-
-    ////////////////////////////////////////////////////////////
-    // Broadcast a buffer and composite larger
-    ////////////////////////////////////////////////////////////
-    void Broadcast(int root,void* data, int bytes);
-    template<class obj> void Broadcast(int root,obj &data)
+  ////////////////////////////////////////////////////////////
+  // Barrier
+  ////////////////////////////////////////////////////////////
+  void Barrier(void);
+  
+  ////////////////////////////////////////////////////////////
+  // Broadcast a buffer and composite larger
+  ////////////////////////////////////////////////////////////
+  void Broadcast(int root,void* data, int bytes);
+  
+  template<class obj> void Broadcast(int root,obj &data)
     {
       Broadcast(root,(void *)&data,sizeof(data));
     };
 
-    static void BroadcastWorld(int root,void* data, int bytes);
-
 }; 
 }
 
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index 4291b319..a638eebb 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -30,19 +30,28 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
-  // Should error check all MPI calls.
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+MPI_Comm CartesianCommunicator::communicator_world;
+
+// Should error check all MPI calls.
 void CartesianCommunicator::Init(int *argc, char ***argv) {
   int flag;
   MPI_Initialized(&flag); // needed to coexist with other libs apparently
   if ( !flag ) {
     MPI_Init(argc,argv);
   }
-}
-
-int Rank(void) {
-  int pe;
-  MPI_Comm_rank(MPI_COMM_WORLD,&pe);
-  return pe;
+  MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
+  MPI_Comm_rank(communicator_world,&WorldRank);
+  MPI_Comm_size(communicator_world,&WorldSize);
+  ShmRank=0;
+  ShmSize=1;
+  GroupRank=WorldRank;
+  GroupSize=WorldSize;
+  Slave    =0;
+  ShmInitGeneric();
 }
 
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
@@ -54,7 +63,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   _processors = processors;
   _processor_coor.resize(_ndimension);
   
-  MPI_Cart_create(MPI_COMM_WORLD, _ndimension,&_processors[0],&periodic[0],1,&communicator);
+  MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator);
   MPI_Comm_rank(communicator,&_processor);
   MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
 
@@ -67,15 +76,6 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   
   assert(Size==_Nprocessors);
 }
-void *CartesianCommunicator::ShmBufferSelf(void)
-{
-  return NULL;
-}
-void *CartesianCommunicator::ShmBuffer(int rank)
-{
-  return NULL;
-}
-
 void CartesianCommunicator::GlobalSum(uint32_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
   assert(ierr==0);
@@ -194,14 +194,17 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
 		     communicator);
   assert(ierr==0);
 }
-
+  ///////////////////////////////////////////////////////
+  // Should only be used prior to Grid Init finished.
+  // Check for this?
+  ///////////////////////////////////////////////////////
 void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
 {
   int ierr= MPI_Bcast(data,
 		      bytes,
 		      MPI_BYTE,
 		      root,
-		      MPI_COMM_WORLD);
+		      communicator_world);
   assert(ierr==0);
 }
 
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index f5bbdbda..00b0ca11 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -1,4 +1,3 @@
-
     /*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
@@ -33,26 +32,197 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 
 
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+int CartesianCommunicator::ShmSetup = 0;
 
-// Global used by Init and nowhere else. How to hide?
-int Rank(void) {
-  int pe;
-  MPI_Comm_rank(MPI_COMM_WORLD,&pe);
-  return pe;
+MPI_Comm CartesianCommunicator::communicator_world;
+MPI_Comm CartesianCommunicator::ShmComm;
+MPI_Win  CartesianCommunicator::ShmWindow;
+
+std::vector<int> CartesianCommunicator::GroupRanks;  
+std::vector<int> CartesianCommunicator::MyGroup;
+std::vector<void *> CartesianCommunicator::ShmCommBufs;
+
+void *CartesianCommunicator::ShmBufferSelf(void)
+{
+  return ShmCommBufs[ShmRank];
 }
-  // Should error check all MPI calls.
+void *CartesianCommunicator::ShmBuffer(int rank)
+{
+  int gpeer = GroupRanks[rank];
+  if (gpeer == MPI_UNDEFINED){
+    return NULL;
+  } else { 
+    return ShmCommBufs[gpeer];
+  }
+}
+void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p)
+{
+  int gpeer = GroupRanks[rank];
+  if (gpeer == MPI_UNDEFINED){
+    return NULL;
+  } else { 
+    uint64_t offset = (uint64_t)local_p - (uint64_t)ShmCommBufs[ShmRank];
+    uint64_t remote = (uint64_t)ShmCommBufs[gpeer]+offset;
+    return (void *) remote;
+  }
+}
+
 void CartesianCommunicator::Init(int *argc, char ***argv) {
   int flag;
   MPI_Initialized(&flag); // needed to coexist with other libs apparently
   if ( !flag ) {
     MPI_Init(argc,argv);
   }
-}
-  ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Want to implement some magic ... Group sub-cubes into those on same node
-  //
-  ////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
+  MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
+  MPI_Comm_rank(communicator_world,&WorldRank);
+  MPI_Comm_size(communicator_world,&WorldSize);
+
+  /////////////////////////////////////////////////////////////////////
+  // Split into groups that can share memory
+  /////////////////////////////////////////////////////////////////////
+  MPI_Comm_split_type(communicator_world, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm);
+  MPI_Comm_rank(ShmComm     ,&ShmRank);
+  MPI_Comm_size(ShmComm     ,&ShmSize);
+  GroupSize = WorldSize/ShmSize;
+
+  /////////////////////////////////////////////////////////////////////
+  // find world ranks in our SHM group (i.e. which ranks are on our node)
+  /////////////////////////////////////////////////////////////////////
+  MPI_Group WorldGroup, ShmGroup;
+  MPI_Comm_group (communicator_world, &WorldGroup); 
+  MPI_Comm_group (ShmComm, &ShmGroup);
+  
+  std::vector<int> world_ranks(WorldSize); 
+  GroupRanks.resize(WorldSize); 
+  MyGroup.resize(ShmSize);
+  for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
+  
+  MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &GroupRanks[0]); 
+
+  ///////////////////////////////////////////////////////////////////
+  // Identify who is in my group and noninate the leader
+    ///////////////////////////////////////////////////////////////////
+  int g=0;
+  for(int rank=0;rank<WorldSize;rank++){
+    if(GroupRanks[rank]!=MPI_UNDEFINED){
+      assert(g<ShmSize);
+      MyGroup[g++] = rank;
+    }
+  }
+  
+  std::sort(MyGroup.begin(),MyGroup.end(),std::less<int>());
+  int myleader = MyGroup[0];
+  
+  std::vector<int> leaders_1hot(WorldSize,0);
+  std::vector<int> leaders_group(GroupSize,0);
+  leaders_1hot [ myleader ] = 1;
+    
+  ///////////////////////////////////////////////////////////////////
+  // global sum leaders over comm world
+  ///////////////////////////////////////////////////////////////////
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,communicator_world);
+  assert(ierr==0);
+  
+  ///////////////////////////////////////////////////////////////////
+  // find the group leaders world rank
+  ///////////////////////////////////////////////////////////////////
+  int group=0;
+  for(int l=0;l<WorldSize;l++){
+    if(leaders_1hot[l]){
+      leaders_group[group++] = l;
+    }
+  }
+  
+  ///////////////////////////////////////////////////////////////////
+  // Identify the rank of the group in which I (and my leader) live
+  ///////////////////////////////////////////////////////////////////
+  GroupRank=-1;
+  for(int g=0;g<GroupSize;g++){
+    if (myleader == leaders_group[g]){
+      GroupRank=g;
+    }
+  }
+  assert(GroupRank!=-1);
+  
+  //////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // allocate the shared window for our group
+  //////////////////////////////////////////////////////////////////////////////////////////////////////////
+  
+  ShmCommBuf = 0;
+  ierr = MPI_Win_allocate_shared(MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,ShmComm,&ShmCommBuf,&ShmWindow);
+  assert(ierr==0);
+  // KNL hack -- force to numa-domain 1 in flat
+#if 0
+  for(uint64_t page=0;page<MAX_MPI_SHM_BYTES;page+=4096){
+    void *pages = (void *) ( page + ShmCommBuf );
+    int status;
+    int flags=MPOL_MF_MOVE_ALL;
+    int nodes=1; // numa domain == MCDRAM
+    unsigned long count=1;
+    ierr= move_pages(0,count, &pages,&nodes,&status,flags);
+    if (ierr && (page==0)) perror("numa relocate command failed");
+  }
+#endif
+  MPI_Win_lock_all (MPI_MODE_NOCHECK, ShmWindow);
+  
+  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Plan: allocate a fixed SHM region. Scratch that is just used via some scheme during stencil comms, with no allocate free.
+  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+  ShmCommBufs.resize(ShmSize);
+  for(int r=0;r<ShmSize;r++){
+    MPI_Aint sz;
+    int dsp_unit;
+    MPI_Win_shared_query (ShmWindow, r, &sz, &dsp_unit, &ShmCommBufs[r]);
+  }
+
+  //////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Verbose for now
+  //////////////////////////////////////////////////////////////////////////////////////////////////////////
+  if (WorldRank == 0){
+    std::cout<<GridLogMessage<< "Grid MPI-3 configuration: detected ";
+    std::cout<< WorldSize << " Ranks " ;
+    std::cout<< GroupSize << " Nodes " ;
+    std::cout<<  ShmSize  << " with ranks-per-node "<<std::endl;
+    
+    std::cout<<GridLogMessage     <<"Grid MPI-3 configuration: allocated shared memory region of size ";
+    std::cout<<std::hex << MAX_MPI_SHM_BYTES <<" ShmCommBuf address = "<<ShmCommBuf << std::dec<<std::endl;
+
+    for(int g=0;g<GroupSize;g++){
+      std::cout<<GridLogMessage<<" Node "<<g<<" led by MPI rank "<<leaders_group[g]<<std::endl;
+    }
+
+    std::cout<<GridLogMessage<<" Boss Node Shm Pointers are {";
+    for(int g=0;g<ShmSize;g++){
+      std::cout<<std::hex<<ShmCommBufs[g]<<std::dec;
+      if(g!=ShmSize-1) std::cout<<",";
+      else std::cout<<"}"<<std::endl;
+    }
+
+  }
+  
+  for(int g=0;g<GroupSize;g++){
+    if ( (ShmRank == 0) && (GroupRank==g) )  std::cout<<GridLogMessage<<"["<<g<<"] Node Group "<<g<<" is ranks {";
+    for(int r=0;r<ShmSize;r++){
+      if ( (ShmRank == 0) && (GroupRank==g) ) {
+	std::cout<<MyGroup[r];
+	if(r<ShmSize-1) std::cout<<",";
+	else std::cout<<"}"<<std::endl;
+      }
+      MPI_Barrier(communicator_world);
+    }
+  }
+  
+  assert(ShmSetup==0);  ShmSetup=1;
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Want to implement some magic ... Group sub-cubes into those on same node
+////////////////////////////////////////////////////////////////////////////////////////////////////////////
 void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
 {
   std::vector<int> coor = _processor_coor;
@@ -80,139 +250,13 @@ void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &c
   rank = LexicographicToWorldRank[rank];
 }
 
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Info that is setup once and indept of cartesian layout
-///////////////////////////////////////////////////////////////////////////////////////////////////
-int CartesianCommunicator::ShmSetup = 0;
-int CartesianCommunicator::ShmRank;
-int CartesianCommunicator::ShmSize;
-int CartesianCommunicator::GroupRank;
-int CartesianCommunicator::GroupSize;
-MPI_Comm CartesianCommunicator::ShmComm;
-MPI_Win  CartesianCommunicator::ShmWindow;
-std::vector<int> CartesianCommunicator::GroupRanks; 
-std::vector<int> CartesianCommunicator::MyGroup;
-
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 { 
-
-  _ndimension = processors.size();
-
-  WorldDims = processors;
-
-  communicator = MPI_COMM_WORLD;
-  MPI_Comm_rank(communicator,&WorldRank);
-  MPI_Comm_size(communicator,&WorldSize);
-
-  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Plan: allocate a fixed SHM region. Scratch that is just used via some scheme during stencil comms, with no allocate free.
-  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Does every grid need one, or could we share across all grids via a singleton/guard?
   int ierr;
 
-  if ( !ShmSetup ) { 
+  communicator=communicator_world;
 
-    MPI_Comm_split_type(communicator, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm);
-    MPI_Comm_rank(ShmComm     ,&ShmRank);
-    MPI_Comm_size(ShmComm     ,&ShmSize);
-    GroupSize = WorldSize/ShmSize;
-
-    /////////////////////////////////////////////////////////////////////
-    // find world ranks in our SHM group (i.e. which ranks are on our node)
-    /////////////////////////////////////////////////////////////////////
-    MPI_Group WorldGroup, ShmGroup;
-    MPI_Comm_group (communicator, &WorldGroup); 
-    MPI_Comm_group (ShmComm, &ShmGroup);
-
-    std::vector<int> world_ranks(WorldSize); 
-    GroupRanks.resize(WorldSize); 
-    MyGroup.resize(ShmSize);
-    for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
-  
-    MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &GroupRanks[0]); 
-
-    ///////////////////////////////////////////////////////////////////
-    // Identify who is in my group and noninate the leader
-    ///////////////////////////////////////////////////////////////////
-    int g=0;
-    for(int rank=0;rank<WorldSize;rank++){
-      if(GroupRanks[rank]!=MPI_UNDEFINED){
-	assert(g<ShmSize);
-	MyGroup[g++] = rank;
-      }
-    }
-  
-    std::sort(MyGroup.begin(),MyGroup.end(),std::greater<int>());
-    int myleader = MyGroup[0];
-    
-    std::vector<int> leaders_1hot(WorldSize,0);
-    std::vector<int> leaders_group(GroupSize,0);
-    leaders_1hot [ myleader ] = 1;
-    
-    ///////////////////////////////////////////////////////////////////
-    // global sum leaders over comm world
-    ///////////////////////////////////////////////////////////////////
-    ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,communicator);
-    assert(ierr==0);
-  
-    ///////////////////////////////////////////////////////////////////
-    // find the group leaders world rank
-    ///////////////////////////////////////////////////////////////////
-    int group=0;
-    for(int l=0;l<WorldSize;l++){
-      if(leaders_1hot[l]){
-	leaders_group[group++] = l;
-      }
-    }
-  
-    ///////////////////////////////////////////////////////////////////
-    // Identify the rank of the group in which I (and my leader) live
-    ///////////////////////////////////////////////////////////////////
-    GroupRank=-1;
-    for(int g=0;g<GroupSize;g++){
-      if (myleader == leaders_group[g]){
-	GroupRank=g;
-      }
-    }
-    assert(GroupRank!=-1);
-    
-    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    // allocate the shared window for our group
-    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    
-    ShmCommBuf = 0;
-    ierr = MPI_Win_allocate_shared(MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,ShmComm,&ShmCommBuf,&ShmWindow);
-    assert(ierr==0);
-    for(uint64_t page=0;page<MAX_MPI_SHM_BYTES;page+=4096){
-      void *pages = (void *) ( page + ShmCommBuf );
-      int status;
-      int flags=MPOL_MF_MOVE_ALL;
-      int nodes=1; // numa domain == MCDRAM
-      unsigned long count=1;
-      ierr= move_pages(0,count, &pages,&nodes,&status,flags);
-      if (ierr && (page==0)) perror("numa relocate command failed");
-    }
-    MPI_Win_lock_all (MPI_MODE_NOCHECK, ShmWindow);
-
-    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    // Verbose for now
-    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    std::cout<<GridLogMessage<< "MPI-3 configuration: Ranks per node "<< ShmSize ;
-    std::cout<< " Nodes "<< GroupSize;
-    std::cout<< " Ranks "<< WorldSize;
-    std::cout<< " Shm CommBuf address"<< std::hex <<ShmCommBuf << std::dec<<std::endl;
-
-    // Done
-    ShmSetup=1;
-
-  }
-
-  ShmCommBufs.resize(ShmSize);
-  for(int r=0;r<ShmSize;r++){
-    MPI_Aint sz;
-    int dsp_unit;
-    MPI_Win_shared_query (ShmWindow, r, &sz, &dsp_unit, &ShmCommBufs[r]);
-  }
+  _ndimension = processors.size();
   
   ////////////////////////////////////////////////////////////////
   // Assert power of two shm_size.
@@ -232,6 +276,8 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
   ////////////////////////////////////////////////////////////////
   int dim = 0;
   
+  std::vector<int> WorldDims = processors;
+
   ShmDims.resize(_ndimension,1);
   GroupDims.resize(_ndimension);
     
@@ -346,21 +392,6 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
   }
 }
 
-
-void *CartesianCommunicator::ShmBufferSelf(void)
-{
-  return ShmCommBufs[ShmRank];
-}
-void *CartesianCommunicator::ShmBuffer(int rank)
-{
-  int gpeer = GroupRanks[rank];
-  if (gpeer == MPI_UNDEFINED){
-    return NULL;
-  } else { 
-    return ShmCommBufs[gpeer];
-  }
-}
-
 // Basic Halo comms primitive
 void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 						void *xmit,
@@ -369,6 +400,7 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 						int from,
 						int bytes)
 {
+#if 1
   MPI_Request xrq;
   MPI_Request rrq;
   
@@ -387,12 +419,11 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 
   sequence++;
   
-  char *to_ptr   = (char *)ShmCommBufs[gdest];
   char *from_ptr = (char *)ShmCommBufs[ShmRank];
 
   int small = (bytes<MAX_MPI_SHM_BYTES);
 
-  typedef vRealD T;
+  typedef uint64_t T;
   int words = bytes/sizeof(T);
 
   assert(((size_t)bytes &(sizeof(T)-1))==0);
@@ -400,13 +431,18 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 
   if ( small && (gdest !=MPI_UNDEFINED) ) {
 
+    char *to_ptr   = (char *)ShmCommBufs[gdest];
+
     assert(gme != gdest);
 
     T *ip = (T *)xmit;
     T *op = (T *)to_ptr;
 PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
-      vstream(op[w],ip[w]);
+      op[w]=ip[w];
+      if ( w == 0 ) { 
+	//	std::cout << " xmit "<< ShmRank <<" -> "<< gdest<<" " <<std::hex<<op[w]<<std::dec<<std::endl;
+      }
     }
 
     bcopy(&_processor,&to_ptr[bytes],sizeof(_processor));
@@ -426,7 +462,10 @@ PARALLEL_FOR_LOOP
     T *op = (T *)recv;
 PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
-      vstream(op[w],ip[w]);
+      op[w]=ip[w];
+      if ( w == 0 ) { 
+	//	std::cout << " recv "<< ShmRank <<" <- "<< gfrom<<" " <<std::hex<<op[w]<<std::dec<<std::endl;
+      }
     }
     bcopy(&from_ptr[bytes]  ,&tag  ,sizeof(tag));
     bcopy(&from_ptr[bytes+4],&check,sizeof(check));
@@ -441,6 +480,19 @@ PARALLEL_FOR_LOOP
   MPI_Win_sync (ShmWindow);   
   MPI_Barrier  (ShmComm);
   MPI_Win_sync (ShmWindow);   
+#else
+  MPI_Request xrq;
+  MPI_Request rrq;
+  int rank = _processor;
+  int ierr;
+  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+  
+  assert(ierr==0);
+
+  list.push_back(xrq);
+  list.push_back(rrq);
+#endif
 }
 
 void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
@@ -476,19 +528,29 @@ void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_
     list.push_back(rrq);
   }
 
+
+  StencilBarrier();
+
+}
+
+
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list)
+{
+  SendToRecvFromComplete(list);
+}
+
+void CartesianCommunicator::StencilBarrier(void)
+{
   MPI_Win_sync (ShmWindow);   
   MPI_Barrier  (ShmComm);
   MPI_Win_sync (ShmWindow);   
-  
 }
 
-
 void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
 {
   int nreq=list.size();
   std::vector<MPI_Status> status(nreq);
   int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
-
   assert(ierr==0);
 }
 
@@ -514,7 +576,7 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
 		      bytes,
 		      MPI_BYTE,
 		      root,
-		      MPI_COMM_WORLD);
+		      communicator_world);
   assert(ierr==0);
 }
 
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index 80b8fb90..d32fe4fa 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -28,18 +28,29 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include "Grid.h"
 namespace Grid {
 
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+int CartesianCommunicator::ShmRank;
+int CartesianCommunicator::ShmSize;
+int CartesianCommunicator::GroupRank;
+int CartesianCommunicator::GroupSize;
+int CartesianCommunicator::WorldRank;
+int CartesianCommunicator::WorldSize;
+int CartesianCommunicator::Slave;
+void *              CartesianCommunicator::ShmCommBuf;
+commVector<uint8_t> CartesianCommunicator::ShmBufStorageVector;
+
 void CartesianCommunicator::Init(int *argc, char *** arv)
 {
-}
-
-int Rank(void ){ return 0; };
-void *CartesianCommunicator::ShmBufferSelf(void)
-{
-  return NULL;
-}
-void *CartesianCommunicator::ShmBuffer(int rank)
-{
-  return NULL;
+  WorldRank = 0;
+  WorldSize = 1;
+  ShmRank=0;
+  ShmSize=1;
+  GroupRank=_WorldRank;
+  GroupSize=_WorldSize;
+  Slave    =0;
+  ShmInitGeneric();
 }
 
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
@@ -97,30 +108,16 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &
   assert(0);
 }
 
-void CartesianCommunicator::Barrier(void)
-{
-}
-
-void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
-{
-}
-void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
-{
-}
-
-
+void CartesianCommunicator::Barrier(void){}
+void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {}
+void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { }
+int  CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor) {  return 0;}
+void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor){  assert(0);}
 void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
 {
   source =0;
   dest=0;
 }
-int CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor)
-{
-  return 0;
-}
-void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor)
-{
-}
 
 
 }
diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc
index 4af719b0..544b37c7 100644
--- a/lib/communicator/Communicator_shmem.cc
+++ b/lib/communicator/Communicator_shmem.cc
@@ -39,25 +39,19 @@ namespace Grid {
     BACKTRACEFILE();		   \
   }\
 }
-int Rank(void) {
-  return shmem_my_pe();
-}
-typedef struct HandShake_t { 
-  uint64_t seq_local;
-  uint64_t seq_remote;
-} HandShake;
 
-static Vector< HandShake > XConnections;
-static Vector< HandShake > RConnections;
 
-void *CartesianCommunicator::ShmBufferSelf(void)
-{
-  return NULL;
-}
-void *CartesianCommunicator::ShmBuffer(int rank)
-{
-  return NULL;
-}
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+int CartesianCommunicator::ShmRank;
+int CartesianCommunicator::ShmSize;
+int CartesianCommunicator::GroupRank;
+int CartesianCommunicator::GroupSize;
+int CartesianCommunicator::WorldRank;
+int CartesianCommunicator::WorldSize;
+int CartesianCommunicator::Slave;
+
 void CartesianCommunicator::Init(int *argc, char ***argv) {
   shmem_init();
   XConnections.resize(shmem_n_pes());
@@ -69,7 +63,36 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     RConnections[pe].seq_remote= 0;
   }
   shmem_barrier_all();
+  ShmInitGeneric();
 }
+
+
+// Should error check all MPI calls.
+void CartesianCommunicator::Init(int *argc, char ***argv) {
+  int flag;
+  MPI_Initialized(&flag); // needed to coexist with other libs apparently
+  if ( !flag ) {
+    MPI_Init(argc,argv);
+    MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
+    MPI_Comm_rank(communicator_world,&_WorldRank);
+    MPI_Comm_size(communicator_world,&_WorldSize);
+    _ShmRank=0;
+    _ShmSize=1;
+    _GroupRank=_WorldRank;
+    _GroupSize=_WorldSize;
+    _Slave    =0;
+  }
+}
+
+
+typedef struct HandShake_t { 
+  uint64_t seq_local;
+  uint64_t seq_remote;
+} HandShake;
+
+static Vector< HandShake > XConnections;
+static Vector< HandShake > RConnections;
+
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
   _ndimension = processors.size();

From 392e0645137a988dea02f74e27ba2a8ad8182328 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 24 Oct 2016 19:24:21 +0100
Subject: [PATCH 090/149] fast local peek-poke

---
 lib/lattice/Lattice_peekpoke.h | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/lattice/Lattice_peekpoke.h b/lib/lattice/Lattice_peekpoke.h
index 9bece943..19d349c4 100644
--- a/lib/lattice/Lattice_peekpoke.h
+++ b/lib/lattice/Lattice_peekpoke.h
@@ -154,7 +154,7 @@ PARALLEL_FOR_LOOP
     template<class vobj,class sobj>
     void peekLocalSite(sobj &s,const Lattice<vobj> &l,std::vector<int> &site){
         
-      GridBase *grid=l._grid;
+      GridBase *grid = l._grid;
 
       typedef typename vobj::scalar_type scalar_type;
       typedef typename vobj::vector_type vector_type;
@@ -164,16 +164,18 @@ PARALLEL_FOR_LOOP
       assert( l.checkerboard== l._grid->CheckerBoard(site));
       assert( sizeof(sobj)*Nsimd == sizeof(vobj));
 
+      static const int words=sizeof(vobj)/sizeof(vector_type);
       int odx,idx;
       idx= grid->iIndex(site);
       odx= grid->oIndex(site);
 
-      std::vector<sobj> buf(Nsimd);
-
-      extract(l._odata[odx],buf);
+      scalar_type * vp = (scalar_type *)&l._odata[odx];
+      scalar_type * pt = (scalar_type *)&s;
+      
+      for(int w=0;w<words;w++){
+        pt[w] = vp[idx+w*Nsimd];
+      }
       
-      s = buf[idx];
-
       return;
     };
 
@@ -190,18 +192,17 @@ PARALLEL_FOR_LOOP
       assert( l.checkerboard== l._grid->CheckerBoard(site));
       assert( sizeof(sobj)*Nsimd == sizeof(vobj));
 
+      static const int words=sizeof(vobj)/sizeof(vector_type);
       int odx,idx;
       idx= grid->iIndex(site);
       odx= grid->oIndex(site);
 
-      std::vector<sobj> buf(Nsimd);
-
-      // extract-modify-merge cycle is easiest way and this is not perf critical
-      extract(l._odata[odx],buf);
+      scalar_type * vp = (scalar_type *)&l._odata[odx];
+      scalar_type * pt = (scalar_type *)&s;
       
-      buf[idx] = s;
-
-      merge(l._odata[odx],buf);
+      for(int w=0;w<words;w++){
+        vp[idx+w*Nsimd] = pt[w];
+      }
 
       return;
     };

From a795b5705e8ab938fcfd244d022bb2a9ce396810 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 24 Oct 2016 19:25:15 +0100
Subject: [PATCH 091/149] memory optimisation

---
 lib/cartesian/Cartesian_base.h      | 2 +-
 lib/cartesian/Cartesian_full.h      | 2 +-
 lib/cartesian/Cartesian_red_black.h | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h
index 8a24c87b..72b21ee3 100644
--- a/lib/cartesian/Cartesian_base.h
+++ b/lib/cartesian/Cartesian_base.h
@@ -77,7 +77,7 @@ public:
     // GridCartesian / GridRedBlackCartesian
     ////////////////////////////////////////////////////////////////
     virtual int CheckerBoarded(int dim)=0;
-    virtual int CheckerBoard(std::vector<int> site)=0;
+    virtual int CheckerBoard(std::vector<int> &site)=0;
     virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0;
     virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0;
     virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int cb)=0;
diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h
index 14ab8b55..b0d20441 100644
--- a/lib/cartesian/Cartesian_full.h
+++ b/lib/cartesian/Cartesian_full.h
@@ -49,7 +49,7 @@ public:
     virtual int CheckerBoarded(int dim){
       return 0;
     }
-    virtual int CheckerBoard(std::vector<int> site){
+    virtual int CheckerBoard(std::vector<int> &site){
         return 0;
     }
     virtual int CheckerBoardDestination(int cb,int shift,int dim){
diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h
index db0508d5..6a4300d7 100644
--- a/lib/cartesian/Cartesian_red_black.h
+++ b/lib/cartesian/Cartesian_red_black.h
@@ -49,7 +49,7 @@ public:
       if( dim==_checker_dim) return 1;
       else return 0;
     }
-    virtual int CheckerBoard(std::vector<int> site){
+    virtual int CheckerBoard(std::vector<int> &site){
       int linear=0;
       assert(site.size()==_ndimension);
       for(int d=0;d<_ndimension;d++){ 

From 13bf0482e3165514feeccf626d60add2aa91910c Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 24 Oct 2016 19:25:40 +0100
Subject: [PATCH 092/149] FFT optimisation

---
 lib/FFT.h | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index 4cda6483..17060dc3 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -200,18 +200,14 @@ namespace Grid {
 					       sign,FFTW_ESTIMATE);
 	}
 
-	double add,mul,fma;
-	FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
-	flops_call = add+mul+2.0*fma;
-
-	GridStopWatch timer;
+    std::vector<int> lcoor(Nd), gcoor(Nd);
 
 	// Barrel shift and collect global pencil
 	for(int p=0;p<processors[dim];p++) { 
 
 	  for(int idx=0;idx<sgrid->lSites();idx++) { 
 
-	    std::vector<int> lcoor(Nd);
+	    
     	    sgrid->LocalIndexToLocalCoor(idx,lcoor);
 
 	    sobj s;
@@ -228,14 +224,11 @@ namespace Grid {
 	
 	// Loop over orthog coords
 	int NN=pencil_g.lSites();
-
-	GridStopWatch Timer;
-	Timer.Start();
+	GridStopWatch timer;
+	timer.Start();
 
 PARALLEL_FOR_LOOP
-	for(int idx=0;idx<NN;idx++) { 
-
-	  std::vector<int> lcoor(Nd);
+	for(int idx=0;idx<NN;idx++) {
 	  pencil_g.LocalIndexToLocalCoor(idx,lcoor);
 
 	  if ( lcoor[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
@@ -245,15 +238,17 @@ PARALLEL_FOR_LOOP
 	  }
 	}
 
-        Timer.Stop();
-	usec += Timer.useconds();
-	flops+= flops_call*NN;
+        timer.Stop();
 
+          double add,mul,fma;
+          FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
+          flops_call = add+mul+2.0*fma;
+          usec += timer.useconds();
+          flops+= flops_call*NN;
         int pc = processor_coor[dim];
-        for(int idx=0;idx<sgrid->lSites();idx++) { 
-	  std::vector<int> lcoor(Nd);
+        for(int idx=0;idx<sgrid->lSites();idx++) {
 	  sgrid->LocalIndexToLocalCoor(idx,lcoor);
-	  std::vector<int> gcoor = lcoor;
+	  gcoor = lcoor;
 	  // extract the result
 	  sobj s;
 	  gcoor[dim] = lcoor[dim]+L*pc;

From b94478fa51c46039bfc6beda741790b65dd2c46e Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Mon, 24 Oct 2016 23:45:31 +0100
Subject: [PATCH 093/149] mpi, mpi3, shmem all compile. mpi, mpi3 pass single
 node multi-rank

---
 lib/AlignedAllocator.h                 |  2 +-
 lib/Stencil.h                          | 31 ++++++-------
 lib/communicator/Communicator_base.cc  | 12 ++---
 lib/communicator/Communicator_mpi3.cc  | 24 +++-------
 lib/communicator/Communicator_none.cc  |  4 +-
 lib/communicator/Communicator_shmem.cc | 61 +++++++-------------------
 6 files changed, 49 insertions(+), 85 deletions(-)

diff --git a/lib/AlignedAllocator.h b/lib/AlignedAllocator.h
index 44929ca8..fa001adc 100644
--- a/lib/AlignedAllocator.h
+++ b/lib/AlignedAllocator.h
@@ -141,7 +141,7 @@ public:
 
     if ( bcast != ptr ) {
       std::printf("inconsistent alloc pe %d %lx %lx \n",shmem_my_pe(),bcast,ptr);std::fflush(stdout);
-      BACKTRACEFILE();
+      //      BACKTRACEFILE();
       exit(0);
     }
     assert( bcast == (void *) ptr);
diff --git a/lib/Stencil.h b/lib/Stencil.h
index f22acb6f..221785b4 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -32,8 +32,6 @@
 
  #include <Grid/stencil/Lebesgue.h>   // subdir aggregate
 
-const int ShmDirectCopy = 1;
-
  //////////////////////////////////////////////////////////////////////////////////////////
  // Must not lose sight that goal is to be able to construct really efficient
  // gather to a point stencil code. CSHIFT is not the best way, so need
@@ -170,13 +168,13 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
     reqs.resize(Packets.size());
     commtime-=usecond();
     for(int i=0;i<Packets.size();i++){
-      if( ShmDirectCopy ) {
 	_grid->StencilSendToRecvFromBegin(reqs[i],
 					  Packets[i].send_buf,
 					  Packets[i].to_rank,
 					  Packets[i].recv_buf,
 					  Packets[i].from_rank,
 					  Packets[i].bytes);
+	/*
       }else{
 	_grid->SendToRecvFromBegin(reqs[i],
 				   Packets[i].send_buf,
@@ -185,17 +183,19 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 				   Packets[i].from_rank,
 				   Packets[i].bytes);
       }
+	*/
     }
     commtime+=usecond();
   }
   void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs)
   {
     commtime-=usecond();
+
     for(int i=0;i<Packets.size();i++){
-      if( ShmDirectCopy ) 
+      //      if( ShmDirectCopy ) 
 	_grid->StencilSendToRecvFromComplete(reqs[i]);
-      else 
-	_grid->SendToRecvFromComplete(reqs[i]);
+	//      else 
+	//	_grid->SendToRecvFromComplete(reqs[i]);
     }
     commtime+=usecond();
   }
@@ -253,8 +253,6 @@ PARALLEL_FOR_LOOP
   // Flat vector, change layout for cache friendly.
   Vector<StencilEntry>  _entries;
   
-  inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; }
-  
   void PrecomputeByteOffsets(void){
     for(int i=0;i<_entries.size();i++){
       if( _entries[i]._is_local ) {
@@ -265,9 +263,7 @@ PARALLEL_FOR_LOOP
     }
   };
 
-  inline uint64_t Touch(int ent) {
-    //	 _mm_prefetch((char *)&_entries[ent],_MM_HINT_T0);
-  }
+  inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; }
   inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
     uint64_t cbase = (uint64_t)&u_recv_buf_p[0];
     local = _entries[ent]._is_local;
@@ -685,7 +681,9 @@ PARALLEL_FOR_LOOP
     _grid->StencilBarrier();
     HaloGather(source,compress);
     this->CommunicateBegin(reqs);
+    _grid->StencilBarrier();
     this->CommunicateComplete(reqs);
+    _grid->StencilBarrier();
     CommsMerge(); // spins
   }
   
@@ -823,11 +821,13 @@ PARALLEL_FOR_LOOP
 
 
 	cobj *send_buf = (cobj *)_grid->ShmBufferTranslate(xmit_to_rank,u_recv_buf_p);
-	if ( (ShmDirectCopy==0)||send_buf==NULL ) { 
-	  cobj *send_buf = u_send_buf_p;
+	if ( (send_buf==NULL) ) { 
+	  send_buf = u_send_buf_p;
 	}
-	
+	//	std::cout << " send_bufs  "<<std::hex<< send_buf <<" ubp "<<u_send_buf_p <<std::dec<<std::endl;
 	t_data-=usecond();
+	assert(u_send_buf_p!=NULL);
+	assert(send_buf!=NULL);
 	Gather_plane_simple_table         (face_table[face_idx],rhs,send_buf,compress,u_comm_offset,so);  face_idx++;
 	t_data+=usecond();
 	
@@ -931,7 +931,8 @@ PARALLEL_FOR_LOOP
 	    _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); 
  
 	    scalar_object *shm = (scalar_object *) _grid->ShmBufferTranslate(recv_from_rank,sp);
-	    if ((ShmDirectCopy==0)||(shm==NULL)) { 
+	    //	    if ((ShmDirectCopy==0)||(shm==NULL)) { 
+	    if (shm==NULL) { 
 	      shm = rp;
 	    } 
 	    
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 1272b6a2..91e9cf9b 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -69,7 +69,7 @@ int                      CartesianCommunicator::ProcessorCount(void)    { return
 ////////////////////////////////////////////////////////////////////////////////
 // very VERY rarely (Log, serial RNG) we need world without a grid
 ////////////////////////////////////////////////////////////////////////////////
-int  CartesianCommunicator::RankWorld(void) { return WorldRank; };
+int  CartesianCommunicator::RankWorld(void){ return WorldRank; };
 int CartesianCommunicator::Ranks    (void) { return WorldSize; };
 int CartesianCommunicator::Nodes    (void) { return GroupSize; };
 int CartesianCommunicator::Cores    (void) { return ShmSize;   };
@@ -108,22 +108,22 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsReque
 {
   SendToRecvFromComplete(waitall);
 }
-void StencilBarrier(void){};
+void CartesianCommunicator::StencilBarrier(void){};
 
 commVector<uint8_t> CartesianCommunicator::ShmBufStorageVector;
 
 void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; }
+
 void *CartesianCommunicator::ShmBuffer(int rank) {
-  if (rank != ShmRank ) return NULL;
-  else                  return ShmCommBuf;
+  return NULL;
 }
 void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { 
-  if (rank != ShmRank ) return NULL;
-  else                  return local_p;
+  return NULL;
 }
 void CartesianCommunicator::ShmInitGeneric(void){
   ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
   ShmCommBuf=(void *)&ShmBufStorageVector[0];
+  std::cout << "allocated persistent buffer"<<std::hex << ShmCommBuf << std::dec<<std::endl;
 }
 
 #endif
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 00b0ca11..61e5e06b 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -400,7 +400,9 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 						int from,
 						int bytes)
 {
-#if 1
+#if 0
+  this->StencilBarrier();
+
   MPI_Request xrq;
   MPI_Request rrq;
   
@@ -440,9 +442,6 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
       op[w]=ip[w];
-      if ( w == 0 ) { 
-	//	std::cout << " xmit "<< ShmRank <<" -> "<< gdest<<" " <<std::hex<<op[w]<<std::dec<<std::endl;
-      }
     }
 
     bcopy(&_processor,&to_ptr[bytes],sizeof(_processor));
@@ -452,20 +451,15 @@ PARALLEL_FOR_LOOP
     assert(ierr==0);
     list.push_back(xrq);
   }
-  
-  MPI_Win_sync (ShmWindow);   
-  MPI_Barrier  (ShmComm);
-  MPI_Win_sync (ShmWindow);   
 
+  this->StencilBarrier();
+  
   if (small && (gfrom !=MPI_UNDEFINED) ) {
     T *ip = (T *)from_ptr;
     T *op = (T *)recv;
 PARALLEL_FOR_LOOP 
     for(int w=0;w<words;w++) {
       op[w]=ip[w];
-      if ( w == 0 ) { 
-	//	std::cout << " recv "<< ShmRank <<" <- "<< gfrom<<" " <<std::hex<<op[w]<<std::dec<<std::endl;
-      }
     }
     bcopy(&from_ptr[bytes]  ,&tag  ,sizeof(tag));
     bcopy(&from_ptr[bytes+4],&check,sizeof(check));
@@ -477,9 +471,8 @@ PARALLEL_FOR_LOOP
     list.push_back(rrq);
   }
 
-  MPI_Win_sync (ShmWindow);   
-  MPI_Barrier  (ShmComm);
-  MPI_Win_sync (ShmWindow);   
+  this->StencilBarrier();
+
 #else
   MPI_Request xrq;
   MPI_Request rrq;
@@ -528,9 +521,6 @@ void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_
     list.push_back(rrq);
   }
 
-
-  StencilBarrier();
-
 }
 
 
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index d32fe4fa..f4e4c575 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -47,8 +47,8 @@ void CartesianCommunicator::Init(int *argc, char *** arv)
   WorldSize = 1;
   ShmRank=0;
   ShmSize=1;
-  GroupRank=_WorldRank;
-  GroupSize=_WorldSize;
+  GroupRank=WorldRank;
+  GroupSize=WorldSize;
   Slave    =0;
   ShmInitGeneric();
 }
diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc
index 544b37c7..d5f3ed76 100644
--- a/lib/communicator/Communicator_shmem.cc
+++ b/lib/communicator/Communicator_shmem.cc
@@ -44,13 +44,16 @@ namespace Grid {
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////////////////////////////////////////
-int CartesianCommunicator::ShmRank;
-int CartesianCommunicator::ShmSize;
-int CartesianCommunicator::GroupRank;
-int CartesianCommunicator::GroupSize;
-int CartesianCommunicator::WorldRank;
-int CartesianCommunicator::WorldSize;
-int CartesianCommunicator::Slave;
+
+typedef struct HandShake_t { 
+  uint64_t seq_local;
+  uint64_t seq_remote;
+} HandShake;
+
+
+static Vector< HandShake > XConnections;
+static Vector< HandShake > RConnections;
+
 
 void CartesianCommunicator::Init(int *argc, char ***argv) {
   shmem_init();
@@ -62,37 +65,17 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     RConnections[pe].seq_local = 0;
     RConnections[pe].seq_remote= 0;
   }
+  WorldSize = shmem_n_pes();
+  WorldRank = shmem_my_pe();
+  ShmRank=0;
+  ShmSize=1;
+  GroupRank=WorldRank;
+  GroupSize=WorldSize;
+  Slave    =0;
   shmem_barrier_all();
   ShmInitGeneric();
 }
 
-
-// Should error check all MPI calls.
-void CartesianCommunicator::Init(int *argc, char ***argv) {
-  int flag;
-  MPI_Initialized(&flag); // needed to coexist with other libs apparently
-  if ( !flag ) {
-    MPI_Init(argc,argv);
-    MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
-    MPI_Comm_rank(communicator_world,&_WorldRank);
-    MPI_Comm_size(communicator_world,&_WorldSize);
-    _ShmRank=0;
-    _ShmSize=1;
-    _GroupRank=_WorldRank;
-    _GroupSize=_WorldSize;
-    _Slave    =0;
-  }
-}
-
-
-typedef struct HandShake_t { 
-  uint64_t seq_local;
-  uint64_t seq_remote;
-} HandShake;
-
-static Vector< HandShake > XConnections;
-static Vector< HandShake > RConnections;
-
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
   _ndimension = processors.size();
@@ -261,12 +244,9 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
 
   if ( _processor == sender ) {
 
-    printf("Sender SHMEM pt2pt %d -> %d\n",sender,receiver);
     // Check he has posted a receive
     while(SendSeq->seq_remote == SendSeq->seq_local);
 
-    printf("Sender receive %d posted\n",sender,receiver);
-
     // Advance our send count
     seq = ++(SendSeq->seq_local);
     
@@ -275,26 +255,19 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
     shmem_putmem(recv,xmit,bytes,receiver);
     shmem_fence();
 
-    printf("Sender sent payload %d\n",seq);
     //Notify him we're done
     shmem_putmem((void *)&(RecvSeq->seq_remote),&seq,sizeof(seq),receiver);
     shmem_fence();
-    printf("Sender ringing door bell  %d\n",seq);
   }
   if ( _processor == receiver ) {
 
-    printf("Receiver SHMEM pt2pt %d->%d\n",sender,receiver);
     // Post a receive
     seq = ++(RecvSeq->seq_local);
     shmem_putmem((void *)&(SendSeq->seq_remote),&seq,sizeof(seq),sender);
 
-    printf("Receiver Opening letter box %d\n",seq);
-
-    
     // Now wait until he has advanced our reception counter
     while(RecvSeq->seq_remote != RecvSeq->seq_local);
 
-    printf("Receiver Got the mail %d\n",seq);
   }
 }
 

From 7c3363b91ecbdb06de96842a610f73697a3de873 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 25 Oct 2016 00:04:17 +0100
Subject: [PATCH 094/149] Compiles all comms targets

---
 lib/communicator/Communicator_none.cc | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index f4e4c575..198c1add 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -31,15 +31,6 @@ namespace Grid {
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////////////////////////////////////////
-int CartesianCommunicator::ShmRank;
-int CartesianCommunicator::ShmSize;
-int CartesianCommunicator::GroupRank;
-int CartesianCommunicator::GroupSize;
-int CartesianCommunicator::WorldRank;
-int CartesianCommunicator::WorldSize;
-int CartesianCommunicator::Slave;
-void *              CartesianCommunicator::ShmCommBuf;
-commVector<uint8_t> CartesianCommunicator::ShmBufStorageVector;
 
 void CartesianCommunicator::Init(int *argc, char *** arv)
 {

From d97a27f4833c6827dc3d344a5186f0a3e4d2d972 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 25 Oct 2016 01:05:31 +0100
Subject: [PATCH 095/149] Verbose

---
 lib/communicator/Communicator_base.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 91e9cf9b..bccba6be 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -123,7 +123,6 @@ void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) {
 void CartesianCommunicator::ShmInitGeneric(void){
   ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
   ShmCommBuf=(void *)&ShmBufStorageVector[0];
-  std::cout << "allocated persistent buffer"<<std::hex << ShmCommBuf << std::dec<<std::endl;
 }
 
 #endif

From 8f8058f8a54e81a26b29a718170d6a8f85d4166e Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 25 Oct 2016 01:05:52 +0100
Subject: [PATCH 096/149] More random bits on parallel seeding

---
 lib/lattice/Lattice_rng.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/lattice/Lattice_rng.h b/lib/lattice/Lattice_rng.h
index 3254af30..2df5e2d1 100644
--- a/lib/lattice/Lattice_rng.h
+++ b/lib/lattice/Lattice_rng.h
@@ -296,9 +296,10 @@ namespace Grid {
 	_grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor);
 
 	int l_idx=generator_idx(o_idx,i_idx);
-	
-	std::vector<int> site_seeds(4);
-	for(int i=0;i<4;i++){
+
+	const int num_rand_seed=16;
+	std::vector<int> site_seeds(num_rand_seed);
+	for(int i=0;i<site_seeds.size();i++){
 	  site_seeds[i]= ui(pseeder);
 	}
 

From d7d92af09db80132959ebd9fdb5ca39516520d20 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 25 Oct 2016 01:45:53 +0100
Subject: [PATCH 097/149] Travis fail fix attempt

---
 lib/Stencil.h                         | 2 +-
 lib/communicator/Communicator_base.cc | 3 ++-
 lib/communicator/Communicator_base.h  | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/Stencil.h b/lib/Stencil.h
index 221785b4..5c3a5ef9 100644
--- a/lib/Stencil.h
+++ b/lib/Stencil.h
@@ -821,7 +821,7 @@ PARALLEL_FOR_LOOP
 
 
 	cobj *send_buf = (cobj *)_grid->ShmBufferTranslate(xmit_to_rank,u_recv_buf_p);
-	if ( (send_buf==NULL) ) { 
+	if ( send_buf==NULL ) { 
 	  send_buf = u_send_buf_p;
 	}
 	//	std::cout << " send_bufs  "<<std::hex<< send_buf <<" ubp "<<u_send_buf_p <<std::dec<<std::endl;
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index bccba6be..9810987d 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -48,10 +48,11 @@ void *CartesianCommunicator::ShmBufferMalloc(size_t bytes){
   void *ptr = (void *)heap_top;
   heap_top  += bytes;
   heap_bytes+= bytes;
+  std::cout <<"Shm alloc "<<ptr<<std::endl;
   assert(heap_bytes < MAX_MPI_SHM_BYTES);
   return ptr;
 }
-void *CartesianCommunicator::ShmBufferFreeAll(void) { 
+void CartesianCommunicator::ShmBufferFreeAll(void) { 
   heap_top  =(size_t)ShmBufferSelf();
   heap_bytes=0;
 }
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 576c749e..9a15d2b0 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -104,7 +104,7 @@ class CartesianCommunicator {
   void *ShmBuffer(int rank);
   void *ShmBufferTranslate(int rank,void * local_p);
   void *ShmBufferMalloc(size_t bytes);
-  void *ShmBufferFreeAll(void) ;
+  void ShmBufferFreeAll(void) ;
   
   ////////////////////////////////////////////////
   // Must call in Grid startup

From 09f66100d3fe9a363d43d8da4e241116b4a8e103 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 25 Oct 2016 06:01:12 +0100
Subject: [PATCH 098/149] MPI 3 compile on non-linux

---
 lib/communicator/Communicator_mpi3.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 61e5e06b..14a152ab 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -27,7 +27,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #include "Grid.h"
 #include <mpi.h>
-#include <numaif.h>
 
 namespace Grid {
 
@@ -157,6 +156,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   assert(ierr==0);
   // KNL hack -- force to numa-domain 1 in flat
 #if 0
+  //#include <numaif.h>
   for(uint64_t page=0;page<MAX_MPI_SHM_BYTES;page+=4096){
     void *pages = (void *) ( page + ShmCommBuf );
     int status;

From 93896ce59ede4bc2d4085f3b97c1b02647a8dc3a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 25 Oct 2016 06:12:49 +0100
Subject: [PATCH 099/149] Roll version number

---
 VERSION | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index c12f9497..e7abbba7 100644
--- a/VERSION
+++ b/VERSION
@@ -1,4 +1,6 @@
-Version : 0.5.0
+Version : 0.6.0
 
 - AVX512, AVX2, AVX, SSE good
 - Clang 3.5 and above, ICPC v16 and above, GCC 4.9 and above
+- MPI and MPI3
+- HiRep, Smearing, Generic gauge group

From 33d199a0ad3d1be0d583961911c2766d8f8874e4 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 25 Oct 2016 12:56:40 +0100
Subject: [PATCH 100/149] temporary thread safety in FFT

---
 lib/FFT.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index 17060dc3..9a59ed01 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -227,7 +227,7 @@ namespace Grid {
 	GridStopWatch timer;
 	timer.Start();
 
-PARALLEL_FOR_LOOP
+//PARALLEL_FOR_LOOP
 	for(int idx=0;idx<NN;idx++) {
 	  pencil_g.LocalIndexToLocalCoor(idx,lcoor);
 

From bca861e112a0473f8215e66310489a9216b7870a Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 25 Oct 2016 14:21:48 +0100
Subject: [PATCH 101/149] Note:FFT shoud be GridFFT (Not change yet).

Gauge fix with FFt is added (tests/core)
---
 lib/qcd/utils/SUn.h         |  31 ++++
 lib/qcd/utils/WilsonLoops.h |   2 +-
 tests/core/Test_fft.cc      |   3 +
 tests/core/Test_fft_gfix.cc | 301 ++++++++++++++++++++++++++++++++++++
 4 files changed, 336 insertions(+), 1 deletion(-)
 create mode 100644 tests/core/Test_fft_gfix.cc

diff --git a/lib/qcd/utils/SUn.h b/lib/qcd/utils/SUn.h
index 3240098a..914c3bad 100644
--- a/lib/qcd/utils/SUn.h
+++ b/lib/qcd/utils/SUn.h
@@ -674,6 +674,37 @@ class SU {
       out += la;
     }
   }
+/*
+ add GaugeTrans
+*/
+
+template<typename GaugeField,typename GaugeMat>
+  static void GaugeTransform( GaugeField &Umu, GaugeMat &g){
+    GridBase *grid = Umu._grid;
+    conformable(grid,g._grid);
+
+    GaugeMat U(grid);
+    GaugeMat ag(grid); ag = adj(g);
+
+    for(int mu=0;mu<Nd;mu++){
+      U= PeekIndex<LorentzIndex>(Umu,mu);
+      U = g*U*Cshift(ag, mu, 1);
+      PokeIndex<LorentzIndex>(Umu,U,mu);
+    }
+  }
+  template<typename GaugeMat>
+    static void GaugeTransform( std::vector<GaugeMat> &U, GaugeMat &g){
+    GridBase *grid = g._grid;
+    GaugeMat ag(grid); ag = adj(g);
+    for(int mu=0;mu<Nd;mu++){
+      U[mu] = g*U[mu]*Cshift(ag, mu, 1);
+    }
+  }
+  template<typename GaugeField,typename GaugeMat>
+  static void RandomGaugeTransform(GridParallelRNG &pRNG, GaugeField &Umu, GaugeMat &g){
+    LieRandomize(pRNG,g,1.0);
+    GaugeTransform(Umu,g);
+  }
 
   // Projects the algebra components a lattice matrix (of dimension ncol*ncol -1 )
   // inverse operation: FundamentalLieAlgebraMatrix
diff --git a/lib/qcd/utils/WilsonLoops.h b/lib/qcd/utils/WilsonLoops.h
index 10022f50..03d45c07 100644
--- a/lib/qcd/utils/WilsonLoops.h
+++ b/lib/qcd/utils/WilsonLoops.h
@@ -522,4 +522,4 @@ typedef WilsonLoops<PeriodicGimplR> SU3WilsonLoops;
 }
 }
 
-#endif
\ No newline at end of file
+#endif
diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index 562a53a1..512d6e62 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -441,6 +441,8 @@ int main (int argc, char ** argv)
   }
 
   {
+    /*
+     * 
     typedef GaugeImplTypes<vComplexD, 1> QEDGimplTypesD;
     typedef Photon<QEDGimplTypesD>       QEDGaction;
 
@@ -450,6 +452,7 @@ int main (int argc, char ** argv)
 
     Maxwell.FreePropagator (Source,Prop);
     std::cout << " MaxwellFree propagator\n";
+    */
   }
   Grid_finalize();
 }
diff --git a/tests/core/Test_fft_gfix.cc b/tests/core/Test_fft_gfix.cc
new file mode 100644
index 00000000..1eda9a67
--- /dev/null
+++ b/tests/core/Test_fft_gfix.cc
@@ -0,0 +1,301 @@
+    /*************************************************************************************
+
+    grid` physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_cshift.cc
+
+    Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/qcd/action/gauge/Photon.h>
+
+using namespace Grid;
+using namespace Grid::QCD;
+
+template <class Gimpl> 
+class FourierAcceleratedGaugeFixer  : public Gimpl {
+  public:
+  INHERIT_GIMPL_TYPES(Gimpl);
+
+  typedef typename Gimpl::GaugeLinkField GaugeMat;
+  typedef typename Gimpl::GaugeField GaugeLorentz;
+
+  static void GaugeLinkToLieAlgebraField(const std::vector<GaugeMat> &U,std::vector<GaugeMat> &A) {
+    for(int mu=0;mu<Nd;mu++){
+//      ImplComplex cmi(0.0,-1.0);
+      ComplexD cmi(0.0,-1.0);
+      A[mu] = Ta(U[mu]) * cmi;
+    }
+  }
+  static void DmuAmu(const std::vector<GaugeMat> &A,GaugeMat &dmuAmu) {
+    dmuAmu=zero;
+    for(int mu=0;mu<Nd;mu++){
+      dmuAmu = dmuAmu + A[mu] - Cshift(A[mu],mu,-1);
+    }
+  }  
+  static void SteepestDescentGaugeFix(GaugeLorentz &Umu,RealD & alpha,int maxiter,RealD Omega_tol, RealD Phi_tol) {
+    GridBase *grid = Umu._grid;
+
+    RealD org_plaq      =WilsonLoops<Gimpl>::avgPlaquette(Umu);
+    RealD org_link_trace=WilsonLoops<Gimpl>::linkTrace(Umu); 
+    RealD old_trace = org_link_trace;
+    RealD trG;
+
+    std::vector<GaugeMat> U(Nd,grid);
+                 GaugeMat dmuAmu(grid);
+
+    for(int i=0;i<maxiter;i++){
+      for(int mu=0;mu<Nd;mu++) U[mu]= PeekIndex<LorentzIndex>(Umu,mu);
+      //trG = SteepestDescentStep(U,alpha,dmuAmu);
+      trG = FourierAccelSteepestDescentStep(U,alpha,dmuAmu);
+      for(int mu=0;mu<Nd;mu++) PokeIndex<LorentzIndex>(Umu,U[mu],mu);
+      // Monitor progress and convergence test 
+      // infrequently to minimise cost overhead
+      if ( i %20 == 0 ) { 
+	RealD plaq      =WilsonLoops<Gimpl>::avgPlaquette(Umu);
+	RealD link_trace=WilsonLoops<Gimpl>::linkTrace(Umu); 
+
+	std::cout << GridLogMessage << " Iteration "<<i<< " plaq= "<<plaq<< " dmuAmu " << norm2(dmuAmu)<< std::endl;
+	
+	RealD Phi  = 1.0 - old_trace / link_trace ;
+	RealD Omega= 1.0 - trG;
+
+
+	std::cout << GridLogMessage << " Iteration "<<i<< " Phi= "<<Phi<< " Omega= " << Omega<< " trG " << trG <<std::endl;
+	if ( (Omega < Omega_tol) && ( ::fabs(Phi) < Phi_tol) ) {
+	  std::cout << GridLogMessage << "Converged ! "<<std::endl;
+	  return;
+	}
+
+	old_trace = link_trace;
+
+      }
+    }
+  };
+  static RealD SteepestDescentStep(std::vector<GaugeMat> &U,RealD & alpha, GaugeMat & dmuAmu) {
+    GridBase *grid = U[0]._grid;
+
+    std::vector<GaugeMat> A(Nd,grid);
+    GaugeMat g(grid);
+
+    GaugeLinkToLieAlgebraField(U,A);
+    ExpiAlphaDmuAmu(A,g,alpha,dmuAmu);
+
+
+    RealD vol = grid->gSites();
+    RealD trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
+
+    SU<Nc>::GaugeTransform(U,g);
+
+    return trG;
+  }
+
+  static RealD FourierAccelSteepestDescentStep(std::vector<GaugeMat> &U,RealD & alpha, GaugeMat & dmuAmu) {
+
+    GridBase *grid = U[0]._grid;
+
+    RealD vol = grid->gSites();
+
+    FFT theFFT((GridCartesian *)grid);
+
+    LatticeComplex  Fp(grid);
+    LatticeComplex  psq(grid); psq=zero;
+    LatticeComplex  pmu(grid); 
+    LatticeComplex   one(grid); one = ComplexD(1.0,0.0);
+
+    GaugeMat g(grid);
+    GaugeMat dmuAmu_p(grid);
+    std::vector<GaugeMat> A(Nd,grid);
+
+    GaugeLinkToLieAlgebraField(U,A);
+
+    DmuAmu(A,dmuAmu);
+
+    theFFT.FFT_all_dim(dmuAmu_p,dmuAmu,FFT::forward);
+
+    //////////////////////////////////
+    // Work out Fp = psq_max/ psq...
+    //////////////////////////////////
+    std::vector<int> latt_size = grid->GlobalDimensions();
+    std::vector<int> coor(grid->_ndimension,0);
+    for(int mu=0;mu<Nd;mu++) {
+
+      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+      LatticeCoordinate(pmu,mu);
+      pmu = TwoPiL * pmu ;
+      psq = psq + 4.0*sin(pmu*0.5)*sin(pmu*0.5); 
+    }
+
+    ComplexD psqMax(16.0);
+    Fp =  psqMax*one/psq;
+
+    static int once;
+    if ( once == 0 ) { 
+      std::cout << " Fp " << Fp <<std::endl;
+      once ++;
+    }
+    pokeSite(TComplex(1.0),Fp,coor);
+
+    dmuAmu_p  = dmuAmu_p * Fp; 
+
+    theFFT.FFT_all_dim(dmuAmu,dmuAmu_p,FFT::backward);
+
+    GaugeMat ciadmam(grid);
+    ComplexD cialpha(0.0,-alpha);
+    ciadmam = dmuAmu*cialpha;
+    SU<Nc>::taExp(ciadmam,g);
+
+    RealD trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
+
+    SU<Nc>::GaugeTransform(U,g);
+
+    return trG;
+  }
+
+  static void ExpiAlphaDmuAmu(const std::vector<GaugeMat> &A,GaugeMat &g,RealD & alpha, GaugeMat &dmuAmu) {
+    GridBase *grid = g._grid;
+    ComplexD cialpha(0.0,-alpha);
+    GaugeMat ciadmam(grid);
+    DmuAmu(A,dmuAmu);
+    ciadmam = dmuAmu*cialpha;
+    SU<Nc>::taExp(ciadmam,g);
+  }  
+/*
+  ////////////////////////////////////////////////////////////////
+  // NB The FT for fields living on links has an extra phase in it
+  // Could add these to the FFT class as a later task since this code
+  // might be reused elsewhere ????
+  ////////////////////////////////////////////////////////////////
+  static void InverseFourierTransformAmu(FFT &theFFT,const std::vector<GaugeMat> &Ap,std::vector<GaugeMat> &Ax) {
+    GridBase * grid = theFFT.Grid();
+    std::vector<int> latt_size = grid->GlobalDimensions();
+
+    ComplexField  pmu(grid);
+    ComplexField  pha(grid);
+    GaugeMat      Apha(grid);
+
+    ComplexD ci(0.0,1.0);
+
+    for(int mu=0;mu<Nd;mu++){
+
+      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+      LatticeCoordinate(pmu,mu);
+      pmu = TwoPiL * pmu ;
+      pha = exp(pmu *  (0.5 *ci)); // e(ipmu/2) since Amu(x+mu/2)
+
+      Apha = Ap[mu] * pha;
+
+      theFFT.FFT_all_dim(Apha,Ax[mu],FFT::backward);
+    }
+  }
+  static void FourierTransformAmu(FFT & theFFT,const std::vector<GaugeMat> &Ax,std::vector<GaugeMat> &Ap) {
+    GridBase * grid = theFFT.Grid();
+    std::vector<int> latt_size = grid->GlobalDimensions();
+
+    ComplexField  pmu(grid);
+    ComplexField  pha(grid);
+    ComplexD ci(0.0,1.0);
+    
+    // Sign convention for FFTW calls:
+    // A(x)= Sum_p e^ipx A(p) / V
+    // A(p)= Sum_p e^-ipx A(x)
+
+    for(int mu=0;mu<Nd;mu++){
+      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+      LatticeCoordinate(pmu,mu);
+      pmu = TwoPiL * pmu ;
+      pha = exp(-pmu *  (0.5 *ci)); // e(+ipmu/2) since Amu(x+mu/2)
+
+      theFFT.FFT_all_dim(Ax[mu],Ap[mu],FFT::backward);
+      Ap[mu] = Ap[mu] * pha;
+    }
+  }
+*/
+};
+
+int main (int argc, char ** argv)
+{
+  std::vector<int> seeds({1,2,3,4});
+
+  Grid_init(&argc,&argv);
+
+  int threads = GridThread::GetThreads();
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout( { vComplexD::Nsimd(),1,1,1});
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  int vol = 1;
+  for(int d=0;d<latt_size.size();d++){
+    vol = vol * latt_size[d];
+  }
+
+  GridCartesian         GRID(latt_size,simd_layout,mpi_layout);
+  GridSerialRNG          sRNG;  sRNG.SeedFixedIntegers(seeds); // naughty seeding
+  GridParallelRNG          pRNG(&GRID);   pRNG.SeedFixedIntegers(seeds);
+
+  FFT theFFT(&GRID);
+
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  std::cout<< "*****************************************************************" <<std::endl;
+  std::cout<< "* Testing we can gauge fix steep descent a RGT of Unit gauge    *" <<std::endl;
+  std::cout<< "*****************************************************************" <<std::endl;
+
+  LatticeGaugeFieldD   Umu(&GRID);
+  LatticeGaugeFieldD   Uorg(&GRID);
+  LatticeColourMatrixD   g(&GRID); // Gauge xform
+
+  
+  SU3::ColdConfiguration(pRNG,Umu); // Unit gauge
+  Uorg=Umu;
+
+  SU3::RandomGaugeTransform(pRNG,Umu,g); // Unit gauge
+  RealD plaq=WilsonLoops<PeriodicGimplD>::avgPlaquette(Umu);
+  std::cout << " Initial plaquette "<<plaq << std::endl;
+
+
+
+  RealD alpha=0.1;
+  FourierAcceleratedGaugeFixer<PeriodicGimplD>::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-10, 1.0e-10);
+
+
+  plaq=WilsonLoops<PeriodicGimplD>::avgPlaquette(Umu);
+  std::cout << " Final plaquette "<<plaq << std::endl;
+
+  Uorg = Uorg - Umu;
+  std::cout << " Norm Difference "<< norm2(Uorg) << std::endl;
+
+
+  //  std::cout<< "*****************************************************************" <<std::endl;
+  //  std::cout<< "* Testing Fourier accelerated fixing                            *" <<std::endl;
+  //  std::cout<< "*****************************************************************" <<std::endl;
+
+  //  std::cout<< "*****************************************************************" <<std::endl;
+  //  std::cout<< "* Testing non-unit configuration                                *" <<std::endl;
+  //  std::cout<< "*****************************************************************" <<std::endl;
+
+
+
+  Grid_finalize();
+}

From 14ddf2c234eda6f62bc8e28bf28d6395a77920d3 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Wed, 26 Oct 2016 17:36:26 +0100
Subject: [PATCH 102/149] more FFT optimisations

---
 lib/FFT.h              | 233 +++++++++++++++++++----------------------
 tests/core/Test_fft.cc |  24 +++--
 2 files changed, 121 insertions(+), 136 deletions(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index 9a59ed01..71cf8a3e 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -98,174 +98,157 @@ namespace Grid {
 #define FFTW_BACKWARD (+1)
 #endif
 
-  class FFT { 
+  class FFT {
   private:
-
+    
     GridCartesian *vgrid;
     GridCartesian *sgrid;
-
+    
     int Nd;
     double flops;
     double flops_call;
     uint64_t usec;
-
+    
     std::vector<int> dimensions;
     std::vector<int> processors;
     std::vector<int> processor_coor;
-
+    
   public:
-
+    
     static const int forward=FFTW_FORWARD;
     static const int backward=FFTW_BACKWARD;
-
+    
     double Flops(void) {return flops;}
     double MFlops(void) {return flops/usec;}
-
-    FFT ( GridCartesian * grid ) : 
-      vgrid(grid),
-      Nd(grid->_ndimension),
-      dimensions(grid->_fdimensions),
-      processors(grid->_processors),
-      processor_coor(grid->_processor_coor)
+    
+    FFT ( GridCartesian * grid ) :
+    vgrid(grid),
+    Nd(grid->_ndimension),
+    dimensions(grid->_fdimensions),
+    processors(grid->_processors),
+    processor_coor(grid->_processor_coor)
     {
       flops=0;
       usec =0;
       std::vector<int> layout(Nd,1);
       sgrid = new GridCartesian(dimensions,layout,processors);
     };
-
-    ~FFT ( void)  { 
-      delete sgrid; 
+    
+    ~FFT ( void)  {
+      delete sgrid;
     }
     
     template<class vobj>
     void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int inverse){
-
+#ifndef HAVE_FFTW
+      assert(0);
+#else
       conformable(result._grid,vgrid);
       conformable(source._grid,vgrid);
-
+      
       int L = vgrid->_ldimensions[dim];
       int G = vgrid->_fdimensions[dim];
-
+      
       std::vector<int> layout(Nd,1);
       std::vector<int> pencil_gd(vgrid->_fdimensions);
-
-      pencil_gd[dim] = G*processors[dim];    
-
+      
+      pencil_gd[dim] = G*processors[dim];
+      
       // Pencil global vol LxLxGxLxL per node
       GridCartesian pencil_g(pencil_gd,layout,processors);
-
+      
       // Construct pencils
       typedef typename vobj::scalar_object sobj;
       typedef typename sobj::scalar_type   scalar;
+      
+      Lattice<sobj> pgbuf(&pencil_g);
+      
 
-      Lattice<vobj> ssource(vgrid); ssource =source;
-      Lattice<sobj> pgsource(&pencil_g);
-      Lattice<sobj> pgresult(&pencil_g); pgresult=zero;
-
-#ifndef HAVE_FFTW	
-      assert(0);
-#else 
       typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
       typedef typename FFTW<scalar>::FFTW_plan   FFTW_plan;
-
-      {
-	int Ncomp = sizeof(sobj)/sizeof(scalar);
-	int Nlow  = 1;
-	for(int d=0;d<dim;d++){
-	  Nlow*=vgrid->_ldimensions[d];
-	}
-
-	int rank = 1;  /* 1d transforms */
-	int n[] = {G}; /* 1d transforms of length G */
-	int howmany = Ncomp;
-	int odist,idist,istride,ostride;
-	idist   = odist   = 1;          /* Distance between consecutive FT's */
-	istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */
-	int *inembed = n, *onembed = n;
-
-	
-	int sign = FFTW_FORWARD;
-	if (inverse) sign = FFTW_BACKWARD;
-
-	FFTW_plan p;
-	{
-	  FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[0];
-	  FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[0];
-	  p = FFTW<scalar>::fftw_plan_many_dft(rank,n,howmany,
-					       in,inembed,
-					       istride,idist,
-					       out,onembed,
-					       ostride, odist,
-					       sign,FFTW_ESTIMATE);
-	}
-
-    std::vector<int> lcoor(Nd), gcoor(Nd);
-
-	// Barrel shift and collect global pencil
-	for(int p=0;p<processors[dim];p++) { 
-
-	  for(int idx=0;idx<sgrid->lSites();idx++) { 
-
-	    
-    	    sgrid->LocalIndexToLocalCoor(idx,lcoor);
-
-	    sobj s;
-
-	    peekLocalSite(s,ssource,lcoor);
-
-	    lcoor[dim]+=p*L;
-	   
-	    pokeLocalSite(s,pgsource,lcoor);
-	  }
-
-	  ssource = Cshift(ssource,dim,L);
-	}
-	
-	// Loop over orthog coords
-	int NN=pencil_g.lSites();
-	GridStopWatch timer;
-	timer.Start();
-
-//PARALLEL_FOR_LOOP
-	for(int idx=0;idx<NN;idx++) {
-	  pencil_g.LocalIndexToLocalCoor(idx,lcoor);
-
-	  if ( lcoor[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
-	    FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[idx];
-	    FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[idx];
-	    FFTW<scalar>::fftw_execute_dft(p,in,out);
-	  }
-	}
-
-        timer.Stop();
-
-          double add,mul,fma;
-          FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
-          flops_call = add+mul+2.0*fma;
-          usec += timer.useconds();
-          flops+= flops_call*NN;
-        int pc = processor_coor[dim];
-        for(int idx=0;idx<sgrid->lSites();idx++) {
-	  sgrid->LocalIndexToLocalCoor(idx,lcoor);
-	  gcoor = lcoor;
-	  // extract the result
-	  sobj s;
-	  gcoor[dim] = lcoor[dim]+L*pc;
-	  peekLocalSite(s,pgresult,gcoor);
-	  pokeLocalSite(s,result,lcoor);
-	}
-      	  
-	FFTW<scalar>::fftw_destroy_plan(p);
+      
+      int Ncomp = sizeof(sobj)/sizeof(scalar);
+      int Nlow  = 1;
+      for(int d=0;d<dim;d++){
+        Nlow*=vgrid->_ldimensions[d];
       }
+      
+      int rank = 1;  /* 1d transforms */
+      int n[] = {G}; /* 1d transforms of length G */
+      int howmany = Ncomp;
+      int odist,idist,istride,ostride;
+      idist   = odist   = 1;          /* Distance between consecutive FT's */
+      istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */
+      int *inembed = n, *onembed = n;
+      
+      int sign = FFTW_FORWARD;
+      if (inverse) sign = FFTW_BACKWARD;
+      
+      FFTW_plan p;
+      {
+        FFTW_scalar *in = (FFTW_scalar *)&pgbuf._odata[0];
+        FFTW_scalar *out= (FFTW_scalar *)&pgbuf._odata[0];
+        p = FFTW<scalar>::fftw_plan_many_dft(rank,n,howmany,
+                                             in,inembed,
+                                             istride,idist,
+                                             out,onembed,
+                                             ostride, odist,
+                                             sign,FFTW_ESTIMATE);
+      }
+      
+      // Barrel shift and collect global pencil
+      std::vector<int> lcoor(Nd), gcoor(Nd);
+      result = source;
+      for(int p=0;p<processors[dim];p++) {
+        for(int idx=0;idx<sgrid->lSites();idx++) {
+          sgrid->LocalIndexToLocalCoor(idx,lcoor);
+          sobj s;
+          peekLocalSite(s,result,lcoor);
+          lcoor[dim]+=p*L;
+          pokeLocalSite(s,pgbuf,lcoor);
+        }
+        result = Cshift(result,dim,L);
+      }
+      
+      // Loop over orthog coords
+      int NN=pencil_g.lSites();
+      GridStopWatch timer;
+      timer.Start();
+      //PARALLEL_FOR_LOOP
+      for(int idx=0;idx<NN;idx++) {
+        pencil_g.LocalIndexToLocalCoor(idx,lcoor);
+        
+        if ( lcoor[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
+          FFTW_scalar *in = (FFTW_scalar *)&pgbuf._odata[idx];
+          FFTW_scalar *out= (FFTW_scalar *)&pgbuf._odata[idx];
+          FFTW<scalar>::fftw_execute_dft(p,in,out);
+        }
+      }
+      timer.Stop();
+      
+      // performance counting
+      double add,mul,fma;
+      FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
+      flops_call = add+mul+2.0*fma;
+      usec += timer.useconds();
+      flops+= flops_call*NN;
+      
+      // writing out result
+      int pc = processor_coor[dim];
+      for(int idx=0;idx<sgrid->lSites();idx++) {
+        sgrid->LocalIndexToLocalCoor(idx,lcoor);
+        gcoor = lcoor;
+        sobj s;
+        gcoor[dim] = lcoor[dim]+L*pc;
+        peekLocalSite(s,pgbuf,gcoor);
+        pokeLocalSite(s,result,lcoor);
+      }
+      
+      // destroying plan
+      FFTW<scalar>::fftw_destroy_plan(p);
 #endif
-
-
     }
-
   };
-
-
 }
 
 #endif
diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index 2fdaeed2..d5b60752 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -76,13 +76,14 @@ int main (int argc, char ** argv)
   S=zero;
   S = S+C;
 
+  Ctilde = C;
   FFT theFFT(&Fine);
 
-  theFFT.FFT_dim(Ctilde,C,0,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Ctilde,C,1,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Ctilde,C,2,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Ctilde,C,3,FFT::forward);  std::cout << theFFT.MFlops()<<std::endl;
-
+  theFFT.FFT_dim(Ctilde,Ctilde,0,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,Ctilde,1,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,Ctilde,2,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,Ctilde,3,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  
   //  C=zero;
   //  Ctilde = where(abs(Ctilde)<1.0e-10,C,Ctilde);
   TComplexD cVol;
@@ -92,12 +93,13 @@ int main (int argc, char ** argv)
   pokeSite(cVol,C,p);
   C=C-Ctilde;
   std::cout << "diff scalar "<<norm2(C) << std::endl;
-
-  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,3,FFT::forward);std::cout << theFFT.MFlops()<<std::endl;
-
+  Stilde = S;
+  
+  theFFT.FFT_dim(Stilde,Stilde,0,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,Stilde,1,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,Stilde,2,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,Stilde,3,FFT::forward); std::cout << theFFT.MFlops()<<std::endl;
+  
   SpinMatrixD Sp; 
   Sp = zero; Sp = Sp+cVol;
 

From a1705a8d53f85e730681d5f90f973d09b15f1b25 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Wed, 26 Oct 2016 18:50:07 +0100
Subject: [PATCH 103/149] debug message removed

---
 lib/communicator/Communicator_base.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 9810987d..8b789f4b 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -48,7 +48,6 @@ void *CartesianCommunicator::ShmBufferMalloc(size_t bytes){
   void *ptr = (void *)heap_top;
   heap_top  += bytes;
   heap_bytes+= bytes;
-  std::cout <<"Shm alloc "<<ptr<<std::endl;
   assert(heap_bytes < MAX_MPI_SHM_BYTES);
   return ptr;
 }

From b4d2af8c899fc78290ed1b1f2b821eef36157831 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Wed, 26 Oct 2016 19:46:36 +0100
Subject: [PATCH 104/149] threaded FFT

---
 lib/FFT.h     | 52 +++++++++++++++++++++++++++++++++------------------
 lib/Threads.h | 11 ++++++++---
 2 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index 71cf8a3e..bb4dcf7c 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -200,12 +200,18 @@ namespace Grid {
       std::vector<int> lcoor(Nd), gcoor(Nd);
       result = source;
       for(int p=0;p<processors[dim];p++) {
-        for(int idx=0;idx<sgrid->lSites();idx++) {
-          sgrid->LocalIndexToLocalCoor(idx,lcoor);
+        PARALLEL_REGION
+        {
+          std::vector<int> cbuf(Nd);
           sobj s;
-          peekLocalSite(s,result,lcoor);
-          lcoor[dim]+=p*L;
-          pokeLocalSite(s,pgbuf,lcoor);
+          
+          PARALLEL_FOR_LOOP_INTERN
+          for(int idx=0;idx<sgrid->lSites();idx++) {
+            sgrid->LocalIndexToLocalCoor(idx,cbuf);
+            peekLocalSite(s,result,cbuf);
+            cbuf[dim]+=p*L;
+            pokeLocalSite(s,pgbuf,cbuf);
+          }
         }
         result = Cshift(result,dim,L);
       }
@@ -214,14 +220,18 @@ namespace Grid {
       int NN=pencil_g.lSites();
       GridStopWatch timer;
       timer.Start();
-      //PARALLEL_FOR_LOOP
-      for(int idx=0;idx<NN;idx++) {
-        pencil_g.LocalIndexToLocalCoor(idx,lcoor);
+      PARALLEL_REGION
+      {
+        std::vector<int> cbuf(Nd);
         
-        if ( lcoor[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
-          FFTW_scalar *in = (FFTW_scalar *)&pgbuf._odata[idx];
-          FFTW_scalar *out= (FFTW_scalar *)&pgbuf._odata[idx];
-          FFTW<scalar>::fftw_execute_dft(p,in,out);
+        PARALLEL_FOR_LOOP_INTERN
+        for(int idx=0;idx<NN;idx++) {
+          pencil_g.LocalIndexToLocalCoor(idx, cbuf);
+          if ( cbuf[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
+            FFTW_scalar *in = (FFTW_scalar *)&pgbuf._odata[idx];
+            FFTW_scalar *out= (FFTW_scalar *)&pgbuf._odata[idx];
+            FFTW<scalar>::fftw_execute_dft(p,in,out);
+          }
         }
       }
       timer.Stop();
@@ -235,13 +245,19 @@ namespace Grid {
       
       // writing out result
       int pc = processor_coor[dim];
-      for(int idx=0;idx<sgrid->lSites();idx++) {
-        sgrid->LocalIndexToLocalCoor(idx,lcoor);
-        gcoor = lcoor;
+      PARALLEL_REGION
+      {
+        std::vector<int> clbuf(Nd), cgbuf(Nd);
         sobj s;
-        gcoor[dim] = lcoor[dim]+L*pc;
-        peekLocalSite(s,pgbuf,gcoor);
-        pokeLocalSite(s,result,lcoor);
+        
+        PARALLEL_FOR_LOOP_INTERN
+        for(int idx=0;idx<sgrid->lSites();idx++) {
+          sgrid->LocalIndexToLocalCoor(idx,clbuf);
+          cgbuf = clbuf;
+          cgbuf[dim] = clbuf[dim]+L*pc;
+          peekLocalSite(s,pgbuf,cgbuf);
+          pokeLocalSite(s,result,clbuf);
+        }
       }
       
       // destroying plan
diff --git a/lib/Threads.h b/lib/Threads.h
index 08e5d545..2f270b73 100644
--- a/lib/Threads.h
+++ b/lib/Threads.h
@@ -38,14 +38,19 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_OMP
 #include <omp.h>
 #ifdef GRID_NUMA
-#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(static)")
+#define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(static)")
+#define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(static)")
 #else
-#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(runtime)")
+#define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(runtime)")
+#define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(runtime)")
 #endif
 #define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)")
+#define PARALLEL_REGION       _Pragma("omp parallel")
 #else
-#define PARALLEL_FOR_LOOP 
+#define PARALLEL_FOR_LOOP
+#define PARALLEL_FOR_LOOP_INTERN
 #define PARALLEL_NESTED_LOOP2
+#define PARALLEL_REGION
 #endif
 
 namespace Grid {

From aa6a839c6069ba32cf9a9d8d1a22ec395b19de49 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 28 Oct 2016 09:13:09 +0100
Subject: [PATCH 105/149] avx512 build fix; detect clang/gcc intrinsics  vs.
 ICPC

---
 lib/simd/Grid_avx512.h | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h
index 62789462..521f804c 100644
--- a/lib/simd/Grid_avx512.h
+++ b/lib/simd/Grid_avx512.h
@@ -371,14 +371,9 @@ namespace Optimization {
   // Some Template specialization
 
   // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases
-<<<<<<< HEAD
-#define GNU_CLANG_COMPILER 
-#ifdef GNU_CLANG_COMPILER
-=======
 
 #ifndef __INTEL_COMPILER
 #warning "Slow reduction due to incomplete reduce intrinsics"
->>>>>>> develop
   //Complex float Reduce
   template<>
     inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){

From e1042aef77f70f2c4ab44d9066fd4c6d093f6e50 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Fri, 28 Oct 2016 17:20:04 +0100
Subject: [PATCH 106/149] First version of the doube prec for testing purposes

It does not compile single and double version at the same time
---
 lib/qcd/action/fermion/WilsonKernelsAsm.cc    | 81 ++++++++++++++++++-
 lib/qcd/action/fermion/WilsonKernelsAsmBody.h |  4 +-
 lib/simd/Grid_avx512.h                        |  6 --
 3 files changed, 80 insertions(+), 11 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsAsm.cc b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
index 74862400..2fc9b035 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsm.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
@@ -53,12 +53,13 @@ WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,
 }
 
 #if defined(AVX512) 
-    
+#include <simd/Intel512wilson.h>
+
+#if defined(GRID_DEFAULT_PRECISION_SINGLE)    
     ///////////////////////////////////////////////////////////
     // If we are AVX512 specialise the single precision routine
     ///////////////////////////////////////////////////////////
-    
-#include <simd/Intel512wilson.h>
+
 #include <simd/Intel512single.h>
     
 static Vector<vComplexF> signs;
@@ -78,6 +79,7 @@ static Vector<vComplexF> signs;
 #define MAYBEPERM(A,perm) if (perm) { A ; }
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
 #define FX(A) WILSONASM_ ##A
+#define COMPLEX_TYPE vComplexF
   
 #undef KERNEL_DAG
 template<> void 
@@ -113,8 +115,79 @@ template<> void
 WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+#undef COMPLEX_TYPE
+	
+#endif //Single precision			    
+
+#if defined(GRID_DEFAULT_PRECISION_DOUBLE)    
+//temporary separating the two sections
+//for debug in isolation
+//can be unified
+
+    ///////////////////////////////////////////////////////////
+    // If we are AVX512 specialise the double precision routine
+    ///////////////////////////////////////////////////////////
+
+#include <simd/Intel512double.h>
+    
+static Vector<vComplexD> signs;
+    
+  int setupSigns(void ){
+    Vector<vComplexD> bother(2);
+    signs = bother;
+    vrsign(signs[0]);
+    visign(signs[1]);
+    return 1;
+  }
+  static int signInit = setupSigns();
+  
+#define label(A)  ilabel(A)
+#define ilabel(A) ".globl\n"  #A ":\n" 
+  
+#define MAYBEPERM(A,perm) if (perm) { A ; }
+#define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
+#define FX(A) WILSONASM_ ##A
+#define COMPLEX_TYPE vComplexD
+  
+#undef KERNEL_DAG
+template<> void 
+WilsonKernels<WilsonImplD>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+      
+#define KERNEL_DAG
+template<> void 
+WilsonKernels<WilsonImplD>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+						   int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
-#endif
+#undef VMOVIDUP
+#undef VMOVRDUP
+#undef MAYBEPERM
+#undef MULT_2SPIN
+#undef FX 
+#define FX(A) DWFASM_ ## A
+#define MAYBEPERM(A,B) 
+#define VMOVIDUP(A,B,C)                                  VBCASTIDUPf(A,B,C)
+#define VMOVRDUP(A,B,C)                                  VBCASTRDUPf(A,B,C)
+#define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
+				    
+#undef KERNEL_DAG
+template<> void 
+WilsonKernels<DomainWallVec5dImplD>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
+#define KERNEL_DAG
+template<> void 
+WilsonKernels<DomainWallVec5dImplD>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+	
+#undef COMPLEX_TYPE
+#endif //Double precision			    
+
+#endif //AVX512
 
 #define INSTANTIATE_ASM(A)\
 template void WilsonKernels<A>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,\
diff --git a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h
index 12579d8c..72e13754 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h
+++ b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h
@@ -5,7 +5,9 @@
   const uint64_t plocal =(uint64_t) & in._odata[0];
 
   //  vComplexF isigns[2] = { signs[0], signs[1] };
-  vComplexF *isigns = &signs[0];
+  //COMPLEX_TYPE is vComplexF of vComplexD depending 
+  //on the chosen precision
+  COMPLEX_TYPE *isigns = &signs[0];
 
   MASK_REGS;
   int nmax=U._grid->oSites();
diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h
index 62789462..136c940e 100644
--- a/lib/simd/Grid_avx512.h
+++ b/lib/simd/Grid_avx512.h
@@ -371,14 +371,8 @@ namespace Optimization {
   // Some Template specialization
 
   // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases
-<<<<<<< HEAD
-#define GNU_CLANG_COMPILER 
-#ifdef GNU_CLANG_COMPILER
-=======
-
 #ifndef __INTEL_COMPILER
 #warning "Slow reduction due to incomplete reduce intrinsics"
->>>>>>> develop
   //Complex float Reduce
   template<>
     inline Grid::ComplexF Reduce<Grid::ComplexF, __m512>::operator()(__m512 in){

From 618abdf302813e38921c506282d559296dcfd7b5 Mon Sep 17 00:00:00 2001
From: James Harrison <jch1g10@soton.ac.uk>
Date: Sat, 29 Oct 2016 11:04:02 +0100
Subject: [PATCH 107/149] Add missing volume factor in stochastic QED field

---
 lib/qcd/action/gauge/Photon.h | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
index 0dd5a9dc..409569d8 100644
--- a/lib/qcd/action/gauge/Photon.h
+++ b/lib/qcd/action/gauge/Photon.h
@@ -151,12 +151,19 @@ namespace QCD{
   {
     auto               *grid = dynamic_cast<GridCartesian *>(out._grid);
     const unsigned int nd = grid->_ndimension;
+    std::vector<int> latt_size   = grid->_fdimensions;
     GaugeLinkField     sqrtK2Inv(grid), r(grid);
     GaugeField         aTilde(grid);
     FFT                fft(grid);
     
+    Integer vol = 1;
+    for(int d = 0; d < nd; d++)
+    {
+      vol = vol * latt_size[d];
+    }
+
     invKHatSquared(sqrtK2Inv);
-    sqrtK2Inv = sqrt(real(sqrtK2Inv));
+    sqrtK2Inv = sqrt(vol*real(sqrtK2Inv));
     zmSub(sqrtK2Inv);
     for(int mu = 0; mu < nd; mu++)
     {

From 9b066e94d00c7190d9bf0531caa8964a4e79cf1f Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Sun, 30 Oct 2016 12:04:06 +0000
Subject: [PATCH 108/149] Compilation with both single and double precision

---
 lib/qcd/action/fermion/WilsonKernelsAsm.cc | 55 ++++++++++------------
 1 file changed, 25 insertions(+), 30 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsAsm.cc b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
index 2fc9b035..8bd55d61 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsm.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
@@ -10,6 +10,7 @@
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -55,23 +56,24 @@ WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,
 #if defined(AVX512) 
 #include <simd/Intel512wilson.h>
 
-#if defined(GRID_DEFAULT_PRECISION_SINGLE)    
     ///////////////////////////////////////////////////////////
     // If we are AVX512 specialise the single precision routine
     ///////////////////////////////////////////////////////////
 
 #include <simd/Intel512single.h>
     
-static Vector<vComplexF> signs;
-    
-  int setupSigns(void ){
-    Vector<vComplexF> bother(2);
+static Vector<vComplexF> signsF;
+
+  template<typename vtype>    
+  int setupSigns(Vector<vtype>& signs ){
+    Vector<vtype> bother(2);
     signs = bother;
     vrsign(signs[0]);
     visign(signs[1]);
     return 1;
   }
-  static int signInit = setupSigns();
+
+  static int signInitF = setupSigns(signsF);
   
 #define label(A)  ilabel(A)
 #define ilabel(A) ".globl\n"  #A ":\n" 
@@ -80,6 +82,7 @@ static Vector<vComplexF> signs;
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
 #define FX(A) WILSONASM_ ##A
 #define COMPLEX_TYPE vComplexF
+#define signs signsF
   
 #undef KERNEL_DAG
 template<> void 
@@ -116,34 +119,22 @@ WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,Lebe
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 #undef COMPLEX_TYPE
+#undef signs
+#undef VMOVRDUP
+#undef MAYBEPERM
+#undef MULT_2SPIN
+#undef FX 
 	
-#endif //Single precision			    
-
-#if defined(GRID_DEFAULT_PRECISION_DOUBLE)    
-//temporary separating the two sections
-//for debug in isolation
-//can be unified
-
-    ///////////////////////////////////////////////////////////
-    // If we are AVX512 specialise the double precision routine
-    ///////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////
+// If we are AVX512 specialise the double precision routine
+///////////////////////////////////////////////////////////
 
 #include <simd/Intel512double.h>
     
-static Vector<vComplexD> signs;
+static Vector<vComplexD> signsD;
+#define signs signsD
+static int signInitD = setupSigns(signsD);
     
-  int setupSigns(void ){
-    Vector<vComplexD> bother(2);
-    signs = bother;
-    vrsign(signs[0]);
-    visign(signs[1]);
-    return 1;
-  }
-  static int signInit = setupSigns();
-  
-#define label(A)  ilabel(A)
-#define ilabel(A) ".globl\n"  #A ":\n" 
-  
 #define MAYBEPERM(A,perm) if (perm) { A ; }
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
 #define FX(A) WILSONASM_ ##A
@@ -185,7 +176,11 @@ WilsonKernels<DomainWallVec5dImplD>::DiracOptAsmDhopSiteDag(StencilImpl &st,Lebe
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 	
 #undef COMPLEX_TYPE
-#endif //Double precision			    
+#undef signs
+#undef VMOVRDUP
+#undef MAYBEPERM
+#undef MULT_2SPIN
+#undef FX 
 
 #endif //AVX512
 

From e8c3174ae28746c8f24377ac7a4fbddc7d9a3831 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Sun, 30 Oct 2016 12:23:11 +0000
Subject: [PATCH 109/149] Small change in the defines

---
 lib/qcd/action/fermion/WilsonKernelsAsm.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsAsm.cc b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
index 8bd55d61..83124d1a 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsm.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
@@ -159,8 +159,8 @@ WilsonKernels<WilsonImplD>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder
 #undef FX 
 #define FX(A) DWFASM_ ## A
 #define MAYBEPERM(A,B) 
-#define VMOVIDUP(A,B,C)                                  VBCASTIDUPf(A,B,C)
-#define VMOVRDUP(A,B,C)                                  VBCASTRDUPf(A,B,C)
+#define VMOVIDUP(A,B,C)                                  VBCASTIDUPd(A,B,C)
+#define VMOVRDUP(A,B,C)                                  VBCASTRDUPd(A,B,C)
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
 				    
 #undef KERNEL_DAG

From e74417ca12b4e416d151f06081fe70dd5f1e6651 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 31 Oct 2016 16:31:27 +0000
Subject: [PATCH 110/149] big build system polish

---
 configure.ac                                  | 266 ++++++++++--------
 lib/FFT.h                                     |   2 +-
 .../iterative/ImplicitlyRestartedLanczos.h    |   6 +-
 3 files changed, 155 insertions(+), 119 deletions(-)

diff --git a/configure.ac b/configure.ac
index d6340a71..9fb35e78 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,5 +1,5 @@
 AC_PREREQ([2.63])
-AC_INIT([Grid], [0.5.1-dev], [https://github.com/paboyle/Grid], [Grid])
+AC_INIT([Grid], [0.6.0], [https://github.com/paboyle/Grid], [Grid])
 AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
@@ -9,22 +9,33 @@ AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
-
 ############### Checks for programs
-AC_LANG(C++)
 CXXFLAGS="-O3 $CXXFLAGS"
 AC_PROG_CXX
 AC_PROG_RANLIB
 
-############ openmp  ###############
+############### Get compiler informations
+AC_LANG([C++])
+AX_CXX_COMPILE_STDCXX_11([noext],[mandatory])
+AX_COMPILER_VENDOR
+AC_DEFINE_UNQUOTED([CXX_COMP_VENDOR],["$ax_cv_cxx_compiler_vendor"],
+      [vendor of C++ compiler that will compile the code])
+AX_GXX_VERSION
+AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
+      [version of g++ that will compile the code])
+
+############### Checks for typedefs, structures, and compiler characteristics
+AC_TYPE_SIZE_T
+AC_TYPE_UINT32_T
+AC_TYPE_UINT64_T
+
+############### OpenMP 
 AC_OPENMP
-
 ac_openmp=no
-
 if test "${OPENMP_CXXFLAGS}X" != "X"; then
-ac_openmp=yes
-AM_CXXFLAGS="$OPENMP_CXXFLAGS $AM_CXXFLAGS"
-AM_LDFLAGS="$OPENMP_CXXFLAGS $AM_LDFLAGS"
+  ac_openmp=yes
+  AM_CXXFLAGS="$OPENMP_CXXFLAGS $AM_CXXFLAGS"
+  AM_LDFLAGS="$OPENMP_CXXFLAGS $AM_LDFLAGS"
 fi
 
 ############### Checks for header files
@@ -37,12 +48,7 @@ AC_CHECK_HEADERS(execinfo.h)
 AC_CHECK_DECLS([ntohll],[], [], [[#include <arpa/inet.h>]])
 AC_CHECK_DECLS([be64toh],[], [], [[#include <arpa/inet.h>]])
 
-############### Checks for typedefs, structures, and compiler characteristics
-AC_TYPE_SIZE_T
-AC_TYPE_UINT32_T
-AC_TYPE_UINT64_T
-
-############### GMP and MPFR #################
+############### GMP and MPFR
 AC_ARG_WITH([gmp],
     [AS_HELP_STRING([--with-gmp=prefix],
     [try this for a non-standard install prefix of the GMP library])],
@@ -54,10 +60,17 @@ AC_ARG_WITH([mpfr],
     [AM_CXXFLAGS="-I$with_mpfr/include $AM_CXXFLAGS"]
     [AM_LDFLAGS="-L$with_mpfr/lib $AM_LDFLAGS"])
 
-################## lapack ####################
+############### FFTW3 
+AC_ARG_WITH([fftw],    
+            [AS_HELP_STRING([--with-fftw=prefix],
+            [try this for a non-standard install prefix of the FFTW3 library])],
+            [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
+            [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
+
+############### lapack 
 AC_ARG_ENABLE([lapack],
     [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])], 
-    [ac_LAPACK=${enable_lapack}],[ac_LAPACK=no])
+    [ac_LAPACK=${enable_lapack}], [ac_LAPACK=no])
 
 case ${ac_LAPACK} in
     no)
@@ -67,10 +80,26 @@ case ${ac_LAPACK} in
     *)
         AM_CXXFLAGS="-I$ac_LAPACK/include $AM_CXXFLAGS"
         AM_LDFLAGS="-L$ac_LAPACK/lib $AM_LDFLAGS"
-        AC_DEFINE([USE_LAPACK],[1],[use LAPACK])
+        AC_DEFINE([USE_LAPACK],[1],[use LAPACK]);;
 esac
 
-################## first-touch ####################
+############### MKL
+AC_ARG_ENABLE([mkl],
+    [AC_HELP_STRING([--enable-mkl=yes|no|prefix], [enable Intel MKL for LAPACK & FFTW])],
+    [ac_MKL=${enable_mkl}], [ac_MKL=no])
+
+case ${ac_MKL} in
+    no)
+        ;;
+    yes)
+        AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
+    *)
+        AM_CXXFLAGS="-I$ac_MKL/include $AM_CXXFLAGS"
+        AM_LDFLAGS="-L$ac_MKL/lib $AM_LDFLAGS"
+        AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
+esac
+
+############### first-touch
 AC_ARG_ENABLE([numa],
     [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
     [ac_NUMA=${enable_NUMA}],[ac_NUMA=no])
@@ -84,56 +113,44 @@ case ${ac_NUMA} in
         AC_DEFINE([GRID_NUMA],[1],[First touch numa locality]);;
 esac
 
-################## FFTW3 ####################
-AC_ARG_WITH([fftw],    
-            [AS_HELP_STRING([--with-fftw=prefix],
-            [try this for a non-standard install prefix of the FFTW3 library])],
-            [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
-            [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
-
-################ Get compiler informations
-AC_LANG([C++])
-AX_CXX_COMPILE_STDCXX_11([noext],[mandatory])
-AX_COMPILER_VENDOR
-AC_DEFINE_UNQUOTED([CXX_COMP_VENDOR],["$ax_cv_cxx_compiler_vendor"],
-      [vendor of C++ compiler that will compile the code])
-AX_GXX_VERSION
-AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
-      [version of g++ that will compile the code])
-
 ############### Checks for library functions
 CXXFLAGS_CPY=$CXXFLAGS
 LDFLAGS_CPY=$LDFLAGS
 CXXFLAGS="$AM_CXXFLAGS $CXXFLAGS"
 LDFLAGS="$AM_LDFLAGS $LDFLAGS"
+
 AC_CHECK_FUNCS([gettimeofday])
-AC_CHECK_LIB([gmp],[__gmpf_init],
-             [AC_CHECK_LIB([mpfr],[mpfr_init],
-                 [AC_DEFINE([HAVE_LIBMPFR], [1], [Define to 1 if you have the `MPFR' library (-lmpfr).])]
-                 [have_mpfr=true]
-                 [LIBS="$LIBS -lmpfr"],
-                 [AC_MSG_ERROR([MPFR library not found])])]
-   	     [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library (-lgmp).])]
-             [have_gmp=true]
-             [LIBS="$LIBS -lgmp"],
-             [AC_MSG_WARN([**** GMP library not found, Grid can still compile but RHMC will not work ****])])
+
+if test "${ac_MKL}x" != "nox"; then
+    AC_SEARCH_LIBS([mkl_set_interface_layer], [mkl_rt], [],
+                   [AC_MSG_ERROR("MKL enabled but library not found")])
+fi
+
+AC_SEARCH_LIBS([__gmpf_init], [gmp],
+               [AC_SEARCH_LIBS([mpfr_init], [mpfr], 
+                               [AC_DEFINE([HAVE_LIBMPFR], [1], 
+                                          [Define to 1 if you have the `MPFR' library])]
+                               [have_mpfr=true], [AC_MSG_ERROR([MPFR library not found])])]
+               [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library])]
+               [have_gmp=true])
 
 if test "${ac_LAPACK}x" != "nox"; then
-    AC_CHECK_LIB([lapack],[LAPACKE_sbdsdc],[],
-                 [AC_MSG_ERROR("LAPACK enabled but library not found")])
-fi
-AC_CHECK_LIB([fftw3],[fftw_execute],
-  [AC_DEFINE([HAVE_FFTW],[1],[Define to 1 if you have the `FFTW' library (-lfftw3).])]
-  [have_fftw=true]
-  [LIBS="$LIBS -lfftw3 -lfftw3f"],
-  [AC_MSG_WARN([**** FFTW library not found, Grid can still compile but FFT-based routines will not work ****])])
+    AC_SEARCH_LIBS([LAPACKE_sbdsdc], [lapack], [],
+                   [AC_MSG_ERROR("LAPACK enabled but library not found")])
+fi   
+
+AC_SEARCH_LIBS([fftw_execute], [fftw3],
+               [AC_SEARCH_LIBS([fftwf_execute], [fftw3f], [],
+                               [AC_MSG_ERROR("single precision FFTW library not found")])]
+               [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
+               [have_fftw=true])
+
 CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
 ############### SIMD instruction selection
-AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=SSE4|AVX|AVXFMA4|AVXFMA|AVX2|AVX512|AVX512MIC|IMCI|KNL|KNC],\
-	[Select instructions to be SSE4.0, AVX 1.0, AVX 2.0+FMA, AVX 512, IMCI])],\
-	[ac_SIMD=${enable_simd}],[ac_SIMD=GEN])
+AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=<code>],
+	            [select SIMD target (cf. README.md)])], [ac_SIMD=${enable_simd}], [ac_SIMD=GEN])
 
 case ${ax_cv_cxx_compiler_vendor} in
   clang|gnu)
@@ -153,12 +170,15 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
         SIMD_FLAGS='-mavx2 -mfma';;
-      AVX512|AVX512MIC|KNL)
+      AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
-      IMCI|KNC)
+      KNC)
         AC_DEFINE([IMCI],[1],[IMCI intrinsics for Knights Corner])
         SIMD_FLAGS='';;
+      KNL)
+        AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
+        SIMD_FLAGS='-march=knl';;
       GEN)
         AC_DEFINE([GENERIC_VEC],[1],[generic vector code])
         SIMD_FLAGS='';;
@@ -176,9 +196,6 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVX)
         AC_DEFINE([AVX1],[1],[AVX intrinsics])
         SIMD_FLAGS='-mavx -xavx';;
-      AVXFMA4)
-        AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
-        SIMD_FLAGS='-mavx -mfma';;
       AVXFMA)
         AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA4])
         SIMD_FLAGS='-mavx -mfma';;
@@ -188,12 +205,12 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-xcore-avx512';;
-      AVX512MIC|KNL)
-        AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing])
-        SIMD_FLAGS='-xmic-avx512';;
-      IMCI|KNC)
+      KNC)
         AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner])
         SIMD_FLAGS='';;
+      KNL)
+        AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing])
+        SIMD_FLAGS='-xmic-avx512';;
       GEN)
         AC_DEFINE([GENERIC_VEC],[1],[generic vector code])
         SIMD_FLAGS='';;
@@ -208,14 +225,18 @@ AM_CXXFLAGS="$SIMD_FLAGS $AM_CXXFLAGS"
 AM_CFLAGS="$SIMD_FLAGS $AM_CFLAGS"
 
 case ${ac_SIMD} in
-  AVX512|AVX512MIC|KNL)
+  AVX512|KNL)
     AC_DEFINE([TEST_ZMM],[1],[compile ZMM test]);;
   *)
 	;;
 esac
 
-############### precision selection
-AC_ARG_ENABLE([precision],[AC_HELP_STRING([--enable-precision=single|double],[Select default word size of Real])],[ac_PRECISION=${enable_precision}],[ac_PRECISION=double])
+############### Precision selection
+AC_ARG_ENABLE([precision],
+              [AC_HELP_STRING([--enable-precision=single|double],
+                              [Select default word size of Real])],
+              [ac_PRECISION=${enable_precision}],[ac_PRECISION=double])
+
 case ${ac_PRECISION} in
      single)
        AC_DEFINE([GRID_DEFAULT_PRECISION_SINGLE],[1],[GRID_DEFAULT_PRECISION is SINGLE] )
@@ -226,43 +247,49 @@ case ${ac_PRECISION} in
 esac
 
 ############### communication type selection
-AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|shmem],[Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
+AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi3-auto|shmem],
+              [Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
 
 case ${ac_COMMS} in
      none)
-       AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
+        AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
      ;;
-     mpi-auto)
-       AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
-       LX_FIND_MPI
-       if test "x$have_CXX_mpi" = 'xno'; then AC_MSG_ERROR(["MPI not found"]); fi
-       AM_CXXFLAGS="$MPI_CXXFLAGS $AM_CXXFLAGS"
-       AM_CFLAGS="$MPI_CFLAGS $AM_CFLAGS"
-       AM_LDFLAGS="`echo $MPI_CXXLDFLAGS | sed -E 's/-l@<:@^ @:>@+//g'` $AM_LDFLAGS"
-       LIBS="`echo $MPI_CXXLDFLAGS | sed -E 's/-L@<:@^ @:>@+//g'` $LIBS"
+     mpi|mpi-auto)
+        AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
      ;;
-     mpi)
-       AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
-     ;;
-     mpi3)
-       AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
+     mpi3|mpi3-auto)
+        AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
      ;;
      shmem)
-       AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
+        AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
      ;;
      *)
-     AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); 
+        AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); 
      ;;
 esac
+case ${ac_COMMS} in
+    *-auto)
+        LX_FIND_MPI
+        if test "x$have_CXX_mpi" = 'xno'; then AC_MSG_ERROR(["MPI not found"]); fi
+        AM_CXXFLAGS="$MPI_CXXFLAGS $AM_CXXFLAGS"
+        AM_CFLAGS="$MPI_CFLAGS $AM_CFLAGS"
+        AM_LDFLAGS="`echo $MPI_CXXLDFLAGS | sed -E 's/-l@<:@^ @:>@+//g'` $AM_LDFLAGS"
+        LIBS="`echo $MPI_CXXLDFLAGS | sed -E 's/-L@<:@^ @:>@+//g'` $LIBS";;
+    *)
+        ;;
+esac
+
 AM_CONDITIONAL(BUILD_COMMS_SHMEM,[ test "X${ac_COMMS}X" == "XshmemX" ])
-AM_CONDITIONAL(BUILD_COMMS_MPI,[ test "X${ac_COMMS}X" == "XmpiX" || test "X${ac_COMMS}X" == "Xmpi-autoX" ])
-AM_CONDITIONAL(BUILD_COMMS_MPI3,[ test "X${ac_COMMS}X" == "Xmpi3X"] )
+AM_CONDITIONAL(BUILD_COMMS_MPI,
+               [ test "X${ac_COMMS}X" == "XmpiX" || test "X${ac_COMMS}X" == "Xmpi-autoX" ])
+AM_CONDITIONAL(BUILD_COMMS_MPI3,
+               [ test "X${ac_COMMS}X" == "Xmpi3X" || test "X${ac_COMMS}X" == "Xmpi3-autoX" ])
 AM_CONDITIONAL(BUILD_COMMS_NONE,[ test "X${ac_COMMS}X" == "XnoneX" ])
 
 ############### RNG selection
 AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937],\
-	[Select Random Number Generator to be used])],\
-	[ac_RNG=${enable_rng}],[ac_RNG=ranlux48])
+	            [Select Random Number Generator to be used])],\
+	            [ac_RNG=${enable_rng}],[ac_RNG=ranlux48])
 
 case ${ac_RNG} in
      ranlux48)
@@ -276,10 +303,11 @@ case ${ac_RNG} in
      ;;
 esac
 
-############### timer option
+############### Timer option
 AC_ARG_ENABLE([timers],[AC_HELP_STRING([--enable-timers],\
-	[Enable system dependent high res timers])],\
-	[ac_TIMERS=${enable_timers}],[ac_TIMERS=yes])
+	            [Enable system dependent high res timers])],\
+	            [ac_TIMERS=${enable_timers}],[ac_TIMERS=yes])
+
 case ${ac_TIMERS} in
      yes)
       AC_DEFINE([TIMERS_ON],[1],[TIMERS_ON] )
@@ -293,7 +321,9 @@ case ${ac_TIMERS} in
 esac
 
 ############### Chroma regression test
-AC_ARG_ENABLE([chroma],[AC_HELP_STRING([--enable-chroma],[Expect chroma compiled under c++11 ])],ac_CHROMA=yes,ac_CHROMA=no)
+AC_ARG_ENABLE([chroma],[AC_HELP_STRING([--enable-chroma],
+              [Expect chroma compiled under c++11 ])],ac_CHROMA=yes,ac_CHROMA=no)
+
 case ${ac_CHROMA} in
      yes|no)
      ;;
@@ -301,6 +331,7 @@ case ${ac_CHROMA} in
        AC_MSG_ERROR([${ac_CHROMA} unsupported --enable-chroma option]); 
      ;;
 esac
+
 AM_CONDITIONAL(BUILD_CHROMA_REGRESSION,[ test "X${ac_CHROMA}X" == "XyesX" ])
 
 ############### Doxygen
@@ -332,35 +363,36 @@ AC_CONFIG_FILES(tests/qdpxx/Makefile)
 AC_CONFIG_FILES(benchmarks/Makefile)
 AC_OUTPUT
 
-echo "
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Summary of configuration for $PACKAGE v$VERSION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ----- PLATFORM ----------------------------------------
-- architecture (build)          : $build_cpu
-- os (build)                    : $build_os
-- architecture (target)         : $target_cpu
-- os (target)                   : $target_os
-- compiler vendor               : ${ax_cv_cxx_compiler_vendor}
-- compiler version              : ${ax_cv_gxx_version}
+architecture (build)        : $build_cpu
+os (build)                  : $build_os
+architecture (target)       : $target_cpu
+os (target)                 : $target_os
+compiler vendor             : ${ax_cv_cxx_compiler_vendor}
+compiler version            : ${ax_cv_gxx_version}
 ----- BUILD OPTIONS -----------------------------------
-- SIMD                          : ${ac_SIMD}
-- Threading                     : ${ac_openmp} 
-- Communications type           : ${ac_COMMS}
-- Default precision             : ${ac_PRECISION}
-- RNG choice                    : ${ac_RNG} 
-- GMP                           : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
-- LAPACK                        : ${ac_LAPACK}
-- FFTW                          : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
-- build DOXYGEN documentation   : `if test "x$enable_doc" = xyes; then echo yes; else echo no; fi`
-- graphs and diagrams           : `if test "x$enable_dot" = xyes; then echo yes; else echo no; fi`
+SIMD                        : ${ac_SIMD}
+Threading                   : ${ac_openmp} 
+Communications type         : ${ac_COMMS}
+Default precision           : ${ac_PRECISION}
+RNG choice                  : ${ac_RNG} 
+GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
+LAPACK                      : ${ac_LAPACK}
+FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
+build DOXYGEN documentation : `if test "x$enable_doc" = xyes; then echo yes; else echo no; fi`
+graphs and diagrams         : `if test "x$enable_dot" = xyes; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
-- CXXFLAGS:
+CXXFLAGS:
 `echo ${AM_CXXFLAGS} ${CXXFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
-- LDFLAGS:
+LDFLAGS:
 `echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
-- LIBS:
+LIBS:
 `echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
--------------------------------------------------------
-"
+-------------------------------------------------------" > config.summary
+echo ""
+cat config.summary
+echo ""
diff --git a/lib/FFT.h b/lib/FFT.h
index 9a59ed01..8d359cba 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -30,7 +30,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #define _GRID_FFT_H_
 
 #ifdef HAVE_FFTW	
-#include <fftw3.h>
+#include <fftw/fftw3.h>
 #endif
 namespace Grid {
 
diff --git a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
index 1a1a0a16..5d6deae0 100644
--- a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
+++ b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
@@ -31,7 +31,11 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #include <string.h> //memset
 #ifdef USE_LAPACK
-#include <lapacke.h>
+void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
+                   double *vl, double *vu, int *il, int *iu, double *abstol,
+                   int *m, double *w, double *z, int *ldz, int *isuppz,
+                   double *work, int *lwork, int *iwork, int *liwork,
+                   int *info);
 #endif
 #include "DenseMatrix.h"
 #include "EigenSort.h"

From 66d832c7339444799452fd8d3510b1d46b88978c Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 31 Oct 2016 16:39:29 +0000
Subject: [PATCH 111/149] FFTW header fix

---
 lib/FFT.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index 8d359cba..f9f91ea6 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -30,7 +30,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #define _GRID_FFT_H_
 
 #ifdef HAVE_FFTW	
-#include <fftw/fftw3.h>
+#include <Grid/fftw/fftw3.h>
 #endif
 namespace Grid {
 

From 07416e4567140655a7153914b8d5f9d69e51598f Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 31 Oct 2016 18:21:52 +0000
Subject: [PATCH 112/149] README update

---
 README.md | 112 +++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 94 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 2a0acad6..faf86faf 100644
--- a/README.md
+++ b/README.md
@@ -16,11 +16,27 @@
 
 **Data parallel C++ mathematical object library.**
 
-Please send all pull requests to the `develop` branch.
-
 License: GPL v2.
 
-Last update 2016/08/03.
+Last update Nov 2016.
+
+_Please send all pull requests to the `develop` branch._
+
+### Bug report
+
+_To help us tracking and solving more efficiently issues with Grid, please report problems using the issue system of GitHub rather than sending emails to Grid developers._
+
+When you file an issue, please go though the following checklist:
+
+1. Check that the code is pointing to the `HEAD` of `develop` or any commit in `master` which is tagged with a version number. 
+2. Give a description of the target platform (CPU, network, compiler).
+3. Give the exact `configure` command used.
+4. Attach `config.log`.
+5. Attach `config.summary`.
+6. Attach the output of `make V=1`.
+7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example.
+
+
 
 ### Description
 This library provides data parallel C++ container classes with internal memory layout
@@ -42,7 +58,7 @@ optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a signifi
 for most programmers.
 
 The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture.
-Presently SSE4 (128 bit) AVX, AVX2 (256 bit) and IMCI and AVX512 (512 bit) targets are supported (ARM NEON and BG/Q QPX on the way).
+Presently SSE4 (128 bit) AVX, AVX2, QPX (256 bit), IMCI, and AVX512 (512 bit) targets are supported (ARM NEON on the way).
 
 These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. These may be useful in themselves for other programmers.
 The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`.
@@ -50,7 +66,7 @@ The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `Compl
 MPI, OpenMP, and SIMD parallelism are present in the library.
 Please see https://arxiv.org/abs/1512.03487 for more detail.
 
-### Installation
+### Quick start
 First, start by cloning the repository:
 
 ``` bash
@@ -71,12 +87,10 @@ mkdir build; cd build
 ../configure --enable-precision=double --enable-simd=AVX --enable-comms=mpi-auto --prefix=<path>
 ```
 
-where `--enable-precision=` set the default precision (`single` or `double`),
-`--enable-simd=` set the SIMD type (see possible values below), `--enable-
-comms=` set the protocol used for communications (`none`, `mpi`, `mpi-auto` or
-`shmem`), and `<path>` should be replaced by the prefix path where you want to
-install Grid. The `mpi-auto` communication option set `configure` to determine
-automatically how to link to MPI. Other options are available, use `configure
+where `--enable-precision=` set the default precision,
+`--enable-simd=` set the SIMD type, `--enable-
+comms=`, and `<path>` should be replaced by the prefix path where you want to
+install Grid. Other options are detailed in the next section, you can also use `configure
 --help` to display them. Like with any other program using GNU autotool, the
 `CXX`, `CXXFLAGS`, `LDFLAGS`, ... environment variables can be modified to
 customise the build.
@@ -93,24 +107,86 @@ To minimise the build time, only the tests at the root of the `tests` directory
 make -C tests/<subdir> tests
 ```
 
+### Build configuration options
+
+- `--prefix=<path>`: installation prefix for Grid.
+- `--with-gmp=<path>`: look for GMP in the UNIX prefix `<path>`
+- `--with-mpfr=<path>`: look for MPFR in the UNIX prefix `<path>`
+- `--with-fftw=<path>`: look for FFTW in the UNIX prefix `<path>`
+- `--enable-lapack[=<path>]`: enable LAPACK support in Lanczos eigensolver. A UNIX prefix containing the library can be specified (optional).
+- `--enable-mkl[=<path>]`: use Intel MKL for FFT (and LAPACK if enabled) routines. A UNIX prefix containing the library can be specified (optional).
+- `--enable-numa`: ???
+- `--enable-simd=<code>`: setup Grid for the SIMD target `<code>` (default: `GEN`). A list of possible SIMD targets is detailed in a section below.
+- `--enable-precision={single|double}`: set the default precision (default: `double`).
+- `--enable-precision=<comm>`: Use `<comm>` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below.
+- `--enable-rng={ranlux48|mt19937}`: choose the RNG (default: `ranlux48 `).
+- `--disable-timers`: disable system dependent high-resolution timers.
+- `--enable-chroma`: enable Chroma regression tests.
+
+### Possible communication interfaces
+
+The following options can be use with the `--enable-simd=` option to target different communication interfaces:
+
+| `<comm>`      | Description                                  |
+| ------------- | -------------------------------------------- |
+| `none`        | no communications                            |
+| `mpi[-auto]`  | MPI communications                           |
+| `mpi3[-auto]` | MPI communications using MPI 3 shared memory |
+| `shmem `      | Cray SHMEM communications                    |
+
+For `mpi` and `mpi3` the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names).
+
 ### Possible SIMD types
 
 The following options can be use with the `--enable-simd=` option to target different SIMD instruction sets:
 
-| String      | Description                            |
+| `<code>`    | Description                            |
 | ----------- | -------------------------------------- |
 | `GEN`       | generic portable vector code           |
 | `SSE4`      | SSE 4.2 (128 bit)                      |
 | `AVX`       | AVX (256 bit)                          |
-| `AVXFMA4`   | AVX (256 bit) + FMA                    |
+| `AVXFMA`    | AVX (256 bit) + FMA                    |
+| `AVXFMA4`   | AVX (256 bit) + FMA4                   |
 | `AVX2`      | AVX 2 (256 bit)                        |
 | `AVX512`    | AVX 512 bit                            |
-| `AVX512MIC` | AVX 512 bit for Intel MIC architecture |
-| `ICMI`      | Intel ICMI instructions (512 bit)      |
+| `QPX`       | QPX (256 bit)                          |
 
 Alternatively, some CPU codenames can be directly used:
 
-| String      | Description                            |
+| `<code>`    | Description                            |
 | ----------- | -------------------------------------- |
-| `KNC`       | [Intel Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) |
-| `KNL`       | [Intel Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |
\ No newline at end of file
+| `KNC`       | [Intel Xeon Phi codename Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) |
+| `KNL`       | [Intel Xeon Phi codename Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |
+| `BGQ`       | Blue Gene/Q                            |
+
+#### Notes:
+- We currently support AVX512 only for the Intel compiler. Support for GCC and clang will appear in future versions.
+- For BG/Q only [bgclang](http://trac.alcf.anl.gov/projects/llvm-bgq) is supported. We do not presently plan to support more compilers for this platform.
+- BG/Q performances are currently rather poor. This is being investigated for future versions.
+
+### Build setup for Intel Knights Landing platform
+
+The following configuration is recommended for the Intel Knights Landing platform:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=KNL        \
+             --enable-comms=mpi3-auto \
+             --with-gmp=<path>        \
+             --with-mpfr=<path>       \
+             --enable-mkl             \
+             CXX=icpc MPICXX=mpiicpc
+```
+
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=KNL        \
+             --enable-comms=mpi3      \
+             --with-gmp=<path>        \
+             --with-mpfr=<path>       \
+             --enable-mkl             \
+             CXX=CC CC=cc
+```
+

From 791cb050c8e79bd0994e474044da5f055a9c05e8 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 1 Nov 2016 11:35:43 +0000
Subject: [PATCH 113/149] Comms improvements

---
 benchmarks/Benchmark_comms.cc                |  86 ++-
 bootstrap.sh                                 |   6 -
 configure.ac                                 |   4 +
 lib/Cshift.h                                 |   4 +
 lib/Makefile.am                              |   5 +
 lib/communicator/Communicator_base.cc        |  15 +-
 lib/communicator/Communicator_base.h         |  43 +-
 lib/communicator/Communicator_mpi.cc         |  12 +-
 lib/communicator/Communicator_mpi3.cc        |  12 +-
 lib/communicator/Communicator_mpi3_leader.cc | 733 +++++++++++++++++++
 lib/communicator/Communicator_none.cc        |   8 +-
 lib/communicator/Communicator_shmem.cc       |  31 +-
 scripts/arm_configure.experimental           |   1 -
 scripts/arm_configure.experimental_cortex57  |   3 -
 scripts/bench_wilson.sh                      |   9 -
 scripts/build-all                            |  46 --
 scripts/configure-all                        |  11 -
 scripts/configure-commands                   |  89 ---
 scripts/configure-cray                       |  10 -
 scripts/configure-mic                        |  10 -
 scripts/copyright                            |  16 +-
 scripts/cray-modules                         |   2 -
 scripts/reconfigure_script                   |   4 -
 scripts/update_fftw.sh                       |  18 -
 scripts/wilson.gnu                           |   7 -
 25 files changed, 898 insertions(+), 287 deletions(-)
 create mode 100644 lib/communicator/Communicator_mpi3_leader.cc
 delete mode 100644 scripts/arm_configure.experimental
 delete mode 100644 scripts/arm_configure.experimental_cortex57
 delete mode 100755 scripts/bench_wilson.sh
 delete mode 100755 scripts/build-all
 delete mode 100755 scripts/configure-all
 delete mode 100755 scripts/configure-commands
 delete mode 100755 scripts/configure-cray
 delete mode 100755 scripts/configure-mic
 delete mode 100644 scripts/cray-modules
 delete mode 100755 scripts/reconfigure_script
 delete mode 100755 scripts/update_fftw.sh
 delete mode 100644 scripts/wilson.gnu

diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index d0cd96c7..234d2fbb 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -42,14 +42,13 @@ int main (int argc, char ** argv)
 
   int Nloop=10;
   int nmu=0;
-  for(int mu=0;mu<4;mu++) if (mpi_layout[mu]>1) nmu++;
+  for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
+
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
 
-
-
   for(int lat=4;lat<=32;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
@@ -194,6 +193,87 @@ int main (int argc, char ** argv)
     }
   }  
 
+
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking concurrent STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+
+  for(int lat=4;lat<=32;lat+=2){
+    for(int Ls=1;Ls<=16;Ls*=2){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],
+      				    lat*mpi_layout[1],
+      				    lat*mpi_layout[2],
+      				    lat*mpi_layout[3]});
+
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+
+      std::vector<HalfSpinColourVectorD *> xbuf(8);
+      std::vector<HalfSpinColourVectorD *> rbuf(8);
+      Grid.ShmBufferFreeAll();
+      for(int d=0;d<8;d++){
+	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+      }
+
+      int ncomm;
+      int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+
+      double start=usecond();
+      for(int i=0;i<Nloop;i++){
+
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
+
+	ncomm=0;
+	for(int mu=0;mu<4;mu++){
+	
+	  if (mpi_layout[mu]>1 ) {
+	  
+	    ncomm++;
+	    int comm_proc=1;
+	    int xmit_to_rank;
+	    int recv_from_rank;
+	    
+	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    Grid.StencilSendToRecvFromBegin(requests,
+					    (void *)&xbuf[mu][0],
+					    xmit_to_rank,
+					    (void *)&rbuf[mu][0],
+					    recv_from_rank,
+					    bytes);
+	
+	    comm_proc = mpi_layout[mu]-1;
+	  
+	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    Grid.StencilSendToRecvFromBegin(requests,
+					    (void *)&xbuf[mu+4][0],
+					    xmit_to_rank,
+					    (void *)&rbuf[mu+4][0],
+					    recv_from_rank,
+					    bytes);
+	  
+	  }
+	}
+	Grid.StencilSendToRecvFromComplete(requests);
+	Grid.Barrier();
+
+      }
+      double stop=usecond();
+
+      double dbytes    = bytes;
+      double xbytes    = Nloop*dbytes*2.0*ncomm;
+      double rbytes    = xbytes;
+      double bidibytes = xbytes+rbytes;
+
+      double time = stop-start; // microseconds
+
+      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+    }
+  }    
+
+
+
 #if 0
 
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
diff --git a/bootstrap.sh b/bootstrap.sh
index f847b7ab..7361161a 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -1,18 +1,12 @@
 #!/usr/bin/env bash
 
 EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2'
-FFTW_URL=http://www.fftw.org/fftw-3.3.4.tar.gz
 
 echo "-- deploying Eigen source..."
 wget ${EIGEN_URL} --no-check-certificate
 ./scripts/update_eigen.sh `basename ${EIGEN_URL}`
 rm `basename ${EIGEN_URL}`
 
-echo "-- copying fftw prototypes..."
-wget ${FFTW_URL}
-./scripts/update_fftw.sh `basename ${FFTW_URL}`
-rm `basename ${FFTW_URL}`
-
 echo '-- generating Make.inc files...'
 ./scripts/filelist
 echo '-- generating configure script...'
diff --git a/configure.ac b/configure.ac
index d6340a71..9587d271 100644
--- a/configure.ac
+++ b/configure.ac
@@ -247,6 +247,9 @@ case ${ac_COMMS} in
      mpi3)
        AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
      ;;
+     mpi3l)
+       AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] )
+     ;;
      shmem)
        AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
      ;;
@@ -257,6 +260,7 @@ esac
 AM_CONDITIONAL(BUILD_COMMS_SHMEM,[ test "X${ac_COMMS}X" == "XshmemX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI,[ test "X${ac_COMMS}X" == "XmpiX" || test "X${ac_COMMS}X" == "Xmpi-autoX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI3,[ test "X${ac_COMMS}X" == "Xmpi3X"] )
+AM_CONDITIONAL(BUILD_COMMS_MPI3L,[ test "X${ac_COMMS}X" == "Xmpi3lX"] )
 AM_CONDITIONAL(BUILD_COMMS_NONE,[ test "X${ac_COMMS}X" == "XnoneX" ])
 
 ############### RNG selection
diff --git a/lib/Cshift.h b/lib/Cshift.h
index 4eeb6cea..cd162e35 100644
--- a/lib/Cshift.h
+++ b/lib/Cshift.h
@@ -42,6 +42,10 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/cshift/Cshift_mpi.h>
 #endif 
 
+#ifdef GRID_COMMS_MPI3L
+#include <Grid/cshift/Cshift_mpi.h>
+#endif 
+
 #ifdef GRID_COMMS_SHMEM
 #include <Grid/cshift/Cshift_mpi.h> // uses same implementation of communicator
 #endif 
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 3285a545..a779135f 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -9,6 +9,11 @@ if BUILD_COMMS_MPI3
   extra_sources+=communicator/Communicator_base.cc
 endif
 
+if BUILD_COMMS_MPI3L
+  extra_sources+=communicator/Communicator_mpi3_leader.cc
+  extra_sources+=communicator/Communicator_base.cc
+endif
+
 if BUILD_COMMS_SHMEM
   extra_sources+=communicator/Communicator_shmem.cc
   extra_sources+=communicator/Communicator_base.cc
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 9810987d..77b4f800 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -31,13 +31,6 @@ namespace Grid {
 ///////////////////////////////////////////////////////////////
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////
-int CartesianCommunicator::ShmRank;
-int CartesianCommunicator::ShmSize;
-int CartesianCommunicator::GroupRank;
-int CartesianCommunicator::GroupSize;
-int CartesianCommunicator::WorldRank;
-int CartesianCommunicator::WorldSize;
-int CartesianCommunicator::Slave;
 void *              CartesianCommunicator::ShmCommBuf;
 
 /////////////////////////////////
@@ -70,12 +63,6 @@ int                      CartesianCommunicator::ProcessorCount(void)    { return
 ////////////////////////////////////////////////////////////////////////////////
 // very VERY rarely (Log, serial RNG) we need world without a grid
 ////////////////////////////////////////////////////////////////////////////////
-int  CartesianCommunicator::RankWorld(void){ return WorldRank; };
-int CartesianCommunicator::Ranks    (void) { return WorldSize; };
-int CartesianCommunicator::Nodes    (void) { return GroupSize; };
-int CartesianCommunicator::Cores    (void) { return ShmSize;   };
-int CartesianCommunicator::NodeRank (void) { return GroupRank; };
-int CartesianCommunicator::CoreRank (void) { return ShmRank;   };
 
 void CartesianCommunicator::GlobalSum(ComplexF &c)
 {
@@ -94,7 +81,7 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
   GlobalSumVector((double *)c,2*N);
 }
 
-#ifndef GRID_COMMS_MPI3
+#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPI3L)
 
 void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 						       void *xmit,
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 9a15d2b0..d4bb50d9 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -1,3 +1,4 @@
+
     /*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
@@ -37,6 +38,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_COMMS_MPI3
 #include <mpi.h>
 #endif
+#ifdef GRID_COMMS_MPI3L
+#include <mpi.h>
+#endif
 #ifdef GRID_COMMS_SHMEM
 #include <mpp/shmem.h>
 #endif
@@ -60,9 +64,9 @@ class CartesianCommunicator {
   std::vector<int> _processor_coor;  // linear processor coordinate
   unsigned long _ndimension;
 
-#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3)
-  MPI_Comm communicator;
+#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPI3L)
   static MPI_Comm communicator_world;
+         MPI_Comm communicator;
   typedef MPI_Request CommsRequest_t;
 #else 
   typedef int CommsRequest_t;
@@ -75,7 +79,15 @@ class CartesianCommunicator {
   // cartesian communicator on a subset of ranks, slave ranks controlled
   // by group leader with data xfer via shared memory
   ////////////////////////////////////////////////////////////////////
-#ifdef  GRID_COMMS_MPI3
+#ifdef GRID_COMMS_MPI3
+
+  static int ShmRank;
+  static int ShmSize;
+  static int GroupRank;
+  static int GroupSize;
+  static int WorldRank;
+  static int WorldSize;
+
   std::vector<int>  WorldDims;
   std::vector<int>  GroupDims;
   std::vector<int>  ShmDims;
@@ -83,7 +95,7 @@ class CartesianCommunicator {
   std::vector<int> GroupCoor;
   std::vector<int> ShmCoor;
   std::vector<int> WorldCoor;
-  
+
   static std::vector<int> GroupRanks; 
   static std::vector<int> MyGroup;
   static int ShmSetup;
@@ -93,13 +105,20 @@ class CartesianCommunicator {
   std::vector<int>  LexicographicToWorldRank;
   
   static std::vector<void *> ShmCommBufs;
+
 #else 
   static void ShmInitGeneric(void);
   static commVector<uint8_t> ShmBufStorageVector;
 #endif 
+
+  /////////////////////////////////
+  // Grid information and queries
+  // Implemented in Communicator_base.C
+  /////////////////////////////////
   static void * ShmCommBuf;
   size_t heap_top;
   size_t heap_bytes;
+
   void *ShmBufferSelf(void);
   void *ShmBuffer(int rank);
   void *ShmBufferTranslate(int rank,void * local_p);
@@ -123,28 +142,12 @@ class CartesianCommunicator {
   int  RankFromProcessorCoor(std::vector<int> &coor);
   void ProcessorCoorFromRank(int rank,std::vector<int> &coor);
   
-  /////////////////////////////////
-  // Grid information and queries
-  /////////////////////////////////
-  static int ShmRank;
-  static int ShmSize;
-  static int GroupSize;
-  static int GroupRank;
-  static int WorldRank;
-  static int WorldSize;
-  static int Slave;
-  
   int                      IsBoss(void)            ;
   int                      BossRank(void)          ;
   int                      ThisRank(void)          ;
   const std::vector<int> & ThisProcessorCoor(void) ;
   const std::vector<int> & ProcessorGrid(void)     ;
   int                      ProcessorCount(void)    ;
-  static int Ranks    (void);
-  static int Nodes    (void);
-  static int Cores    (void);
-  static int NodeRank (void);
-  static int CoreRank (void);
 
   ////////////////////////////////////////////////////////////////////////////////
   // very VERY rarely (Log, serial RNG) we need world without a grid
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index a638eebb..65ced9c7 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -44,13 +44,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     MPI_Init(argc,argv);
   }
   MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
-  MPI_Comm_rank(communicator_world,&WorldRank);
-  MPI_Comm_size(communicator_world,&WorldSize);
-  ShmRank=0;
-  ShmSize=1;
-  GroupRank=WorldRank;
-  GroupSize=WorldSize;
-  Slave    =0;
   ShmInitGeneric();
 }
 
@@ -198,6 +191,11 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
   // Should only be used prior to Grid Init finished.
   // Check for this?
   ///////////////////////////////////////////////////////
+int CartesianCommunicator::RankWorld(void){ 
+  int r; 
+  MPI_Comm_rank(communicator_world,&r);
+  return r;
+}
 void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
 {
   int ierr= MPI_Bcast(data,
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 14a152ab..c707ec1f 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -30,12 +30,18 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
-
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 int CartesianCommunicator::ShmSetup = 0;
 
+int CartesianCommunicator::ShmRank;
+int CartesianCommunicator::ShmSize;
+int CartesianCommunicator::GroupRank;
+int CartesianCommunicator::GroupSize;
+int CartesianCommunicator::WorldRank;
+int CartesianCommunicator::WorldSize;
+
 MPI_Comm CartesianCommunicator::communicator_world;
 MPI_Comm CartesianCommunicator::ShmComm;
 MPI_Win  CartesianCommunicator::ShmWindow;
@@ -97,15 +103,15 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   
   std::vector<int> world_ranks(WorldSize); 
   GroupRanks.resize(WorldSize); 
-  MyGroup.resize(ShmSize);
   for(int r=0;r<WorldSize;r++) world_ranks[r]=r;
   
   MPI_Group_translate_ranks (WorldGroup,WorldSize,&world_ranks[0],ShmGroup, &GroupRanks[0]); 
 
   ///////////////////////////////////////////////////////////////////
   // Identify who is in my group and noninate the leader
-    ///////////////////////////////////////////////////////////////////
+  ///////////////////////////////////////////////////////////////////
   int g=0;
+  MyGroup.resize(ShmSize);
   for(int rank=0;rank<WorldSize;rank++){
     if(GroupRanks[rank]!=MPI_UNDEFINED){
       assert(g<ShmSize);
diff --git a/lib/communicator/Communicator_mpi3_leader.cc b/lib/communicator/Communicator_mpi3_leader.cc
new file mode 100644
index 00000000..50e63eb7
--- /dev/null
+++ b/lib/communicator/Communicator_mpi3_leader.cc
@@ -0,0 +1,733 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/communicator/Communicator_mpi.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include "Grid.h"
+#include <mpi.h>
+
+namespace Grid {
+
+enum { COMMAND_ISEND, COMMAND_IRECV, COMMAND_WAITALL };
+
+struct Descriptor {
+  uint64_t buf;
+  size_t bytes;
+  int rank;
+  int tag;
+  int command;
+  MPI_Request request;
+};
+
+const int pool = 48;
+
+class SlaveState {
+public:
+  volatile int head;
+  volatile int start;
+  volatile int tail;
+  volatile Descriptor Descrs[pool];
+};
+
+class Slave {
+public:
+  SlaveState *state;
+  MPI_Comm squadron;
+  uint64_t     base;
+  ////////////////////////////////////////////////////////////
+  // Descriptor circular pointers
+  ////////////////////////////////////////////////////////////
+  Slave() {};
+
+  void Init(SlaveState * _state,MPI_Comm _squadron);
+  
+  void EventLoop (void) {
+    std::cerr<< " Entering even loop "<<std::endl;
+    while(1) {
+      Event();
+    }
+  }
+
+  void Event (void) ;
+
+  uint64_t QueueCommand(int command,void *buf, int bytes, int hashtag, MPI_Comm comm,int rank) ;
+
+  void WaitAll() {
+    QueueCommand(COMMAND_WAITALL,0,0,0,squadron,0);
+    std::cerr<< " Waiting on FIFO drain "<<std::endl;
+    while ( state->tail != state->head );
+    std::cerr<< " FIFO drained "<< state->tail <<std::endl;
+  }
+};
+
+////////////////////////////////////////////////////////////////////////
+// One instance of a data mover.
+// Master and Slave must agree on location in shared memory
+////////////////////////////////////////////////////////////////////////
+
+class MPIoffloadEngine { 
+public:
+
+  static std::vector<Slave> Slaves;
+
+  static int ShmSetup;
+  
+  static int UniverseRank;
+  static int UniverseSize;
+  
+  static MPI_Comm communicator_universe;
+  static MPI_Comm communicator_cached;
+
+  static MPI_Comm HorizontalComm;
+  static int HorizontalRank;
+  static int HorizontalSize;
+  
+  static MPI_Comm VerticalComm;
+  static MPI_Win  VerticalWindow; 
+  static int VerticalSize;
+  static int VerticalRank;
+  
+  static std::vector<void *> VerticalShmBufs;
+  static std::vector<std::vector<int> > UniverseRanks;
+  static std::vector<int> UserCommunicatorToWorldRanks; 
+  
+  static MPI_Group WorldGroup, CachedGroup;
+  
+  static void CommunicatorInit (MPI_Comm &communicator_world,
+				MPI_Comm &ShmComm,
+				void * &ShmCommBuf);
+
+  static void MapCommRankToWorldRank(int &hashtag, int & comm_world_peer,int tag, MPI_Comm comm,int rank);
+
+  /////////////////////////////////////////////////////////
+  // routines for master proc must handle any communicator
+  /////////////////////////////////////////////////////////
+
+  static uint64_t QueueSend(int slave,void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
+    std::cerr<< " Queueing send  "<< bytes<<std::endl;
+    return Slaves[slave].QueueCommand(COMMAND_ISEND,buf,bytes,tag,comm,rank);
+  };
+
+  static uint64_t QueueRecv(int slave, void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
+    std::cerr<< " Queueing receive  "<< bytes<<std::endl;
+    return Slaves[slave].QueueCommand(COMMAND_IRECV,buf,bytes,tag,comm,rank);
+  };
+
+  static void WaitAll() {
+    for(int s=1;s<VerticalSize;s++) {
+      Slaves[s].WaitAll();
+    }
+  };
+
+  static void GetWork(int nwork, int me, int & mywork, int & myoff,int units){
+    int basework = nwork/units;
+    int backfill = units-(nwork%units);
+    if ( me >= units ) { 
+      mywork = myoff = 0;
+    } else { 
+      mywork = (nwork+me)/units;
+      myoff  = basework * me;
+      if ( me > backfill ) 
+	myoff+= (me-backfill);
+    }
+    return;
+  };
+
+  static void QueueMultiplexedSend(void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
+    uint8_t * cbuf = (uint8_t *) buf;
+    int mywork, myoff, procs;
+    procs = VerticalSize-1;
+    for(int s=0;s<procs;s++) {
+      GetWork(bytes,s,mywork,myoff,procs);
+      QueueSend(s+1,&cbuf[myoff],mywork,tag,comm,rank);
+    }
+  };
+
+  static void QueueMultiplexedRecv(void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
+    uint8_t * cbuf = (uint8_t *) buf;
+    int mywork, myoff, procs;
+    procs = VerticalSize-1;
+    for(int s=0;s<procs;s++) {
+      GetWork(bytes,s,mywork,myoff,procs);
+      QueueRecv(s+1,&cbuf[myoff],mywork,tag,comm,rank);
+    }
+  };
+
+};
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+std::vector<Slave> MPIoffloadEngine::Slaves;
+    
+int MPIoffloadEngine::UniverseRank;
+int MPIoffloadEngine::UniverseSize;
+
+MPI_Comm  MPIoffloadEngine::communicator_universe;
+MPI_Comm  MPIoffloadEngine::communicator_cached;
+MPI_Group MPIoffloadEngine::WorldGroup;
+MPI_Group MPIoffloadEngine::CachedGroup;
+
+MPI_Comm MPIoffloadEngine::HorizontalComm;
+int      MPIoffloadEngine::HorizontalRank;
+int      MPIoffloadEngine::HorizontalSize;
+
+MPI_Comm MPIoffloadEngine::VerticalComm;
+MPI_Win  MPIoffloadEngine::VerticalWindow; 
+int      MPIoffloadEngine::VerticalSize;
+int      MPIoffloadEngine::VerticalRank;
+
+std::vector<void *>            MPIoffloadEngine::VerticalShmBufs;
+std::vector<std::vector<int> > MPIoffloadEngine::UniverseRanks;
+std::vector<int>               MPIoffloadEngine::UserCommunicatorToWorldRanks; 
+
+int MPIoffloadEngine::ShmSetup = 0;
+
+void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
+					 MPI_Comm &ShmComm,
+					 void * &ShmCommBuf)
+{      
+  int flag;
+  assert(ShmSetup==0);  
+  
+  //////////////////////////////////////////////////////////////////////
+  // Universe is all nodes prior to squadron grouping
+  //////////////////////////////////////////////////////////////////////
+  MPI_Comm_dup (MPI_COMM_WORLD,&communicator_universe);
+  MPI_Comm_rank(communicator_universe,&UniverseRank);
+  MPI_Comm_size(communicator_universe,&UniverseSize);
+  
+  /////////////////////////////////////////////////////////////////////
+  // Split into groups that can share memory (Verticals)
+  /////////////////////////////////////////////////////////////////////
+  //  MPI_Comm_split_type(communicator_universe, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&VerticalComm);
+  MPI_Comm_split(communicator_universe,(UniverseRank&0x1),UniverseRank,&VerticalComm);
+  MPI_Comm_rank(VerticalComm     ,&VerticalRank);
+  MPI_Comm_size(VerticalComm     ,&VerticalSize);
+  
+  //////////////////////////////////////////////////////////////////////
+  // Split into horizontal groups by rank in squadron
+  //////////////////////////////////////////////////////////////////////
+  MPI_Comm_split(communicator_universe,VerticalRank,UniverseRank,&HorizontalComm);
+  MPI_Comm_rank(HorizontalComm,&HorizontalRank);
+  MPI_Comm_size(HorizontalComm,&HorizontalSize);
+  assert(HorizontalSize*VerticalSize==UniverseSize);
+  
+  ////////////////////////////////////////////////////////////////////////////////
+  // What is my place in the world
+  ////////////////////////////////////////////////////////////////////////////////
+  int WorldRank=0;
+  if(VerticalRank==0) WorldRank = HorizontalRank;
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&WorldRank,1,MPI_INT,MPI_SUM,VerticalComm);
+  assert(ierr==0);
+  
+  ////////////////////////////////////////////////////////////////////////////////
+  // Where is the world in the universe?
+  ////////////////////////////////////////////////////////////////////////////////
+  UniverseRanks = std::vector<std::vector<int> >(HorizontalSize,std::vector<int>(VerticalSize,0));
+  UniverseRanks[WorldRank][VerticalRank] = UniverseRank;
+  for(int w=0;w<HorizontalSize;w++){
+    ierr=MPI_Allreduce(MPI_IN_PLACE,&UniverseRanks[w][0],VerticalSize,MPI_INT,MPI_SUM,communicator_universe);
+    assert(ierr==0);
+  }
+  
+  //////////////////////////////////////////////////////////////////////////////////////////////////////////
+  // allocate the shared window for our group, pass back Shm info to CartesianCommunicator
+  //////////////////////////////////////////////////////////////////////////////////////////////////////////
+  ierr = MPI_Win_allocate_shared(CartesianCommunicator::MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,VerticalComm,&ShmCommBuf,&VerticalWindow);
+  ierr|= MPI_Win_lock_all (MPI_MODE_NOCHECK, VerticalWindow);
+  assert(ierr==0);
+  
+  std::cerr<<"SHM "<<ShmCommBuf<<std::endl;
+
+  VerticalShmBufs.resize(VerticalSize);
+  for(int r=0;r<VerticalSize;r++){
+    MPI_Aint sz;
+    int dsp_unit;
+    MPI_Win_shared_query (VerticalWindow, r, &sz, &dsp_unit, &VerticalShmBufs[r]);
+    std::cerr<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // Map rank of leader on node in their in new world, to the
+  // rank in this vertical plane's horizontal communicator
+  //////////////////////////////////////////////////////////////////////
+  communicator_world = HorizontalComm;
+  ShmComm            = VerticalComm;
+  ShmCommBuf         = VerticalShmBufs[0];
+  MPI_Comm_group (communicator_world, &WorldGroup); 
+  
+  ///////////////////////////////////////////////////////////
+  // Start the slave data movers
+  ///////////////////////////////////////////////////////////
+  if ( VerticalRank != 0 ) {
+    Slave indentured;
+    indentured.Init( (SlaveState *) VerticalShmBufs[VerticalRank], VerticalComm);
+    indentured.EventLoop();
+    assert(0);
+  } else {
+    Slaves.resize(VerticalSize);
+    for(int i=1;i<VerticalSize;i++){
+      Slaves[i].Init((SlaveState *)VerticalShmBufs[i],VerticalComm);
+    }
+  }
+  
+  ///////////////////////////////////////////////////////////
+  // Verbose for now
+  ///////////////////////////////////////////////////////////
+  
+  ShmSetup=1;
+  
+  if (UniverseRank == 0){
+      
+    std::cout<<GridLogMessage << "Grid MPI-3 configuration: detected ";
+    std::cout<<UniverseSize   << " Ranks " ;
+    std::cout<<HorizontalSize << " Nodes " ;
+    std::cout<<VerticalSize   << " with ranks-per-node "<<std::endl;
+    
+    std::cout<<GridLogMessage << "Grid MPI-3 configuration: using one lead process per node " << std::endl;
+    std::cout<<GridLogMessage << "Grid MPI-3 configuration: reduced communicator has size " << HorizontalSize << std::endl;
+    
+    for(int g=0;g<HorizontalSize;g++){
+      std::cout<<GridLogMessage<<" Node "<<g<<" led by MPI rank "<< UniverseRanks[g][0]<<std::endl;
+    }
+    
+    for(int g=0;g<HorizontalSize;g++){
+      std::cout<<GridLogMessage<<" { ";
+      for(int s=0;s<VerticalSize;s++){
+	std::cout<< UniverseRanks[g][s];
+	if ( s<VerticalSize-1 ) {
+	  std::cout<<",";
+	}
+      }
+      std::cout<<" } "<<std::endl;
+    }
+  }
+};
+
+  ///////////////////////////////////////////////////////////////////////////////////////////////
+  // Map the communicator into communicator_world, and find the neighbour.
+  // Cache the mappings; cache size is 1.
+  ///////////////////////////////////////////////////////////////////////////////////////////////
+void MPIoffloadEngine::MapCommRankToWorldRank(int &hashtag, int & comm_world_peer,int tag, MPI_Comm comm,int rank) {
+
+  if ( comm == HorizontalComm ) {
+    comm_world_peer = rank;
+  } else if ( comm == communicator_cached ) {
+    comm_world_peer = UserCommunicatorToWorldRanks[rank];
+  } else { 
+    
+    int size;
+
+    MPI_Comm_size(comm,&size);
+
+    UserCommunicatorToWorldRanks.resize(size);
+
+    std::vector<int> cached_ranks(size); 
+
+    for(int r=0;r<size;r++) {
+      cached_ranks[r]=r;
+    }
+
+    communicator_cached=comm;
+    
+    MPI_Comm_group(communicator_cached, &CachedGroup);
+    
+    MPI_Group_translate_ranks(CachedGroup,size,&cached_ranks[0],WorldGroup, &UserCommunicatorToWorldRanks[0]); 
+    
+    comm_world_peer = UserCommunicatorToWorldRanks[rank];
+    
+    assert(comm_world_peer != MPI_UNDEFINED);
+  }
+
+  assert( (tag & (~0xFFFFL)) ==0); 
+  
+  uint64_t icomm = (uint64_t)comm;
+  int comm_hash = ((icomm>>0 )&0xFFFF)^((icomm>>16)&0xFFFF)
+                ^ ((icomm>>32)&0xFFFF)^((icomm>>48)&0xFFFF);
+  
+  hashtag = (comm_hash<<15) | tag;      
+
+};
+
+void Slave::Init(SlaveState * _state,MPI_Comm _squadron)
+{
+  squadron=_squadron;
+  state   =_state;
+  state->head = state->tail = state->start = 0;
+  MPI_Barrier(squadron);
+  base = (uint64_t)MPIoffloadEngine::VerticalShmBufs[0];
+  int rank; MPI_Comm_rank(_squadron,&rank);
+}
+#define PERI_PLUS(A) ( (A+1)%pool )
+void Slave::Event (void) {
+
+  static int tail_last;
+  static int head_last;
+  static int start_last;
+  int ierr;
+
+  if (   (state->tail != tail_last)
+       ||(state->head != head_last)
+       ||(state->start != start_last)
+       ) { 
+    std::cerr<< " Event loop "<< state->tail << " "<< state->start<< " "<< state->head <<std::endl;
+  }
+
+  ////////////////////////////////////////////////////
+  // Try to advance the tail pointers
+  ////////////////////////////////////////////////////
+  /*
+  int t=state->tail;
+  if ( t != state->start ) {
+    int flag=0;
+    
+    std::cerr<< " Testing tail "<< t<<" "<< (void *)&state->Descrs[t].request
+	     << " "<<state->Descrs[t].request<<std::endl;
+    //    ierr=MPI_Test((MPI_Request *)&state->Descrs[t].request,&flag,MPI_STATUS_IGNORE);
+    //    ierr=MPI_Test((MPI_Request *)&state->Descrs[t].request,&flag,MPI_STATUS_IGNORE);
+    assert(ierr==0);
+    if ( flag ) {
+      state->tail = PERI_PLUS(t);
+      std::cerr<< " Tail advanced from "<< t<<std::endl;
+      return;
+    }
+  }
+  */
+
+  ////////////////////////////////////////////////////
+  // Try to advance the start pointers
+  ////////////////////////////////////////////////////
+  int s=state->start;
+  if ( s != state->head ) {
+    switch ( state->Descrs[s].command ) {
+    case COMMAND_ISEND:
+      std::cerr<< " Send "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
+	       << " to " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
+	       << " Comm " << MPIoffloadEngine::communicator_universe<< std::endl;
+
+      std::cerr<< " Request was "<<state->Descrs[s].request<<std::endl;
+      ierr = MPI_Isend((void *)(state->Descrs[s].buf+base), 
+		       state->Descrs[s].bytes, 
+		       MPI_CHAR,
+		       state->Descrs[s].rank,
+		       state->Descrs[s].tag,
+		       MPIoffloadEngine::communicator_universe,
+		       (MPI_Request *)&state->Descrs[s].request);
+      std::cerr<< " Request is "<<state->Descrs[s].request<<std::endl;
+      std::cerr<< " Request0 is "<<state->Descrs[0].request<<std::endl;
+      assert(ierr==0);
+      state->start = PERI_PLUS(s);
+      break;
+
+    case COMMAND_IRECV:
+      std::cerr<< " Recv "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
+	       << " to " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
+	       << " Comm " << MPIoffloadEngine::communicator_universe<< std::endl;
+
+      std::cerr<< " Request was "<<state->Descrs[s].request<<std::endl;
+      ierr=MPI_Irecv((void *)(state->Descrs[s].buf+base), 
+		     state->Descrs[s].bytes, 
+		     MPI_CHAR,
+		     state->Descrs[s].rank,
+		     state->Descrs[s].tag,
+		     MPIoffloadEngine::communicator_universe,
+		     (MPI_Request *)&state->Descrs[s].request);
+      std::cerr<< " Request is "<<state->Descrs[s].request<<std::endl;
+      std::cerr<< " Request0 is "<<state->Descrs[0].request<<std::endl;
+      assert(ierr==0);
+      state->start = PERI_PLUS(s);
+      break;
+
+    case COMMAND_WAITALL:
+      std::cerr<< " Wait all "<<std::endl;
+      for(int t=state->tail;t!=s; t=PERI_PLUS(t) ){
+	std::cerr<< " Wait ["<<t<<"] "<<state->Descrs[t].request <<std::endl;
+	std::cerr<< " Request0 is "<<state->Descrs[0].request<<std::endl;
+	MPI_Wait((MPI_Request *)&state->Descrs[t].request,MPI_STATUS_IGNORE);
+      };
+      s=PERI_PLUS(s);
+      state->start = s;
+      state->tail  = s;
+      break;
+
+    default:
+      assert(0);
+      break;
+    }
+    return;
+  }
+}
+  //////////////////////////////////////////////////////////////////////////////
+  // External interaction with the queue
+  //////////////////////////////////////////////////////////////////////////////
+  
+uint64_t Slave::QueueCommand(int command,void *buf, int bytes, int tag, MPI_Comm comm,int commrank) 
+{
+  /////////////////////////////////////////
+  // Spin; if FIFO is full until not full
+  /////////////////////////////////////////
+  int head =state->head;
+  int next = PERI_PLUS(head);
+    
+  // Set up descriptor
+  int worldrank;
+  int hashtag;
+  MPI_Comm    communicator;
+  MPI_Request request;
+  
+  MPIoffloadEngine::MapCommRankToWorldRank(hashtag,commrank,tag,comm,worldrank);
+
+  int VerticalRank = MPIoffloadEngine::VerticalRank;
+  uint64_t relative= (uint64_t)buf - base;
+  state->Descrs[head].buf    = relative;
+  state->Descrs[head].bytes  = bytes;
+  state->Descrs[head].rank   = MPIoffloadEngine::UniverseRanks[worldrank][VerticalRank];
+  state->Descrs[head].tag    = hashtag;
+  state->Descrs[head].command= command;
+  std::cerr<< " QueueCommand "<<buf<<"["<<bytes<<"]" << std::endl;
+
+  // Block until FIFO has space
+  while( state->tail==next );
+
+  // Msync on weak order architectures
+  // Advance pointer
+  state->head = next;
+
+  return 0;
+}
+  
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+MPI_Comm CartesianCommunicator::communicator_world;
+
+void CartesianCommunicator::Init(int *argc, char ***argv) 
+{
+  int flag;
+  MPI_Initialized(&flag); // needed to coexist with other libs apparently
+  if ( !flag ) {
+    MPI_Init(argc,argv);
+  }
+  communicator_world = MPI_COMM_WORLD;
+  MPI_Comm ShmComm;
+  MPIoffloadEngine::CommunicatorInit (communicator_world,ShmComm,ShmCommBuf);
+}
+void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
+{
+  int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest);
+  assert(ierr==0);
+}
+int CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor)
+{
+  int rank;
+  int ierr=MPI_Cart_rank  (communicator, &coor[0], &rank);
+  assert(ierr==0);
+  return rank;
+}
+void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor)
+{
+  coor.resize(_ndimension);
+  int ierr=MPI_Cart_coords  (communicator, rank, _ndimension,&coor[0]);
+  assert(ierr==0);
+}
+
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
+{ 
+  _ndimension = processors.size();
+  std::vector<int> periodic(_ndimension,1);
+
+  _Nprocessors=1;
+  _processors = processors;
+
+  for(int i=0;i<_ndimension;i++){
+    _Nprocessors*=_processors[i];
+  }
+
+  int Size; 
+  MPI_Comm_size(communicator_world,&Size);
+  assert(Size==_Nprocessors);
+
+  _processor_coor.resize(_ndimension);
+  MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator);
+  MPI_Comm_rank  (communicator,&_processor);
+  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
+};
+
+void CartesianCommunicator::GlobalSum(uint32_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(uint64_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(float &f){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSumVector(float *f,int N)
+{
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(double &d)
+{
+  int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSumVector(double *d,int N)
+{
+  int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+
+// Basic Halo comms primitive
+void CartesianCommunicator::SendToRecvFrom(void *xmit,
+					   int dest,
+					   void *recv,
+					   int from,
+					   int bytes)
+{
+  std::vector<CommsRequest_t> reqs(0);
+  SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes);
+  SendToRecvFromComplete(reqs);
+}
+
+void CartesianCommunicator::SendRecvPacket(void *xmit,
+					   void *recv,
+					   int sender,
+					   int receiver,
+					   int bytes)
+{
+  MPI_Status stat;
+  assert(sender != receiver);
+  int tag = sender;
+  if ( _processor == sender ) {
+    MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator);
+  }
+  if ( _processor == receiver ) { 
+    MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat);
+  }
+}
+
+// Basic Halo comms primitive
+void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						void *xmit,
+						int dest,
+						void *recv,
+						int from,
+						int bytes)
+{
+  MPI_Request xrq;
+  MPI_Request rrq;
+  int rank = _processor;
+  int ierr;
+  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+  
+  assert(ierr==0);
+
+  list.push_back(xrq);
+  list.push_back(rrq);
+}
+
+void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						       void *xmit,
+						       int dest,
+						       void *recv,
+						       int from,
+						       int bytes)
+{
+  uint64_t xmit_i = (uint64_t) xmit;
+  uint64_t recv_i = (uint64_t) recv;
+  uint64_t shm    = (uint64_t) ShmCommBuf;
+  // assert xmit and recv lie in shared memory region
+  assert( (xmit_i >= shm) && (xmit_i+bytes <= shm+MAX_MPI_SHM_BYTES) );
+  assert( (recv_i >= shm) && (recv_i+bytes <= shm+MAX_MPI_SHM_BYTES) );
+  MPIoffloadEngine::QueueMultiplexedSend(xmit,bytes,_processor,communicator,dest);
+  MPIoffloadEngine::QueueMultiplexedRecv(recv,bytes,from,communicator,from);
+}
+
+
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list)
+{
+  MPIoffloadEngine::WaitAll();
+}
+
+void CartesianCommunicator::StencilBarrier(void)
+{
+}
+
+void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
+{
+  int nreq=list.size();
+  std::vector<MPI_Status> status(nreq);
+  int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
+  assert(ierr==0);
+}
+
+void CartesianCommunicator::Barrier(void)
+{
+  int ierr = MPI_Barrier(communicator);
+  assert(ierr==0);
+}
+
+void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
+{
+  int ierr=MPI_Bcast(data,
+		     bytes,
+		     MPI_BYTE,
+		     root,
+		     communicator);
+  assert(ierr==0);
+}
+
+void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
+{
+  int ierr= MPI_Bcast(data,
+		      bytes,
+		      MPI_BYTE,
+		      root,
+		      communicator_world);
+  assert(ierr==0);
+}
+
+void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; }
+
+void *CartesianCommunicator::ShmBuffer(int rank) {
+  return NULL;
+}
+void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { 
+  return NULL;
+}
+
+
+};
+
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index 198c1add..0f43f1f5 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -34,13 +34,6 @@ namespace Grid {
 
 void CartesianCommunicator::Init(int *argc, char *** arv)
 {
-  WorldRank = 0;
-  WorldSize = 1;
-  ShmRank=0;
-  ShmSize=1;
-  GroupRank=WorldRank;
-  GroupSize=WorldSize;
-  Slave    =0;
   ShmInitGeneric();
 }
 
@@ -99,6 +92,7 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &
   assert(0);
 }
 
+int  CartesianCommunicator::RankWorld(void){return 0;}
 void CartesianCommunicator::Barrier(void){}
 void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {}
 void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { }
diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc
index d5f3ed76..56e03224 100644
--- a/lib/communicator/Communicator_shmem.cc
+++ b/lib/communicator/Communicator_shmem.cc
@@ -50,11 +50,16 @@ typedef struct HandShake_t {
   uint64_t seq_remote;
 } HandShake;
 
+std::array<long,_SHMEM_REDUCE_SYNC_SIZE> make_psync_init(void) {
+  array<long,_SHMEM_REDUCE_SYNC_SIZE> ret;
+  ret.fill(SHMEM_SYNC_VALUE);
+  return ret;
+}
+static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync_init = make_psync_init();
 
 static Vector< HandShake > XConnections;
 static Vector< HandShake > RConnections;
 
-
 void CartesianCommunicator::Init(int *argc, char ***argv) {
   shmem_init();
   XConnections.resize(shmem_n_pes());
@@ -65,13 +70,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     RConnections[pe].seq_local = 0;
     RConnections[pe].seq_remote= 0;
   }
-  WorldSize = shmem_n_pes();
-  WorldRank = shmem_my_pe();
-  ShmRank=0;
-  ShmSize=1;
-  GroupRank=WorldRank;
-  GroupSize=WorldSize;
-  Slave    =0;
   shmem_barrier_all();
   ShmInitGeneric();
 }
@@ -103,7 +101,7 @@ void CartesianCommunicator::GlobalSum(uint32_t &u){
   static long long source ;
   static long long dest   ;
   static long long llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static long      psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
 
   //  int nreduce=1;
   //  int pestart=0;
@@ -119,7 +117,7 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){
   static long long source ;
   static long long dest   ;
   static long long llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static long      psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
 
   //  int nreduce=1;
   //  int pestart=0;
@@ -135,7 +133,7 @@ void CartesianCommunicator::GlobalSum(float &f){
   static float source ;
   static float dest   ;
   static float llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
 
   source = f;
   dest   =0.0;
@@ -147,7 +145,7 @@ void CartesianCommunicator::GlobalSumVector(float *f,int N)
   static float source ;
   static float dest   = 0 ;
   static float llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
 
   if ( shmem_addr_accessible(f,_processor)  ){
     shmem_float_sum_to_all(f,f,N,0,0,_Nprocessors,llwrk,psync);
@@ -166,7 +164,7 @@ void CartesianCommunicator::GlobalSum(double &d)
   static double source;
   static double dest  ;
   static double llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
 
   source = d;
   dest   = 0;
@@ -178,7 +176,8 @@ void CartesianCommunicator::GlobalSumVector(double *d,int N)
   static double source ;
   static double dest   ;
   static double llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE];
-  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
+
 
   if ( shmem_addr_accessible(d,_processor)  ){
     shmem_double_sum_to_all(d,d,N,0,0,_Nprocessors,llwrk,psync);
@@ -295,7 +294,7 @@ void CartesianCommunicator::Barrier(void)
 }
 void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
 {
-  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
   static uint32_t word;
   uint32_t *array = (uint32_t *) data;
   assert( (bytes % 4)==0);
@@ -318,7 +317,7 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
 }
 void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
 {
-  static long  psync[_SHMEM_REDUCE_SYNC_SIZE];
+  static std::array<long,_SHMEM_REDUCE_SYNC_SIZE> psync =  psync_init;
   static uint32_t word;
   uint32_t *array = (uint32_t *) data;
   assert( (bytes % 4)==0);
diff --git a/scripts/arm_configure.experimental b/scripts/arm_configure.experimental
deleted file mode 100644
index fbad84a5..00000000
--- a/scripts/arm_configure.experimental
+++ /dev/null
@@ -1 +0,0 @@
-./configure --host=arm-linux-gnueabihf  CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target arm-linux-gnueabihf -I/usr/arm-linux-gnueabihf/include/ -I/home/neo/Codes/gmp6.0/gmp-arm/include/ -I/usr/arm-linux-gnueabihf/include/c++/4.8.2/arm-linux-gnueabihf/ -L/home/neo/Codes/gmp6.0/gmp-arm/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-arm/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-arm/lib/ -static -mcpu=cortex-a7' --enable-simd=NEONv7
diff --git a/scripts/arm_configure.experimental_cortex57 b/scripts/arm_configure.experimental_cortex57
deleted file mode 100644
index d229763e..00000000
--- a/scripts/arm_configure.experimental_cortex57
+++ /dev/null
@@ -1,3 +0,0 @@
-#./configure --host=arm-linux-gnueabihf  CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target arm-linux-gnueabihf -I/usr/arm-linux-gnueabihf/include/ -I/home/neo/Codes/gmp6.0/gmp-arm/include/ -I/usr/lib/llvm-3.5/lib/clang/3.5.0/include/ -L/home/neo/Codes/gmp6.0/gmp-arm/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-arm/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-arm/lib/ -static -mcpu=cortex-a57' --enable-simd=NEONv7
-
-./configure --host=aarch64-linux-gnu  CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target aarch64-linux-gnu -static -I/home/neo/Codes/gmp6.0/gmp-armv8/include/ -L/home/neo/Codes/gmp6.0/gmp-armv8/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-armv8/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-armv8/lib/ -I/usr/aarch64-linux-gnu/include/ -I/usr/aarch64-linux-gnu/include/c++/4.8.2/aarch64-linux-gnu/' --enable-simd=NEONv7
diff --git a/scripts/bench_wilson.sh b/scripts/bench_wilson.sh
deleted file mode 100755
index af73d591..00000000
--- a/scripts/bench_wilson.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-for omp in 1 2 4
-do
-echo > wilson.t$omp
-for vol in 4.4.4.4 4.4.4.8 4.4.8.8  4.8.8.8  8.8.8.8   8.8.8.16 8.8.16.16  8.16.16.16
-do   
-perf=` ./benchmarks/Grid_wilson --grid $vol --omp $omp  | grep mflop | awk '{print $3}'`
-echo $vol $perf >> wilson.t$omp
-done
-done
\ No newline at end of file
diff --git a/scripts/build-all b/scripts/build-all
deleted file mode 100755
index b97dca19..00000000
--- a/scripts/build-all
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash  -e
-
-DIRS="clang-avx clang-avx-openmp clang-avx-openmp-mpi clang-avx-mpi clang-avx2 clang-avx2-openmp clang-avx2-openmp-mpi clang-avx2-mpi clang-sse"
-EXTRADIRS="g++-avx g++-sse4 icpc-avx icpc-avx2 icpc-avx512"
-BLACK="\033[30m"
-RED="\033[31m"
-GREEN="\033[32m"
-YELLOW="\033[33m"
-BLUE="\033[34m"
-PINK="\033[35m"
-CYAN="\033[36m"
-WHITE="\033[37m"
-NORMAL="\033[0;39m"
-
-for D in $DIRS
-do
-
-echo
-echo -e $RED ==============================
-echo -e $GREEN $D
-echo -e $RED ==============================
-echo -e $BLUE
-
-  cd builds/$D
-  make clean all -j 8
-  cd ../../
-echo -e $NORMAL
-done
-
-if [ "X$1" == "Xextra" ]
-then
-for D in $EXTRADIRS
-do
-
-echo
-echo -e $RED ==============================
-echo -e $RED $D
-echo -e $RED ==============================
-echo -e $BLUE
-
-  cd builds/$D
-  make clean all -j 8
-  cd ../../
-echo -e $NORMAL
-done
-fi
\ No newline at end of file
diff --git a/scripts/configure-all b/scripts/configure-all
deleted file mode 100755
index ad91034d..00000000
--- a/scripts/configure-all
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-DIRS="clang-avx clang-avx-openmp clang-avx-openmp-mpi clang-avx-mpi clang-avx2 clang-avx2-openmp clang-avx2-openmp-mpi clang-avx2-mpi icpc-avx icpc-avx2 icpc-avx512 g++-sse4 g++-avx clang-sse icpc-avx-openmp-mpi icpc-avx-openmp"
-
-for D in $DIRS
-do
-  mkdir -p builds/$D
-  cd builds/$D
-  ../../scripts/configure-commands $D
-  cd ../..
-done
diff --git a/scripts/configure-commands b/scripts/configure-commands
deleted file mode 100755
index a3599d1f..00000000
--- a/scripts/configure-commands
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/bin/bash
-WD=$1
-BLACK="\033[30m"
-RED="\033[31m"
-GREEN="\033[32m"
-YELLOW="\033[33m"
-BLUE="\033[34m"
-PINK="\033[35m"
-CYAN="\033[36m"
-WHITE="\033[37m"
-NORMAL="\033[0;39m"
-echo
-echo -e $RED ==============================
-echo -e $GREEN $WD
-echo -e $RED ==============================
-echo -e $YELLOW
-
-case $WD in
-g++-avx)
-  CXX=g++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-g++-avx-openmp)
-  CXX=g++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11" LIBS="-fopenmp -lgmp -lmpfr" --enable-comms=none
-  ;;
-g++5-sse4)
-  CXX=g++-5 ../../configure --enable-simd=SSE4 CXXFLAGS="-msse4 -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-g++5-avx)
-  CXX=g++-5 ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-icpc-avx)
-  CXX=icpc ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-icpc-avx-openmp-mpi)
-CXX=icpc ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -fopenmp -lgmp -lmpfr" --enable-comms=mpi
-;;
-icpc-avx-openmp)
-CXX=icpc ../../configure --enable-precision=single --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11" LIBS="-fopenmp -lgmp -lmpfr" --enable-comms=mpi
-;;
-icpc-avx2)
-  CXX=icpc ../../configure --enable-simd=AVX2 CXXFLAGS="-march=core-avx2 -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-icpc-avx512)
-  CXX=icpc ../../configure --enable-simd=AVX512 CXXFLAGS="-xCOMMON-AVX512 -O3  -std=c++11" --host=none  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-icpc-mic)
-  CXX=icpc ../../configure --host=none --enable-simd=IMCI CXXFLAGS="-mmic -O3  -std=c++11" LDFLAGS=-mmic LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-icpc-mic-avx512)
-  CXX=icpc ../../configure --host=none --enable-simd=IMCI CXXFLAGS="-xCOMMON_AVX512 -O3  -std=c++11" LDFLAGS=-xCOMMON_AVX512 LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-sse)
-CXX=clang++ ../../configure --enable-precision=single --enable-simd=SSE4 CXXFLAGS="-msse4 -O3 -std=c++11"  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-avx)
-CXX=clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11"  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-avx2)
-CXX=clang++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -O3 -std=c++11"  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-avx-openmp)
-CXX=clang-omp++ ../../configure --enable-precision=double --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11" LDFLAGS="-fopenmp"  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-xc30)
-CXX=$HOME/Clang/install/bin/clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11 -I/opt/gcc/4.9.2/snos/include/g++/x86_64-suse-linux/ -I/opt/gcc/4.9.2/snos/include/g++/ " LDFLAGS=""  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-xc30-openmp)
-CXX=$HOME/Clang/install/bin/clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11 -I/opt/gcc/4.9.2/snos/include/g++/x86_64-suse-linux/ -I/opt/gcc/4.9.2/snos/include/g++/ " LDFLAGS="-fopenmp"  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-avx2-openmp)
-CXX=clang-omp++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -fopenmp -O3 -std=c++11" LDFLAGS="-fopenmp"  LIBS="-lgmp -lmpfr" --enable-comms=none
-  ;;
-clang-avx-openmp-mpi)
-CXX=clang-omp++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -fopenmp -lgmp -lmpfr" --enable-comms=mpi
-;;
-clang-avx2-openmp-mpi)
-CXX=clang-omp++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -fopenmp -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -fopenmp -lgmp -lmpfr" --enable-comms=mpi
-;;
-clang-avx-mpi)
-CXX=clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -lgmp -lmpfr" --enable-comms=mpi
-;;
-clang-avx2-mpi)
-CXX=clang++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -lgmp -lmpfr" --enable-comms=mpi
-;;
-clang-avx2)
-CXX=clang++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -O3 -std=c++11" LDFLAGS="-L/usr/local/lib/" LIBS="-lgmp -lmpfr" --enable-comms=none
-;;
-esac
-echo -e $NORMAL
diff --git a/scripts/configure-cray b/scripts/configure-cray
deleted file mode 100755
index db581493..00000000
--- a/scripts/configure-cray
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-DIRS="g++-avx-openmp g++-avx clang-xc30 clang-xc30-openmp"
-
-for D in $DIRS
-do
-  mkdir -p builds/$D
-  cd builds/$D
-  ../../scripts/configure-commands $D
-  cd ../..
-done
diff --git a/scripts/configure-mic b/scripts/configure-mic
deleted file mode 100755
index 668845fe..00000000
--- a/scripts/configure-mic
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-DIRS="build-icpc-mic"
-
-for D in $DIRS
-do
-  mkdir -p $D
- cd $D
-  ../configure-commands
-  cd ..
-done
diff --git a/scripts/copyright b/scripts/copyright
index 92772f16..cc9ed6e5 100755
--- a/scripts/copyright
+++ b/scripts/copyright
@@ -12,6 +12,7 @@ Grid physics library, www.github.com/paboyle/Grid
 Source file: $1
 
 Copyright (C) 2015
+Copyright (C) 2016
 
 EOF
 
@@ -38,8 +39,21 @@ See the full license in the file "LICENSE" in the top level distribution directo
 /*  END LEGAL */
 EOF
 
+
 cat message > tmp.fil
-cat $1 >> tmp.fil
+
+NOTICE=`grep -n "END LEGAL" $1 | awk '{ print $1 }'  `
+
+if [ "X$NOTICE" != "X" ]
+then
+    echo "found notice ending on line $NOTICE"
+    awk 'BEGIN { P=0 } { if ( P ) print } /END LEGAL/{P=1} ' $1 >> tmp.fil
+else
+    cat $1 >> tmp.fil
+      
+fi
+
+
 cp tmp.fil $1
 
 shift
diff --git a/scripts/cray-modules b/scripts/cray-modules
deleted file mode 100644
index 95de2b0b..00000000
--- a/scripts/cray-modules
+++ /dev/null
@@ -1,2 +0,0 @@
-module swap PrgEnv-cray PrgEnv-intel
-module swap intel/14.0.4.211 intel/15.0.2.164
diff --git a/scripts/reconfigure_script b/scripts/reconfigure_script
deleted file mode 100755
index d8d7212d..00000000
--- a/scripts/reconfigure_script
+++ /dev/null
@@ -1,4 +0,0 @@
-aclocal -I m4
-autoheader -f
-automake -f --add-missing
-autoconf -f
diff --git a/scripts/update_fftw.sh b/scripts/update_fftw.sh
deleted file mode 100755
index 20e42080..00000000
--- a/scripts/update_fftw.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-if (( $# != 1 )); then
-    echo "usage: `basename $0` <archive>" 1>&2
-    exit 1
-fi
-ARC=$1
-
-INITDIR=`pwd`
-rm -rf lib/fftw
-mkdir lib/fftw
-
-ARCDIR=`tar -tf ${ARC} | head -n1 | sed -e 's@/.*@@'`
-tar -xf ${ARC}
-cp ${ARCDIR}/api/fftw3.h lib/fftw/
-
-cd ${INITDIR}
-rm -rf ${ARCDIR}
diff --git a/scripts/wilson.gnu b/scripts/wilson.gnu
deleted file mode 100644
index 69bca5b5..00000000
--- a/scripts/wilson.gnu
+++ /dev/null
@@ -1,7 +0,0 @@
-plot 'wilson.t1' u 2 w l t "AVX1-OMP=1"
-replot 'wilson.t2' u 2 w l t "AVX1-OMP=2"
-replot 'wilson.t4' u 2 w l t "AVX1-OMP=4"
-set terminal 'pdf'
-set output 'wilson_clang.pdf'
-replot
-quit

From 7f0fc0eff5ee11e218462bab1b64a28de71c837e Mon Sep 17 00:00:00 2001
From: James Harrison <jch1g10@soton.ac.uk>
Date: Tue, 1 Nov 2016 16:02:35 +0000
Subject: [PATCH 114/149] Remove explicit use of double-precision types in
 photon.h

---
 lib/qcd/action/gauge/Photon.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
index 409569d8..852ecb3e 100644
--- a/lib/qcd/action/gauge/Photon.h
+++ b/lib/qcd/action/gauge/Photon.h
@@ -78,14 +78,14 @@ namespace QCD{
     const unsigned int nd    = grid->_ndimension;
     std::vector<int>   &l    = grid->_fdimensions;
     std::vector<int>   zm(nd,0);
-    TComplexD          Tone = ComplexD(1.0,0.0);
-    TComplexD          Tzero= ComplexD(0.0,0.0);
+    TComplex           Tone = Complex(1.0,0.0);
+    TComplex           Tzero= Complex(0.0,0.0);
     
-    one = ComplexD(1.0,0.0);
+    one = Complex(1.0,0.0);
     out = zero;
     for(int mu = 0; mu < nd; mu++)
     {
-      RealD twoPiL = M_PI*2./l[mu];
+      Real twoPiL = M_PI*2./l[mu];
       
       LatticeCoordinate(kmu,mu);
       kmu = 2.*sin(.5*twoPiL*kmu);
@@ -93,7 +93,7 @@ namespace QCD{
     }
     pokeSite(Tone, out, zm);
     out = one/out;
-    pokeSite(Tzero, out,zm);
+    pokeSite(Tzero, out, zm);
   }
   
   template<class Gimpl>
@@ -107,7 +107,7 @@ namespace QCD{
       case ZmScheme::QedTL:
       {
         std::vector<int> zm(nd,0);
-        TComplexD        Tzero = ComplexD(0.0,0.0);
+        TComplex         Tzero = Complex(0.0,0.0);
         
         pokeSite(Tzero, out, zm);
         

From bb94ddd0ebca02fb7c82b47e1befd411fafe0283 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 2 Nov 2016 08:07:09 +0000
Subject: [PATCH 115/149] Tidy up of mpi3; also some cleaning of the dslash
 controls.

---
 benchmarks/Benchmark_comms.cc                | 134 +-------------
 benchmarks/Benchmark_dwf.cc                  |  63 +++++--
 benchmarks/Benchmark_dwf_ntpf.cc             | 153 ----------------
 benchmarks/Benchmark_dwf_sweep.cc            |  16 +-
 benchmarks/Benchmark_wilson_sweep.cc         |  13 ++
 benchmarks/Benchmark_zmm.cc                  | 175 -------------------
 lib/Init.cc                                  | 156 +++++++++++------
 lib/communicator/Communicator_base.cc        |   9 +-
 lib/communicator/Communicator_base.h         |   2 +-
 lib/communicator/Communicator_mpi3_leader.cc | 147 ++++++++--------
 lib/qcd/action/fermion/WilsonKernels.cc      |   3 +-
 lib/qcd/action/fermion/WilsonKernels.h       |  80 ++++++---
 lib/stencil/Lebesgue.cc                      |   2 +-
 13 files changed, 321 insertions(+), 632 deletions(-)
 delete mode 100644 benchmarks/Benchmark_dwf_ntpf.cc
 delete mode 100644 benchmarks/Benchmark_zmm.cc

diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index 234d2fbb..eab9b9b9 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -48,8 +48,8 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "= Benchmarking concurrent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-
-  for(int lat=4;lat<=32;lat+=2){
+  int maxlat=32;
+  for(int lat=4;lat<=maxlat;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
@@ -124,7 +124,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
 
 
-  for(int lat=4;lat<=32;lat+=2){
+  for(int lat=4;lat<=maxlat;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
       std::vector<int> latt_size  ({lat,lat,lat,lat});
@@ -199,7 +199,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
 
-  for(int lat=4;lat<=32;lat+=2){
+  for(int lat=4;lat<=maxlat;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
@@ -271,131 +271,5 @@ int main (int argc, char ** argv)
       std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
-
-
-
-#if 0
-
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "= Benchmarking sequential persistent halo exchange in "<<nmu<<" dimensions"<<std::endl;
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-
-
-  for(int lat=4;lat<=32;lat+=2){
-    for(int Ls=1;Ls<=16;Ls*=2){
-
-      std::vector<int> latt_size  ({lat,lat,lat,lat});
-
-      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-
-
-      int ncomm;
-      int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
-
-
-      std::vector<CartesianCommunicator::CommsRequest_t> empty;
-      std::vector<std::vector<CartesianCommunicator::CommsRequest_t> > requests_fwd(Nd,empty);
-      std::vector<std::vector<CartesianCommunicator::CommsRequest_t> > requests_bwd(Nd,empty);
-
-      for(int mu=0;mu<4;mu++){
-	ncomm=0;
-	if (mpi_layout[mu]>1 ) {
-	  ncomm++;
-
-	  int comm_proc;
-	  int xmit_to_rank;
-	  int recv_from_rank;
-
-	  comm_proc=1;
-	  Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	  Grid.SendToRecvFromInit(requests_fwd[mu],
-				  (void *)&xbuf[mu][0],
-				  xmit_to_rank,
-				  (void *)&rbuf[mu][0],
-				  recv_from_rank,
-				  bytes);
-
-	  comm_proc = mpi_layout[mu]-1;
-	  Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	  Grid.SendToRecvFromInit(requests_bwd[mu],
-				  (void *)&xbuf[mu+4][0],
-				  xmit_to_rank,
-				  (void *)&rbuf[mu+4][0],
-				  recv_from_rank,
-				  bytes);
-
-	}
-      }
-
-      {
-	double start=usecond();
-	for(int i=0;i<Nloop;i++){
-	  
-	  for(int mu=0;mu<4;mu++){
-	    
-	    if (mpi_layout[mu]>1 ) {
-	      
-	      Grid.SendToRecvFromBegin(requests_fwd[mu]);
-	      Grid.SendToRecvFromComplete(requests_fwd[mu]);
-	      Grid.SendToRecvFromBegin(requests_bwd[mu]);
-	      Grid.SendToRecvFromComplete(requests_bwd[mu]);
-	    }
-	  }
-	  Grid.Barrier();
-	}
-	
-	double stop=usecond();
-	
-	double dbytes    = bytes;
-	double xbytes    = Nloop*dbytes*2.0*ncomm;
-	double rbytes    = xbytes;
-	double bidibytes = xbytes+rbytes;
-	
-	double time = stop-start;
-	
-	std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
-
-      }
-
-
-      {
-	double start=usecond();
-	for(int i=0;i<Nloop;i++){
-	  
-	  for(int mu=0;mu<4;mu++){
-	    
-	    if (mpi_layout[mu]>1 ) {
-	      
-	      Grid.SendToRecvFromBegin(requests_fwd[mu]);
-	      Grid.SendToRecvFromBegin(requests_bwd[mu]);
-	      Grid.SendToRecvFromComplete(requests_fwd[mu]);
-	      Grid.SendToRecvFromComplete(requests_bwd[mu]);
-	    }
-	  }
-	  Grid.Barrier();
-	}
-	
-	double stop=usecond();
-	
-	double dbytes    = bytes;
-	double xbytes    = Nloop*dbytes*2.0*ncomm;
-	double rbytes    = xbytes;
-	double bidibytes = xbytes+rbytes;
-	
-	double time = stop-start;
-	
-	std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
-
-      }
-
-    }
-  }
-
-#endif
-
   Grid_finalize();
 }
diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index c9e2fa22..5d42a1f9 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -44,7 +44,6 @@ struct scal {
     Gamma::GammaT
   };
 
-bool overlapComms = false;
 typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
 typedef WilsonFermion5D<DomainWallVec5dImplF> WilsonFermion5DF;
 typedef WilsonFermion5D<DomainWallVec5dImplD> WilsonFermion5DD;
@@ -54,10 +53,6 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
-  if( GridCmdOptionExists(argv,argv+argc,"--asynch") ){
-    overlapComms = true;
-  }
-
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
@@ -126,14 +121,21 @@ int main (int argc, char ** argv)
 
   RealD NP = UGrid->_Nprocessors;
 
-  for(int doasm=1;doasm<2;doasm++){
-
-    QCD::WilsonKernelsStatic::AsmOpt=doasm;
-
   DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-  
-  std::cout<<GridLogMessage << "Naive wilson implementation "<<std::endl;
-  std::cout << GridLogMessage<< "Calling Dw"<<std::endl;
+
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::Dhop                  "<<std::endl;
+  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
+  if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+  if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+
   int ncall =100;
   if (1) {
 
@@ -162,6 +164,17 @@ int main (int argc, char ** argv)
 
   if (1)
   {
+
+    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
+    std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::Dhop "<<std::endl;
+    std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
+    if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+    if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+    if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+    if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+    if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+    std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
+
     typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
     LatticeFermion ssrc(sFGrid);
     LatticeFermion sref(sFGrid);
@@ -248,6 +261,16 @@ int main (int argc, char ** argv)
       sr_e = zero;
       sr_o = zero;
 
+      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
+      std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::DhopEO "<<std::endl;
+      std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
+      if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+      if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
+
       sDw.ZeroCounters();
       sDw.stat.init("DhopEO");
       double t0=usecond();
@@ -308,7 +331,7 @@ int main (int argc, char ** argv)
     ref = -0.5*ref;
   }
   Dw.Dhop(src,result,1);
-  std::cout << GridLogMessage << "Naive wilson implementation Dag" << std::endl;
+  std::cout << GridLogMessage << "Compare to naive wilson implementation Dag to verify correctness" << std::endl;
   std::cout<<GridLogMessage << "Called DwDag"<<std::endl;
   std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
   std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
@@ -322,13 +345,22 @@ int main (int argc, char ** argv)
   LatticeFermion r_eo  (FGrid);
 
 
-  std::cout<<GridLogMessage << "Calling Deo and Doe"<<std::endl;
+  std::cout<<GridLogMessage << "Calling Deo and Doe and assert Deo+Doe == Dunprec"<<std::endl;
   pickCheckerboard(Even,src_e,src);
   pickCheckerboard(Odd,src_o,src);
 
   std::cout<<GridLogMessage << "src_e"<<norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "src_o"<<norm2(src_o)<<std::endl;
 
+  std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::DhopEO                "<<std::endl;
+  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
+  if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+  if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
   {
     Dw.ZeroCounters();
     double t0=usecond();
@@ -366,8 +398,5 @@ int main (int argc, char ** argv)
   assert(norm2(src_e)<1.0e-5);
   assert(norm2(src_o)<1.0e-5);
 
-
-  }
-
   Grid_finalize();
 }
diff --git a/benchmarks/Benchmark_dwf_ntpf.cc b/benchmarks/Benchmark_dwf_ntpf.cc
deleted file mode 100644
index 0d1e322f..00000000
--- a/benchmarks/Benchmark_dwf_ntpf.cc
+++ /dev/null
@@ -1,153 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./benchmarks/Benchmark_dwf.cc
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#include <Grid/Grid.h>
-
-using namespace std;
-using namespace Grid;
-using namespace Grid::QCD;
-
-template<class d>
-struct scal {
-  d internal;
-};
-
-  Gamma::GammaMatrix Gmu [] = {
-    Gamma::GammaX,
-    Gamma::GammaY,
-    Gamma::GammaZ,
-    Gamma::GammaT
-  };
-
-bool overlapComms = false;
-
-
-int main (int argc, char ** argv)
-{
-  Grid_init(&argc,&argv);
-
-  if( GridCmdOptionExists(argv,argv+argc,"--asynch") ){
-    overlapComms = true;
-  }
-
-  int threads = GridThread::GetThreads();
-  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
-
-  std::vector<int> latt4 = GridDefaultLatt();
-  const int Ls=16;
-  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
-  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
-  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
-  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
-
-  std::vector<int> seeds4({1,2,3,4});
-  std::vector<int> seeds5({5,6,7,8});
-
-  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
-  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
-
-  LatticeFermion src   (FGrid); random(RNG5,src);
-  LatticeFermion result(FGrid); result=zero;
-  LatticeFermion    ref(FGrid);    ref=zero;
-  LatticeFermion    tmp(FGrid);
-  LatticeFermion    err(FGrid);
-
-  ColourMatrix cm = Complex(1.0,0.0);
-
-  LatticeGaugeField Umu(UGrid); 
-  random(RNG4,Umu);
-
-  LatticeGaugeField Umu5d(FGrid); 
-
-  // replicate across fifth dimension
-  for(int ss=0;ss<Umu._grid->oSites();ss++){
-    for(int s=0;s<Ls;s++){
-      Umu5d._odata[Ls*ss+s] = Umu._odata[ss];
-    }
-  }
-
-  ////////////////////////////////////
-  // Naive wilson implementation
-  ////////////////////////////////////
-  std::vector<LatticeColourMatrix> U(4,FGrid);
-  for(int mu=0;mu<Nd;mu++){
-    U[mu] = PeekIndex<LorentzIndex>(Umu5d,mu);
-  }
-
-  if (1)
-  {
-    ref = zero;
-    for(int mu=0;mu<Nd;mu++){
-
-      tmp = U[mu]*Cshift(src,mu+1,1);
-      ref=ref + tmp - Gamma(Gmu[mu])*tmp;
-
-      tmp =adj(U[mu])*src;
-      tmp =Cshift(tmp,mu+1,-1);
-      ref=ref + tmp + Gamma(Gmu[mu])*tmp;
-    }
-    ref = -0.5*ref;
-  }
-
-  RealD mass=0.1;
-  RealD M5  =1.8;
-
-  typename DomainWallFermionR::ImplParams params; 
-  params.overlapCommsCompute = overlapComms;
-  
-  RealD NP = UGrid->_Nprocessors;
-
-
-  QCD::WilsonKernelsStatic::AsmOpt=1;
-
-  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,params);
-  
-  std::cout<<GridLogMessage << "Calling Dw"<<std::endl;
-  int ncall =50;
-  if (1) {
-
-    double t0=usecond();
-    for(int i=0;i<ncall;i++){
-      Dw.Dhop(src,result,0);
-    }
-    double t1=usecond();
-    
-    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=1344*volume*ncall;
-
-    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
-    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
-    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
-    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
-    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
-    err = ref-result; 
-    std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-    //    Dw.Report();
-  }
-  Grid_finalize();
-}
diff --git a/benchmarks/Benchmark_dwf_sweep.cc b/benchmarks/Benchmark_dwf_sweep.cc
index ee78a33d..a3803b87 100644
--- a/benchmarks/Benchmark_dwf_sweep.cc
+++ b/benchmarks/Benchmark_dwf_sweep.cc
@@ -51,16 +51,18 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+
   const int Ls=8;
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
-
-  if ( getenv("ASMOPT") )  {
-    QCD::WilsonKernelsStatic::AsmOpt=1;
-  } else { 
-    QCD::WilsonKernelsStatic::AsmOpt=0;
-  }
-
   std::cout<<GridLogMessage << "=========================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking DWF"<<std::endl;
   std::cout<<GridLogMessage << "=========================================================================="<<std::endl;
diff --git a/benchmarks/Benchmark_wilson_sweep.cc b/benchmarks/Benchmark_wilson_sweep.cc
index d7c712d3..96e5b5e4 100644
--- a/benchmarks/Benchmark_wilson_sweep.cc
+++ b/benchmarks/Benchmark_wilson_sweep.cc
@@ -58,6 +58,19 @@ int main (int argc, char ** argv)
   std::vector<int> seeds({1,2,3,4});
   RealD mass = 0.1;
 
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Benchmarking WilsonFermionR::Dhop                  "<<std::endl;
+  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
+  if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+  if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+
   std::cout<<GridLogMessage << "============================================================================="<< std::endl;
   std::cout<<GridLogMessage << "= Benchmarking Wilson" << std::endl;
   std::cout<<GridLogMessage << "============================================================================="<< std::endl;
diff --git a/benchmarks/Benchmark_zmm.cc b/benchmarks/Benchmark_zmm.cc
deleted file mode 100644
index c8ece12c..00000000
--- a/benchmarks/Benchmark_zmm.cc
+++ /dev/null
@@ -1,175 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./tests/Test_zmm.cc
-
-    Copyright (C) 2015
-
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#include <Grid/Grid.h>
-
-
-using namespace Grid;
-using namespace Grid::QCD;
-
-
-int bench(std::ofstream &os, std::vector<int> &latt4,int Ls);
-
-int main(int argc,char **argv)
-{
-  Grid_init(&argc,&argv);
-  std::ofstream os("zmm.dat");
-
-  os << "#V Ls Lxy Lzt C++ Asm OMP L1 " <<std::endl;
-  std::cout<<GridLogMessage << "====================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "= Benchmarking ZMM"<<std::endl;
-  std::cout<<GridLogMessage << "====================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "Volume \t\t\t\tC++DW/MFLOPs\tASM-DW/MFLOPs\tdiff"<<std::endl;
-  std::cout<<GridLogMessage << "====================================================================="<<std::endl;
-  for(int L=4;L<=32;L+=4){
-    for(int m=1;m<=2;m++){
-      for(int Ls=8;Ls<=16;Ls+=8){
-	std::vector<int> grid({L,L,m*L,m*L});
-  std::cout << GridLogMessage <<"\t";
-	for(int i=0;i<4;i++) { 
-	  std::cout << grid[i]<<"x";
-	}
-	std::cout << Ls<<"\t\t";
-	bench(os,grid,Ls);
-      }
-    }
-  }
-}
-
-int bench(std::ofstream &os, std::vector<int> &latt4,int Ls)
-{
-
-  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
-  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
-  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
-  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
-
-  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
-  std::vector<int> mpi_layout  = GridDefaultMpi();
-  int threads = GridThread::GetThreads();
-
-  std::vector<int> seeds4({1,2,3,4});
-  std::vector<int> seeds5({5,6,7,8});
-
-  GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds4);
-
-  LatticeFermion src (FGrid);
-  LatticeFermion tmp (FGrid);
-  LatticeFermion srce(FrbGrid);
-
-  LatticeFermion resulto(FrbGrid); resulto=zero;
-  LatticeFermion resulta(FrbGrid); resulta=zero;
-  LatticeFermion junk(FrbGrid); junk=zero;
-  LatticeFermion diff(FrbGrid); 
-  LatticeGaugeField Umu(UGrid);
-
-  double mfc, mfa, mfo, mfl1;
-
-  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
-  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
-  random(RNG5,src);
-#if 1
-  random(RNG4,Umu);
-#else
-  int mmu=2;
-  std::vector<LatticeColourMatrix> U(4,UGrid);
-  for(int mu=0;mu<Nd;mu++){
-    U[mu] = PeekIndex<LorentzIndex>(Umu,mu);
-    if ( mu!=mmu ) U[mu] = zero;
-    if ( mu==mmu ) U[mu] = 1.0;
-    PokeIndex<LorentzIndex>(Umu,U[mu],mu);
-  }
-#endif
- pickCheckerboard(Even,srce,src);
-
-  RealD mass=0.1;
-  RealD M5  =1.8;
-  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-
-  int ncall=50;
-  double t0=usecond();
-  for(int i=0;i<ncall;i++){
-    Dw.DhopOE(srce,resulto,0);
-  }
-  double t1=usecond();
-
-  double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-  double flops=1344*volume/2;
-
-  mfc = flops*ncall/(t1-t0);
-  std::cout<<mfc<<"\t\t";
-
-  QCD::WilsonKernelsStatic::AsmOpt=1;
-  t0=usecond();
-  for(int i=0;i<ncall;i++){
-    Dw.DhopOE(srce,resulta,0);
-  }
-  t1=usecond();
-  mfa = flops*ncall/(t1-t0);
-  std::cout<<mfa<<"\t\t";
-  /*
-  int dag=DaggerNo;
-  t0=usecond();
-  for(int i=0;i<1;i++){
-    Dw.DhopInternalOMPbench(Dw.StencilEven,Dw.LebesgueEvenOdd,Dw.UmuOdd,srce,resulta,dag);
-  }
-  t1=usecond();
-  mfo = flops*100/(t1-t0);
-  std::cout<<GridLogMessage << "Called ASM-OMP Dw"<< " mflop/s =   "<< mfo<<std::endl;
-
-  t0=usecond();
-  for(int i=0;i<1;i++){
-    Dw.DhopInternalL1bench(Dw.StencilEven,Dw.LebesgueEvenOdd,Dw.UmuOdd,srce,resulta,dag);
-  }
-  t1=usecond();
-  mfl1= flops*100/(t1-t0);
-  std::cout<<GridLogMessage << "Called ASM-L1 Dw"<< " mflop/s =   "<< mfl1<<std::endl;
-  os << latt4[0]*latt4[1]*latt4[2]*latt4[3]<< " "<<Ls<<" "<< latt4[0] <<" " <<latt4[2]<< " "
-     << mfc<<" "
-     << mfa<<" "
-     << mfo<<" "
-     << mfl1<<std::endl;
-  */
-
-#if 0
-  for(int i=0;i< PerformanceCounter::NumTypes(); i++ ){
-    Dw.DhopOE(srce,resulta,0);
-    PerformanceCounter Counter(i);
-    Counter.Start();
-    Dw.DhopOE(srce,resulta,0);
-    Counter.Stop();
-    Counter.Report();
-  }
-#endif
-  //resulta = (-0.5) * resulta;
-
-  diff = resulto-resulta;
-  std::cout<<norm2(diff)<<std::endl;
-  return 0;
-}
-
-
diff --git a/lib/Init.cc b/lib/Init.cc
index a2a8526a..d15e4bd1 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -123,6 +123,13 @@ void GridCmdOptionIntVector(std::string &str,std::vector<int> & vec)
   return;
 }
 
+void GridCmdOptionInt(std::string &str,int & val)
+{
+  std::stringstream ss(str);
+  ss>>val;
+  return;
+}
+
 
 void GridParseLayout(char **argv,int argc,
 		     std::vector<int> &latt,
@@ -153,14 +160,12 @@ void GridParseLayout(char **argv,int argc,
     assert(ompthreads.size()==1);
     GridThread::SetThreads(ompthreads[0]);
   }
-
   if( GridCmdOptionExists(argv,argv+argc,"--cores") ){
-    std::vector<int> cores(0);
+    int cores;
     arg= GridCmdOptionPayload(argv,argv+argc,"--cores");
-    GridCmdOptionIntVector(arg,cores);
-    GridThread::SetCores(cores[0]);
+    GridCmdOptionInt(arg,cores);
+    GridThread::SetCores(cores);
   }
-
 }
 
 std::string GridCmdVectorIntToString(const std::vector<int> & vec){
@@ -169,7 +174,7 @@ std::string GridCmdVectorIntToString(const std::vector<int> & vec){
   return oss.str();
 }
 /////////////////////////////////////////////////////////
-//
+// Reinit guard
 /////////////////////////////////////////////////////////
 static int Grid_is_initialised = 0;
 
@@ -178,27 +183,31 @@ void Grid_init(int *argc,char ***argv)
 {
   GridLogger::StopWatch.Start();
 
+  std::string arg;
+
+  ////////////////////////////////////
+  // Shared memory block size
+  ////////////////////////////////////
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--shm") ){
+    int MB;
+    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--shm");
+    GridCmdOptionInt(arg,MB);
+    CartesianCommunicator::MAX_MPI_SHM_BYTES = MB*1024*1024;
+  }
+
   CartesianCommunicator::Init(argc,argv);
 
-  // Parse command line args.
+  ////////////////////////////////////
+  // Logging
+  ////////////////////////////////////
 
-  std::string arg;
   std::vector<std::string> logstreams;
   std::string defaultLog("Error,Warning,Message,Performance");
-
   GridCmdOptionCSL(defaultLog,logstreams);
   GridLogConfigure(logstreams);
 
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--help") ){
-    std::cout<<GridLogMessage<<"--help : this message"<<std::endl;
-    std::cout<<GridLogMessage<<"--debug-signals : catch sigsegv and print a blame report"<<std::endl;
-    std::cout<<GridLogMessage<<"--debug-stdout  : print stdout from EVERY node"<<std::endl;    
-    std::cout<<GridLogMessage<<"--decomposition : report on default omp,mpi and simd decomposition"<<std::endl;    
-    std::cout<<GridLogMessage<<"--mpi n.n.n.n   : default MPI decomposition"<<std::endl;    
-    std::cout<<GridLogMessage<<"--threads n     : default number of OMP threads"<<std::endl;
-    std::cout<<GridLogMessage<<"--grid n.n.n.n  : default Grid size"<<std::endl;    
-    std::cout<<GridLogMessage<<"--log list      : comma separted list of streams from Error,Warning,Message,Performance,Iterative,Integrator,Debug,Colours"<<std::endl;
-    exit(EXIT_SUCCESS);
+  if( !GridCmdOptionExists(*argv,*argv+*argc,"--debug-stdout") ){
+    Grid_quiesce_nodes();
   }
 
   if( GridCmdOptionExists(*argv,*argv+*argc,"--log") ){
@@ -207,38 +216,39 @@ void Grid_init(int *argc,char ***argv)
     GridLogConfigure(logstreams);
   }
 
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-signals") ){
-    Grid_debug_handler_init();
-  }
-  if( !GridCmdOptionExists(*argv,*argv+*argc,"--debug-stdout") ){
-    Grid_quiesce_nodes();
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-opt") ){
-    QCD::WilsonKernelsStatic::HandOpt=1;
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--lebesgue") ){
-    LebesgueOrder::UseLebesgueOrder=1;
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--cacheblocking") ){
-    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--cacheblocking");
-    GridCmdOptionIntVector(arg,LebesgueOrder::Block);
-  }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--timestamp") ){
-    GridLogTimestamp(1);
+  ////////////////////////////////////
+  // Help message
+  ////////////////////////////////////
+
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--help") ){
+    std::cout<<GridLogMessage<<"  --help : this message"<<std::endl;
+    std::cout<<GridLogMessage<<std::endl;
+    std::cout<<GridLogMessage<<"Geometry:"<<std::endl;
+    std::cout<<GridLogMessage<<"  --mpi n.n.n.n   : default MPI decomposition"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --threads n     : default number of OMP threads"<<std::endl;
+    std::cout<<GridLogMessage<<"  --grid n.n.n.n  : default Grid size"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --shm  M        : allocate M megabytes of shared memory for comms"<<std::endl;    
+    std::cout<<GridLogMessage<<std::endl;
+    std::cout<<GridLogMessage<<"Verbose and debug:"<<std::endl;
+    std::cout<<GridLogMessage<<"  --log list      : comma separted list of streams from Error,Warning,Message,Performance,Iterative,Integrator,Debug,Colours"<<std::endl;
+    std::cout<<GridLogMessage<<"  --decomposition : report on default omp,mpi and simd decomposition"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --debug-signals : catch sigsegv and print a blame report"<<std::endl;
+    std::cout<<GridLogMessage<<"  --debug-stdout  : print stdout from EVERY node"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --timestamp     : tag with millisecond resolution stamps"<<std::endl;    
+    std::cout<<GridLogMessage<<std::endl;
+    std::cout<<GridLogMessage<<"Performance:"<<std::endl;
+    std::cout<<GridLogMessage<<"  --dslash-generic: Wilson kernel for generic Nc"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --dslash-unroll : Wilson kernel for Nc=3"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --dslash-asm    : Wilson kernel for AVX512"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --lebesgue      : Cache oblivious Lebesgue curve/Morton order/Z-graph stencil looping"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --cacheblocking n.m.o.p : Hypercuboidal cache blocking"<<std::endl;    
+    std::cout<<GridLogMessage<<std::endl;
+    exit(EXIT_SUCCESS);
   }
 
-  GridParseLayout(*argv,*argc,
-		  Grid_default_latt,
-		  Grid_default_mpi);
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--decomposition") ){
-    std::cout<<GridLogMessage<<"Grid Decomposition\n";
-    std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
-    std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
-    std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
-    std::cout<<GridLogMessage<<"\tvRealD         : "<<sizeof(vRealD)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealD::Nsimd()))<<std::endl;
-    std::cout<<GridLogMessage<<"\tvComplexF      : "<<sizeof(vComplexF)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexF::Nsimd()))<<std::endl;
-    std::cout<<GridLogMessage<<"\tvComplexD      : "<<sizeof(vComplexD)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexD::Nsimd()))<<std::endl;
-  }
+  ////////////////////////////////////
+  // Banner
+  ////////////////////////////////////
 
   std::string COL_RED    = GridLogColours.colour["RED"];
   std::string COL_PURPLE = GridLogColours.colour["PURPLE"];
@@ -247,7 +257,6 @@ void Grid_init(int *argc,char ***argv)
   std::string COL_BLUE   = GridLogColours.colour["BLUE"];
   std::string COL_YELLOW = GridLogColours.colour["YELLOW"];
   std::string COL_BACKGROUND = GridLogColours.colour["NORMAL"];
-
   
   std::cout <<std::endl;
   std::cout <<COL_RED  << "__|__|__|__|__"<<             "|__|__|_"<<COL_PURPLE<<"_|__|__|"<<                "__|__|__|__|__"<<std::endl; 
@@ -281,6 +290,53 @@ void Grid_init(int *argc,char ***argv)
   std::cout << COL_BACKGROUND <<std::endl;
   std::cout << std::endl;
 
+  ////////////////////////////////////
+  // Debug and performance options
+  ////////////////////////////////////
+
+
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-signals") ){
+    Grid_debug_handler_init();
+  }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-unroll") ){
+    QCD::WilsonKernelsStatic::Opt=QCD::WilsonKernelsStatic::OptHandUnroll;
+  }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-asm") ){
+    QCD::WilsonKernelsStatic::Opt=QCD::WilsonKernelsStatic::OptInlineAsm;
+  }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--dslash-generic") ){
+    QCD::WilsonKernelsStatic::Opt=QCD::WilsonKernelsStatic::OptGeneric;
+  }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--lebesgue") ){
+    LebesgueOrder::UseLebesgueOrder=1;
+  }
+
+
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--cacheblocking") ){
+    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--cacheblocking");
+    GridCmdOptionIntVector(arg,LebesgueOrder::Block);
+  }
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--timestamp") ){
+    GridLogTimestamp(1);
+  }
+
+  GridParseLayout(*argv,*argc,
+		  Grid_default_latt,
+		  Grid_default_mpi);
+
+  std::cout << GridLogMessage << "Requesting "<< CartesianCommunicator::MAX_MPI_SHM_BYTES <<" byte stencil comms buffers "<<std::endl;
+
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--decomposition") ){
+    std::cout<<GridLogMessage<<"Grid Decomposition\n";
+    std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
+    std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealD         : "<<sizeof(vRealD)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexF      : "<<sizeof(vComplexF)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexD      : "<<sizeof(vComplexD)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexD::Nsimd()))<<std::endl;
+  }
+
+
   Grid_is_initialised = 1;
 }
 
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 77b4f800..b003d867 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -32,6 +32,7 @@ namespace Grid {
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////
 void *              CartesianCommunicator::ShmCommBuf;
+uint64_t            CartesianCommunicator::MAX_MPI_SHM_BYTES   = 128*1024*1024; 
 
 /////////////////////////////////
 // Alloc, free shmem region
@@ -41,8 +42,12 @@ void *CartesianCommunicator::ShmBufferMalloc(size_t bytes){
   void *ptr = (void *)heap_top;
   heap_top  += bytes;
   heap_bytes+= bytes;
-  std::cout <<"Shm alloc "<<ptr<<std::endl;
-  assert(heap_bytes < MAX_MPI_SHM_BYTES);
+  if (heap_bytes >= MAX_MPI_SHM_BYTES) {
+    std::cout<< " ShmBufferMalloc exceeded shared heap size -- try increasing with --shm <MB> flag" <<std::endl;
+    std::cout<< " Parameter specified in units of MB (megabytes) " <<std::endl;
+    std::cout<< " Current value is " << (MAX_MPI_SHM_BYTES/(1024*1024)) <<std::endl;
+    assert(heap_bytes<MAX_MPI_SHM_BYTES);
+  }
   return ptr;
 }
 void CartesianCommunicator::ShmBufferFreeAll(void) { 
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index d4bb50d9..94ad1093 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -55,7 +55,7 @@ class CartesianCommunicator {
   // Give external control (command line override?) of this
 
   static const int      MAXLOG2RANKSPERNODE = 16;            
-  static const uint64_t MAX_MPI_SHM_BYTES   = 128*1024*1024; 
+  static uint64_t MAX_MPI_SHM_BYTES;
 
   // Communicator should know nothing of the physics grid, only processor grid.
   int              _Nprocessors;     // How many in all
diff --git a/lib/communicator/Communicator_mpi3_leader.cc b/lib/communicator/Communicator_mpi3_leader.cc
index 50e63eb7..db0638da 100644
--- a/lib/communicator/Communicator_mpi3_leader.cc
+++ b/lib/communicator/Communicator_mpi3_leader.cc
@@ -27,6 +27,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #include "Grid.h"
 #include <mpi.h>
+#include <semaphore.h>
 
 namespace Grid {
 
@@ -45,6 +46,7 @@ const int pool = 48;
 
 class SlaveState {
 public:
+  sem_t    sem;
   volatile int head;
   volatile int start;
   volatile int tail;
@@ -56,29 +58,32 @@ public:
   SlaveState *state;
   MPI_Comm squadron;
   uint64_t     base;
+  int universe_rank;
+  int vertical_rank;
   ////////////////////////////////////////////////////////////
   // Descriptor circular pointers
   ////////////////////////////////////////////////////////////
   Slave() {};
 
-  void Init(SlaveState * _state,MPI_Comm _squadron);
+  void Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank);
   
   void EventLoop (void) {
-    std::cerr<< " Entering even loop "<<std::endl;
-    while(1) {
+    std::cout<< " Entering event loop "<<std::endl;
+    while(1){
       Event();
+      MPI_Barrier(squadron);
     }
   }
 
-  void Event (void) ;
+  int Event (void) ;
 
-  uint64_t QueueCommand(int command,void *buf, int bytes, int hashtag, MPI_Comm comm,int rank) ;
+  uint64_t QueueCommand(int command,void *buf, int bytes, int hashtag, MPI_Comm comm,int u_rank) ;
 
-  void WaitAll() {
+  void QueueWaitAll() {
     QueueCommand(COMMAND_WAITALL,0,0,0,squadron,0);
-    std::cerr<< " Waiting on FIFO drain "<<std::endl;
+  }
+  void WaitAll() {
     while ( state->tail != state->head );
-    std::cerr<< " FIFO drained "<< state->tail <<std::endl;
   }
 };
 
@@ -119,26 +124,31 @@ public:
 				MPI_Comm &ShmComm,
 				void * &ShmCommBuf);
 
-  static void MapCommRankToWorldRank(int &hashtag, int & comm_world_peer,int tag, MPI_Comm comm,int rank);
+  static void MapCommRankToWorldRank(int &hashtag, int & comm_world_peer,int tag, MPI_Comm comm,int commrank);
 
   /////////////////////////////////////////////////////////
   // routines for master proc must handle any communicator
   /////////////////////////////////////////////////////////
 
   static uint64_t QueueSend(int slave,void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
-    std::cerr<< " Queueing send  "<< bytes<<std::endl;
+     //    std::cout<< " Queueing send  "<< bytes<< " slave "<< slave << " to comm "<<rank  <<std::endl;
     return Slaves[slave].QueueCommand(COMMAND_ISEND,buf,bytes,tag,comm,rank);
   };
 
   static uint64_t QueueRecv(int slave, void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
-    std::cerr<< " Queueing receive  "<< bytes<<std::endl;
+    //    std::cout<< " Queueing recv "<< bytes<< " slave "<< slave << " from comm "<<rank  <<std::endl;
     return Slaves[slave].QueueCommand(COMMAND_IRECV,buf,bytes,tag,comm,rank);
   };
 
   static void WaitAll() {
+    for(int s=1;s<VerticalSize;s++) {
+      Slaves[s].QueueWaitAll();
+    }
+    MPI_Barrier(VerticalComm);
     for(int s=1;s<VerticalSize;s++) {
       Slaves[s].WaitAll();
     }
+    MPI_Barrier(VerticalComm);
   };
 
   static void GetWork(int nwork, int me, int & mywork, int & myoff,int units){
@@ -163,6 +173,7 @@ public:
       GetWork(bytes,s,mywork,myoff,procs);
       QueueSend(s+1,&cbuf[myoff],mywork,tag,comm,rank);
     }
+    MPI_Barrier(VerticalComm);
   };
 
   static void QueueMultiplexedRecv(void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
@@ -173,6 +184,7 @@ public:
       GetWork(bytes,s,mywork,myoff,procs);
       QueueRecv(s+1,&cbuf[myoff],mywork,tag,comm,rank);
     }
+    MPI_Barrier(VerticalComm);
   };
 
 };
@@ -225,7 +237,7 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
   // Split into groups that can share memory (Verticals)
   /////////////////////////////////////////////////////////////////////
   //  MPI_Comm_split_type(communicator_universe, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&VerticalComm);
-  MPI_Comm_split(communicator_universe,(UniverseRank&0x1),UniverseRank,&VerticalComm);
+  MPI_Comm_split(communicator_universe,(UniverseRank/2),UniverseRank,&VerticalComm);
   MPI_Comm_rank(VerticalComm     ,&VerticalRank);
   MPI_Comm_size(VerticalComm     ,&VerticalSize);
   
@@ -262,14 +274,14 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
   ierr|= MPI_Win_lock_all (MPI_MODE_NOCHECK, VerticalWindow);
   assert(ierr==0);
   
-  std::cerr<<"SHM "<<ShmCommBuf<<std::endl;
+  std::cout<<"SHM "<<ShmCommBuf<<std::endl;
 
   VerticalShmBufs.resize(VerticalSize);
   for(int r=0;r<VerticalSize;r++){
     MPI_Aint sz;
     int dsp_unit;
     MPI_Win_shared_query (VerticalWindow, r, &sz, &dsp_unit, &VerticalShmBufs[r]);
-    std::cerr<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
+    std::cout<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
   }
 
   //////////////////////////////////////////////////////////////////////
@@ -286,14 +298,16 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
   ///////////////////////////////////////////////////////////
   if ( VerticalRank != 0 ) {
     Slave indentured;
-    indentured.Init( (SlaveState *) VerticalShmBufs[VerticalRank], VerticalComm);
+    indentured.Init( (SlaveState *) VerticalShmBufs[VerticalRank], VerticalComm, UniverseRank,VerticalRank);
+    MPI_Barrier(VerticalComm);
     indentured.EventLoop();
     assert(0);
   } else {
     Slaves.resize(VerticalSize);
     for(int i=1;i<VerticalSize;i++){
-      Slaves[i].Init((SlaveState *)VerticalShmBufs[i],VerticalComm);
+      Slaves[i].Init((SlaveState *)VerticalShmBufs[i],VerticalComm, UniverseRanks[HorizontalRank][i],i);
     }
+    MPI_Barrier(VerticalComm);
   }
   
   ///////////////////////////////////////////////////////////
@@ -337,8 +351,10 @@ void MPIoffloadEngine::MapCommRankToWorldRank(int &hashtag, int & comm_world_pee
 
   if ( comm == HorizontalComm ) {
     comm_world_peer = rank;
+    //    std::cout << " MapCommRankToWorldRank  horiz " <<rank<<"->"<<comm_world_peer<<std::endl;
   } else if ( comm == communicator_cached ) {
     comm_world_peer = UserCommunicatorToWorldRanks[rank];
+    //    std::cout << " MapCommRankToWorldRank  cached " <<rank<<"->"<<comm_world_peer<<std::endl;
   } else { 
     
     int size;
@@ -360,6 +376,7 @@ void MPIoffloadEngine::MapCommRankToWorldRank(int &hashtag, int & comm_world_pee
     MPI_Group_translate_ranks(CachedGroup,size,&cached_ranks[0],WorldGroup, &UserCommunicatorToWorldRanks[0]); 
     
     comm_world_peer = UserCommunicatorToWorldRanks[rank];
+    //    std::cout << " MapCommRankToWorldRank  cache miss " <<rank<<"->"<<comm_world_peer<<std::endl;
     
     assert(comm_world_peer != MPI_UNDEFINED);
   }
@@ -370,55 +387,30 @@ void MPIoffloadEngine::MapCommRankToWorldRank(int &hashtag, int & comm_world_pee
   int comm_hash = ((icomm>>0 )&0xFFFF)^((icomm>>16)&0xFFFF)
                 ^ ((icomm>>32)&0xFFFF)^((icomm>>48)&0xFFFF);
   
-  hashtag = (comm_hash<<15) | tag;      
+  //  hashtag = (comm_hash<<15) | tag;      
+  hashtag = tag;      
 
 };
 
-void Slave::Init(SlaveState * _state,MPI_Comm _squadron)
+void Slave::Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank)
 {
   squadron=_squadron;
+  universe_rank=_universe_rank;
+  vertical_rank=_vertical_rank;
   state   =_state;
+  std::cout << "state "<<_state<<" comm "<<_squadron<<" universe_rank"<<universe_rank <<std::endl;
   state->head = state->tail = state->start = 0;
-  MPI_Barrier(squadron);
   base = (uint64_t)MPIoffloadEngine::VerticalShmBufs[0];
   int rank; MPI_Comm_rank(_squadron,&rank);
 }
 #define PERI_PLUS(A) ( (A+1)%pool )
-void Slave::Event (void) {
+int Slave::Event (void) {
 
   static int tail_last;
   static int head_last;
   static int start_last;
   int ierr;
 
-  if (   (state->tail != tail_last)
-       ||(state->head != head_last)
-       ||(state->start != start_last)
-       ) { 
-    std::cerr<< " Event loop "<< state->tail << " "<< state->start<< " "<< state->head <<std::endl;
-  }
-
-  ////////////////////////////////////////////////////
-  // Try to advance the tail pointers
-  ////////////////////////////////////////////////////
-  /*
-  int t=state->tail;
-  if ( t != state->start ) {
-    int flag=0;
-    
-    std::cerr<< " Testing tail "<< t<<" "<< (void *)&state->Descrs[t].request
-	     << " "<<state->Descrs[t].request<<std::endl;
-    //    ierr=MPI_Test((MPI_Request *)&state->Descrs[t].request,&flag,MPI_STATUS_IGNORE);
-    //    ierr=MPI_Test((MPI_Request *)&state->Descrs[t].request,&flag,MPI_STATUS_IGNORE);
-    assert(ierr==0);
-    if ( flag ) {
-      state->tail = PERI_PLUS(t);
-      std::cerr<< " Tail advanced from "<< t<<std::endl;
-      return;
-    }
-  }
-  */
-
   ////////////////////////////////////////////////////
   // Try to advance the start pointers
   ////////////////////////////////////////////////////
@@ -426,11 +418,11 @@ void Slave::Event (void) {
   if ( s != state->head ) {
     switch ( state->Descrs[s].command ) {
     case COMMAND_ISEND:
-      std::cerr<< " Send "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
-	       << " to " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
-	       << " Comm " << MPIoffloadEngine::communicator_universe<< std::endl;
-
-      std::cerr<< " Request was "<<state->Descrs[s].request<<std::endl;
+      /*
+            std::cout<< " Send "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
+      	       << " to " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
+       << " Comm " << MPIoffloadEngine::communicator_universe<< " me " <<universe_rank<< std::endl;
+      */
       ierr = MPI_Isend((void *)(state->Descrs[s].buf+base), 
 		       state->Descrs[s].bytes, 
 		       MPI_CHAR,
@@ -438,18 +430,17 @@ void Slave::Event (void) {
 		       state->Descrs[s].tag,
 		       MPIoffloadEngine::communicator_universe,
 		       (MPI_Request *)&state->Descrs[s].request);
-      std::cerr<< " Request is "<<state->Descrs[s].request<<std::endl;
-      std::cerr<< " Request0 is "<<state->Descrs[0].request<<std::endl;
       assert(ierr==0);
       state->start = PERI_PLUS(s);
+      return 1;
       break;
 
     case COMMAND_IRECV:
-      std::cerr<< " Recv "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
-	       << " to " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
-	       << " Comm " << MPIoffloadEngine::communicator_universe<< std::endl;
-
-      std::cerr<< " Request was "<<state->Descrs[s].request<<std::endl;
+      /*
+      std::cout<< " Recv "<<s << " ptr "<< state<<" "<< state->Descrs[s].buf<< "["<<state->Descrs[s].bytes<<"]"
+	       << " from " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag
+	       << " Comm " << MPIoffloadEngine::communicator_universe<< " me "<< universe_rank<< std::endl;
+      */
       ierr=MPI_Irecv((void *)(state->Descrs[s].buf+base), 
 		     state->Descrs[s].bytes, 
 		     MPI_CHAR,
@@ -457,30 +448,32 @@ void Slave::Event (void) {
 		     state->Descrs[s].tag,
 		     MPIoffloadEngine::communicator_universe,
 		     (MPI_Request *)&state->Descrs[s].request);
-      std::cerr<< " Request is "<<state->Descrs[s].request<<std::endl;
-      std::cerr<< " Request0 is "<<state->Descrs[0].request<<std::endl;
+
+      //      std::cout<< " Request is "<<state->Descrs[s].request<<std::endl;
+      //      std::cout<< " Request0 is "<<state->Descrs[0].request<<std::endl;
       assert(ierr==0);
       state->start = PERI_PLUS(s);
+      return 1;
       break;
 
     case COMMAND_WAITALL:
-      std::cerr<< " Wait all "<<std::endl;
+
       for(int t=state->tail;t!=s; t=PERI_PLUS(t) ){
-	std::cerr<< " Wait ["<<t<<"] "<<state->Descrs[t].request <<std::endl;
-	std::cerr<< " Request0 is "<<state->Descrs[0].request<<std::endl;
 	MPI_Wait((MPI_Request *)&state->Descrs[t].request,MPI_STATUS_IGNORE);
       };
       s=PERI_PLUS(s);
       state->start = s;
       state->tail  = s;
+      MPI_Barrier(squadron);
+      return 1;
       break;
 
     default:
       assert(0);
       break;
     }
-    return;
   }
+  return 0;
 }
   //////////////////////////////////////////////////////////////////////////////
   // External interaction with the queue
@@ -500,17 +493,29 @@ uint64_t Slave::QueueCommand(int command,void *buf, int bytes, int tag, MPI_Comm
   MPI_Comm    communicator;
   MPI_Request request;
   
-  MPIoffloadEngine::MapCommRankToWorldRank(hashtag,commrank,tag,comm,worldrank);
+  MPIoffloadEngine::MapCommRankToWorldRank(hashtag,worldrank,tag,comm,commrank);
 
-  int VerticalRank = MPIoffloadEngine::VerticalRank;
   uint64_t relative= (uint64_t)buf - base;
   state->Descrs[head].buf    = relative;
   state->Descrs[head].bytes  = bytes;
-  state->Descrs[head].rank   = MPIoffloadEngine::UniverseRanks[worldrank][VerticalRank];
+  state->Descrs[head].rank   = MPIoffloadEngine::UniverseRanks[worldrank][vertical_rank];
   state->Descrs[head].tag    = hashtag;
   state->Descrs[head].command= command;
-  std::cerr<< " QueueCommand "<<buf<<"["<<bytes<<"]" << std::endl;
 
+  /*  
+  if ( command == COMMAND_ISEND ) { 
+  std::cout << "QueueSend from "<< universe_rank <<" to commrank " << commrank 
+            << " to worldrank " << worldrank <<std::endl;
+  std::cout << " via VerticalRank "<< vertical_rank <<" to universerank " << MPIoffloadEngine::UniverseRanks[worldrank][vertical_rank]<<std::endl;
+  std::cout << " QueueCommand "<<buf<<"["<<bytes<<"]" << std::endl;
+  } 
+  if ( command == COMMAND_IRECV ) { 
+  std::cout << "QueueRecv on "<< universe_rank <<" from commrank " << commrank 
+            << " from worldrank " << worldrank <<std::endl;
+  std::cout << " via VerticalRank "<< vertical_rank <<" from universerank " << MPIoffloadEngine::UniverseRanks[worldrank][vertical_rank]<<std::endl;
+  std::cout << " QueueSend "<<buf<<"["<<bytes<<"]" << std::endl;
+  } 
+  */
   // Block until FIFO has space
   while( state->tail==next );
 
@@ -671,6 +676,8 @@ void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_
   // assert xmit and recv lie in shared memory region
   assert( (xmit_i >= shm) && (xmit_i+bytes <= shm+MAX_MPI_SHM_BYTES) );
   assert( (recv_i >= shm) && (recv_i+bytes <= shm+MAX_MPI_SHM_BYTES) );
+  assert(from!=_processor);
+  assert(dest!=_processor);
   MPIoffloadEngine::QueueMultiplexedSend(xmit,bytes,_processor,communicator,dest);
   MPIoffloadEngine::QueueMultiplexedRecv(recv,bytes,from,communicator,from);
 }
diff --git a/lib/qcd/action/fermion/WilsonKernels.cc b/lib/qcd/action/fermion/WilsonKernels.cc
index 52ee8704..43776c86 100644
--- a/lib/qcd/action/fermion/WilsonKernels.cc
+++ b/lib/qcd/action/fermion/WilsonKernels.cc
@@ -32,8 +32,7 @@ directory
 namespace Grid {
 namespace QCD {
 
-int WilsonKernelsStatic::HandOpt;
-int WilsonKernelsStatic::AsmOpt;
+int WilsonKernelsStatic::Opt;
 
 template <class Impl>
 WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
diff --git a/lib/qcd/action/fermion/WilsonKernels.h b/lib/qcd/action/fermion/WilsonKernels.h
index 669ee4be..919f7540 100644
--- a/lib/qcd/action/fermion/WilsonKernels.h
+++ b/lib/qcd/action/fermion/WilsonKernels.h
@@ -40,9 +40,9 @@ namespace QCD {
   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 class WilsonKernelsStatic { 
  public:
+  enum { OptGeneric, OptHandUnroll, OptInlineAsm };
   // S-direction is INNERMOST and takes no part in the parity.
-  static int AsmOpt;  // these are a temporary hack
-  static int HandOpt; // these are a temporary hack
+  static int Opt;  // these are a temporary hack
 };
  
 template<class Impl> class WilsonKernels : public FermionOperator<Impl> , public WilsonKernelsStatic { 
@@ -56,24 +56,40 @@ public:
   template <bool EnableBool = true>
   typename std::enable_if<Impl::Dimension == 3 && Nc == 3 &&EnableBool, void>::type
   DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
+		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) 
+  {
+    switch(Opt) {
 #ifdef AVX512
-    if (AsmOpt) {
-      WilsonKernels<Impl>::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
-    } else {
-#else
-    {
-#endif
+    case OptInlineAsm:
       for (int site = 0; site < Ns; site++) {
 	for (int s = 0; s < Ls; s++) {
-	  if (HandOpt)
-	    WilsonKernels<Impl>::DiracOptHandDhopSite(st,lo,U,buf,sF,sU,in,out);
-	  else
-	    WilsonKernels<Impl>::DiracOptGenericDhopSite(st,lo,U,buf,sF,sU,in,out);
+	  WilsonKernels<Impl>::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
 	  sF++;
 	}
 	sU++;
       }
+      break;
+#endif
+    case OptHandUnroll:
+      for (int site = 0; site < Ns; site++) {
+	for (int s = 0; s < Ls; s++) {
+	  WilsonKernels<Impl>::DiracOptHandDhopSite(st,lo,U,buf,sF,sU,in,out);
+	  sF++;
+	}
+	sU++;
+      }
+      break;
+    case OptGeneric:
+      for (int site = 0; site < Ns; site++) {
+	for (int s = 0; s < Ls; s++) {
+	  WilsonKernels<Impl>::DiracOptGenericDhopSite(st,lo,U,buf,sF,sU,in,out);
+	  sF++;
+	}
+	sU++;
+      }
+      break;
+    default:
+      assert(0);
     }
   }
      
@@ -81,7 +97,7 @@ public:
   typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type
   DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
 		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
-     
+    // no kernel choice  
     for (int site = 0; site < Ns; site++) {
       for (int s = 0; s < Ls; s++) {
 	WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in, out);
@@ -95,23 +111,39 @@ public:
   typename std::enable_if<Impl::Dimension == 3 && Nc == 3 && EnableBool,void>::type
   DiracOptDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
 		      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) {
+
+    switch(Opt) {
 #ifdef AVX512
-    if (AsmOpt) {
-      WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
-    } else {
-#else
-    {
-#endif
+    case OptInlineAsm:
       for (int site = 0; site < Ns; site++) {
 	for (int s = 0; s < Ls; s++) {
-	  if (HandOpt)
-	    WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
-	  else
-	    WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	  WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
 	  sF++;
 	}
 	sU++;
       }
+      break;
+#endif
+    case OptHandUnroll:
+      for (int site = 0; site < Ns; site++) {
+	for (int s = 0; s < Ls; s++) {
+	  WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	  sF++;
+	}
+	sU++;
+      }
+      break;
+    case OptGeneric:
+      for (int site = 0; site < Ns; site++) {
+	for (int s = 0; s < Ls; s++) {
+	  WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	  sF++;
+	}
+	sU++;
+      }
+      break;
+    default:
+      assert(0);
     }
   }
 
diff --git a/lib/stencil/Lebesgue.cc b/lib/stencil/Lebesgue.cc
index c34b5c96..c30c14c7 100644
--- a/lib/stencil/Lebesgue.cc
+++ b/lib/stencil/Lebesgue.cc
@@ -32,7 +32,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 
 int LebesgueOrder::UseLebesgueOrder;
-std::vector<int> LebesgueOrder::Block({2,2,2,2});
+std::vector<int> LebesgueOrder::Block({8,2,2,2});
 
 LebesgueOrder::IndexInteger LebesgueOrder::alignup(IndexInteger n){
   n--;           // 1000 0011 --> 1000 0010

From 32375aca657c87674c321ce5e8fc2a82b48dcec2 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 2 Nov 2016 09:27:20 +0000
Subject: [PATCH 116/149] Semaphore sleep/wake up on remote processes.

---
 lib/communicator/Communicator_mpi3_leader.cc | 84 +++++++++++++++-----
 1 file changed, 65 insertions(+), 19 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3_leader.cc b/lib/communicator/Communicator_mpi3_leader.cc
index db0638da..f160c681 100644
--- a/lib/communicator/Communicator_mpi3_leader.cc
+++ b/lib/communicator/Communicator_mpi3_leader.cc
@@ -27,7 +27,24 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #include "Grid.h"
 #include <mpi.h>
+
+/// bloody mac os doesn't implement unnamed semaphores since it is "optional" posix.
+/// darwin dispatch semaphores don't seem to be multiprocess.
+//#ifdef __APPLE__
 #include <semaphore.h>
+typedef sem_t *Grid_semaphore;
+#define SEM_INIT(S)      S = sem_open(sem_name,0,00777,0);
+#define SEM_INIT_EXCL(S) sem_unlink(sem_name); S = sem_open(sem_name,O_CREAT|O_EXCL,00777,0);
+#define SEM_POST(S) sem_post(S);
+#define SEM_WAIT(S) sem_wait(S);
+//#else
+//#include <semaphore.h>
+//typedef sem_t Grid_semaphore;
+//#define SEM_INIT(S)      
+//#define SEM_INIT_EXCL(S) sem_init(&S,0);
+//#define SEM_POST(S) sem_post(&S);
+//#define SEM_WAIT(S) sem_wait(&S);
+//#endif
 
 namespace Grid {
 
@@ -46,7 +63,8 @@ const int pool = 48;
 
 class SlaveState {
 public:
-  sem_t    sem;
+  Grid_semaphore  sem_head;
+  Grid_semaphore  sem_tail;
   volatile int head;
   volatile int start;
   volatile int tail;
@@ -60,18 +78,43 @@ public:
   uint64_t     base;
   int universe_rank;
   int vertical_rank;
+  char sem_name [NAME_MAX];
   ////////////////////////////////////////////////////////////
   // Descriptor circular pointers
   ////////////////////////////////////////////////////////////
   Slave() {};
 
   void Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank);
-  
+
+  void SemInit(void) {
+    sprintf(sem_name,"/Grid_mpi3_head_%d",universe_rank);
+    SEM_INIT(state->sem_head);
+    sprintf(sem_name,"/Grid_mpi3_tail_%d",universe_rank);
+    SEM_INIT(state->sem_tail);
+  }  
+  void SemInitExcl(void) {
+    sprintf(sem_name,"/Grid_mpi3_head_%d",universe_rank);
+    SEM_INIT_EXCL(state->sem_head);
+    sprintf(sem_name,"/Grid_mpi3_tail_%d",universe_rank);
+    SEM_INIT_EXCL(state->sem_tail);
+  }  
+  void WakeUpDMA(void) { 
+    SEM_POST(state->sem_head);
+  };
+  void WakeUpCompute(void) { 
+    SEM_POST(state->sem_tail);
+  };
+  void WaitForCommand(void) { 
+    SEM_WAIT(state->sem_head);
+  };
+  void WaitForComplete(void) { 
+    SEM_WAIT(state->sem_tail);
+  };
   void EventLoop (void) {
     std::cout<< " Entering event loop "<<std::endl;
     while(1){
+      WaitForCommand();
       Event();
-      MPI_Barrier(squadron);
     }
   }
 
@@ -79,11 +122,11 @@ public:
 
   uint64_t QueueCommand(int command,void *buf, int bytes, int hashtag, MPI_Comm comm,int u_rank) ;
 
-  void QueueWaitAll() {
-    QueueCommand(COMMAND_WAITALL,0,0,0,squadron,0);
-  }
   void WaitAll() {
-    while ( state->tail != state->head );
+    QueueCommand(COMMAND_WAITALL,0,0,0,squadron,0);
+    WakeUpDMA();
+    WaitForComplete();
+    assert ( state->tail == state->head );
   }
 };
 
@@ -130,25 +173,22 @@ public:
   // routines for master proc must handle any communicator
   /////////////////////////////////////////////////////////
 
-  static uint64_t QueueSend(int slave,void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
+  static void QueueSend(int slave,void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
      //    std::cout<< " Queueing send  "<< bytes<< " slave "<< slave << " to comm "<<rank  <<std::endl;
-    return Slaves[slave].QueueCommand(COMMAND_ISEND,buf,bytes,tag,comm,rank);
+    Slaves[slave].QueueCommand(COMMAND_ISEND,buf,bytes,tag,comm,rank);
+    Slaves[slave].WakeUpDMA();
   };
 
-  static uint64_t QueueRecv(int slave, void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
+  static void QueueRecv(int slave, void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
     //    std::cout<< " Queueing recv "<< bytes<< " slave "<< slave << " from comm "<<rank  <<std::endl;
-    return Slaves[slave].QueueCommand(COMMAND_IRECV,buf,bytes,tag,comm,rank);
+    Slaves[slave].QueueCommand(COMMAND_IRECV,buf,bytes,tag,comm,rank);
+    Slaves[slave].WakeUpDMA();
   };
 
   static void WaitAll() {
-    for(int s=1;s<VerticalSize;s++) {
-      Slaves[s].QueueWaitAll();
-    }
-    MPI_Barrier(VerticalComm);
     for(int s=1;s<VerticalSize;s++) {
       Slaves[s].WaitAll();
     }
-    MPI_Barrier(VerticalComm);
   };
 
   static void GetWork(int nwork, int me, int & mywork, int & myoff,int units){
@@ -173,7 +213,6 @@ public:
       GetWork(bytes,s,mywork,myoff,procs);
       QueueSend(s+1,&cbuf[myoff],mywork,tag,comm,rank);
     }
-    MPI_Barrier(VerticalComm);
   };
 
   static void QueueMultiplexedRecv(void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
@@ -184,7 +223,6 @@ public:
       GetWork(bytes,s,mywork,myoff,procs);
       QueueRecv(s+1,&cbuf[myoff],mywork,tag,comm,rank);
     }
-    MPI_Barrier(VerticalComm);
   };
 
 };
@@ -299,6 +337,8 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
   if ( VerticalRank != 0 ) {
     Slave indentured;
     indentured.Init( (SlaveState *) VerticalShmBufs[VerticalRank], VerticalComm, UniverseRank,VerticalRank);
+    indentured.SemInitExcl();// init semaphore in shared memory
+    MPI_Barrier(VerticalComm);
     MPI_Barrier(VerticalComm);
     indentured.EventLoop();
     assert(0);
@@ -308,6 +348,10 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
       Slaves[i].Init((SlaveState *)VerticalShmBufs[i],VerticalComm, UniverseRanks[HorizontalRank][i],i);
     }
     MPI_Barrier(VerticalComm);
+    for(int i=1;i<VerticalSize;i++){
+      Slaves[i].SemInit();// init semaphore in shared memory
+    }
+    MPI_Barrier(VerticalComm);
   }
   
   ///////////////////////////////////////////////////////////
@@ -464,7 +508,9 @@ int Slave::Event (void) {
       s=PERI_PLUS(s);
       state->start = s;
       state->tail  = s;
-      MPI_Barrier(squadron);
+
+      WakeUpCompute();
+
       return 1;
       break;
 

From ae8561892ef8e6763454a552d9aba45fa33074c6 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Wed, 2 Nov 2016 10:21:06 +0000
Subject: [PATCH 117/149] Eliminating useless defines

---
 lib/qcd/action/fermion/WilsonKernelsAsm.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsAsm.cc b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
index 83124d1a..d7a9edd3 100644
--- a/lib/qcd/action/fermion/WilsonKernelsAsm.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsAsm.cc
@@ -103,8 +103,8 @@ WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder
 #undef FX 
 #define FX(A) DWFASM_ ## A
 #define MAYBEPERM(A,B) 
-#define VMOVIDUP(A,B,C)                                  VBCASTIDUPf(A,B,C)
-#define VMOVRDUP(A,B,C)                                  VBCASTRDUPf(A,B,C)
+//#define VMOVIDUP(A,B,C)                                  VBCASTIDUPf(A,B,C)
+//#define VMOVRDUP(A,B,C)                                  VBCASTRDUPf(A,B,C)
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
 				    
 #undef KERNEL_DAG
@@ -159,8 +159,8 @@ WilsonKernels<WilsonImplD>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder
 #undef FX 
 #define FX(A) DWFASM_ ## A
 #define MAYBEPERM(A,B) 
-#define VMOVIDUP(A,B,C)                                  VBCASTIDUPd(A,B,C)
-#define VMOVRDUP(A,B,C)                                  VBCASTRDUPd(A,B,C)
+//#define VMOVIDUP(A,B,C)                                  VBCASTIDUPd(A,B,C)
+//#define VMOVRDUP(A,B,C)                                  VBCASTRDUPd(A,B,C)
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
 				    
 #undef KERNEL_DAG

From 757a928f9a70c0f2448cc6844f1112bab9cdf7d9 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 2 Nov 2016 12:37:46 +0000
Subject: [PATCH 118/149] Improvement to use own SHM_OPEN call to avoid openmpi
 bug.

---
 benchmarks/Benchmark_comms.cc                |   4 +-
 lib/communicator/Communicator_mpi3_leader.cc | 159 ++++++++++++++-----
 2 files changed, 124 insertions(+), 39 deletions(-)

diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index eab9b9b9..de73bc81 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -48,7 +48,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "= Benchmarking concurrent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-  int maxlat=32;
+  int maxlat=16;
   for(int lat=4;lat<=maxlat;lat+=2){
     for(int Ls=1;Ls<=16;Ls*=2){
 
@@ -193,7 +193,7 @@ int main (int argc, char ** argv)
     }
   }  
 
-
+  Nloop=100;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
diff --git a/lib/communicator/Communicator_mpi3_leader.cc b/lib/communicator/Communicator_mpi3_leader.cc
index f160c681..b6995a44 100644
--- a/lib/communicator/Communicator_mpi3_leader.cc
+++ b/lib/communicator/Communicator_mpi3_leader.cc
@@ -28,23 +28,26 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include "Grid.h"
 #include <mpi.h>
 
-/// bloody mac os doesn't implement unnamed semaphores since it is "optional" posix.
-/// darwin dispatch semaphores don't seem to be multiprocess.
-//#ifdef __APPLE__
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/// Workarounds:
+/// i) bloody mac os doesn't implement unnamed semaphores since it is "optional" posix.
+///    darwin dispatch semaphores don't seem to be multiprocess.
+///
+/// ii) openmpi under --mca shmem posix works with two squadrons per node; 
+///     openmpi under default mca settings (I think --mca shmem mmap) on MacOS makes two squadrons map the SAME
+///     memory as each other, despite their living on different communicators. This appears to be a bug in OpenMPI.
+///
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 #include <semaphore.h>
 typedef sem_t *Grid_semaphore;
-#define SEM_INIT(S)      S = sem_open(sem_name,0,00777,0);
-#define SEM_INIT_EXCL(S) sem_unlink(sem_name); S = sem_open(sem_name,O_CREAT|O_EXCL,00777,0);
-#define SEM_POST(S) sem_post(S);
-#define SEM_WAIT(S) sem_wait(S);
-//#else
-//#include <semaphore.h>
-//typedef sem_t Grid_semaphore;
-//#define SEM_INIT(S)      
-//#define SEM_INIT_EXCL(S) sem_init(&S,0);
-//#define SEM_POST(S) sem_post(&S);
-//#define SEM_WAIT(S) sem_wait(&S);
-//#endif
+
+#define SEM_INIT(S)      S = sem_open(sem_name,0,0600,0); assert ( S != SEM_FAILED );
+#define SEM_INIT_EXCL(S) sem_unlink(sem_name); S = sem_open(sem_name,O_CREAT|O_EXCL,0600,0); assert ( S != SEM_FAILED );
+#define SEM_POST(S) assert ( sem_post(S) == 0 ); 
+#define SEM_WAIT(S) assert ( sem_wait(S) == 0 );
+
+#include <sys/mman.h>
+
 
 namespace Grid {
 
@@ -63,8 +66,6 @@ const int pool = 48;
 
 class SlaveState {
 public:
-  Grid_semaphore  sem_head;
-  Grid_semaphore  sem_tail;
   volatile int head;
   volatile int start;
   volatile int tail;
@@ -73,6 +74,8 @@ public:
 
 class Slave {
 public:
+  Grid_semaphore  sem_head;
+  Grid_semaphore  sem_tail;
   SlaveState *state;
   MPI_Comm squadron;
   uint64_t     base;
@@ -87,33 +90,38 @@ public:
   void Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank);
 
   void SemInit(void) {
-    sprintf(sem_name,"/Grid_mpi3_head_%d",universe_rank);
-    SEM_INIT(state->sem_head);
-    sprintf(sem_name,"/Grid_mpi3_tail_%d",universe_rank);
-    SEM_INIT(state->sem_tail);
+    sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank);
+    printf("SEM_NAME: %s \n",sem_name);
+    SEM_INIT(sem_head);
+    sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank);
+    printf("SEM_NAME: %s \n",sem_name);
+    SEM_INIT(sem_tail);
   }  
   void SemInitExcl(void) {
-    sprintf(sem_name,"/Grid_mpi3_head_%d",universe_rank);
-    SEM_INIT_EXCL(state->sem_head);
-    sprintf(sem_name,"/Grid_mpi3_tail_%d",universe_rank);
-    SEM_INIT_EXCL(state->sem_tail);
+    sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank);
+    printf("SEM_INIT_EXCL: %s \n",sem_name);
+    SEM_INIT_EXCL(sem_head);
+    sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank);
+    printf("SEM_INIT_EXCL: %s \n",sem_name);
+    SEM_INIT_EXCL(sem_tail);
   }  
   void WakeUpDMA(void) { 
-    SEM_POST(state->sem_head);
+    SEM_POST(sem_head);
   };
   void WakeUpCompute(void) { 
-    SEM_POST(state->sem_tail);
+    SEM_POST(sem_tail);
   };
   void WaitForCommand(void) { 
-    SEM_WAIT(state->sem_head);
+    SEM_WAIT(sem_head);
   };
   void WaitForComplete(void) { 
-    SEM_WAIT(state->sem_tail);
+    SEM_WAIT(sem_tail);
   };
   void EventLoop (void) {
     std::cout<< " Entering event loop "<<std::endl;
     while(1){
       WaitForCommand();
+      //      std::cout << "Getting command "<<std::endl;
       Event();
     }
   }
@@ -123,9 +131,13 @@ public:
   uint64_t QueueCommand(int command,void *buf, int bytes, int hashtag, MPI_Comm comm,int u_rank) ;
 
   void WaitAll() {
+    //    std::cout << "Queueing WAIT command  "<<std::endl;
     QueueCommand(COMMAND_WAITALL,0,0,0,squadron,0);
+    //    std::cout << "Waking up DMA "<<std::endl;
     WakeUpDMA();
+    //    std::cout << "Waiting from semaphore "<<std::endl;
     WaitForComplete();
+    //    std::cout << "Checking FIFO is empty "<<std::endl;
     assert ( state->tail == state->head );
   }
 };
@@ -176,19 +188,25 @@ public:
   static void QueueSend(int slave,void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
      //    std::cout<< " Queueing send  "<< bytes<< " slave "<< slave << " to comm "<<rank  <<std::endl;
     Slaves[slave].QueueCommand(COMMAND_ISEND,buf,bytes,tag,comm,rank);
+    //    std::cout << "Queued send command to rank "<< rank<< " via "<<slave <<std::endl;
     Slaves[slave].WakeUpDMA();
+    //    std::cout << "Waking up DMA "<< slave<<std::endl;
   };
 
   static void QueueRecv(int slave, void *buf, int bytes, int tag, MPI_Comm comm,int rank) {
     //    std::cout<< " Queueing recv "<< bytes<< " slave "<< slave << " from comm "<<rank  <<std::endl;
     Slaves[slave].QueueCommand(COMMAND_IRECV,buf,bytes,tag,comm,rank);
+    //    std::cout << "Queued recv command from rank "<< rank<< " via "<<slave <<std::endl;
     Slaves[slave].WakeUpDMA();
+    //    std::cout << "Waking up DMA "<< slave<<std::endl;
   };
 
   static void WaitAll() {
     for(int s=1;s<VerticalSize;s++) {
+      //      std::cout << "Waiting for slave "<< s<<std::endl;
       Slaves[s].WaitAll();
     }
+    //    std::cout << " Wait all Complete "<<std::endl;
   };
 
   static void GetWork(int nwork, int me, int & mywork, int & myoff,int units){
@@ -247,10 +265,9 @@ int      MPIoffloadEngine::HorizontalRank;
 int      MPIoffloadEngine::HorizontalSize;
 
 MPI_Comm MPIoffloadEngine::VerticalComm;
-MPI_Win  MPIoffloadEngine::VerticalWindow; 
 int      MPIoffloadEngine::VerticalSize;
 int      MPIoffloadEngine::VerticalRank;
-
+MPI_Win  MPIoffloadEngine::VerticalWindow; 
 std::vector<void *>            MPIoffloadEngine::VerticalShmBufs;
 std::vector<std::vector<int> > MPIoffloadEngine::UniverseRanks;
 std::vector<int>               MPIoffloadEngine::UserCommunicatorToWorldRanks; 
@@ -274,8 +291,12 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
   /////////////////////////////////////////////////////////////////////
   // Split into groups that can share memory (Verticals)
   /////////////////////////////////////////////////////////////////////
-  //  MPI_Comm_split_type(communicator_universe, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&VerticalComm);
-  MPI_Comm_split(communicator_universe,(UniverseRank/2),UniverseRank,&VerticalComm);
+#define MPI_SHARED_MEM_DEBUG
+#ifdef  MPI_SHARED_MEM_DEBUG
+  MPI_Comm_split(communicator_universe,(UniverseRank/4),UniverseRank,&VerticalComm);
+#else 
+  MPI_Comm_split_type(communicator_universe, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&VerticalComm);
+#endif
   MPI_Comm_rank(VerticalComm     ,&VerticalRank);
   MPI_Comm_size(VerticalComm     ,&VerticalSize);
   
@@ -308,19 +329,83 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
   //////////////////////////////////////////////////////////////////////////////////////////////////////////
   // allocate the shared window for our group, pass back Shm info to CartesianCommunicator
   //////////////////////////////////////////////////////////////////////////////////////////////////////////
+  VerticalShmBufs.resize(VerticalSize);
+
+#undef MPI_SHARED_MEM
+#ifdef MPI_SHARED_MEM
   ierr = MPI_Win_allocate_shared(CartesianCommunicator::MAX_MPI_SHM_BYTES,1,MPI_INFO_NULL,VerticalComm,&ShmCommBuf,&VerticalWindow);
   ierr|= MPI_Win_lock_all (MPI_MODE_NOCHECK, VerticalWindow);
   assert(ierr==0);
-  
-  std::cout<<"SHM "<<ShmCommBuf<<std::endl;
+  //  std::cout<<"SHM "<<ShmCommBuf<<std::endl;
 
-  VerticalShmBufs.resize(VerticalSize);
   for(int r=0;r<VerticalSize;r++){
     MPI_Aint sz;
     int dsp_unit;
     MPI_Win_shared_query (VerticalWindow, r, &sz, &dsp_unit, &VerticalShmBufs[r]);
-    std::cout<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
+    //    std::cout<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
   }
+#else 
+  char shm_name [NAME_MAX];
+  MPI_Barrier(VerticalComm);
+
+  if ( VerticalRank == 0 ) {
+    for(int r=0;r<VerticalSize;r++){
+
+      size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES;
+      if ( r>0 ) size = sizeof(SlaveState);
+
+      sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",WorldRank,r);
+      
+      shm_unlink(shm_name);
+
+      int fd=shm_open(shm_name,O_RDWR|O_CREAT,0600);
+      if ( fd < 0 ) {
+	perror("failed shm_open");
+	assert(0);
+      }
+
+      ftruncate(fd, size);
+
+      VerticalShmBufs[r] = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+
+      if ( VerticalShmBufs[r] == MAP_FAILED ) { 
+	perror("failed mmap");
+	assert(0);
+      }
+
+      uint64_t * check = (uint64_t *) VerticalShmBufs[r];
+      check[0] = WorldRank;
+      check[1] = r;
+
+      //      std::cout<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
+    }
+  }
+
+  MPI_Barrier(VerticalComm);
+
+  if ( VerticalRank != 0 ) { 
+  for(int r=0;r<VerticalSize;r++){
+
+    size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES ;
+    if ( r>0 ) size = sizeof(SlaveState);
+    
+    sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",WorldRank,r);
+    
+    int fd=shm_open(shm_name,O_RDWR|O_CREAT,0600);
+    if ( fd<0 ) {
+      perror("failed shm_open");
+      assert(0);
+    }
+    VerticalShmBufs[r] = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+
+    uint64_t * check = (uint64_t *) VerticalShmBufs[r];
+    assert(check[0]== WorldRank);
+    assert(check[1]== r);
+    std::cerr<<"SHM "<<r<<" " <<VerticalShmBufs[r]<<std::endl;
+  }
+  }
+#endif
+  MPI_Barrier(VerticalComm);
 
   //////////////////////////////////////////////////////////////////////
   // Map rank of leader on node in their in new world, to the

From f41a230b32cba75c9730aa4a5ebdf1d5e6760508 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 2 Nov 2016 19:54:03 +0000
Subject: [PATCH 119/149] Decrease mpi3l verbose

---
 lib/communicator/Communicator_mpi3_leader.cc | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3_leader.cc b/lib/communicator/Communicator_mpi3_leader.cc
index b6995a44..49a3e948 100644
--- a/lib/communicator/Communicator_mpi3_leader.cc
+++ b/lib/communicator/Communicator_mpi3_leader.cc
@@ -48,7 +48,6 @@ typedef sem_t *Grid_semaphore;
 
 #include <sys/mman.h>
 
-
 namespace Grid {
 
 enum { COMMAND_ISEND, COMMAND_IRECV, COMMAND_WAITALL };
@@ -91,18 +90,18 @@ public:
 
   void SemInit(void) {
     sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank);
-    printf("SEM_NAME: %s \n",sem_name);
+    //    printf("SEM_NAME: %s \n",sem_name);
     SEM_INIT(sem_head);
     sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank);
-    printf("SEM_NAME: %s \n",sem_name);
+    //    printf("SEM_NAME: %s \n",sem_name);
     SEM_INIT(sem_tail);
   }  
   void SemInitExcl(void) {
     sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank);
-    printf("SEM_INIT_EXCL: %s \n",sem_name);
+    //    printf("SEM_INIT_EXCL: %s \n",sem_name);
     SEM_INIT_EXCL(sem_head);
     sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank);
-    printf("SEM_INIT_EXCL: %s \n",sem_name);
+    //    printf("SEM_INIT_EXCL: %s \n",sem_name);
     SEM_INIT_EXCL(sem_tail);
   }  
   void WakeUpDMA(void) { 
@@ -118,7 +117,7 @@ public:
     SEM_WAIT(sem_tail);
   };
   void EventLoop (void) {
-    std::cout<< " Entering event loop "<<std::endl;
+    //    std::cout<< " Entering event loop "<<std::endl;
     while(1){
       WaitForCommand();
       //      std::cout << "Getting command "<<std::endl;
@@ -291,7 +290,7 @@ void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world,
   /////////////////////////////////////////////////////////////////////
   // Split into groups that can share memory (Verticals)
   /////////////////////////////////////////////////////////////////////
-#define MPI_SHARED_MEM_DEBUG
+#undef MPI_SHARED_MEM_DEBUG
 #ifdef  MPI_SHARED_MEM_DEBUG
   MPI_Comm_split(communicator_universe,(UniverseRank/4),UniverseRank,&VerticalComm);
 #else 
@@ -527,7 +526,7 @@ void Slave::Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _
   universe_rank=_universe_rank;
   vertical_rank=_vertical_rank;
   state   =_state;
-  std::cout << "state "<<_state<<" comm "<<_squadron<<" universe_rank"<<universe_rank <<std::endl;
+  //  std::cout << "state "<<_state<<" comm "<<_squadron<<" universe_rank"<<universe_rank <<std::endl;
   state->head = state->tail = state->start = 0;
   base = (uint64_t)MPIoffloadEngine::VerticalShmBufs[0];
   int rank; MPI_Comm_rank(_squadron,&rank);

From 111bfbc6bc60b4edcb696f21c2edff4606fdb5f0 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 3 Nov 2016 11:40:26 +0000
Subject: [PATCH 120/149] notimestamp by default

---
 lib/Init.cc                       |  6 ++++--
 tests/forces/Test_wilson_force.cc | 18 ++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/Init.cc b/lib/Init.cc
index d15e4bd1..695bb01b 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -234,7 +234,7 @@ void Grid_init(int *argc,char ***argv)
     std::cout<<GridLogMessage<<"  --decomposition : report on default omp,mpi and simd decomposition"<<std::endl;    
     std::cout<<GridLogMessage<<"  --debug-signals : catch sigsegv and print a blame report"<<std::endl;
     std::cout<<GridLogMessage<<"  --debug-stdout  : print stdout from EVERY node"<<std::endl;    
-    std::cout<<GridLogMessage<<"  --timestamp     : tag with millisecond resolution stamps"<<std::endl;    
+    std::cout<<GridLogMessage<<"  --notimestamp   : suppress millisecond resolution stamps"<<std::endl;    
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"Performance:"<<std::endl;
     std::cout<<GridLogMessage<<"  --dslash-generic: Wilson kernel for generic Nc"<<std::endl;    
@@ -316,7 +316,9 @@ void Grid_init(int *argc,char ***argv)
     arg= GridCmdOptionPayload(*argv,*argv+*argc,"--cacheblocking");
     GridCmdOptionIntVector(arg,LebesgueOrder::Block);
   }
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--timestamp") ){
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--notimestamp") ){
+    GridLogTimestamp(0);
+  } else { 
     GridLogTimestamp(1);
   }
 
diff --git a/tests/forces/Test_wilson_force.cc b/tests/forces/Test_wilson_force.cc
index 540a2ffa..1af156cc 100644
--- a/tests/forces/Test_wilson_force.cc
+++ b/tests/forces/Test_wilson_force.cc
@@ -102,16 +102,14 @@ int main (int argc, char ** argv)
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 
     // fourth order exponential approx
-    parallel_for(auto i=mom.begin();i<mom.end();i++){
-      Uprime[i](mu) =
-	  U[i](mu)
-	+ mom[i](mu)*U[i](mu)*dt 
-	+ mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt/2.0)
-	+ mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt/6.0)
-	+ mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt/24.0)
-	+ mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt/120.0)
-	+ mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt*dt/720.0)
-	;
+    parallel_for(auto i=mom.begin();i<mom.end();i++) {
+      Uprime[i](mu)  =	  U[i](mu);
+      Uprime[i](mu) += mom[i](mu)*U[i](mu)*dt ;
+      Uprime[i](mu) += mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt/2.0);
+      Uprime[i](mu) += mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt/6.0);
+      Uprime[i](mu) += mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt/24.0);
+      Uprime[i](mu) += mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt/120.0);
+      Uprime[i](mu) += mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt*dt/720.0);
     }
 
   }

From 92cd797636f628ab721ab5666e6ee42cc8470956 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Thu, 3 Nov 2016 13:48:07 +0000
Subject: [PATCH 121/149] MPI auto configure fix

---
 configure.ac | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/configure.ac b/configure.ac
index a8ad40dd..9178a11d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -253,18 +253,23 @@ AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi
 case ${ac_COMMS} in
      none)
         AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
+        comms_type='none'
      ;;
-     mpi|mpi-auto)
-        AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
-     ;;
-     mpi3|mpi3-auto)
-        AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
-     ;;
-     mpi3l)
+     mpi3l*)
        AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] )
+       comms_type='mpi3l'
+     ;;
+     mpi3*)
+        AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
+        comms_type='mpi3'
+     ;;
+     mpi*)
+        AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
+        comms_type='mpi'
      ;;
      shmem)
         AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
+        comms_type='shmem'
      ;;
      *)
         AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); 
@@ -282,11 +287,11 @@ case ${ac_COMMS} in
         ;;
 esac
 
-AM_CONDITIONAL(BUILD_COMMS_SHMEM,[ test "X${ac_COMMS}X" == "XshmemX" ])
-AM_CONDITIONAL(BUILD_COMMS_MPI,[ test "X${ac_COMMS}X" == "XmpiX" || test "X${ac_COMMS}X" == "Xmpi-autoX" ])
-AM_CONDITIONAL(BUILD_COMMS_MPI3,[ test "X${ac_COMMS}X" == "Xmpi3X"] )
-AM_CONDITIONAL(BUILD_COMMS_MPI3L,[ test "X${ac_COMMS}X" == "Xmpi3lX"] )
-AM_CONDITIONAL(BUILD_COMMS_NONE,[ test "X${ac_COMMS}X" == "XnoneX" ])
+AM_CONDITIONAL(BUILD_COMMS_SHMEM, [ test "${comms_type}X" == "shmemX" ])
+AM_CONDITIONAL(BUILD_COMMS_MPI,   [ test "${comms_type}X" == "mpiX" ])
+AM_CONDITIONAL(BUILD_COMMS_MPI3,  [ test "${comms_type}X" == "mpi3X" ] )
+AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] )
+AM_CONDITIONAL(BUILD_COMMS_NONE,  [ test "${comms_type}X" == "noneX" ])
 
 ############### RNG selection
 AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937],\
@@ -379,7 +384,7 @@ compiler version            : ${ax_cv_gxx_version}
 ----- BUILD OPTIONS -----------------------------------
 SIMD                        : ${ac_SIMD}
 Threading                   : ${ac_openmp} 
-Communications type         : ${ac_COMMS}
+Communications type         : ${comms_type}
 Default precision           : ${ac_PRECISION}
 RNG choice                  : ${ac_RNG} 
 GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`

From c65d23935a4e30d8c3a501772c5afd270a27fccc Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Thu, 3 Nov 2016 13:48:20 +0000
Subject: [PATCH 122/149] README update

---
 README.md | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index faf86faf..c7461368 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ License: GPL v2.
 
 Last update Nov 2016.
 
-_Please send all pull requests to the `develop` branch._
+_Please do not send pull requests to the `master` branch which is reserved for releases._
 
 ### Bug report
 
@@ -29,7 +29,7 @@ _To help us tracking and solving more efficiently issues with Grid, please repor
 When you file an issue, please go though the following checklist:
 
 1. Check that the code is pointing to the `HEAD` of `develop` or any commit in `master` which is tagged with a version number. 
-2. Give a description of the target platform (CPU, network, compiler).
+2. Give a description of the target platform (CPU, network, compiler). Please give the full CPU part description, using for example `cat /proc/cpuinfo | grep 'model name' | uniq` (Linux) or `sysctl machdep.cpu.brand_string` (macOS) and the full output the `--version` option of your compiler.
 3. Give the exact `configure` command used.
 4. Attach `config.log`.
 5. Attach `config.summary`.
@@ -45,7 +45,7 @@ are provided, similar to HPF and cmfortran, and user control is given over the m
 array indices to both MPI tasks and SIMD processing elements.
 
 * Identically shaped arrays then be processed with perfect data parallelisation.
-* Such identically shapped arrays are called conformable arrays.
+* Such identically shaped arrays are called conformable arrays.
 
 The transformation is based on the observation that Cartesian array processing involves
 identical processing to be performed on different regions of the Cartesian array.
@@ -127,14 +127,15 @@ make -C tests/<subdir> tests
 
 The following options can be use with the `--enable-simd=` option to target different communication interfaces:
 
-| `<comm>`      | Description                                  |
-| ------------- | -------------------------------------------- |
-| `none`        | no communications                            |
-| `mpi[-auto]`  | MPI communications                           |
-| `mpi3[-auto]` | MPI communications using MPI 3 shared memory |
-| `shmem `      | Cray SHMEM communications                    |
+| `<comm>`       | Description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| `none`         | no communications                                             |
+| `mpi[-auto]`   | MPI communications                                            |
+| `mpi3[-auto]`  | MPI communications using MPI 3 shared memory                  |
+| `mpi3l[-auto]` | MPI communications using MPI 3 shared memory and leader model |
+| `shmem `       | Cray SHMEM communications                                     |
 
-For `mpi` and `mpi3` the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names).
+For the MPI interfaces the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names).
 
 ### Possible SIMD types
 
@@ -160,7 +161,7 @@ Alternatively, some CPU codenames can be directly used:
 | `BGQ`       | Blue Gene/Q                            |
 
 #### Notes:
-- We currently support AVX512 only for the Intel compiler. Support for GCC and clang will appear in future versions.
+- We currently support AVX512 only for the Intel compiler. Support for GCC and clang will appear in future versions of Grid when the AVX512 support within GCC and clang will be more advanced.
 - For BG/Q only [bgclang](http://trac.alcf.anl.gov/projects/llvm-bgq) is supported. We do not presently plan to support more compilers for this platform.
 - BG/Q performances are currently rather poor. This is being investigated for future versions.
 
@@ -171,7 +172,7 @@ The following configuration is recommended for the Intel Knights Landing platfor
 ``` bash
 ../configure --enable-precision=double\
              --enable-simd=KNL        \
-             --enable-comms=mpi3-auto \
+             --enable-comms=mpi-auto \
              --with-gmp=<path>        \
              --with-mpfr=<path>       \
              --enable-mkl             \
@@ -183,10 +184,9 @@ where `<path>` is the UNIX prefix where GMP and MPFR are installed. If you are w
 ``` bash
 ../configure --enable-precision=double\
              --enable-simd=KNL        \
-             --enable-comms=mpi3      \
+             --enable-comms=mpi       \
              --with-gmp=<path>        \
              --with-mpfr=<path>       \
              --enable-mkl             \
              CXX=CC CC=cc
-```
-
+```
\ No newline at end of file

From aee44dc69472e610cf2b3041ac203e99459d5276 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Thu, 3 Nov 2016 13:54:15 +0000
Subject: [PATCH 123/149] Photon.h removed from develop branch

---
 lib/qcd/action/gauge/Photon.h | 240 ----------------------------------
 1 file changed, 240 deletions(-)
 delete mode 100644 lib/qcd/action/gauge/Photon.h

diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h
deleted file mode 100644
index 852ecb3e..00000000
--- a/lib/qcd/action/gauge/Photon.h
+++ /dev/null
@@ -1,240 +0,0 @@
-/*************************************************************************************
- 
- Grid physics library, www.github.com/paboyle/Grid
- 
- Source file: ./lib/qcd/action/gauge/Photon.h
- 
- Copyright (C) 2015
- 
- Author: Peter Boyle <paboyle@ph.ed.ac.uk>
- 
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
- 
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- GNU General Public License for more details.
- 
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- 
- See the full license in the file "LICENSE" in the top level distribution directory
- *************************************************************************************/
-/*  END LEGAL */
-#ifndef QCD_PHOTON_ACTION_H
-#define QCD_PHOTON_ACTION_H
-
-namespace Grid{
-namespace QCD{
-  
-  template<class Gimpl>
-  class Photon
-  {
-  public:
-    INHERIT_GIMPL_TYPES(Gimpl);
-    enum class Gauge    {Feynman, Coulomb, Landau};
-    enum class ZmScheme {QedL, QedTL};
-  public:
-    Photon(Gauge gauge, ZmScheme zmScheme);
-    virtual ~Photon(void) = default;
-    void FreePropagator(const GaugeField &in, GaugeField &out);
-    void MomentumSpacePropagator(const GaugeField &in, GaugeField &out);
-    void StochasticField(GaugeField &out, GridParallelRNG &rng);
-  private:
-    void invKHatSquared(GaugeLinkField &out);
-    void zmSub(GaugeLinkField &out);
-  private:
-    Gauge    gauge_;
-    ZmScheme zmScheme_;
-  };
-
-  template<class Gimpl>
-  Photon<Gimpl>::Photon(Gauge gauge, ZmScheme zmScheme)
-  : gauge_(gauge), zmScheme_(zmScheme)
-  {}
-  
-  template<class Gimpl>
-  void Photon<Gimpl>::FreePropagator (const GaugeField &in,GaugeField &out)
-  {
-    FFT theFFT(in._grid);
-    
-    GaugeField in_k(in._grid);
-    GaugeField prop_k(in._grid);
-    
-    theFFT.FFT_all_dim(in_k,in,FFT::forward);
-    MomentumSpacePropagator(prop_k,in_k);
-    theFFT.FFT_all_dim(out,prop_k,FFT::backward);
-  }
-  
-  template<class Gimpl>
-  void Photon<Gimpl>::invKHatSquared(GaugeLinkField &out)
-  {
-    GridBase           *grid = out._grid;
-    GaugeLinkField     kmu(grid), one(grid);
-    const unsigned int nd    = grid->_ndimension;
-    std::vector<int>   &l    = grid->_fdimensions;
-    std::vector<int>   zm(nd,0);
-    TComplex           Tone = Complex(1.0,0.0);
-    TComplex           Tzero= Complex(0.0,0.0);
-    
-    one = Complex(1.0,0.0);
-    out = zero;
-    for(int mu = 0; mu < nd; mu++)
-    {
-      Real twoPiL = M_PI*2./l[mu];
-      
-      LatticeCoordinate(kmu,mu);
-      kmu = 2.*sin(.5*twoPiL*kmu);
-      out = out + kmu*kmu;
-    }
-    pokeSite(Tone, out, zm);
-    out = one/out;
-    pokeSite(Tzero, out, zm);
-  }
-  
-  template<class Gimpl>
-  void Photon<Gimpl>::zmSub(GaugeLinkField &out)
-  {
-    GridBase           *grid = out._grid;
-    const unsigned int nd    = grid->_ndimension;
-    
-    switch (zmScheme_)
-    {
-      case ZmScheme::QedTL:
-      {
-        std::vector<int> zm(nd,0);
-        TComplex         Tzero = Complex(0.0,0.0);
-        
-        pokeSite(Tzero, out, zm);
-        
-        break;
-      }
-      case ZmScheme::QedL:
-      {
-        LatticeInteger spNrm(grid), coor(grid);
-        GaugeLinkField z(grid);
-        
-        spNrm = zero;
-        for(int d = 0; d < grid->_ndimension - 1; d++)
-        {
-          LatticeCoordinate(coor,d);
-          spNrm = spNrm + coor*coor;
-        }
-        out = where(spNrm == Integer(0), 0.*out, out);
-        
-        break;
-      }
-      default:
-        break;
-    }
-  }
-  
-  template<class Gimpl>
-  void Photon<Gimpl>::MomentumSpacePropagator(const GaugeField &in,
-                                               GaugeField &out)
-  {
-    GridBase           *grid = out._grid;
-    LatticeComplex     k2Inv(grid);
-    
-    invKHatSquared(k2Inv);
-    zmSub(k2Inv);
-    
-    out = in*k2Inv;
-  }
-  
-  template<class Gimpl>
-  void Photon<Gimpl>::StochasticField(GaugeField &out, GridParallelRNG &rng)
-  {
-    auto               *grid = dynamic_cast<GridCartesian *>(out._grid);
-    const unsigned int nd = grid->_ndimension;
-    std::vector<int> latt_size   = grid->_fdimensions;
-    GaugeLinkField     sqrtK2Inv(grid), r(grid);
-    GaugeField         aTilde(grid);
-    FFT                fft(grid);
-    
-    Integer vol = 1;
-    for(int d = 0; d < nd; d++)
-    {
-      vol = vol * latt_size[d];
-    }
-
-    invKHatSquared(sqrtK2Inv);
-    sqrtK2Inv = sqrt(vol*real(sqrtK2Inv));
-    zmSub(sqrtK2Inv);
-    for(int mu = 0; mu < nd; mu++)
-    {
-      gaussian(rng, r);
-      r = sqrtK2Inv*r;
-      pokeLorentz(aTilde, r, mu);
-    }
-    fft.FFT_all_dim(out, aTilde, FFT::backward);
-  }
-//  template<class Gimpl>
-//  void Photon<Gimpl>::FeynmanGaugeMomentumSpacePropagator_L(GaugeField &out,
-//                                                            const GaugeField &in)
-//  {
-//    
-//    FeynmanGaugeMomentumSpacePropagator_TL(out,in);
-//    
-//    GridBase *grid = out._grid;
-//    LatticeInteger     coor(grid);
-//    GaugeField zz(grid); zz=zero;
-//    
-//    // xyzt
-//    for(int d = 0; d < grid->_ndimension-1;d++){
-//      LatticeCoordinate(coor,d);
-//      out = where(coor==Integer(0),zz,out);
-//    }
-//  }
-//  
-//  template<class Gimpl>
-//  void Photon<Gimpl>::FeynmanGaugeMomentumSpacePropagator_TL(GaugeField &out,
-//                                                             const GaugeField &in)
-//  {
-//    
-//    // what type LatticeComplex
-//    GridBase *grid = out._grid;
-//    int nd = grid->_ndimension;
-//    
-//    typedef typename GaugeField::vector_type vector_type;
-//    typedef typename GaugeField::scalar_type ScalComplex;
-//    typedef Lattice<iSinglet<vector_type> > LatComplex;
-//    
-//    std::vector<int> latt_size   = grid->_fdimensions;
-//    
-//    LatComplex denom(grid); denom= zero;
-//    LatComplex   one(grid); one = ScalComplex(1.0,0.0);
-//    LatComplex   kmu(grid);
-//    
-//    ScalComplex ci(0.0,1.0);
-//    // momphase = n * 2pi / L
-//    for(int mu=0;mu<Nd;mu++) {
-//      
-//      LatticeCoordinate(kmu,mu);
-//      
-//      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
-//      
-//      kmu = TwoPiL * kmu ;
-//      
-//      denom = denom + 4.0*sin(kmu*0.5)*sin(kmu*0.5); // Wilson term
-//    }
-//    std::vector<int> zero_mode(nd,0);
-//    TComplexD Tone = ComplexD(1.0,0.0);
-//    TComplexD Tzero= ComplexD(0.0,0.0);
-//    
-//    pokeSite(Tone,denom,zero_mode);
-//    
-//    denom= one/denom;
-//    
-//    pokeSite(Tzero,denom,zero_mode);
-//    
-//    out = zero;
-//    out = in*denom;
-//  };
-  
-}}
-#endif

From 2854e601e6b1720da5b0ae3dbc9bec9906667bb4 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Thu, 3 Nov 2016 14:09:47 +0000
Subject: [PATCH 124/149] FFT test typo

---
 tests/core/Test_fft.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index 6872f0e1..93596066 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -70,7 +70,7 @@ int main (int argc, char ** argv)
 
 
   std::cout<<"*************************************************"<<std::endl;
-  std::cout<<"Testing Fourier xfrom of known plane wave        "<<std::endl;
+  std::cout<<"Testing Fourier from of known plane wave         "<<std::endl;
   std::cout<<"*************************************************"<<std::endl;
   C=zero;
   for(int mu=0;mu<4;mu++){

From ec232af8518f36b6e2581ffefc2cda797898e214 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Thu, 3 Nov 2016 14:34:16 +0000
Subject: [PATCH 125/149] Photon.h references removed

---
 lib/qcd/action/Actions.h    | 1 -
 tests/core/Test_fft.cc      | 1 -
 tests/core/Test_fft_gfix.cc | 1 -
 3 files changed, 3 deletions(-)

diff --git a/lib/qcd/action/Actions.h b/lib/qcd/action/Actions.h
index a23c6495..ba6e577d 100644
--- a/lib/qcd/action/Actions.h
+++ b/lib/qcd/action/Actions.h
@@ -57,7 +57,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 ////////////////////////////////////////////
 // Gauge Actions
 ////////////////////////////////////////////
-#include <Grid/qcd/action/gauge/Photon.h>
 #include <Grid/qcd/action/gauge/WilsonGaugeAction.h>
 #include <Grid/qcd/action/gauge/PlaqPlusRectangleAction.h>
 
diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index 93596066..03c8bc52 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -27,7 +27,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 #include <Grid/Grid.h>
-#include <Grid/qcd/action/gauge/Photon.h>
 
 using namespace Grid;
 using namespace Grid::QCD;
diff --git a/tests/core/Test_fft_gfix.cc b/tests/core/Test_fft_gfix.cc
index 1eda9a67..6a2868b0 100644
--- a/tests/core/Test_fft_gfix.cc
+++ b/tests/core/Test_fft_gfix.cc
@@ -27,7 +27,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 #include <Grid/Grid.h>
-#include <Grid/qcd/action/gauge/Photon.h>
 
 using namespace Grid;
 using namespace Grid::QCD;

From 6e548a8ad5641db9421db435d68ee6f6d3b9dba1 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 4 Nov 2016 11:34:16 +0000
Subject: [PATCH 126/149] Linux compile needed

---
 lib/communicator/Communicator_mpi3_leader.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/communicator/Communicator_mpi3_leader.cc b/lib/communicator/Communicator_mpi3_leader.cc
index 49a3e948..71f1a913 100644
--- a/lib/communicator/Communicator_mpi3_leader.cc
+++ b/lib/communicator/Communicator_mpi3_leader.cc
@@ -39,6 +39,10 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 ///
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 #include <semaphore.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+
 typedef sem_t *Grid_semaphore;
 
 #define SEM_INIT(S)      S = sem_open(sem_name,0,0600,0); assert ( S != SEM_FAILED );

From b7d55f7dfb4c6b11f0daf573aebfa3a5755aab96 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 4 Nov 2016 11:35:08 +0000
Subject: [PATCH 127/149] Fix a typo in reorg of the --dslash-asm

---
 lib/qcd/action/fermion/WilsonKernels.h | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernels.h b/lib/qcd/action/fermion/WilsonKernels.h
index 919f7540..47da2b14 100644
--- a/lib/qcd/action/fermion/WilsonKernels.h
+++ b/lib/qcd/action/fermion/WilsonKernels.h
@@ -61,14 +61,8 @@ public:
     switch(Opt) {
 #ifdef AVX512
     case OptInlineAsm:
-      for (int site = 0; site < Ns; site++) {
-	for (int s = 0; s < Ls; s++) {
-	  WilsonKernels<Impl>::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
-	  sF++;
-	}
-	sU++;
-      }
-      break;
+       WilsonKernels<Impl>::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
+       break;
 #endif
     case OptHandUnroll:
       for (int site = 0; site < Ns; site++) {
@@ -115,13 +109,7 @@ public:
     switch(Opt) {
 #ifdef AVX512
     case OptInlineAsm:
-      for (int site = 0; site < Ns; site++) {
-	for (int s = 0; s < Ls; s++) {
-	  WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
-	  sF++;
-	}
-	sU++;
-      }
+      WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
       break;
 #endif
     case OptHandUnroll:

From f2f16eb972011081680d1805981d3809ada0449a Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 4 Nov 2016 13:11:05 +0000
Subject: [PATCH 128/149] fftw3.h removed, please don't commit this file back

---
 lib/fftw/fftw3.h | 412 -----------------------------------------------
 1 file changed, 412 deletions(-)
 delete mode 100644 lib/fftw/fftw3.h

diff --git a/lib/fftw/fftw3.h b/lib/fftw/fftw3.h
deleted file mode 100644
index 1bf34396..00000000
--- a/lib/fftw/fftw3.h
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Copyright (c) 2003, 2007-14 Matteo Frigo
- * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
- *
- * The following statement of license applies *only* to this header file,
- * and *not* to the other files distributed with FFTW or derived therefrom:
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/***************************** NOTE TO USERS *********************************
- *
- *                 THIS IS A HEADER FILE, NOT A MANUAL
- *
- *    If you want to know how to use FFTW, please read the manual,
- *    online at http://www.fftw.org/doc/ and also included with FFTW.
- *    For a quick start, see the manual's tutorial section.
- *
- *   (Reading header files to learn how to use a library is a habit
- *    stemming from code lacking a proper manual.  Arguably, it's a
- *    *bad* habit in most cases, because header files can contain
- *    interfaces that are not part of the public, stable API.)
- *
- ****************************************************************************/
-
-#ifndef FFTW3_H
-#define FFTW3_H
-
-#include <stdio.h>
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif /* __cplusplus */
-
-/* If <complex.h> is included, use the C99 complex type.  Otherwise
-   define a type bit-compatible with C99 complex */
-#if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I)
-#  define FFTW_DEFINE_COMPLEX(R, C) typedef R _Complex C
-#else
-#  define FFTW_DEFINE_COMPLEX(R, C) typedef R C[2]
-#endif
-
-#define FFTW_CONCAT(prefix, name) prefix ## name
-#define FFTW_MANGLE_DOUBLE(name) FFTW_CONCAT(fftw_, name)
-#define FFTW_MANGLE_FLOAT(name) FFTW_CONCAT(fftwf_, name)
-#define FFTW_MANGLE_LONG_DOUBLE(name) FFTW_CONCAT(fftwl_, name)
-#define FFTW_MANGLE_QUAD(name) FFTW_CONCAT(fftwq_, name)
-
-/* IMPORTANT: for Windows compilers, you should add a line
-        #define FFTW_DLL
-   here and in kernel/ifftw.h if you are compiling/using FFTW as a
-   DLL, in order to do the proper importing/exporting, or
-   alternatively compile with -DFFTW_DLL or the equivalent
-   command-line flag.  This is not necessary under MinGW/Cygwin, where
-   libtool does the imports/exports automatically. */
-#if defined(FFTW_DLL) && (defined(_WIN32) || defined(__WIN32__))
-   /* annoying Windows syntax for shared-library declarations */
-#  if defined(COMPILING_FFTW) /* defined in api.h when compiling FFTW */
-#    define FFTW_EXTERN extern __declspec(dllexport) 
-#  else /* user is calling FFTW; import symbol */
-#    define FFTW_EXTERN extern __declspec(dllimport) 
-#  endif
-#else
-#  define FFTW_EXTERN extern
-#endif
-
-enum fftw_r2r_kind_do_not_use_me {
-     FFTW_R2HC=0, FFTW_HC2R=1, FFTW_DHT=2,
-     FFTW_REDFT00=3, FFTW_REDFT01=4, FFTW_REDFT10=5, FFTW_REDFT11=6,
-     FFTW_RODFT00=7, FFTW_RODFT01=8, FFTW_RODFT10=9, FFTW_RODFT11=10
-};
-
-struct fftw_iodim_do_not_use_me {
-     int n;                     /* dimension size */
-     int is;			/* input stride */
-     int os;			/* output stride */
-};
-
-#include <stddef.h> /* for ptrdiff_t */
-struct fftw_iodim64_do_not_use_me {
-     ptrdiff_t n;                     /* dimension size */
-     ptrdiff_t is;			/* input stride */
-     ptrdiff_t os;			/* output stride */
-};
-
-typedef void (*fftw_write_char_func_do_not_use_me)(char c, void *);
-typedef int (*fftw_read_char_func_do_not_use_me)(void *);
-
-/*
-  huge second-order macro that defines prototypes for all API
-  functions.  We expand this macro for each supported precision
- 
-  X: name-mangling macro
-  R: real data type
-  C: complex data type
-*/
-
-#define FFTW_DEFINE_API(X, R, C)					   \
-									   \
-FFTW_DEFINE_COMPLEX(R, C);						   \
-									   \
-typedef struct X(plan_s) *X(plan);					   \
-									   \
-typedef struct fftw_iodim_do_not_use_me X(iodim);			   \
-typedef struct fftw_iodim64_do_not_use_me X(iodim64);			   \
-									   \
-typedef enum fftw_r2r_kind_do_not_use_me X(r2r_kind);			   \
-									   \
-typedef fftw_write_char_func_do_not_use_me X(write_char_func);		   \
-typedef fftw_read_char_func_do_not_use_me X(read_char_func);		   \
-									   \
-FFTW_EXTERN void X(execute)(const X(plan) p);				   \
-									   \
-FFTW_EXTERN X(plan) X(plan_dft)(int rank, const int *n,			   \
-		    C *in, C *out, int sign, unsigned flags);		   \
-									   \
-FFTW_EXTERN X(plan) X(plan_dft_1d)(int n, C *in, C *out, int sign,	   \
-		       unsigned flags);					   \
-FFTW_EXTERN X(plan) X(plan_dft_2d)(int n0, int n1,			   \
-		       C *in, C *out, int sign, unsigned flags);	   \
-FFTW_EXTERN X(plan) X(plan_dft_3d)(int n0, int n1, int n2,		   \
-		       C *in, C *out, int sign, unsigned flags);	   \
-									   \
-FFTW_EXTERN X(plan) X(plan_many_dft)(int rank, const int *n,		   \
-                         int howmany,					   \
-                         C *in, const int *inembed,			   \
-                         int istride, int idist,			   \
-                         C *out, const int *onembed,			   \
-                         int ostride, int odist,			   \
-                         int sign, unsigned flags);			   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru_dft)(int rank, const X(iodim) *dims,	   \
-			 int howmany_rank,				   \
-			 const X(iodim) *howmany_dims,			   \
-			 C *in, C *out,					   \
-			 int sign, unsigned flags);			   \
-FFTW_EXTERN X(plan) X(plan_guru_split_dft)(int rank, const X(iodim) *dims, \
-			 int howmany_rank,				   \
-			 const X(iodim) *howmany_dims,			   \
-			 R *ri, R *ii, R *ro, R *io,			   \
-			 unsigned flags);				   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru64_dft)(int rank,			   \
-                         const X(iodim64) *dims,			   \
-			 int howmany_rank,				   \
-			 const X(iodim64) *howmany_dims,		   \
-			 C *in, C *out,					   \
-			 int sign, unsigned flags);			   \
-FFTW_EXTERN X(plan) X(plan_guru64_split_dft)(int rank,			   \
-                         const X(iodim64) *dims,			   \
-			 int howmany_rank,				   \
-			 const X(iodim64) *howmany_dims,		   \
-			 R *ri, R *ii, R *ro, R *io,			   \
-			 unsigned flags);				   \
-									   \
-FFTW_EXTERN void X(execute_dft)(const X(plan) p, C *in, C *out);	   \
-FFTW_EXTERN void X(execute_split_dft)(const X(plan) p, R *ri, R *ii,	   \
-                                      R *ro, R *io);			   \
-									   \
-FFTW_EXTERN X(plan) X(plan_many_dft_r2c)(int rank, const int *n,	   \
-                             int howmany,				   \
-                             R *in, const int *inembed,			   \
-                             int istride, int idist,			   \
-                             C *out, const int *onembed,		   \
-                             int ostride, int odist,			   \
-                             unsigned flags);				   \
-									   \
-FFTW_EXTERN X(plan) X(plan_dft_r2c)(int rank, const int *n,		   \
-                        R *in, C *out, unsigned flags);			   \
-									   \
-FFTW_EXTERN X(plan) X(plan_dft_r2c_1d)(int n,R *in,C *out,unsigned flags); \
-FFTW_EXTERN X(plan) X(plan_dft_r2c_2d)(int n0, int n1,			   \
-			   R *in, C *out, unsigned flags);		   \
-FFTW_EXTERN X(plan) X(plan_dft_r2c_3d)(int n0, int n1,			   \
-			   int n2,					   \
-			   R *in, C *out, unsigned flags);		   \
-									   \
-									   \
-FFTW_EXTERN X(plan) X(plan_many_dft_c2r)(int rank, const int *n,	   \
-			     int howmany,				   \
-			     C *in, const int *inembed,			   \
-			     int istride, int idist,			   \
-			     R *out, const int *onembed,		   \
-			     int ostride, int odist,			   \
-			     unsigned flags);				   \
-									   \
-FFTW_EXTERN X(plan) X(plan_dft_c2r)(int rank, const int *n,		   \
-                        C *in, R *out, unsigned flags);			   \
-									   \
-FFTW_EXTERN X(plan) X(plan_dft_c2r_1d)(int n,C *in,R *out,unsigned flags); \
-FFTW_EXTERN X(plan) X(plan_dft_c2r_2d)(int n0, int n1,			   \
-			   C *in, R *out, unsigned flags);		   \
-FFTW_EXTERN X(plan) X(plan_dft_c2r_3d)(int n0, int n1,			   \
-			   int n2,					   \
-			   C *in, R *out, unsigned flags);		   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru_dft_r2c)(int rank, const X(iodim) *dims,   \
-			     int howmany_rank,				   \
-			     const X(iodim) *howmany_dims,		   \
-			     R *in, C *out,				   \
-			     unsigned flags);				   \
-FFTW_EXTERN X(plan) X(plan_guru_dft_c2r)(int rank, const X(iodim) *dims,   \
-			     int howmany_rank,				   \
-			     const X(iodim) *howmany_dims,		   \
-			     C *in, R *out,				   \
-			     unsigned flags);				   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru_split_dft_r2c)(				   \
-                             int rank, const X(iodim) *dims,		   \
-			     int howmany_rank,				   \
-			     const X(iodim) *howmany_dims,		   \
-			     R *in, R *ro, R *io,			   \
-			     unsigned flags);				   \
-FFTW_EXTERN X(plan) X(plan_guru_split_dft_c2r)(				   \
-                             int rank, const X(iodim) *dims,		   \
-			     int howmany_rank,				   \
-			     const X(iodim) *howmany_dims,		   \
-			     R *ri, R *ii, R *out,			   \
-			     unsigned flags);				   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru64_dft_r2c)(int rank,			   \
-                             const X(iodim64) *dims,			   \
-			     int howmany_rank,				   \
-			     const X(iodim64) *howmany_dims,		   \
-			     R *in, C *out,				   \
-			     unsigned flags);				   \
-FFTW_EXTERN X(plan) X(plan_guru64_dft_c2r)(int rank,			   \
-                             const X(iodim64) *dims,			   \
-			     int howmany_rank,				   \
-			     const X(iodim64) *howmany_dims,		   \
-			     C *in, R *out,				   \
-			     unsigned flags);				   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru64_split_dft_r2c)(			   \
-                             int rank, const X(iodim64) *dims,		   \
-			     int howmany_rank,				   \
-			     const X(iodim64) *howmany_dims,		   \
-			     R *in, R *ro, R *io,			   \
-			     unsigned flags);				   \
-FFTW_EXTERN X(plan) X(plan_guru64_split_dft_c2r)(			   \
-                             int rank, const X(iodim64) *dims,		   \
-			     int howmany_rank,				   \
-			     const X(iodim64) *howmany_dims,		   \
-			     R *ri, R *ii, R *out,			   \
-			     unsigned flags);				   \
-									   \
-FFTW_EXTERN void X(execute_dft_r2c)(const X(plan) p, R *in, C *out);	   \
-FFTW_EXTERN void X(execute_dft_c2r)(const X(plan) p, C *in, R *out);	   \
-									   \
-FFTW_EXTERN void X(execute_split_dft_r2c)(const X(plan) p,		   \
-                                          R *in, R *ro, R *io);		   \
-FFTW_EXTERN void X(execute_split_dft_c2r)(const X(plan) p,		   \
-                                          R *ri, R *ii, R *out);	   \
-									   \
-FFTW_EXTERN X(plan) X(plan_many_r2r)(int rank, const int *n,		   \
-                         int howmany,					   \
-                         R *in, const int *inembed,			   \
-                         int istride, int idist,			   \
-                         R *out, const int *onembed,			   \
-                         int ostride, int odist,			   \
-                         const X(r2r_kind) *kind, unsigned flags);	   \
-									   \
-FFTW_EXTERN X(plan) X(plan_r2r)(int rank, const int *n, R *in, R *out,	   \
-                    const X(r2r_kind) *kind, unsigned flags);		   \
-									   \
-FFTW_EXTERN X(plan) X(plan_r2r_1d)(int n, R *in, R *out,		   \
-                       X(r2r_kind) kind, unsigned flags);		   \
-FFTW_EXTERN X(plan) X(plan_r2r_2d)(int n0, int n1, R *in, R *out,	   \
-                       X(r2r_kind) kind0, X(r2r_kind) kind1,		   \
-                       unsigned flags);					   \
-FFTW_EXTERN X(plan) X(plan_r2r_3d)(int n0, int n1, int n2,		   \
-                       R *in, R *out, X(r2r_kind) kind0,		   \
-                       X(r2r_kind) kind1, X(r2r_kind) kind2,		   \
-                       unsigned flags);					   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru_r2r)(int rank, const X(iodim) *dims,	   \
-                         int howmany_rank,				   \
-                         const X(iodim) *howmany_dims,			   \
-                         R *in, R *out,					   \
-                         const X(r2r_kind) *kind, unsigned flags);	   \
-									   \
-FFTW_EXTERN X(plan) X(plan_guru64_r2r)(int rank, const X(iodim64) *dims,   \
-                         int howmany_rank,				   \
-                         const X(iodim64) *howmany_dims,		   \
-                         R *in, R *out,					   \
-                         const X(r2r_kind) *kind, unsigned flags);	   \
-									   \
-FFTW_EXTERN void X(execute_r2r)(const X(plan) p, R *in, R *out);	   \
-									   \
-FFTW_EXTERN void X(destroy_plan)(X(plan) p);				   \
-FFTW_EXTERN void X(forget_wisdom)(void);				   \
-FFTW_EXTERN void X(cleanup)(void);					   \
-									   \
-FFTW_EXTERN void X(set_timelimit)(double t);				   \
-									   \
-FFTW_EXTERN void X(plan_with_nthreads)(int nthreads);			   \
-FFTW_EXTERN int X(init_threads)(void);					   \
-FFTW_EXTERN void X(cleanup_threads)(void);				   \
-									   \
-FFTW_EXTERN int X(export_wisdom_to_filename)(const char *filename);	   \
-FFTW_EXTERN void X(export_wisdom_to_file)(FILE *output_file);		   \
-FFTW_EXTERN char *X(export_wisdom_to_string)(void);			   \
-FFTW_EXTERN void X(export_wisdom)(X(write_char_func) write_char,   	   \
-                                  void *data);				   \
-FFTW_EXTERN int X(import_system_wisdom)(void);				   \
-FFTW_EXTERN int X(import_wisdom_from_filename)(const char *filename);	   \
-FFTW_EXTERN int X(import_wisdom_from_file)(FILE *input_file);		   \
-FFTW_EXTERN int X(import_wisdom_from_string)(const char *input_string);	   \
-FFTW_EXTERN int X(import_wisdom)(X(read_char_func) read_char, void *data); \
-									   \
-FFTW_EXTERN void X(fprint_plan)(const X(plan) p, FILE *output_file);	   \
-FFTW_EXTERN void X(print_plan)(const X(plan) p);			   \
-FFTW_EXTERN char *X(sprint_plan)(const X(plan) p);			   \
-									   \
-FFTW_EXTERN void *X(malloc)(size_t n);					   \
-FFTW_EXTERN R *X(alloc_real)(size_t n);					   \
-FFTW_EXTERN C *X(alloc_complex)(size_t n);				   \
-FFTW_EXTERN void X(free)(void *p);					   \
-									   \
-FFTW_EXTERN void X(flops)(const X(plan) p,				   \
-                          double *add, double *mul, double *fmas);	   \
-FFTW_EXTERN double X(estimate_cost)(const X(plan) p);			   \
-FFTW_EXTERN double X(cost)(const X(plan) p);				   \
-									   \
-FFTW_EXTERN int X(alignment_of)(R *p);                                     \
-FFTW_EXTERN const char X(version)[];                                       \
-FFTW_EXTERN const char X(cc)[];						   \
-FFTW_EXTERN const char X(codelet_optim)[];
-
-
-/* end of FFTW_DEFINE_API macro */
-
-FFTW_DEFINE_API(FFTW_MANGLE_DOUBLE, double, fftw_complex)
-FFTW_DEFINE_API(FFTW_MANGLE_FLOAT, float, fftwf_complex)
-FFTW_DEFINE_API(FFTW_MANGLE_LONG_DOUBLE, long double, fftwl_complex)
-
-/* __float128 (quad precision) is a gcc extension on i386, x86_64, and ia64
-   for gcc >= 4.6 (compiled in FFTW with --enable-quad-precision) */
-#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) \
- && !(defined(__ICC) || defined(__INTEL_COMPILER)) \
- && (defined(__i386__) || defined(__x86_64__) || defined(__ia64__))
-#  if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I)
-/* note: __float128 is a typedef, which is not supported with the _Complex
-         keyword in gcc, so instead we use this ugly __attribute__ version.
-         However, we can't simply pass the __attribute__ version to
-         FFTW_DEFINE_API because the __attribute__ confuses gcc in pointer
-         types.  Hence redefining FFTW_DEFINE_COMPLEX.  Ugh. */
-#    undef FFTW_DEFINE_COMPLEX
-#    define FFTW_DEFINE_COMPLEX(R, C) typedef _Complex float __attribute__((mode(TC))) C
-#  endif
-FFTW_DEFINE_API(FFTW_MANGLE_QUAD, __float128, fftwq_complex)
-#endif
-
-#define FFTW_FORWARD (-1)
-#define FFTW_BACKWARD (+1)
-
-#define FFTW_NO_TIMELIMIT (-1.0)
-
-/* documented flags */
-#define FFTW_MEASURE (0U)
-#define FFTW_DESTROY_INPUT (1U << 0)
-#define FFTW_UNALIGNED (1U << 1)
-#define FFTW_CONSERVE_MEMORY (1U << 2)
-#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */
-#define FFTW_PRESERVE_INPUT (1U << 4) /* cancels FFTW_DESTROY_INPUT */
-#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */
-#define FFTW_ESTIMATE (1U << 6)
-#define FFTW_WISDOM_ONLY (1U << 21)
-
-/* undocumented beyond-guru flags */
-#define FFTW_ESTIMATE_PATIENT (1U << 7)
-#define FFTW_BELIEVE_PCOST (1U << 8)
-#define FFTW_NO_DFT_R2HC (1U << 9)
-#define FFTW_NO_NONTHREADED (1U << 10)
-#define FFTW_NO_BUFFERING (1U << 11)
-#define FFTW_NO_INDIRECT_OP (1U << 12)
-#define FFTW_ALLOW_LARGE_GENERIC (1U << 13) /* NO_LARGE_GENERIC is default */
-#define FFTW_NO_RANK_SPLITS (1U << 14)
-#define FFTW_NO_VRANK_SPLITS (1U << 15)
-#define FFTW_NO_VRECURSE (1U << 16)
-#define FFTW_NO_SIMD (1U << 17)
-#define FFTW_NO_SLOW (1U << 18)
-#define FFTW_NO_FIXED_RADIX_LARGE_N (1U << 19)
-#define FFTW_ALLOW_PRUNING (1U << 20)
-
-#ifdef __cplusplus
-}  /* extern "C" */
-#endif /* __cplusplus */
-
-#endif /* FFTW3_H */

From ad971ca07beead6482c69f656e5cb80eb4ed1c73 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 4 Nov 2016 13:12:35 +0000
Subject: [PATCH 129/149] fftw3.h is now expected to be an external header

---
 lib/FFT.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index a16e775b..fda43eb8 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -30,7 +30,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #define _GRID_FFT_H_
 
 #ifdef HAVE_FFTW	
-#include <Grid/fftw/fftw3.h>
+#include <fftw3.h>
 #endif
 
 

From 6592078feffb83e01e82dcb9144ebd551900201b Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 4 Nov 2016 13:43:40 +0000
Subject: [PATCH 130/149] Make.inc removed, once again don't commit it!

---
 tests/hmc/Make.inc | 63 ----------------------------------------------
 1 file changed, 63 deletions(-)
 delete mode 100644 tests/hmc/Make.inc

diff --git a/tests/hmc/Make.inc b/tests/hmc/Make.inc
deleted file mode 100644
index f6f9c6b5..00000000
--- a/tests/hmc/Make.inc
+++ /dev/null
@@ -1,63 +0,0 @@
-tests: Test_hmc_EODWFRatio Test_hmc_EODWFRatio_Gparity Test_hmc_EOWilsonFermionGauge Test_hmc_EOWilsonRatio Test_hmc_GparityIwasakiGauge Test_hmc_GparityWilsonGauge Test_hmc_IwasakiGauge Test_hmc_RectGauge Test_hmc_WilsonAdjointFermionGauge Test_hmc_WilsonFermionGauge Test_hmc_WilsonGauge Test_hmc_WilsonMixedRepresentationsFermionGauge Test_hmc_WilsonRatio Test_hmc_WilsonTwoIndexSymmetricFermionGauge Test_multishift_sqrt Test_remez Test_rhmc_EOWilson1p1 Test_rhmc_EOWilsonRatio Test_rhmc_Wilson1p1 Test_rhmc_WilsonRatio
-EXTRA_PROGRAMS = Test_hmc_EODWFRatio Test_hmc_EODWFRatio_Gparity Test_hmc_EOWilsonFermionGauge Test_hmc_EOWilsonRatio Test_hmc_GparityIwasakiGauge Test_hmc_GparityWilsonGauge Test_hmc_IwasakiGauge Test_hmc_RectGauge Test_hmc_WilsonAdjointFermionGauge Test_hmc_WilsonFermionGauge Test_hmc_WilsonGauge Test_hmc_WilsonMixedRepresentationsFermionGauge Test_hmc_WilsonRatio Test_hmc_WilsonTwoIndexSymmetricFermionGauge Test_multishift_sqrt Test_remez Test_rhmc_EOWilson1p1 Test_rhmc_EOWilsonRatio Test_rhmc_Wilson1p1 Test_rhmc_WilsonRatio
-
-Test_hmc_EODWFRatio_SOURCES=Test_hmc_EODWFRatio.cc
-Test_hmc_EODWFRatio_LDADD=-lGrid
-
-Test_hmc_EODWFRatio_Gparity_SOURCES=Test_hmc_EODWFRatio_Gparity.cc
-Test_hmc_EODWFRatio_Gparity_LDADD=-lGrid
-
-Test_hmc_EOWilsonFermionGauge_SOURCES=Test_hmc_EOWilsonFermionGauge.cc
-Test_hmc_EOWilsonFermionGauge_LDADD=-lGrid
-
-Test_hmc_EOWilsonRatio_SOURCES=Test_hmc_EOWilsonRatio.cc
-Test_hmc_EOWilsonRatio_LDADD=-lGrid
-
-Test_hmc_GparityIwasakiGauge_SOURCES=Test_hmc_GparityIwasakiGauge.cc
-Test_hmc_GparityIwasakiGauge_LDADD=-lGrid
-
-Test_hmc_GparityWilsonGauge_SOURCES=Test_hmc_GparityWilsonGauge.cc
-Test_hmc_GparityWilsonGauge_LDADD=-lGrid
-
-Test_hmc_IwasakiGauge_SOURCES=Test_hmc_IwasakiGauge.cc
-Test_hmc_IwasakiGauge_LDADD=-lGrid
-
-Test_hmc_RectGauge_SOURCES=Test_hmc_RectGauge.cc
-Test_hmc_RectGauge_LDADD=-lGrid
-
-Test_hmc_WilsonAdjointFermionGauge_SOURCES=Test_hmc_WilsonAdjointFermionGauge.cc
-Test_hmc_WilsonAdjointFermionGauge_LDADD=-lGrid
-
-Test_hmc_WilsonFermionGauge_SOURCES=Test_hmc_WilsonFermionGauge.cc
-Test_hmc_WilsonFermionGauge_LDADD=-lGrid
-
-Test_hmc_WilsonGauge_SOURCES=Test_hmc_WilsonGauge.cc
-Test_hmc_WilsonGauge_LDADD=-lGrid
-
-Test_hmc_WilsonMixedRepresentationsFermionGauge_SOURCES=Test_hmc_WilsonMixedRepresentationsFermionGauge.cc
-Test_hmc_WilsonMixedRepresentationsFermionGauge_LDADD=-lGrid
-
-Test_hmc_WilsonRatio_SOURCES=Test_hmc_WilsonRatio.cc
-Test_hmc_WilsonRatio_LDADD=-lGrid
-
-Test_hmc_WilsonTwoIndexSymmetricFermionGauge_SOURCES=Test_hmc_WilsonTwoIndexSymmetricFermionGauge.cc
-Test_hmc_WilsonTwoIndexSymmetricFermionGauge_LDADD=-lGrid
-
-Test_multishift_sqrt_SOURCES=Test_multishift_sqrt.cc
-Test_multishift_sqrt_LDADD=-lGrid
-
-Test_remez_SOURCES=Test_remez.cc
-Test_remez_LDADD=-lGrid
-
-Test_rhmc_EOWilson1p1_SOURCES=Test_rhmc_EOWilson1p1.cc
-Test_rhmc_EOWilson1p1_LDADD=-lGrid
-
-Test_rhmc_EOWilsonRatio_SOURCES=Test_rhmc_EOWilsonRatio.cc
-Test_rhmc_EOWilsonRatio_LDADD=-lGrid
-
-Test_rhmc_Wilson1p1_SOURCES=Test_rhmc_Wilson1p1.cc
-Test_rhmc_Wilson1p1_LDADD=-lGrid
-
-Test_rhmc_WilsonRatio_SOURCES=Test_rhmc_WilsonRatio.cc
-Test_rhmc_WilsonRatio_LDADD=-lGrid
-

From 8af8b047fde3aaf17ed6a19df139788c73798217 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 4 Nov 2016 13:44:21 +0000
Subject: [PATCH 131/149] tests is now a recusrsive target

---
 configure.ac     | 1 +
 scripts/filelist | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 9178a11d..2b759dde 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4,6 +4,7 @@ AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
 AM_INIT_AUTOMAKE(subdir-objects)
+AM_EXTRA_RECURSIVE_TARGETS([tests])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h])
diff --git a/scripts/filelist b/scripts/filelist
index 0802c2a6..52252ca9 100755
--- a/scripts/filelist
+++ b/scripts/filelist
@@ -20,7 +20,7 @@ for subdir in $dirs; do
     TESTS=`ls T*.cc`
     TESTLIST=`echo ${TESTS} | sed s/.cc//g `
     PREF=`[ $subdir = '.' ] && echo noinst || echo EXTRA`
-    echo "tests: ${TESTLIST}" > Make.inc
+    echo "tests-local: ${TESTLIST}" > Make.inc
     echo ${PREF}_PROGRAMS = ${TESTLIST} >> Make.inc
     echo >> Make.inc
     for f in $TESTS; do

From 7df940dc3eaee162dca8a4fd81dd299d32d342b9 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Fri, 4 Nov 2016 22:32:25 +0000
Subject: [PATCH 132/149] homemade test recusrive target for old autotools
 versions

---
 Makefile.am       |  5 +++++
 README.md         |  1 +
 configure.ac      |  1 -
 scripts/filelist  | 15 ++++++++++-----
 tests/Makefile.am |  5 +++++
 5 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 90c5cd71..049220e8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,5 +1,10 @@
 # additional include paths necessary to compile the C++ library
 SUBDIRS = lib benchmarks tests
 
+.PHONY: tests
+
+tests:
+	make -C tests tests
+
 AM_CXXFLAGS += -I$(top_builddir)/include
 ACLOCAL_AMFLAGS = -I m4
diff --git a/README.md b/README.md
index c7461368..49f08237 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,7 @@ To minimise the build time, only the tests at the root of the `tests` directory
 ``` bash
 make -C tests/<subdir> tests
 ```
+If you want to build all the tests at once just use `make tests`.
 
 ### Build configuration options
 
diff --git a/configure.ac b/configure.ac
index 2b759dde..9178a11d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4,7 +4,6 @@ AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
 AM_INIT_AUTOMAKE(subdir-objects)
-AM_EXTRA_RECURSIVE_TARGETS([tests])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h])
diff --git a/scripts/filelist b/scripts/filelist
index 52252ca9..dddb605b 100755
--- a/scripts/filelist
+++ b/scripts/filelist
@@ -20,15 +20,20 @@ for subdir in $dirs; do
     TESTS=`ls T*.cc`
     TESTLIST=`echo ${TESTS} | sed s/.cc//g `
     PREF=`[ $subdir = '.' ] && echo noinst || echo EXTRA`
-    echo "tests-local: ${TESTLIST}" > Make.inc
+    SUB=`[ $subdir = '.' ] && echo subtests`
+    echo "tests: ${TESTLIST} ${SUB}" > Make.inc
     echo ${PREF}_PROGRAMS = ${TESTLIST} >> Make.inc
     echo >> Make.inc
     for f in $TESTS; do
-	BNAME=`basename $f .cc`
-	echo ${BNAME}_SOURCES=$f  >> Make.inc
-	echo ${BNAME}_LDADD=-lGrid>> Make.inc
-	echo >> Make.inc
+	   BNAME=`basename $f .cc`
+	   echo ${BNAME}_SOURCES=$f  >> Make.inc
+	   echo ${BNAME}_LDADD=-lGrid>> Make.inc
+	   echo >> Make.inc
     done
+    if [ $subdir != '.' ]; then
+       echo CLEANFILES = ${TESTLIST} >> Make.inc
+       echo >> Make.inc
+    fi
 done
 
 # benchmarks Make.inc
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 2b8cc2d7..c98bc2d0 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -4,4 +4,9 @@ if BUILD_CHROMA_REGRESSION
   SUBDIRS+= qdpxx
 endif
 
+.PHONY: subtests
+
 include Make.inc
+
+subtests:
+	for d in $(SUBDIRS); do make -C $${d} tests; done

From 96ba42a2976b4eed3406ddca8013030e4423f4dd Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Fri, 4 Nov 2016 22:47:25 +0000
Subject: [PATCH 133/149] omm buf

---
 lib/algorithms/CoarsenedMatrix.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/algorithms/CoarsenedMatrix.h b/lib/algorithms/CoarsenedMatrix.h
index 3ae0af75..fd9acc91 100644
--- a/lib/algorithms/CoarsenedMatrix.h
+++ b/lib/algorithms/CoarsenedMatrix.h
@@ -282,7 +282,7 @@ PARALLEL_FOR_LOOP
 	  } else if(SE->_is_local) { 
 	    nbr = in._odata[SE->_offset];
 	  } else {
-	    nbr = Stencil.comm_buf[SE->_offset];
+	    nbr = Stencil.CommBuf()[SE->_offset];
 	  }
 	  res = res + A[point]._odata[ss]*nbr;
 	}

From afc8d3e524b6ceff65c415ad05b41384fc420e2a Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Mon, 7 Nov 2016 11:13:43 +0000
Subject: [PATCH 134/149] Adding support for parallel recursive compilation for
 the tests

---
 Makefile.am       | 2 +-
 tests/Makefile.am | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 049220e8..2606b88c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -4,7 +4,7 @@ SUBDIRS = lib benchmarks tests
 .PHONY: tests
 
 tests:
-	make -C tests tests
+	$(MAKE) -C tests tests
 
 AM_CXXFLAGS += -I$(top_builddir)/include
 ACLOCAL_AMFLAGS = -I m4
diff --git a/tests/Makefile.am b/tests/Makefile.am
index c98bc2d0..2e7c1f0a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -9,4 +9,4 @@ endif
 include Make.inc
 
 subtests:
-	for d in $(SUBDIRS); do make -C $${d} tests; done
+	for d in $(SUBDIRS); do $(MAKE) -C $${d} tests; done

From 0cff8754d1b8e2c46b972dcc02b0e80069c2bcd7 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 8 Nov 2016 11:35:41 +0000
Subject: [PATCH 135/149] Usecs

---
 lib/FFT.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index fda43eb8..52d08cbe 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -122,7 +122,8 @@ namespace Grid {
     
     double Flops(void) {return flops;}
     double MFlops(void) {return flops/usec;}
-    
+    double USec(void)   {return (double)usec;}    
+
     FFT ( GridCartesian * grid ) :
     vgrid(grid),
     Nd(grid->_ndimension),

From 3dc2e05d6edb5acd0582293017d4a7fe5753297d Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 8 Nov 2016 11:36:18 +0000
Subject: [PATCH 136/149] Time as well since MKL returns zero for Mflops

---
 tests/core/Test_fftf.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/core/Test_fftf.cc b/tests/core/Test_fftf.cc
index 54b1ddb3..4eb4398d 100644
--- a/tests/core/Test_fftf.cc
+++ b/tests/core/Test_fftf.cc
@@ -93,10 +93,10 @@ int main (int argc, char ** argv)
   C=C-Ctilde;
   std::cout << "diff scalar "<<norm2(C) << std::endl;
 
-  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde; std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
-  theFFT.FFT_dim(Stilde,S,3,FFT::forward);std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<< " "<<theFFT.USec() <<std::endl;
+  theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<< " "<<theFFT.USec() <<std::endl;
+  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<< " "<<theFFT.USec() <<std::endl;
+  theFFT.FFT_dim(Stilde,S,3,FFT::forward);std::cout << theFFT.MFlops()<<" "<<theFFT.USec() <<std::endl;
 
   SpinMatrixF Sp; 
   Sp = zero; Sp = Sp+cVol;

From f85b35314dafe12ba6a497c46d0b2c0bbbc373a2 Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 8 Nov 2016 11:49:13 +0000
Subject: [PATCH 137/149] Fix a routine for single node processor coor from
 rank

---
 lib/communicator/Communicator_none.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index 0f43f1f5..5e91b305 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -97,7 +97,7 @@ void CartesianCommunicator::Barrier(void){}
 void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {}
 void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { }
 int  CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor) {  return 0;}
-void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor){  assert(0);}
+void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor){ coor = _processor_coor ;}
 void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
 {
   source =0;

From 343f3e829f0d8cd4cb5913a150db3039bec7437d Mon Sep 17 00:00:00 2001
From: azusayamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 8 Nov 2016 13:42:12 +0000
Subject: [PATCH 138/149] Fixes prerelease to make all tests

---
 Makefile.am                 |  2 +-
 tests/Makefile.am           |  2 +-
 tests/core/Test_fft_gfix.cc | 14 +++++++-------
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 049220e8..2606b88c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -4,7 +4,7 @@ SUBDIRS = lib benchmarks tests
 .PHONY: tests
 
 tests:
-	make -C tests tests
+	$(MAKE) -C tests tests
 
 AM_CXXFLAGS += -I$(top_builddir)/include
 ACLOCAL_AMFLAGS = -I m4
diff --git a/tests/Makefile.am b/tests/Makefile.am
index c98bc2d0..2e7c1f0a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -9,4 +9,4 @@ endif
 include Make.inc
 
 subtests:
-	for d in $(SUBDIRS); do make -C $${d} tests; done
+	for d in $(SUBDIRS); do $(MAKE) -C $${d} tests; done
diff --git a/tests/core/Test_fft_gfix.cc b/tests/core/Test_fft_gfix.cc
index 6a2868b0..d5779726 100644
--- a/tests/core/Test_fft_gfix.cc
+++ b/tests/core/Test_fft_gfix.cc
@@ -120,7 +120,7 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
     LatticeComplex  Fp(grid);
     LatticeComplex  psq(grid); psq=zero;
     LatticeComplex  pmu(grid); 
-    LatticeComplex   one(grid); one = ComplexD(1.0,0.0);
+    LatticeComplex   one(grid); one = Complex(1.0,0.0);
 
     GaugeMat g(grid);
     GaugeMat dmuAmu_p(grid);
@@ -261,25 +261,25 @@ int main (int argc, char ** argv)
   std::cout<< "* Testing we can gauge fix steep descent a RGT of Unit gauge    *" <<std::endl;
   std::cout<< "*****************************************************************" <<std::endl;
 
-  LatticeGaugeFieldD   Umu(&GRID);
-  LatticeGaugeFieldD   Uorg(&GRID);
-  LatticeColourMatrixD   g(&GRID); // Gauge xform
+  LatticeGaugeField   Umu(&GRID);
+  LatticeGaugeField   Uorg(&GRID);
+  LatticeColourMatrix   g(&GRID); // Gauge xform
 
   
   SU3::ColdConfiguration(pRNG,Umu); // Unit gauge
   Uorg=Umu;
 
   SU3::RandomGaugeTransform(pRNG,Umu,g); // Unit gauge
-  RealD plaq=WilsonLoops<PeriodicGimplD>::avgPlaquette(Umu);
+  RealD plaq=WilsonLoops<PeriodicGimplF>::avgPlaquette(Umu);
   std::cout << " Initial plaquette "<<plaq << std::endl;
 
 
 
   RealD alpha=0.1;
-  FourierAcceleratedGaugeFixer<PeriodicGimplD>::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-10, 1.0e-10);
+  FourierAcceleratedGaugeFixer<PeriodicGimplF>::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-10, 1.0e-10);
 
 
-  plaq=WilsonLoops<PeriodicGimplD>::avgPlaquette(Umu);
+  plaq=WilsonLoops<PeriodicGimplF>::avgPlaquette(Umu);
   std::cout << " Final plaquette "<<plaq << std::endl;
 
   Uorg = Uorg - Umu;

From c5a025d421cb30f9ff114bf23a53b60195427ae6 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 8 Nov 2016 14:07:59 +0000
Subject: [PATCH 139/149] README typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 49f08237..f4a376f1 100644
--- a/README.md
+++ b/README.md
@@ -126,7 +126,7 @@ If you want to build all the tests at once just use `make tests`.
 
 ### Possible communication interfaces
 
-The following options can be use with the `--enable-simd=` option to target different communication interfaces:
+The following options can be use with the `--enable-comms=` option to target different communication interfaces:
 
 | `<comm>`       | Description                                                   |
 | -------------- | ------------------------------------------------------------- |

From f6e1a5b348a107e58e8bb9a94a2323047630a473 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 8 Nov 2016 14:08:33 +0000
Subject: [PATCH 140/149] building tests depends on building the library at the
 top level

---
 Makefile.am | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.am b/Makefile.am
index 2606b88c..18b3ddc3 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3,7 +3,7 @@ SUBDIRS = lib benchmarks tests
 
 .PHONY: tests
 
-tests:
+tests: all
 	$(MAKE) -C tests tests
 
 AM_CXXFLAGS += -I$(top_builddir)/include

From a26adfb0908ebe4c72c977b78e0b803742c06a2e Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 8 Nov 2016 14:11:18 +0000
Subject: [PATCH 141/149] README: only markdown

---
 README | 44 --------------------------------------------
 1 file changed, 44 deletions(-)
 delete mode 100644 README

diff --git a/README b/README
deleted file mode 100644
index 17e92fa0..00000000
--- a/README
+++ /dev/null
@@ -1,44 +0,0 @@
-This library provides data parallel C++ container classes with internal memory layout
-that is transformed to map efficiently to SIMD architectures. CSHIFT facilities
-are provided, similar to HPF and cmfortran, and user control is given over the mapping of
-array indices to both MPI tasks and SIMD processing elements.
-
-* Identically shaped arrays then be processed with perfect data parallelisation.
-* Such identically shapped arrays are called conformable arrays.
-
-The transformation is based on the observation that Cartesian array processing involves
-identical processing to be performed on different regions of the Cartesian array.
-
-The library will (eventually) both geometrically decompose into MPI tasks and across SIMD lanes.
-
-Data parallel array operations can then be specified with a SINGLE data parallel paradigm, but
-optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a significant simplification
-for most programmers.
-
-The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture.
-Presently SSE2 (128 bit) AVX, AVX2 (256 bit) and IMCI and AVX512 (512 bit) targets are supported.
-
-These are presented as 
-
-  vRealF, vRealD, vComplexF, vComplexD 
-
-internal vector data types. These may be useful in themselves for other programmers.
-The corresponding scalar types are named
-
-  RealF, RealD, ComplexF, ComplexD
-
-MPI parallelism is UNIMPLEMENTED and for now only OpenMP and SIMD parallelism is present in the library.
-
-   You can give `configure' initial values for configuration parameters
-by setting variables in the command line or in the environment.  Here
-is are examples:
-
-     ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -msse4" --enable-simd=SSE4
-
-     ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -mavx" --enable-simd=AVX1
-
-     ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -mavx2" --enable-simd=AVX2
-
-     ./configure CXX=icpc CXXFLAGS="-std=c++11 -O3 -mmic" --enable-simd=AVX512 --host=none
-     
-     

From 3d2a22a14d9a74bf3dd7788a41d69adb3759b722 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 8 Nov 2016 15:31:47 +0000
Subject: [PATCH 142/149] include fix for MKL

---
 lib/FFT.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/FFT.h b/lib/FFT.h
index fda43eb8..c1cd1980 100644
--- a/lib/FFT.h
+++ b/lib/FFT.h
@@ -29,9 +29,13 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifndef _GRID_FFT_H_
 #define _GRID_FFT_H_
 
-#ifdef HAVE_FFTW	
+#ifdef HAVE_FFTW
+#ifdef USE_MKL
+#include <fftw/fftw3.h>
+#else
 #include <fftw3.h>
 #endif
+#endif
 
 
 namespace Grid {

From cd0be8cb24f29afe2a09010d7d87524af13aa543 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 8 Nov 2016 15:32:05 +0000
Subject: [PATCH 143/149] Test_fft_gfix.c precision fix

---
 tests/core/Test_fft_gfix.cc | 68 ++++++++++++++++++-------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/tests/core/Test_fft_gfix.cc b/tests/core/Test_fft_gfix.cc
index 6a2868b0..c6b77a13 100644
--- a/tests/core/Test_fft_gfix.cc
+++ b/tests/core/Test_fft_gfix.cc
@@ -42,7 +42,7 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
   static void GaugeLinkToLieAlgebraField(const std::vector<GaugeMat> &U,std::vector<GaugeMat> &A) {
     for(int mu=0;mu<Nd;mu++){
 //      ImplComplex cmi(0.0,-1.0);
-      ComplexD cmi(0.0,-1.0);
+      Complex cmi(0.0,-1.0);
       A[mu] = Ta(U[mu]) * cmi;
     }
   }
@@ -52,13 +52,13 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
       dmuAmu = dmuAmu + A[mu] - Cshift(A[mu],mu,-1);
     }
   }  
-  static void SteepestDescentGaugeFix(GaugeLorentz &Umu,RealD & alpha,int maxiter,RealD Omega_tol, RealD Phi_tol) {
+  static void SteepestDescentGaugeFix(GaugeLorentz &Umu,Real & alpha,int maxiter,Real Omega_tol, Real Phi_tol) {
     GridBase *grid = Umu._grid;
 
-    RealD org_plaq      =WilsonLoops<Gimpl>::avgPlaquette(Umu);
-    RealD org_link_trace=WilsonLoops<Gimpl>::linkTrace(Umu); 
-    RealD old_trace = org_link_trace;
-    RealD trG;
+    Real org_plaq      =WilsonLoops<Gimpl>::avgPlaquette(Umu);
+    Real org_link_trace=WilsonLoops<Gimpl>::linkTrace(Umu); 
+    Real old_trace = org_link_trace;
+    Real trG;
 
     std::vector<GaugeMat> U(Nd,grid);
                  GaugeMat dmuAmu(grid);
@@ -71,13 +71,13 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
       // Monitor progress and convergence test 
       // infrequently to minimise cost overhead
       if ( i %20 == 0 ) { 
-	RealD plaq      =WilsonLoops<Gimpl>::avgPlaquette(Umu);
-	RealD link_trace=WilsonLoops<Gimpl>::linkTrace(Umu); 
+	Real plaq      =WilsonLoops<Gimpl>::avgPlaquette(Umu);
+	Real link_trace=WilsonLoops<Gimpl>::linkTrace(Umu); 
 
 	std::cout << GridLogMessage << " Iteration "<<i<< " plaq= "<<plaq<< " dmuAmu " << norm2(dmuAmu)<< std::endl;
 	
-	RealD Phi  = 1.0 - old_trace / link_trace ;
-	RealD Omega= 1.0 - trG;
+	Real Phi  = 1.0 - old_trace / link_trace ;
+	Real Omega= 1.0 - trG;
 
 
 	std::cout << GridLogMessage << " Iteration "<<i<< " Phi= "<<Phi<< " Omega= " << Omega<< " trG " << trG <<std::endl;
@@ -91,7 +91,7 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
       }
     }
   };
-  static RealD SteepestDescentStep(std::vector<GaugeMat> &U,RealD & alpha, GaugeMat & dmuAmu) {
+  static Real SteepestDescentStep(std::vector<GaugeMat> &U,Real & alpha, GaugeMat & dmuAmu) {
     GridBase *grid = U[0]._grid;
 
     std::vector<GaugeMat> A(Nd,grid);
@@ -101,26 +101,26 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
     ExpiAlphaDmuAmu(A,g,alpha,dmuAmu);
 
 
-    RealD vol = grid->gSites();
-    RealD trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
+    Real vol = grid->gSites();
+    Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
 
     SU<Nc>::GaugeTransform(U,g);
 
     return trG;
   }
 
-  static RealD FourierAccelSteepestDescentStep(std::vector<GaugeMat> &U,RealD & alpha, GaugeMat & dmuAmu) {
+  static Real FourierAccelSteepestDescentStep(std::vector<GaugeMat> &U,Real & alpha, GaugeMat & dmuAmu) {
 
     GridBase *grid = U[0]._grid;
 
-    RealD vol = grid->gSites();
+    Real vol = grid->gSites();
 
     FFT theFFT((GridCartesian *)grid);
 
     LatticeComplex  Fp(grid);
     LatticeComplex  psq(grid); psq=zero;
     LatticeComplex  pmu(grid); 
-    LatticeComplex   one(grid); one = ComplexD(1.0,0.0);
+    LatticeComplex   one(grid); one = Complex(1.0,0.0);
 
     GaugeMat g(grid);
     GaugeMat dmuAmu_p(grid);
@@ -139,13 +139,13 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
     std::vector<int> coor(grid->_ndimension,0);
     for(int mu=0;mu<Nd;mu++) {
 
-      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+      Real TwoPiL =  M_PI * 2.0/ latt_size[mu];
       LatticeCoordinate(pmu,mu);
       pmu = TwoPiL * pmu ;
       psq = psq + 4.0*sin(pmu*0.5)*sin(pmu*0.5); 
     }
 
-    ComplexD psqMax(16.0);
+    Complex psqMax(16.0);
     Fp =  psqMax*one/psq;
 
     static int once;
@@ -160,20 +160,20 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
     theFFT.FFT_all_dim(dmuAmu,dmuAmu_p,FFT::backward);
 
     GaugeMat ciadmam(grid);
-    ComplexD cialpha(0.0,-alpha);
+    Complex cialpha(0.0,-alpha);
     ciadmam = dmuAmu*cialpha;
     SU<Nc>::taExp(ciadmam,g);
 
-    RealD trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
+    Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
 
     SU<Nc>::GaugeTransform(U,g);
 
     return trG;
   }
 
-  static void ExpiAlphaDmuAmu(const std::vector<GaugeMat> &A,GaugeMat &g,RealD & alpha, GaugeMat &dmuAmu) {
+  static void ExpiAlphaDmuAmu(const std::vector<GaugeMat> &A,GaugeMat &g,Real & alpha, GaugeMat &dmuAmu) {
     GridBase *grid = g._grid;
-    ComplexD cialpha(0.0,-alpha);
+    Complex cialpha(0.0,-alpha);
     GaugeMat ciadmam(grid);
     DmuAmu(A,dmuAmu);
     ciadmam = dmuAmu*cialpha;
@@ -193,11 +193,11 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
     ComplexField  pha(grid);
     GaugeMat      Apha(grid);
 
-    ComplexD ci(0.0,1.0);
+    Complex ci(0.0,1.0);
 
     for(int mu=0;mu<Nd;mu++){
 
-      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+      Real TwoPiL =  M_PI * 2.0/ latt_size[mu];
       LatticeCoordinate(pmu,mu);
       pmu = TwoPiL * pmu ;
       pha = exp(pmu *  (0.5 *ci)); // e(ipmu/2) since Amu(x+mu/2)
@@ -213,14 +213,14 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
 
     ComplexField  pmu(grid);
     ComplexField  pha(grid);
-    ComplexD ci(0.0,1.0);
+    Complex ci(0.0,1.0);
     
     // Sign convention for FFTW calls:
     // A(x)= Sum_p e^ipx A(p) / V
     // A(p)= Sum_p e^-ipx A(x)
 
     for(int mu=0;mu<Nd;mu++){
-      RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+      Real TwoPiL =  M_PI * 2.0/ latt_size[mu];
       LatticeCoordinate(pmu,mu);
       pmu = TwoPiL * pmu ;
       pha = exp(-pmu *  (0.5 *ci)); // e(+ipmu/2) since Amu(x+mu/2)
@@ -241,7 +241,7 @@ int main (int argc, char ** argv)
   int threads = GridThread::GetThreads();
 
   std::vector<int> latt_size   = GridDefaultLatt();
-  std::vector<int> simd_layout( { vComplexD::Nsimd(),1,1,1});
+  std::vector<int> simd_layout( { vComplex::Nsimd(),1,1,1});
   std::vector<int> mpi_layout  = GridDefaultMpi();
 
   int vol = 1;
@@ -261,25 +261,25 @@ int main (int argc, char ** argv)
   std::cout<< "* Testing we can gauge fix steep descent a RGT of Unit gauge    *" <<std::endl;
   std::cout<< "*****************************************************************" <<std::endl;
 
-  LatticeGaugeFieldD   Umu(&GRID);
-  LatticeGaugeFieldD   Uorg(&GRID);
-  LatticeColourMatrixD   g(&GRID); // Gauge xform
+  LatticeGaugeField   Umu(&GRID);
+  LatticeGaugeField   Uorg(&GRID);
+  LatticeColourMatrix   g(&GRID); // Gauge xform
 
   
   SU3::ColdConfiguration(pRNG,Umu); // Unit gauge
   Uorg=Umu;
 
   SU3::RandomGaugeTransform(pRNG,Umu,g); // Unit gauge
-  RealD plaq=WilsonLoops<PeriodicGimplD>::avgPlaquette(Umu);
+  Real plaq=WilsonLoops<PeriodicGimplR>::avgPlaquette(Umu);
   std::cout << " Initial plaquette "<<plaq << std::endl;
 
 
 
-  RealD alpha=0.1;
-  FourierAcceleratedGaugeFixer<PeriodicGimplD>::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-10, 1.0e-10);
+  Real alpha=0.1;
+  FourierAcceleratedGaugeFixer<PeriodicGimplR>::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-10, 1.0e-10);
 
 
-  plaq=WilsonLoops<PeriodicGimplD>::avgPlaquette(Umu);
+  plaq=WilsonLoops<PeriodicGimplR>::avgPlaquette(Umu);
   std::cout << " Final plaquette "<<plaq << std::endl;
 
   Uorg = Uorg - Umu;

From 34cf702b24b34379ac5aa8d355fdad83fbf7b78f Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 8 Nov 2016 17:00:38 +0000
Subject: [PATCH 144/149] README is now a symlink to README.md

---
 README | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 README

diff --git a/README b/README
new file mode 120000
index 00000000..42061c01
--- /dev/null
+++ b/README
@@ -0,0 +1 @@
+README.md
\ No newline at end of file

From 9576f0903dd0c9f3c44fbedf6fe5d14bfc22d798 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 8 Nov 2016 19:07:47 +0000
Subject: [PATCH 145/149] namespace fix

---
 lib/PerfCount.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/PerfCount.h b/lib/PerfCount.h
index 9ac58883..5ab07c02 100644
--- a/lib/PerfCount.h
+++ b/lib/PerfCount.h
@@ -43,6 +43,9 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #else
 #include <sys/syscall.h>
 #endif
+#ifdef __x86_64__
+#include <x86intrin.h>
+#endif
 
 namespace Grid {
 
@@ -86,7 +89,6 @@ inline uint64_t cyclecount(void){
    return tmp;
 }
 #elif defined __x86_64__
-#include <x86intrin.h>
 inline uint64_t cyclecount(void){ 
   return __rdtsc();
   //  unsigned int dummy;

From 33dc1f51b51a08a0b79b65276173d8c0b03fc582 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 9 Nov 2016 04:11:03 -0800
Subject: [PATCH 146/149] Final  sign off commits from Cori-1

---
 benchmarks/Benchmark_comms.cc                | 86 ++++++++++++++++++++
 benchmarks/Benchmark_dwf.cc                  | 37 +++++----
 benchmarks/Benchmark_dwf_sweep.cc            |  4 +-
 lib/Init.cc                                  |  2 +-
 lib/Log.cc                                   |  2 +-
 lib/algorithms/iterative/ConjugateGradient.h |  2 +-
 lib/qcd/hmc/HmcRunner.h                      |  2 +-
 tests/hmc/Test_hmc_WilsonFermionGauge.cc     |  2 +-
 8 files changed, 115 insertions(+), 22 deletions(-)

diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index de73bc81..969a2a42 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -193,6 +193,7 @@ int main (int argc, char ** argv)
     }
   }  
 
+
   Nloop=100;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
@@ -271,5 +272,90 @@ int main (int argc, char ** argv)
       std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
+
+
+
+  Nloop=100;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking sequential STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+
+  for(int lat=4;lat<=maxlat;lat+=2){
+    for(int Ls=1;Ls<=16;Ls*=2){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],
+      				    lat*mpi_layout[1],
+      				    lat*mpi_layout[2],
+      				    lat*mpi_layout[3]});
+
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+
+      std::vector<HalfSpinColourVectorD *> xbuf(8);
+      std::vector<HalfSpinColourVectorD *> rbuf(8);
+      Grid.ShmBufferFreeAll();
+      for(int d=0;d<8;d++){
+	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+      }
+
+      int ncomm;
+      int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+
+      double start=usecond();
+      for(int i=0;i<Nloop;i++){
+
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
+
+	ncomm=0;
+	for(int mu=0;mu<4;mu++){
+	
+	  if (mpi_layout[mu]>1 ) {
+	  
+	    ncomm++;
+	    int comm_proc=1;
+	    int xmit_to_rank;
+	    int recv_from_rank;
+	    
+	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    Grid.StencilSendToRecvFromBegin(requests,
+					    (void *)&xbuf[mu][0],
+					    xmit_to_rank,
+					    (void *)&rbuf[mu][0],
+					    recv_from_rank,
+					    bytes);
+	    //	    Grid.StencilSendToRecvFromComplete(requests);
+	    //	    requests.resize(0);
+
+	    comm_proc = mpi_layout[mu]-1;
+	  
+	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    Grid.StencilSendToRecvFromBegin(requests,
+					    (void *)&xbuf[mu+4][0],
+					    xmit_to_rank,
+					    (void *)&rbuf[mu+4][0],
+					    recv_from_rank,
+					    bytes);
+	    Grid.StencilSendToRecvFromComplete(requests);
+	    requests.resize(0);
+	  
+	  }
+	}
+	Grid.Barrier();
+
+      }
+      double stop=usecond();
+
+      double dbytes    = bytes;
+      double xbytes    = Nloop*dbytes*2.0*ncomm;
+      double rbytes    = xbytes;
+      double bidibytes = xbytes+rbytes;
+
+      double time = stop-start; // microseconds
+
+      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+    }
+  }    
+
   Grid_finalize();
 }
diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index 5d42a1f9..10e4521b 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -57,7 +57,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
   std::vector<int> latt4 = GridDefaultLatt();
-  const int Ls=16;
+  const int Ls=8;
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
@@ -138,7 +138,7 @@ int main (int argc, char ** argv)
 
   int ncall =100;
   if (1) {
-
+    FGrid->Barrier();
     Dw.ZeroCounters();
     double t0=usecond();
     for(int i=0;i<ncall;i++){
@@ -147,6 +147,7 @@ int main (int argc, char ** argv)
       __SSC_STOP;
     }
     double t1=usecond();
+    FGrid->Barrier();
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=1344*volume*ncall;
@@ -158,7 +159,7 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-    assert (norm2(err)< 1.0e-5 );
+    assert (norm2(err)< 1.0e-4 );
     Dw.Report();
   }
 
@@ -193,6 +194,7 @@ int main (int argc, char ** argv)
       pokeSite(tmp,ssrc,site);
     }}}}}
     std::cout<<GridLogMessage<< "src norms "<< norm2(src)<<" " <<norm2(ssrc)<<std::endl;
+    FGrid->Barrier();
     double t0=usecond();
     sDw.ZeroCounters();
     for(int i=0;i<ncall;i++){
@@ -201,6 +203,7 @@ int main (int argc, char ** argv)
       __SSC_STOP;
     }
     double t1=usecond();
+    FGrid->Barrier();
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=1344*volume*ncall;
 
@@ -211,12 +214,12 @@ int main (int argc, char ** argv)
   
     if(0){
       for(int i=0;i< PerformanceCounter::NumTypes(); i++ ){
-  sDw.Dhop(ssrc,sresult,0);
-  PerformanceCounter Counter(i);
-  Counter.Start();
-  sDw.Dhop(ssrc,sresult,0);
-  Counter.Stop();
-  Counter.Report();
+	sDw.Dhop(ssrc,sresult,0);
+	PerformanceCounter Counter(i);
+	Counter.Start();
+	sDw.Dhop(ssrc,sresult,0);
+	Counter.Stop();
+	Counter.Report();
       }
     }
 
@@ -240,7 +243,7 @@ int main (int argc, char ** argv)
       }
     }}}}}
     std::cout<<GridLogMessage<<" difference between normal and simd is "<<sum<<std::endl;
-    assert (sum< 1.0e-5 );
+    assert (sum< 1.0e-4 );
 
 
     if (1) {
@@ -271,6 +274,7 @@ int main (int argc, char ** argv)
       if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
       std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
 
+      FGrid->Barrier();
       sDw.ZeroCounters();
       sDw.stat.init("DhopEO");
       double t0=usecond();
@@ -278,6 +282,7 @@ int main (int argc, char ** argv)
         sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
       }
       double t1=usecond();
+      FGrid->Barrier();
       sDw.stat.print();
 
       double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
@@ -301,7 +306,7 @@ int main (int argc, char ** argv)
 
       error+= norm2(ssrc_o);
       std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm"<<norm2(sr_o) <<std::endl;
-      if(error>1.0e-5) { 
+      if(error>1.0e-4) { 
 	setCheckerboard(ssrc,ssrc_o);
 	setCheckerboard(ssrc,ssrc_e);
 	std::cout<< ssrc << std::endl;
@@ -337,7 +342,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
   err = ref-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-  assert(norm2(err)<1.0e-5);
+  assert(norm2(err)<1.0e-4);
   LatticeFermion src_e (FrbGrid);
   LatticeFermion src_o (FrbGrid);
   LatticeFermion r_e   (FrbGrid);
@@ -363,11 +368,13 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
   {
     Dw.ZeroCounters();
+    FGrid->Barrier();
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       Dw.DhopEO(src_o,r_e,DaggerNo);
     }
     double t1=usecond();
+    FGrid->Barrier();
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=(1344.0*volume*ncall)/2;
@@ -389,14 +396,14 @@ int main (int argc, char ** argv)
 
   err = r_eo-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-  assert(norm2(err)<1.0e-5);
+  assert(norm2(err)<1.0e-4);
 
   pickCheckerboard(Even,src_e,err);
   pickCheckerboard(Odd,src_o,err);
   std::cout<<GridLogMessage << "norm diff even  "<< norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "norm diff odd   "<< norm2(src_o)<<std::endl;
-  assert(norm2(src_e)<1.0e-5);
-  assert(norm2(src_o)<1.0e-5);
+  assert(norm2(src_e)<1.0e-4);
+  assert(norm2(src_o)<1.0e-4);
 
   Grid_finalize();
 }
diff --git a/benchmarks/Benchmark_dwf_sweep.cc b/benchmarks/Benchmark_dwf_sweep.cc
index a3803b87..21db823d 100644
--- a/benchmarks/Benchmark_dwf_sweep.cc
+++ b/benchmarks/Benchmark_dwf_sweep.cc
@@ -69,8 +69,8 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "Volume \t\t\tProcs \t Dw \t eoDw \t sDw \t eosDw (Mflop/s)  "<<std::endl;
   std::cout<<GridLogMessage << "=========================================================================="<<std::endl;
 
-  int Lmax=32;
-  int dmin=0;
+  int Lmax=16;
+  int dmin=2;
   if ( getenv("LMAX") ) Lmax=atoi(getenv("LMAX"));
   if ( getenv("DMIN") ) dmin=atoi(getenv("DMIN"));
   for (int L=8;L<=Lmax;L*=2){
diff --git a/lib/Init.cc b/lib/Init.cc
index b5ef0a95..d6d6b9f8 100644
--- a/lib/Init.cc
+++ b/lib/Init.cc
@@ -369,7 +369,7 @@ void Grid_init(int *argc,char ***argv)
   
 void Grid_finalize(void)
 {
-#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3)
+#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) 
   MPI_Finalize();
   Grid_unquiesce_nodes();
 #endif
diff --git a/lib/Log.cc b/lib/Log.cc
index 733bfc58..7521657b 100644
--- a/lib/Log.cc
+++ b/lib/Log.cc
@@ -93,7 +93,7 @@ void GridLogConfigure(std::vector<std::string> &logstreams) {
 ////////////////////////////////////////////////////////////
 void Grid_quiesce_nodes(void) {
   int me = 0;
-#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3)
+#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPI3L)
   MPI_Comm_rank(MPI_COMM_WORLD, &me);
 #endif
 #ifdef GRID_COMMS_SHMEM
diff --git a/lib/algorithms/iterative/ConjugateGradient.h b/lib/algorithms/iterative/ConjugateGradient.h
index f340eb38..cf3872c8 100644
--- a/lib/algorithms/iterative/ConjugateGradient.h
+++ b/lib/algorithms/iterative/ConjugateGradient.h
@@ -154,7 +154,7 @@ class ConjugateGradient : public OperatorFunction<Field> {
                   << LinalgTimer.Elapsed();
         std::cout << std::endl;
 
-        if (ErrorOnNoConverge) assert(true_residual / Tolerance < 1000.0);
+        if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
 
         return;
       }
diff --git a/lib/qcd/hmc/HmcRunner.h b/lib/qcd/hmc/HmcRunner.h
index a31ba784..53b127cf 100644
--- a/lib/qcd/hmc/HmcRunner.h
+++ b/lib/qcd/hmc/HmcRunner.h
@@ -116,7 +116,7 @@ class NerscHmcRunnerTemplate {
     NoSmearing<Gimpl> SmearingPolicy;
     typedef MinimumNorm2<GaugeField, NoSmearing<Gimpl>, RepresentationsPolicy >
         IntegratorType;  // change here to change the algorithm
-    IntegratorParameters MDpar(20, 1.0);
+    IntegratorParameters MDpar(40, 1.0);
     IntegratorType MDynamics(UGrid, MDpar, TheAction, SmearingPolicy);
 
     // Checkpoint strategy
diff --git a/tests/hmc/Test_hmc_WilsonFermionGauge.cc b/tests/hmc/Test_hmc_WilsonFermionGauge.cc
index 9dcf6343..351d1e68 100644
--- a/tests/hmc/Test_hmc_WilsonFermionGauge.cc
+++ b/tests/hmc/Test_hmc_WilsonFermionGauge.cc
@@ -68,7 +68,7 @@ class HmcRunner : public NerscHmcRunner {
     TwoFlavourPseudoFermionAction<ImplPolicy> Nf2(FermOp, CG, CG);
 
     // Set smearing (true/false), default: false
-    Nf2.is_smeared = true;
+    Nf2.is_smeared = false;
 
     // Collect actions
     ActionLevel<LatticeGaugeField> Level1(1);

From 042ae5b87c0534348984e87a55c3cdbced132405 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 15 Nov 2016 12:15:33 +0000
Subject: [PATCH 147/149] generic 256bits SIMD

---
 benchmarks/simple_su3_test.cc |   2 +-
 configure.ac                  |   8 +-
 lib/simd/Grid_generic.h       | 496 --------------------------
 lib/simd/Grid_generic_256.h   | 644 ++++++++++++++++++++++++++++++++++
 lib/simd/Grid_vector_types.h  |   4 +-
 5 files changed, 651 insertions(+), 503 deletions(-)
 create mode 100644 lib/simd/Grid_generic_256.h

diff --git a/benchmarks/simple_su3_test.cc b/benchmarks/simple_su3_test.cc
index e5cca98d..79437c23 100644
--- a/benchmarks/simple_su3_test.cc
+++ b/benchmarks/simple_su3_test.cc
@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid.h>
+#include <Grid/Grid.h>
 
 using namespace std;
 using namespace Grid;
diff --git a/configure.ac b/configure.ac
index 9178a11d..fb80b1ae 100644
--- a/configure.ac
+++ b/configure.ac
@@ -179,8 +179,8 @@ case ${ax_cv_cxx_compiler_vendor} in
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-march=knl';;
-      GEN)
-        AC_DEFINE([GENERIC_VEC],[1],[generic vector code])
+      GEN256)
+        AC_DEFINE([GEN256],[1],[generic vector code])
         SIMD_FLAGS='';;
       QPX|BGQ)
         AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q])
@@ -211,8 +211,8 @@ case ${ax_cv_cxx_compiler_vendor} in
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing])
         SIMD_FLAGS='-xmic-avx512';;
-      GEN)
-        AC_DEFINE([GENERIC_VEC],[1],[generic vector code])
+      GEN256)
+        AC_DEFINE([GEN256],[1],[generic vector code])
         SIMD_FLAGS='';;
       *)
         AC_MSG_ERROR(["SIMD option ${ac_SIMD} not supported by the Intel compiler"]);;
diff --git a/lib/simd/Grid_generic.h b/lib/simd/Grid_generic.h
index ca0029ef..e69de29b 100644
--- a/lib/simd/Grid_generic.h
+++ b/lib/simd/Grid_generic.h
@@ -1,496 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/simd/Grid_generic.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: neo <cossu@post.kek.jp>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-
-namespace Grid {
-namespace Optimization {
-
-  template<class vtype>
-  union uconv {
-    float f;
-    vtype v;
-  };
-
-  union u128f {
-    float v;
-    float f[4];
-  };
-  union u128d {
-    double v;
-    double f[2];
-  };
-  
-  struct Vsplat{
-    //Complex float
-    inline u128f operator()(float a, float b){
-      u128f out; 
-      out.f[0] = a;
-      out.f[1] = b;
-      out.f[2] = a;
-      out.f[3] = b;
-      return out;
-    }
-    // Real float
-    inline u128f operator()(float a){
-      u128f out; 
-      out.f[0] = a;
-      out.f[1] = a;
-      out.f[2] = a;
-      out.f[3] = a;
-      return out;
-    }
-    //Complex double
-    inline u128d operator()(double a, double b){
-      u128d out; 
-      out.f[0] = a;
-      out.f[1] = b;
-      return out;
-    }
-    //Real double
-    inline u128d operator()(double a){
-      u128d out; 
-      out.f[0] = a;
-      out.f[1] = a;
-      return out;
-    }
-    //Integer
-    inline int operator()(Integer a){
-      return a;
-    }
-  };
-
-  struct Vstore{
-    //Float 
-    inline void operator()(u128f a, float* F){
-      memcpy(F,a.f,4*sizeof(float));
-    }
-    //Double
-    inline void operator()(u128d a, double* D){
-      memcpy(D,a.f,2*sizeof(double));
-    }
-    //Integer
-    inline void operator()(int a, Integer* I){
-      I[0] = a;
-    }
-
-  };
-
-  struct Vstream{
-    //Float
-    inline void operator()(float * a, u128f b){
-      memcpy(a,b.f,4*sizeof(float));
-    }
-    //Double
-    inline void operator()(double * a, u128d b){
-      memcpy(a,b.f,2*sizeof(double));
-    }
-
-
-  };
-
-  struct Vset{
-    // Complex float 
-    inline u128f operator()(Grid::ComplexF *a){
-      u128f out; 
-      out.f[0] = a[0].real();
-      out.f[1] = a[0].imag();
-      out.f[2] = a[1].real();
-      out.f[3] = a[1].imag();
-      return out;
-    }
-    // Complex double 
-    inline u128d operator()(Grid::ComplexD *a){
-      u128d out; 
-      out.f[0] = a[0].real();
-      out.f[1] = a[0].imag();
-      return out;
-    }
-    // Real float 
-    inline u128f operator()(float *a){
-      u128f out; 
-      out.f[0] = a[0];
-      out.f[1] = a[1];
-      out.f[2] = a[2];
-      out.f[3] = a[3];
-      return out;
-    }
-    // Real double
-    inline u128d operator()(double *a){
-      u128d out; 
-      out.f[0] = a[0];
-      out.f[1] = a[1];
-      return out;
-    }
-    // Integer
-    inline int operator()(Integer *a){
-      return a[0];
-    }
-
-
-  };
-
-  template <typename Out_type, typename In_type>
-  struct Reduce{
-    //Need templated class to overload output type
-    //General form must generate error if compiled
-    inline Out_type operator()(In_type in){
-      printf("Error, using wrong Reduce function\n");
-      exit(1);
-      return 0;
-    }
-  };
-
-  /////////////////////////////////////////////////////
-  // Arithmetic operations
-  /////////////////////////////////////////////////////
-  struct Sum{
-    //Complex/Real float
-    inline u128f operator()(u128f a, u128f b){
-      u128f out;
-      out.f[0] = a.f[0] + b.f[0];
-      out.f[1] = a.f[1] + b.f[1];
-      out.f[2] = a.f[2] + b.f[2];
-      out.f[3] = a.f[3] + b.f[3];
-      return out;
-    }
-    //Complex/Real double
-    inline u128d operator()(u128d a, u128d b){
-      u128d out;
-      out.f[0] = a.f[0] + b.f[0];
-      out.f[1] = a.f[1] + b.f[1];
-      return out;
-    }
-    //Integer
-    inline int operator()(int a, int b){
-      return a + b;
-    }
-  };
-
-  struct Sub{
-    //Complex/Real float
-    inline u128f operator()(u128f a, u128f b){
-      u128f out;
-      out.f[0] = a.f[0] - b.f[0];
-      out.f[1] = a.f[1] - b.f[1];
-      out.f[2] = a.f[2] - b.f[2];
-      out.f[3] = a.f[3] - b.f[3];
-      return out;
-    }
-    //Complex/Real double
-    inline u128d operator()(u128d a, u128d b){
-      u128d out;
-      out.f[0] = a.f[0] - b.f[0];
-      out.f[1] = a.f[1] - b.f[1];
-      return out;
-    }
-    //Integer
-    inline int operator()(int a, int b){
-      return a-b;
-    }
-  };
-
-  struct MultComplex{
-    // Complex float
-    inline u128f operator()(u128f a, u128f b){
-      u128f out;
-      out.f[0] = a.f[0]*b.f[0] - a.f[1]*b.f[1];
-      out.f[1] = a.f[0]*b.f[1] + a.f[1]*b.f[0];
-      out.f[2] = a.f[2]*b.f[2] - a.f[3]*b.f[3];
-      out.f[3] = a.f[2]*b.f[3] + a.f[3]*b.f[2];
-      return out;
-    }
-    // Complex double
-    inline u128d operator()(u128d a, u128d b){
-      u128d out;
-      out.f[0] = a.f[0]*b.f[0] - a.f[1]*b.f[1];
-      out.f[1] = a.f[0]*b.f[1] + a.f[1]*b.f[0];
-      return out;
-    }
-  };
-
-  struct Mult{
-    //CK: Appear unneeded
-    // inline float  mac(float a, float b,double c){
-    //   return 0;
-    // }
-    // inline double mac(double a, double b,double c){
-    //   return 0;
-    // }
-
-    // Real float
-    inline u128f operator()(u128f a, u128f b){
-      u128f out;
-      out.f[0] = a.f[0]*b.f[0];
-      out.f[1] = a.f[1]*b.f[1];
-      out.f[2] = a.f[2]*b.f[2];
-      out.f[3] = a.f[3]*b.f[3];
-      return out;
-    }
-    // Real double
-    inline u128d operator()(u128d a, u128d b){
-      u128d out;
-      out.f[0] = a.f[0]*b.f[0];
-      out.f[1] = a.f[1]*b.f[1];
-      return out;
-    }
-    // Integer
-    inline int operator()(int a, int b){
-      return a*b;
-    }
-  };
-
-  struct Conj{
-    // Complex single
-    inline u128f operator()(u128f in){
-      u128f out;
-      out.f[0] = in.f[0];
-      out.f[1] = -in.f[1];
-      out.f[2] = in.f[2];
-      out.f[3] = -in.f[3];
-      return out;
-    }
-    // Complex double
-    inline u128d operator()(u128d in){
-      u128d out;
-      out.f[0] = in.f[0];
-      out.f[1] = -in.f[1];
-      return out;
-    }
-    // do not define for integer input
-  };
-
-  struct TimesMinusI{
-    //Complex single
-    inline u128f operator()(u128f in, u128f ret){ //note ret is ignored
-      u128f out;
-      out.f[0] = in.f[1];
-      out.f[1] = -in.f[0];
-      out.f[2] = in.f[3];
-      out.f[3] = -in.f[2];
-      return out;
-    }
-    //Complex double
-    inline u128d operator()(u128d in, u128d ret){
-      u128d out;
-      out.f[0] = in.f[1];
-      out.f[1] = -in.f[0];
-      return out;
-    }
-  };
-
-  struct TimesI{
-    //Complex single
-    inline u128f operator()(u128f in, u128f ret){ //note ret is ignored
-      u128f out;
-      out.f[0] = -in.f[1];
-      out.f[1] = in.f[0];
-      out.f[2] = -in.f[3];
-      out.f[3] = in.f[2];
-      return out;
-    }
-    //Complex double
-    inline u128d operator()(u128d in, u128d ret){
-      u128d out;
-      out.f[0] = -in.f[1];
-      out.f[1] = in.f[0];
-      return out;
-    }
-  };
-
-  //////////////////////////////////////////////
-  // Some Template specialization
-  struct Permute{
-    //We just have to mirror the permutes of Grid_sse4.h
-    static inline u128f Permute0(u128f in){ //AB CD -> CD AB
-      u128f out;
-      out.f[0] = in.f[2];
-      out.f[1] = in.f[3];
-      out.f[2] = in.f[0];
-      out.f[3] = in.f[1];
-      return out;
-    };
-    static inline u128f Permute1(u128f in){ //AB CD -> BA DC
-      u128f out;
-      out.f[0] = in.f[1];
-      out.f[1] = in.f[0];
-      out.f[2] = in.f[3];
-      out.f[3] = in.f[2];
-      return out;
-    };
-    static inline u128f Permute2(u128f in){
-      return in;
-    };
-    static inline u128f Permute3(u128f in){
-      return in;
-    };
-
-    static inline u128d Permute0(u128d in){ //AB -> BA
-      u128d out;
-      out.f[0] = in.f[1];
-      out.f[1] = in.f[0];
-      return out;      
-    };
-    static inline u128d Permute1(u128d in){
-      return in;
-    };
-    static inline u128d Permute2(u128d in){
-      return in;
-    };
-    static inline u128d Permute3(u128d in){
-      return in;
-    };
-
-  };
-  
-  template < typename vtype > 
-    void permute(vtype &a, vtype b, int perm) {
-   };
-    
-  struct Rotate{
-
-    static inline u128f rotate(u128f in,int n){
-      u128f out;
-      switch(n){
-      case 0:
-        out.f[0] = in.f[0];
-        out.f[1] = in.f[1];
-        out.f[2] = in.f[2];
-        out.f[3] = in.f[3];
-        break;
-      case 1:
-        out.f[0] = in.f[1];
-        out.f[1] = in.f[2];
-        out.f[2] = in.f[3];
-        out.f[3] = in.f[0];
-        break;
-      case 2:
-        out.f[0] = in.f[2];
-        out.f[1] = in.f[3];
-        out.f[2] = in.f[0];
-        out.f[3] = in.f[1];
-        break;
-      case 3:
-        out.f[0] = in.f[3];
-        out.f[1] = in.f[0];
-        out.f[2] = in.f[1];
-        out.f[3] = in.f[2];
-        break;
-      default: assert(0);
-      }
-      return out;
-    }
-    static inline u128d rotate(u128d in,int n){
-      u128d out;
-      switch(n){
-      case 0:
-        out.f[0] = in.f[0];
-        out.f[1] = in.f[1];
-        break;
-      case 1:
-        out.f[0] = in.f[1];
-        out.f[1] = in.f[0];
-        break;
-      default: assert(0);
-      }
-      return out;
-    }
-  };
-
-  //Complex float Reduce
-  template<>
-  inline Grid::ComplexF Reduce<Grid::ComplexF, u128f>::operator()(u128f in){ //2 complex
-    return Grid::ComplexF(in.f[0] + in.f[2], in.f[1] + in.f[3]);
-  }
-  //Real float Reduce
-  template<>
-  inline Grid::RealF Reduce<Grid::RealF, u128f>::operator()(u128f in){ //4 floats
-    return in.f[0] + in.f[1] + in.f[2] + in.f[3];
-  }
-  
-  
-  //Complex double Reduce
-  template<>
-  inline Grid::ComplexD Reduce<Grid::ComplexD, u128d>::operator()(u128d in){ //1 complex
-    return Grid::ComplexD(in.f[0],in.f[1]);
-  }
-  
-  //Real double Reduce
-  template<>
-  inline Grid::RealD Reduce<Grid::RealD, u128d>::operator()(u128d in){ //2 doubles
-    return in.f[0] + in.f[1];
-  }
-
-  //Integer Reduce
-  template<>
-  inline Integer Reduce<Integer, int>::operator()(int in){
-    // FIXME unimplemented
-   printf("Reduce : Missing integer implementation -> FIX\n");
-    assert(0);
-  }
-}
-
-//////////////////////////////////////////////////////////////////////////////////////
-// Here assign types 
-
-  typedef Optimization::u128f SIMD_Ftype;  // Single precision type
-  typedef Optimization::u128d SIMD_Dtype; // Double precision type
-  typedef int SIMD_Itype; // Integer type
-
-  // prefetch utilities
-  inline void v_prefetch0(int size, const char *ptr){};
-  inline void prefetch_HINT_T0(const char *ptr){};
-
-
-
-  // Gpermute function
-  template < typename VectorSIMD > 
-    inline void Gpermute(VectorSIMD &y,const VectorSIMD &b, int perm ) {
-    Optimization::permute(y.v,b.v,perm);
-  }
-
-
-  // Function name aliases
-  typedef Optimization::Vsplat   VsplatSIMD;
-  typedef Optimization::Vstore   VstoreSIMD;
-  typedef Optimization::Vset     VsetSIMD;
-  typedef Optimization::Vstream  VstreamSIMD;
-  template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
-
- 
-
-
-  // Arithmetic operations
-  typedef Optimization::Sum         SumSIMD;
-  typedef Optimization::Sub         SubSIMD;
-  typedef Optimization::Mult        MultSIMD;
-  typedef Optimization::MultComplex MultComplexSIMD;
-  typedef Optimization::Conj        ConjSIMD;
-  typedef Optimization::TimesMinusI TimesMinusISIMD;
-  typedef Optimization::TimesI      TimesISIMD;
-
-}
diff --git a/lib/simd/Grid_generic_256.h b/lib/simd/Grid_generic_256.h
new file mode 100644
index 00000000..42df6cf3
--- /dev/null
+++ b/lib/simd/Grid_generic_256.h
@@ -0,0 +1,644 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/simd/Grid_generic.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+
+#ifndef GEN_SIMD_WIDTH
+#define GEN_SIMD_DCOMPLEX_WIDTH 2
+#endif
+
+#include "Grid_generic.h"
+
+namespace Grid {
+namespace Optimization {
+
+  constexpr unsigned int dcw = GEN_SIMD_DCOMPLEX_WIDTH;
+  constexpr unsigned int fcw = 2*dcw;
+  constexpr unsigned int dw  = 2*dcw;
+  constexpr unsigned int fw  = 2*fcw;
+
+  struct vecf {
+    float v[fw];
+  };
+  
+  struct vecd {
+    double v[dw];
+  };
+  
+  struct Vsplat{
+    //Complex float
+    inline vecf operator()(float a, float b){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fw; i += 2)
+      {
+        out.v[i]   = a;
+        out.v[i+1] = b;
+      }
+
+      return out;
+    }
+    
+    // Real float
+    inline vecf operator()(float a){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fw; ++i)
+      {
+        out.v[i] = a;
+      }
+      
+      return out;
+    }
+    
+    //Complex double
+    inline vecd operator()(double a, double b){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dw; i += 2)
+      {
+        out.v[i]   = a;
+        out.v[i+1] = b;
+      }
+      
+      return out;
+    }
+    
+    //Real double
+    inline vecd operator()(double a){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dw; ++i)
+      {
+        out.v[i] = a;
+      }
+      
+      return out;
+    }
+    
+    //Integer
+    inline int operator()(Integer a){
+      return a;
+    }
+  };
+
+  struct Vstore{
+    //Float 
+    inline void operator()(vecf a, float* F){
+      memcpy(F,a.v,fw*sizeof(float));
+    }
+    //Double
+    inline void operator()(vecd a, double* D){
+      memcpy(D,a.v,dw*sizeof(double));
+    }
+    //Integer
+    inline void operator()(int a, Integer* I){
+      I[0] = a;
+    }
+
+  };
+
+  struct Vstream{
+    //Float
+    inline void operator()(float * a, vecf b){
+      memcpy(a,b.v,fw*sizeof(float));
+    }
+    //Double
+    inline void operator()(double * a, vecd b){
+      memcpy(a,b.v,dw*sizeof(double));
+    }
+
+
+  };
+
+  struct Vset{
+    // Complex float 
+    inline vecf operator()(Grid::ComplexF *a){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fcw; ++i)
+      {
+        out.v[2*i]   = a[i].real();
+        out.v[2*i+1] = a[i].imag();
+      }
+      
+      return out;
+    }
+    
+    // Complex double 
+    inline vecd operator()(Grid::ComplexD *a){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dcw; ++i)
+      {
+        out.v[2*i]   = a[i].real();
+        out.v[2*i+1] = a[i].imag();
+      }
+      
+      return out;
+    }
+    
+    // Real float 
+    inline vecf operator()(float *a){
+      vecf out;
+      
+      memcpy(out.v,a,fw*sizeof(float));
+      
+      return out;
+    }
+    // Real double
+    inline vecd operator()(double *a){
+      vecd out; 
+      
+      memcpy(out.v,a,dw*sizeof(float));
+      
+      return out;
+    }
+    // Integer
+    inline int operator()(Integer *a){
+      return a[0];
+    }
+
+
+  };
+
+  /////////////////////////////////////////////////////
+  // Arithmetic operations
+  /////////////////////////////////////////////////////
+  struct Sum{
+    //Complex/Real float
+    inline vecf operator()(vecf a, vecf b){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fw; ++i)
+      {
+        out.v[i] = a.v[i] + b.v[i];
+      }
+      
+      return out;
+    }
+    
+    //Complex/Real double
+    inline vecd operator()(vecd a, vecd b){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dw; ++i)
+      {
+        out.v[i] = a.v[i] + b.v[i];
+      }
+
+      return out;
+    }
+    
+    //Integer
+    inline int operator()(int a, int b){
+      return a + b;
+    }
+  };
+
+  struct Sub{
+    //Complex/Real float
+    inline vecf operator()(vecf a, vecf b){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fw; ++i)
+      {
+        out.v[i] = a.v[i] - b.v[i];
+      }
+      
+      return out;
+    }
+    
+    //Complex/Real double
+    inline vecd operator()(vecd a, vecd b){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dw; ++i)
+      {
+        out.v[i] = a.v[i] - b.v[i];
+      }
+      
+      return out;
+    }
+    
+    //Integer
+    inline int operator()(int a, int b){
+      return a-b;
+    }
+  };
+
+  #define cmul(a, b, c, i)\
+  c[i]   = a[i]*b[i]   - a[i+1]*b[i+1];\
+  c[i+1] = a[i]*b[i+1] + a[i+1]*b[i];
+  
+  struct MultComplex{
+    // Complex float
+    inline vecf operator()(vecf a, vecf b){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fcw; ++i)
+      {
+        cmul(a.v, b.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+    
+    // Complex double
+    inline vecd operator()(vecd a, vecd b){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dcw; ++i)
+      {
+        cmul(a.v, b.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+  };
+  
+  #undef cmul
+
+  struct Mult{
+    // Real float
+    inline vecf operator()(vecf a, vecf b){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fw; ++i)
+      {
+        out.v[i] = a.v[i]*b.v[i];
+      }
+      
+      return out;
+    }
+    
+    // Real double
+    inline vecd operator()(vecd a, vecd b){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dw; ++i)
+      {
+        out.v[i] = a.v[i]*b.v[i];
+      }
+      
+      return out;
+    }
+    
+    // Integer
+    inline int operator()(int a, int b){
+      return a*b;
+    }
+  };
+
+  struct Div{
+    // Real float
+    inline vecf operator()(vecf a, vecf b){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fw; ++i)
+      {
+        out.v[i] = a.v[i]/b.v[i];
+      }
+      
+      return out;
+    }
+    // Real double
+    inline vecd operator()(vecd a, vecd b){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dw; ++i)
+      {
+        out.v[i] = a.v[i]/b.v[i];
+      }
+      
+      return out;
+    }
+  };
+  
+  #define conj(a, b, i)\
+  b[i]   = a[i];\
+  b[i+1] = -a[i+1];
+  
+  struct Conj{
+    // Complex single
+    inline vecf operator()(vecf in){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fcw; ++i)
+      {
+        conj(in.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+    
+    // Complex double
+    inline vecd operator()(vecd in){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dcw; ++i)
+      {
+        conj(in.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+  };
+  
+  #undef conj
+
+  #define timesmi(a, b, i)\
+  b[i]   = a[i+1];\
+  b[i+1] = -a[i];
+  
+  struct TimesMinusI{
+    // Complex single
+    inline vecf operator()(vecf in, vecf ret){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fcw; ++i)
+      {
+        timesmi(in.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+    
+    // Complex double
+    inline vecd operator()(vecd in, vecd ret){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dcw; ++i)
+      {
+        timesmi(in.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+  };
+
+  #undef timesmi
+  
+  #define timespi(a, b, i)\
+  b[i]   = -a[i+1];\
+  b[i+1] = a[i];
+  
+  struct TimesI{
+    // Complex single
+    inline vecf operator()(vecf in, vecf ret){
+      vecf out;
+      
+      for (unsigned int i = 0; i < fcw; ++i)
+      {
+        timespi(in.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+    
+    // Complex double
+    inline vecd operator()(vecd in, vecd ret){
+      vecd out;
+      
+      for (unsigned int i = 0; i < dcw; ++i)
+      {
+        timespi(in.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+  };
+  
+  #undef timespi
+
+  //////////////////////////////////////////////
+  // Some Template specialization
+  struct Permute{
+    static inline vecf Permute0(vecf in){ //AB CD -> CD AB
+      vecf out;
+      
+      out.v[0] = in.v[4];
+      out.v[1] = in.v[5];
+      out.v[2] = in.v[6];
+      out.v[3] = in.v[7];
+      out.v[4] = in.v[0];
+      out.v[5] = in.v[1];
+      out.v[6] = in.v[2];
+      out.v[7] = in.v[3];
+      
+      return out;
+    };
+    
+    static inline vecf Permute1(vecf in){ //AB CD -> BA DC
+      vecf out;
+      
+      out.v[0] = in.v[2];
+      out.v[1] = in.v[3];
+      out.v[2] = in.v[0];
+      out.v[3] = in.v[1];
+      out.v[4] = in.v[6];
+      out.v[5] = in.v[7];
+      out.v[6] = in.v[4];
+      out.v[7] = in.v[5];
+      
+      return out;
+    };
+    
+    static inline vecf Permute2(vecf in){
+      vecf out;
+      
+      out.v[0] = in.v[1];
+      out.v[1] = in.v[0];
+      out.v[2] = in.v[3];
+      out.v[3] = in.v[2];
+      out.v[4] = in.v[5];
+      out.v[5] = in.v[4];
+      out.v[6] = in.v[7];
+      out.v[7] = in.v[6];
+      
+      return out;
+    };
+
+    static inline vecf Permute3(vecf in){
+      return in;
+    };
+
+    static inline vecd Permute0(vecd in){ //AB -> BA
+      vecd out;
+      
+      out.v[0] = in.v[2];
+      out.v[1] = in.v[3];
+      out.v[2] = in.v[0];
+      out.v[3] = in.v[1];
+      
+      return out;
+    };
+    
+    static inline vecd Permute1(vecd in){
+      vecd out;
+      
+      out.v[0] = in.v[1];
+      out.v[1] = in.v[0];
+      out.v[2] = in.v[3];
+      out.v[3] = in.v[2];
+      
+      return out;
+    };
+    
+    static inline vecd Permute2(vecd in){
+      return in;
+    };
+    
+    static inline vecd Permute3(vecd in){
+      return in;
+    };
+
+  };
+  
+  #define rot(a, b, n, w)\
+  for (unsigned int i = 0; i < w; ++i)\
+  {\
+    b[i] = a[(i + n)%w];\
+  }
+  
+  struct Rotate{
+
+    static inline vecf rotate(vecf in, int n){
+      vecf out;
+      
+      rot(in.v, out.v, n, fw);
+      
+      return out;
+    }
+    
+    static inline vecd rotate(vecd in,int n){
+      vecd out;
+      
+      rot(in.v, out.v, n, dw);
+      
+      return out;
+    }
+  };
+
+  #undef rot
+  
+  #define acc(v, a, off, step, n)\
+  for (unsigned int i = off; i < n; i += step)\
+  {\
+    a += v[i];\
+  }
+  
+  template <typename Out_type, typename In_type>
+  struct Reduce{
+    //Need templated class to overload output type
+    //General form must generate error if compiled
+    inline Out_type operator()(In_type in){
+      printf("Error, using wrong Reduce function\n");
+      exit(1);
+      return 0;
+    }
+  };
+  
+  //Complex float Reduce
+  template<>
+  inline Grid::ComplexF Reduce<Grid::ComplexF, vecf>::operator()(vecf in){
+    float a = 0.f, b = 0.f;
+    
+    acc(in.v, a, 0, 2, fw);
+    acc(in.v, b, 1, 2, fw);
+    
+    return Grid::ComplexF(a, b);
+  }
+  
+  //Real float Reduce
+  template<>
+  inline Grid::RealF Reduce<Grid::RealF, vecf>::operator()(vecf in){
+    float a = 0.;
+    
+    acc(in.v, a, 0, 1, fw);
+    
+    return a;
+  }
+  
+  //Complex double Reduce
+  template<>
+  inline Grid::ComplexD Reduce<Grid::ComplexD, vecd>::operator()(vecd in){
+    double a = 0., b = 0.;
+    
+    acc(in.v, a, 0, 2, dw);
+    acc(in.v, b, 1, 2, dw);
+    
+    return Grid::ComplexD(a, b);
+  }
+  
+  //Real double Reduce
+  template<>
+  inline Grid::RealD Reduce<Grid::RealD, vecd>::operator()(vecd in){
+    double a = 0.f;
+    
+    acc(in.v, a, 0, 1, dw);
+    
+    return a;
+  }
+
+  //Integer Reduce
+  template<>
+  inline Integer Reduce<Integer, int>::operator()(int in){
+    return in;
+  }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+// Here assign types 
+
+  typedef Optimization::vecf SIMD_Ftype;  // Single precision type
+  typedef Optimization::vecd SIMD_Dtype; // Double precision type
+  typedef int SIMD_Itype; // Integer type
+
+  // prefetch utilities
+  inline void v_prefetch0(int size, const char *ptr){};
+  inline void prefetch_HINT_T0(const char *ptr){};
+
+
+  // Function name aliases
+  typedef Optimization::Vsplat   VsplatSIMD;
+  typedef Optimization::Vstore   VstoreSIMD;
+  typedef Optimization::Vset     VsetSIMD;
+  typedef Optimization::Vstream  VstreamSIMD;
+  template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
+
+  // Arithmetic operations
+  typedef Optimization::Sum         SumSIMD;
+  typedef Optimization::Sub         SubSIMD;
+  typedef Optimization::Div         DivSIMD;
+  typedef Optimization::Mult        MultSIMD;
+  typedef Optimization::MultComplex MultComplexSIMD;
+  typedef Optimization::Conj        ConjSIMD;
+  typedef Optimization::TimesMinusI TimesMinusISIMD;
+  typedef Optimization::TimesI      TimesISIMD;
+
+}
diff --git a/lib/simd/Grid_vector_types.h b/lib/simd/Grid_vector_types.h
index e1592eef..9b9ad18b 100644
--- a/lib/simd/Grid_vector_types.h
+++ b/lib/simd/Grid_vector_types.h
@@ -38,8 +38,8 @@ directory
 #ifndef GRID_VECTOR_TYPES
 #define GRID_VECTOR_TYPES
 
-#ifdef GENERIC_VEC
-#include "Grid_generic.h"
+#ifdef GEN256
+#include "Grid_generic_256.h"
 #endif
 #ifdef SSE4
 #include "Grid_sse4.h"

From b873504b9031399ebbbf825679659d869f25c198 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Sat, 19 Nov 2016 01:32:39 +0100
Subject: [PATCH 148/149] fully generic SIMD

---
 configure.ac                 |  26 +-
 lib/simd/Grid_generic.h      | 462 +++++++++++++++++++++++++
 lib/simd/Grid_generic_256.h  | 644 -----------------------------------
 lib/simd/Grid_vector_types.h |   4 +-
 4 files changed, 483 insertions(+), 653 deletions(-)
 delete mode 100644 lib/simd/Grid_generic_256.h

diff --git a/configure.ac b/configure.ac
index fb80b1ae..7c81e5b5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -149,8 +149,14 @@ CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
 ############### SIMD instruction selection
-AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=<code>],
-	            [select SIMD target (cf. README.md)])], [ac_SIMD=${enable_simd}], [ac_SIMD=GEN])
+AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=code],
+              [select SIMD target (cf. README.md)])], [ac_SIMD=${enable_simd}], [ac_SIMD=GEN])
+
+AC_ARG_ENABLE([gen-simd-width],
+            [AS_HELP_STRING([--enable-gen-simd-width=size],
+            [size (in bytes) of the generic SIMD vectors (default: 32)])],
+            [ac_gen_simd_width=$enable_gen_simd_width],
+            [ac_gen_simd_width=32])
 
 case ${ax_cv_cxx_compiler_vendor} in
   clang|gnu)
@@ -179,8 +185,11 @@ case ${ax_cv_cxx_compiler_vendor} in
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-march=knl';;
-      GEN256)
-        AC_DEFINE([GEN256],[1],[generic vector code])
+      GEN)
+        AC_DEFINE([GEN],[1],[generic vector code])
+        AC_DEFINE_UNQUOTED([GEN_SIMD_WIDTH],[$ac_gen_simd_width],
+                           [generic SIMD vector width (in bytes)])
+        SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)"
         SIMD_FLAGS='';;
       QPX|BGQ)
         AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q])
@@ -211,8 +220,11 @@ case ${ax_cv_cxx_compiler_vendor} in
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing])
         SIMD_FLAGS='-xmic-avx512';;
-      GEN256)
-        AC_DEFINE([GEN256],[1],[generic vector code])
+      GEN)
+        AC_DEFINE([GEN],[1],[generic vector code])
+        AC_DEFINE([GEN_SIMD_WIDTH],[$ac_gen_simd_width],
+                  [generic SIMD vector width (in bytes)])
+        SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)"
         SIMD_FLAGS='';;
       *)
         AC_MSG_ERROR(["SIMD option ${ac_SIMD} not supported by the Intel compiler"]);;
@@ -382,7 +394,7 @@ os (target)                 : $target_os
 compiler vendor             : ${ax_cv_cxx_compiler_vendor}
 compiler version            : ${ax_cv_gxx_version}
 ----- BUILD OPTIONS -----------------------------------
-SIMD                        : ${ac_SIMD}
+SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
 Threading                   : ${ac_openmp} 
 Communications type         : ${comms_type}
 Default precision           : ${ac_PRECISION}
diff --git a/lib/simd/Grid_generic.h b/lib/simd/Grid_generic.h
index e69de29b..62c78afb 100644
--- a/lib/simd/Grid_generic.h
+++ b/lib/simd/Grid_generic.h
@@ -0,0 +1,462 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/simd/Grid_generic.h
+
+    Copyright (C) 2015
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+
+static_assert(GEN_SIMD_WIDTH % 16u == 0, "SIMD vector size is not an integer multiple of 16 bytes");
+
+//#define VECTOR_LOOPS
+
+// playing with compiler pragmas
+#ifdef VECTOR_LOOPS
+#ifdef __clang__
+#define VECTOR_FOR(i, w, inc)\
+_Pragma("clang loop unroll(full) vectorize(enable) interleave(enable) vectorize_width(w)")\
+for (unsigned int i = 0; i < w; i += inc)
+#elif defined __INTEL_COMPILER
+#define VECTOR_FOR(i, w, inc)\
+_Pragma("simd vectorlength(w*8)")\
+for (unsigned int i = 0; i < w; i += inc)
+#else
+#define VECTOR_FOR(i, w, inc)\
+for (unsigned int i = 0; i < w; i += inc)
+#endif
+#else
+#define VECTOR_FOR(i, w, inc)\
+for (unsigned int i = 0; i < w; i += inc)
+#endif
+
+namespace Grid {
+namespace Optimization {
+
+  // type traits giving the number of elements for each vector type
+  template <typename T> struct W;
+  template <> struct W<double> {
+    constexpr static unsigned int c = GEN_SIMD_WIDTH/16u;
+    constexpr static unsigned int r = GEN_SIMD_WIDTH/8u;
+  };
+  template <> struct W<float> {
+    constexpr static unsigned int c = GEN_SIMD_WIDTH/8u;
+    constexpr static unsigned int r = GEN_SIMD_WIDTH/4u;
+  };
+  
+  // SIMD vector types
+  template <typename T>
+  struct vec {
+    alignas(GEN_SIMD_WIDTH) T v[W<T>::r];
+  };
+  
+  typedef vec<float>   vecf;
+  typedef vec<double>  vecd;
+  
+  struct Vsplat{
+    // Complex
+    template <typename T>
+    inline vec<T> operator()(T a, T b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::r, 2)
+      {
+        out.v[i]   = a;
+        out.v[i+1] = b;
+      }
+
+      return out;
+    }
+    
+    // Real
+    template <typename T>
+    inline vec<T> operator()(T a){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::r, 1)
+      {
+        out.v[i] = a;
+      }
+      
+      return out;
+    }
+    
+    // Integer
+    inline int operator()(Integer a){
+      return a;
+    }
+  };
+
+  struct Vstore{
+    // Real
+    template <typename T>
+    inline void operator()(vec<T> a, T *D){
+      *((vec<T> *)D) = a;
+    }
+    //Integer
+    inline void operator()(int a, Integer *I){
+      *I = a;
+    }
+
+  };
+
+  struct Vstream{
+    // Real
+    template <typename T>
+    inline void operator()(T * a, vec<T> b){
+      *((vec<T> *)a) = b;
+    }
+  };
+
+  struct Vset{
+    // Complex
+    template <typename T>
+    inline vec<T> operator()(std::complex<T> *a){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::c, 1)
+      {
+        out.v[2*i]   = a[i].real();
+        out.v[2*i+1] = a[i].imag();
+      }
+      
+      return out;
+    }
+    
+    // Real
+    template <typename T>
+    inline vec<T> operator()(T *a){
+      vec<T> out;
+      
+      out = *((vec<T> *)a);
+      
+      return out;
+    }
+
+    // Integer
+    inline int operator()(Integer *a){
+      return *a;
+    }
+  };
+
+  /////////////////////////////////////////////////////
+  // Arithmetic operations
+  /////////////////////////////////////////////////////
+  struct Sum{
+    // Complex/Real
+    template <typename T>
+    inline vec<T> operator()(vec<T> a, vec<T> b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::r, 1)
+      {
+        out.v[i] = a.v[i] + b.v[i];
+      }
+      
+      return out;
+    }
+    
+    //I nteger
+    inline int operator()(int a, int b){
+      return a + b;
+    }
+  };
+
+  struct Sub{
+    // Complex/Real
+    template <typename T>
+    inline vec<T> operator()(vec<T> a, vec<T> b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::r, 1)
+      {
+        out.v[i] = a.v[i] - b.v[i];
+      }
+      
+      return out;
+    }
+    
+    //Integer
+    inline int operator()(int a, int b){
+      return a-b;
+    }
+  };
+
+  struct Mult{
+    // Real
+    template <typename T>
+    inline vec<T> operator()(vec<T> a, vec<T> b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::r, 1)
+      {
+        out.v[i] = a.v[i]*b.v[i];
+      }
+      
+      return out;
+    }
+    
+    // Integer
+    inline int operator()(int a, int b){
+      return a*b;
+    }
+  };
+  
+  #define cmul(a, b, c, i)\
+  c[i]   = a[i]*b[i]   - a[i+1]*b[i+1];\
+  c[i+1] = a[i]*b[i+1] + a[i+1]*b[i];
+  
+  struct MultComplex{
+    // Complex
+    template <typename T>
+    inline vec<T> operator()(vec<T> a, vec<T> b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::c, 1)
+      {
+        cmul(a.v, b.v, out.v, 2*i);
+      }      
+      
+      return out;
+    }
+  };
+  
+  #undef cmul
+
+  struct Div{
+    // Real
+    template <typename T>
+    inline vec<T> operator()(vec<T> a, vec<T> b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::r, 1)
+      {
+        out.v[i] = a.v[i]/b.v[i];
+      }
+      
+      return out;
+    }
+  };
+  
+  #define conj(a, b, i)\
+  b[i]   = a[i];\
+  b[i+1] = -a[i+1];
+  
+  struct Conj{
+    // Complex
+    template <typename T>
+    inline vec<T> operator()(vec<T> a){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::c, 1)
+      {
+        conj(a.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+  };
+  
+  #undef conj
+
+  #define timesmi(a, b, i)\
+  b[i]   = a[i+1];\
+  b[i+1] = -a[i];
+  
+  struct TimesMinusI{
+    // Complex
+    template <typename T>
+    inline vec<T> operator()(vec<T> a, vec<T> b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::c, 1)
+      {
+        timesmi(a.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+  };
+
+  #undef timesmi
+  
+  #define timesi(a, b, i)\
+  b[i]   = -a[i+1];\
+  b[i+1] = a[i];
+  
+  struct TimesI{
+    // Complex
+    template <typename T>
+    inline vec<T> operator()(vec<T> a, vec<T> b){
+      vec<T> out;
+      
+      VECTOR_FOR(i, W<T>::c, 1)
+      {
+        timesi(a.v, out.v, 2*i);
+      }
+      
+      return out;
+    }
+  };
+  
+  #undef timesi
+
+  //////////////////////////////////////////////
+  // Some Template specialization
+  #define perm(a, b, n, w)\
+  unsigned int _mask = w >> (n + 1);\
+  VECTOR_FOR(i, w, 1)\
+  {\
+    b[i] = a[i^_mask];\
+  }
+  
+  #define DECL_PERMUTE_N(n)\
+  template <typename T>\
+  static inline vec<T> Permute##n(vec<T> in) {\
+    vec<T> out;\
+    perm(in.v, out.v, n, W<T>::r);\
+    return out;\
+  }
+  
+  struct Permute{
+    DECL_PERMUTE_N(0);
+    DECL_PERMUTE_N(1);
+    DECL_PERMUTE_N(2);
+    DECL_PERMUTE_N(3);
+  };
+  
+  #undef perm
+  #undef DECL_PERMUTE_N
+  
+  #define rot(a, b, n, w)\
+  VECTOR_FOR(i, w, 1)\
+  {\
+    b[i] = a[(i + n)%w];\
+  }
+  
+  struct Rotate{
+    template <typename T>
+    static inline vec<T> rotate(vec<T> in, int n){
+      vec<T> out;
+      
+      rot(in.v, out.v, n, W<T>::r);
+      
+      return out;
+    }
+  };
+
+  #undef rot
+  
+  #define acc(v, a, off, step, n)\
+  for (unsigned int i = off; i < n; i += step)\
+  {\
+    a += v[i];\
+  }
+  
+  template <typename Out_type, typename In_type>
+  struct Reduce{
+    //Need templated class to overload output type
+    //General form must generate error if compiled
+    inline Out_type operator()(In_type in){
+      printf("Error, using wrong Reduce function\n");
+      exit(1);
+      return 0;
+    }
+  };
+  
+  //Complex float Reduce
+  template <>
+  inline Grid::ComplexF Reduce<Grid::ComplexF, vecf>::operator()(vecf in){
+    float a = 0.f, b = 0.f;
+    
+    acc(in.v, a, 0, 2, W<float>::r);
+    acc(in.v, b, 1, 2, W<float>::r);
+    
+    return Grid::ComplexF(a, b);
+  }
+  
+  //Real float Reduce
+  template<>
+  inline Grid::RealF Reduce<Grid::RealF, vecf>::operator()(vecf in){
+    float a = 0.;
+    
+    acc(in.v, a, 0, 1, W<float>::r);
+    
+    return a;
+  }
+  
+  //Complex double Reduce
+  template<>
+  inline Grid::ComplexD Reduce<Grid::ComplexD, vecd>::operator()(vecd in){
+    double a = 0., b = 0.;
+    
+    acc(in.v, a, 0, 2, W<double>::r);
+    acc(in.v, b, 1, 2, W<double>::r);
+    
+    return Grid::ComplexD(a, b);
+  }
+  
+  //Real double Reduce
+  template<>
+  inline Grid::RealD Reduce<Grid::RealD, vecd>::operator()(vecd in){
+    double a = 0.f;
+    
+    acc(in.v, a, 0, 1, W<double>::r);
+    
+    return a;
+  }
+
+  //Integer Reduce
+  template<>
+  inline Integer Reduce<Integer, int>::operator()(int in){
+    return in;
+  }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+// Here assign types 
+
+  typedef Optimization::vecf SIMD_Ftype; // Single precision type
+  typedef Optimization::vecd SIMD_Dtype; // Double precision type
+  typedef int SIMD_Itype; // Integer type
+
+  // prefetch utilities
+  inline void v_prefetch0(int size, const char *ptr){};
+  inline void prefetch_HINT_T0(const char *ptr){};
+
+  // Function name aliases
+  typedef Optimization::Vsplat   VsplatSIMD;
+  typedef Optimization::Vstore   VstoreSIMD;
+  typedef Optimization::Vset     VsetSIMD;
+  typedef Optimization::Vstream  VstreamSIMD;
+  template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
+
+  // Arithmetic operations
+  typedef Optimization::Sum         SumSIMD;
+  typedef Optimization::Sub         SubSIMD;
+  typedef Optimization::Div         DivSIMD;
+  typedef Optimization::Mult        MultSIMD;
+  typedef Optimization::MultComplex MultComplexSIMD;
+  typedef Optimization::Conj        ConjSIMD;
+  typedef Optimization::TimesMinusI TimesMinusISIMD;
+  typedef Optimization::TimesI      TimesISIMD;
+}
diff --git a/lib/simd/Grid_generic_256.h b/lib/simd/Grid_generic_256.h
deleted file mode 100644
index 42df6cf3..00000000
--- a/lib/simd/Grid_generic_256.h
+++ /dev/null
@@ -1,644 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/simd/Grid_generic.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: neo <cossu@post.kek.jp>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-
-#ifndef GEN_SIMD_WIDTH
-#define GEN_SIMD_DCOMPLEX_WIDTH 2
-#endif
-
-#include "Grid_generic.h"
-
-namespace Grid {
-namespace Optimization {
-
-  constexpr unsigned int dcw = GEN_SIMD_DCOMPLEX_WIDTH;
-  constexpr unsigned int fcw = 2*dcw;
-  constexpr unsigned int dw  = 2*dcw;
-  constexpr unsigned int fw  = 2*fcw;
-
-  struct vecf {
-    float v[fw];
-  };
-  
-  struct vecd {
-    double v[dw];
-  };
-  
-  struct Vsplat{
-    //Complex float
-    inline vecf operator()(float a, float b){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fw; i += 2)
-      {
-        out.v[i]   = a;
-        out.v[i+1] = b;
-      }
-
-      return out;
-    }
-    
-    // Real float
-    inline vecf operator()(float a){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fw; ++i)
-      {
-        out.v[i] = a;
-      }
-      
-      return out;
-    }
-    
-    //Complex double
-    inline vecd operator()(double a, double b){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dw; i += 2)
-      {
-        out.v[i]   = a;
-        out.v[i+1] = b;
-      }
-      
-      return out;
-    }
-    
-    //Real double
-    inline vecd operator()(double a){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dw; ++i)
-      {
-        out.v[i] = a;
-      }
-      
-      return out;
-    }
-    
-    //Integer
-    inline int operator()(Integer a){
-      return a;
-    }
-  };
-
-  struct Vstore{
-    //Float 
-    inline void operator()(vecf a, float* F){
-      memcpy(F,a.v,fw*sizeof(float));
-    }
-    //Double
-    inline void operator()(vecd a, double* D){
-      memcpy(D,a.v,dw*sizeof(double));
-    }
-    //Integer
-    inline void operator()(int a, Integer* I){
-      I[0] = a;
-    }
-
-  };
-
-  struct Vstream{
-    //Float
-    inline void operator()(float * a, vecf b){
-      memcpy(a,b.v,fw*sizeof(float));
-    }
-    //Double
-    inline void operator()(double * a, vecd b){
-      memcpy(a,b.v,dw*sizeof(double));
-    }
-
-
-  };
-
-  struct Vset{
-    // Complex float 
-    inline vecf operator()(Grid::ComplexF *a){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fcw; ++i)
-      {
-        out.v[2*i]   = a[i].real();
-        out.v[2*i+1] = a[i].imag();
-      }
-      
-      return out;
-    }
-    
-    // Complex double 
-    inline vecd operator()(Grid::ComplexD *a){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dcw; ++i)
-      {
-        out.v[2*i]   = a[i].real();
-        out.v[2*i+1] = a[i].imag();
-      }
-      
-      return out;
-    }
-    
-    // Real float 
-    inline vecf operator()(float *a){
-      vecf out;
-      
-      memcpy(out.v,a,fw*sizeof(float));
-      
-      return out;
-    }
-    // Real double
-    inline vecd operator()(double *a){
-      vecd out; 
-      
-      memcpy(out.v,a,dw*sizeof(float));
-      
-      return out;
-    }
-    // Integer
-    inline int operator()(Integer *a){
-      return a[0];
-    }
-
-
-  };
-
-  /////////////////////////////////////////////////////
-  // Arithmetic operations
-  /////////////////////////////////////////////////////
-  struct Sum{
-    //Complex/Real float
-    inline vecf operator()(vecf a, vecf b){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fw; ++i)
-      {
-        out.v[i] = a.v[i] + b.v[i];
-      }
-      
-      return out;
-    }
-    
-    //Complex/Real double
-    inline vecd operator()(vecd a, vecd b){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dw; ++i)
-      {
-        out.v[i] = a.v[i] + b.v[i];
-      }
-
-      return out;
-    }
-    
-    //Integer
-    inline int operator()(int a, int b){
-      return a + b;
-    }
-  };
-
-  struct Sub{
-    //Complex/Real float
-    inline vecf operator()(vecf a, vecf b){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fw; ++i)
-      {
-        out.v[i] = a.v[i] - b.v[i];
-      }
-      
-      return out;
-    }
-    
-    //Complex/Real double
-    inline vecd operator()(vecd a, vecd b){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dw; ++i)
-      {
-        out.v[i] = a.v[i] - b.v[i];
-      }
-      
-      return out;
-    }
-    
-    //Integer
-    inline int operator()(int a, int b){
-      return a-b;
-    }
-  };
-
-  #define cmul(a, b, c, i)\
-  c[i]   = a[i]*b[i]   - a[i+1]*b[i+1];\
-  c[i+1] = a[i]*b[i+1] + a[i+1]*b[i];
-  
-  struct MultComplex{
-    // Complex float
-    inline vecf operator()(vecf a, vecf b){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fcw; ++i)
-      {
-        cmul(a.v, b.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-    
-    // Complex double
-    inline vecd operator()(vecd a, vecd b){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dcw; ++i)
-      {
-        cmul(a.v, b.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-  };
-  
-  #undef cmul
-
-  struct Mult{
-    // Real float
-    inline vecf operator()(vecf a, vecf b){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fw; ++i)
-      {
-        out.v[i] = a.v[i]*b.v[i];
-      }
-      
-      return out;
-    }
-    
-    // Real double
-    inline vecd operator()(vecd a, vecd b){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dw; ++i)
-      {
-        out.v[i] = a.v[i]*b.v[i];
-      }
-      
-      return out;
-    }
-    
-    // Integer
-    inline int operator()(int a, int b){
-      return a*b;
-    }
-  };
-
-  struct Div{
-    // Real float
-    inline vecf operator()(vecf a, vecf b){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fw; ++i)
-      {
-        out.v[i] = a.v[i]/b.v[i];
-      }
-      
-      return out;
-    }
-    // Real double
-    inline vecd operator()(vecd a, vecd b){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dw; ++i)
-      {
-        out.v[i] = a.v[i]/b.v[i];
-      }
-      
-      return out;
-    }
-  };
-  
-  #define conj(a, b, i)\
-  b[i]   = a[i];\
-  b[i+1] = -a[i+1];
-  
-  struct Conj{
-    // Complex single
-    inline vecf operator()(vecf in){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fcw; ++i)
-      {
-        conj(in.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-    
-    // Complex double
-    inline vecd operator()(vecd in){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dcw; ++i)
-      {
-        conj(in.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-  };
-  
-  #undef conj
-
-  #define timesmi(a, b, i)\
-  b[i]   = a[i+1];\
-  b[i+1] = -a[i];
-  
-  struct TimesMinusI{
-    // Complex single
-    inline vecf operator()(vecf in, vecf ret){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fcw; ++i)
-      {
-        timesmi(in.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-    
-    // Complex double
-    inline vecd operator()(vecd in, vecd ret){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dcw; ++i)
-      {
-        timesmi(in.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-  };
-
-  #undef timesmi
-  
-  #define timespi(a, b, i)\
-  b[i]   = -a[i+1];\
-  b[i+1] = a[i];
-  
-  struct TimesI{
-    // Complex single
-    inline vecf operator()(vecf in, vecf ret){
-      vecf out;
-      
-      for (unsigned int i = 0; i < fcw; ++i)
-      {
-        timespi(in.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-    
-    // Complex double
-    inline vecd operator()(vecd in, vecd ret){
-      vecd out;
-      
-      for (unsigned int i = 0; i < dcw; ++i)
-      {
-        timespi(in.v, out.v, 2*i);
-      }
-      
-      return out;
-    }
-  };
-  
-  #undef timespi
-
-  //////////////////////////////////////////////
-  // Some Template specialization
-  struct Permute{
-    static inline vecf Permute0(vecf in){ //AB CD -> CD AB
-      vecf out;
-      
-      out.v[0] = in.v[4];
-      out.v[1] = in.v[5];
-      out.v[2] = in.v[6];
-      out.v[3] = in.v[7];
-      out.v[4] = in.v[0];
-      out.v[5] = in.v[1];
-      out.v[6] = in.v[2];
-      out.v[7] = in.v[3];
-      
-      return out;
-    };
-    
-    static inline vecf Permute1(vecf in){ //AB CD -> BA DC
-      vecf out;
-      
-      out.v[0] = in.v[2];
-      out.v[1] = in.v[3];
-      out.v[2] = in.v[0];
-      out.v[3] = in.v[1];
-      out.v[4] = in.v[6];
-      out.v[5] = in.v[7];
-      out.v[6] = in.v[4];
-      out.v[7] = in.v[5];
-      
-      return out;
-    };
-    
-    static inline vecf Permute2(vecf in){
-      vecf out;
-      
-      out.v[0] = in.v[1];
-      out.v[1] = in.v[0];
-      out.v[2] = in.v[3];
-      out.v[3] = in.v[2];
-      out.v[4] = in.v[5];
-      out.v[5] = in.v[4];
-      out.v[6] = in.v[7];
-      out.v[7] = in.v[6];
-      
-      return out;
-    };
-
-    static inline vecf Permute3(vecf in){
-      return in;
-    };
-
-    static inline vecd Permute0(vecd in){ //AB -> BA
-      vecd out;
-      
-      out.v[0] = in.v[2];
-      out.v[1] = in.v[3];
-      out.v[2] = in.v[0];
-      out.v[3] = in.v[1];
-      
-      return out;
-    };
-    
-    static inline vecd Permute1(vecd in){
-      vecd out;
-      
-      out.v[0] = in.v[1];
-      out.v[1] = in.v[0];
-      out.v[2] = in.v[3];
-      out.v[3] = in.v[2];
-      
-      return out;
-    };
-    
-    static inline vecd Permute2(vecd in){
-      return in;
-    };
-    
-    static inline vecd Permute3(vecd in){
-      return in;
-    };
-
-  };
-  
-  #define rot(a, b, n, w)\
-  for (unsigned int i = 0; i < w; ++i)\
-  {\
-    b[i] = a[(i + n)%w];\
-  }
-  
-  struct Rotate{
-
-    static inline vecf rotate(vecf in, int n){
-      vecf out;
-      
-      rot(in.v, out.v, n, fw);
-      
-      return out;
-    }
-    
-    static inline vecd rotate(vecd in,int n){
-      vecd out;
-      
-      rot(in.v, out.v, n, dw);
-      
-      return out;
-    }
-  };
-
-  #undef rot
-  
-  #define acc(v, a, off, step, n)\
-  for (unsigned int i = off; i < n; i += step)\
-  {\
-    a += v[i];\
-  }
-  
-  template <typename Out_type, typename In_type>
-  struct Reduce{
-    //Need templated class to overload output type
-    //General form must generate error if compiled
-    inline Out_type operator()(In_type in){
-      printf("Error, using wrong Reduce function\n");
-      exit(1);
-      return 0;
-    }
-  };
-  
-  //Complex float Reduce
-  template<>
-  inline Grid::ComplexF Reduce<Grid::ComplexF, vecf>::operator()(vecf in){
-    float a = 0.f, b = 0.f;
-    
-    acc(in.v, a, 0, 2, fw);
-    acc(in.v, b, 1, 2, fw);
-    
-    return Grid::ComplexF(a, b);
-  }
-  
-  //Real float Reduce
-  template<>
-  inline Grid::RealF Reduce<Grid::RealF, vecf>::operator()(vecf in){
-    float a = 0.;
-    
-    acc(in.v, a, 0, 1, fw);
-    
-    return a;
-  }
-  
-  //Complex double Reduce
-  template<>
-  inline Grid::ComplexD Reduce<Grid::ComplexD, vecd>::operator()(vecd in){
-    double a = 0., b = 0.;
-    
-    acc(in.v, a, 0, 2, dw);
-    acc(in.v, b, 1, 2, dw);
-    
-    return Grid::ComplexD(a, b);
-  }
-  
-  //Real double Reduce
-  template<>
-  inline Grid::RealD Reduce<Grid::RealD, vecd>::operator()(vecd in){
-    double a = 0.f;
-    
-    acc(in.v, a, 0, 1, dw);
-    
-    return a;
-  }
-
-  //Integer Reduce
-  template<>
-  inline Integer Reduce<Integer, int>::operator()(int in){
-    return in;
-  }
-}
-
-//////////////////////////////////////////////////////////////////////////////////////
-// Here assign types 
-
-  typedef Optimization::vecf SIMD_Ftype;  // Single precision type
-  typedef Optimization::vecd SIMD_Dtype; // Double precision type
-  typedef int SIMD_Itype; // Integer type
-
-  // prefetch utilities
-  inline void v_prefetch0(int size, const char *ptr){};
-  inline void prefetch_HINT_T0(const char *ptr){};
-
-
-  // Function name aliases
-  typedef Optimization::Vsplat   VsplatSIMD;
-  typedef Optimization::Vstore   VstoreSIMD;
-  typedef Optimization::Vset     VsetSIMD;
-  typedef Optimization::Vstream  VstreamSIMD;
-  template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
-
-  // Arithmetic operations
-  typedef Optimization::Sum         SumSIMD;
-  typedef Optimization::Sub         SubSIMD;
-  typedef Optimization::Div         DivSIMD;
-  typedef Optimization::Mult        MultSIMD;
-  typedef Optimization::MultComplex MultComplexSIMD;
-  typedef Optimization::Conj        ConjSIMD;
-  typedef Optimization::TimesMinusI TimesMinusISIMD;
-  typedef Optimization::TimesI      TimesISIMD;
-
-}
diff --git a/lib/simd/Grid_vector_types.h b/lib/simd/Grid_vector_types.h
index 9b9ad18b..184baad9 100644
--- a/lib/simd/Grid_vector_types.h
+++ b/lib/simd/Grid_vector_types.h
@@ -38,8 +38,8 @@ directory
 #ifndef GRID_VECTOR_TYPES
 #define GRID_VECTOR_TYPES
 
-#ifdef GEN256
-#include "Grid_generic_256.h"
+#ifdef GEN
+#include "Grid_generic.h"
 #endif
 #ifdef SSE4
 #include "Grid_sse4.h"

From 433afd36f51ffe2e6fe024357a3671615be808f8 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Sat, 19 Nov 2016 01:33:13 +0100
Subject: [PATCH 149/149] Makefile rule for simple_* objects

---
 benchmarks/Makefile.am         | 10 ++++++++++
 benchmarks/simple_simd_test.cc | 11 +++++++++++
 benchmarks/simple_su3_expr.cc  |  2 +-
 3 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 benchmarks/simple_simd_test.cc

diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am
index 60b82dd7..1b345fd2 100644
--- a/benchmarks/Makefile.am
+++ b/benchmarks/Makefile.am
@@ -1 +1,11 @@
 include Make.inc
+
+simple: simple_su3_test.o simple_su3_expr.o simple_simd_test.o
+
+EXTRA_LIBRARIES = libsimple_su3_test.a libsimple_su3_expr.a libsimple_simd_test.a
+
+libsimple_su3_test_a_SOURCES = simple_su3_test.cc
+
+libsimple_su3_expr_a_SOURCES = simple_su3_expr.cc
+
+libsimple_simd_test_a_SOURCES = simple_simd_test.cc
diff --git a/benchmarks/simple_simd_test.cc b/benchmarks/simple_simd_test.cc
new file mode 100644
index 00000000..3d5651e4
--- /dev/null
+++ b/benchmarks/simple_simd_test.cc
@@ -0,0 +1,11 @@
+#include <Grid/Grid.h>
+
+Grid::vRealD add(const Grid::vRealD &x, const Grid::vRealD &y)
+{
+	return x+y;
+}
+
+Grid::vRealD sub(const Grid::vRealD &x, const Grid::vRealD &y)
+{
+	return x-y;
+}
diff --git a/benchmarks/simple_su3_expr.cc b/benchmarks/simple_su3_expr.cc
index 11a4816e..0c6092f6 100644
--- a/benchmarks/simple_su3_expr.cc
+++ b/benchmarks/simple_su3_expr.cc
@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid.h>
+#include <Grid/Grid.h>
 
 using namespace std;
 using namespace Grid;