Merge branch 'develop' into feature/scalar_adjointFT

.travis.yml

@@ -7,7 +7,7 @@ cache:
 matrix:
   include:
     - os:        osx
-      osx_image: xcode7.3
+      osx_image: xcode8.3
       compiler: clang
     - compiler: gcc
       sudo: required
@@ -75,8 +75,6 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openmpi; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]] && [[ "$CC" == "gcc" ]]; then brew install gcc5; fi
     
 install:
     - export CC=$CC$VERSION
@@ -94,15 +92,14 @@ script:
     - cd build
     - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=none
     - make -j4 
-    - ./benchmarks/Benchmark_dwf --threads 1
+    - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - echo make clean
     - ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none
     - make -j4
-    - ./benchmarks/Benchmark_dwf --threads 1
+    - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - echo make clean
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXXFLAGS='-DMPI_UINT32_T=MPI_UNSIGNED -DMPI_UINT64_T=MPI_UNSIGNED_LONG'; fi
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto CXXFLAGS='-DMPI_UINT32_T=MPI_UNSIGNED -DMPI_UINT64_T=MPI_UNSIGNED_LONG'; fi
-    - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make -j4; fi
-    - make -j4
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
 
 

Makefile.am

@@ -3,6 +3,8 @@ SUBDIRS = lib benchmarks tests extras
 
 include $(top_srcdir)/doxygen.inc
 
+bin_SCRIPTS=grid-config
+
 tests: all
 	$(MAKE) -C tests tests
 

TODO

@@ -1,6 +1,26 @@
 TODO:
 ---------------
 
+Peter's work list:
+2)- Precision conversion and sort out localConvert      <-- 
+3)- Remove DenseVector, DenseMatrix; Use Eigen instead. <-- started 
+4)- Binary I/O speed up & x-strips
+-- Profile CG, BlockCG, etc... Flop count/rate -- PARTIAL, time but no flop/s yet
+-- Physical propagator interface
+-- Conserved currents
+-- GaugeFix into central location
+-- Multigrid Wilson and DWF, compare to other Multigrid implementations
+-- HDCR resume
+
+Recent DONE 
+-- Cut down the exterior overhead                      <-- DONE
+-- Interior legs from SHM comms                        <-- DONE
+-- Half-precision comms                                <-- DONE
+-- Merge high precision reduction into develop        
+-- multiRHS DWF; benchmark on Cori/BNL for comms elimination
+   -- slice* linalg routines for multiRHS, BlockCG    
+
+-----
 * Forces; the UdSdU  term in gauge force term is half of what I think it should
   be. This is a consequence of taking ONLY the first term in:
 
@@ -21,16 +41,8 @@ TODO:
   This means we must double the force in the Test_xxx_force routines, and is the origin of the factor of two.
   This 2x is applied by hand in the fermion routines and in the Test_rect_force routine.
 
-
-Policies:
-
-* Link smearing/boundary conds; Policy class based implementation ; framework more in place
-
 * Support different boundary conditions (finite temp, chem. potential ... )
 
-* Support different fermion representations? 
-  - contained entirely within the integrator presently
-
 - Sign of force term.
 
 - Reversibility test.
@@ -41,11 +53,6 @@ Policies:
 
 - Audit oIndex usage for cb behaviour
 
-- Rectangle gauge actions.
-  Iwasaki,
-  Symanzik,
-  ... etc...
-
 - Prepare multigrid for HMC. - Alternate setup schemes.
 
 - Support for ILDG --- ugly, not done
@@ -55,9 +62,11 @@ Policies:
 - FFTnD ?
 
 - Gparity; hand opt use template specialisation elegance to enable the optimised paths ?
+
 - Gparity force term; Gparity (R)HMC.
-- Random number state save restore
+
 - Mobius implementation clean up to rmove #if 0 stale code sequences
+
 - CG -- profile carefully, kernel fusion, whole CG performance measurements.
 
 ================================================================
@@ -90,6 +99,7 @@ Insert/Extract
 Not sure of status of this -- reverify. Things are working nicely now though.
 
 * Make the Tensor types and Complex etc... play more nicely.
+
   - TensorRemove is a hack, come up with a long term rationalised approach to Complex vs. Scalar<Scalar<Scalar<Complex > > >
     QDP forces use of "toDouble" to get back to non tensor scalar. This role is presently taken TensorRemove, but I
     want to introduce a syntax that does not require this.
@@ -112,6 +122,8 @@ Not sure of status of this -- reverify. Things are working nicely now though.
 RECENT
 ---------------
 
+  - Support different fermion representations? -- DONE
+  - contained entirely within the integrator presently
   - Clean up HMC                                                             -- DONE
   - LorentzScalar<GaugeField> gets Gauge link type (cleaner).                -- DONE
   - Simplified the integrators a bit.                                        -- DONE
@@ -123,6 +135,26 @@ RECENT
   - Parallel io improvements                                  -- DONE
   - Plaquette and link trace checks into nersc reader from the Grid_nersc_io.cc test. -- DONE
 
+
+DONE:
+- MultiArray -- MultiRHS done
+- ConjugateGradientMultiShift -- DONE
+- MCR                         -- DONE
+- Remez -- Mike or Boost?     -- DONE
+- Proto (ET)                  -- DONE
+- uBlas                       -- DONE ; Eigen
+- Potentially Useful Boost libraries -- DONE ; Eigen
+- Aligned allocator; memory pool -- DONE
+- Multiprecision              -- DONE
+- Serialization               -- DONE
+- Regex -- Not needed
+- Tokenize -- Why?
+
+- Random number state save restore -- DONE
+- Rectangle gauge actions. -- DONE
+  Iwasaki,
+  Symanzik,
+  ... etc...
 Done: Cayley, Partial , ContFrac force terms.
 
 DONE
@@ -207,6 +239,7 @@ Done
 FUNCTIONALITY: it pleases me to keep track of things I have done (keeps me arguably sane)
 ======================================================================================================
 
+* Link smearing/boundary conds; Policy class based implementation ; framework more in place -- DONE
 * Command line args for geometry, simd, etc. layout. Is it necessary to have -- DONE
   user pass these? Is this a QCD specific?
 

benchmarks/Benchmark_dwf.cc

@@ -1,28 +1,22 @@
-    /*************************************************************************************
+ /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./benchmarks/Benchmark_dwf.cc
-
     Copyright (C) 2015
 
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -37,12 +31,12 @@ struct scal {
   d internal;
 };
 
-Gamma::Algebra Gmu [] = {
+  Gamma::Algebra Gmu [] = {
-  Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaX,
-  Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaY,
-  Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaZ,
-  Gamma::Algebra::GammaT
+    Gamma::Algebra::GammaT
-};
+  };
 
 typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
 typedef WilsonFermion5D<DomainWallVec5dImplF> WilsonFermion5DF;
@@ -51,24 +45,24 @@ typedef WilsonFermion5D<DomainWallVec5dImplD> WilsonFermion5DD;
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
-  
+
-  
+
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
-  
+
   std::vector<int> latt4 = GridDefaultLatt();
   const int Ls=16;
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
   GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
-  
+
   std::cout << GridLogMessage << "Making s innermost grids"<<std::endl;
   GridCartesian         * sUGrid   = SpaceTimeGrid::makeFourDimDWFGrid(GridDefaultLatt(),GridDefaultMpi());
   GridRedBlackCartesian * sUrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(sUGrid);
   GridCartesian         * sFGrid   = SpaceTimeGrid::makeFiveDimDWFGrid(Ls,UGrid);
   GridRedBlackCartesian * sFrbGrid = SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(Ls,UGrid);
-  
+
   std::vector<int> seeds4({1,2,3,4});
   std::vector<int> seeds5({5,6,7,8});
   
@@ -77,7 +71,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "Initialising 5d RNG" << std::endl;
   GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
   std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
-  
+
   LatticeFermion src   (FGrid); random(RNG5,src);
 #if 0
   src = zero;
@@ -93,14 +87,13 @@ int main (int argc, char ** argv)
   RealD N2 = 1.0/::sqrt(norm2(src));
   src = src*N2;
 #endif
-  
+
-  
+
   LatticeFermion result(FGrid); result=zero;
   LatticeFermion    ref(FGrid);    ref=zero;
-  LatticeFermion    refDag(FGrid);    refDag=zero;
   LatticeFermion    tmp(FGrid);
   LatticeFermion    err(FGrid);
-  
+
   std::cout << GridLogMessage << "Drawing gauge field" << std::endl;
   LatticeGaugeField Umu(UGrid); 
   SU3::HotConfiguration(RNG4,Umu); 
@@ -116,7 +109,7 @@ int main (int argc, char ** argv)
   }
   std::cout << GridLogMessage << "Forced to diagonal " << std::endl;
 #endif
-  
+
   ////////////////////////////////////
   // Naive wilson implementation
   ////////////////////////////////////
@@ -132,30 +125,28 @@ int main (int argc, char ** argv)
     U[mu] = PeekIndex<LorentzIndex>(Umu5d,mu);
   }
   std::cout << GridLogMessage << "Setting up Cshift based reference " << std::endl;
-  
+
   if (1)
-    {
+  {
-      ref = zero;
+    ref = zero;
-      for(int mu=0;mu<Nd;mu++){
+    for(int mu=0;mu<Nd;mu++){
-	
+
-	tmp = U[mu]*Cshift(src,mu+1,1);
+      tmp = U[mu]*Cshift(src,mu+1,1);
-	ref=ref + tmp - Gamma(Gmu[mu])*tmp;
+      ref=ref + tmp - Gamma(Gmu[mu])*tmp;
-	
+
-	tmp =adj(U[mu])*src;
+      tmp =adj(U[mu])*src;
-	tmp =Cshift(tmp,mu+1,-1);
+      tmp =Cshift(tmp,mu+1,-1);
-	ref=ref + tmp + Gamma(Gmu[mu])*tmp;
+      ref=ref + tmp + Gamma(Gmu[mu])*tmp;
-      }
-      ref = -0.5*ref;
     }
-  
+    ref = -0.5*ref;
+  }
+
   RealD mass=0.1;
   RealD M5  =1.8;
-  
+
   RealD NP = UGrid->_Nprocessors;
-  
+  RealD NN = UGrid->NodeCount();
-  std::cout << GridLogMessage << "Creating action operator " << std::endl;
+
-  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-  
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
   std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
@@ -164,16 +155,22 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
   if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
   if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
-  
+
+  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
   int ncall =1000;
   if (1) {
     FGrid->Barrier();
     Dw.ZeroCounters();
     Dw.Dhop(src,result,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       __SSC_START;
@@ -185,65 +182,81 @@ int main (int argc, char ** argv)
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=1344*volume*ncall;
-    
+
     std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
     //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-    
+
     /*
-      if(( norm2(err)>1.0e-4) ) { 
+    if(( norm2(err)>1.0e-4) ) { 
       std::cout << "RESULT\n " << result<<std::endl;
       std::cout << "REF   \n " << ref   <<std::endl;
       std::cout << "ERR   \n " << err   <<std::endl;
       FGrid->Barrier();
       exit(-1);
-      }
+    }
     */
     assert (norm2(err)< 1.0e-4 );
     Dw.Report();
   }
-  
+
-  if (1) {  // Naive wilson dag implementation
+  DomainWallFermionRL DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-    refDag = zero;
-    for (int mu = 0; mu < Nd; mu++) {
-      //    ref =  src - Gamma(Gamma::GammaX)* src ; // 1+gamma_x
-      tmp = U[mu] * Cshift(src, mu + 1, 1);
-      for (int i = 0; i < refDag._odata.size(); i++) {
-        refDag._odata[i] += tmp._odata[i] + Gamma(Gmu[mu]) * tmp._odata[i];
-      }
-      
-      tmp = adj(U[mu]) * src;
-      tmp = Cshift(tmp, mu + 1, -1);
-      for (int i = 0; i < refDag._odata.size(); i++) {
-        refDag._odata[i] += tmp._odata[i] - Gamma(Gmu[mu]) * tmp._odata[i];
-      }
-    }
-    refDag = -0.5 * refDag;
-  }
-  
   if (1) {
+    FGrid->Barrier();
+    DwH.ZeroCounters();
+    DwH.Dhop(src,result,0);
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwH.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
     
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called half prec comms Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    err = ref-result; 
+    std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+
+    assert (norm2(err)< 1.0e-3 );
+    DwH.Report();
+  }
+
+  if (1)
+  {
+
     std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
     std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::Dhop "<<std::endl;
     std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
     if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
     if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
     std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-    
+
     typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
     LatticeFermion ssrc(sFGrid);
     LatticeFermion sref(sFGrid);
     LatticeFermion sresult(sFGrid);
-    
+
     WilsonFermion5DR sDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,M5);
-    
+
     localConvert(src,ssrc);
     std::cout<<GridLogMessage<< "src norms "<< norm2(src)<<" " <<norm2(ssrc)<<std::endl;
     FGrid->Barrier();
@@ -259,11 +272,12 @@ int main (int argc, char ** argv)
     FGrid->Barrier();
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=1344*volume*ncall;
-    
+
     std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
-    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    //    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
     sDw.Report();
     RealD sum=0;
 
@@ -276,6 +290,7 @@ int main (int argc, char ** argv)
       std::cout<< "sD REF\n " <<ref << std::endl;
       std::cout<< "sD ERR   \n " <<err  <<std::endl;
     }
+    //    assert(sum < 1.0e-4);
 
     err=zero;
     localConvert(sresult,err);
@@ -288,29 +303,16 @@ int main (int argc, char ** argv)
     }
     assert(sum < 1.0e-4);
     
-    
-    // Check Dag
-    std::cout << GridLogMessage << "Compare WilsonFermion5D<DomainWallVec5dImplR>::Dhop to naive wilson implementation Dag to verify correctness" << std::endl;
-    sDw.Dhop(ssrc,sresult,1);
-    err=zero;
-    localConvert(sresult,err);
-    err = err - refDag;
-    sum = norm2(err);
-    std::cout<<GridLogMessage<<" difference between normal dag ref and simd is "<<sum<<std::endl;
-    if(sum > 1.0e-4 ){
-      std::cout<< "sD REF\n " <<result << std::endl;
-      std::cout<< "sD ERR   \n " << err  <<std::endl;
-    }
-    assert(sum < 1.0e-4);
-    
-    
-    
     if(1){
       std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
       std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::DhopEO "<<std::endl;
       std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
       if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
       if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
       if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) 
 	std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
       if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) 
@@ -318,58 +320,57 @@ int main (int argc, char ** argv)
       if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) 
 	std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
       std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-      
+
       LatticeFermion sr_eo(sFGrid);
       LatticeFermion ssrc_e (sFrbGrid);
       LatticeFermion ssrc_o (sFrbGrid);
       LatticeFermion sr_e   (sFrbGrid);
       LatticeFermion sr_o   (sFrbGrid);
-      
+
       pickCheckerboard(Even,ssrc_e,ssrc);
       pickCheckerboard(Odd,ssrc_o,ssrc);
       //      setCheckerboard(sr_eo,ssrc_o);
       //      setCheckerboard(sr_eo,ssrc_e);
-      
+
       sr_e = zero;
       sr_o = zero;
-      
+
       FGrid->Barrier();
       sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
       sDw.ZeroCounters();
       //      sDw.stat.init("DhopEO");
       double t0=usecond();
       for (int i = 0; i < ncall; i++) {
-	sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
+        sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
       }
       double t1=usecond();
       FGrid->Barrier();
       //      sDw.stat.print();
-      
+
       double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
       double flops=(1344.0*volume*ncall)/2;
-      
+
       std::cout<<GridLogMessage << "sDeo mflop/s =   "<< flops/(t1-t0)<<std::endl;
       std::cout<<GridLogMessage << "sDeo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+      std::cout<<GridLogMessage << "sDeo mflop/s per node   "<< flops/(t1-t0)/NN<<std::endl;
       sDw.Report();
-      
+
       sDw.DhopEO(ssrc_o,sr_e,DaggerNo);
       sDw.DhopOE(ssrc_e,sr_o,DaggerNo);
       sDw.Dhop  (ssrc  ,sresult,DaggerNo);
-      
+
       pickCheckerboard(Even,ssrc_e,sresult);
       pickCheckerboard(Odd ,ssrc_o,sresult);
 
-      // Check even part
       ssrc_e = ssrc_e - sr_e;
       RealD error = norm2(ssrc_e);
-      std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm: "<<norm2(sr_e) <<std::endl;
+      std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm"<<norm2(sr_e) <<std::endl;
 
-      // Check odd part
       ssrc_o = ssrc_o - sr_o;
       error+= norm2(ssrc_o);
-      std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm: "<<norm2(sr_o) <<std::endl;
+      std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm"<<norm2(sr_o) <<std::endl;
 
-      if(error>1.0e-4) { 
+      if(( error>1.0e-4) ) { 
 	setCheckerboard(ssrc,ssrc_o);
 	setCheckerboard(ssrc,ssrc_e);
 	std::cout<< "DIFF\n " <<ssrc << std::endl;
@@ -379,43 +380,20 @@ int main (int argc, char ** argv)
 	std::cout<< "RESULT\n " <<sresult<< std::endl;
       }
       assert(error<1.0e-4);
-      
-      // Check the dag
-      std::cout << GridLogMessage << "Compare WilsonFermion5D<DomainWallVec5dImplR>::DhopEO to Dhop to verify correctness" << std::endl;
-      pickCheckerboard(Even,ssrc_e,ssrc);
-      pickCheckerboard(Odd,ssrc_o,ssrc);
-      sDw.DhopEO(ssrc_o,sr_e,DaggerYes);
-      sDw.DhopOE(ssrc_e,sr_o,DaggerYes);
-      sDw.Dhop  (ssrc  ,sresult,DaggerYes);
-      
-      pickCheckerboard(Even,ssrc_e,sresult);
-      pickCheckerboard(Odd ,ssrc_o,sresult);
-
-      ssrc_e = ssrc_e - sr_e;
-      error = norm2(ssrc_e);
-      std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm: "<<norm2(sr_e) <<std::endl;
-      ssrc_o = ssrc_o - sr_o;
-      error+= norm2(ssrc_o);
-      std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm: "<<norm2(sr_o) <<std::endl;
-
-      if(error>1.0e-4) { 
-	setCheckerboard(ssrc,ssrc_o);
-	setCheckerboard(ssrc,ssrc_e);
-	std::cout<< ssrc << std::endl;
-      } 
-
     }
-  }  
+  }
-      
+
-  if (1) { // Naive wilson dag implementation
+  if (1)
+  { // Naive wilson dag implementation
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
+
       //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu+1,1);
       for(int i=0;i<ref._odata.size();i++){
 	ref._odata[i]+= tmp._odata[i] + Gamma(Gmu[mu])*tmp._odata[i]; ;
       }
-      
+
       tmp =adj(U[mu])*src;
       tmp =Cshift(tmp,mu+1,-1);
       for(int i=0;i<ref._odata.size();i++){
@@ -424,43 +402,44 @@ int main (int argc, char ** argv)
     }
     ref = -0.5*ref;
   }
-  
+  //  dump=1;
   Dw.Dhop(src,result,1);
-  std::cout << GridLogMessage << "Compare DomainWallFermionR::Dhop to naive wilson implementation Dag to verify correctness" << std::endl;
+  std::cout << GridLogMessage << "Compare to naive wilson implementation Dag to verify correctness" << std::endl;
   std::cout<<GridLogMessage << "Called DwDag"<<std::endl;
   std::cout<<GridLogMessage << "norm dag result "<< norm2(result)<<std::endl;
   std::cout<<GridLogMessage << "norm dag ref    "<< norm2(ref)<<std::endl;
   err = ref-result; 
   std::cout<<GridLogMessage << "norm dag diff   "<< norm2(err)<<std::endl;
   if((norm2(err)>1.0e-4)){
-    std::cout<< "DAG RESULT\n "  <<ref     << std::endl;
+	std::cout<< "DAG RESULT\n "  <<ref     << std::endl;
-    std::cout<< "DAG sRESULT\n " <<result  << std::endl;
+	std::cout<< "DAG sRESULT\n " <<result  << std::endl;
-    std::cout<< "DAG ERR   \n "  << err    <<std::endl;
+	std::cout<< "DAG ERR   \n "  << err    <<std::endl;
   }
   LatticeFermion src_e (FrbGrid);
   LatticeFermion src_o (FrbGrid);
   LatticeFermion r_e   (FrbGrid);
   LatticeFermion r_o   (FrbGrid);
   LatticeFermion r_eo  (FGrid);
-  
+
-  
+
   std::cout<<GridLogMessage << "Calling Deo and Doe and //assert Deo+Doe == Dunprec"<<std::endl;
   pickCheckerboard(Even,src_e,src);
   pickCheckerboard(Odd,src_o,src);
-  
+
   std::cout<<GridLogMessage << "src_e"<<norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "src_o"<<norm2(src_o)<<std::endl;
-  
+
-  
+
   // S-direction is INNERMOST and takes no part in the parity.
-  static int Opt;  // these are a temporary hack
-  static int Comms;  // these are a temporary hack
-  
   std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
   std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::DhopEO                "<<std::endl;
   std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
   if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
   if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
@@ -478,38 +457,39 @@ int main (int argc, char ** argv)
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=(1344.0*volume*ncall)/2;
-    
+
     std::cout<<GridLogMessage << "Deo mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "Deo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "Deo mflop/s per node   "<< flops/(t1-t0)/NN<<std::endl;
     Dw.Report();
   }
   Dw.DhopEO(src_o,r_e,DaggerNo);
   Dw.DhopOE(src_e,r_o,DaggerNo);
   Dw.Dhop  (src  ,result,DaggerNo);
-  
+
   std::cout<<GridLogMessage << "r_e"<<norm2(r_e)<<std::endl;
   std::cout<<GridLogMessage << "r_o"<<norm2(r_o)<<std::endl;
   std::cout<<GridLogMessage << "res"<<norm2(result)<<std::endl;
-  
+
   setCheckerboard(r_eo,r_o);
   setCheckerboard(r_eo,r_e);
-  
+
   err = r_eo-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
   if((norm2(err)>1.0e-4)){
-    std::cout<< "Deo RESULT\n " <<r_eo << std::endl;
+	std::cout<< "Deo RESULT\n " <<r_eo << std::endl;
-    std::cout<< "Deo REF\n " <<result  << std::endl;
+	std::cout<< "Deo REF\n " <<result  << std::endl;
-    std::cout<< "Deo ERR   \n " << err <<std::endl;
+	std::cout<< "Deo ERR   \n " << err <<std::endl;
   }
-  
+
   pickCheckerboard(Even,src_e,err);
   pickCheckerboard(Odd,src_o,err);
   std::cout<<GridLogMessage << "norm diff even  "<< norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "norm diff odd   "<< norm2(src_o)<<std::endl;
-  
+
   //assert(norm2(src_e)<1.0e-4);
   //assert(norm2(src_o)<1.0e-4);
-  
+
   Grid_finalize();
 }
-  
+

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.63])
-AC_INIT([Grid], [0.6.0], [https://github.com/paboyle/Grid], [Grid])
+AC_INIT([Grid], [0.6.0-dev], [https://github.com/paboyle/Grid], [Grid])
 AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
@@ -32,7 +32,7 @@ AC_TYPE_SIZE_T
 AC_TYPE_UINT32_T
 AC_TYPE_UINT64_T
 
-############### OpenMP 
+############### OpenMP
 AC_OPENMP
 ac_openmp=no
 if test "${OPENMP_CXXFLAGS}X" != "X"; then
@@ -63,8 +63,8 @@ AC_ARG_WITH([mpfr],
     [AM_CXXFLAGS="-I$with_mpfr/include $AM_CXXFLAGS"]
     [AM_LDFLAGS="-L$with_mpfr/lib $AM_LDFLAGS"])
 
-############### FFTW3 
+############### FFTW3
-AC_ARG_WITH([fftw],    
+AC_ARG_WITH([fftw],
             [AS_HELP_STRING([--with-fftw=prefix],
             [try this for a non-standard install prefix of the FFTW3 library])],
             [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
@@ -77,9 +77,9 @@ AC_ARG_WITH([lime],
             [AM_CXXFLAGS="-I$with_lime/include $AM_CXXFLAGS"]
             [AM_LDFLAGS="-L$with_lime/lib $AM_LDFLAGS"])
 
-############### lapack 
+############### lapack
 AC_ARG_ENABLE([lapack],
-    [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])], 
+    [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])],
     [ac_LAPACK=${enable_lapack}], [ac_LAPACK=no])
 
 case ${ac_LAPACK} in
@@ -93,6 +93,18 @@ case ${ac_LAPACK} in
         AC_DEFINE([USE_LAPACK],[1],[use LAPACK]);;
 esac
 
+############### FP16 conversions
+AC_ARG_ENABLE([sfw-fp16],
+    [AC_HELP_STRING([--enable-sfw-fp16=yes|no], [enable software fp16 comms])],
+    [ac_SFW_FP16=${enable_sfw_fp16}], [ac_SFW_FP16=yes])
+case ${ac_SFW_FP16} in
+    yes)
+      AC_DEFINE([SFW_FP16],[1],[software conversion to fp16]);;
+    no);;
+    *)
+      AC_MSG_ERROR(["SFW FP16 option not supported ${ac_SFW_FP16}"]);;
+esac
+
 ############### MKL
 AC_ARG_ENABLE([mkl],
     [AC_HELP_STRING([--enable-mkl=yes|no|prefix], [enable Intel MKL for LAPACK & FFTW])],
@@ -118,7 +130,7 @@ AC_ARG_WITH([hdf5],
 
 ############### first-touch
 AC_ARG_ENABLE([numa],
-    [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
+    [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])],
     [ac_NUMA=${enable_NUMA}],[ac_NUMA=no])
 
 case ${ac_NUMA} in
@@ -144,8 +156,8 @@ if test "${ac_MKL}x" != "nox"; then
 fi
 
 AC_SEARCH_LIBS([__gmpf_init], [gmp],
-               [AC_SEARCH_LIBS([mpfr_init], [mpfr], 
+               [AC_SEARCH_LIBS([mpfr_init], [mpfr],
-                               [AC_DEFINE([HAVE_LIBMPFR], [1], 
+                               [AC_DEFINE([HAVE_LIBMPFR], [1],
                                           [Define to 1 if you have the `MPFR' library])]
                                [have_mpfr=true], [AC_MSG_ERROR([MPFR library not found])])]
                [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library])]
@@ -154,7 +166,7 @@ AC_SEARCH_LIBS([__gmpf_init], [gmp],
 if test "${ac_LAPACK}x" != "nox"; then
     AC_SEARCH_LIBS([LAPACKE_sbdsdc], [lapack], [],
                    [AC_MSG_ERROR("LAPACK enabled but library not found")])
-fi   
+fi
 
 AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_SEARCH_LIBS([fftwf_execute], [fftw3f], [],
@@ -194,19 +206,26 @@ case ${ax_cv_cxx_compiler_vendor} in
     case ${ac_SIMD} in
       SSE4)
         AC_DEFINE([SSE4],[1],[SSE4 intrinsics])
-        SIMD_FLAGS='-msse4.2';;
+	case ${ac_SFW_FP16} in
+	  yes)
+	  SIMD_FLAGS='-msse4.2';;
+	  no)
+	  SIMD_FLAGS='-msse4.2 -mf16c';;
+	  *)
+          AC_MSG_ERROR(["SFW_FP16 must be either yes or no value ${ac_SFW_FP16} "]);;
+	esac;;
       AVX)
         AC_DEFINE([AVX1],[1],[AVX intrinsics])
-        SIMD_FLAGS='-mavx';;
+        SIMD_FLAGS='-mavx -mf16c';;
       AVXFMA4)
         AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
-        SIMD_FLAGS='-mavx -mfma4';;
+        SIMD_FLAGS='-mavx -mfma4 -mf16c';;
       AVXFMA)
         AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA3])
-        SIMD_FLAGS='-mavx -mfma';;
+        SIMD_FLAGS='-mavx -mfma -mf16c';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
-        SIMD_FLAGS='-mavx2 -mfma';;
+        SIMD_FLAGS='-mavx2 -mfma -mf16c';;
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
@@ -315,7 +334,7 @@ case ${ac_COMMS} in
         comms_type='shmem'
      ;;
      *)
-        AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); 
+        AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]);
      ;;
 esac
 case ${ac_COMMS} in
@@ -352,7 +371,7 @@ case ${ac_RNG} in
       AC_DEFINE([RNG_SITMO],[1],[RNG_SITMO] )
      ;;
      *)
-      AC_MSG_ERROR([${ac_RNG} unsupported --enable-rng option]); 
+      AC_MSG_ERROR([${ac_RNG} unsupported --enable-rng option]);
      ;;
 esac
 
@@ -369,7 +388,7 @@ case ${ac_TIMERS} in
       AC_DEFINE([TIMERS_OFF],[1],[TIMERS_OFF] )
      ;;
      *)
-      AC_MSG_ERROR([${ac_TIMERS} unsupported --enable-timers option]); 
+      AC_MSG_ERROR([${ac_TIMERS} unsupported --enable-timers option]);
      ;;
 esac
 
@@ -381,7 +400,7 @@ case ${ac_CHROMA} in
      yes|no)
      ;;
      *)
-       AC_MSG_ERROR([${ac_CHROMA} unsupported --enable-chroma option]); 
+       AC_MSG_ERROR([${ac_CHROMA} unsupported --enable-chroma option]);
      ;;
 esac
 
@@ -402,39 +421,31 @@ DX_INIT_DOXYGEN([$PACKAGE_NAME], [doxygen.cfg])
 
 ############### Ouput
 cwd=`pwd -P`; cd ${srcdir}; abs_srcdir=`pwd -P`; cd ${cwd}
+GRID_CXXFLAGS="$AM_CXXFLAGS $CXXFLAGS"
+GRID_LDFLAGS="$AM_LDFLAGS $LDFLAGS"
+GRID_LIBS=$LIBS
+GRID_SHORT_SHA=`git rev-parse --short HEAD`
+GRID_SHA=`git rev-parse HEAD`
+GRID_BRANCH=`git rev-parse --abbrev-ref HEAD`
 AM_CXXFLAGS="-I${abs_srcdir}/include $AM_CXXFLAGS"
 AM_CFLAGS="-I${abs_srcdir}/include $AM_CFLAGS"
 AM_LDFLAGS="-L${cwd}/lib $AM_LDFLAGS"
 AC_SUBST([AM_CFLAGS])
 AC_SUBST([AM_CXXFLAGS])
 AC_SUBST([AM_LDFLAGS])
-AC_CONFIG_FILES(Makefile)
+AC_SUBST([GRID_CXXFLAGS])
-AC_CONFIG_FILES(lib/Makefile)
+AC_SUBST([GRID_LDFLAGS])
-AC_CONFIG_FILES(tests/Makefile)
+AC_SUBST([GRID_LIBS])
-AC_CONFIG_FILES(tests/IO/Makefile)
+AC_SUBST([GRID_SHA])
-AC_CONFIG_FILES(tests/core/Makefile)
+AC_SUBST([GRID_BRANCH])
-AC_CONFIG_FILES(tests/debug/Makefile)
-AC_CONFIG_FILES(tests/forces/Makefile)
-AC_CONFIG_FILES(tests/hadrons/Makefile)
-AC_CONFIG_FILES(tests/hmc/Makefile)
-AC_CONFIG_FILES(tests/solver/Makefile)
-AC_CONFIG_FILES(tests/qdpxx/Makefile)
-AC_CONFIG_FILES(tests/smearing/Makefile)
-AC_CONFIG_FILES(tests/testu01/Makefile)
-AC_CONFIG_FILES(benchmarks/Makefile)
-AC_CONFIG_FILES(extras/Makefile)
-AC_CONFIG_FILES(extras/Hadrons/Makefile)
-AC_OUTPUT
 
 git_commit=`cd $srcdir && ./scripts/configure.commit`
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Summary of configuration for $PACKAGE v$VERSION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
 ----- GIT VERSION -------------------------------------
 $git_commit
-
 ----- PLATFORM ----------------------------------------
 architecture (build)        : $build_cpu
 os (build)                  : $build_os
@@ -444,10 +455,11 @@ compiler vendor             : ${ax_cv_cxx_compiler_vendor}
 compiler version            : ${ax_cv_gxx_version}
 ----- BUILD OPTIONS -----------------------------------
 SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
-Threading                   : ${ac_openmp} 
+Threading                   : ${ac_openmp}
 Communications type         : ${comms_type}
 Default precision           : ${ac_PRECISION}
-RNG choice                  : ${ac_RNG} 
+Software FP16 conversion    : ${ac_SFW_FP16}
+RNG choice                  : ${ac_RNG}
 GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
 LAPACK                      : ${ac_LAPACK}
 FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
@@ -462,6 +474,30 @@ LDFLAGS:
 LIBS:
 `echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
 -------------------------------------------------------" > grid.configure.summary
+
+GRID_SUMMARY="`cat grid.configure.summary`"
+AM_SUBST_NOTMAKE([GRID_SUMMARY])
+AC_SUBST([GRID_SUMMARY])
+
+AC_CONFIG_FILES([grid-config], [chmod +x grid-config])
+AC_CONFIG_FILES(Makefile)
+AC_CONFIG_FILES(lib/Makefile)
+AC_CONFIG_FILES(tests/Makefile)
+AC_CONFIG_FILES(tests/IO/Makefile)
+AC_CONFIG_FILES(tests/core/Makefile)
+AC_CONFIG_FILES(tests/debug/Makefile)
+AC_CONFIG_FILES(tests/forces/Makefile)
+AC_CONFIG_FILES(tests/hadrons/Makefile)
+AC_CONFIG_FILES(tests/hmc/Makefile)
+AC_CONFIG_FILES(tests/solver/Makefile)
+AC_CONFIG_FILES(tests/smearing/Makefile)
+AC_CONFIG_FILES(tests/qdpxx/Makefile)
+AC_CONFIG_FILES(tests/testu01/Makefile)
+AC_CONFIG_FILES(benchmarks/Makefile)
+AC_CONFIG_FILES(extras/Makefile)
+AC_CONFIG_FILES(extras/Hadrons/Makefile)
+AC_OUTPUT
+
 echo ""
 cat grid.configure.summary
 echo ""

extras/Hadrons/Application.cc

@@ -162,7 +162,8 @@ void Application::saveParameterFile(const std::string parameterFileName)
 sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
 
 #define DEFINE_MEMPEAK \
-auto memPeak = [this](const std::vector<unsigned int> &program)\
+GeneticScheduler<unsigned int>::ObjFunc memPeak = \
+[this](const std::vector<unsigned int> &program)\
 {\
     unsigned int memPeak;\
     bool         msg;\

extras/Hadrons/Global.hpp

@@ -145,6 +145,15 @@ std::string typeName(void)
     return typeName(typeIdPt<T>());
 }
 
+// default writers/readers
+#ifdef HAVE_HDF5
+typedef Hdf5Reader CorrReader;
+typedef Hdf5Writer CorrWriter;
+#else
+typedef XmlReader CorrReader;
+typedef XmlWriter CorrWriter;
+#endif
+
 END_HADRONS_NAMESPACE
 
 #endif // Hadrons_Global_hpp_

extras/Hadrons/Modules.hpp

@@ -29,12 +29,20 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #include <Grid/Hadrons/Modules/MAction/DWF.hpp>
 #include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
 #include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
+#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
 #include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
 #include <Grid/Hadrons/Modules/MGauge/Load.hpp>
 #include <Grid/Hadrons/Modules/MGauge/Random.hpp>
 #include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
+#include <Grid/Hadrons/Modules/MLoop/NoiseLoop.hpp>
 #include <Grid/Hadrons/Modules/MSolver/RBPrecCG.hpp>
 #include <Grid/Hadrons/Modules/MSource/Point.hpp>
 #include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
+#include <Grid/Hadrons/Modules/MSource/Wall.hpp>
 #include <Grid/Hadrons/Modules/MSource/Z2.hpp>
 #include <Grid/Hadrons/Modules/Quark.hpp>

extras/Hadrons/Modules/MContraction/Baryon.hpp

@@ -112,7 +112,7 @@ void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
                  << " quarks '" << par().q1 << "', '" << par().q2 << "', and '"
                  << par().q3 << "'" << std::endl;
     
-    XmlWriter             writer(par().output);
+    CorrWriter             writer(par().output);
     PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
     PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
     PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q2);

extras/Hadrons/Modules/MContraction/DiscLoop.hpp

@@ -0,0 +1,144 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/DiscLoop.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_DiscLoop_hpp_
+#define Hadrons_DiscLoop_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                                DiscLoop                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class DiscLoopPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(DiscLoopPar,
+                                    std::string,    q_loop,
+                                    Gamma::Algebra, gamma,
+                                    std::string,    output);
+};
+
+template <typename FImpl>
+class TDiscLoop: public Module<DiscLoopPar>
+{
+    TYPE_ALIASES(FImpl,);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma,
+                                        std::vector<Complex>, corr);
+    };
+public:
+    // constructor
+    TDiscLoop(const std::string name);
+    // destructor
+    virtual ~TDiscLoop(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(DiscLoop, TDiscLoop<FIMPL>, MContraction);
+
+/******************************************************************************
+ *                       TDiscLoop implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TDiscLoop<FImpl>::TDiscLoop(const std::string name)
+: Module<DiscLoopPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TDiscLoop<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q_loop};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TDiscLoop<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDiscLoop<FImpl>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDiscLoop<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing disconnected loop contraction '" << getName() 
+                 << "' using '" << par().q_loop << "' with " << par().gamma 
+                 << " insertion." << std::endl;
+
+    CorrWriter            writer(par().output);
+    PropagatorField       &q_loop = *env().template getObject<PropagatorField>(par().q_loop);
+    LatticeComplex        c(env().getGrid());
+    Gamma                 gamma(par().gamma);
+    std::vector<TComplex> buf;
+    Result                result;
+
+    c = trace(gamma*q_loop);
+    sliceSum(c, buf, Tp);
+
+    result.gamma = par().gamma;
+    result.corr.resize(buf.size());
+    for (unsigned int t = 0; t < buf.size(); ++t)
+    {
+        result.corr[t] = TensorRemove(buf[t]);
+    }
+
+    write(writer, "disc", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_DiscLoop_hpp_

extras/Hadrons/Modules/MContraction/Gamma3pt.hpp

@@ -0,0 +1,170 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Gamma3pt.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Gamma3pt_hpp_
+#define Hadrons_Gamma3pt_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ * 3pt contraction with gamma matrix insertion.
+ *
+ * Schematic:
+ *
+ *             q2           q3
+ *        /----<------*------<----¬
+ *       /          gamma          \
+ *      /                           \
+ *   i *                            * f
+ *      \                          /
+ *       \                        /
+ *        \----------->----------/
+ *                   q1
+ *
+ *      trace(g5*q1*adj(q2)*g5*gamma*q3)
+ */
+
+/******************************************************************************
+ *                               Gamma3pt                                     *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class Gamma3ptPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(Gamma3ptPar,
+                                    std::string,    q1,
+                                    std::string,    q2,
+                                    std::string,    q3,
+                                    Gamma::Algebra, gamma,
+                                    std::string,    output);
+};
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+class TGamma3pt: public Module<Gamma3ptPar>
+{
+    TYPE_ALIASES(FImpl1, 1);
+    TYPE_ALIASES(FImpl2, 2);
+    TYPE_ALIASES(FImpl3, 3);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma,
+                                        std::vector<Complex>, corr);
+    };
+public:
+    // constructor
+    TGamma3pt(const std::string name);
+    // destructor
+    virtual ~TGamma3pt(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Gamma3pt, ARG(TGamma3pt<FIMPL, FIMPL, FIMPL>), MContraction);
+
+/******************************************************************************
+ *                       TGamma3pt implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+TGamma3pt<FImpl1, FImpl2, FImpl3>::TGamma3pt(const std::string name)
+: Module<Gamma3ptPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3};
+    
+    return in;
+}
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TGamma3pt<FImpl1, FImpl2, FImpl3>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TGamma3pt<FImpl1, FImpl2, FImpl3>::execute(void)
+{
+    LOG(Message) << "Computing 3pt contractions '" << getName() << "' using"
+                 << " quarks '" << par().q1 << "', '" << par().q2 << "' and '"
+                 << par().q3 << "', with " << par().gamma << " insertion." 
+                 << std::endl;
+
+    CorrWriter            writer(par().output);
+    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q3);
+    LatticeComplex        c(env().getGrid());
+    Gamma                 g5(Gamma::Algebra::Gamma5);
+    Gamma                 gamma(par().gamma);
+    std::vector<TComplex> buf;
+    Result                result;
+
+    c = trace(g5*q1*adj(q2)*(g5*gamma)*q3);
+    sliceSum(c, buf, Tp);
+
+    result.gamma = par().gamma;
+    result.corr.resize(buf.size());
+    for (unsigned int t = 0; t < buf.size(); ++t)
+    {
+        result.corr[t] = TensorRemove(buf[t]);
+    }
+
+    write(writer, "gamma3pt", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Gamma3pt_hpp_

extras/Hadrons/Modules/MContraction/Meson.hpp

@@ -6,8 +6,10 @@ Source file: extras/Hadrons/Modules/MContraction/Meson.hpp
 
 Copyright (C) 2015
 Copyright (C) 2016
+Copyright (C) 2017
 
 Author: Antonin Portelli <antonin.portelli@me.com>
+        Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -36,20 +38,39 @@ See the full license in the file "LICENSE" in the top level distribution directo
 
 BEGIN_HADRONS_NAMESPACE
 
+/*
+ 
+ Meson contractions
+ -----------------------------
+ 
+ * options:
+ - q1: input propagator 1 (string)
+ - q2: input propagator 2 (string)
+ - gammas: gamma products to insert at sink & source, pairs of gamma matrices 
+           (space-separated strings) in angled brackets (i.e. <g_sink g_src>),
+           in a sequence (e.g. "<Gamma5 Gamma5><Gamma5 GammaT>").
+
+           Special values: "all" - perform all possible contractions.
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0."),
+        given as multiples of (2*pi) / L.
+*/
+
 /******************************************************************************
  *                                TMeson                                       *
  ******************************************************************************/
 BEGIN_MODULE_NAMESPACE(MContraction)
 
+typedef std::pair<Gamma::Algebra, Gamma::Algebra> GammaPair;
+
 class MesonPar: Serializable
 {
 public:
     GRID_SERIALIZABLE_CLASS_MEMBERS(MesonPar,
-                                    std::string,    q1,
+                                    std::string, q1,
-                                    std::string,    q2,
+                                    std::string, q2,
-                                    std::string,    output,
+                                    std::string, gammas,
-                                    Gamma::Algebra, gammaSource,
+                                    std::string, mom,
-                                    Gamma::Algebra, gammaSink);
+                                    std::string, output);
 };
 
 template <typename FImpl1, typename FImpl2>
@@ -61,7 +82,10 @@ public:
     class Result: Serializable
     {
     public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(Result, std::vector<Complex>, corr);
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma_snk,
+                                        Gamma::Algebra, gamma_src,
+                                        std::vector<Complex>, corr);
     };
 public:
     // constructor
@@ -71,6 +95,7 @@ public:
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+    virtual void parseGammaString(std::vector<GammaPair> &gammaList);
     // execution
     virtual void execute(void);
 };
@@ -103,6 +128,32 @@ std::vector<std::string> TMeson<FImpl1, FImpl2>::getOutput(void)
     return output;
 }
 
+template <typename FImpl1, typename FImpl2>
+void TMeson<FImpl1, FImpl2>::parseGammaString(std::vector<GammaPair> &gammaList)
+{
+    // Determine gamma matrices to insert at source/sink.
+    if (par().gammas.compare("all") == 0)
+    {
+        // Do all contractions.
+        unsigned int n_gam = Ns * Ns;
+        gammaList.resize(n_gam*n_gam);
+        for (unsigned int i = 1; i < Gamma::nGamma; i += 2)
+        {
+            for (unsigned int j = 1; j < Gamma::nGamma; j += 2)
+            {
+                gammaList.push_back(std::make_pair((Gamma::Algebra)i, 
+                                                   (Gamma::Algebra)j));
+            }
+        }
+    }
+    else
+    {
+        // Parse individual contractions from input string.
+        gammaList = strToVec<GammaPair>(par().gammas);
+    }
+}
+
+
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl1, typename FImpl2>
 void TMeson<FImpl1, FImpl2>::execute(void)
@@ -111,21 +162,44 @@ void TMeson<FImpl1, FImpl2>::execute(void)
                  << " quarks '" << par().q1 << "' and '" << par().q2 << "'"
                  << std::endl;
     
-    XmlWriter             writer(par().output);
+    CorrWriter              writer(par().output);
-    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField1       &q1 = *env().template getObject<PropagatorField1>(par().q1);
-    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    PropagatorField2       &q2 = *env().template getObject<PropagatorField2>(par().q2);
-    LatticeComplex        c(env().getGrid());
+    LatticeComplex         c(env().getGrid());
-    Gamma                 gSrc(par().gammaSource), gSnk(par().gammaSink);
+    Gamma                  g5(Gamma::Algebra::Gamma5);
-    Gamma                 g5(Gamma::Algebra::Gamma5);
+    std::vector<GammaPair> gammaList;
-    std::vector<TComplex> buf;
+    std::vector<TComplex>  buf;
-    Result                result;
+    std::vector<Result>    result;
-    
+    std::vector<Real>      p;
-    c = trace(gSnk*q1*adj(gSrc)*g5*adj(q2)*g5);
+
-    sliceSum(c, buf, Tp);
+    p  = strToVec<Real>(par().mom);
-    result.corr.resize(buf.size());
+    LatticeComplex         ph(env().getGrid()), coor(env().getGrid());
-    for (unsigned int t = 0; t < buf.size(); ++t)
+    Complex                i(0.0,1.0);
+    ph = zero;
+    for(unsigned int mu = 0; mu < env().getNd(); mu++)
     {
-        result.corr[t] = TensorRemove(buf[t]);
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
+    }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    
+    parseGammaString(gammaList);
+
+    result.resize(gammaList.size());
+    for (unsigned int i = 0; i < result.size(); ++i)
+    {
+        Gamma gSnk(gammaList[i].first);
+        Gamma gSrc(gammaList[i].second);
+        c = trace((g5*gSnk)*q1*(adj(gSrc)*g5)*adj(q2))*ph;
+        sliceSum(c, buf, Tp);
+
+        result[i].gamma_snk = gammaList[i].first;
+        result[i].gamma_src = gammaList[i].second;
+        result[i].corr.resize(buf.size());
+        for (unsigned int t = 0; t < buf.size(); ++t)
+        {
+            result[i].corr[t] = TensorRemove(buf[t]);
+        }
     }
     write(writer, "meson", result);
 }

extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp

@@ -0,0 +1,114 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakHamiltonian_hpp_
+#define Hadrons_WeakHamiltonian_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonian                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+/*******************************************************************************
+ * Utilities for contractions involving the Weak Hamiltonian.
+ ******************************************************************************/
+//// Sum and store correlator.
+#define MAKE_DIAG(exp, buf, res, n)\
+sliceSum(exp, buf, Tp);\
+res.name = (n);\
+res.corr.resize(buf.size());\
+for (unsigned int t = 0; t < buf.size(); ++t)\
+{\
+    res.corr[t] = TensorRemove(buf[t]);\
+}
+
+//// Contraction of mu index: use 'mu' variable in exp.
+#define SUM_MU(buf,exp)\
+buf = zero;\
+for (unsigned int mu = 0; mu < ndim; ++mu)\
+{\
+    buf += exp;\
+}
+
+enum 
+{
+  i_V = 0,
+  i_A = 1,
+  n_i = 2
+};
+
+class WeakHamiltonianPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WeakHamiltonianPar,
+                                    std::string, q1,
+                                    std::string, q2,
+                                    std::string, q3,
+                                    std::string, q4,
+                                    std::string, output);
+};
+
+#define MAKE_WEAK_MODULE(modname)\
+class T##modname: public Module<WeakHamiltonianPar>\
+{\
+public:\
+    TYPE_ALIASES(FIMPL,)\
+    class Result: Serializable\
+    {\
+    public:\
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,\
+                                        std::string, name,\
+                                        std::vector<Complex>, corr);\
+    };\
+public:\
+    /* constructor */ \
+    T##modname(const std::string name);\
+    /* destructor */ \
+    virtual ~T##modname(void) = default;\
+    /* dependency relation */ \
+    virtual std::vector<std::string> getInput(void);\
+    virtual std::vector<std::string> getOutput(void);\
+    /* setup */ \
+    virtual void setup(void);\
+    /* execution */ \
+    virtual void execute(void);\
+    std::vector<std::string> VA_label = {"V", "A"};\
+};\
+MODULE_REGISTER_NS(modname, T##modname, MContraction);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakHamiltonian_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc

@@ -0,0 +1,137 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian current-current contractions, Eye-type.
+ * 
+ * These contractions are generated by the Q1 and Q2 operators in the physical
+ * basis (see e.g. Fig 3 of arXiv:1507.03094).
+ * 
+ * Schematics:        q4                 |                  
+ *                  /-<-¬                |                             
+ *                 /     \               |             q2           q3
+ *                 \     /               |        /----<------*------<----¬                        
+ *            q2    \   /    q3          |       /          /-*-¬          \
+ *       /-----<-----* *-----<----¬      |      /          /     \          \
+ *    i *            H_W           * f   |   i *           \     /  q4      * f
+ *       \                        /      |      \           \->-/          /   
+ *        \                      /       |       \                        /       
+ *         \---------->---------/        |        \----------->----------/        
+ *                   q1                  |                   q1                  
+ *                                       |
+ *                Saucer (S)             |                  Eye (E)
+ * 
+ * S: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1]*q4*gL[mu][p_2])
+ * E: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1])*trace(q4*gL[mu][p_2])
+ */
+
+/******************************************************************************
+ *                  TWeakHamiltonianEye implementation                        *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakHamiltonianEye::TWeakHamiltonianEye(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakHamiltonianEye::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakHamiltonianEye::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakHamiltonianEye::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakHamiltonianEye::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian (Eye type) contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_eye_diag);
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> S_body(ndim, tmp1);
+    std::vector<PropagatorField> S_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  E_body(ndim, tmp2);
+    std::vector<LatticeComplex>  E_loop(ndim, tmp2);
+
+    // Setup for S-type contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        S_body[mu] = MAKE_SE_BODY(q1, q2, q3, GammaL(Gamma::gmu[mu]));
+        S_loop[mu] = MAKE_SE_LOOP(q4, GammaL(Gamma::gmu[mu]));
+    }
+
+    // Perform S-type contractions.    
+    SUM_MU(expbuf, trace(S_body[mu]*S_loop[mu]))
+    MAKE_DIAG(expbuf, corrbuf, result[S_diag], "HW_S")
+
+    // Recycle sub-expressions for E-type contractions.
+    for (unsigned int mu = 0; mu < ndim; ++mu)
+    {
+        E_body[mu] = trace(S_body[mu]);
+        E_loop[mu] = trace(S_loop[mu]);
+    }
+
+    // Perform E-type contractions.
+    SUM_MU(expbuf, E_body[mu]*E_loop[mu])
+    MAKE_DIAG(expbuf, corrbuf, result[E_diag], "HW_E")
+
+    write(writer, "HW_Eye", result);
+}

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp

@@ -0,0 +1,58 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakHamiltonianEye_hpp_
+#define Hadrons_WeakHamiltonianEye_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonianEye                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    S_diag = 0,
+    E_diag = 1,
+    n_eye_diag = 2
+};
+
+// Saucer and Eye subdiagram contractions.
+#define MAKE_SE_BODY(Q_1, Q_2, Q_3, gamma) (Q_3*g5*Q_1*adj(Q_2)*g5*gamma)
+#define MAKE_SE_LOOP(Q_loop, gamma) (Q_loop*gamma)
+
+MAKE_WEAK_MODULE(WeakHamiltonianEye)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakHamiltonianEye_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc

@@ -0,0 +1,139 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian current-current contractions, Non-Eye-type.
+ * 
+ * These contractions are generated by the Q1 and Q2 operators in the physical
+ * basis (see e.g. Fig 3 of arXiv:1507.03094).
+ * 
+ * Schematic:     
+ *            q2             q3          |           q2              q3
+ *          /--<--¬       /--<--¬        |        /--<--¬         /--<--¬       
+ *         /       \     /       \       |       /       \       /       \      
+ *        /         \   /         \      |      /         \     /         \     
+ *       /           \ /           \     |     /           \   /           \    
+ *    i *             * H_W         *  f |  i *             * * H_W         * f 
+ *      \             *             |    |     \           /   \           /
+ *       \           / \           /     |      \         /     \         /    
+ *        \         /   \         /      |       \       /       \       /  
+ *         \       /     \       /       |        \-->--/         \-->--/      
+ *          \-->--/       \-->--/        |          q1               q4 
+ *            q1             q4          |
+ *                Connected (C)          |                 Wing (W)
+ *
+ * C: trace(q1*adj(q2)*g5*gL[mu]*q3*adj(q4)*g5*gL[mu])
+ * W: trace(q1*adj(q2)*g5*gL[mu])*trace(q3*adj(q4)*g5*gL[mu])
+ * 
+ */
+
+/******************************************************************************
+ *                  TWeakHamiltonianNonEye implementation                     *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakHamiltonianNonEye::TWeakHamiltonianNonEye(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakHamiltonianNonEye::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakHamiltonianNonEye::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakHamiltonianNonEye::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakHamiltonianNonEye::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian (Non-Eye type) contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+    
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_noneye_diag); 
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> C_i_side_loop(ndim, tmp1);
+    std::vector<PropagatorField> C_f_side_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  W_i_side_loop(ndim, tmp2);
+    std::vector<LatticeComplex>  W_f_side_loop(ndim, tmp2);
+
+    // Setup for C-type contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        C_i_side_loop[mu] = MAKE_CW_SUBDIAG(q1, q2, GammaL(Gamma::gmu[mu]));
+        C_f_side_loop[mu] = MAKE_CW_SUBDIAG(q3, q4, GammaL(Gamma::gmu[mu]));
+    }
+
+    // Perform C-type contractions.    
+    SUM_MU(expbuf, trace(C_i_side_loop[mu]*C_f_side_loop[mu]))
+    MAKE_DIAG(expbuf, corrbuf, result[C_diag], "HW_C")
+
+    // Recycle sub-expressions for W-type contractions.
+    for (unsigned int mu = 0; mu < ndim; ++mu)
+    {
+        W_i_side_loop[mu] = trace(C_i_side_loop[mu]);
+        W_f_side_loop[mu] = trace(C_f_side_loop[mu]);
+    }
+
+    // Perform W-type contractions.
+    SUM_MU(expbuf, W_i_side_loop[mu]*W_f_side_loop[mu])
+    MAKE_DIAG(expbuf, corrbuf, result[W_diag], "HW_W")
+
+    write(writer, "HW_NonEye", result);
+}

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp

@@ -0,0 +1,57 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakHamiltonianNonEye_hpp_
+#define Hadrons_WeakHamiltonianNonEye_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonianNonEye                              *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    W_diag = 0,
+    C_diag = 1,
+    n_noneye_diag = 2
+};
+
+// Wing and Connected subdiagram contractions
+#define MAKE_CW_SUBDIAG(Q_1, Q_2, gamma) (Q_1*adj(Q_2)*g5*gamma)
+
+MAKE_WEAK_MODULE(WeakHamiltonianNonEye)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakHamiltonianNonEye_hpp_

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc

@@ -0,0 +1,135 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian + current contractions, disconnected topology for neutral 
+ * mesons.
+ * 
+ * These contractions are generated by operators Q_1,...,10 of the dS=1 Weak
+ * Hamiltonian in the physical basis and an additional current J (see e.g. 
+ * Fig 11 of arXiv:1507.03094).
+ * 
+ * Schematic:
+ *                        
+ *           q2          q4             q3
+ *       /--<--¬     /---<--¬       /---<--¬
+ *     /         \ /         \     /        \
+ *  i *           * H_W      |  J *          * f
+ *     \         / \         /     \        /
+ *      \--->---/   \-------/       \------/
+ *          q1 
+ * 
+ * options
+ * - q1: input propagator 1 (string)
+ * - q2: input propagator 2 (string)
+ * - q3: input propagator 3 (string), assumed to be sequential propagator 
+ * - q4: input propagator 4 (string), assumed to be a loop
+ * 
+ * type 1: trace(q1*adj(q2)*g5*gL[mu])*trace(loop*gL[mu])*trace(q3*g5)
+ * type 2: trace(q1*adj(q2)*g5*gL[mu]*loop*gL[mu])*trace(q3*g5)
+ */
+
+/*******************************************************************************
+ *                  TWeakNeutral4ptDisc implementation                         *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakNeutral4ptDisc::TWeakNeutral4ptDisc(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakNeutral4ptDisc::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakNeutral4ptDisc::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakNeutral4ptDisc::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakNeutral4ptDisc::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian neutral disconnected contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_neut_disc_diag);
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp(env().getGrid());
+    std::vector<PropagatorField> meson(ndim, tmp);
+    std::vector<PropagatorField> loop(ndim, tmp);
+    LatticeComplex               curr(env().getGrid());
+
+    // Setup for type 1 contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        meson[mu] = MAKE_DISC_MESON(q1, q2, GammaL(Gamma::gmu[mu]));
+        loop[mu] = MAKE_DISC_LOOP(q4, GammaL(Gamma::gmu[mu]));
+    }
+    curr = MAKE_DISC_CURR(q3, GammaL(Gamma::Algebra::Gamma5));
+
+    // Perform type 1 contractions.    
+    SUM_MU(expbuf, trace(meson[mu]*loop[mu]))
+    expbuf *= curr;
+    MAKE_DIAG(expbuf, corrbuf, result[neut_disc_1_diag], "HW_disc0_1")
+
+    // Perform type 2 contractions.
+    SUM_MU(expbuf, trace(meson[mu])*trace(loop[mu]))
+    expbuf *= curr;
+    MAKE_DIAG(expbuf, corrbuf, result[neut_disc_2_diag], "HW_disc0_2")
+
+    write(writer, "HW_disc0", result);
+}

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp

@@ -0,0 +1,59 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakNeutral4ptDisc_hpp_
+#define Hadrons_WeakNeutral4ptDisc_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakNeutral4ptDisc                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    neut_disc_1_diag = 0,
+    neut_disc_2_diag = 1,
+    n_neut_disc_diag = 2
+};
+
+// Neutral 4pt disconnected subdiagram contractions.
+#define MAKE_DISC_MESON(Q_1, Q_2, gamma) (Q_1*adj(Q_2)*g5*gamma)
+#define MAKE_DISC_LOOP(Q_LOOP, gamma) (Q_LOOP*gamma)
+#define MAKE_DISC_CURR(Q_c, gamma) (trace(Q_c*gamma))
+
+MAKE_WEAK_MODULE(WeakNeutral4ptDisc)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakNeutral4ptDisc_hpp_

extras/Hadrons/Modules/MLoop/NoiseLoop.hpp

@@ -0,0 +1,132 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MLoop/NoiseLoop.hpp
+
+Copyright (C) 2016
+
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_NoiseLoop_hpp_
+#define Hadrons_NoiseLoop_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Noise loop propagator
+ -----------------------------
+ * loop_x = q_x * adj(eta_x)
+ 
+ * options:
+ - q = Result of inversion on noise source.
+ - eta = noise source.
+
+ */
+
+
+/******************************************************************************
+ *                         NoiseLoop                                          *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MLoop)
+
+class NoiseLoopPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(NoiseLoopPar,
+                                    std::string, q,
+                                    std::string, eta);
+};
+
+template <typename FImpl>
+class TNoiseLoop: public Module<NoiseLoopPar>
+{
+public:
+    TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TNoiseLoop(const std::string name);
+    // destructor
+    virtual ~TNoiseLoop(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(NoiseLoop, TNoiseLoop<FIMPL>, MLoop);
+
+/******************************************************************************
+ *                 TNoiseLoop implementation                                  *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TNoiseLoop<FImpl>::TNoiseLoop(const std::string name)
+: Module<NoiseLoopPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TNoiseLoop<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q, par().eta};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TNoiseLoop<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TNoiseLoop<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TNoiseLoop<FImpl>::execute(void)
+{
+    PropagatorField &loop = *env().template createLattice<PropagatorField>(getName());
+    PropagatorField &q    = *env().template getObject<PropagatorField>(par().q);
+    PropagatorField &eta  = *env().template getObject<PropagatorField>(par().eta);
+    loop = q*adj(eta);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_NoiseLoop_hpp_

extras/Hadrons/Modules/MSource/SeqGamma.hpp

@@ -6,6 +6,7 @@ Source file: extras/Hadrons/Modules/MSource/SeqGamma.hpp
 
 Copyright (C) 2015
 Copyright (C) 2016
+Copyright (C) 2017
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -149,9 +150,9 @@ void TSeqGamma<FImpl>::execute(void)
     for(unsigned int mu = 0; mu < env().getNd(); mu++)
     {
         LatticeCoordinate(coor, mu);
-        ph = ph + p[mu]*coor;
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
     }
-    ph = exp(i*ph);
+    ph = exp((Real)(2*M_PI)*i*ph);
     LatticeCoordinate(t, Tp);
     src = where((t >= par().tA) and (t <= par().tB), ph*(g*q), 0.*q);
 }

extras/Hadrons/Modules/MSource/Wall.hpp

@@ -0,0 +1,147 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MSource/Wall.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WallSource_hpp_
+#define Hadrons_WallSource_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Wall source
+ -----------------------------
+ * src_x = delta(x_3 - tW) * exp(i x.mom)
+ 
+ * options:
+ - tW: source timeslice (integer)
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0.")
+ 
+ */
+
+/******************************************************************************
+ *                         Wall                                               *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSource)
+
+class WallPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WallPar,
+                                    unsigned int, tW,
+                                    std::string, mom);
+};
+
+template <typename FImpl>
+class TWall: public Module<WallPar>
+{
+public:
+    TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TWall(const std::string name);
+    // destructor
+    virtual ~TWall(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Wall, TWall<FIMPL>, MSource);
+
+/******************************************************************************
+ *                 TWall implementation                                       *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TWall<FImpl>::TWall(const std::string name)
+: Module<WallPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TWall<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TWall<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWall<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWall<FImpl>::execute(void)
+{    
+    LOG(Message) << "Generating wall source at t = " << par().tW 
+                 << " with momentum " << par().mom << std::endl;
+    
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             ph(env().getGrid()), coor(env().getGrid());
+    std::vector<Real>          p;
+    Complex                    i(0.0,1.0);
+    
+    p  = strToVec<Real>(par().mom);
+    ph = zero;
+    for(unsigned int mu = 0; mu < Nd; mu++)
+    {
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
+    }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    LatticeCoordinate(t, Tp);
+    src = 1.;
+    src = where((t == par().tW), src*ph, 0.*src);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WallSource_hpp_

extras/Hadrons/Modules/Quark.hpp

@@ -173,7 +173,7 @@ void TQuark<FImpl>::execute(void)
                 *env().template getObject<PropagatorField>(getName());
             
             axpby_ssp_pminus(sol, 0., sol, 1., sol, 0, 0);
-            axpby_ssp_pplus(sol, 0., sol, 1., sol, 0, Ls_-1);
+            axpby_ssp_pplus(sol, 1., sol, 1., sol, 0, Ls_-1);
             ExtractSlice(tmp, sol, 0, 0);
             FermToProp(p4d, tmp, s, c);
         }

extras/Hadrons/modules.inc

@@ -1,4 +1,7 @@
 modules_cc =\
+  Modules/MContraction/WeakHamiltonianEye.cc \
+  Modules/MContraction/WeakHamiltonianNonEye.cc \
+  Modules/MContraction/WeakNeutral4ptDisc.cc \
   Modules/MGauge/Load.cc \
   Modules/MGauge/Random.cc \
   Modules/MGauge/Unit.cc
@@ -7,13 +10,21 @@ modules_hpp =\
   Modules/MAction/DWF.hpp \
   Modules/MAction/Wilson.hpp \
   Modules/MContraction/Baryon.hpp \
+  Modules/MContraction/DiscLoop.hpp \
+  Modules/MContraction/Gamma3pt.hpp \
   Modules/MContraction/Meson.hpp \
+  Modules/MContraction/WeakHamiltonian.hpp \
+  Modules/MContraction/WeakHamiltonianEye.hpp \
+  Modules/MContraction/WeakHamiltonianNonEye.hpp \
+  Modules/MContraction/WeakNeutral4ptDisc.hpp \
   Modules/MGauge/Load.hpp \
   Modules/MGauge/Random.hpp \
   Modules/MGauge/Unit.hpp \
+  Modules/MLoop/NoiseLoop.hpp \
   Modules/MSolver/RBPrecCG.hpp \
   Modules/MSource/Point.hpp \
   Modules/MSource/SeqGamma.hpp \
+  Modules/MSource/Wall.hpp \
   Modules/MSource/Z2.hpp \
   Modules/Quark.hpp
 

grid-config.in

@@ -0,0 +1,86 @@
+#! /bin/sh
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+includedir=@includedir@
+
+usage()
+{
+  cat <<EOF
+Usage: grid-config [OPTION]
+
+Known values for OPTION are:
+
+  --prefix     show Grid installation prefix
+  --cxxflags   print pre-processor and compiler flags
+  --ldflags    print library linking flags
+  --libs       print library linking information
+  --summary    print full build summary
+  --help       display this help and exit
+  --version    output version information
+  --git        print git revision
+
+EOF
+  
+  exit $1
+}
+
+if test $# -eq 0; then
+  usage 1
+fi
+
+cflags=false
+libs=false
+
+while test $# -gt 0; do
+  case "$1" in
+    -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+    *) optarg= ;;
+  esac
+  
+  case "$1" in
+    --prefix)
+      echo $prefix
+    ;;
+    
+    --version)
+      echo @VERSION@
+      exit 0
+    ;;
+    
+    --git)
+      echo "@GRID_BRANCH@ @GRID_SHA@"
+      exit 0
+    ;;
+    
+    --help)
+      usage 0
+    ;;
+    
+    --cxxflags)
+      echo @GRID_CXXFLAGS@
+    ;;
+    
+    --ldflags)
+      echo @GRID_LDFLAGS@
+    ;;
+    
+    --libs)
+      echo @GRID_LIBS@
+    ;;
+    
+    --summary)
+      echo ""
+      echo "@GRID_SUMMARY@"
+      echo ""
+    ;;
+    
+    *)
+      usage
+      exit 1
+    ;;
+  esac
+  shift
+done
+
+exit 0

lib/GridCore.h

@@ -38,28 +38,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_BASE_H
 #define GRID_BASE_H
 
-///////////////////
+#include <Grid/GridStd.h>
-// Std C++ dependencies
-///////////////////
-#include <cassert>
-#include <complex>
-#include <vector>
-#include <iostream>
-#include <iomanip>
-#include <random>
-#include <functional>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <signal.h>
-#include <ctime>
-#include <sys/time.h>
-#include <chrono>
-
-///////////////////
-// Grid headers
-///////////////////
-#include "Config.h"
 
 #include <Grid/perfmon/Timer.h>
 #include <Grid/perfmon/PerfCount.h>

lib/GridStd.h

@@ -0,0 +1,27 @@
+#ifndef GRID_STD_H
+#define GRID_STD_H
+
+///////////////////
+// Std C++ dependencies
+///////////////////
+#include <cassert>
+#include <complex>
+#include <vector>
+#include <iostream>
+#include <iomanip>
+#include <random>
+#include <functional>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <ctime>
+#include <sys/time.h>
+#include <chrono>
+
+///////////////////
+// Grid config
+///////////////////
+#include "Config.h"
+
+#endif /* GRID_STD_H */

lib/algorithms/Algorithms.h

@@ -46,7 +46,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/iterative/ConjugateGradientMixedPrec.h>
 
 // Lanczos support
-#include <Grid/algorithms/iterative/MatrixUtils.h>
+//#include <Grid/algorithms/iterative/MatrixUtils.h>
 #include <Grid/algorithms/iterative/ImplicitlyRestartedLanczos.h>
 #include <Grid/algorithms/CoarsenedMatrix.h>
 #include <Grid/algorithms/FFT.h>

lib/algorithms/approx/Remez.h

@@ -16,7 +16,7 @@
 #define INCLUDED_ALG_REMEZ_H
 
 #include <stddef.h>
-#include <Config.h>
+#include <Grid/GridStd.h>
 
 #ifdef HAVE_LIBGMP
 #include "bigfloat.h"

lib/algorithms/iterative/BlockConjugateGradient.h

@@ -0,0 +1,366 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/algorithms/iterative/BlockConjugateGradient.h
+
+Copyright (C) 2017
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_BLOCK_CONJUGATE_GRADIENT_H
+#define GRID_BLOCK_CONJUGATE_GRADIENT_H
+
+
+namespace Grid {
+
+//////////////////////////////////////////////////////////////////////////
+// Block conjugate gradient. Dimension zero should be the block direction
+//////////////////////////////////////////////////////////////////////////
+template <class Field>
+class BlockConjugateGradient : public OperatorFunction<Field> {
+ public:
+
+  typedef typename Field::scalar_type scomplex;
+
+  const int blockDim = 0;
+
+  int Nblock;
+  bool ErrorOnNoConverge;  // throw an assert when the CG fails to converge.
+                           // Defaults true.
+  RealD Tolerance;
+  Integer MaxIterations;
+  Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
+  
+  BlockConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
+    : Tolerance(tol),
+    MaxIterations(maxit),
+    ErrorOnNoConverge(err_on_no_conv){};
+
+void operator()(LinearOperatorBase<Field> &Linop, const Field &Src, Field &Psi) 
+{
+  int Orthog = 0; // First dimension is block dim
+  Nblock = Src._grid->_fdimensions[Orthog];
+
+  std::cout<<GridLogMessage<<" Block Conjugate Gradient : Orthog "<<Orthog<<" Nblock "<<Nblock<<std::endl;
+
+  Psi.checkerboard = Src.checkerboard;
+  conformable(Psi, Src);
+
+  Field P(Src);
+  Field AP(Src);
+  Field R(Src);
+  
+  Eigen::MatrixXcd m_pAp    = Eigen::MatrixXcd::Identity(Nblock,Nblock);
+  Eigen::MatrixXcd m_pAp_inv= Eigen::MatrixXcd::Identity(Nblock,Nblock);
+  Eigen::MatrixXcd m_rr     = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+  Eigen::MatrixXcd m_rr_inv = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+
+  Eigen::MatrixXcd m_alpha      = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+  Eigen::MatrixXcd m_beta   = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+
+  // Initial residual computation & set up
+  std::vector<RealD> residuals(Nblock);
+  std::vector<RealD> ssq(Nblock);
+
+  sliceNorm(ssq,Src,Orthog);
+  RealD sssum=0;
+  for(int b=0;b<Nblock;b++) sssum+=ssq[b];
+
+  sliceNorm(residuals,Src,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  sliceNorm(residuals,Psi,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  // Initial search dir is guess
+  Linop.HermOp(Psi, AP);
+  
+
+  /************************************************************************
+   * Block conjugate gradient (Stephen Pickles, thesis 1995, pp 71, O Leary 1980)
+   ************************************************************************
+   * O'Leary : R = B - A X
+   * O'Leary : P = M R ; preconditioner M = 1
+   * O'Leary : alpha = PAP^{-1} RMR
+   * O'Leary : beta  = RMR^{-1}_old RMR_new
+   * O'Leary : X=X+Palpha
+   * O'Leary : R_new=R_old-AP alpha
+   * O'Leary : P=MR_new+P beta
+   */
+
+  R = Src - AP;  
+  P = R;
+  sliceInnerProductMatrix(m_rr,R,R,Orthog);
+
+  GridStopWatch sliceInnerTimer;
+  GridStopWatch sliceMaddTimer;
+  GridStopWatch MatrixTimer;
+  GridStopWatch SolverTimer;
+  SolverTimer.Start();
+
+  int k;
+  for (k = 1; k <= MaxIterations; k++) {
+
+    RealD rrsum=0;
+    for(int b=0;b<Nblock;b++) rrsum+=real(m_rr(b,b));
+
+    std::cout << GridLogIterative << "\titeration "<<k<<" rr_sum "<<rrsum<<" ssq_sum "<< sssum
+	      <<" / "<<std::sqrt(rrsum/sssum) <<std::endl;
+
+    MatrixTimer.Start();
+    Linop.HermOp(P, AP);
+    MatrixTimer.Stop();
+
+    // Alpha
+    sliceInnerTimer.Start();
+    sliceInnerProductMatrix(m_pAp,P,AP,Orthog);
+    sliceInnerTimer.Stop();
+    m_pAp_inv = m_pAp.inverse();
+    m_alpha   = m_pAp_inv * m_rr ;
+
+    // Psi, R update
+    sliceMaddTimer.Start();
+    sliceMaddMatrix(Psi,m_alpha, P,Psi,Orthog);     // add alpha *  P to psi
+    sliceMaddMatrix(R  ,m_alpha,AP,  R,Orthog,-1.0);// sub alpha * AP to resid
+    sliceMaddTimer.Stop();
+
+    // Beta
+    m_rr_inv = m_rr.inverse();
+    sliceInnerTimer.Start();
+    sliceInnerProductMatrix(m_rr,R,R,Orthog);
+    sliceInnerTimer.Stop();
+    m_beta = m_rr_inv *m_rr;
+
+    // Search update
+    sliceMaddTimer.Start();
+    sliceMaddMatrix(AP,m_beta,P,R,Orthog);
+    sliceMaddTimer.Stop();
+    P= AP;
+
+    /*********************
+     * convergence monitor
+     *********************
+     */
+    RealD max_resid=0;
+    for(int b=0;b<Nblock;b++){
+      RealD rr = real(m_rr(b,b))/ssq[b];
+      if ( rr > max_resid ) max_resid = rr;
+    }
+    
+    if ( max_resid < Tolerance*Tolerance ) { 
+
+      SolverTimer.Stop();
+
+      std::cout << GridLogMessage<<"BlockCG converged in "<<k<<" iterations"<<std::endl;
+      for(int b=0;b<Nblock;b++){
+	std::cout << GridLogMessage<< "\t\tblock "<<b<<" resid "<< std::sqrt(real(m_rr(b,b))/ssq[b])<<std::endl;
+      }
+      std::cout << GridLogMessage<<"\tMax residual is "<<std::sqrt(max_resid)<<std::endl;
+
+      Linop.HermOp(Psi, AP);
+      AP = AP-Src;
+      std::cout << GridLogMessage <<"\tTrue residual is " << std::sqrt(norm2(AP)/norm2(Src)) <<std::endl;
+
+      std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
+      std::cout << GridLogMessage << "\tElapsed    " << SolverTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tMatrix     " << MatrixTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tInnerProd  " << sliceInnerTimer.Elapsed() <<std::endl;
+      std::cout << GridLogMessage << "\tMaddMatrix " << sliceMaddTimer.Elapsed()  <<std::endl;
+	    
+      IterationsToComplete = k;
+      return;
+    }
+
+  }
+  std::cout << GridLogMessage << "BlockConjugateGradient did NOT converge" << std::endl;
+
+  if (ErrorOnNoConverge) assert(0);
+  IterationsToComplete = k;
+}
+};
+
+
+//////////////////////////////////////////////////////////////////////////
+// multiRHS conjugate gradient. Dimension zero should be the block direction
+//////////////////////////////////////////////////////////////////////////
+template <class Field>
+class MultiRHSConjugateGradient : public OperatorFunction<Field> {
+ public:
+
+  typedef typename Field::scalar_type scomplex;
+
+  const int blockDim = 0;
+
+  int Nblock;
+  bool ErrorOnNoConverge;  // throw an assert when the CG fails to converge.
+                           // Defaults true.
+  RealD Tolerance;
+  Integer MaxIterations;
+  Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
+  
+   MultiRHSConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
+    : Tolerance(tol),
+    MaxIterations(maxit),
+    ErrorOnNoConverge(err_on_no_conv){};
+
+void operator()(LinearOperatorBase<Field> &Linop, const Field &Src, Field &Psi) 
+{
+  int Orthog = 0; // First dimension is block dim
+  Nblock = Src._grid->_fdimensions[Orthog];
+
+  std::cout<<GridLogMessage<<"MultiRHS Conjugate Gradient : Orthog "<<Orthog<<" Nblock "<<Nblock<<std::endl;
+
+  Psi.checkerboard = Src.checkerboard;
+  conformable(Psi, Src);
+
+  Field P(Src);
+  Field AP(Src);
+  Field R(Src);
+  
+  std::vector<ComplexD> v_pAp(Nblock);
+  std::vector<RealD> v_rr (Nblock);
+  std::vector<RealD> v_rr_inv(Nblock);
+  std::vector<RealD> v_alpha(Nblock);
+  std::vector<RealD> v_beta(Nblock);
+
+  // Initial residual computation & set up
+  std::vector<RealD> residuals(Nblock);
+  std::vector<RealD> ssq(Nblock);
+
+  sliceNorm(ssq,Src,Orthog);
+  RealD sssum=0;
+  for(int b=0;b<Nblock;b++) sssum+=ssq[b];
+
+  sliceNorm(residuals,Src,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  sliceNorm(residuals,Psi,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  // Initial search dir is guess
+  Linop.HermOp(Psi, AP);
+
+  R = Src - AP;  
+  P = R;
+  sliceNorm(v_rr,R,Orthog);
+
+  GridStopWatch sliceInnerTimer;
+  GridStopWatch sliceMaddTimer;
+  GridStopWatch sliceNormTimer;
+  GridStopWatch MatrixTimer;
+  GridStopWatch SolverTimer;
+
+  SolverTimer.Start();
+  int k;
+  for (k = 1; k <= MaxIterations; k++) {
+
+    RealD rrsum=0;
+    for(int b=0;b<Nblock;b++) rrsum+=real(v_rr[b]);
+
+    std::cout << GridLogIterative << "\titeration "<<k<<" rr_sum "<<rrsum<<" ssq_sum "<< sssum
+	      <<" / "<<std::sqrt(rrsum/sssum) <<std::endl;
+
+    MatrixTimer.Start();
+    Linop.HermOp(P, AP);
+    MatrixTimer.Stop();
+
+    // Alpha
+    //    sliceInnerProductVectorTest(v_pAp_test,P,AP,Orthog);
+    sliceInnerTimer.Start();
+    sliceInnerProductVector(v_pAp,P,AP,Orthog);
+    sliceInnerTimer.Stop();
+    for(int b=0;b<Nblock;b++){
+      //      std::cout << " "<< v_pAp[b]<<" "<< v_pAp_test[b]<<std::endl;
+      v_alpha[b] = v_rr[b]/real(v_pAp[b]);
+    }
+
+    // Psi, R update
+    sliceMaddTimer.Start();
+    sliceMaddVector(Psi,v_alpha, P,Psi,Orthog);     // add alpha *  P to psi
+    sliceMaddVector(R  ,v_alpha,AP,  R,Orthog,-1.0);// sub alpha * AP to resid
+    sliceMaddTimer.Stop();
+
+    // Beta
+    for(int b=0;b<Nblock;b++){
+      v_rr_inv[b] = 1.0/v_rr[b];
+    }
+    sliceNormTimer.Start();
+    sliceNorm(v_rr,R,Orthog);
+    sliceNormTimer.Stop();
+    for(int b=0;b<Nblock;b++){
+      v_beta[b] = v_rr_inv[b] *v_rr[b];
+    }
+
+    // Search update
+    sliceMaddTimer.Start();
+    sliceMaddVector(P,v_beta,P,R,Orthog);
+    sliceMaddTimer.Stop();
+
+    /*********************
+     * convergence monitor
+     *********************
+     */
+    RealD max_resid=0;
+    for(int b=0;b<Nblock;b++){
+      RealD rr = v_rr[b]/ssq[b];
+      if ( rr > max_resid ) max_resid = rr;
+    }
+    
+    if ( max_resid < Tolerance*Tolerance ) { 
+
+      SolverTimer.Stop();
+
+      std::cout << GridLogMessage<<"MultiRHS solver converged in " <<k<<" iterations"<<std::endl;
+      for(int b=0;b<Nblock;b++){
+	std::cout << GridLogMessage<< "\t\tBlock "<<b<<" resid "<< std::sqrt(v_rr[b]/ssq[b])<<std::endl;
+      }
+      std::cout << GridLogMessage<<"\tMax residual is "<<std::sqrt(max_resid)<<std::endl;
+
+      Linop.HermOp(Psi, AP);
+      AP = AP-Src;
+      std::cout <<GridLogMessage << "\tTrue residual is " << std::sqrt(norm2(AP)/norm2(Src)) <<std::endl;
+
+      std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
+      std::cout << GridLogMessage << "\tElapsed    " << SolverTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tMatrix     " << MatrixTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tInnerProd  " << sliceInnerTimer.Elapsed() <<std::endl;
+      std::cout << GridLogMessage << "\tNorm       " << sliceNormTimer.Elapsed() <<std::endl;
+      std::cout << GridLogMessage << "\tMaddMatrix " << sliceMaddTimer.Elapsed()  <<std::endl;
+
+
+      IterationsToComplete = k;
+      return;
+    }
+
+  }
+  std::cout << GridLogMessage << "MultiRHSConjugateGradient did NOT converge" << std::endl;
+
+  if (ErrorOnNoConverge) assert(0);
+  IterationsToComplete = k;
+}
+};
+
+
+
+}
+#endif

lib/algorithms/iterative/ConjugateGradient.h

@@ -78,18 +78,12 @@ class ConjugateGradient : public OperatorFunction<Field> {
     cp = a;
     ssq = norm2(src);
 
-    std::cout << GridLogIterative << std::setprecision(4)
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient: guess " << guess << std::endl;
-              << "ConjugateGradient: guess " << guess << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:   src " << ssq << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:    mp " << d << std::endl;
-              << "ConjugateGradient:   src " << ssq << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:   mmp " << b << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:  cp,r " << cp << std::endl;
-              << "ConjugateGradient:    mp " << d << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:     p " << a << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:   mmp " << b << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:  cp,r " << cp << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:     p " << a << std::endl;
 
     RealD rsq = Tolerance * Tolerance * ssq;
 
@@ -99,8 +93,7 @@ class ConjugateGradient : public OperatorFunction<Field> {
     }
 
     std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient: k=0 residual " << cp << " target " << rsq
+              << "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
-              << std::endl;
 
     GridStopWatch LinalgTimer;
     GridStopWatch MatrixTimer;
@@ -148,19 +141,20 @@ class ConjugateGradient : public OperatorFunction<Field> {
         RealD resnorm = sqrt(norm2(p));
         RealD true_residual = resnorm / srcnorm;
 
-        std::cout << GridLogMessage
+        std::cout << GridLogMessage << "ConjugateGradient Converged on iteration " << k << std::endl;
-                  << "ConjugateGradient: Converged on iteration " << k << std::endl;
+        std::cout << GridLogMessage << "\tComputed residual " << sqrt(cp / ssq)<<std::endl;
-        std::cout << GridLogMessage << "Computed residual " << sqrt(cp / ssq)
+	std::cout << GridLogMessage << "\tTrue residual " << true_residual<<std::endl;
-                  << " true residual " << true_residual << " target "
+	std::cout << GridLogMessage << "\tTarget " << Tolerance << std::endl;
-                  << Tolerance << std::endl;
+
-        std::cout << GridLogMessage << "Time elapsed: Iterations "
+        std::cout << GridLogMessage << "Time breakdown "<<std::endl;
-                  << SolverTimer.Elapsed() << " Matrix  "
+	std::cout << GridLogMessage << "\tElapsed    " << SolverTimer.Elapsed() <<std::endl;
-                  << MatrixTimer.Elapsed() << " Linalg "
+	std::cout << GridLogMessage << "\tMatrix     " << MatrixTimer.Elapsed() <<std::endl;
-                  << LinalgTimer.Elapsed();
+	std::cout << GridLogMessage << "\tLinalg     " << LinalgTimer.Elapsed() <<std::endl;
-        std::cout << std::endl;
 
         if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
+
 	IterationsToComplete = k;	
+
         return;
       }
     }

lib/algorithms/iterative/ImplicitlyRestartedLanczos.h

@@ -30,6 +30,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define GRID_IRL_H
 
 #include <string.h> //memset
+
 #ifdef USE_LAPACK
 void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
                    double *vl, double *vu, int *il, int *iu, double *abstol,
@@ -37,8 +38,9 @@ void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
                    double *work, int *lwork, int *iwork, int *liwork,
                    int *info);
 #endif
-#include "DenseMatrix.h"
+
-#include "EigenSort.h"
+#include <Grid/algorithms/densematrix/DenseMatrix.h>
+#include <Grid/algorithms/iterative/EigenSort.h>
 
 namespace Grid {
 
@@ -1088,8 +1090,6 @@ static void Lock(DenseMatrix<T> &H, 	// Hess mtx
 		 int dfg,
 		 bool herm)
 {	
-
-
   //ForceTridiagonal(H);
 
   int M = H.dim;
@@ -1121,7 +1121,6 @@ static void Lock(DenseMatrix<T> &H, 	// Hess mtx
 
   AH = Hermitian(QQ)*AH;
   AH = AH*QQ;
-	
 
   for(int i=con;i<M;i++){
     for(int j=con;j<M;j++){

lib/algorithms/iterative/Matrix.h

@@ -1,453 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/iterative/Matrix.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef MATRIX_H
-#define MATRIX_H
-
-#include <cstdlib>
-#include <string>
-#include <cmath>
-#include <vector>
-#include <iostream>
-#include <iomanip>
-#include <complex>
-#include <typeinfo>
-#include <Grid/Grid.h>
-
-
-/** Sign function **/
-template <class T> T sign(T p){return ( p/abs(p) );}
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////////
-///////////////////// Hijack STL containers for our wicked means /////////////////////////////////////////
-/////////////////////////////////////////////////////////////////////////////////////////////////////////
-template<class T> using Vector = Vector<T>;
-template<class T> using Matrix = Vector<Vector<T> >;
-
-template<class T> void Resize(Vector<T > & vec, int N) { vec.resize(N); }
-
-template<class T> void Resize(Matrix<T > & mat, int N, int M) { 
-  mat.resize(N);
-  for(int i=0;i<N;i++){
-    mat[i].resize(M);
-  }
-}
-template<class T> void Size(Vector<T> & vec, int &N) 
-{ 
-  N= vec.size();
-}
-template<class T> void Size(Matrix<T> & mat, int &N,int &M) 
-{ 
-  N= mat.size();
-  M= mat[0].size();
-}
-template<class T> void SizeSquare(Matrix<T> & mat, int &N) 
-{ 
-  int M; Size(mat,N,M);
-  assert(N==M);
-}
-template<class T> void SizeSame(Matrix<T> & mat1,Matrix<T> &mat2, int &N1,int &M1) 
-{ 
-  int N2,M2;
-  Size(mat1,N1,M1);
-  Size(mat2,N2,M2);
-  assert(N1==N2);
-  assert(M1==M2);
-}
-
-//*****************************************
-//*	(Complex) Vector operations	*
-//*****************************************
-
-/**Conj of a Vector **/
-template <class T> Vector<T> conj(Vector<T> p){
-	Vector<T> q(p.size());
-	for(int i=0;i<p.size();i++){q[i] = conj(p[i]);}
-	return q;
-}
-
-/** Norm of a Vector**/
-template <class T> T norm(Vector<T> p){
-	T sum = 0;
-	for(int i=0;i<p.size();i++){sum = sum + p[i]*conj(p[i]);}
-	return abs(sqrt(sum));
-}
-
-/** Norm squared of a Vector **/
-template <class T> T norm2(Vector<T> p){
-	T sum = 0;
-	for(int i=0;i<p.size();i++){sum = sum + p[i]*conj(p[i]);}
-	return abs((sum));
-}
-
-/** Sum elements of a Vector **/
-template <class T> T trace(Vector<T> p){
-	T sum = 0;
-	for(int i=0;i<p.size();i++){sum = sum + p[i];}
-	return sum;
-}
-
-/** Fill a Vector with constant c **/
-template <class T> void Fill(Vector<T> &p, T c){
-	for(int i=0;i<p.size();i++){p[i] = c;}
-}
-/** Normalize a Vector **/
-template <class T> void normalize(Vector<T> &p){
-	T m = norm(p);
-	if( abs(m) > 0.0) for(int i=0;i<p.size();i++){p[i] /= m;}
-}
-/** Vector by scalar **/
-template <class T, class U> Vector<T> times(Vector<T> p, U s){
-	for(int i=0;i<p.size();i++){p[i] *= s;}
-	return p;
-}
-template <class T, class U> Vector<T> times(U s, Vector<T> p){
-	for(int i=0;i<p.size();i++){p[i] *= s;}
-	return p;
-}
-/** inner product of a and b = conj(a) . b **/
-template <class T> T inner(Vector<T> a, Vector<T> b){
-	T m = 0.;
-	for(int i=0;i<a.size();i++){m = m + conj(a[i])*b[i];}
-	return m;
-}
-/** sum of a and b = a + b **/
-template <class T> Vector<T> add(Vector<T> a, Vector<T> b){
-	Vector<T> m(a.size());
-	for(int i=0;i<a.size();i++){m[i] = a[i] + b[i];}
-	return m;
-}
-/** sum of a and b = a - b **/
-template <class T> Vector<T> sub(Vector<T> a, Vector<T> b){
-	Vector<T> m(a.size());
-	for(int i=0;i<a.size();i++){m[i] = a[i] - b[i];}
-	return m;
-}
-
-/** 
- *********************************
- *	Matrices	         *
- *********************************
- **/
-
-template<class T> void Fill(Matrix<T> & mat, T&val) { 
-  int N,M;
-  Size(mat,N,M);
-  for(int i=0;i<N;i++){
-  for(int j=0;j<M;j++){
-    mat[i][j] = val;
-  }}
-}
-
-/** Transpose of a matrix **/
-Matrix<T> Transpose(Matrix<T> & mat){
-  int N,M;
-  Size(mat,N,M);
-  Matrix C; Resize(C,M,N);
-  for(int i=0;i<M;i++){
-  for(int j=0;j<N;j++){
-    C[i][j] = mat[j][i];
-  }} 
-  return C;
-}
-/** Set Matrix to unit matrix **/
-template<class T> void Unity(Matrix<T> &mat){
-  int N;  SizeSquare(mat,N);
-  for(int i=0;i<N;i++){
-    for(int j=0;j<N;j++){
-      if ( i==j ) A[i][j] = 1;
-      else        A[i][j] = 0;
-    } 
-  } 
-}
-/** Add C * I to matrix **/
-template<class T>
-void PlusUnit(Matrix<T> & A,T c){
-  int dim;  SizeSquare(A,dim);
-  for(int i=0;i<dim;i++){A[i][i] = A[i][i] + c;} 
-}
-
-/** return the Hermitian conjugate of matrix **/
-Matrix<T> HermitianConj(Matrix<T> &mat){
-
-  int dim; SizeSquare(mat,dim);
-
-  Matrix<T> C; Resize(C,dim,dim);
-
-  for(int i=0;i<dim;i++){
-    for(int j=0;j<dim;j++){
-      C[i][j] = conj(mat[j][i]);
-    } 
-  } 
-  return C;
-}
-
-/** return diagonal entries as a Vector **/
-Vector<T> diag(Matrix<T> &A)
-{
-  int dim; SizeSquare(A,dim);
-  Vector<T> d; Resize(d,dim);
-
-  for(int i=0;i<dim;i++){
-    d[i] = A[i][i];
-  }
-  return d;
-}
-
-/** Left multiply by a Vector **/
-Vector<T> operator *(Vector<T> &B,Matrix<T> &A)
-{
-  int K,M,N; 
-  Size(B,K);
-  Size(A,M,N);
-  assert(K==M);
-  
-  Vector<T> C; Resize(C,N);
-
-  for(int j=0;j<N;j++){
-    T sum = 0.0;
-    for(int i=0;i<M;i++){
-      sum += B[i] * A[i][j];
-    }
-    C[j] =  sum;
-  }
-  return C; 
-}
-
-/** return 1/diagonal entries as a Vector **/
-Vector<T> inv_diag(Matrix<T> & A){
-  int dim; SizeSquare(A,dim);
-  Vector<T> d; Resize(d,dim);
-  for(int i=0;i<dim;i++){
-    d[i] = 1.0/A[i][i];
-  }
-  return d;
-}
-/** Matrix Addition **/
-inline Matrix<T> operator + (Matrix<T> &A,Matrix<T> &B)
-{
-  int N,M  ; SizeSame(A,B,N,M);
-  Matrix C; Resize(C,N,M);
-  for(int i=0;i<N;i++){
-    for(int j=0;j<M;j++){
-      C[i][j] = A[i][j] +  B[i][j];
-    } 
-  } 
-  return C;
-} 
-/** Matrix Subtraction **/
-inline Matrix<T> operator- (Matrix<T> & A,Matrix<T> &B){
-  int N,M  ; SizeSame(A,B,N,M);
-  Matrix C; Resize(C,N,M);
-  for(int i=0;i<N;i++){
-  for(int j=0;j<M;j++){
-    C[i][j] = A[i][j] -  B[i][j];
-  }}
-  return C;
-} 
-
-/** Matrix scalar multiplication **/
-inline Matrix<T> operator* (Matrix<T> & A,T c){
-  int N,M; Size(A,N,M);
-  Matrix C; Resize(C,N,M);
-  for(int i=0;i<N;i++){
-  for(int j=0;j<M;j++){
-    C[i][j] = A[i][j]*c;
-  }} 
-  return C;
-} 
-/** Matrix Matrix multiplication **/
-inline Matrix<T> operator* (Matrix<T> &A,Matrix<T> &B){
-  int K,L,N,M;
-  Size(A,K,L);
-  Size(B,N,M); assert(L==N);
-  Matrix C; Resize(C,K,M);
-
-  for(int i=0;i<K;i++){
-    for(int j=0;j<M;j++){
-      T sum = 0.0;
-      for(int k=0;k<N;k++) sum += A[i][k]*B[k][j];
-      C[i][j] =sum;
-    }
-  }
-  return C; 
-} 
-/** Matrix Vector multiplication **/
-inline Vector<T> operator* (Matrix<T> &A,Vector<T> &B){
-  int M,N,K;
-  Size(A,N,M);
-  Size(B,K); assert(K==M);
-  Vector<T> C; Resize(C,N);
-  for(int i=0;i<N;i++){
-    T sum = 0.0;
-    for(int j=0;j<M;j++) sum += A[i][j]*B[j];
-    C[i] =  sum;
-  }
-  return C; 
-} 
-
-/** Some version of Matrix norm **/
-/*
-inline T Norm(){ // this is not a usual L2 norm
-    T norm = 0;
-    for(int i=0;i<dim;i++){
-      for(int j=0;j<dim;j++){
-	norm += abs(A[i][j]);
-    }}
-    return norm;
-  }
-*/
-
-/** Some version of Matrix norm **/
-template<class T> T LargestDiag(Matrix<T> &A)
-{
-  int dim ; SizeSquare(A,dim); 
-
-  T ld = abs(A[0][0]);
-  for(int i=1;i<dim;i++){
-    T cf = abs(A[i][i]);
-    if(abs(cf) > abs(ld) ){ld = cf;}
-  }
-  return ld;
-}
-
-/** Look for entries on the leading subdiagonal that are smaller than 'small' **/
-template <class T,class U> int Chop_subdiag(Matrix<T> &A,T norm, int offset, U small)
-{
-  int dim; SizeSquare(A,dim);
-  for(int l = dim - 1 - offset; l >= 1; l--) {             		
-    if((U)abs(A[l][l - 1]) < (U)small) {
-      A[l][l-1]=(U)0.0;
-      return l;
-    }
-  }
-  return 0;
-}
-
-/** Look for entries on the leading subdiagonal that are smaller than 'small' **/
-template <class T,class U> int Chop_symm_subdiag(Matrix<T> & A,T norm, int offset, U small) 
-{
-  int dim; SizeSquare(A,dim);
-  for(int l = dim - 1 - offset; l >= 1; l--) {
-    if((U)abs(A[l][l - 1]) < (U)small) {
-      A[l][l - 1] = (U)0.0;
-      A[l - 1][l] = (U)0.0;
-      return l;
-    }
-  }
-  return 0;
-}
-/**Assign a submatrix to a larger one**/
-template<class T>
-void AssignSubMtx(Matrix<T> & A,int row_st, int row_end, int col_st, int col_end, Matrix<T> &S)
-{
-  for(int i = row_st; i<row_end; i++){
-    for(int j = col_st; j<col_end; j++){
-      A[i][j] = S[i - row_st][j - col_st];
-    }
-  }
-}
-
-/**Get a square submatrix**/
-template <class T>
-Matrix<T> GetSubMtx(Matrix<T> &A,int row_st, int row_end, int col_st, int col_end)
-{
-  Matrix<T> H; Resize(row_end - row_st,col_end-col_st);
-
-  for(int i = row_st; i<row_end; i++){
-  for(int j = col_st; j<col_end; j++){
-    H[i-row_st][j-col_st]=A[i][j];
-  }}
-  return H;
-}
-  
- /**Assign a submatrix to a larger one NB remember Vector Vectors are transposes of the matricies they represent**/
-template<class T>
-void AssignSubMtx(Matrix<T> & A,int row_st, int row_end, int col_st, int col_end, Matrix<T> &S)
-{
-  for(int i = row_st; i<row_end; i++){
-  for(int j = col_st; j<col_end; j++){
-    A[i][j] = S[i - row_st][j - col_st];
-  }}
-}
-  
-/** compute b_i A_ij b_j **/ // surprised no Conj
-template<class T> T proj(Matrix<T> A, Vector<T> B){
-  int dim; SizeSquare(A,dim);
-  int dimB; Size(B,dimB);
-  assert(dimB==dim);
-  T C = 0;
-  for(int i=0;i<dim;i++){
-    T sum = 0.0;
-    for(int j=0;j<dim;j++){
-      sum += A[i][j]*B[j];
-    }
-    C +=  B[i]*sum; // No conj?
-  }
-  return C; 
-}
-
-
-/*
- *************************************************************
- *
- * Matrix Vector products
- *
- *************************************************************
- */
-// Instead make a linop and call my CG;
-
-/// q -> q Q
-template <class T,class Fermion> void times(Vector<Fermion> &q, Matrix<T> &Q)
-{
-  int M; SizeSquare(Q,M);
-  int N; Size(q,N); 
-  assert(M==N);
-
-  times(q,Q,N);
-}
-
-/// q -> q Q
-template <class T> void times(multi1d<LatticeFermion> &q, Matrix<T> &Q, int N)
-{
-  GridBase *grid = q[0]._grid;
-  int M; SizeSquare(Q,M);
-  int K; Size(q,K); 
-  assert(N<M);
-  assert(N<K);
-  Vector<Fermion> S(N,grid );
-  for(int j=0;j<N;j++){
-    S[j] = zero;
-    for(int k=0;k<N;k++){
-      S[j] = S[j] +  q[k]* Q[k][j]; 
-    }
-  }
-  for(int j=0;j<q.size();j++){
-    q[j] = S[j];
-  }
-}
-#endif

lib/algorithms/iterative/MatrixUtils.h

@@ -1,75 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/iterative/MatrixUtils.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef GRID_MATRIX_UTILS_H
-#define GRID_MATRIX_UTILS_H
-
-namespace Grid {
-
-  namespace MatrixUtils { 
-
-    template<class T> inline void Size(Matrix<T>& A,int &N,int &M){
-      N=A.size(); assert(N>0);
-      M=A[0].size();
-      for(int i=0;i<N;i++){
-	assert(A[i].size()==M);
-      }
-    }
-
-    template<class T> inline void SizeSquare(Matrix<T>& A,int &N)
-    {
-      int M;
-      Size(A,N,M);
-      assert(N==M);
-    }
-
-    template<class T> inline void Fill(Matrix<T>& A,T & val)
-    { 
-      int N,M;
-      Size(A,N,M);
-      for(int i=0;i<N;i++){
-      for(int j=0;j<M;j++){
-	A[i][j]=val;
-      }}
-    }
-    template<class T> inline void Diagonal(Matrix<T>& A,T & val)
-    { 
-      int N;
-      SizeSquare(A,N);
-      for(int i=0;i<N;i++){
-	A[i][i]=val;
-      }
-    }
-    template<class T> inline void Identity(Matrix<T>& A)
-    {
-      Fill(A,0.0);
-      Diagonal(A,1.0);
-    }
-
-  };
-}
-#endif

lib/algorithms/iterative/TODO

@@ -1,15 +0,0 @@
-- ConjugateGradientMultiShift
-- MCR
-
-- Potentially Useful Boost libraries
-
-- MultiArray
-- Aligned allocator; memory pool
-- Remez -- Mike or Boost?
-- Multiprecision
-- quaternians
-- Tokenize
-- Serialization
-- Regex
-- Proto (ET)
-- uBlas

lib/algorithms/iterative/bisec.c

@@ -1,122 +0,0 @@
-#include <math.h>
-#include <stdlib.h>
-#include <vector>
-
-struct Bisection {
-
-static void get_eig2(int row_num,std::vector<RealD> &ALPHA,std::vector<RealD> &BETA, std::vector<RealD> & eig)
-{
-  int i,j;
-  std::vector<RealD> evec1(row_num+3);
-  std::vector<RealD> evec2(row_num+3);
-  RealD eps2;
-  ALPHA[1]=0.;
-  BETHA[1]=0.;
-  for(i=0;i<row_num-1;i++) {
-    ALPHA[i+1] = A[i*(row_num+1)].real();
-    BETHA[i+2] = A[i*(row_num+1)+1].real();
-  }
-  ALPHA[row_num] = A[(row_num-1)*(row_num+1)].real();
-  bisec(ALPHA,BETHA,row_num,1,row_num,1e-10,1e-10,evec1,eps2);
-  bisec(ALPHA,BETHA,row_num,1,row_num,1e-16,1e-16,evec2,eps2);
-
-  // Do we really need to sort here?
-  int begin=1;
-  int end = row_num;
-  int swapped=1;
-  while(swapped) {
-    swapped=0;
-    for(i=begin;i<end;i++){
-      if(mag(evec2[i])>mag(evec2[i+1]))	{
-	swap(evec2+i,evec2+i+1);
-	swapped=1;
-      }
-    }
-    end--;
-    for(i=end-1;i>=begin;i--){
-      if(mag(evec2[i])>mag(evec2[i+1]))	{
-	swap(evec2+i,evec2+i+1);
-	swapped=1;
-      }
-    }
-    begin++;
-  }
-
-  for(i=0;i<row_num;i++){
-    for(j=0;j<row_num;j++) {
-      if(i==j) H[i*row_num+j]=evec2[i+1];
-      else H[i*row_num+j]=0.;
-    }
-  }
-}
-
-static void bisec(std::vector<RealD> &c,   
-		  std::vector<RealD> &b,
-		  int n,
-		  int m1,
-		  int m2,
-		  RealD eps1,
-		  RealD relfeh,
-		  std::vector<RealD> &x,
-		  RealD &eps2)
-{
-  std::vector<RealD> wu(n+2);
-
-  RealD h,q,x1,xu,x0,xmin,xmax; 
-  int i,a,k;
-
-  b[1]=0.0;
-  xmin=c[n]-fabs(b[n]);
-  xmax=c[n]+fabs(b[n]);
-  for(i=1;i<n;i++){
-    h=fabs(b[i])+fabs(b[i+1]);
-    if(c[i]+h>xmax) xmax= c[i]+h;
-    if(c[i]-h<xmin) xmin= c[i]-h;
-  }
-  xmax *=2.;
-
-  eps2=relfeh*((xmin+xmax)>0.0 ? xmax : -xmin);
-  if(eps1<=0.0) eps1=eps2;
-  eps2=0.5*eps1+7.0*(eps2);
-  x0=xmax;
-  for(i=m1;i<=m2;i++){
-    x[i]=xmax;
-    wu[i]=xmin;
-  }
-
-  for(k=m2;k>=m1;k--){
-    xu=xmin;
-    i=k;
-    do{
-      if(xu<wu[i]){
-	xu=wu[i];
-	i=m1-1;
-      }
-      i--;
-    }while(i>=m1);
-    if(x0>x[k]) x0=x[k];
-    while((x0-xu)>2*relfeh*(fabs(xu)+fabs(x0))+eps1){
-      x1=(xu+x0)/2;
-
-      a=0;
-      q=1.0;
-      for(i=1;i<=n;i++){
-	q=c[i]-x1-((q!=0.0)? b[i]*b[i]/q:fabs(b[i])/relfeh);
-	if(q<0) a++;
-      }
-      //			printf("x1=%e a=%d\n",x1,a);
-      if(a<k){
-	if(a<m1){
-	  xu=x1;
-	  wu[m1]=x1;
-	}else {
-	  xu=x1;
-	  wu[a+1]=x1;
-	  if(x[a]>x1) x[a]=x1;
-	}
-      }else x0=x1;
-    }
-    x[k]=(x0+xu)/2;
-  }
-}
-}

lib/algorithms/iterative/get_eig.c

@@ -1 +0,0 @@
-

lib/cshift/Cshift_common.h

@@ -30,21 +30,11 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
-template<class vobj>
-class SimpleCompressor {
-public:
-  void Point(int) {};
-
-  vobj operator() (const vobj &arg) {
-    return arg;
-  }
-};
-
 ///////////////////////////////////////////////////////////////////
-// Gather for when there is no need to SIMD split with compression
+// Gather for when there is no need to SIMD split 
 ///////////////////////////////////////////////////////////////////
-template<class vobj,class cobj,class compressor> void 
+template<class vobj> void 
-Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0)
+Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 
@@ -62,7 +52,7 @@ Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimen
       for(int b=0;b<e2;b++){
 	int o  = n*stride;
 	int bo = n*e2;
-	buffer[off+bo+b]=compress(rhs._odata[so+o+b]);
+	buffer[off+bo+b]=rhs._odata[so+o+b];
       }
     }
   } else { 
@@ -78,17 +68,16 @@ Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimen
        }
      }
      parallel_for(int i=0;i<table.size();i++){
-       buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
+       buffer[off+table[i].first]=rhs._odata[so+table[i].second];
      }
   }
 }
 
-
 ///////////////////////////////////////////////////////////////////
-// Gather for when there *is* need to SIMD split with compression
+// Gather for when there *is* need to SIMD split 
 ///////////////////////////////////////////////////////////////////
-template<class cobj,class vobj,class compressor> void 
+template<class vobj> void 
-Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename cobj::scalar_object *> pointers,int dimension,int plane,int cbmask,compressor &compress)
+Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename vobj::scalar_object *> pointers,int dimension,int plane,int cbmask)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 
@@ -109,8 +98,8 @@ Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename cobj::scalar_
 	int o      =   n*n1;
 	int offset = b+n*e2;
 	
-	cobj temp =compress(rhs._odata[so+o+b]);
+	vobj temp =rhs._odata[so+o+b];
-	extract<cobj>(temp,pointers,offset);
+	extract<vobj>(temp,pointers,offset);
 
       }
     }
@@ -127,32 +116,14 @@ Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename cobj::scalar_
 	int offset = b+n*e2;
 
 	if ( ocb & cbmask ) {
-	  cobj temp =compress(rhs._odata[so+o+b]);
+	  vobj temp =rhs._odata[so+o+b];
-	  extract<cobj>(temp,pointers,offset);
+	  extract<vobj>(temp,pointers,offset);
 	}
       }
     }
   }
 }
 
-//////////////////////////////////////////////////////
-// Gather for when there is no need to SIMD split
-//////////////////////////////////////////////////////
-template<class vobj> void Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
-{
-  SimpleCompressor<vobj> dontcompress;
-  Gather_plane_simple (rhs,buffer,dimension,plane,cbmask,dontcompress);
-}
-
-//////////////////////////////////////////////////////
-// Gather for when there *is* need to SIMD split
-//////////////////////////////////////////////////////
-template<class vobj> void Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename vobj::scalar_object *> pointers,int dimension,int plane,int cbmask)
-{
-  SimpleCompressor<vobj> dontcompress;
-  Gather_plane_extract<vobj,vobj,decltype(dontcompress)>(rhs,pointers,dimension,plane,cbmask,dontcompress);
-}
-
 //////////////////////////////////////////////////////
 // Scatter for when there is no need to SIMD split
 //////////////////////////////////////////////////////
@@ -200,7 +171,7 @@ template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,commVector<vo
 //////////////////////////////////////////////////////
 // Scatter for when there *is* need to SIMD split
 //////////////////////////////////////////////////////
- template<class vobj,class cobj> void Scatter_plane_merge(Lattice<vobj> &rhs,std::vector<cobj *> pointers,int dimension,int plane,int cbmask)
+template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,std::vector<typename vobj::scalar_object *> pointers,int dimension,int plane,int cbmask)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 

lib/cshift/Cshift_mpi.h

@@ -154,13 +154,7 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
 			   recv_from_rank,
 			   bytes);
       grid->Barrier();
-      /*
+
-      for(int i=0;i<send_buf.size();i++){
-	assert(recv_buf.size()==buffer_size);
-	assert(send_buf.size()==buffer_size);
-	std::cout << "SendRecv_Cshift_comms ["<<i<<" "<< dimension<<"] snd "<<send_buf[i]<<" rcv " << recv_buf[i] << "  0x" << cbmask<<std::endl;
-      }
-      */
       Scatter_plane_simple (ret,recv_buf,dimension,x,cbmask);
     }
   }
@@ -246,13 +240,6 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
 			     (void *)&recv_buf_extract[i][0],
 			     recv_from_rank,
 			     bytes);
-	/*
-	for(int w=0;w<recv_buf_extract[i].size();w++){
-	  assert(recv_buf_extract[i].size()==buffer_size);
-	  assert(send_buf_extract[i].size()==buffer_size);
-	  std::cout << "SendRecv_Cshift_comms ["<<w<<" "<< dimension<<"] recv "<<recv_buf_extract[i][w]<<" send " << send_buf_extract[nbr_lane][w]  << cbmask<<std::endl;
-	}
-	*/	
 	grid->Barrier();
 	rpointers[i] = &recv_buf_extract[i][0];
       } else { 

lib/lattice/Lattice_reduction.h

@@ -1,224 +1,521 @@
-    /*************************************************************************************
+ /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./lib/lattice/Lattice_reduction.h
-
     Copyright (C) 2015
-
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
-
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
 #ifndef GRID_LATTICE_REDUCTION_H
 #define GRID_LATTICE_REDUCTION_H
 
+#include <Grid/Eigen/Dense>
+
 namespace Grid {
 #ifdef GRID_WARN_SUBOPTIMAL
 #warning "Optimisation alert all these reduction loops are NOT threaded "
 #endif     
-  
+
   ////////////////////////////////////////////////////////////////////////////////////////////////////
   // Deterministic Reduction operations
   ////////////////////////////////////////////////////////////////////////////////////////////////////
-  template <class vobj>
+template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
-  inline RealD norm2(const Lattice<vobj> &arg) {
+  ComplexD nrm = innerProduct(arg,arg);
-    ComplexD nrm = innerProduct(arg, arg);
+  return std::real(nrm); 
-    return std::real(nrm);
+}
+
+// Double inner product
+template<class vobj>
+inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right) 
+{
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_typeD vector_type;
+  scalar_type  nrm;
+  
+  GridBase *grid = left._grid;
+  
+  std::vector<vector_type,alignedAllocator<vector_type> > sumarray(grid->SumArraySize());
+  
+  parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
+    int nwork, mywork, myoff;
+    GridThread::GetWork(left._grid->oSites(),thr,mywork,myoff);
+    
+    decltype(innerProductD(left._odata[0],right._odata[0])) vnrm=zero; // private to thread; sub summation
+    for(int ss=myoff;ss<mywork+myoff; ss++){
+      vnrm = vnrm + innerProductD(left._odata[ss],right._odata[ss]);
+    }
+    sumarray[thr]=TensorRemove(vnrm) ;
   }
   
-  template <class vobj>
+  vector_type vvnrm; vvnrm=zero;  // sum across threads
-  inline ComplexD innerProduct(const Lattice<vobj> &left,
+  for(int i=0;i<grid->SumArraySize();i++){
-			       const Lattice<vobj> &right) {
+    vvnrm = vvnrm+sumarray[i];
+  } 
+  nrm = Reduce(vvnrm);// sum across simd
+  right._grid->GlobalSum(nrm);
+  return nrm;
+}
+ 
+template<class Op,class T1>
+inline auto sum(const LatticeUnaryExpression<Op,T1> & expr)
+  ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second))))::scalar_object
+{
+  return sum(closure(expr));
+}
+
+template<class Op,class T1,class T2>
+inline auto sum(const LatticeBinaryExpression<Op,T1,T2> & expr)
+      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),eval(0,std::get<1>(expr.second))))::scalar_object
+{
+  return sum(closure(expr));
+}
+
+
+template<class Op,class T1,class T2,class T3>
+inline auto sum(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr)
+  ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),
+				      eval(0,std::get<1>(expr.second)),
+				      eval(0,std::get<2>(expr.second))
+				      ))::scalar_object
+{
+  return sum(closure(expr));
+}
+
+template<class vobj>
+inline typename vobj::scalar_object sum(const Lattice<vobj> &arg)
+{
+  GridBase *grid=arg._grid;
+  int Nsimd = grid->Nsimd();
+  
+  std::vector<vobj,alignedAllocator<vobj> > sumarray(grid->SumArraySize());
+  for(int i=0;i<grid->SumArraySize();i++){
+    sumarray[i]=zero;
+  }
+  
+  parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
+    int nwork, mywork, myoff;
+    GridThread::GetWork(grid->oSites(),thr,mywork,myoff);
+    
+    vobj vvsum=zero;
+    for(int ss=myoff;ss<mywork+myoff; ss++){
+      vvsum = vvsum + arg._odata[ss];
+    }
+    sumarray[thr]=vvsum;
+  }
+  
+  vobj vsum=zero;  // sum across threads
+  for(int i=0;i<grid->SumArraySize();i++){
+    vsum = vsum+sumarray[i];
+  } 
+  
+  typedef typename vobj::scalar_object sobj;
+  sobj ssum=zero;
+  
+  std::vector<sobj>               buf(Nsimd);
+  extract(vsum,buf);
+  
+  for(int i=0;i<Nsimd;i++) ssum = ssum + buf[i];
+  arg._grid->GlobalSum(ssum);
+  
+  return ssum;
+}
+
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// sliceSum, sliceInnerProduct, sliceAxpy, sliceNorm etc...
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<typename vobj::scalar_object> &result,int orthogdim)
+{
+  ///////////////////////////////////////////////////////
+  // FIXME precision promoted summation
+  // may be important for correlation functions
+  // But easily avoided by using double precision fields
+  ///////////////////////////////////////////////////////
+  typedef typename vobj::scalar_object sobj;
+  GridBase  *grid = Data._grid;
+  assert(grid!=NULL);
+
+  const int    Nd = grid->_ndimension;
+  const int Nsimd = grid->Nsimd();
+
+  assert(orthogdim >= 0);
+  assert(orthogdim < Nd);
+
+  int fd=grid->_fdimensions[orthogdim];
+  int ld=grid->_ldimensions[orthogdim];
+  int rd=grid->_rdimensions[orthogdim];
+
+  std::vector<vobj,alignedAllocator<vobj> > lvSum(rd); // will locally sum vectors first
+  std::vector<sobj> lsSum(ld,zero);                    // sum across these down to scalars
+  std::vector<sobj> extracted(Nsimd);                  // splitting the SIMD
+
+  result.resize(fd); // And then global sum to return the same vector to every node 
+  for(int r=0;r<rd;r++){
+    lvSum[r]=zero;
+  }
+
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
+
+  // sum over reduced dimension planes, breaking out orthog dir
+  // Parallel over orthog direction
+  parallel_for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	lvSum[r]=lvSum[r]+Data._odata[ss];
+      }
+    }
+  }
+
+  // Sum across simd lanes in the plane, breaking out orthog dir.
+  std::vector<int> icoor(Nd);
+
+  for(int rt=0;rt<rd;rt++){
+
+    extract(lvSum[rt],extracted);
+
+    for(int idx=0;idx<Nsimd;idx++){
+
+      grid->iCoorFromIindex(icoor,idx);
+
+      int ldx =rt+icoor[orthogdim]*rd;
+
+      lsSum[ldx]=lsSum[ldx]+extracted[idx];
+
+    }
+  }
+  
+  // sum over nodes.
+  sobj gsum;
+  for(int t=0;t<fd;t++){
+    int pt = t/ld; // processor plane
+    int lt = t%ld;
+    if ( pt == grid->_processor_coor[orthogdim] ) {
+      gsum=lsSum[lt];
+    } else {
+      gsum=zero;
+    }
+
+    grid->GlobalSum(gsum);
+
+    result[t]=gsum;
+  }
+}
+
+template<class vobj>
+static void sliceInnerProductVector( std::vector<ComplexD> & result, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int orthogdim) 
+{
+  typedef typename vobj::vector_type   vector_type;
+  typedef typename vobj::scalar_type   scalar_type;
+  GridBase  *grid = lhs._grid;
+  assert(grid!=NULL);
+  conformable(grid,rhs._grid);
+
+  const int    Nd = grid->_ndimension;
+  const int Nsimd = grid->Nsimd();
+
+  assert(orthogdim >= 0);
+  assert(orthogdim < Nd);
+
+  int fd=grid->_fdimensions[orthogdim];
+  int ld=grid->_ldimensions[orthogdim];
+  int rd=grid->_rdimensions[orthogdim];
+
+  std::vector<vector_type,alignedAllocator<vector_type> > lvSum(rd); // will locally sum vectors first
+  std::vector<scalar_type > lsSum(ld,scalar_type(0.0));                    // sum across these down to scalars
+  std::vector<iScalar<scalar_type> > extracted(Nsimd);                  // splitting the SIMD
+
+  result.resize(fd); // And then global sum to return the same vector to every node for IO to file
+  for(int r=0;r<rd;r++){
+    lvSum[r]=zero;
+  }
+
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
+
+  parallel_for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	vector_type vv = TensorRemove(innerProduct(lhs._odata[ss],rhs._odata[ss]));
+	lvSum[r]=lvSum[r]+vv;
+      }
+    }
+  }
+
+  // Sum across simd lanes in the plane, breaking out orthog dir.
+  std::vector<int> icoor(Nd);
+  for(int rt=0;rt<rd;rt++){
+
+    iScalar<vector_type> temp; 
+    temp._internal = lvSum[rt];
+    extract(temp,extracted);
+
+    for(int idx=0;idx<Nsimd;idx++){
+
+      grid->iCoorFromIindex(icoor,idx);
+
+      int ldx =rt+icoor[orthogdim]*rd;
+
+      lsSum[ldx]=lsSum[ldx]+extracted[idx]._internal;
+
+    }
+  }
+  
+  // sum over nodes.
+  scalar_type gsum;
+  for(int t=0;t<fd;t++){
+    int pt = t/ld; // processor plane
+    int lt = t%ld;
+    if ( pt == grid->_processor_coor[orthogdim] ) {
+      gsum=lsSum[lt];
+    } else {
+      gsum=scalar_type(0.0);
+    }
+
+    grid->GlobalSum(gsum);
+
+    result[t]=gsum;
+  }
+}
+template<class vobj>
+static void sliceNorm (std::vector<RealD> &sn,const Lattice<vobj> &rhs,int Orthog) 
+{
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  int Nblock = rhs._grid->GlobalDimensions()[Orthog];
+  std::vector<ComplexD> ip(Nblock);
+  sn.resize(Nblock);
+  
+  sliceInnerProductVector(ip,rhs,rhs,Orthog);
+  for(int ss=0;ss<Nblock;ss++){
+    sn[ss] = real(ip[ss]);
+  }
+};
+
+
+template<class vobj>
+static void sliceMaddVector(Lattice<vobj> &R,std::vector<RealD> &a,const Lattice<vobj> &X,const Lattice<vobj> &Y,
+			    int orthogdim,RealD scale=1.0) 
+{    
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  typedef typename vobj::tensor_reduced tensor_reduced;
+  
+  GridBase *grid  = X._grid;
+
+  int Nsimd  =grid->Nsimd();
+  int Nblock =grid->GlobalDimensions()[orthogdim];
+
+  int fd     =grid->_fdimensions[orthogdim];
+  int ld     =grid->_ldimensions[orthogdim];
+  int rd     =grid->_rdimensions[orthogdim];
+
+  int e1     =grid->_slice_nblock[orthogdim];
+  int e2     =grid->_slice_block [orthogdim];
+  int stride =grid->_slice_stride[orthogdim];
+
+  std::vector<int> icoor;
+
+  for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    vector_type    av;
+
+    for(int l=0;l<Nsimd;l++){
+      grid->iCoorFromIindex(icoor,l);
+      int ldx =r+icoor[orthogdim]*rd;
+      scalar_type *as =(scalar_type *)&av;
+      as[l] = scalar_type(a[ldx])*scale;
+    }
+
+    tensor_reduced at; at=av;
+
+    parallel_for_nest2(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	R._odata[ss] = at*X._odata[ss]+Y._odata[ss];
+      }
+    }
+  }
+};
+
+
+/*
+template<class vobj>
+static void sliceMaddVectorSlow (Lattice<vobj> &R,std::vector<RealD> &a,const Lattice<vobj> &X,const Lattice<vobj> &Y,
+			     int Orthog,RealD scale=1.0) 
+{    
+  // FIXME: Implementation is slow
+  // Best base the linear combination by constructing a 
+  // set of vectors of size grid->_rdimensions[Orthog].
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  int Nblock = X._grid->GlobalDimensions()[Orthog];
+  
+  GridBase *FullGrid  = X._grid;
+  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  
+  Lattice<vobj> Xslice(SliceGrid);
+  Lattice<vobj> Rslice(SliceGrid);
+  // If we based this on Cshift it would work for spread out
+  // but it would be even slower
+  for(int i=0;i<Nblock;i++){
+    ExtractSlice(Rslice,Y,i,Orthog);
+    ExtractSlice(Xslice,X,i,Orthog);
+    Rslice = Rslice + Xslice*(scale*a[i]);
+    InsertSlice(Rslice,R,i,Orthog);
+  }
+};
+template<class vobj>
+static void sliceInnerProductVectorSlow( std::vector<ComplexD> & vec, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog) 
+  {
+    // FIXME: Implementation is slow
+    // Look at localInnerProduct implementation,
+    // and do inside a site loop with block strided iterators
+    typedef typename vobj::scalar_object sobj;
     typedef typename vobj::scalar_type scalar_type;
     typedef typename vobj::vector_type vector_type;
-    scalar_type nrm;
+    typedef typename vobj::tensor_reduced scalar;
-    
+    typedef typename scalar::scalar_object  scomplex;
-    GridBase *grid = left._grid;
+  
-    
+    int Nblock = lhs._grid->GlobalDimensions()[Orthog];
-    std::vector<vector_type, alignedAllocator<vector_type> > sumarray(grid->SumArraySize());
+    vec.resize(Nblock);
-    for (int i = 0; i < grid->SumArraySize(); i++) {
+    std::vector<scomplex> sip(Nblock);
-      sumarray[i] = zero;
+    Lattice<scalar> IP(lhs._grid); 
+    IP=localInnerProduct(lhs,rhs);
+    sliceSum(IP,sip,Orthog);
+  
+    for(int ss=0;ss<Nblock;ss++){
+      vec[ss] = TensorRemove(sip[ss]);
     }
-    
-    parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
-      int nwork, mywork, myoff;
-      GridThread::GetWork(left._grid->oSites(), thr, mywork, myoff);
-      
-      decltype(innerProduct(left._odata[0], right._odata[0])) vnrm=zero; // private to thread; sub summation
-      for(int ss = myoff; ss<mywork + myoff; ss++){
-	vnrm = vnrm + innerProduct(left._odata[ss],right._odata[ss]);
-      }
-      sumarray[thr]=TensorRemove(vnrm) ;
-    }
-    
-    vector_type vvnrm;
-    vvnrm=zero;  // sum across threads
-    for(int i=0; i < grid->SumArraySize(); i++){
-      vvnrm = vvnrm + sumarray[i];
-    } 
-    nrm = Reduce(vvnrm);// sum across simd
-    right._grid->GlobalSum(nrm);
-    return nrm;
   }
+*/
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// FIXME: Implementation is slow
+// If we based this on Cshift it would work for spread out
+// but it would be even slower
+//
+// Repeated extract slice is inefficient
+//
+// Best base the linear combination by constructing a 
+// set of vectors of size grid->_rdimensions[Orthog].
+//////////////////////////////////////////////////////////////////////////////////////////
+
+inline GridBase         *makeSubSliceGrid(const GridBase *BlockSolverGrid,int Orthog)
+{
+  int NN    = BlockSolverGrid->_ndimension;
+  int nsimd = BlockSolverGrid->Nsimd();
   
-  template <class Op, class T1>
+  std::vector<int> latt_phys(0);
-  inline auto sum(const LatticeUnaryExpression<Op, T1> &expr) ->
+  std::vector<int> simd_phys(0);
-    typename decltype(expr.first.func(eval(0, std::get<0>(expr.second))))::scalar_object {
+  std::vector<int>  mpi_phys(0);
-    return sum(closure(expr));
+  
+  for(int d=0;d<NN;d++){
+    if( d!=Orthog ) { 
+      latt_phys.push_back(BlockSolverGrid->_fdimensions[d]);
+      simd_phys.push_back(BlockSolverGrid->_simd_layout[d]);
+      mpi_phys.push_back(BlockSolverGrid->_processors[d]);
+    }
   }
-						  
+  return (GridBase *)new GridCartesian(latt_phys,simd_phys,mpi_phys); 
-    template<class Op,class T1,class T2>
-    inline auto sum(const LatticeBinaryExpression<Op,T1,T2> & expr)
-      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),eval(0,std::get<1>(expr.second))))::scalar_object
-    {
-      return sum(closure(expr));
-    }
-						     
-						     
-    template<class Op,class T1,class T2,class T3>
-    inline auto sum(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr)
-      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),
-					  eval(0,std::get<1>(expr.second)),
-					  eval(0,std::get<2>(expr.second))
-					  ))::scalar_object
-    {
-      return sum(closure(expr));
-    }
-						     
-    template<class vobj>
-    inline typename vobj::scalar_object sum(const Lattice<vobj> &arg){
-      
-      GridBase *grid=arg._grid;
-      int Nsimd = grid->Nsimd();
-      
-      std::vector<vobj,alignedAllocator<vobj> > sumarray(grid->SumArraySize());
-      for(int i=0;i<grid->SumArraySize();i++){
-	sumarray[i]=zero;
-      }
-      
-      parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
-	int nwork, mywork, myoff;
-	GridThread::GetWork(grid->oSites(),thr,mywork,myoff);
-	
-	vobj vvsum=zero;
-        for(int ss=myoff;ss<mywork+myoff; ss++){
-	  vvsum = vvsum + arg._odata[ss];
-	}
-	sumarray[thr]=vvsum;
-      }
-      
-      vobj vsum=zero;  // sum across threads
-      for(int i=0;i<grid->SumArraySize();i++){
-	vsum = vsum+sumarray[i];
-      } 
-      
-      typedef typename vobj::scalar_object sobj;
-      sobj ssum=zero;
-      
-      std::vector<sobj>               buf(Nsimd);
-      extract(vsum,buf);
-      
-      for(int i=0;i<Nsimd;i++) ssum = ssum + buf[i];
-      arg._grid->GlobalSum(ssum);
-      
-      return ssum;
-    }
-  
-  
-  
-  template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<typename vobj::scalar_object> &result,int orthogdim)
-  {
-    typedef typename vobj::scalar_object sobj;
-    GridBase  *grid = Data._grid;
-    assert(grid!=NULL);
-    
-    // FIXME
-    // std::cout<<GridLogMessage<<"WARNING ! SliceSum is unthreaded "<<grid->SumArraySize()<<" threads "<<std::endl;
-    
-    const int    Nd = grid->_ndimension;
-    const int Nsimd = grid->Nsimd();
-    
-    assert(orthogdim >= 0);
-    assert(orthogdim < Nd);
-    
-    int fd=grid->_fdimensions[orthogdim];
-    int ld=grid->_ldimensions[orthogdim];
-    int rd=grid->_rdimensions[orthogdim];
-    
-    std::vector<vobj,alignedAllocator<vobj> > lvSum(rd); // will locally sum vectors first
-    std::vector<sobj> lsSum(ld,zero); // sum across these down to scalars
-    std::vector<sobj> extracted(Nsimd);     // splitting the SIMD
-    
-    result.resize(fd); // And then global sum to return the same vector to every node for IO to file
-    for(int r=0;r<rd;r++){
-      lvSum[r]=zero;
-    }
-    
-    std::vector<int>  coor(Nd);  
-    
-    // sum over reduced dimension planes, breaking out orthog dir
-    
-    for(int ss=0;ss<grid->oSites();ss++){
-      Lexicographic::CoorFromIndex(coor,ss,grid->_rdimensions);
-      int r = coor[orthogdim];
-      lvSum[r]=lvSum[r]+Data._odata[ss];
-    }  
-    
-    // Sum across simd lanes in the plane, breaking out orthog dir.
-    std::vector<int> icoor(Nd);
-    
-    for(int rt=0;rt<rd;rt++){
-      
-      extract(lvSum[rt],extracted);
-      
-      for(int idx=0;idx<Nsimd;idx++){
-	
-	grid->iCoorFromIindex(icoor,idx);
-	
-	int ldx =rt+icoor[orthogdim]*rd;
-	
-	lsSum[ldx]=lsSum[ldx]+extracted[idx];
-	
-      }
-    }
-    
-    // sum over nodes.
-    sobj gsum;
-    for(int t=0;t<fd;t++){
-      int pt = t/ld; // processor plane
-      int lt = t%ld;
-      if ( pt == grid->_processor_coor[orthogdim] ) {
-	gsum=lsSum[lt];
-      } else {
-	gsum=zero;
-      }
-      
-      grid->GlobalSum(gsum);
-      
-      result[t]=gsum;
-    }
-    
-  }
-  
-  
 }
+
+
+template<class vobj>
+static void sliceMaddMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,const Lattice<vobj> &Y,int Orthog,RealD scale=1.0) 
+{    
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+
+  int Nblock = X._grid->GlobalDimensions()[Orthog];
+  
+  GridBase *FullGrid  = X._grid;
+  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  
+  Lattice<vobj> Xslice(SliceGrid);
+  Lattice<vobj> Rslice(SliceGrid);
+  
+  for(int i=0;i<Nblock;i++){
+    ExtractSlice(Rslice,Y,i,Orthog);
+    for(int j=0;j<Nblock;j++){
+      ExtractSlice(Xslice,X,j,Orthog);
+      Rslice = Rslice + Xslice*(scale*aa(j,i));
+    }
+    InsertSlice(Rslice,R,i,Orthog);
+  }
+};
+
+template<class vobj>
+static void sliceInnerProductMatrix(  Eigen::MatrixXcd &mat, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog) 
+{
+  // FIXME: Implementation is slow
+  // Not sure of best solution.. think about it
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  GridBase *FullGrid  = lhs._grid;
+  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  
+  int Nblock = FullGrid->GlobalDimensions()[Orthog];
+  
+  Lattice<vobj> Lslice(SliceGrid);
+  Lattice<vobj> Rslice(SliceGrid);
+  
+  mat = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+  
+  for(int i=0;i<Nblock;i++){
+    ExtractSlice(Lslice,lhs,i,Orthog);
+    for(int j=0;j<Nblock;j++){
+      ExtractSlice(Rslice,rhs,j,Orthog);
+      mat(i,j) = innerProduct(Lslice,Rslice);
+    }
+  }
+#undef FORCE_DIAG
+#ifdef FORCE_DIAG
+  for(int i=0;i<Nblock;i++){
+    for(int j=0;j<Nblock;j++){
+      if ( i != j ) mat(i,j)=0.0;
+    }
+  }
+#endif
+  return;
+}
+
+} /*END NAMESPACE GRID*/
 #endif
 
+
+

lib/lattice/Lattice_transfer.h

@@ -1,4 +1,4 @@
-    /*************************************************************************************
+/*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
 
@@ -359,7 +359,7 @@ void localConvert(const Lattice<vobj> &in,Lattice<vvobj> &out)
 
 
 template<class vobj>
-void InsertSlice(Lattice<vobj> &lowDim,Lattice<vobj> & higherDim,int slice, int orthog)
+void InsertSlice(const Lattice<vobj> &lowDim,Lattice<vobj> & higherDim,int slice, int orthog)
 {
   typedef typename vobj::scalar_object sobj;
 
@@ -401,7 +401,7 @@ void InsertSlice(Lattice<vobj> &lowDim,Lattice<vobj> & higherDim,int slice, int
 }
 
 template<class vobj>
-void ExtractSlice(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice, int orthog)
+void ExtractSlice(Lattice<vobj> &lowDim,const Lattice<vobj> & higherDim,int slice, int orthog)
 {
   typedef typename vobj::scalar_object sobj;
 
@@ -444,7 +444,7 @@ void ExtractSlice(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice, in
 
 
 template<class vobj>
-void InsertSliceLocal(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice_lo,int slice_hi, int orthog)
+void InsertSliceLocal(const Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice_lo,int slice_hi, int orthog)
 {
   typedef typename vobj::scalar_object sobj;
 

lib/log/Log.h

@@ -110,8 +110,8 @@ public:
   friend std::ostream& operator<< (std::ostream& stream, Logger& log){
 
     if ( log.active ) {
-      stream << log.background()<< std::setw(10) << std::left << log.topName << log.background()<< " : ";
+      stream << log.background()<< std::setw(8) << std::left << log.topName << log.background()<< " : ";
-      stream << log.colour() << std::setw(14) << std::left << log.name << log.background() << " : ";
+      stream << log.colour() << std::setw(10) << std::left << log.name << log.background() << " : ";
       if ( log.timestamp ) {
 	StopWatch.Stop();
 	GridTime now = StopWatch.Elapsed();

lib/parallelIO/BinaryIO.h

@@ -563,7 +563,7 @@ class BinaryIO {
       	if (ILDG.is_ILDG){
       		// use C-LIME to populate the record
           #ifdef HAVE_LIME
-          size_t sizeFO = sizeof(fileObj);
+          uint64_t sizeFO = sizeof(fileObj);
           limeReaderSeek(ILDG.LR, g_idx*sizeFO, SEEK_SET);
           int status = limeReaderReadData((void *)&fileObj, &sizeFO, ILDG.LR);
           #endif
@@ -762,7 +762,7 @@ class BinaryIO {
 
         if (ILDG.is_ILDG) {
           #ifdef HAVE_LIME
-          size_t sizeFO = sizeof(fileObj);
+          uint64_t sizeFO = sizeof(fileObj);
  					limeWriterSeek(ILDG.LW, g_idx*sizeFO, SEEK_SET);
           int status = limeWriteRecordData((void *)&fileObj, &sizeFO, ILDG.LW);
           #endif

lib/qcd/action/ActionParams.h

@@ -44,7 +44,12 @@ namespace QCD {
   
   struct WilsonImplParams {
     bool overlapCommsCompute;
-    WilsonImplParams() : overlapCommsCompute(false){};
+    std::vector<Complex> boundary_phases;
+    WilsonImplParams() : overlapCommsCompute(false) {
+      boundary_phases.resize(Nd, 1.0);
+    };
+    WilsonImplParams(const std::vector<Complex> phi)
+      : boundary_phases(phi), overlapCommsCompute(false) {}
   };
 
   struct StaggeredImplParams {

lib/qcd/action/fermion/CayleyFermion5D.cc

@@ -320,7 +320,7 @@ void CayleyFermion5D<Impl>::MDeriv  (GaugeField &mat,const FermionField &U,const
     this->DhopDeriv(mat,U,Din,dag);
   } else {
     //      U d/du [D_w D5]^dag V = U D5^dag d/du DW^dag Y // implicit adj on U in call
-    MeooeDag5D(U,Din);
+    Meooe5D(U,Din);
     this->DhopDeriv(mat,Din,V,dag);
   }
 };
@@ -335,8 +335,8 @@ void CayleyFermion5D<Impl>::MoeDeriv(GaugeField &mat,const FermionField &U,const
     this->DhopDerivOE(mat,U,Din,dag);
   } else {
     //      U d/du [D_w D5]^dag V = U D5^dag d/du DW^dag Y // implicit adj on U in call
-      MeooeDag5D(U,Din);
+    Meooe5D(U,Din);
-      this->DhopDerivOE(mat,Din,V,dag);
+    this->DhopDerivOE(mat,Din,V,dag);
   }
 };
 template<class Impl>
@@ -350,7 +350,7 @@ void CayleyFermion5D<Impl>::MeoDeriv(GaugeField &mat,const FermionField &U,const
     this->DhopDerivEO(mat,U,Din,dag);
   } else {
     //      U d/du [D_w D5]^dag V = U D5^dag d/du DW^dag Y // implicit adj on U in call
-    MeooeDag5D(U,Din);
+    Meooe5D(U,Din);
     this->DhopDerivEO(mat,Din,V,dag);
   }
 };
@@ -380,6 +380,8 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   ///////////////////////////////////////////////////////////
   // The Cayley coeffs (unprec)
   ///////////////////////////////////////////////////////////
+  assert(gamma.size()==Ls);
+
   omega.resize(Ls);
   bs.resize(Ls);
   cs.resize(Ls);
@@ -412,10 +414,11 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   for(int i=0; i < Ls; i++){
     as[i] = 1.0;
     omega[i] = gamma[i]*zolo_hi; //NB reciprocal relative to Chroma NEF code
+    //    assert(fabs(omega[i])>0.0);
     bs[i] = 0.5*(bpc/omega[i] + bmc);
     cs[i] = 0.5*(bpc/omega[i] - bmc);
   }
-  
+
   ////////////////////////////////////////////////////////
   // Constants for the preconditioned matrix Cayley form
   ////////////////////////////////////////////////////////
@@ -425,12 +428,12 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   ceo.resize(Ls);
   
   for(int i=0;i<Ls;i++){
-    bee[i]=as[i]*(bs[i]*(4.0-this->M5) +1.0);
+    bee[i]=as[i]*(bs[i]*(4.0-this->M5) +1.0);     
+    //    assert(fabs(bee[i])>0.0);
     cee[i]=as[i]*(1.0-cs[i]*(4.0-this->M5));
     beo[i]=as[i]*bs[i];
     ceo[i]=-as[i]*cs[i];
   }
-  
   aee.resize(Ls);
   aeo.resize(Ls);
   for(int i=0;i<Ls;i++){
@@ -474,14 +477,16 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
 	
   { 
     Coeff_t delta_d=mass*cee[Ls-1];
-    for(int j=0;j<Ls-1;j++) delta_d *= cee[j]/bee[j];
+    for(int j=0;j<Ls-1;j++) {
+      //      assert(fabs(bee[j])>0.0);
+      delta_d *= cee[j]/bee[j];
+    }
     dee[Ls-1] += delta_d;
   }  
 
   int inv=1;
   this->MooeeInternalCompute(0,inv,MatpInv,MatmInv);
   this->MooeeInternalCompute(1,inv,MatpInvDag,MatmInvDag);
-
 }
 
 
@@ -495,7 +500,9 @@ void CayleyFermion5D<Impl>::MooeeInternalCompute(int dag, int inv,
   GridBase *grid = this->FermionRedBlackGrid();
   int LLs = grid->_rdimensions[0];
 
-  if ( LLs == Ls ) return; // Not vectorised in 5th direction
+  if ( LLs == Ls ) {
+    return; // Not vectorised in 5th direction
+  }
 
   Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
   Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);

lib/qcd/action/fermion/CayleyFermion5Dcache.cc

@@ -237,6 +237,13 @@ void CayleyFermion5D<Impl>::MooeeInvDag (const FermionField &psi, FermionField &
   INSTANTIATE_DPERP(GparityWilsonImplD);
   INSTANTIATE_DPERP(ZWilsonImplF);
   INSTANTIATE_DPERP(ZWilsonImplD);
+
+  INSTANTIATE_DPERP(WilsonImplFH);
+  INSTANTIATE_DPERP(WilsonImplDF);
+  INSTANTIATE_DPERP(GparityWilsonImplFH);
+  INSTANTIATE_DPERP(GparityWilsonImplDF);
+  INSTANTIATE_DPERP(ZWilsonImplFH);
+  INSTANTIATE_DPERP(ZWilsonImplDF);
 #endif
 
 }}

lib/qcd/action/fermion/CayleyFermion5Ddense.cc

@@ -137,6 +137,20 @@ template void CayleyFermion5D<WilsonImplF>::MooeeInternal(const FermionField &ps
 template void CayleyFermion5D<WilsonImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZWilsonImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZWilsonImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+
+INSTANTIATE_DPERP(GparityWilsonImplFH);
+INSTANTIATE_DPERP(GparityWilsonImplDF);
+INSTANTIATE_DPERP(WilsonImplFH);
+INSTANTIATE_DPERP(WilsonImplDF);
+INSTANTIATE_DPERP(ZWilsonImplFH);
+INSTANTIATE_DPERP(ZWilsonImplDF);
+
+template void CayleyFermion5D<GparityWilsonImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<GparityWilsonImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<WilsonImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<WilsonImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZWilsonImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZWilsonImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 #endif
 
 }}

lib/qcd/action/fermion/CayleyFermion5Dssp.cc

@@ -37,7 +37,6 @@ namespace Grid {
 namespace QCD {
 
   // FIXME -- make a version of these routines with site loop outermost for cache reuse.
-
   // Pminus fowards
   // Pplus  backwards
 template<class Impl>  
@@ -152,6 +151,13 @@ void CayleyFermion5D<Impl>::MooeeInvDag (const FermionField &psi, FermionField &
   INSTANTIATE_DPERP(GparityWilsonImplD);
   INSTANTIATE_DPERP(ZWilsonImplF);
   INSTANTIATE_DPERP(ZWilsonImplD);
+
+  INSTANTIATE_DPERP(WilsonImplFH);
+  INSTANTIATE_DPERP(WilsonImplDF);
+  INSTANTIATE_DPERP(GparityWilsonImplFH);
+  INSTANTIATE_DPERP(GparityWilsonImplDF);
+  INSTANTIATE_DPERP(ZWilsonImplFH);
+  INSTANTIATE_DPERP(ZWilsonImplDF);
 #endif
 
 }

lib/qcd/action/fermion/CayleyFermion5Dvec.cc

@@ -808,10 +808,21 @@ INSTANTIATE_DPERP(DomainWallVec5dImplF);
 INSTANTIATE_DPERP(ZDomainWallVec5dImplD);
 INSTANTIATE_DPERP(ZDomainWallVec5dImplF);
 
+INSTANTIATE_DPERP(DomainWallVec5dImplDF);
+INSTANTIATE_DPERP(DomainWallVec5dImplFH);
+INSTANTIATE_DPERP(ZDomainWallVec5dImplDF);
+INSTANTIATE_DPERP(ZDomainWallVec5dImplFH);
+
 template void CayleyFermion5D<DomainWallVec5dImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<DomainWallVec5dImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZDomainWallVec5dImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZDomainWallVec5dImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 
+template void CayleyFermion5D<DomainWallVec5dImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<DomainWallVec5dImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZDomainWallVec5dImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZDomainWallVec5dImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+
+
 
 }}

lib/qcd/action/fermion/DomainWallFermion.h

@@ -68,7 +68,7 @@ namespace Grid {
 	Approx::zolotarev_data *zdata = Approx::higham(eps,this->Ls);// eps is ignored for higham
 	assert(zdata->n==this->Ls);
 	
-	//	std::cout<<GridLogMessage << "DomainWallFermion with Ls="<<this->Ls<<std::endl;
+	std::cout<<GridLogMessage << "DomainWallFermion with Ls="<<this->Ls<<std::endl;
 	// Call base setter
 	this->SetCoefficientsTanh(zdata,1.0,0.0);
 

lib/qcd/action/fermion/Fermion.h

@@ -58,6 +58,7 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
 #include <Grid/qcd/action/fermion/DomainWallFermion.h>
 #include <Grid/qcd/action/fermion/MobiusFermion.h>
 #include <Grid/qcd/action/fermion/ZMobiusFermion.h>
+#include <Grid/qcd/action/fermion/SchurDiagTwoKappa.h>
 #include <Grid/qcd/action/fermion/ScaledShamirFermion.h>
 #include <Grid/qcd/action/fermion/MobiusZolotarevFermion.h>
 #include <Grid/qcd/action/fermion/ShamirZolotarevFermion.h>
@@ -88,6 +89,10 @@ typedef WilsonFermion<WilsonImplR> WilsonFermionR;
 typedef WilsonFermion<WilsonImplF> WilsonFermionF;
 typedef WilsonFermion<WilsonImplD> WilsonFermionD;
 
+typedef WilsonFermion<WilsonImplRL> WilsonFermionRL;
+typedef WilsonFermion<WilsonImplFH> WilsonFermionFH;
+typedef WilsonFermion<WilsonImplDF> WilsonFermionDF;
+
 typedef WilsonFermion<WilsonAdjImplR> WilsonAdjFermionR;
 typedef WilsonFermion<WilsonAdjImplF> WilsonAdjFermionF;
 typedef WilsonFermion<WilsonAdjImplD> WilsonAdjFermionD;
@@ -104,27 +109,50 @@ typedef DomainWallFermion<WilsonImplR> DomainWallFermionR;
 typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;
 typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
 
+typedef DomainWallFermion<WilsonImplRL> DomainWallFermionRL;
+typedef DomainWallFermion<WilsonImplFH> DomainWallFermionFH;
+typedef DomainWallFermion<WilsonImplDF> DomainWallFermionDF;
+
 typedef MobiusFermion<WilsonImplR> MobiusFermionR;
 typedef MobiusFermion<WilsonImplF> MobiusFermionF;
 typedef MobiusFermion<WilsonImplD> MobiusFermionD;
 
+typedef MobiusFermion<WilsonImplRL> MobiusFermionRL;
+typedef MobiusFermion<WilsonImplFH> MobiusFermionFH;
+typedef MobiusFermion<WilsonImplDF> MobiusFermionDF;
+
 typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
 typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
 typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
 
+typedef ZMobiusFermion<ZWilsonImplRL> ZMobiusFermionRL;
+typedef ZMobiusFermion<ZWilsonImplFH> ZMobiusFermionFH;
+typedef ZMobiusFermion<ZWilsonImplDF> ZMobiusFermionDF;
+
 // Ls vectorised 
 typedef DomainWallFermion<DomainWallVec5dImplR> DomainWallFermionVec5dR;
 typedef DomainWallFermion<DomainWallVec5dImplF> DomainWallFermionVec5dF;
 typedef DomainWallFermion<DomainWallVec5dImplD> DomainWallFermionVec5dD;
 
+typedef DomainWallFermion<DomainWallVec5dImplRL> DomainWallFermionVec5dRL;
+typedef DomainWallFermion<DomainWallVec5dImplFH> DomainWallFermionVec5dFH;
+typedef DomainWallFermion<DomainWallVec5dImplDF> DomainWallFermionVec5dDF;
+
 typedef MobiusFermion<DomainWallVec5dImplR> MobiusFermionVec5dR;
 typedef MobiusFermion<DomainWallVec5dImplF> MobiusFermionVec5dF;
 typedef MobiusFermion<DomainWallVec5dImplD> MobiusFermionVec5dD;
 
+typedef MobiusFermion<DomainWallVec5dImplRL> MobiusFermionVec5dRL;
+typedef MobiusFermion<DomainWallVec5dImplFH> MobiusFermionVec5dFH;
+typedef MobiusFermion<DomainWallVec5dImplDF> MobiusFermionVec5dDF;
+
 typedef ZMobiusFermion<ZDomainWallVec5dImplR> ZMobiusFermionVec5dR;
 typedef ZMobiusFermion<ZDomainWallVec5dImplF> ZMobiusFermionVec5dF;
 typedef ZMobiusFermion<ZDomainWallVec5dImplD> ZMobiusFermionVec5dD;
 
+typedef ZMobiusFermion<ZDomainWallVec5dImplRL> ZMobiusFermionVec5dRL;
+typedef ZMobiusFermion<ZDomainWallVec5dImplFH> ZMobiusFermionVec5dFH;
+typedef ZMobiusFermion<ZDomainWallVec5dImplDF> ZMobiusFermionVec5dDF;
 
 typedef ScaledShamirFermion<WilsonImplR> ScaledShamirFermionR;
 typedef ScaledShamirFermion<WilsonImplF> ScaledShamirFermionF;
@@ -165,17 +193,35 @@ typedef OverlapWilsonPartialFractionZolotarevFermion<WilsonImplD> OverlapWilsonP
 typedef WilsonFermion<GparityWilsonImplR>     GparityWilsonFermionR;
 typedef WilsonFermion<GparityWilsonImplF>     GparityWilsonFermionF;
 typedef WilsonFermion<GparityWilsonImplD>     GparityWilsonFermionD;
+
+typedef WilsonFermion<GparityWilsonImplRL>     GparityWilsonFermionRL;
+typedef WilsonFermion<GparityWilsonImplFH>     GparityWilsonFermionFH;
+typedef WilsonFermion<GparityWilsonImplDF>     GparityWilsonFermionDF;
+
 typedef DomainWallFermion<GparityWilsonImplR> GparityDomainWallFermionR;
 typedef DomainWallFermion<GparityWilsonImplF> GparityDomainWallFermionF;
 typedef DomainWallFermion<GparityWilsonImplD> GparityDomainWallFermionD;
 
+typedef DomainWallFermion<GparityWilsonImplRL> GparityDomainWallFermionRL;
+typedef DomainWallFermion<GparityWilsonImplFH> GparityDomainWallFermionFH;
+typedef DomainWallFermion<GparityWilsonImplDF> GparityDomainWallFermionDF;
+
 typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionR;
 typedef WilsonTMFermion<GparityWilsonImplF> GparityWilsonTMFermionF;
 typedef WilsonTMFermion<GparityWilsonImplD> GparityWilsonTMFermionD;
+
+typedef WilsonTMFermion<GparityWilsonImplRL> GparityWilsonTMFermionRL;
+typedef WilsonTMFermion<GparityWilsonImplFH> GparityWilsonTMFermionFH;
+typedef WilsonTMFermion<GparityWilsonImplDF> GparityWilsonTMFermionDF;
+
 typedef MobiusFermion<GparityWilsonImplR> GparityMobiusFermionR;
 typedef MobiusFermion<GparityWilsonImplF> GparityMobiusFermionF;
 typedef MobiusFermion<GparityWilsonImplD> GparityMobiusFermionD;
 
+typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
+typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
+typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
+
 typedef ImprovedStaggeredFermion<StaggeredImplR> ImprovedStaggeredFermionR;
 typedef ImprovedStaggeredFermion<StaggeredImplF> ImprovedStaggeredFermionF;
 typedef ImprovedStaggeredFermion<StaggeredImplD> ImprovedStaggeredFermionD;

lib/qcd/action/fermion/FermionCore.h

@@ -55,7 +55,14 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
   template class A<ZWilsonImplF>;		\
   template class A<ZWilsonImplD>;		\
   template class A<GparityWilsonImplF>;		\
-  template class A<GparityWilsonImplD>;		
+  template class A<GparityWilsonImplD>;		\
+  template class A<WilsonImplFH>;		\
+  template class A<WilsonImplDF>;		\
+  template class A<ZWilsonImplFH>;		\
+  template class A<ZWilsonImplDF>;		\
+  template class A<GparityWilsonImplFH>;		\
+  template class A<GparityWilsonImplDF>;		
+
 
 #define AdjointFermOpTemplateInstantiate(A) \
   template class A<WilsonAdjImplF>; \
@@ -69,7 +76,11 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
   template class A<DomainWallVec5dImplF>;	\
   template class A<DomainWallVec5dImplD>;	\
   template class A<ZDomainWallVec5dImplF>;	\
-  template class A<ZDomainWallVec5dImplD>;	
+  template class A<ZDomainWallVec5dImplD>;	\
+  template class A<DomainWallVec5dImplFH>;	\
+  template class A<DomainWallVec5dImplDF>;	\
+  template class A<ZDomainWallVec5dImplFH>;	\
+  template class A<ZDomainWallVec5dImplDF>;	
 
 #define FermOpTemplateInstantiate(A) \
  FermOp4dVecTemplateInstantiate(A) \

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -35,7 +35,6 @@ directory
 namespace Grid {
 namespace QCD {
 
-
   //////////////////////////////////////////////
   // Template parameter class constructs to package
   // externally control Fermion implementations
@@ -89,7 +88,53 @@ namespace QCD {
   //    
   //  }
   //////////////////////////////////////////////
-  
+
+  template <class T> struct SamePrecisionMapper {
+    typedef T HigherPrecVector ;
+    typedef T LowerPrecVector ;
+  };
+  template <class T> struct LowerPrecisionMapper {  };
+  template <> struct LowerPrecisionMapper<vRealF> {
+    typedef vRealF HigherPrecVector ;
+    typedef vRealH LowerPrecVector ;
+  };
+  template <> struct LowerPrecisionMapper<vRealD> {
+    typedef vRealD HigherPrecVector ;
+    typedef vRealF LowerPrecVector ;
+  };
+  template <> struct LowerPrecisionMapper<vComplexF> {
+    typedef vComplexF HigherPrecVector ;
+    typedef vComplexH LowerPrecVector ;
+  };
+  template <> struct LowerPrecisionMapper<vComplexD> {
+    typedef vComplexD HigherPrecVector ;
+    typedef vComplexF LowerPrecVector ;
+  };
+
+  struct CoeffReal {
+  public:
+    typedef RealD _Coeff_t;
+    static const int Nhcs = 2;
+    template<class Simd> using PrecisionMapper = SamePrecisionMapper<Simd>;
+  };
+  struct CoeffRealHalfComms {
+  public:
+    typedef RealD _Coeff_t;
+    static const int Nhcs = 1;
+    template<class Simd> using PrecisionMapper = LowerPrecisionMapper<Simd>;
+  };
+  struct CoeffComplex {
+  public:
+    typedef ComplexD _Coeff_t;
+    static const int Nhcs = 2;
+    template<class Simd> using PrecisionMapper = SamePrecisionMapper<Simd>;
+  };
+  struct CoeffComplexHalfComms {
+  public:
+    typedef ComplexD _Coeff_t;
+    static const int Nhcs = 1;
+    template<class Simd> using PrecisionMapper = LowerPrecisionMapper<Simd>;
+  };
 
   ////////////////////////////////////////////////////////////////////////
   // Implementation dependent fermion types
@@ -114,43 +159,48 @@ namespace QCD {
   /////////////////////////////////////////////////////////////////////////////
   // Single flavour four spinors with colour index
   /////////////////////////////////////////////////////////////////////////////
-  template <class S, class Representation = FundamentalRepresentation,class _Coeff_t = RealD >
+  template <class S, class Representation = FundamentalRepresentation,class Options = CoeffReal >
   class WilsonImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
-
     public:
 
     static const int Dimension = Representation::Dimension;
+    static const bool LsVectorised=false;
+    static const int Nhcs = Options::Nhcs;
+
     typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
+    INHERIT_GIMPL_TYPES(Gimpl);
       
     //Necessary?
     constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
     
-    const bool LsVectorised=false;
+    typedef typename Options::_Coeff_t Coeff_t;
-    typedef _Coeff_t Coeff_t;
+    typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
-
-    INHERIT_GIMPL_TYPES(Gimpl);
       
     template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
     template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Dimension>, Ns> >;
     template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
+    template <typename vtype> using iImplHalfCommSpinor    = iScalar<iVector<iVector<vtype, Dimension>, Nhcs> >;
     template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
     
     typedef iImplSpinor<Simd>            SiteSpinor;
     typedef iImplPropagator<Simd>        SitePropagator;
     typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
+    typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
     typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
     
     typedef Lattice<SiteSpinor>            FermionField;
     typedef Lattice<SitePropagator>        PropagatorField;
     typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
     
-    typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+    typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
     typedef WilsonImplParams ImplParams;
     typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
     
     ImplParams Params;
     
-    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
+    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){
+      assert(Params.boundary_phases.size() == Nd);
+    };
       
     bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
       
@@ -174,10 +224,16 @@ namespace QCD {
       conformable(Uds._grid, GaugeGrid);
       conformable(Umu._grid, GaugeGrid);
       GaugeLinkField U(GaugeGrid);
+      GaugeLinkField tmp(GaugeGrid);
+      Lattice<iScalar<vInteger> > coor(GaugeGrid);
       for (int mu = 0; mu < Nd; mu++) {
+        LatticeCoordinate(coor, mu);
+        int Lmu = GaugeGrid->GlobalDimensions()[mu] - 1;
         U = PeekIndex<LorentzIndex>(Umu, mu);
-        PokeIndex<LorentzIndex>(Uds, U, mu);
+        tmp = where(coor == Lmu, Params.boundary_phases[mu] * U, U);
+        PokeIndex<LorentzIndex>(Uds, tmp, mu);
         U = adj(Cshift(U, mu, -1));
+        U = where(coor == 0, Params.boundary_phases[mu] * U, U);
         PokeIndex<LorentzIndex>(Uds, U, mu + 4);
       }
     }
@@ -209,31 +265,34 @@ namespace QCD {
   ////////////////////////////////////////////////////////////////////////////////////
   // Single flavour four spinors with colour index, 5d redblack
   ////////////////////////////////////////////////////////////////////////////////////
-
+template<class S,int Nrepresentation=Nc, class Options=CoeffReal>
-template<class S,int Nrepresentation=Nc,class _Coeff_t = RealD>
 class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { 
   public:
-      
+
-  static const int Dimension = Nrepresentation;
-  const bool LsVectorised=true;
-  typedef _Coeff_t Coeff_t;      
   typedef PeriodicGaugeImpl<GaugeImplTypes<S, Nrepresentation> > Gimpl;
-  
   INHERIT_GIMPL_TYPES(Gimpl);
+
+  static const int Dimension = Nrepresentation;
+  static const bool LsVectorised=true;
+  static const int Nhcs = Options::Nhcs;
+      
+  typedef typename Options::_Coeff_t Coeff_t;      
+  typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
   
   template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Nrepresentation>, Ns> >;
   template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Nrepresentation>, Ns> >;
   template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhs> >;
+  template <typename vtype> using iImplHalfCommSpinor    = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhcs> >;
   template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>;
   template <typename vtype> using iImplGaugeField        = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nd>;
   template <typename vtype> using iImplGaugeLink         = iScalar<iScalar<iMatrix<vtype, Nrepresentation> > >;
   
-  typedef iImplSpinor<Simd> SiteSpinor;
+  typedef iImplSpinor<Simd>            SiteSpinor;
-  typedef iImplPropagator<Simd> SitePropagator;
+  typedef iImplPropagator<Simd>        SitePropagator;
-  typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
+  typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
-  typedef Lattice<SiteSpinor> FermionField;
+  typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
-  typedef Lattice<SitePropagator> PropagatorField;
+  typedef Lattice<SiteSpinor>          FermionField;
-  
+  typedef Lattice<SitePropagator>      PropagatorField;
 
   /////////////////////////////////////////////////
   // Make the doubled gauge field a *scalar*
@@ -241,9 +300,9 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
   typedef iImplDoubledGaugeField<typename Simd::scalar_type>  SiteDoubledGaugeField;  // This is a scalar
   typedef iImplGaugeField<typename Simd::scalar_type>         SiteScalarGaugeField;  // scalar
   typedef iImplGaugeLink<typename Simd::scalar_type>          SiteScalarGaugeLink;  // scalar
-  typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
+  typedef Lattice<SiteDoubledGaugeField>                      DoubledGaugeField;
       
-  typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+  typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
   typedef WilsonImplParams ImplParams;
   typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
   
@@ -303,6 +362,11 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
   }
 
   inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde, int mu) {
+
+    assert(0);
+    // Following lines to be revised after Peter's addition of half prec
+    // missing put lane...
+    /*
     typedef decltype(traceIndex<SpinIndex>(outerProduct(Btilde[0], Atilde[0]))) result_type;
     unsigned int LLs = Btilde._grid->_rdimensions[0];
     conformable(Atilde._grid,Btilde._grid);
@@ -342,41 +406,44 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
       }
     }
     PokeIndex<LorentzIndex>(mat, tmp, mu);
+    */
   }
 };
     
     ////////////////////////////////////////////////////////////////////////////////////////
     // Flavour doubled spinors; is Gparity the only? what about C*?
     ////////////////////////////////////////////////////////////////////////////////////////
-    
+template <class S, int Nrepresentation, class Options=CoeffReal>
-template <class S, int Nrepresentation,class _Coeff_t = RealD>
 class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
  public:
 
  static const int Dimension = Nrepresentation;
+ static const int Nhcs = Options::Nhcs;
+ static const bool LsVectorised=false;
 
- const bool LsVectorised=false;
-
- typedef _Coeff_t Coeff_t;
  typedef ConjugateGaugeImpl< GaugeImplTypes<S,Nrepresentation> > Gimpl;
- 
  INHERIT_GIMPL_TYPES(Gimpl);
+
+ typedef typename Options::_Coeff_t Coeff_t;
+ typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
       
- template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
+ template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>,   Ngp>;
- template <typename vtype> using iImplPropagator        = iVector<iMatrix<iMatrix<vtype, Nrepresentation>, Ns>, Ngp >;
+ template <typename vtype> using iImplPropagator        = iVector<iMatrix<iMatrix<vtype, Nrepresentation>, Ns>,   Ngp>;
- template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
+ template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>,  Ngp>;
+ template <typename vtype> using iImplHalfCommSpinor    = iVector<iVector<iVector<vtype, Nrepresentation>, Nhcs>, Ngp>;
  template <typename vtype> using iImplDoubledGaugeField = iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
-      
+
- typedef iImplSpinor<Simd> SiteSpinor;
+ typedef iImplSpinor<Simd>            SiteSpinor;
- typedef iImplPropagator<Simd> SitePropagator;
+ typedef iImplPropagator<Simd>        SitePropagator;
- typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
+ typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
+ typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
  typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
 
  typedef Lattice<SiteSpinor> FermionField;
  typedef Lattice<SitePropagator> PropagatorField;
  typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
  
- typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+ typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
  typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
  
  typedef GparityWilsonImplParams ImplParams;
@@ -393,8 +460,8 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
                       const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
                       StencilImpl &St) {
 
-  typedef SiteHalfSpinor vobj;
+   typedef SiteHalfSpinor vobj;
-  typedef typename SiteHalfSpinor::scalar_object sobj;
+   typedef typename SiteHalfSpinor::scalar_object sobj;
 	
    vobj vtmp;
    sobj stmp;
@@ -513,7 +580,6 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
    }
  }
       
-      
  inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A, int mu) {
 
    // DhopDir provides U or Uconj depending on coor/flavour.
@@ -546,23 +612,22 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 
 };
 
-
+/////////////////////////////////////////////////////////////////////////////
-  /////////////////////////////////////////////////////////////////////////////
+// Single flavour one component spinors with colour index
-  // Single flavour one component spinors with colour index
+/////////////////////////////////////////////////////////////////////////////
-  /////////////////////////////////////////////////////////////////////////////
+template <class S, class Representation = FundamentalRepresentation >
-  template <class S, class Representation = FundamentalRepresentation >
+class StaggeredImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
-  class StaggeredImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
 
     public:
 
     typedef RealD  _Coeff_t ;
     static const int Dimension = Representation::Dimension;
+    static const bool LsVectorised=false;
     typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
       
     //Necessary?
     constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
     
-    const bool LsVectorised=false;
     typedef _Coeff_t Coeff_t;
 
     INHERIT_GIMPL_TYPES(Gimpl);
@@ -679,8 +744,6 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
     }
   };
 
-
-
   /////////////////////////////////////////////////////////////////////////////
   // Single flavour one component spinors with colour index. 5d vec
   /////////////////////////////////////////////////////////////////////////////
@@ -689,16 +752,14 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 
     public:
 
-    typedef RealD  _Coeff_t ;
     static const int Dimension = Representation::Dimension;
+    static const bool LsVectorised=true;
+    typedef RealD   Coeff_t ;
     typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
       
     //Necessary?
     constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
-    
-    const bool LsVectorised=true;
 
-    typedef _Coeff_t Coeff_t;
 
     INHERIT_GIMPL_TYPES(Gimpl);
 
@@ -861,43 +922,61 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
     }
   };
 
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffReal > WilsonImplR;  // Real.. whichever prec
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffReal > WilsonImplF;  // Float
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffReal > WilsonImplD;  // Double
 
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffRealHalfComms > WilsonImplRL;  // Real.. whichever prec
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplFH;  // Float
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplDF;  // Double
 
- typedef WilsonImpl<vComplex,  FundamentalRepresentation > WilsonImplR;   // Real.. whichever prec
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffComplex > ZWilsonImplR; // Real.. whichever prec
- typedef WilsonImpl<vComplexF, FundamentalRepresentation > WilsonImplF;  // Float
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplex > ZWilsonImplF; // Float
- typedef WilsonImpl<vComplexD, FundamentalRepresentation > WilsonImplD;  // Double
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplex > ZWilsonImplD; // Double
 
- typedef WilsonImpl<vComplex,  FundamentalRepresentation, ComplexD > ZWilsonImplR; // Real.. whichever prec
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplRL; // Real.. whichever prec
- typedef WilsonImpl<vComplexF, FundamentalRepresentation, ComplexD > ZWilsonImplF; // Float
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplFH; // Float
- typedef WilsonImpl<vComplexD, FundamentalRepresentation, ComplexD > ZWilsonImplD; // Double
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplDF; // Double
  
- typedef WilsonImpl<vComplex,  AdjointRepresentation > WilsonAdjImplR;   // Real.. whichever prec
+typedef WilsonImpl<vComplex,  AdjointRepresentation, CoeffReal > WilsonAdjImplR;   // Real.. whichever prec
- typedef WilsonImpl<vComplexF, AdjointRepresentation > WilsonAdjImplF;  // Float
+typedef WilsonImpl<vComplexF, AdjointRepresentation, CoeffReal > WilsonAdjImplF;  // Float
- typedef WilsonImpl<vComplexD, AdjointRepresentation > WilsonAdjImplD;  // Double
+typedef WilsonImpl<vComplexD, AdjointRepresentation, CoeffReal > WilsonAdjImplD;  // Double
  
- typedef WilsonImpl<vComplex,  TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplR;   // Real.. whichever prec
+typedef WilsonImpl<vComplex,  TwoIndexSymmetricRepresentation, CoeffReal > WilsonTwoIndexSymmetricImplR;   // Real.. whichever prec
- typedef WilsonImpl<vComplexF, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplF;  // Float
+typedef WilsonImpl<vComplexF, TwoIndexSymmetricRepresentation, CoeffReal > WilsonTwoIndexSymmetricImplF;  // Float
- typedef WilsonImpl<vComplexD, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplD;  // Double
+typedef WilsonImpl<vComplexD, TwoIndexSymmetricRepresentation, CoeffReal > WilsonTwoIndexSymmetricImplD;  // Double
  
- typedef DomainWallVec5dImpl<vComplex ,Nc> DomainWallVec5dImplR; // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplex ,Nc, CoeffReal> DomainWallVec5dImplR; // Real.. whichever prec
- typedef DomainWallVec5dImpl<vComplexF,Nc> DomainWallVec5dImplF; // Float
+typedef DomainWallVec5dImpl<vComplexF,Nc, CoeffReal> DomainWallVec5dImplF; // Float
- typedef DomainWallVec5dImpl<vComplexD,Nc> DomainWallVec5dImplD; // Double
+typedef DomainWallVec5dImpl<vComplexD,Nc, CoeffReal> DomainWallVec5dImplD; // Double
  
- typedef DomainWallVec5dImpl<vComplex ,Nc,ComplexD> ZDomainWallVec5dImplR; // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplex ,Nc, CoeffRealHalfComms> DomainWallVec5dImplRL; // Real.. whichever prec
- typedef DomainWallVec5dImpl<vComplexF,Nc,ComplexD> ZDomainWallVec5dImplF; // Float
+typedef DomainWallVec5dImpl<vComplexF,Nc, CoeffRealHalfComms> DomainWallVec5dImplFH; // Float
- typedef DomainWallVec5dImpl<vComplexD,Nc,ComplexD> ZDomainWallVec5dImplD; // Double
+typedef DomainWallVec5dImpl<vComplexD,Nc, CoeffRealHalfComms> DomainWallVec5dImplDF; // Double
  
- typedef GparityWilsonImpl<vComplex , Nc> GparityWilsonImplR;  // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplex ,Nc,CoeffComplex> ZDomainWallVec5dImplR; // Real.. whichever prec
- typedef GparityWilsonImpl<vComplexF, Nc> GparityWilsonImplF;  // Float
+typedef DomainWallVec5dImpl<vComplexF,Nc,CoeffComplex> ZDomainWallVec5dImplF; // Float
- typedef GparityWilsonImpl<vComplexD, Nc> GparityWilsonImplD;  // Double
+typedef DomainWallVec5dImpl<vComplexD,Nc,CoeffComplex> ZDomainWallVec5dImplD; // Double
+ 
+typedef DomainWallVec5dImpl<vComplex ,Nc,CoeffComplexHalfComms> ZDomainWallVec5dImplRL; // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplexF,Nc,CoeffComplexHalfComms> ZDomainWallVec5dImplFH; // Float
+typedef DomainWallVec5dImpl<vComplexD,Nc,CoeffComplexHalfComms> ZDomainWallVec5dImplDF; // Double
+ 
+typedef GparityWilsonImpl<vComplex , Nc,CoeffReal> GparityWilsonImplR;  // Real.. whichever prec
+typedef GparityWilsonImpl<vComplexF, Nc,CoeffReal> GparityWilsonImplF;  // Float
+typedef GparityWilsonImpl<vComplexD, Nc,CoeffReal> GparityWilsonImplD;  // Double
+ 
+typedef GparityWilsonImpl<vComplex , Nc,CoeffRealHalfComms> GparityWilsonImplRL;  // Real.. whichever prec
+typedef GparityWilsonImpl<vComplexF, Nc,CoeffRealHalfComms> GparityWilsonImplFH;  // Float
+typedef GparityWilsonImpl<vComplexD, Nc,CoeffRealHalfComms> GparityWilsonImplDF;  // Double
 
- typedef StaggeredImpl<vComplex,  FundamentalRepresentation > StaggeredImplR;   // Real.. whichever prec
+typedef StaggeredImpl<vComplex,  FundamentalRepresentation > StaggeredImplR;   // Real.. whichever prec
- typedef StaggeredImpl<vComplexF, FundamentalRepresentation > StaggeredImplF;  // Float
+typedef StaggeredImpl<vComplexF, FundamentalRepresentation > StaggeredImplF;  // Float
- typedef StaggeredImpl<vComplexD, FundamentalRepresentation > StaggeredImplD;  // Double
+typedef StaggeredImpl<vComplexD, FundamentalRepresentation > StaggeredImplD;  // Double
 
- typedef StaggeredVec5dImpl<vComplex,  FundamentalRepresentation > StaggeredVec5dImplR;   // Real.. whichever prec
+typedef StaggeredVec5dImpl<vComplex,  FundamentalRepresentation > StaggeredVec5dImplR;   // Real.. whichever prec
- typedef StaggeredVec5dImpl<vComplexF, FundamentalRepresentation > StaggeredVec5dImplF;  // Float
+typedef StaggeredVec5dImpl<vComplexF, FundamentalRepresentation > StaggeredVec5dImplF;  // Float
- typedef StaggeredVec5dImpl<vComplexD, FundamentalRepresentation > StaggeredVec5dImplD;  // Double
+typedef StaggeredVec5dImpl<vComplexD, FundamentalRepresentation > StaggeredVec5dImplD;  // Double
 
 }}
 

lib/qcd/action/fermion/ImprovedStaggeredFermion.cc

@@ -160,8 +160,6 @@ void ImprovedStaggeredFermion<Impl>::ImportGauge(const GaugeField &_Uthin,const
     PokeIndex<LorentzIndex>(UUUmu, U*(-0.5*c2/u0/u0/u0), mu+4);
   }
 
-  std::cout << " Umu " << Umu._odata[0]<<std::endl;
-  std::cout << " UUUmu " << UUUmu._odata[0]<<std::endl;
   pickCheckerboard(Even, UmuEven, Umu);
   pickCheckerboard(Odd,  UmuOdd , Umu);
   pickCheckerboard(Even, UUUmuEven, UUUmu);

lib/qcd/action/fermion/SchurDiagTwoKappa.h

@@ -0,0 +1,102 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: SchurDiagTwoKappa.h
+
+    Copyright (C) 2017
+
+Author: Christoph Lehner
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef  _SCHUR_DIAG_TWO_KAPPA_H
+#define  _SCHUR_DIAG_TWO_KAPPA_H
+
+namespace Grid {
+
+  // This is specific to (Z)mobius fermions
+  template<class Matrix, class Field>
+    class KappaSimilarityTransform {
+  public:
+    INHERIT_IMPL_TYPES(Matrix);
+    std::vector<Coeff_t> kappa, kappaDag, kappaInv, kappaInvDag;
+
+    KappaSimilarityTransform (Matrix &zmob) {
+      for (int i=0;i<(int)zmob.bs.size();i++) {
+	Coeff_t k = 1.0 / ( 2.0 * (zmob.bs[i] *(4 - zmob.M5) + 1.0) );
+	kappa.push_back( k );
+	kappaDag.push_back( conj(k) );
+	kappaInv.push_back( 1.0 / k );
+	kappaInvDag.push_back( 1.0 / conj(k) );
+      }
+    }
+
+  template<typename vobj>
+    void sscale(const Lattice<vobj>& in, Lattice<vobj>& out, Coeff_t* s) {
+    GridBase *grid=out._grid;
+    out.checkerboard = in.checkerboard;
+    assert(grid->_simd_layout[0] == 1); // should be fine for ZMobius for now
+    int Ls = grid->_rdimensions[0];
+    parallel_for(int ss=0;ss<grid->oSites();ss++){
+      vobj tmp = s[ss % Ls]*in._odata[ss];
+      vstream(out._odata[ss],tmp);
+    }
+  }
+
+  RealD sscale_norm(const Field& in, Field& out, Coeff_t* s) {
+    sscale(in,out,s);
+    return norm2(out);
+  }
+
+  virtual RealD M       (const Field& in, Field& out) { return sscale_norm(in,out,&kappa[0]);   }
+  virtual RealD MDag    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaDag[0]);}
+  virtual RealD MInv    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInv[0]);}
+  virtual RealD MInvDag (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInvDag[0]);}
+
+  };
+
+  template<class Matrix,class Field>
+    class SchurDiagTwoKappaOperator :  public SchurOperatorBase<Field> {
+  public:
+    KappaSimilarityTransform<Matrix, Field> _S;
+    SchurDiagTwoOperator<Matrix, Field> _Mat;
+
+    SchurDiagTwoKappaOperator (Matrix &Mat): _S(Mat), _Mat(Mat) {};
+
+    virtual  RealD Mpc      (const Field &in, Field &out) {
+      Field tmp(in._grid);
+
+      _S.MInv(in,out);
+      _Mat.Mpc(out,tmp);
+      return _S.M(tmp,out);
+
+    }
+    virtual  RealD MpcDag   (const Field &in, Field &out){
+      Field tmp(in._grid);
+
+      _S.MDag(in,out);
+      _Mat.MpcDag(out,tmp);
+      return _S.MInvDag(tmp,out);
+    }
+  };
+
+}
+
+#endif

lib/qcd/action/fermion/WilsonCompressor.h

@@ -33,228 +33,321 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 namespace QCD {
 
-  template<class SiteHalfSpinor,class SiteSpinor>
+/////////////////////////////////////////////////////////////////////////////////////////////
-  class WilsonCompressor {
+// optimised versions supporting half precision too
-  public:
+/////////////////////////////////////////////////////////////////////////////////////////////
-    int mu;
-    int dag;
 
-    WilsonCompressor(int _dag){
+template<class _HCspinor,class _Hspinor,class _Spinor, class projector,typename SFINAE = void >
-      mu=0;
+class WilsonCompressorTemplate;
-      dag=_dag;
-      assert((dag==0)||(dag==1));
-    }
-    void Point(int p) { 
-      mu=p;
-    };
 
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      int mudag=mu;
-      if (!dag) {
-	mudag=(mu+Nd)%(2*Nd);
-      }
-      switch(mudag) {
-      case Xp:
-	spProjXp(ret,in);
-	break;
-      case Yp:
-	spProjYp(ret,in);
-	break;
-      case Zp:
-	spProjZp(ret,in);
-	break;
-      case Tp:
-	spProjTp(ret,in);
-	break;
-      case Xm:
-	spProjXm(ret,in);
-	break;
-      case Ym:
-	spProjYm(ret,in);
-	break;
-      case Zm:
-	spProjZm(ret,in);
-	break;
-      case Tm:
-	spProjTm(ret,in);
-	break;
-      default: 
-	assert(0);
-	break;
-      }
-      return ret;
-    }
-  };
 
-  /////////////////////////
+template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
-  // optimised versions
+class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector,
-  /////////////////////////
+  typename std::enable_if<std::is_same<_HCspinor,_Hspinor>::value>::type >
+{
+ public:
+  
+  int mu,dag;  
 
-  template<class SiteHalfSpinor,class SiteSpinor>
+  void Point(int p) { mu=p; };
-  class WilsonXpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjXp(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonYpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjYp(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonZpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjZp(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonTpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjTp(ret,in);
-      return ret;
-    }
-  };
 
-  template<class SiteHalfSpinor,class SiteSpinor>
+  WilsonCompressorTemplate(int _dag=0){
-  class WilsonXmCompressor {
+    dag = _dag;
-  public:
+  }
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjXm(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonYmCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjYm(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonZmCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjZm(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonTmCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjTm(ret,in);
-      return ret;
-    }
-  };
 
-    // Fast comms buffer manipulation which should inline right through (avoid direction
+  typedef _Spinor         SiteSpinor;
-    // dependent logic that prevents inlining
+  typedef _Hspinor     SiteHalfSpinor;
-  template<class vobj,class cobj>
+  typedef _HCspinor SiteHalfCommSpinor;
-  class WilsonStencil : public CartesianStencil<vobj,cobj> {
+  typedef typename SiteHalfCommSpinor::vector_type vComplexLow;
-  public:
+  typedef typename SiteHalfSpinor::vector_type     vComplexHigh;
+  constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
 
-    typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
+  inline int CommDatumSize(void) {
+    return sizeof(SiteHalfCommSpinor);
+  }
 
-    WilsonStencil(GridBase *grid,
+  /*****************************************************/
+  /* Compress includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Compress(SiteHalfSpinor *buf,Integer o,const SiteSpinor &in) {
+    projector::Proj(buf[o],in,mu,dag);
+  }
+
+  /*****************************************************/
+  /* Exchange includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Exchange(SiteHalfSpinor *mp,
+                       SiteHalfSpinor *vp0,
+                       SiteHalfSpinor *vp1,
+		       Integer type,Integer o){
+    exchange(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type);
+  }
+
+  /*****************************************************/
+  /* Have a decompression step if mpi data is not same */
+  /*****************************************************/
+  inline void Decompress(SiteHalfSpinor *out,
+			 SiteHalfSpinor *in, Integer o) {    
+    assert(0);
+  }
+
+  /*****************************************************/
+  /* Compress Exchange                                 */
+  /*****************************************************/
+  inline void CompressExchange(SiteHalfSpinor *out0,
+			       SiteHalfSpinor *out1,
+			       const SiteSpinor *in,
+			       Integer j,Integer k, Integer m,Integer type){
+    SiteHalfSpinor temp1, temp2,temp3,temp4;
+    projector::Proj(temp1,in[k],mu,dag);
+    projector::Proj(temp2,in[m],mu,dag);
+    exchange(out0[j],out1[j],temp1,temp2,type);
+  }
+
+  /*****************************************************/
+  /* Pass the info to the stencil */
+  /*****************************************************/
+  inline bool DecompressionStep(void) { return false; }
+
+};
+
+template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
+class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector,
+  typename std::enable_if<!std::is_same<_HCspinor,_Hspinor>::value>::type >
+{
+ public:
+  
+  int mu,dag;  
+
+  void Point(int p) { mu=p; };
+
+  WilsonCompressorTemplate(int _dag=0){
+    dag = _dag;
+  }
+
+  typedef _Spinor         SiteSpinor;
+  typedef _Hspinor     SiteHalfSpinor;
+  typedef _HCspinor SiteHalfCommSpinor;
+  typedef typename SiteHalfCommSpinor::vector_type vComplexLow;
+  typedef typename SiteHalfSpinor::vector_type     vComplexHigh;
+  constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
+
+  inline int CommDatumSize(void) {
+    return sizeof(SiteHalfCommSpinor);
+  }
+
+  /*****************************************************/
+  /* Compress includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Compress(SiteHalfSpinor *buf,Integer o,const SiteSpinor &in) {
+    SiteHalfSpinor hsp;
+    SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf;
+    projector::Proj(hsp,in,mu,dag);
+    precisionChange((vComplexLow *)&hbuf[o],(vComplexHigh *)&hsp,Nw);
+  }
+
+  /*****************************************************/
+  /* Exchange includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Exchange(SiteHalfSpinor *mp,
+                       SiteHalfSpinor *vp0,
+                       SiteHalfSpinor *vp1,
+		       Integer type,Integer o){
+    SiteHalfSpinor vt0,vt1;
+    SiteHalfCommSpinor *vpp0 = (SiteHalfCommSpinor *)vp0;
+    SiteHalfCommSpinor *vpp1 = (SiteHalfCommSpinor *)vp1;
+    precisionChange((vComplexHigh *)&vt0,(vComplexLow *)&vpp0[o],Nw);
+    precisionChange((vComplexHigh *)&vt1,(vComplexLow *)&vpp1[o],Nw);
+    exchange(mp[2*o],mp[2*o+1],vt0,vt1,type);
+  }
+
+  /*****************************************************/
+  /* Have a decompression step if mpi data is not same */
+  /*****************************************************/
+  inline void Decompress(SiteHalfSpinor *out,
+			 SiteHalfSpinor *in, Integer o){
+    SiteHalfCommSpinor *hin=(SiteHalfCommSpinor *)in;
+    precisionChange((vComplexHigh *)&out[o],(vComplexLow *)&hin[o],Nw);
+  }
+
+  /*****************************************************/
+  /* Compress Exchange                                 */
+  /*****************************************************/
+  inline void CompressExchange(SiteHalfSpinor *out0,
+			       SiteHalfSpinor *out1,
+			       const SiteSpinor *in,
+			       Integer j,Integer k, Integer m,Integer type){
+    SiteHalfSpinor temp1, temp2,temp3,temp4;
+    SiteHalfCommSpinor *hout0 = (SiteHalfCommSpinor *)out0;
+    SiteHalfCommSpinor *hout1 = (SiteHalfCommSpinor *)out1;
+    projector::Proj(temp1,in[k],mu,dag);
+    projector::Proj(temp2,in[m],mu,dag);
+    exchange(temp3,temp4,temp1,temp2,type);
+    precisionChange((vComplexLow *)&hout0[j],(vComplexHigh *)&temp3,Nw);
+    precisionChange((vComplexLow *)&hout1[j],(vComplexHigh *)&temp4,Nw);
+  }
+
+  /*****************************************************/
+  /* Pass the info to the stencil */
+  /*****************************************************/
+  inline bool DecompressionStep(void) { return true; }
+
+};
+
+#define DECLARE_PROJ(Projector,Compressor,spProj)			\
+  class Projector {							\
+  public:								\
+    template<class hsp,class fsp>					\
+    static void Proj(hsp &result,const fsp &in,int mu,int dag){			\
+      spProj(result,in);						\
+    }									\
+  };									\
+template<typename HCS,typename HS,typename S> using Compressor = WilsonCompressorTemplate<HCS,HS,S,Projector>;
+
+DECLARE_PROJ(WilsonXpProjector,WilsonXpCompressor,spProjXp);
+DECLARE_PROJ(WilsonYpProjector,WilsonYpCompressor,spProjYp);
+DECLARE_PROJ(WilsonZpProjector,WilsonZpCompressor,spProjZp);
+DECLARE_PROJ(WilsonTpProjector,WilsonTpCompressor,spProjTp);
+DECLARE_PROJ(WilsonXmProjector,WilsonXmCompressor,spProjXm);
+DECLARE_PROJ(WilsonYmProjector,WilsonYmCompressor,spProjYm);
+DECLARE_PROJ(WilsonZmProjector,WilsonZmCompressor,spProjZm);
+DECLARE_PROJ(WilsonTmProjector,WilsonTmCompressor,spProjTm);
+
+class WilsonProjector {
+ public:
+  template<class hsp,class fsp>
+  static void Proj(hsp &result,const fsp &in,int mu,int dag){
+    int mudag=dag? mu : (mu+Nd)%(2*Nd);
+    switch(mudag) {
+    case Xp:	spProjXp(result,in);	break;
+    case Yp:	spProjYp(result,in);	break;
+    case Zp:	spProjZp(result,in);	break;
+    case Tp:	spProjTp(result,in);	break;
+    case Xm:	spProjXm(result,in);	break;
+    case Ym:	spProjYm(result,in);	break;
+    case Zm:	spProjZm(result,in);	break;
+    case Tm:	spProjTm(result,in);	break;
+    default: 	assert(0);	        break;
+    }
+  }
+};
+template<typename HCS,typename HS,typename S> using WilsonCompressor = WilsonCompressorTemplate<HCS,HS,S,WilsonProjector>;
+
+// Fast comms buffer manipulation which should inline right through (avoid direction
+// dependent logic that prevents inlining
+template<class vobj,class cobj>
+class WilsonStencil : public CartesianStencil<vobj,cobj> {
+public:
+
+  typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
+
+  std::vector<int> same_node;
+  std::vector<int> surface_list;
+
+  WilsonStencil(GridBase *grid,
 		int npoints,
 		int checkerboard,
 		const std::vector<int> &directions,
-		const std::vector<int> &distances)  : CartesianStencil<vobj,cobj> (grid,npoints,checkerboard,directions,distances) 
+		const std::vector<int> &distances)  
-      {    };
+    : CartesianStencil<vobj,cobj> (grid,npoints,checkerboard,directions,distances) ,
-
+    same_node(npoints)
-    template < class compressor>
+  { 
-    void HaloExchangeOpt(const Lattice<vobj> &source,compressor &compress) 
+    surface_list.resize(0);
-    {
-      std::vector<std::vector<CommsRequest_t> > reqs;
-      HaloExchangeOptGather(source,compress);
-      this->CommunicateBegin(reqs);
-      this->calls++;
-      this->CommunicateComplete(reqs);
-      this->CommsMerge();
-    }
-
-    template < class compressor>
-    void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress) 
-    {
-      this->calls++;
-      this->Mergers.resize(0); 
-      this->Packets.resize(0);
-      this->HaloGatherOpt(source,compress);
-    }
-
-
-    template < class compressor>
-    void HaloGatherOpt(const Lattice<vobj> &source,compressor &compress)
-    {
-      this->_grid->StencilBarrier();
-      // conformable(source._grid,_grid);
-      assert(source._grid==this->_grid);
-      this->halogtime-=usecond();
-      
-      this->u_comm_offset=0;
-      
-      int dag = compress.dag;
-      
-      WilsonXpCompressor<cobj,vobj> XpCompress; 
-      WilsonYpCompressor<cobj,vobj> YpCompress; 
-      WilsonZpCompressor<cobj,vobj> ZpCompress; 
-      WilsonTpCompressor<cobj,vobj> TpCompress;
-      WilsonXmCompressor<cobj,vobj> XmCompress;
-      WilsonYmCompressor<cobj,vobj> YmCompress;
-      WilsonZmCompressor<cobj,vobj> ZmCompress;
-      WilsonTmCompressor<cobj,vobj> TmCompress;
-
-      // Gather all comms buffers
-      //    for(int point = 0 ; point < _npoints; point++) {
-      //      compress.Point(point);
-      //      HaloGatherDir(source,compress,point,face_idx);
-      //    }
-      int face_idx=0;
-      if ( dag ) { 
-	//	std::cout << " Optimised Dagger compress " <<std::endl;
-	this->HaloGatherDir(source,XpCompress,Xp,face_idx);
-	this->HaloGatherDir(source,YpCompress,Yp,face_idx);
-	this->HaloGatherDir(source,ZpCompress,Zp,face_idx);
-	this->HaloGatherDir(source,TpCompress,Tp,face_idx);
-	this->HaloGatherDir(source,XmCompress,Xm,face_idx);
-	this->HaloGatherDir(source,YmCompress,Ym,face_idx);
-	this->HaloGatherDir(source,ZmCompress,Zm,face_idx);
-	this->HaloGatherDir(source,TmCompress,Tm,face_idx);
-      } else {
-	this->HaloGatherDir(source,XmCompress,Xp,face_idx);
-	this->HaloGatherDir(source,YmCompress,Yp,face_idx);
-	this->HaloGatherDir(source,ZmCompress,Zp,face_idx);
-	this->HaloGatherDir(source,TmCompress,Tp,face_idx);
-	this->HaloGatherDir(source,XpCompress,Xm,face_idx);
-	this->HaloGatherDir(source,YpCompress,Ym,face_idx);
-	this->HaloGatherDir(source,ZpCompress,Zm,face_idx);
-	this->HaloGatherDir(source,TpCompress,Tm,face_idx);
-      }
-      this->face_table_computed=1;
-      assert(this->u_comm_offset==this->_unified_buffer_size);
-      this->halogtime+=usecond();
-    }
-
   };
 
+  void BuildSurfaceList(int Ls,int vol4){
+
+    // find same node for SHM
+    // Here we know the distance is 1 for WilsonStencil
+    for(int point=0;point<this->_npoints;point++){
+      same_node[point] = this->SameNode(point);
+      //      std::cout << " dir " <<point<<" same_node " <<same_node[point]<<std::endl;
+    }
+    
+    for(int site = 0 ;site< vol4;site++){
+      int local = 1;
+      for(int point=0;point<this->_npoints;point++){
+	if( (!this->GetNodeLocal(site*Ls,point)) && (!same_node[point]) ){ 
+	  local = 0;
+	}
+      }
+      if(local == 0) { 
+	surface_list.push_back(site);
+      }
+    }
+  }
+
+  template < class compressor>
+  void HaloExchangeOpt(const Lattice<vobj> &source,compressor &compress) 
+  {
+    std::vector<std::vector<CommsRequest_t> > reqs;
+    this->HaloExchangeOptGather(source,compress);
+    this->CommunicateBegin(reqs);
+    this->CommunicateComplete(reqs);
+    this->CommsMerge(compress);
+    this->CommsMergeSHM(compress);
+  }
+  
+  template <class compressor>
+  void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress) 
+  {
+    this->Prepare();
+    this->HaloGatherOpt(source,compress);
+  }
+
+  template <class compressor>
+  void HaloGatherOpt(const Lattice<vobj> &source,compressor &compress)
+  {
+    // Strategy. Inherit types from Compressor.
+    // Use types to select the write direction by directon compressor
+    typedef typename compressor::SiteSpinor         SiteSpinor;
+    typedef typename compressor::SiteHalfSpinor     SiteHalfSpinor;
+    typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor;
+
+    this->_grid->StencilBarrier();
+
+    assert(source._grid==this->_grid);
+    this->halogtime-=usecond();
+    
+    this->u_comm_offset=0;
+      
+    WilsonXpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> XpCompress; 
+    WilsonYpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> YpCompress; 
+    WilsonZpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> ZpCompress; 
+    WilsonTpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> TpCompress;
+    WilsonXmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> XmCompress; 
+    WilsonYmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> YmCompress; 
+    WilsonZmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> ZmCompress; 
+    WilsonTmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> TmCompress;
+
+    int dag = compress.dag;
+    int face_idx=0;
+    if ( dag ) { 
+      //	std::cout << " Optimised Dagger compress " <<std::endl;
+      assert(same_node[Xp]==this->HaloGatherDir(source,XpCompress,Xp,face_idx));
+      assert(same_node[Yp]==this->HaloGatherDir(source,YpCompress,Yp,face_idx));
+      assert(same_node[Zp]==this->HaloGatherDir(source,ZpCompress,Zp,face_idx));
+      assert(same_node[Tp]==this->HaloGatherDir(source,TpCompress,Tp,face_idx));
+      assert(same_node[Xm]==this->HaloGatherDir(source,XmCompress,Xm,face_idx));
+      assert(same_node[Ym]==this->HaloGatherDir(source,YmCompress,Ym,face_idx));
+      assert(same_node[Zm]==this->HaloGatherDir(source,ZmCompress,Zm,face_idx));
+      assert(same_node[Tm]==this->HaloGatherDir(source,TmCompress,Tm,face_idx));
+    } else {
+      assert(same_node[Xp]==this->HaloGatherDir(source,XmCompress,Xp,face_idx));
+      assert(same_node[Yp]==this->HaloGatherDir(source,YmCompress,Yp,face_idx));
+      assert(same_node[Zp]==this->HaloGatherDir(source,ZmCompress,Zp,face_idx));
+      assert(same_node[Tp]==this->HaloGatherDir(source,TmCompress,Tp,face_idx));
+      assert(same_node[Xm]==this->HaloGatherDir(source,XpCompress,Xm,face_idx));
+      assert(same_node[Ym]==this->HaloGatherDir(source,YpCompress,Ym,face_idx));
+      assert(same_node[Zm]==this->HaloGatherDir(source,ZpCompress,Zm,face_idx));
+      assert(same_node[Tm]==this->HaloGatherDir(source,TpCompress,Tm,face_idx));
+    }
+    this->face_table_computed=1;
+    assert(this->u_comm_offset==this->_unified_buffer_size);
+    this->halogtime+=usecond();
+  }
+
+ };
 
 }} // namespace close
 #endif

lib/qcd/action/fermion/WilsonFermion5D.cc

@@ -118,6 +118,18 @@ WilsonFermion5D<Impl>::WilsonFermion5D(GaugeField &_Umu,
     
   // Allocate the required comms buffer
   ImportGauge(_Umu);
+  // Build lists of exterior only nodes
+  int LLs = FiveDimGrid._rdimensions[0];
+  int vol4;
+  vol4=FourDimGrid.oSites();
+  Stencil.BuildSurfaceList(LLs,vol4);
+  vol4=FourDimRedBlackGrid.oSites();
+  StencilEven.BuildSurfaceList(LLs,vol4);
+   StencilOdd.BuildSurfaceList(LLs,vol4);
+
+  std::cout << GridLogMessage << " SurfaceLists "<< Stencil.surface_list.size()
+                       <<" " << StencilEven.surface_list.size()<<std::endl;
+
 }
      
 template<class Impl>
@@ -359,6 +371,7 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
   DhopTotalTime+=usecond();
 }
 
+
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, LebesgueOrder &lo,
 							DoubledGaugeField & U,
@@ -372,12 +385,21 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
 
   int LLs = in._grid->_rdimensions[0];
   int len =  U._grid->oSites();
-  
+
   DhopFaceTime-=usecond();
   st.HaloExchangeOptGather(in,compressor);
   DhopFaceTime+=usecond();
   std::vector<std::vector<CommsRequest_t> > reqs;
 
+  // Rely on async comms; start comms before merge of local data
+  DhopCommTime-=usecond();
+  st.CommunicateBegin(reqs);
+
+  DhopFaceTime-=usecond();
+  st.CommsMergeSHM(compressor);
+  DhopFaceTime+=usecond();
+
+  // Perhaps use omp task and region
 #pragma omp parallel 
   { 
     int nthreads = omp_get_num_threads();
@@ -388,8 +410,6 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
     int sF = LLs * myoff;
 
     if ( me == 0 ) {
-      DhopCommTime-=usecond();
-      st.CommunicateBegin(reqs);
       st.CommunicateComplete(reqs);
       DhopCommTime+=usecond();
     } else { 
@@ -402,28 +422,37 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
   }
 
   DhopFaceTime-=usecond();
-  st.CommsMerge();
+  st.CommsMerge(compressor);
   DhopFaceTime+=usecond();
 
-#pragma omp parallel 
+  // Load imbalance alert. Should use dynamic schedule OMP for loop
-  {
+  // Perhaps create a list of only those sites with face work, and 
-    int nthreads = omp_get_num_threads();
+  // load balance process the list.
-    int me = omp_get_thread_num();
+  DhopComputeTime2-=usecond();
-    int myoff, mywork;
+  if (dag == DaggerYes) {
-
+    int sz=st.surface_list.size();
-    GridThread::GetWork(len,me,mywork,myoff,nthreads);
+    parallel_for (int ss = 0; ss < sz; ss++) {
-    int sF = LLs * myoff;
+      int sU = st.surface_list[ss];
-
+      int sF = LLs * sU;
-    // Exterior links in stencil
+      Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,0,1);
-    if ( me==0 ) DhopComputeTime2-=usecond();
+    }
-    if (dag == DaggerYes) Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,myoff,LLs,mywork,in,out,0,1);
+  } else {
-    else                  Kernels::DhopSite   (st,lo,U,st.CommBuf(),sF,myoff,LLs,mywork,in,out,0,1);
+    int sz=st.surface_list.size();
-    if ( me==0 ) DhopComputeTime2+=usecond();
+    parallel_for (int ss = 0; ss < sz; ss++) {
-  }// end parallel region
+      int sU = st.surface_list[ss];
+      int sF = LLs * sU;
+      Kernels::DhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,0,1);
+    }
+  }
+  DhopComputeTime2+=usecond();
 #else 
   assert(0);
 #endif
+
 }
+
+
+
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopInternalSerialComms(StencilImpl & st, LebesgueOrder &lo,
 					 DoubledGaugeField & U,
@@ -642,7 +671,6 @@ void WilsonFermion5D<Impl>::MomentumSpacePropagatorHw(FermionField &out,const Fe
 
 }
 
-
 FermOpTemplateInstantiate(WilsonFermion5D);
 GparityFermOpTemplateInstantiate(WilsonFermion5D);
   

lib/qcd/action/fermion/WilsonKernels.cc

@@ -33,52 +33,8 @@ directory
 namespace Grid {
 namespace QCD {
 
-  int WilsonKernelsStatic::Opt   = WilsonKernelsStatic::OptGeneric;
+int WilsonKernelsStatic::Opt   = WilsonKernelsStatic::OptGeneric;
-  int WilsonKernelsStatic::Comms = WilsonKernelsStatic::CommsAndCompute;
+int WilsonKernelsStatic::Comms = WilsonKernelsStatic::CommsAndCompute;
-
-#ifdef QPX
-#include <spi/include/kernel/location.h>
-#include <spi/include/l1p/types.h>
-#include <hwi/include/bqc/l1p_mmio.h>
-#include <hwi/include/bqc/A2_inlines.h>
-#endif
-
-void bgq_l1p_optimisation(int mode)
-{
-#ifdef QPX
-#undef L1P_CFG_PF_USR
-#define L1P_CFG_PF_USR  (0x3fde8000108ll)   /*  (64 bit reg, 23 bits wide, user/unpriv) */
-
-  uint64_t cfg_pf_usr;
-  if ( mode ) { 
-    cfg_pf_usr =
-        L1P_CFG_PF_USR_ifetch_depth(0)       
-      | L1P_CFG_PF_USR_ifetch_max_footprint(1)   
-      | L1P_CFG_PF_USR_pf_stream_est_on_dcbt 
-      | L1P_CFG_PF_USR_pf_stream_establish_enable
-      | L1P_CFG_PF_USR_pf_stream_optimistic
-      | L1P_CFG_PF_USR_pf_adaptive_throttle(0xF) ;
-    //    if ( sizeof(Float) == sizeof(double) ) {
-      cfg_pf_usr |=  L1P_CFG_PF_USR_dfetch_depth(2)| L1P_CFG_PF_USR_dfetch_max_footprint(3)   ;
-      //    } else {
-      //      cfg_pf_usr |=  L1P_CFG_PF_USR_dfetch_depth(1)| L1P_CFG_PF_USR_dfetch_max_footprint(2)   ;
-      //    }
-  } else { 
-    cfg_pf_usr = L1P_CFG_PF_USR_dfetch_depth(1)
-      | L1P_CFG_PF_USR_dfetch_max_footprint(2)   
-      | L1P_CFG_PF_USR_ifetch_depth(0)       
-      | L1P_CFG_PF_USR_ifetch_max_footprint(1)   
-      | L1P_CFG_PF_USR_pf_stream_est_on_dcbt 
-      | L1P_CFG_PF_USR_pf_stream_establish_enable
-      | L1P_CFG_PF_USR_pf_stream_optimistic
-      | L1P_CFG_PF_USR_pf_stream_prefetch_enable;
-  }
-  *((uint64_t *)L1P_CFG_PF_USR) = cfg_pf_usr;
-
-#endif
-
-}
-
 
 template <class Impl>
 WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
@@ -86,12 +42,72 @@ WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
 ////////////////////////////////////////////
 // Generic implementation; move to different file?
 ////////////////////////////////////////////
+  
+#define GENERIC_STENCIL_LEG(Dir,spProj,Recon)			\
+  SE = st.GetEntry(ptype, Dir, sF);				\
+  if (SE->_is_local) {						\
+    chi_p = &chi;						\
+    if (SE->_permute) {						\
+      spProj(tmp, in._odata[SE->_offset]);			\
+      permute(chi, tmp, ptype);					\
+    } else {							\
+      spProj(chi, in._odata[SE->_offset]);			\
+    }								\
+  } else {							\
+    chi_p = &buf[SE->_offset];					\
+  }								\
+  Impl::multLink(Uchi, U._odata[sU], *chi_p, Dir, SE, st);	\
+  Recon(result, Uchi);
+  
+#define GENERIC_STENCIL_LEG_INT(Dir,spProj,Recon)		\
+  SE = st.GetEntry(ptype, Dir, sF);				\
+  if (SE->_is_local) {						\
+    chi_p = &chi;						\
+    if (SE->_permute) {						\
+      spProj(tmp, in._odata[SE->_offset]);			\
+      permute(chi, tmp, ptype);					\
+    } else {							\
+      spProj(chi, in._odata[SE->_offset]);			\
+    }								\
+  } else if ( st.same_node[Dir] ) {				\
+      chi_p = &buf[SE->_offset];				\
+  }								\
+  if (SE->_is_local || st.same_node[Dir] ) {			\
+    Impl::multLink(Uchi, U._odata[sU], *chi_p, Dir, SE, st);	\
+    Recon(result, Uchi);					\
+  }
 
+#define GENERIC_STENCIL_LEG_EXT(Dir,spProj,Recon)		\
+  SE = st.GetEntry(ptype, Dir, sF);				\
+  if ((!SE->_is_local) && (!st.same_node[Dir]) ) {		\
+    chi_p = &buf[SE->_offset];					\
+    Impl::multLink(Uchi, U._odata[sU], *chi_p, Dir, SE, st);	\
+    Recon(result, Uchi);					\
+    nmu++;							\
+  }
+
+#define GENERIC_DHOPDIR_LEG(Dir,spProj,Recon)			\
+  if (gamma == Dir) {						\
+    if (SE->_is_local && SE->_permute) {			\
+      spProj(tmp, in._odata[SE->_offset]);			\
+      permute(chi, tmp, ptype);					\
+    } else if (SE->_is_local) {					\
+      spProj(chi, in._odata[SE->_offset]);			\
+    } else {							\
+      chi = buf[SE->_offset];					\
+    }								\
+    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);	\
+    Recon(result, Uchi);					\
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // All legs kernels ; comms then compute
+  ////////////////////////////////////////////////////////////////////
 template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-						     SiteHalfSpinor *buf, int sF,
+					     SiteHalfSpinor *buf, int sF,
-						     int sU, const FermionField &in, FermionField &out,
+					     int sU, const FermionField &in, FermionField &out)
-						     int interior,int exterior) {
+{
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteHalfSpinor *chi_p;
@@ -100,174 +116,22 @@ void WilsonKernels<Impl>::GenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo,
   StencilEntry *SE;
   int ptype;
 
-  ///////////////////////////
+  GENERIC_STENCIL_LEG(Xp,spProjXp,spReconXp);
-  // Xp
+  GENERIC_STENCIL_LEG(Yp,spProjYp,accumReconYp);
-  ///////////////////////////
+  GENERIC_STENCIL_LEG(Zp,spProjZp,accumReconZp);
-  SE = st.GetEntry(ptype, Xp, sF);
+  GENERIC_STENCIL_LEG(Tp,spProjTp,accumReconTp);
-
+  GENERIC_STENCIL_LEG(Xm,spProjXm,accumReconXm);
-  if (SE->_is_local) {
+  GENERIC_STENCIL_LEG(Ym,spProjYm,accumReconYm);
-    chi_p = &chi;
+  GENERIC_STENCIL_LEG(Zm,spProjZm,accumReconZm);
-    if (SE->_permute) {
+  GENERIC_STENCIL_LEG(Tm,spProjTm,accumReconTm);
-      spProjXp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjXp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Xp, SE, st);
-  spReconXp(result, Uchi);
-
-  ///////////////////////////
-  // Yp
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Yp, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjYp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjYp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Yp, SE, st);
-  accumReconYp(result, Uchi);
-
-  ///////////////////////////
-  // Zp
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Zp, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjZp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjZp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Zp, SE, st);
-  accumReconZp(result, Uchi);
-
-  ///////////////////////////
-  // Tp
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Tp, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjTp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjTp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Tp, SE, st);
-  accumReconTp(result, Uchi);
-
-  ///////////////////////////
-  // Xm
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Xm, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjXm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjXm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Xm, SE, st);
-  accumReconXm(result, Uchi);
-
-  ///////////////////////////
-  // Ym
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Ym, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjYm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjYm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Ym, SE, st);
-  accumReconYm(result, Uchi);
-
-  ///////////////////////////
-  // Zm
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Zm, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjZm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjZm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Zm, SE, st);
-  accumReconZm(result, Uchi);
-
-  ///////////////////////////
-  // Tm
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Tm, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjTm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjTm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Tm, SE, st);
-  accumReconTm(result, Uchi);
-
   vstream(out._odata[sF], result);
 };
 
-// Need controls to do interior, exterior, or both
 template <class Impl>
 void WilsonKernels<Impl>::GenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-						  SiteHalfSpinor *buf, int sF,
+					  SiteHalfSpinor *buf, int sF,
-						  int sU, const FermionField &in, FermionField &out,int interior,int exterior) {
+					  int sU, const FermionField &in, FermionField &out) 
+{
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteHalfSpinor *chi_p;
@@ -276,168 +140,123 @@ void WilsonKernels<Impl>::GenericDhopSite(StencilImpl &st, LebesgueOrder &lo, Do
   StencilEntry *SE;
   int ptype;
 
-  ///////////////////////////
+  GENERIC_STENCIL_LEG(Xm,spProjXp,spReconXp);
-  // Xp
+  GENERIC_STENCIL_LEG(Ym,spProjYp,accumReconYp);
-  ///////////////////////////
+  GENERIC_STENCIL_LEG(Zm,spProjZp,accumReconZp);
-  SE = st.GetEntry(ptype, Xm, sF);
+  GENERIC_STENCIL_LEG(Tm,spProjTp,accumReconTp);
-
+  GENERIC_STENCIL_LEG(Xp,spProjXm,accumReconXm);
-  if (SE->_is_local) {
+  GENERIC_STENCIL_LEG(Yp,spProjYm,accumReconYm);
-    chi_p = &chi;
+  GENERIC_STENCIL_LEG(Zp,spProjZm,accumReconZm);
-    if (SE->_permute) {
+  GENERIC_STENCIL_LEG(Tp,spProjTm,accumReconTm);
-      spProjXp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjXp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Xm, SE, st);
-  spReconXp(result, Uchi);
-
-  ///////////////////////////
-  // Yp
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Ym, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjYp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjYp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Ym, SE, st);
-  accumReconYp(result, Uchi);
-
-  ///////////////////////////
-  // Zp
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Zm, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjZp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjZp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Zm, SE, st);
-  accumReconZp(result, Uchi);
-
-  ///////////////////////////
-  // Tp
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Tm, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjTp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjTp(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Tm, SE, st);
-  accumReconTp(result, Uchi);
-
-  ///////////////////////////
-  // Xm
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Xp, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjXm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjXm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Xp, SE, st);
-  accumReconXm(result, Uchi);
-
-  ///////////////////////////
-  // Ym
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Yp, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjYm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjYm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Yp, SE, st);
-  accumReconYm(result, Uchi);
-
-  ///////////////////////////
-  // Zm
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Zp, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjZm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjZm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Zp, SE, st);
-  accumReconZm(result, Uchi);
-
-  ///////////////////////////
-  // Tm
-  ///////////////////////////
-  SE = st.GetEntry(ptype, Tp, sF);
-
-  if (SE->_is_local) {
-    chi_p = &chi;
-    if (SE->_permute) {
-      spProjTm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else {
-      spProjTm(chi, in._odata[SE->_offset]);
-    }
-  } else {
-    chi_p = &buf[SE->_offset];
-  }
-
-  Impl::multLink(Uchi, U._odata[sU], *chi_p, Tp, SE, st);
-  accumReconTm(result, Uchi);
-
   vstream(out._odata[sF], result);
 };
+  ////////////////////////////////////////////////////////////////////
+  // Interior kernels
+  ////////////////////////////////////////////////////////////////////
+template <class Impl>
+void WilsonKernels<Impl>::GenericDhopSiteDagInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+						SiteHalfSpinor *buf, int sF,
+						int sU, const FermionField &in, FermionField &out)
+{
+  SiteHalfSpinor tmp;
+  SiteHalfSpinor chi;
+  SiteHalfSpinor *chi_p;
+  SiteHalfSpinor Uchi;
+  SiteSpinor result;
+  StencilEntry *SE;
+  int ptype;
+
+  result=zero;
+  GENERIC_STENCIL_LEG_INT(Xp,spProjXp,accumReconXp);
+  GENERIC_STENCIL_LEG_INT(Yp,spProjYp,accumReconYp);
+  GENERIC_STENCIL_LEG_INT(Zp,spProjZp,accumReconZp);
+  GENERIC_STENCIL_LEG_INT(Tp,spProjTp,accumReconTp);
+  GENERIC_STENCIL_LEG_INT(Xm,spProjXm,accumReconXm);
+  GENERIC_STENCIL_LEG_INT(Ym,spProjYm,accumReconYm);
+  GENERIC_STENCIL_LEG_INT(Zm,spProjZm,accumReconZm);
+  GENERIC_STENCIL_LEG_INT(Tm,spProjTm,accumReconTm);
+  vstream(out._odata[sF], result);
+};
+
+template <class Impl>
+void WilsonKernels<Impl>::GenericDhopSiteInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+					     SiteHalfSpinor *buf, int sF,
+					     int sU, const FermionField &in, FermionField &out) 
+{
+  SiteHalfSpinor tmp;
+  SiteHalfSpinor chi;
+  SiteHalfSpinor *chi_p;
+  SiteHalfSpinor Uchi;
+  SiteSpinor result;
+  StencilEntry *SE;
+  int ptype;
+  result=zero;
+  GENERIC_STENCIL_LEG_INT(Xm,spProjXp,accumReconXp);
+  GENERIC_STENCIL_LEG_INT(Ym,spProjYp,accumReconYp);
+  GENERIC_STENCIL_LEG_INT(Zm,spProjZp,accumReconZp);
+  GENERIC_STENCIL_LEG_INT(Tm,spProjTp,accumReconTp);
+  GENERIC_STENCIL_LEG_INT(Xp,spProjXm,accumReconXm);
+  GENERIC_STENCIL_LEG_INT(Yp,spProjYm,accumReconYm);
+  GENERIC_STENCIL_LEG_INT(Zp,spProjZm,accumReconZm);
+  GENERIC_STENCIL_LEG_INT(Tp,spProjTm,accumReconTm);
+  vstream(out._odata[sF], result);
+};
+////////////////////////////////////////////////////////////////////
+// Exterior kernels
+////////////////////////////////////////////////////////////////////
+template <class Impl>
+void WilsonKernels<Impl>::GenericDhopSiteDagExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+						SiteHalfSpinor *buf, int sF,
+						int sU, const FermionField &in, FermionField &out)
+{
+  SiteHalfSpinor tmp;
+  SiteHalfSpinor chi;
+  SiteHalfSpinor *chi_p;
+  SiteHalfSpinor Uchi;
+  SiteSpinor result;
+  StencilEntry *SE;
+  int ptype;
+  int nmu=0;
+  result=zero;
+  GENERIC_STENCIL_LEG_EXT(Xp,spProjXp,accumReconXp);
+  GENERIC_STENCIL_LEG_EXT(Yp,spProjYp,accumReconYp);
+  GENERIC_STENCIL_LEG_EXT(Zp,spProjZp,accumReconZp);
+  GENERIC_STENCIL_LEG_EXT(Tp,spProjTp,accumReconTp);
+  GENERIC_STENCIL_LEG_EXT(Xm,spProjXm,accumReconXm);
+  GENERIC_STENCIL_LEG_EXT(Ym,spProjYm,accumReconYm);
+  GENERIC_STENCIL_LEG_EXT(Zm,spProjZm,accumReconZm);
+  GENERIC_STENCIL_LEG_EXT(Tm,spProjTm,accumReconTm);
+  if ( nmu ) { 
+    out._odata[sF] = out._odata[sF] + result; 
+  }
+};
+
+template <class Impl>
+void WilsonKernels<Impl>::GenericDhopSiteExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+					     SiteHalfSpinor *buf, int sF,
+					     int sU, const FermionField &in, FermionField &out) 
+{
+  SiteHalfSpinor tmp;
+  SiteHalfSpinor chi;
+  SiteHalfSpinor *chi_p;
+  SiteHalfSpinor Uchi;
+  SiteSpinor result;
+  StencilEntry *SE;
+  int ptype;
+  int nmu=0;
+  result=zero;
+  GENERIC_STENCIL_LEG_EXT(Xm,spProjXp,accumReconXp);
+  GENERIC_STENCIL_LEG_EXT(Ym,spProjYp,accumReconYp);
+  GENERIC_STENCIL_LEG_EXT(Zm,spProjZp,accumReconZp);
+  GENERIC_STENCIL_LEG_EXT(Tm,spProjTp,accumReconTp);
+  GENERIC_STENCIL_LEG_EXT(Xp,spProjXm,accumReconXm);
+  GENERIC_STENCIL_LEG_EXT(Yp,spProjYm,accumReconYm);
+  GENERIC_STENCIL_LEG_EXT(Zp,spProjZm,accumReconZm);
+  GENERIC_STENCIL_LEG_EXT(Tp,spProjTm,accumReconTm);
+  if ( nmu ) { 
+    out._odata[sF] = out._odata[sF] + result; 
+  }
+};
 
 template <class Impl>
 void WilsonKernels<Impl>::DhopDir( StencilImpl &st, DoubledGaugeField &U,SiteHalfSpinor *buf, int sF,
@@ -451,119 +270,14 @@ void WilsonKernels<Impl>::DhopDir( StencilImpl &st, DoubledGaugeField &U,SiteHal
   int ptype;
 
   SE = st.GetEntry(ptype, dir, sF);
-
+  GENERIC_DHOPDIR_LEG(Xp,spProjXp,spReconXp);
-  // Xp
+  GENERIC_DHOPDIR_LEG(Yp,spProjYp,spReconYp);
-  if (gamma == Xp) {
+  GENERIC_DHOPDIR_LEG(Zp,spProjZp,spReconZp);
-    if (SE->_is_local && SE->_permute) {
+  GENERIC_DHOPDIR_LEG(Tp,spProjTp,spReconTp);
-      spProjXp(tmp, in._odata[SE->_offset]);
+  GENERIC_DHOPDIR_LEG(Xm,spProjXm,spReconXm);
-      permute(chi, tmp, ptype);
+  GENERIC_DHOPDIR_LEG(Ym,spProjYm,spReconYm);
-    } else if (SE->_is_local) {
+  GENERIC_DHOPDIR_LEG(Zm,spProjZm,spReconZm);
-      spProjXp(chi, in._odata[SE->_offset]);
+  GENERIC_DHOPDIR_LEG(Tm,spProjTm,spReconTm);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconXp(result, Uchi);
-  }
-
-  // Yp
-  if (gamma == Yp) {
-    if (SE->_is_local && SE->_permute) {
-      spProjYp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else if (SE->_is_local) {
-      spProjYp(chi, in._odata[SE->_offset]);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconYp(result, Uchi);
-  }
-
-  // Zp
-  if (gamma == Zp) {
-    if (SE->_is_local && SE->_permute) {
-      spProjZp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else if (SE->_is_local) {
-      spProjZp(chi, in._odata[SE->_offset]);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconZp(result, Uchi);
-  }
-
-  // Tp
-  if (gamma == Tp) {
-    if (SE->_is_local && SE->_permute) {
-      spProjTp(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else if (SE->_is_local) {
-      spProjTp(chi, in._odata[SE->_offset]);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconTp(result, Uchi);
-  }
-
-  // Xm
-  if (gamma == Xm) {
-    if (SE->_is_local && SE->_permute) {
-      spProjXm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else if (SE->_is_local) {
-      spProjXm(chi, in._odata[SE->_offset]);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconXm(result, Uchi);
-  }
-
-  // Ym
-  if (gamma == Ym) {
-    if (SE->_is_local && SE->_permute) {
-      spProjYm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else if (SE->_is_local) {
-      spProjYm(chi, in._odata[SE->_offset]);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconYm(result, Uchi);
-  }
-
-  // Zm
-  if (gamma == Zm) {
-    if (SE->_is_local && SE->_permute) {
-      spProjZm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else if (SE->_is_local) {
-      spProjZm(chi, in._odata[SE->_offset]);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconZm(result, Uchi);
-  }
-
-  // Tm
-  if (gamma == Tm) {
-    if (SE->_is_local && SE->_permute) {
-      spProjTm(tmp, in._odata[SE->_offset]);
-      permute(chi, tmp, ptype);
-    } else if (SE->_is_local) {
-      spProjTm(chi, in._odata[SE->_offset]);
-    } else {
-      chi = buf[SE->_offset];
-    }
-    Impl::multLink(Uchi, U._odata[sU], chi, dir, SE, st);
-    spReconTm(result, Uchi);
-  }
-
   vstream(out._odata[sF], result);
 }
 

lib/qcd/action/fermion/WilsonKernels.h

@@ -34,8 +34,6 @@ directory
 namespace Grid {
 namespace QCD {
 
-void bgq_l1p_optimisation(int mode);
-
   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
   // Helper routines that implement Wilson stencil for a single site.
   // Common to both the WilsonFermion and WilsonFermion5D
@@ -44,9 +42,8 @@ class WilsonKernelsStatic {
  public:
   enum { OptGeneric, OptHandUnroll, OptInlineAsm };
   enum { CommsAndCompute, CommsThenCompute };
-  // S-direction is INNERMOST and takes no part in the parity.
+  static int Opt;  
-  static int Opt;  // these are a temporary hack
+  static int Comms;
-  static int Comms;  // these are a temporary hack
 };
  
 template<class Impl> class WilsonKernels : public FermionOperator<Impl> , public WilsonKernelsStatic { 
@@ -66,7 +63,7 @@ public:
     switch(Opt) {
 #if defined(AVX512) || defined (QPX)
     case OptInlineAsm:
-      if(interior&&exterior) WilsonKernels<Impl>::AsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
+      if(interior&&exterior) WilsonKernels<Impl>::AsmDhopSite   (st,lo,U,buf,sF,sU,Ls,Ns,in,out);
       else if (interior)     WilsonKernels<Impl>::AsmDhopSiteInt(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
       else if (exterior)     WilsonKernels<Impl>::AsmDhopSiteExt(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
       else assert(0);
@@ -75,7 +72,9 @@ public:
     case OptHandUnroll:
       for (int site = 0; site < Ns; site++) {
 	for (int s = 0; s < Ls; s++) {
-	  if( exterior) WilsonKernels<Impl>::HandDhopSite(st,lo,U,buf,sF,sU,in,out,interior,exterior);
+	  if(interior&&exterior) WilsonKernels<Impl>::HandDhopSite(st,lo,U,buf,sF,sU,in,out);
+	  else if (interior)     WilsonKernels<Impl>::HandDhopSiteInt(st,lo,U,buf,sF,sU,in,out);
+	  else if (exterior)     WilsonKernels<Impl>::HandDhopSiteExt(st,lo,U,buf,sF,sU,in,out);
 	  sF++;
 	}
 	sU++;
@@ -84,7 +83,10 @@ public:
     case OptGeneric:
       for (int site = 0; site < Ns; site++) {
 	for (int s = 0; s < Ls; s++) {
-	  if( exterior) WilsonKernels<Impl>::GenericDhopSite(st,lo,U,buf,sF,sU,in,out,interior,exterior);
+	  if(interior&&exterior) WilsonKernels<Impl>::GenericDhopSite(st,lo,U,buf,sF,sU,in,out);
+	  else if (interior)     WilsonKernels<Impl>::GenericDhopSiteInt(st,lo,U,buf,sF,sU,in,out);
+	  else if (exterior)     WilsonKernels<Impl>::GenericDhopSiteExt(st,lo,U,buf,sF,sU,in,out);
+	  else assert(0);
 	  sF++;
 	}
 	sU++;
@@ -99,11 +101,14 @@ public:
   template <bool EnableBool = true>
   typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type
   DhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-		   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out,int interior=1,int exterior=1 ) {
+	   int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out,int interior=1,int exterior=1 ) {
     // no kernel choice  
     for (int site = 0; site < Ns; site++) {
       for (int s = 0; s < Ls; s++) {
-	if( exterior) WilsonKernels<Impl>::GenericDhopSite(st, lo, U, buf, sF, sU, in, out,interior,exterior);
+	if(interior&&exterior) WilsonKernels<Impl>::GenericDhopSite(st,lo,U,buf,sF,sU,in,out);
+	else if (interior)     WilsonKernels<Impl>::GenericDhopSiteInt(st,lo,U,buf,sF,sU,in,out);
+	else if (exterior)     WilsonKernels<Impl>::GenericDhopSiteExt(st,lo,U,buf,sF,sU,in,out);
+	else assert(0);
 	sF++;
       }
       sU++;
@@ -113,13 +118,13 @@ public:
   template <bool EnableBool = true>
   typename std::enable_if<Impl::Dimension == 3 && Nc == 3 && EnableBool,void>::type
   DhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-		      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out,int interior=1,int exterior=1) {
+	      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out,int interior=1,int exterior=1) 
-
+{
     bgq_l1p_optimisation(1);
     switch(Opt) {
 #if defined(AVX512) || defined (QPX)
     case OptInlineAsm:
-      if(interior&&exterior) WilsonKernels<Impl>::AsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
+      if(interior&&exterior) WilsonKernels<Impl>::AsmDhopSiteDag   (st,lo,U,buf,sF,sU,Ls,Ns,in,out);
       else if (interior)     WilsonKernels<Impl>::AsmDhopSiteDagInt(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
       else if (exterior)     WilsonKernels<Impl>::AsmDhopSiteDagExt(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
       else assert(0);
@@ -128,7 +133,10 @@ public:
     case OptHandUnroll:
       for (int site = 0; site < Ns; site++) {
 	for (int s = 0; s < Ls; s++) {
-	  if( exterior) WilsonKernels<Impl>::HandDhopSiteDag(st,lo,U,buf,sF,sU,in,out,interior,exterior);
+	  if(interior&&exterior) WilsonKernels<Impl>::HandDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	  else if (interior)     WilsonKernels<Impl>::HandDhopSiteDagInt(st,lo,U,buf,sF,sU,in,out);
+	  else if (exterior)     WilsonKernels<Impl>::HandDhopSiteDagExt(st,lo,U,buf,sF,sU,in,out);
+	  else assert(0);
 	  sF++;
 	}
 	sU++;
@@ -137,7 +145,10 @@ public:
     case OptGeneric:
       for (int site = 0; site < Ns; site++) {
 	for (int s = 0; s < Ls; s++) {
-	  if( exterior) WilsonKernels<Impl>::GenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out,interior,exterior);
+	  if(interior&&exterior) WilsonKernels<Impl>::GenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	  else if (interior)     WilsonKernels<Impl>::GenericDhopSiteDagInt(st,lo,U,buf,sF,sU,in,out);
+	  else if (exterior)     WilsonKernels<Impl>::GenericDhopSiteDagExt(st,lo,U,buf,sF,sU,in,out);
+	  else assert(0);
 	  sF++;
 	}
 	sU++;
@@ -156,7 +167,10 @@ public:
 
     for (int site = 0; site < Ns; site++) {
       for (int s = 0; s < Ls; s++) {
-	if( exterior) WilsonKernels<Impl>::GenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out,interior,exterior);
+	if(interior&&exterior) WilsonKernels<Impl>::GenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
+	else if (interior)     WilsonKernels<Impl>::GenericDhopSiteDagInt(st,lo,U,buf,sF,sU,in,out);
+	else if (exterior)     WilsonKernels<Impl>::GenericDhopSiteDagExt(st,lo,U,buf,sF,sU,in,out);
+	else assert(0);
 	sF++;
       }
       sU++;
@@ -169,36 +183,60 @@ public:
 private:
      // Specialised variants
   void GenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			       int sF, int sU, const FermionField &in, FermionField &out,int interior,int exterior);
+		       int sF, int sU, const FermionField &in, FermionField &out);
       
   void GenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-				  int sF, int sU, const FermionField &in, FermionField &out,int interior,int exterior);
+			  int sF, int sU, const FermionField &in, FermionField &out);
+
+  void GenericDhopSiteInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			  int sF, int sU, const FermionField &in, FermionField &out);
+      
+  void GenericDhopSiteDagInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			     int sF, int sU, const FermionField &in, FermionField &out);
+
+  void GenericDhopSiteExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			  int sF, int sU, const FermionField &in, FermionField &out);
+      
+  void GenericDhopSiteDagExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			     int sF, int sU, const FermionField &in, FermionField &out);
 
   void AsmDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			   int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
+		   int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
 
   void AsmDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
+		      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
 
   void AsmDhopSiteInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			   int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
+		      int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
 
   void AsmDhopSiteDagInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			      int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
+			 int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
 
   void AsmDhopSiteExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			      int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
+		      int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out);
 
   void AsmDhopSiteDagExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-				 int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
+			 int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out);
 
 
   void HandDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			    int sF, int sU, const FermionField &in, FermionField &out,int interior,int exterior);
+		    int sF, int sU, const FermionField &in, FermionField &out);
 
   void HandDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
-			       int sF, int sU, const FermionField &in, FermionField &out,int interior,int exterior);
+		       int sF, int sU, const FermionField &in, FermionField &out);
       
+  void HandDhopSiteInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+		       int sF, int sU, const FermionField &in, FermionField &out);
+  
+  void HandDhopSiteDagInt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			  int sF, int sU, const FermionField &in, FermionField &out);
+  
+  void HandDhopSiteExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+		       int sF, int sU, const FermionField &in, FermionField &out);
+  
+  void HandDhopSiteDagExt(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf,
+			  int sF, int sU, const FermionField &in, FermionField &out);
+  
 public:
 
   WilsonKernels(const ImplParams &p = ImplParams());

lib/qcd/action/fermion/WilsonKernelsAsm.cc

@@ -112,5 +112,16 @@ INSTANTIATE_ASM(DomainWallVec5dImplD);
 INSTANTIATE_ASM(ZDomainWallVec5dImplF);
 INSTANTIATE_ASM(ZDomainWallVec5dImplD);
 
+INSTANTIATE_ASM(WilsonImplFH);
+INSTANTIATE_ASM(WilsonImplDF);
+INSTANTIATE_ASM(ZWilsonImplFH);
+INSTANTIATE_ASM(ZWilsonImplDF);
+INSTANTIATE_ASM(GparityWilsonImplFH);
+INSTANTIATE_ASM(GparityWilsonImplDF);
+INSTANTIATE_ASM(DomainWallVec5dImplFH);
+INSTANTIATE_ASM(DomainWallVec5dImplDF);
+INSTANTIATE_ASM(ZDomainWallVec5dImplFH);
+INSTANTIATE_ASM(ZDomainWallVec5dImplDF);
+
 }}
 

lib/qcd/action/fermion/WilsonKernelsAsmAvx512.h

@@ -71,6 +71,16 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,Doub
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplFH>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
+template<> void 
+WilsonKernels<ZWilsonImplFH>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -84,6 +94,16 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,D
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplFH>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
+template<> void 
+WilsonKernels<ZWilsonImplFH>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
@@ -97,6 +117,16 @@ template<> void
 WilsonKernels<ZWilsonImplF>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
+template<> void 
+WilsonKernels<WilsonImplFH>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
+template<> void 
+WilsonKernels<ZWilsonImplFH>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
       
 /////////////////////////////////////////////////////////////////
 // XYZT vectorised, dag Kernel, single
@@ -115,6 +145,16 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,D
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplFH>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
+template<> void 
+WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -128,6 +168,16 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & l
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplFH>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
+template<> void 
+WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
 #define EXTERIOR
@@ -141,6 +191,16 @@ WilsonKernels<ZWilsonImplF>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & l
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
+template<> void 
+WilsonKernels<WilsonImplFH>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
+template<> void 
+WilsonKernels<ZWilsonImplFH>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
 #undef MAYBEPERM
 #undef MULT_2SPIN
 #define MAYBEPERM(A,B) 
@@ -162,6 +222,15 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSite(StencilImpl &st,LebesgueOrder
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -174,6 +243,15 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrd
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
 #define EXTERIOR
@@ -189,6 +267,16 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrd
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
+template<> void 
+WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
 /////////////////////////////////////////////////////////////////
 // Ls vectorised, dag Kernel, single
 /////////////////////////////////////////////////////////////////
@@ -205,6 +293,15 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrd
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -217,6 +314,15 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteDagInt(StencilImpl &st,Lebesgue
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
 #define EXTERIOR
@@ -229,6 +335,15 @@ WilsonKernels<ZDomainWallVec5dImplF>::AsmDhopSiteDagExt(StencilImpl &st,Lebesgue
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplFH>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplFH>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef COMPLEX_SIGNS
 #undef MAYBEPERM
 #undef MULT_2SPIN
@@ -269,6 +384,15 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,Doub
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplDF>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZWilsonImplDF>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -281,6 +405,15 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,D
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplDF>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZWilsonImplDF>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
 #define EXTERIOR
@@ -293,6 +426,15 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,D
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
       
+template<> void 
+WilsonKernels<WilsonImplDF>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZWilsonImplDF>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+      
 /////////////////////////////////////////////////////////////////
 // XYZT vectorised, dag Kernel, single
 /////////////////////////////////////////////////////////////////
@@ -309,6 +451,15 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,D
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplDF>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -321,6 +472,15 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & l
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<WilsonImplDF>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
 #define EXTERIOR
@@ -333,6 +493,15 @@ WilsonKernels<ZWilsonImplD>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & l
 						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
+template<> void 
+WilsonKernels<WilsonImplDF>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZWilsonImplDF>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+						int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
 #undef MAYBEPERM
 #undef MULT_2SPIN
 #define MAYBEPERM(A,B) 
@@ -354,6 +523,15 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSite(StencilImpl &st,LebesgueOrder
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -366,6 +544,15 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrd
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
 #define EXTERIOR
@@ -380,6 +567,15 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrd
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
+template<> void 
+WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,
+							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
 /////////////////////////////////////////////////////////////////
 // Ls vectorised, dag Kernel, single
 /////////////////////////////////////////////////////////////////
@@ -396,6 +592,15 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrd
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #define INTERIOR
 #undef EXTERIOR
@@ -408,6 +613,15 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteDagInt(StencilImpl &st,Lebesgue
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDagInt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef INTERIOR_AND_EXTERIOR
 #undef INTERIOR
 #define EXTERIOR
@@ -420,6 +634,15 @@ WilsonKernels<ZDomainWallVec5dImplD>::AsmDhopSiteDagExt(StencilImpl &st,Lebesgue
 							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 
+template<> void 
+WilsonKernels<DomainWallVec5dImplDF>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+template<> void 
+WilsonKernels<ZDomainWallVec5dImplDF>::AsmDhopSiteDagExt(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+
 #undef COMPLEX_SIGNS
 #undef MAYBEPERM
 #undef MULT_2SPIN

lib/qcd/action/fermion/WilsonKernelsAsmBody.h

@@ -39,24 +39,26 @@
 ////////////////////////////////////////////////////////////////////////////////
 #ifdef INTERIOR_AND_EXTERIOR
 
-#define ZERO_NMU(A) 
+#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-#define INTERIOR_BLOCK_XP(a,b,PERMUTE_DIR,PROJMEM,RECON) INTERIOR_BLOCK(a,b,PERMUTE_DIR,PROJMEM,RECON)
+      basep = st.GetPFInfo(nent,plocal); nent++;			\
-#define EXTERIOR_BLOCK_XP(a,b,RECON) EXTERIOR_BLOCK(a,b,RECON)
+      if ( local ) {							\
+	LOAD64(%r10,isigns);						\
+	PROJ(base);							\
+	MAYBEPERM(PERMUTE_DIR,perm);					\
+      } else {								\
+	LOAD_CHI(base);							\
+      }									\
+      base = st.GetInfo(ptype,local,perm,NxtDir,ent,plocal); ent++;	\
+      PREFETCH_CHIMU(base);						\
+      MULT_2SPIN_DIR_PF(Dir,basep);					\
+      LOAD64(%r10,isigns);						\
+      RECON;								\
 
-#define INTERIOR_BLOCK(a,b,PERMUTE_DIR,PROJMEM,RECON)	\
+#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-  LOAD64(%r10,isigns);                                  \
+  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++;		\
-  PROJMEM(base);                                        \
+  PF_GAUGE(Xp);								\
-  MAYBEPERM(PERMUTE_DIR,perm);                                  
+  PREFETCH1_CHIMU(base);						\
-
+  ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) 
-#define EXTERIOR_BLOCK(a,b,RECON)             \
-  LOAD_CHI(base);
-
-#define COMMON_BLOCK(a,b,RECON)               \
-  base = st.GetInfo(ptype,local,perm,b,ent,plocal); ent++;     \
-  PREFETCH_CHIMU(base);                                         \
-  MULT_2SPIN_DIR_PF(a,basep);					\
-  LOAD64(%r10,isigns);                                          \
-  RECON;                                                        
 
 #define RESULT(base,basep) SAVE_RESULT(base,basep);
 
@@ -67,62 +69,62 @@
 ////////////////////////////////////////////////////////////////////////////////
 #ifdef INTERIOR
 
-#define COMMON_BLOCK(a,b,RECON)       
+#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-#define ZERO_NMU(A) 
+      basep = st.GetPFInfo(nent,plocal); nent++;			\
+      if ( local ) {							\
+	LOAD64(%r10,isigns);						\
+	PROJ(base);							\
+	MAYBEPERM(PERMUTE_DIR,perm);					\
+      }else if ( st.same_node[Dir] ) {LOAD_CHI(base);}			\
+      if ( local || st.same_node[Dir] ) {				\
+	MULT_2SPIN_DIR_PF(Dir,basep);					\
+	LOAD64(%r10,isigns);						\
+	RECON;								\
+      }									\
+      base = st.GetInfo(ptype,local,perm,NxtDir,ent,plocal); ent++;	\
+      PREFETCH_CHIMU(base);						\
 
-// No accumulate for DIR0
+#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-#define EXTERIOR_BLOCK_XP(a,b,RECON)				\
+  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++;		\
-  ZERO_PSI;							\
+  PF_GAUGE(Xp);								\
-  base = st.GetInfo(ptype,local,perm,b,ent,plocal); ent++;	
+  PREFETCH1_CHIMU(base);						\
-
+  { ZERO_PSI; }								\
-#define EXTERIOR_BLOCK(a,b,RECON)  \
+  ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON) 
-  base = st.GetInfo(ptype,local,perm,b,ent,plocal); ent++;     
-
-#define INTERIOR_BLOCK_XP(a,b,PERMUTE_DIR,PROJMEM,RECON) INTERIOR_BLOCK(a,b,PERMUTE_DIR,PROJMEM,RECON)
-
-#define INTERIOR_BLOCK(a,b,PERMUTE_DIR,PROJMEM,RECON)		\
-  LOAD64(%r10,isigns);						\
-  PROJMEM(base);                                                \
-  MAYBEPERM(PERMUTE_DIR,perm);                                  \
-  base = st.GetInfo(ptype,local,perm,b,ent,plocal); ent++;	\
-  PREFETCH_CHIMU(base);						\
-  MULT_2SPIN_DIR_PF(a,basep);					\
-  LOAD64(%r10,isigns);                                          \
-  RECON;                                                        
 
 #define RESULT(base,basep) SAVE_RESULT(base,basep);
 
 #endif
-
 ////////////////////////////////////////////////////////////////////////////////
 // Post comms kernel
 ////////////////////////////////////////////////////////////////////////////////
 #ifdef EXTERIOR
 
-#define ZERO_NMU(A) nmu=0;
 
-#define INTERIOR_BLOCK_XP(a,b,PERMUTE_DIR,PROJMEM,RECON) \
+#define ASM_LEG(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
-  ZERO_PSI;   base = st.GetInfo(ptype,local,perm,b,ent,plocal); ent++;		
+  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++;		\
+  if((!local)&&(!st.same_node[Dir]) ) {					\
+    LOAD_CHI(base);							\
+    MULT_2SPIN_DIR_PF(Dir,base);					\
+    LOAD64(%r10,isigns);						\
+    RECON;								\
+    nmu++;								\
+  }									
 
-#define EXTERIOR_BLOCK_XP(a,b,RECON) EXTERIOR_BLOCK(a,b,RECON)
+#define ASM_LEG_XP(Dir,NxtDir,PERMUTE_DIR,PROJ,RECON)			\
+  nmu=0;								\
+  { ZERO_PSI;}								\
+  base = st.GetInfo(ptype,local,perm,Dir,ent,plocal); ent++;		\
+  if((!local)&&(!st.same_node[Dir]) ) {					\
+    LOAD_CHI(base);							\
+    MULT_2SPIN_DIR_PF(Dir,base);					\
+    LOAD64(%r10,isigns);						\
+    RECON;								\
+    nmu++;								\
+  }
 
-#define INTERIOR_BLOCK(a,b,PERMUTE_DIR,PROJMEM,RECON)			\
+#define RESULT(base,basep) if (nmu){ ADD_RESULT(base,base);}
-  base = st.GetInfo(ptype,local,perm,b,ent,plocal); ent++;		
-
-#define EXTERIOR_BLOCK(a,b,RECON)				\
-    nmu++;							\
-    LOAD_CHI(base);						\
-    MULT_2SPIN_DIR_PF(a,base);					\
-    base = st.GetInfo(ptype,local,perm,b,ent,plocal); ent++;	\
-    LOAD64(%r10,isigns);					\
-    RECON;                                                        
-
-#define COMMON_BLOCK(a,b,RECON)			
-
-#define RESULT(base,basep) if (nmu){  ADD_RESULT(base,base);}
 
 #endif
-
 {
   int nmu;
   int local,perm, ptype;
@@ -134,11 +136,15 @@
   MASK_REGS;
   int nmax=U._grid->oSites();
   for(int site=0;site<Ns;site++) {
+#ifndef EXTERIOR
     int sU =lo.Reorder(ssU);
     int ssn=ssU+1;     if(ssn>=nmax) ssn=0;
     int sUn=lo.Reorder(ssn);
-#ifndef EXTERIOR
     LOCK_GAUGE(0);
+#else
+    int sU =ssU;
+    int ssn=ssU+1;     if(ssn>=nmax) ssn=0;
+    int sUn=ssn;
 #endif
     for(int s=0;s<Ls;s++) {
       ss =sU*Ls+s;
@@ -146,93 +152,20 @@
       int  ent=ss*8;// 2*Ndim
       int nent=ssn*8;
 
-      ZERO_NMU(0);
+   ASM_LEG_XP(Xp,Yp,PERMUTE_DIR3,DIR0_PROJMEM,DIR0_RECON);
-      base  = st.GetInfo(ptype,local,perm,Xp,ent,plocal); ent++;
+      ASM_LEG(Yp,Zp,PERMUTE_DIR2,DIR1_PROJMEM,DIR1_RECON);
-#ifndef EXTERIOR
+      ASM_LEG(Zp,Tp,PERMUTE_DIR1,DIR2_PROJMEM,DIR2_RECON);
-      PF_GAUGE(Xp); 
+      ASM_LEG(Tp,Xm,PERMUTE_DIR0,DIR3_PROJMEM,DIR3_RECON);
-      PREFETCH1_CHIMU(base);
-#endif
-      ////////////////////////////////
-      // Xp
-      ////////////////////////////////
-      basep = st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK_XP(Xp,Yp,PERMUTE_DIR3,DIR0_PROJMEM,DIR0_RECON);
-      } else { 
-	EXTERIOR_BLOCK_XP(Xp,Yp,DIR0_RECON);
-      }
-      COMMON_BLOCK(Xp,Yp,DIR0_RECON);
-      ////////////////////////////////
-      // Yp
-      ////////////////////////////////
-      basep = st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK(Yp,Zp,PERMUTE_DIR2,DIR1_PROJMEM,DIR1_RECON);
-      } else { 
-	EXTERIOR_BLOCK(Yp,Zp,DIR1_RECON);
-      }
-      COMMON_BLOCK(Yp,Zp,DIR1_RECON);
-      ////////////////////////////////
-      // Zp
-      ////////////////////////////////
-      basep = st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK(Zp,Tp,PERMUTE_DIR1,DIR2_PROJMEM,DIR2_RECON);
-      } else { 
-	EXTERIOR_BLOCK(Zp,Tp,DIR2_RECON);
-      }
-      COMMON_BLOCK(Zp,Tp,DIR2_RECON);
-      ////////////////////////////////
-      // Tp
-      ////////////////////////////////
-      basep = st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK(Tp,Xm,PERMUTE_DIR0,DIR3_PROJMEM,DIR3_RECON);
-      } else { 
-	EXTERIOR_BLOCK(Tp,Xm,DIR3_RECON);
-      }
-      COMMON_BLOCK(Tp,Xm,DIR3_RECON);
-      ////////////////////////////////
-      // Xm
-      ////////////////////////////////
-      //  basep= st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK(Xm,Ym,PERMUTE_DIR3,DIR4_PROJMEM,DIR4_RECON);
-      } else { 
-	EXTERIOR_BLOCK(Xm,Ym,DIR4_RECON);
-      }
-      COMMON_BLOCK(Xm,Ym,DIR4_RECON);
-      ////////////////////////////////
-      // Ym
-      ////////////////////////////////
-      basep= st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK(Ym,Zm,PERMUTE_DIR2,DIR5_PROJMEM,DIR5_RECON);
-      } else { 
-	EXTERIOR_BLOCK(Ym,Zm,DIR5_RECON);
-      }
-      COMMON_BLOCK(Ym,Zm,DIR5_RECON);
-      ////////////////////////////////
-      // Zm
-      ////////////////////////////////
-      basep= st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK(Zm,Tm,PERMUTE_DIR1,DIR6_PROJMEM,DIR6_RECON);
-      } else { 
-	EXTERIOR_BLOCK(Zm,Tm,DIR6_RECON);
-      }
-      COMMON_BLOCK(Zm,Tm,DIR6_RECON);
-      ////////////////////////////////
-      // Tm
-      ////////////////////////////////
-      basep= st.GetPFInfo(nent,plocal); nent++;
-      if ( local ) {
-	INTERIOR_BLOCK(Tm,Xp,PERMUTE_DIR0,DIR7_PROJMEM,DIR7_RECON);
-      } else { 
-	EXTERIOR_BLOCK(Tm,Xp,DIR7_RECON);
-      }
-      COMMON_BLOCK(Tm,Xp,DIR7_RECON);
 
+      ASM_LEG(Xm,Ym,PERMUTE_DIR3,DIR4_PROJMEM,DIR4_RECON);
+      ASM_LEG(Ym,Zm,PERMUTE_DIR2,DIR5_PROJMEM,DIR5_RECON);
+      ASM_LEG(Zm,Tm,PERMUTE_DIR1,DIR6_PROJMEM,DIR6_RECON);
+      ASM_LEG(Tm,Xp,PERMUTE_DIR0,DIR7_PROJMEM,DIR7_RECON);
+
+#ifdef EXTERIOR
+      if (nmu==0) break;
+      //      if (nmu!=0) std::cout << "EXT "<<sU<<std::endl;
+#endif
       base = (uint64_t) &out._odata[ss];
       basep= st.GetPFInfo(nent,plocal); nent++;
       RESULT(base,basep);
@@ -258,10 +191,6 @@
 #undef DIR5_RECON
 #undef DIR6_RECON
 #undef DIR7_RECON
-#undef EXTERIOR_BLOCK
+#undef ASM_LEG
-#undef INTERIOR_BLOCK
+#undef ASM_LEG_XP
-#undef EXTERIOR_BLOCK_XP
-#undef INTERIOR_BLOCK_XP
-#undef COMMON_BLOCK
-#undef ZERO_NMU
 #undef RESULT

lib/qcd/action/fermion/WilsonKernelsHand.cc

@@ -31,7 +31,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define REGISTER
 
 #define LOAD_CHIMU \
-  const SiteSpinor & ref (in._odata[offset]);	\
+  {const SiteSpinor & ref (in._odata[offset]);	\
     Chimu_00=ref()(0)(0);\
     Chimu_01=ref()(0)(1);\
     Chimu_02=ref()(0)(2);\
@@ -43,20 +43,20 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     Chimu_22=ref()(2)(2);\
     Chimu_30=ref()(3)(0);\
     Chimu_31=ref()(3)(1);\
-    Chimu_32=ref()(3)(2);
+    Chimu_32=ref()(3)(2);}
 
 #define LOAD_CHI\
-  const SiteHalfSpinor &ref(buf[offset]);	\
+  {const SiteHalfSpinor &ref(buf[offset]);	\
     Chi_00 = ref()(0)(0);\
     Chi_01 = ref()(0)(1);\
     Chi_02 = ref()(0)(2);\
     Chi_10 = ref()(1)(0);\
     Chi_11 = ref()(1)(1);\
-    Chi_12 = ref()(1)(2);
+    Chi_12 = ref()(1)(2);}
 
 // To splat or not to splat depends on the implementation
 #define MULT_2SPIN(A)\
-   auto & ref(U._odata[sU](A));	\
+  {auto & ref(U._odata[sU](A));			\
    Impl::loadLinkElement(U_00,ref()(0,0));	\
    Impl::loadLinkElement(U_10,ref()(1,0));	\
    Impl::loadLinkElement(U_20,ref()(2,0));	\
@@ -83,7 +83,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     UChi_01+= U_10*Chi_02;\
     UChi_11+= U_10*Chi_12;\
     UChi_02+= U_20*Chi_02;\
-    UChi_12+= U_20*Chi_12;
+    UChi_12+= U_20*Chi_12;}
 
 
 #define PERMUTE_DIR(dir)			\
@@ -307,55 +307,132 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   result_31-= UChi_11;	\
   result_32-= UChi_12;
 
-namespace Grid {
+#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON)	\
-namespace QCD {
+  SE=st.GetEntry(ptype,DIR,ss);			\
+  offset = SE->_offset;				\
+  local  = SE->_is_local;			\
+  perm   = SE->_permute;			\
+  if ( local ) {				\
+    LOAD_CHIMU;					\
+    PROJ;					\
+    if ( perm) {				\
+      PERMUTE_DIR(PERM);			\
+    }						\
+  } else {					\
+    LOAD_CHI;					\
+  }						\
+  MULT_2SPIN(DIR);				\
+  RECON;					
+
+#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON)	\
+  SE=st.GetEntry(ptype,DIR,ss);			\
+  offset = SE->_offset;				\
+  local  = SE->_is_local;			\
+  perm   = SE->_permute;			\
+  if ( local ) {				\
+    LOAD_CHIMU;					\
+    PROJ;					\
+    if ( perm) {				\
+      PERMUTE_DIR(PERM);			\
+    }						\
+  } else if ( st.same_node[DIR] ) {		\
+    LOAD_CHI;					\
+  }						\
+  if (local || st.same_node[DIR] ) {		\
+    MULT_2SPIN(DIR);				\
+    RECON;					\
+  }
+
+#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON)	\
+  SE=st.GetEntry(ptype,DIR,ss);			\
+  offset = SE->_offset;				\
+  if((!SE->_is_local)&&(!st.same_node[DIR]) ) {	\
+    LOAD_CHI;					\
+    MULT_2SPIN(DIR);				\
+    RECON;					\
+    nmu++;					\
+  }
+
+#define HAND_RESULT(ss)				\
+  {						\
+    SiteSpinor & ref (out._odata[ss]);		\
+    vstream(ref()(0)(0),result_00);		\
+    vstream(ref()(0)(1),result_01);		\
+    vstream(ref()(0)(2),result_02);		\
+    vstream(ref()(1)(0),result_10);		\
+    vstream(ref()(1)(1),result_11);		\
+    vstream(ref()(1)(2),result_12);		\
+    vstream(ref()(2)(0),result_20);		\
+    vstream(ref()(2)(1),result_21);		\
+    vstream(ref()(2)(2),result_22);		\
+    vstream(ref()(3)(0),result_30);		\
+    vstream(ref()(3)(1),result_31);		\
+    vstream(ref()(3)(2),result_32);		\
+  }
+
+#define HAND_RESULT_EXT(ss)			\
+  if (nmu){					\
+    SiteSpinor & ref (out._odata[ss]);		\
+    ref()(0)(0)+=result_00;		\
+    ref()(0)(1)+=result_01;		\
+    ref()(0)(2)+=result_02;		\
+    ref()(1)(0)+=result_10;		\
+    ref()(1)(1)+=result_11;		\
+    ref()(1)(2)+=result_12;		\
+    ref()(2)(0)+=result_20;		\
+    ref()(2)(1)+=result_21;		\
+    ref()(2)(2)+=result_22;		\
+    ref()(3)(0)+=result_30;		\
+    ref()(3)(1)+=result_31;		\
+    ref()(3)(2)+=result_32;		\
+  }
 
 
-template<class Impl> void 
+#define HAND_DECLARATIONS(a)			\
-WilsonKernels<Impl>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
+  Simd result_00;				\
-					  int ss,int sU,const FermionField &in, FermionField &out,int interior,int exterior)
+  Simd result_01;				\
-{
+  Simd result_02;				\
-  typedef typename Simd::scalar_type S;
+  Simd result_10;				\
-  typedef typename Simd::vector_type V;
+  Simd result_11;				\
+  Simd result_12;				\
+  Simd result_20;				\
+  Simd result_21;				\
+  Simd result_22;				\
+  Simd result_30;				\
+  Simd result_31;				\
+  Simd result_32;				\
+  Simd Chi_00;					\
+  Simd Chi_01;					\
+  Simd Chi_02;					\
+  Simd Chi_10;					\
+  Simd Chi_11;					\
+  Simd Chi_12;					\
+  Simd UChi_00;					\
+  Simd UChi_01;					\
+  Simd UChi_02;					\
+  Simd UChi_10;					\
+  Simd UChi_11;					\
+  Simd UChi_12;					\
+  Simd U_00;					\
+  Simd U_10;					\
+  Simd U_20;					\
+  Simd U_01;					\
+  Simd U_11;					\
+  Simd U_21;
 
-  REGISTER Simd result_00; // 12 regs on knc
+#define ZERO_RESULT				\
-  REGISTER Simd result_01;
+  result_00=zero;				\
-  REGISTER Simd result_02;
+  result_01=zero;				\
-
+  result_02=zero;				\
-  REGISTER Simd result_10;
+  result_10=zero;				\
-  REGISTER Simd result_11;
+  result_11=zero;				\
-  REGISTER Simd result_12;
+  result_12=zero;				\
-
+  result_20=zero;				\
-  REGISTER Simd result_20;
+  result_21=zero;				\
-  REGISTER Simd result_21;
+  result_22=zero;				\
-  REGISTER Simd result_22;
+  result_30=zero;				\
-
+  result_31=zero;				\
-  REGISTER Simd result_30;
+  result_32=zero;			
-  REGISTER Simd result_31;
-  REGISTER Simd result_32; // 20 left
-
-  REGISTER Simd Chi_00;    // two spinor; 6 regs
-  REGISTER Simd Chi_01;
-  REGISTER Simd Chi_02;
-
-  REGISTER Simd Chi_10;
-  REGISTER Simd Chi_11;
-  REGISTER Simd Chi_12;   // 14 left
-
-  REGISTER Simd UChi_00;  // two spinor; 6 regs
-  REGISTER Simd UChi_01;
-  REGISTER Simd UChi_02;
-
-  REGISTER Simd UChi_10;
-  REGISTER Simd UChi_11;
-  REGISTER Simd UChi_12;  // 8 left
-
-  REGISTER Simd U_00;  // two rows of U matrix
-  REGISTER Simd U_10;
-  REGISTER Simd U_20;  
-  REGISTER Simd U_01;
-  REGISTER Simd U_11;
-  REGISTER Simd U_21;  // 2 reg left.
 
 #define Chimu_00 Chi_00
 #define Chimu_01 Chi_01
@@ -370,475 +447,225 @@ WilsonKernels<Impl>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGauge
 #define Chimu_31 UChi_11
 #define Chimu_32 UChi_12
 
+namespace Grid {
+namespace QCD {
+
+template<class Impl> void 
+WilsonKernels<Impl>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
+					  int ss,int sU,const FermionField &in, FermionField &out)
+{
+// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+
+  HAND_DECLARATIONS(ignore);
 
   int offset,local,perm, ptype;
   StencilEntry *SE;
 
-  // Xp
+  HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
-  SE=st.GetEntry(ptype,Xp,ss);
+  HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  offset = SE->_offset;
+  HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-  local  = SE->_is_local;
+  HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  perm   = SE->_permute;
+  HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  
+  HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-  if ( local ) {
+  HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-    LOAD_CHIMU;
+  HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-    XM_PROJ;
+  HAND_RESULT(ss);
-    if ( perm) {
-      PERMUTE_DIR(3); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Xp);
-  }
-  XM_RECON;
-  
-  // Yp
-  SE=st.GetEntry(ptype,Yp,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    YM_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(2); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Yp);
-  }
-  YM_RECON_ACCUM;
-
-
-  // Zp
-  SE=st.GetEntry(ptype,Zp,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    ZM_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(1); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Zp);
-  }
-  ZM_RECON_ACCUM;
-
-  // Tp
-  SE=st.GetEntry(ptype,Tp,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    TM_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(0); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Tp);
-  }
-  TM_RECON_ACCUM;
-  
-  // Xm
-  SE=st.GetEntry(ptype,Xm,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    XP_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(3); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Xm);
-  }
-  XP_RECON_ACCUM;
-  
-  
-  // Ym
-  SE=st.GetEntry(ptype,Ym,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    YP_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(2); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Ym);
-  }
-  YP_RECON_ACCUM;
-
-  // Zm
-  SE=st.GetEntry(ptype,Zm,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    ZP_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(1); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Zm);
-  }
-  ZP_RECON_ACCUM;
-
-  // Tm
-  SE=st.GetEntry(ptype,Tm,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    TP_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(0); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Tm);
-  }
-  TP_RECON_ACCUM;
-
-  {
-    SiteSpinor & ref (out._odata[ss]);
-    vstream(ref()(0)(0),result_00);
-    vstream(ref()(0)(1),result_01);
-    vstream(ref()(0)(2),result_02);
-    vstream(ref()(1)(0),result_10);
-    vstream(ref()(1)(1),result_11);
-    vstream(ref()(1)(2),result_12);
-    vstream(ref()(2)(0),result_20);
-    vstream(ref()(2)(1),result_21);
-    vstream(ref()(2)(2),result_22);
-    vstream(ref()(3)(0),result_30);
-    vstream(ref()(3)(1),result_31);
-    vstream(ref()(3)(2),result_32);
-  }
 }
 
 template<class Impl>
 void WilsonKernels<Impl>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-						  int ss,int sU,const FermionField &in, FermionField &out,int interior,int exterior)
+						  int ss,int sU,const FermionField &in, FermionField &out)
 {
-  //  std::cout << "Hand op Dhop "<<std::endl;
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
 
-  REGISTER Simd result_00; // 12 regs on knc
+  HAND_DECLARATIONS(ignore);
-  REGISTER Simd result_01;
-  REGISTER Simd result_02;
-  
-  REGISTER Simd result_10;
-  REGISTER Simd result_11;
-  REGISTER Simd result_12;
-
-  REGISTER Simd result_20;
-  REGISTER Simd result_21;
-  REGISTER Simd result_22;
-
-  REGISTER Simd result_30;
-  REGISTER Simd result_31;
-  REGISTER Simd result_32; // 20 left
-
-  REGISTER Simd Chi_00;    // two spinor; 6 regs
-  REGISTER Simd Chi_01;
-  REGISTER Simd Chi_02;
-
-  REGISTER Simd Chi_10;
-  REGISTER Simd Chi_11;
-  REGISTER Simd Chi_12;   // 14 left
-
-  REGISTER Simd UChi_00;  // two spinor; 6 regs
-  REGISTER Simd UChi_01;
-  REGISTER Simd UChi_02;
-
-  REGISTER Simd UChi_10;
-  REGISTER Simd UChi_11;
-  REGISTER Simd UChi_12;  // 8 left
-
-  REGISTER Simd U_00;  // two rows of U matrix
-  REGISTER Simd U_10;
-  REGISTER Simd U_20;  
-  REGISTER Simd U_01;
-  REGISTER Simd U_11;
-  REGISTER Simd U_21;  // 2 reg left.
-
-#define Chimu_00 Chi_00
-#define Chimu_01 Chi_01
-#define Chimu_02 Chi_02
-#define Chimu_10 Chi_10
-#define Chimu_11 Chi_11
-#define Chimu_12 Chi_12
-#define Chimu_20 UChi_00
-#define Chimu_21 UChi_01
-#define Chimu_22 UChi_02
-#define Chimu_30 UChi_10
-#define Chimu_31 UChi_11
-#define Chimu_32 UChi_12
-
 
   StencilEntry *SE;
   int offset,local,perm, ptype;
   
-  // Xp
+  HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON);
-  SE=st.GetEntry(ptype,Xp,ss);
+  HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM);
-  offset = SE->_offset;
+  HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
-  local  = SE->_is_local;
+  HAND_STENCIL_LEG(TP_PROJ,0,Tp,TP_RECON_ACCUM);
-  perm   = SE->_permute;
+  HAND_STENCIL_LEG(XM_PROJ,3,Xm,XM_RECON_ACCUM);
-  
+  HAND_STENCIL_LEG(YM_PROJ,2,Ym,YM_RECON_ACCUM);
-  if ( local ) {
+  HAND_STENCIL_LEG(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
-    LOAD_CHIMU;
+  HAND_STENCIL_LEG(TM_PROJ,0,Tm,TM_RECON_ACCUM);
-    XP_PROJ;
+  HAND_RESULT(ss);
-    if ( perm) {
+}
-      PERMUTE_DIR(3); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
 
-  {
+template<class Impl> void 
-    MULT_2SPIN(Xp);
+WilsonKernels<Impl>::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
-  }
+					  int ss,int sU,const FermionField &in, FermionField &out)
-  XP_RECON;
+{
+// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
 
-  // Yp
+  HAND_DECLARATIONS(ignore);
-  SE=st.GetEntry(ptype,Yp,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    YP_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(2); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Yp);
-  }
-  YP_RECON_ACCUM;
 
+  int offset,local,perm, ptype;
+  StencilEntry *SE;
+  ZERO_RESULT;
+  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
+  HAND_RESULT(ss);
+}
 
-  // Zp
+template<class Impl>
-  SE=st.GetEntry(ptype,Zp,ss);
+void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-  offset = SE->_offset;
+						  int ss,int sU,const FermionField &in, FermionField &out)
-  local  = SE->_is_local;
+{
-  perm   = SE->_permute;
+  typedef typename Simd::scalar_type S;
-  
+  typedef typename Simd::vector_type V;
-  if ( local ) {
-    LOAD_CHIMU;
-    ZP_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(1); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Zp);
-  }
-  ZP_RECON_ACCUM;
 
-  // Tp
+  HAND_DECLARATIONS(ignore);
-  SE=st.GetEntry(ptype,Tp,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    TP_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(0); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Tp);
-  }
-  TP_RECON_ACCUM;
-  
-  // Xm
-  SE=st.GetEntry(ptype,Xm,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    XM_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(3); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Xm);
-  }
-  XM_RECON_ACCUM;
-  
-  // Ym
-  SE=st.GetEntry(ptype,Ym,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
-  
-  if ( local ) {
-    LOAD_CHIMU;
-    YM_PROJ;
-    if ( perm) {
-      PERMUTE_DIR(2); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
-  } else { 
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Ym);
-  }
-  YM_RECON_ACCUM;
 
-  // Zm
+  StencilEntry *SE;
-  SE=st.GetEntry(ptype,Zm,ss);
+  int offset,local,perm, ptype;
-  offset = SE->_offset;
+  ZERO_RESULT;
-  local  = SE->_is_local;
+  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
-  perm   = SE->_permute;
+  HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
+  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
+  HAND_RESULT(ss);
+}
 
-  if ( local ) {
+template<class Impl> void 
-    LOAD_CHIMU;
+WilsonKernels<Impl>::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
-    ZM_PROJ;
+					  int ss,int sU,const FermionField &in, FermionField &out)
-    if ( perm) {
+{
-      PERMUTE_DIR(1); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-    }
+  typedef typename Simd::scalar_type S;
-  } else { 
+  typedef typename Simd::vector_type V;
-    LOAD_CHI;
-  }
-  {
-    MULT_2SPIN(Zm);
-  }
-  ZM_RECON_ACCUM;
 
-  // Tm
+  HAND_DECLARATIONS(ignore);
-  SE=st.GetEntry(ptype,Tm,ss);
-  offset = SE->_offset;
-  local  = SE->_is_local;
-  perm   = SE->_permute;
 
-  if ( local ) {
+  int offset,local,perm, ptype;
-    LOAD_CHIMU;
+  StencilEntry *SE;
-    TM_PROJ;
+  int nmu=0;
-    if ( perm) {
+  ZERO_RESULT;
-      PERMUTE_DIR(0); // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
-    }
+  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  } else { 
+  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-    LOAD_CHI;
+  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  }
+  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  {
+  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-    MULT_2SPIN(Tm);
+  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-  }
+  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-  TM_RECON_ACCUM;
+  HAND_RESULT_EXT(ss);
+}
 
-  {
+template<class Impl>
-    SiteSpinor & ref (out._odata[ss]);
+void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-    vstream(ref()(0)(0),result_00);
+						  int ss,int sU,const FermionField &in, FermionField &out)
-    vstream(ref()(0)(1),result_01);
+{
-    vstream(ref()(0)(2),result_02);
+  typedef typename Simd::scalar_type S;
-    vstream(ref()(1)(0),result_10);
+  typedef typename Simd::vector_type V;
-    vstream(ref()(1)(1),result_11);
+
-    vstream(ref()(1)(2),result_12);
+  HAND_DECLARATIONS(ignore);
-    vstream(ref()(2)(0),result_20);
+
-    vstream(ref()(2)(1),result_21);
+  StencilEntry *SE;
-    vstream(ref()(2)(2),result_22);
+  int offset,local,perm, ptype;
-    vstream(ref()(3)(0),result_30);
+  int nmu=0;
-    vstream(ref()(3)(1),result_31);
+  ZERO_RESULT;
-    vstream(ref()(3)(2),result_32);
+  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
-  }
+  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
+  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
+  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
+  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
+  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
+  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
+  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
+  HAND_RESULT_EXT(ss);
 }
 
   ////////////////////////////////////////////////
   // Specialise Gparity to simple implementation
   ////////////////////////////////////////////////
-template<> void 
+#define HAND_SPECIALISE_EMPTY(IMPL)					\
-WilsonKernels<GparityWilsonImplF>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+  template<> void							\
-							SiteHalfSpinor *buf,
+  WilsonKernels<IMPL>::HandDhopSite(StencilImpl &st,			\
-							int sF,int sU,const FermionField &in, FermionField &out,int internal,int external)
+				    LebesgueOrder &lo,			\
-{
+				    DoubledGaugeField &U,		\
-  assert(0);
+				    SiteHalfSpinor *buf,		\
-}
+				    int sF,int sU,			\
-
+				    const FermionField &in,		\
-template<> void 
+				    FermionField &out){ assert(0); }	\
-WilsonKernels<GparityWilsonImplF>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+  template<> void							\
-							   SiteHalfSpinor *buf,
+  WilsonKernels<IMPL>::HandDhopSiteDag(StencilImpl &st,			\
-							   int sF,int sU,const FermionField &in, FermionField &out,int internal,int external)
+				    LebesgueOrder &lo,			\
-{
+				    DoubledGaugeField &U,		\
-  assert(0);
+				    SiteHalfSpinor *buf,		\
-}
+				    int sF,int sU,			\
-
+				    const FermionField &in,		\
-template<> void 
+				    FermionField &out){ assert(0); }	\
-WilsonKernels<GparityWilsonImplD>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+  template<> void							\
-							int sF,int sU,const FermionField &in, FermionField &out,int internal,int external)
+  WilsonKernels<IMPL>::HandDhopSiteInt(StencilImpl &st,			\
-{
+				    LebesgueOrder &lo,			\
-  assert(0);
+				    DoubledGaugeField &U,		\
-}
+				    SiteHalfSpinor *buf,		\
-
+				    int sF,int sU,			\
-template<> void 
+				    const FermionField &in,		\
-WilsonKernels<GparityWilsonImplD>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+				    FermionField &out){ assert(0); }	\
-							   int sF,int sU,const FermionField &in, FermionField &out,int internal,int external)
+  template<> void							\
-{
+  WilsonKernels<IMPL>::HandDhopSiteExt(StencilImpl &st,			\
-  assert(0);
+				    LebesgueOrder &lo,			\
-}
+				    DoubledGaugeField &U,		\
-
+				    SiteHalfSpinor *buf,		\
+				    int sF,int sU,			\
+				    const FermionField &in,		\
+				    FermionField &out){ assert(0); }	\
+  template<> void							\
+  WilsonKernels<IMPL>::HandDhopSiteDagInt(StencilImpl &st,	       	\
+				    LebesgueOrder &lo,			\
+				    DoubledGaugeField &U,		\
+				    SiteHalfSpinor *buf,		\
+				    int sF,int sU,			\
+				    const FermionField &in,		\
+				    FermionField &out){ assert(0); }	\
+  template<> void							\
+  WilsonKernels<IMPL>::HandDhopSiteDagExt(StencilImpl &st,	       	\
+				    LebesgueOrder &lo,			\
+				    DoubledGaugeField &U,		\
+				    SiteHalfSpinor *buf,		\
+				    int sF,int sU,			\
+				    const FermionField &in,		\
+				    FermionField &out){ assert(0); }	\
 
+  HAND_SPECIALISE_EMPTY(GparityWilsonImplF);
+  HAND_SPECIALISE_EMPTY(GparityWilsonImplD);
+  HAND_SPECIALISE_EMPTY(GparityWilsonImplFH);
+  HAND_SPECIALISE_EMPTY(GparityWilsonImplDF);
 
 ////////////// Wilson ; uses this implementation /////////////////////
-// Need Nc=3 though //
 
 #define INSTANTIATE_THEM(A) \
 template void WilsonKernels<A>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
-						     int ss,int sU,const FermionField &in, FermionField &out,int interior,int exterior); \
+					     int ss,int sU,const FermionField &in, FermionField &out); \
-template void WilsonKernels<A>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
+template void WilsonKernels<A>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \
-							int ss,int sU,const FermionField &in, FermionField &out,int interior,int exterior);
+						int ss,int sU,const FermionField &in, FermionField &out);\
+template void WilsonKernels<A>::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
+						int ss,int sU,const FermionField &in, FermionField &out); \
+template void WilsonKernels<A>::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \
+						   int ss,int sU,const FermionField &in, FermionField &out); \
+template void WilsonKernels<A>::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\
+						int ss,int sU,const FermionField &in, FermionField &out); \
+template void WilsonKernels<A>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \
+						   int ss,int sU,const FermionField &in, FermionField &out); 
 
 INSTANTIATE_THEM(WilsonImplF);
 INSTANTIATE_THEM(WilsonImplD);
@@ -850,5 +677,15 @@ INSTANTIATE_THEM(DomainWallVec5dImplF);
 INSTANTIATE_THEM(DomainWallVec5dImplD);
 INSTANTIATE_THEM(ZDomainWallVec5dImplF);
 INSTANTIATE_THEM(ZDomainWallVec5dImplD);
+INSTANTIATE_THEM(WilsonImplFH);
+INSTANTIATE_THEM(WilsonImplDF);
+INSTANTIATE_THEM(ZWilsonImplFH);
+INSTANTIATE_THEM(ZWilsonImplDF);
+INSTANTIATE_THEM(GparityWilsonImplFH);
+INSTANTIATE_THEM(GparityWilsonImplDF);
+INSTANTIATE_THEM(DomainWallVec5dImplFH);
+INSTANTIATE_THEM(DomainWallVec5dImplDF);
+INSTANTIATE_THEM(ZDomainWallVec5dImplFH);
+INSTANTIATE_THEM(ZDomainWallVec5dImplDF);
 
 }}

lib/qcd/hmc/GenericHMCrunner.h

@@ -56,6 +56,7 @@ class HMCWrapperTemplate: public HMCRunnerBase<ReaderClass> {
   using IntegratorType = Integrator<Implementation, S, RepresentationsPolicy>;
 
   HMCparameters Parameters;
+  std::string ParameterFile;
   HMCResourceManager<Implementation> Resources;
 
   // The set of actions (keep here for lower level users, for now)
@@ -114,6 +115,10 @@ class HMCWrapperTemplate: public HMCRunnerBase<ReaderClass> {
       GridCmdOptionIntVector(arg, ivec);
       Parameters.NoMetropolisUntil = ivec[0];
     }
+    if (GridCmdOptionExists(argv, argv + argc, "--ParameterFile")) {
+      arg = GridCmdOptionPayload(argv, argv + argc, "--ParameterFile");
+      ParameterFile = arg;
+    }
   }
 
 

lib/qcd/observables/topological_charge.h

@@ -49,24 +49,7 @@ class TopologicalCharge : public HmcObservable<typename Impl::Field> {
                             GridSerialRNG &sRNG,
                             GridParallelRNG &pRNG) {
 
-    // 4d topological charge
+    Real q = WilsonLoops<Impl>::TopologicalCharge(U);
-    // Bx = -iF(y,z), By = -iF(z,y), Bz = -iF(x,y)
-    GaugeLinkField Bx(U._grid), By(U._grid), Bz(U._grid);
-    WilsonLoops<Impl>::FieldStrength(Bx, U, Ydir, Zdir);
-    WilsonLoops<Impl>::FieldStrength(By, U, Zdir, Xdir);
-    WilsonLoops<Impl>::FieldStrength(Bz, U, Xdir, Ydir);
-
-    // Ex = -iF(t,x), Ey = -iF(t,y), Ez = -iF(t,z)
-    GaugeLinkField Ex(U._grid), Ey(U._grid), Ez(U._grid);
-    WilsonLoops<Impl>::FieldStrength(Ex, U, Tdir, Xdir);
-    WilsonLoops<Impl>::FieldStrength(Ey, U, Tdir, Ydir);
-    WilsonLoops<Impl>::FieldStrength(Ez, U, Tdir, Zdir);
-
-    double coeff = 8.0/(32.0*M_PI*M_PI);
-
-    LatticeComplex qfield = coeff*trace(Bx*Ex + By*Ey + Bz*Ez);
-    TComplex Tq = sum(qfield);
-    Real q = TensorRemove(Tq).real();
 
     int def_prec = std::cout.precision();
 

lib/qcd/utils/WilsonLoops.h

@@ -313,6 +313,27 @@ public:
       FS = 0.25*Ta(u*v + Cshift(vu, mu, +1));
   }
 
+  static Real TopologicalCharge(GaugeLorentz &U){
+    // 4d topological charge
+    assert(Nd==4);
+    // Bx = -iF(y,z), By = -iF(z,y), Bz = -iF(x,y)
+    GaugeMat Bx(U._grid), By(U._grid), Bz(U._grid);
+    FieldStrength(Bx, U, Ydir, Zdir);
+    FieldStrength(By, U, Zdir, Xdir);
+    FieldStrength(Bz, U, Xdir, Ydir);
+
+    // Ex = -iF(t,x), Ey = -iF(t,y), Ez = -iF(t,z)
+    GaugeMat Ex(U._grid), Ey(U._grid), Ez(U._grid);
+    FieldStrength(Ex, U, Tdir, Xdir);
+    FieldStrength(Ey, U, Tdir, Ydir);
+    FieldStrength(Ez, U, Tdir, Zdir);
+
+    double coeff = 8.0/(32.0*M_PI*M_PI);
+
+    LatticeComplex qfield = coeff*trace(Bx*Ex + By*Ey + Bz*Ez);
+    TComplex Tq = sum(qfield);
+    return TensorRemove(Tq).real();
+  }
 
   //////////////////////////////////////////////////////
   // Similar to above for rectangle is required

lib/serialisation/BaseIO.h

@@ -138,6 +138,54 @@ namespace Grid {
     unsigned int               dimInd_{0};
   };
   
+  // Pair IO utilities /////////////////////////////////////////////////////////
+  // helper function to parse input in the format "<obj1 obj2>"
+  template <typename T1, typename T2>
+  inline std::istream & operator>>(std::istream &is, std::pair<T1, T2> &buf)
+  {
+    T1 buf1;
+    T2 buf2;
+    char c;
+
+    // Search for "pair" delimiters.
+    do
+    {
+      is.get(c);
+    } while (c != '<' && !is.eof());
+    if (c == '<')
+    {
+      int start = is.tellg();
+      do
+      {
+        is.get(c);
+      } while (c != '>' && !is.eof());
+      if (c == '>')
+      {
+        int end = is.tellg();
+        int psize = end - start - 1;
+
+        // Only read data between pair limiters.
+        is.seekg(start);
+        std::string tmpstr(psize, ' ');
+        is.read(&tmpstr[0], psize);
+        std::istringstream temp(tmpstr);
+        temp >> buf1 >> buf2;
+        buf = std::make_pair(buf1, buf2);
+        is.seekg(end);
+      }
+    }
+    is.peek();
+    return is;
+  }
+  
+  // output to streams for pairs
+  template <class T1, class T2>
+  inline std::ostream & operator<<(std::ostream &os, const std::pair<T1, T2> &p)
+  {
+    os << "<" << p.first << " " << p.second << ">";
+    return os;
+  }
+
   // Abstract writer/reader classes ////////////////////////////////////////////
   // static polymorphism implemented using CRTP idiom
   class Serializable;

lib/serialisation/MacroMagic.h

@@ -54,7 +54,7 @@ THE SOFTWARE.
 
 #define GRID_MACRO_EMPTY()
 
-#define GRID_MACRO_EVAL(...)     GRID_MACRO_EVAL1024(__VA_ARGS__)
+#define GRID_MACRO_EVAL(...)     GRID_MACRO_EVAL64(__VA_ARGS__)
 #define GRID_MACRO_EVAL1024(...) GRID_MACRO_EVAL512(GRID_MACRO_EVAL512(__VA_ARGS__))
 #define GRID_MACRO_EVAL512(...)  GRID_MACRO_EVAL256(GRID_MACRO_EVAL256(__VA_ARGS__))
 #define GRID_MACRO_EVAL256(...)  GRID_MACRO_EVAL128(GRID_MACRO_EVAL128(__VA_ARGS__))

lib/simd/Grid_avx.h

@@ -368,8 +368,8 @@ namespace Optimization {
       b0 = _mm256_extractf128_si256(b,0);
       a1 = _mm256_extractf128_si256(a,1);
       b1 = _mm256_extractf128_si256(b,1);
-      a0 = _mm_mul_epi32(a0,b0);
+      a0 = _mm_mullo_epi32(a0,b0);
-      a1 = _mm_mul_epi32(a1,b1);
+      a1 = _mm_mullo_epi32(a1,b1);
       return _mm256_set_m128i(a1,a0);
 #endif
 #if defined (AVX2)
@@ -461,7 +461,52 @@ namespace Optimization {
       return in;
     };
   };
-
+#define USE_FP16
+  struct PrecisionChange {
+    static inline __m256i StoH (__m256 a,__m256 b) {
+      __m256i h;
+#ifdef USE_FP16
+      __m128i ha = _mm256_cvtps_ph(a,0);
+      __m128i hb = _mm256_cvtps_ph(b,0);
+      h =(__m256i) _mm256_castps128_ps256((__m128)ha);
+      h =(__m256i) _mm256_insertf128_ps((__m256)h,(__m128)hb,1);
+#else 
+      assert(0);
+#endif
+      return h;
+    }
+    static inline void  HtoS (__m256i h,__m256 &sa,__m256 &sb) {
+#ifdef USE_FP16
+      sa = _mm256_cvtph_ps((__m128i)_mm256_extractf128_ps((__m256)h,0));
+      sb = _mm256_cvtph_ps((__m128i)_mm256_extractf128_ps((__m256)h,1));
+#else 
+      assert(0);
+#endif
+    }
+    static inline __m256 DtoS (__m256d a,__m256d b) {
+      __m128 sa = _mm256_cvtpd_ps(a);
+      __m128 sb = _mm256_cvtpd_ps(b);
+      __m256 s = _mm256_castps128_ps256(sa);
+      s = _mm256_insertf128_ps(s,sb,1);
+      return s;
+    }
+    static inline void StoD (__m256 s,__m256d &a,__m256d &b) {
+      a = _mm256_cvtps_pd(_mm256_extractf128_ps(s,0));
+      b = _mm256_cvtps_pd(_mm256_extractf128_ps(s,1));
+    }
+    static inline __m256i DtoH (__m256d a,__m256d b,__m256d c,__m256d d) {
+      __m256 sa,sb;
+      sa = DtoS(a,b);
+      sb = DtoS(c,d);
+      return StoH(sa,sb);
+    }
+    static inline void HtoD (__m256i h,__m256d &a,__m256d &b,__m256d &c,__m256d &d) {
+      __m256 sa,sb;
+      HtoS(h,sa,sb);
+      StoD(sa,a,b);
+      StoD(sb,c,d);
+    }
+  };
   struct Exchange{
     // 3210 ordering
     static inline void Exchange0(__m256 &out1,__m256 &out2,__m256 in1,__m256 in2){
@@ -666,6 +711,7 @@ namespace Optimization {
 //////////////////////////////////////////////////////////////////////////////////////
 // Here assign types
 
+  typedef __m256i SIMD_Htype;  // Single precision type
   typedef __m256  SIMD_Ftype; // Single precision type
   typedef __m256d SIMD_Dtype; // Double precision type
   typedef __m256i SIMD_Itype; // Integer type

lib/simd/Grid_avx512.h

@@ -235,11 +235,9 @@ namespace Optimization {
     inline void mac(__m512 &a, __m512 b, __m512 c){         
        a= _mm512_fmadd_ps( b, c, a);                         
     }
-
     inline void mac(__m512d &a, __m512d b, __m512d c){
       a= _mm512_fmadd_pd( b, c, a);                   
     }                                             
-
     // Real float
     inline __m512 operator()(__m512 a, __m512 b){
       return _mm512_mul_ps(a,b);
@@ -342,7 +340,52 @@ namespace Optimization {
     };
 
   };
-
+#define USE_FP16
+  struct PrecisionChange {
+    static inline __m512i StoH (__m512 a,__m512 b) {
+      __m512i h;
+#ifdef USE_FP16
+      __m256i ha = _mm512_cvtps_ph(a,0);
+      __m256i hb = _mm512_cvtps_ph(b,0);
+      h =(__m512i) _mm512_castps256_ps512((__m256)ha);
+      h =(__m512i) _mm512_insertf64x4((__m512d)h,(__m256d)hb,1);
+#else
+      assert(0);
+#endif
+      return h;
+    }
+    static inline void  HtoS (__m512i h,__m512 &sa,__m512 &sb) {
+#ifdef USE_FP16
+      sa = _mm512_cvtph_ps((__m256i)_mm512_extractf64x4_pd((__m512d)h,0));
+      sb = _mm512_cvtph_ps((__m256i)_mm512_extractf64x4_pd((__m512d)h,1));
+#else
+      assert(0);
+#endif
+    }
+    static inline __m512 DtoS (__m512d a,__m512d b) {
+      __m256 sa = _mm512_cvtpd_ps(a);
+      __m256 sb = _mm512_cvtpd_ps(b);
+      __m512 s = _mm512_castps256_ps512(sa);
+      s =(__m512) _mm512_insertf64x4((__m512d)s,(__m256d)sb,1);
+      return s;
+    }
+    static inline void StoD (__m512 s,__m512d &a,__m512d &b) {
+      a = _mm512_cvtps_pd((__m256)_mm512_extractf64x4_pd((__m512d)s,0));
+      b = _mm512_cvtps_pd((__m256)_mm512_extractf64x4_pd((__m512d)s,1));
+    }
+    static inline __m512i DtoH (__m512d a,__m512d b,__m512d c,__m512d d) {
+      __m512 sa,sb;
+      sa = DtoS(a,b);
+      sb = DtoS(c,d);
+      return StoH(sa,sb);
+    }
+    static inline void HtoD (__m512i h,__m512d &a,__m512d &b,__m512d &c,__m512d &d) {
+      __m512 sa,sb;
+      HtoS(h,sa,sb);
+      StoD(sa,a,b);
+      StoD(sb,c,d);
+    }
+  };
   // On extracting face: Ah Al , Bh Bl -> Ah Bh, Al Bl
   // On merging buffers: Ah,Bh , Al Bl -> Ah Al, Bh, Bl
   // The operation is its own inverse
@@ -539,7 +582,9 @@ namespace Optimization {
 //////////////////////////////////////////////////////////////////////////////////////
 // Here assign types 
 
-  typedef __m512 SIMD_Ftype;  // Single precision type
+
+  typedef __m512i SIMD_Htype;  // Single precision type
+  typedef __m512  SIMD_Ftype;  // Single precision type
   typedef __m512d SIMD_Dtype; // Double precision type
   typedef __m512i SIMD_Itype; // Integer type
 

lib/simd/Grid_generic.h

@@ -279,6 +279,101 @@ namespace Optimization {
   
   #undef timesi
 
+  struct PrecisionChange {
+    static inline vech StoH (const vecf &a,const vecf &b) {
+#ifdef USE_FP16
+      vech ret;
+      vech *ha = (vech *)&a;
+      vech *hb = (vech *)&b;
+      const int nf = W<float>::r;
+      //      VECTOR_FOR(i, nf,1){ ret.v[i]    = ( (uint16_t *) &a.v[i])[1] ; }
+      //      VECTOR_FOR(i, nf,1){ ret.v[i+nf] = ( (uint16_t *) &b.v[i])[1] ; }
+      VECTOR_FOR(i, nf,1){ ret.v[i]    = ha->v[2*i+1]; }
+      VECTOR_FOR(i, nf,1){ ret.v[i+nf] = hb->v[2*i+1]; }
+#else
+      assert(0);
+#endif
+      return ret;
+    }
+    static inline void  HtoS (vech h,vecf &sa,vecf &sb) {
+#ifdef USE_FP16
+      const int nf = W<float>::r;
+      const int nh = W<uint16_t>::r;
+      vech *ha = (vech *)&sa;
+      vech *hb = (vech *)&sb;
+      VECTOR_FOR(i, nf, 1){ sb.v[i]= sa.v[i] = 0; }
+      //      VECTOR_FOR(i, nf, 1){ ( (uint16_t *) (&sa.v[i]))[1] = h.v[i];}
+      //      VECTOR_FOR(i, nf, 1){ ( (uint16_t *) (&sb.v[i]))[1] = h.v[i+nf];}
+      VECTOR_FOR(i, nf, 1){ ha->v[2*i+1]=h.v[i]; }
+      VECTOR_FOR(i, nf, 1){ hb->v[2*i+1]=h.v[i+nf]; }
+#else
+      assert(0);
+#endif
+    }
+    static inline vecf DtoS (vecd a,vecd b) {
+      const int nd = W<double>::r;
+      const int nf = W<float>::r;
+      vecf ret;
+      VECTOR_FOR(i, nd,1){ ret.v[i]    = a.v[i] ; }
+      VECTOR_FOR(i, nd,1){ ret.v[i+nd] = b.v[i] ; }
+      return ret;
+    }
+    static inline void StoD (vecf s,vecd &a,vecd &b) {
+      const int nd = W<double>::r;
+      VECTOR_FOR(i, nd,1){ a.v[i] = s.v[i] ; }
+      VECTOR_FOR(i, nd,1){ b.v[i] = s.v[i+nd] ; }
+    }
+    static inline vech DtoH (vecd a,vecd b,vecd c,vecd d) {
+      vecf sa,sb;
+      sa = DtoS(a,b);
+      sb = DtoS(c,d);
+      return StoH(sa,sb);
+    }
+    static inline void HtoD (vech h,vecd &a,vecd &b,vecd &c,vecd &d) {
+      vecf sa,sb;
+      HtoS(h,sa,sb);
+      StoD(sa,a,b);
+      StoD(sb,c,d);
+    }
+  };
+
+  //////////////////////////////////////////////
+  // Exchange support
+  struct Exchange{
+
+    template <typename T,int n>
+    static inline void ExchangeN(vec<T> &out1,vec<T> &out2,vec<T> &in1,vec<T> &in2){
+      const int w = W<T>::r;
+      unsigned int mask = w >> (n + 1);
+      //      std::cout << " Exchange "<<n<<" nsimd "<<w<<" mask 0x" <<std::hex<<mask<<std::dec<<std::endl;
+      VECTOR_FOR(i, w, 1) {	
+	int j1 = i&(~mask);
+	if  ( (i&mask) == 0 ) { out1.v[i]=in1.v[j1];}
+	else                  { out1.v[i]=in2.v[j1];}
+	int j2 = i|mask;
+	if  ( (i&mask) == 0 ) { out2.v[i]=in1.v[j2];}
+	else                  { out2.v[i]=in2.v[j2];}
+      }      
+    }
+    template <typename T>
+    static inline void Exchange0(vec<T> &out1,vec<T> &out2,vec<T> &in1,vec<T> &in2){
+      ExchangeN<T,0>(out1,out2,in1,in2);
+    };
+    template <typename T>
+    static inline void Exchange1(vec<T> &out1,vec<T> &out2,vec<T> &in1,vec<T> &in2){
+      ExchangeN<T,1>(out1,out2,in1,in2);
+    };
+    template <typename T>
+    static inline void Exchange2(vec<T> &out1,vec<T> &out2,vec<T> &in1,vec<T> &in2){
+      ExchangeN<T,2>(out1,out2,in1,in2);
+    };
+    template <typename T>
+    static inline void Exchange3(vec<T> &out1,vec<T> &out2,vec<T> &in1,vec<T> &in2){
+      ExchangeN<T,3>(out1,out2,in1,in2);
+    };
+  };
+
+
   //////////////////////////////////////////////
   // Some Template specialization
   #define perm(a, b, n, w)\
@@ -403,6 +498,7 @@ namespace Optimization {
 //////////////////////////////////////////////////////////////////////////////////////
 // Here assign types 
 
+  typedef Optimization::vech SIMD_Htype; // Reduced precision type
   typedef Optimization::vecf SIMD_Ftype; // Single precision type
   typedef Optimization::vecd SIMD_Dtype; // Double precision type
   typedef Optimization::veci SIMD_Itype; // Integer type

lib/simd/Grid_generic_types.h

@@ -66,6 +66,10 @@ namespace Optimization {
   template <> struct W<Integer> {
     constexpr static unsigned int r = GEN_SIMD_WIDTH/4u;
   };
+  template <> struct W<uint16_t> {
+    constexpr static unsigned int c = GEN_SIMD_WIDTH/4u;
+    constexpr static unsigned int r = GEN_SIMD_WIDTH/2u;
+  };
   
   // SIMD vector types
   template <typename T>
@@ -73,8 +77,9 @@ namespace Optimization {
     alignas(GEN_SIMD_WIDTH) T v[W<T>::r];
   };
 
-  typedef vec<float>   vecf;
+  typedef vec<float>     vecf;
-  typedef vec<double>  vecd;
+  typedef vec<double>    vecd;
-  typedef vec<Integer> veci;
+  typedef vec<uint16_t>  vech; // half precision comms
+  typedef vec<Integer>   veci;
   
 }}

lib/simd/Grid_qpx.h

@@ -33,6 +33,14 @@
 #include "Grid_generic_types.h" // Definitions for simulated integer SIMD.
 
 namespace Grid {
+
+#ifdef QPX
+#include <spi/include/kernel/location.h>
+#include <spi/include/l1p/types.h>
+#include <hwi/include/bqc/l1p_mmio.h>
+#include <hwi/include/bqc/A2_inlines.h>
+#endif
+
 namespace Optimization {
   typedef struct 
   {
@@ -125,7 +133,6 @@ namespace Optimization {
       f[2] = a.v2;
       f[3] = a.v3;
     }
-
     //Double
     inline void operator()(double *d, vector4double a){
       vec_st(a, 0, d);

lib/simd/Grid_sse4.h

@@ -328,6 +328,140 @@ namespace Optimization {
     };
   };
 
+  
+#define _my_alignr_epi32(a,b,n) _mm_alignr_epi8(a,b,(n*4)%16)
+#define _my_alignr_epi64(a,b,n) _mm_alignr_epi8(a,b,(n*8)%16)
+
+#ifdef SFW_FP16
+
+  struct Grid_half {
+    Grid_half(){}
+    Grid_half(uint16_t raw) : x(raw) {}
+    uint16_t x;
+  };
+  union FP32 {
+    unsigned int u;
+    float f;
+  };
+
+  // PAB - Lifted and adapted from Eigen, which is GPL V2
+  inline float sfw_half_to_float(Grid_half h) {
+    const FP32 magic = { 113 << 23 };
+    const unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift
+    FP32 o;
+    o.u = (h.x & 0x7fff) << 13;             // exponent/mantissa bits
+    unsigned int exp = shifted_exp & o.u;   // just the exponent
+    o.u += (127 - 15) << 23;                // exponent adjust
+    // handle exponent special cases
+    if (exp == shifted_exp) {     // Inf/NaN?
+      o.u += (128 - 16) << 23;    // extra exp adjust
+    } else if (exp == 0) {        // Zero/Denormal?
+      o.u += 1 << 23;             // extra exp adjust
+      o.f -= magic.f;             // renormalize
+    }
+    o.u |= (h.x & 0x8000) << 16;    // sign bit
+    return o.f;
+  }
+  inline Grid_half sfw_float_to_half(float ff) {
+    FP32 f; f.f = ff;
+    const FP32 f32infty = { 255 << 23 };
+    const FP32 f16max = { (127 + 16) << 23 };
+    const FP32 denorm_magic = { ((127 - 15) + (23 - 10) + 1) << 23 };
+    unsigned int sign_mask = 0x80000000u;
+    Grid_half o;
+    
+    o.x = static_cast<unsigned short>(0x0u);
+    unsigned int sign = f.u & sign_mask;
+    f.u ^= sign;
+    // NOTE all the integer compares in this function can be safely
+    // compiled into signed compares since all operands are below
+    // 0x80000000. Important if you want fast straight SSE2 code
+    // (since there's no unsigned PCMPGTD).
+    if (f.u >= f16max.u) {  // result is Inf or NaN (all exponent bits set)
+      o.x = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf
+    } else {  // (De)normalized number or zero
+      if (f.u < (113 << 23)) {  // resulting FP16 is subnormal or zero
+	// use a magic value to align our 10 mantissa bits at the bottom of
+	// the float. as long as FP addition is round-to-nearest-even this
+	// just works.
+	f.f += denorm_magic.f;
+	// and one integer subtract of the bias later, we have our final float!
+	o.x = static_cast<unsigned short>(f.u - denorm_magic.u);
+      } else {
+	unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd
+	
+	// update exponent, rounding bias part 1
+	f.u += ((unsigned int)(15 - 127) << 23) + 0xfff;
+	// rounding bias part 2
+	f.u += mant_odd;
+	// take the bits!
+	o.x = static_cast<unsigned short>(f.u >> 13);
+      }
+    } 
+    o.x |= static_cast<unsigned short>(sign >> 16);
+    return o;
+  }
+  static inline __m128i Grid_mm_cvtps_ph(__m128 f,int discard) {
+    __m128i ret=(__m128i)_mm_setzero_ps();
+    float *fp = (float *)&f;
+    Grid_half *hp = (Grid_half *)&ret;
+    hp[0] = sfw_float_to_half(fp[0]);
+    hp[1] = sfw_float_to_half(fp[1]);
+    hp[2] = sfw_float_to_half(fp[2]);
+    hp[3] = sfw_float_to_half(fp[3]);
+    return ret;
+  }
+  static inline __m128 Grid_mm_cvtph_ps(__m128i h,int discard) {
+    __m128 ret=_mm_setzero_ps();
+    float *fp = (float *)&ret;
+    Grid_half  *hp = (Grid_half *)&h;
+    fp[0] = sfw_half_to_float(hp[0]);
+    fp[1] = sfw_half_to_float(hp[1]);
+    fp[2] = sfw_half_to_float(hp[2]);
+    fp[3] = sfw_half_to_float(hp[3]);
+    return ret;
+  }
+#else 
+#define Grid_mm_cvtps_ph _mm_cvtps_ph
+#define Grid_mm_cvtph_ps _mm_cvtph_ps
+#endif
+  struct PrecisionChange {
+    static inline __m128i StoH (__m128 a,__m128 b) {
+      __m128i ha = Grid_mm_cvtps_ph(a,0);
+      __m128i hb = Grid_mm_cvtps_ph(b,0);
+      __m128i h =(__m128i) _mm_shuffle_ps((__m128)ha,(__m128)hb,_MM_SELECT_FOUR_FOUR(1,0,1,0));
+      return h;
+    }
+    static inline void  HtoS (__m128i h,__m128 &sa,__m128 &sb) {
+      sa = Grid_mm_cvtph_ps(h,0); 
+      h =  (__m128i)_my_alignr_epi32((__m128i)h,(__m128i)h,2);
+      sb = Grid_mm_cvtph_ps(h,0);
+    }
+    static inline __m128 DtoS (__m128d a,__m128d b) {
+      __m128 sa = _mm_cvtpd_ps(a);
+      __m128 sb = _mm_cvtpd_ps(b);
+      __m128 s = _mm_shuffle_ps(sa,sb,_MM_SELECT_FOUR_FOUR(1,0,1,0));
+      return s;
+    }
+    static inline void StoD (__m128 s,__m128d &a,__m128d &b) {
+      a = _mm_cvtps_pd(s);
+      s = (__m128)_my_alignr_epi32((__m128i)s,(__m128i)s,2);
+      b = _mm_cvtps_pd(s);
+    }
+    static inline __m128i DtoH (__m128d a,__m128d b,__m128d c,__m128d d) {
+      __m128 sa,sb;
+      sa = DtoS(a,b);
+      sb = DtoS(c,d);
+      return StoH(sa,sb);
+    }
+    static inline void HtoD (__m128i h,__m128d &a,__m128d &b,__m128d &c,__m128d &d) {
+      __m128 sa,sb;
+      HtoS(h,sa,sb);
+      StoD(sa,a,b);
+      StoD(sb,c,d);
+    }
+  };
+
   struct Exchange{
     // 3210 ordering
     static inline void Exchange0(__m128 &out1,__m128 &out2,__m128 in1,__m128 in2){
@@ -335,8 +469,10 @@ namespace Optimization {
       out2= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(3,2,3,2));
     };
     static inline void Exchange1(__m128 &out1,__m128 &out2,__m128 in1,__m128 in2){
-      out1= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(2,0,2,0));
+      out1= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(2,0,2,0)); /*ACEG*/
-      out2= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(3,1,3,1));
+      out2= _mm_shuffle_ps(in1,in2,_MM_SELECT_FOUR_FOUR(3,1,3,1)); /*BDFH*/
+      out1= _mm_shuffle_ps(out1,out1,_MM_SELECT_FOUR_FOUR(3,1,2,0)); /*AECG*/
+      out2= _mm_shuffle_ps(out2,out2,_MM_SELECT_FOUR_FOUR(3,1,2,0)); /*AECG*/
     };
     static inline void Exchange2(__m128 &out1,__m128 &out2,__m128 in1,__m128 in2){
       assert(0);
@@ -383,14 +519,9 @@ namespace Optimization {
       default: assert(0);
       }
     }
-  
-#ifndef _mm_alignr_epi64
-#define _mm_alignr_epi32(a,b,n) _mm_alignr_epi8(a,b,(n*4)%16)
-#define _mm_alignr_epi64(a,b,n) _mm_alignr_epi8(a,b,(n*8)%16)
-#endif 
 
-    template<int n> static inline __m128  tRotate(__m128  in){ return (__m128)_mm_alignr_epi32((__m128i)in,(__m128i)in,n); };
+    template<int n> static inline __m128  tRotate(__m128  in){ return (__m128)_my_alignr_epi32((__m128i)in,(__m128i)in,n); };
-    template<int n> static inline __m128d tRotate(__m128d in){ return (__m128d)_mm_alignr_epi64((__m128i)in,(__m128i)in,n); };
+    template<int n> static inline __m128d tRotate(__m128d in){ return (__m128d)_my_alignr_epi64((__m128i)in,(__m128i)in,n); };
 
   };
   //////////////////////////////////////////////
@@ -450,7 +581,8 @@ namespace Optimization {
 //////////////////////////////////////////////////////////////////////////////////////
 // Here assign types 
 
-  typedef __m128 SIMD_Ftype;  // Single precision type
+  typedef __m128i SIMD_Htype;  // Single precision type
+  typedef __m128  SIMD_Ftype;  // Single precision type
   typedef __m128d SIMD_Dtype; // Double precision type
   typedef __m128i SIMD_Itype; // Integer type
 

lib/simd/Grid_vector_types.h

@@ -2,7 +2,7 @@
 
 Grid physics library, www.github.com/paboyle/Grid
 
-Source file: ./lib/simd/Grid_vector_types.h
+Source file: ./lib/simd/Grid_vector_type.h
 
 Copyright (C) 2015
 
@@ -53,12 +53,14 @@ directory
 #if defined IMCI
 #include "Grid_imci.h"
 #endif
-#if defined QPX
-#include "Grid_qpx.h"
-#endif
 #ifdef NEONv8
 #include "Grid_neon.h"
 #endif
+#if defined QPX
+#include "Grid_qpx.h"
+#endif
+
+#include "l1p.h"
 
 namespace Grid {
 
@@ -74,12 +76,14 @@ struct RealPart<std::complex<T> > {
   typedef T type;
 };
 
+#include <type_traits>
+
 //////////////////////////////////////
 // demote a vector to real type
 //////////////////////////////////////
 // type alias used to simplify the syntax of std::enable_if
 template <typename T> using Invoke = typename T::type;
-template <typename Condition, typename ReturnType> using EnableIf = Invoke<std::enable_if<Condition::value, ReturnType> >;
+template <typename Condition, typename ReturnType> using EnableIf    = Invoke<std::enable_if<Condition::value, ReturnType> >;
 template <typename Condition, typename ReturnType> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
 
 ////////////////////////////////////////////////////////
@@ -88,13 +92,15 @@ template <typename T> struct is_complex : public std::false_type {};
 template <> struct is_complex<std::complex<double> > : public std::true_type {};
 template <> struct is_complex<std::complex<float> > : public std::true_type {};
 
-template <typename T> using IfReal       = Invoke<std::enable_if<std::is_floating_point<T>::value, int> >;
+template <typename T>              using IfReal    = Invoke<std::enable_if<std::is_floating_point<T>::value, int> >;
-template <typename T> using IfComplex    = Invoke<std::enable_if<is_complex<T>::value, int> >;
+template <typename T>              using IfComplex = Invoke<std::enable_if<is_complex<T>::value, int> >;
-template <typename T> using IfInteger    = Invoke<std::enable_if<std::is_integral<T>::value, int> >;
+template <typename T>              using IfInteger = Invoke<std::enable_if<std::is_integral<T>::value, int> >;
+template <typename T1,typename T2> using IfSame    = Invoke<std::enable_if<std::is_same<T1,T2>::value, int> >;
 
-template <typename T> using IfNotReal    = Invoke<std::enable_if<!std::is_floating_point<T>::value, int> >;
+template <typename T>              using IfNotReal    = Invoke<std::enable_if<!std::is_floating_point<T>::value, int> >;
-template <typename T> using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
+template <typename T>              using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
-template <typename T> using IfNotInteger = Invoke<std::enable_if<!std::is_integral<T>::value, int> >;
+template <typename T>              using IfNotInteger = Invoke<std::enable_if<!std::is_integral<T>::value, int> >;
+template <typename T1,typename T2> using IfNotSame    = Invoke<std::enable_if<!std::is_same<T1,T2>::value, int> >;
 
 ////////////////////////////////////////////////////////
 // Define the operation templates functors
@@ -358,16 +364,12 @@ class Grid_simd {
   {
     if       (n==3) {
       Optimization::Exchange::Exchange3(out1.v,out2.v,in1.v,in2.v);
-      //      std::cout << " Exchange3 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
     } else if(n==2) {
       Optimization::Exchange::Exchange2(out1.v,out2.v,in1.v,in2.v);
-      //      std::cout << " Exchange2 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
     } else if(n==1) {
       Optimization::Exchange::Exchange1(out1.v,out2.v,in1.v,in2.v);
-      //      std::cout << " Exchange1 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
     } else if(n==0) { 
       Optimization::Exchange::Exchange0(out1.v,out2.v,in1.v,in2.v);
-      //      std::cout << " Exchange0 "<< out1<<" "<< out2<<" <- " << in1 << " "<<in2<<std::endl;
     }
   }
 
@@ -428,7 +430,6 @@ template <class S, class V, IfNotComplex<S> = 0>
 inline Grid_simd<S, V> rotate(Grid_simd<S, V> b, int nrot) {
   nrot = nrot % Grid_simd<S, V>::Nsimd();
   Grid_simd<S, V> ret;
-  //    std::cout << "Rotate Real by "<<nrot<<std::endl;
   ret.v = Optimization::Rotate::rotate(b.v, nrot);
   return ret;
 }
@@ -436,7 +437,6 @@ template <class S, class V, IfComplex<S> = 0>
 inline Grid_simd<S, V> rotate(Grid_simd<S, V> b, int nrot) {
   nrot = nrot % Grid_simd<S, V>::Nsimd();
   Grid_simd<S, V> ret;
-  //    std::cout << "Rotate Complex by "<<nrot<<std::endl;
   ret.v = Optimization::Rotate::rotate(b.v, 2 * nrot);
   return ret;
 }
@@ -444,14 +444,12 @@ template <class S, class V, IfNotComplex<S> =0>
 inline void rotate( Grid_simd<S,V> &ret,Grid_simd<S,V> b,int nrot)
 {
   nrot = nrot % Grid_simd<S,V>::Nsimd();
-  //    std::cout << "Rotate Real by "<<nrot<<std::endl;
   ret.v = Optimization::Rotate::rotate(b.v,nrot);
 }
 template <class S, class V, IfComplex<S> =0> 
 inline void rotate(Grid_simd<S,V> &ret,Grid_simd<S,V> b,int nrot)
 {
   nrot = nrot % Grid_simd<S,V>::Nsimd();
-  //    std::cout << "Rotate Complex by "<<nrot<<std::endl;
   ret.v = Optimization::Rotate::rotate(b.v,2*nrot);
 }
 
@@ -711,7 +709,6 @@ inline Grid_simd<S, V> innerProduct(const Grid_simd<S, V> &l,
                                     const Grid_simd<S, V> &r) {
   return conjugate(l) * r;
 }
-
 template <class S, class V>
 inline Grid_simd<S, V> outerProduct(const Grid_simd<S, V> &l,
                                     const Grid_simd<S, V> &r) {
@@ -771,6 +768,67 @@ typedef Grid_simd<std::complex<float>, SIMD_Ftype> vComplexF;
 typedef Grid_simd<std::complex<double>, SIMD_Dtype> vComplexD;
 typedef Grid_simd<Integer, SIMD_Itype> vInteger;
 
+// Half precision; no arithmetic support
+typedef Grid_simd<uint16_t, SIMD_Htype>               vRealH;
+typedef Grid_simd<std::complex<uint16_t>, SIMD_Htype> vComplexH;
+
+inline void precisionChange(vRealF    *out,vRealD    *in,int nvec)
+{
+  assert((nvec&0x1)==0);
+  for(int m=0;m*2<nvec;m++){
+    int n=m*2;
+    out[m].v=Optimization::PrecisionChange::DtoS(in[n].v,in[n+1].v);
+  }
+}
+inline void precisionChange(vRealH    *out,vRealD    *in,int nvec)
+{
+  assert((nvec&0x3)==0);
+  for(int m=0;m*4<nvec;m++){
+    int n=m*4;
+    out[m].v=Optimization::PrecisionChange::DtoH(in[n].v,in[n+1].v,in[n+2].v,in[n+3].v);
+  }
+}
+inline void precisionChange(vRealH    *out,vRealF    *in,int nvec)
+{
+  assert((nvec&0x1)==0);
+  for(int m=0;m*2<nvec;m++){
+    int n=m*2;
+    out[m].v=Optimization::PrecisionChange::StoH(in[n].v,in[n+1].v);
+  }
+}
+inline void precisionChange(vRealD    *out,vRealF    *in,int nvec)
+{
+  assert((nvec&0x1)==0);
+  for(int m=0;m*2<nvec;m++){
+    int n=m*2;
+    Optimization::PrecisionChange::StoD(in[m].v,out[n].v,out[n+1].v);
+  }
+}
+inline void precisionChange(vRealD    *out,vRealH    *in,int nvec)
+{
+  assert((nvec&0x3)==0);
+  for(int m=0;m*4<nvec;m++){
+    int n=m*4;
+    Optimization::PrecisionChange::HtoD(in[m].v,out[n].v,out[n+1].v,out[n+2].v,out[n+3].v);
+  }
+}
+inline void precisionChange(vRealF    *out,vRealH    *in,int nvec)
+{
+  assert((nvec&0x1)==0);
+  for(int m=0;m*2<nvec;m++){
+    int n=m*2;
+    Optimization::PrecisionChange::HtoS(in[m].v,out[n].v,out[n+1].v);
+  }
+}
+inline void precisionChange(vComplexF *out,vComplexD *in,int nvec){ precisionChange((vRealF *)out,(vRealD *)in,nvec);}
+inline void precisionChange(vComplexH *out,vComplexD *in,int nvec){ precisionChange((vRealH *)out,(vRealD *)in,nvec);}
+inline void precisionChange(vComplexH *out,vComplexF *in,int nvec){ precisionChange((vRealH *)out,(vRealF *)in,nvec);}
+inline void precisionChange(vComplexD *out,vComplexF *in,int nvec){ precisionChange((vRealD *)out,(vRealF *)in,nvec);}
+inline void precisionChange(vComplexD *out,vComplexH *in,int nvec){ precisionChange((vRealD *)out,(vRealH *)in,nvec);}
+inline void precisionChange(vComplexF *out,vComplexH *in,int nvec){ precisionChange((vRealF *)out,(vRealH *)in,nvec);}
+
+
+
 // Check our vector types are of an appropriate size.
 #if defined QPX
 static_assert(2*sizeof(SIMD_Ftype) == sizeof(SIMD_Dtype), "SIMD vector lengths incorrect");

lib/simd/l1p.h

@@ -0,0 +1,37 @@
+#pragma once
+namespace Grid {
+// L1p optimisation 
+inline void bgq_l1p_optimisation(int mode)
+{
+#ifdef QPX
+#undef L1P_CFG_PF_USR
+#define L1P_CFG_PF_USR  (0x3fde8000108ll)   /*  (64 bit reg, 23 bits wide, user/unpriv) */
+
+  uint64_t cfg_pf_usr;
+  if ( mode ) { 
+    cfg_pf_usr =
+        L1P_CFG_PF_USR_ifetch_depth(0)       
+      | L1P_CFG_PF_USR_ifetch_max_footprint(1)   
+      | L1P_CFG_PF_USR_pf_stream_est_on_dcbt 
+      | L1P_CFG_PF_USR_pf_stream_establish_enable
+      | L1P_CFG_PF_USR_pf_stream_optimistic
+      | L1P_CFG_PF_USR_pf_adaptive_throttle(0xF) ;
+    //    if ( sizeof(Float) == sizeof(double) ) {
+      cfg_pf_usr |=  L1P_CFG_PF_USR_dfetch_depth(2)| L1P_CFG_PF_USR_dfetch_max_footprint(3)   ;
+      //    } else {
+      //      cfg_pf_usr |=  L1P_CFG_PF_USR_dfetch_depth(1)| L1P_CFG_PF_USR_dfetch_max_footprint(2)   ;
+      //    }
+  } else { 
+    cfg_pf_usr = L1P_CFG_PF_USR_dfetch_depth(1)
+      | L1P_CFG_PF_USR_dfetch_max_footprint(2)   
+      | L1P_CFG_PF_USR_ifetch_depth(0)       
+      | L1P_CFG_PF_USR_ifetch_max_footprint(1)   
+      | L1P_CFG_PF_USR_pf_stream_est_on_dcbt 
+      | L1P_CFG_PF_USR_pf_stream_establish_enable
+      | L1P_CFG_PF_USR_pf_stream_optimistic
+      | L1P_CFG_PF_USR_pf_stream_prefetch_enable;
+  }
+  *((uint64_t *)L1P_CFG_PF_USR) = cfg_pf_usr;
+#endif
+}
+}

lib/stencil/SimpleCompressor.h

@@ -0,0 +1,29 @@
+#ifndef _STENCIL_SIMPLE_COMPRESSOR_H_
+#define _STENCIL_SIMPLE_COMPRESSOR_H_
+
+namespace Grid {
+
+template<class vobj>
+class SimpleCompressor {
+public:
+  void Point(int) {};
+  inline int  CommDatumSize(void) { return sizeof(vobj); }
+  inline bool DecompressionStep(void) { return false; }
+  inline void Compress(vobj *buf,int o,const vobj &in) { buf[o]=in; }
+  inline void Exchange(vobj *mp,vobj *vp0,vobj *vp1,Integer type,Integer o){
+    exchange(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type);
+  }
+  inline void Decompress(vobj *out,vobj *in, int o){ assert(0); }
+  inline void CompressExchange(vobj *out0,vobj *out1,const vobj *in,
+			       int j,int k, int m,int type){
+    exchange(out0[j],out1[j],in[k],in[m],type);
+  }
+  // For cshift. Cshift should drop compressor coupling altogether 
+  // because I had to decouple the code from the Stencil anyway
+  inline vobj operator() (const vobj &arg) {
+    return arg;
+  }
+};
+
+}
+#endif

lib/tensors/Tensor_class.h

@@ -1,9 +1,6 @@
 /*************************************************************************************
-
 Grid physics library, www.github.com/paboyle/Grid
-
 Source file: ./lib/tensors/Tensor_class.h
-
 Copyright (C) 2015
 
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
@@ -13,16 +10,13 @@ This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.
-
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
-
 You should have received a copy of the GNU General Public License along
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
 See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
@@ -56,18 +50,18 @@ class iScalar {
   typedef vtype element;
   typedef typename GridTypeMapper<vtype>::scalar_type scalar_type;
   typedef typename GridTypeMapper<vtype>::vector_type vector_type;
+  typedef typename GridTypeMapper<vtype>::vector_typeD vector_typeD;
   typedef typename GridTypeMapper<vtype>::tensor_reduced tensor_reduced_v;
-  typedef iScalar<tensor_reduced_v> tensor_reduced;
   typedef typename GridTypeMapper<vtype>::scalar_object recurse_scalar_object;
+  typedef iScalar<tensor_reduced_v> tensor_reduced;
   typedef iScalar<recurse_scalar_object> scalar_object;
-
   // substitutes a real or complex version with same tensor structure
   typedef iScalar<typename GridTypeMapper<vtype>::Complexified> Complexified;
   typedef iScalar<typename GridTypeMapper<vtype>::Realified> Realified;
 
   // get double precision version
   typedef iScalar<typename GridTypeMapper<vtype>::DoublePrecision> DoublePrecision;
-
+  
   enum { TensorLevel = GridTypeMapper<vtype>::TensorLevel + 1 };
 
   // Scalar no action
@@ -80,29 +74,18 @@ class iScalar {
   iScalar<vtype> & operator= (const iScalar<vtype> &copyme) = default;
   iScalar<vtype> & operator= (iScalar<vtype> &&copyme) = default;
   */
-  iScalar(scalar_type s)
+
-      : _internal(s){};  // recurse down and hit the constructor for vector_type
+  //  template<int N=0>
+  //  iScalar(EnableIf<isSIMDvectorized<vector_type>, vector_type> s) : _internal(s){};  // recurse down and hit the constructor for vector_type
+
+  iScalar(scalar_type s) : _internal(s){};  // recurse down and hit the constructor for vector_type
+
   iScalar(const Zero &z) { *this = zero; };
 
   iScalar<vtype> &operator=(const Zero &hero) {
     zeroit(*this);
     return *this;
   }
-
-
-  // managing the internal vector structure
-  strong_inline scalar_object getlane(int lane){
-    scalar_object ret;
-    ret._internal = _internal.getlane(lane);
-    return ret;
-  }
-
-  strong_inline void putlane(scalar_object &s, int lane){
-    _internal.putlane(s._internal,lane);
-  }
-
-
-  
   friend strong_inline void vstream(iScalar<vtype> &out,
                                     const iScalar<vtype> &in) {
     vstream(out._internal, in._internal);
@@ -152,42 +135,38 @@ class iScalar {
   strong_inline const vtype &operator()(void) const { return _internal; }
 
   // Type casts meta programmed, must be pure scalar to match TensorRemove
-  template <class U = vtype, class V = scalar_type, IfComplex<V> = 0,
+  template <class U = vtype, class V = scalar_type, IfComplex<V> = 0, IfNotSimd<U> = 0>
-            IfNotSimd<U> = 0>
   operator ComplexF() const {
     return (TensorRemove(_internal));
   };
-  template <class U = vtype, class V = scalar_type, IfComplex<V> = 0,
+  template <class U = vtype, class V = scalar_type, IfComplex<V> = 0, IfNotSimd<U> = 0>
-            IfNotSimd<U> = 0>
   operator ComplexD() const {
     return (TensorRemove(_internal));
   };
   //  template<class U=vtype,class V=scalar_type,IfComplex<V> = 0,IfNotSimd<U> =
   //  0> operator RealD    () const { return(real(TensorRemove(_internal))); }
-  template <class U = vtype, class V = scalar_type, IfReal<V> = 0,
+  template <class U = vtype, class V = scalar_type, IfReal<V> = 0,IfNotSimd<U> = 0>
-            IfNotSimd<U> = 0>
   operator RealD() const {
     return TensorRemove(_internal);
   }
-  template <class U = vtype, class V = scalar_type, IfInteger<V> = 0,
+  template <class U = vtype, class V = scalar_type, IfInteger<V> = 0, IfNotSimd<U> = 0>
-            IfNotSimd<U> = 0>
   operator Integer() const {
     return Integer(TensorRemove(_internal));
   }
 
   // convert from a something to a scalar via constructor of something arg
-  template <class T, typename std::enable_if<!isGridTensor<T>::value, T>::type
+  template <class T, typename std::enable_if<!isGridTensor<T>::value, T>::type * = nullptr>
-                         * = nullptr>
+    strong_inline iScalar<vtype> operator=(T arg) {
-  strong_inline iScalar<vtype> operator=(T arg) {
     _internal = arg;
     return *this;
   }
 
-  friend std::ostream &operator<<(std::ostream &stream,
+  friend std::ostream &operator<<(std::ostream &stream,const iScalar<vtype> &o) {
-                                  const iScalar<vtype> &o) {
     stream << "S {" << o._internal << "}";
     return stream;
   };
+
+
 };
 ///////////////////////////////////////////////////////////
 // Allows to turn scalar<scalar<scalar<double>>>> back to double.
@@ -211,6 +190,7 @@ class iVector {
   typedef vtype element;
   typedef typename GridTypeMapper<vtype>::scalar_type scalar_type;
   typedef typename GridTypeMapper<vtype>::vector_type vector_type;
+  typedef typename GridTypeMapper<vtype>::vector_typeD vector_typeD;
   typedef typename GridTypeMapper<vtype>::tensor_reduced tensor_reduced_v;
   typedef typename GridTypeMapper<vtype>::scalar_object recurse_scalar_object;
   typedef iScalar<tensor_reduced_v> tensor_reduced;
@@ -222,8 +202,7 @@ class iVector {
 
   // get double precision version
   typedef iVector<typename GridTypeMapper<vtype>::DoublePrecision, N> DoublePrecision;
-
+  
-
   template <class T, typename std::enable_if<!isGridTensor<T>::value, T>::type
                          * = nullptr>
   strong_inline auto operator=(T arg) -> iVector<vtype, N> {
@@ -246,20 +225,6 @@ class iVector {
     zeroit(*this);
     return *this;
   }
-
-  strong_inline scalar_object getlane(int lane){
-    scalar_object ret;
-    for (int i = 0; i < N; i++) ret._internal[i] = _internal[i].getlane(lane);
-    return ret;
-  }
-
-  strong_inline void putlane(scalar_object &s, int lane){
-    for (int i = 0; i < N; i++) _internal[i].putlane(s._internal[i],lane);
-  }
-  
-
-
-  
   friend strong_inline void zeroit(iVector<vtype, N> &that) {
     for (int i = 0; i < N; i++) {
       zeroit(that._internal[i]);
@@ -341,6 +306,7 @@ class iMatrix {
   typedef vtype element;
   typedef typename GridTypeMapper<vtype>::scalar_type scalar_type;
   typedef typename GridTypeMapper<vtype>::vector_type vector_type;
+  typedef typename GridTypeMapper<vtype>::vector_typeD vector_typeD;
   typedef typename GridTypeMapper<vtype>::tensor_reduced tensor_reduced_v;
   typedef typename GridTypeMapper<vtype>::scalar_object recurse_scalar_object;
 
@@ -350,8 +316,7 @@ class iMatrix {
 
   // get double precision version
   typedef iMatrix<typename GridTypeMapper<vtype>::DoublePrecision, N> DoublePrecision;
-
+  
-
   // Tensor removal
   typedef iScalar<tensor_reduced_v> tensor_reduced;
   typedef iMatrix<recurse_scalar_object, N> scalar_object;
@@ -390,25 +355,6 @@ class iMatrix {
     return *this;
   }
 
-
-  strong_inline scalar_object getlane(int lane){
-    scalar_object ret;
-    for (int i = 0; i < N; i++) {
-      for (int j = 0; j < N; j++) {
-	ret._internal[i][j] = _internal[i][j].getlane(lane);
-      }
-    }
-    return ret;
-  }
-
-  strong_inline void putlane(scalar_object &s, int lane){
-    for (int i = 0; i < N; i++)
-      for (int j = 0; j < N; j++) _internal[i][j].putlane(s._internal[i][j],lane);
-  }
-  
-
-  
-
   friend strong_inline void zeroit(iMatrix<vtype,N> &that){
     for(int i=0;i<N;i++){
       for(int j=0;j<N;j++){
@@ -527,3 +473,6 @@ void vprefetch(const iMatrix<v, N> &vv) {
 }
 }
 #endif
+
+
+

lib/tensors/Tensor_inner.h

@@ -29,51 +29,109 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_MATH_INNER_H
 #define GRID_MATH_INNER_H
 namespace Grid {
-    ///////////////////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////////////////////////////////////////////////////////////
-    // innerProduct Scalar x Scalar -> Scalar
+  // innerProduct Scalar x Scalar -> Scalar
-    // innerProduct Vector x Vector -> Scalar
+  // innerProduct Vector x Vector -> Scalar
-    // innerProduct Matrix x Matrix -> Scalar
+  // innerProduct Matrix x Matrix -> Scalar
-    ///////////////////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////////////////////////////////////////////////////////////
-    template<class sobj> inline RealD norm2(const sobj &arg){
+  template<class sobj> inline RealD norm2(const sobj &arg){
-      typedef typename sobj::scalar_type scalar;
+    auto nrm = innerProductD(arg,arg);
-      decltype(innerProduct(arg,arg)) nrm;
+    RealD ret = real(nrm);
-      nrm = innerProduct(arg,arg);
+    return ret;
-      RealD ret = real(nrm);
+  }
-      return ret;
+  //////////////////////////////////////
-    }
+  // If single promote to double and sum 2x
+  //////////////////////////////////////
 
-    template<class l,class r,int N> inline
+inline ComplexD innerProductD(const ComplexF &l,const ComplexF &r){  return innerProduct(l,r); }
-    auto innerProduct (const iVector<l,N>& lhs,const iVector<r,N>& rhs) -> iScalar<decltype(innerProduct(lhs._internal[0],rhs._internal[0]))>
+inline ComplexD innerProductD(const ComplexD &l,const ComplexD &r){  return innerProduct(l,r); }
-    {
+inline RealD    innerProductD(const RealD    &l,const RealD    &r){  return innerProduct(l,r); }
-        typedef decltype(innerProduct(lhs._internal[0],rhs._internal[0])) ret_t;
+inline RealD    innerProductD(const RealF    &l,const RealF    &r){  return innerProduct(l,r); }
-        iScalar<ret_t> ret;
+
-	ret=zero;
+inline vComplexD innerProductD(const vComplexD &l,const vComplexD &r){  return innerProduct(l,r); }
-        for(int c1=0;c1<N;c1++){
+inline vRealD    innerProductD(const vRealD    &l,const vRealD    &r){  return innerProduct(l,r); }
-            ret._internal += innerProduct(lhs._internal[c1],rhs._internal[c1]);
+inline vComplexD innerProductD(const vComplexF &l,const vComplexF &r){  
-        }
+  vComplexD la,lb;
-        return ret;
+  vComplexD ra,rb;
+  Optimization::PrecisionChange::StoD(l.v,la.v,lb.v);
+  Optimization::PrecisionChange::StoD(r.v,ra.v,rb.v);
+  return innerProduct(la,ra) + innerProduct(lb,rb); 
+}
+inline vRealD innerProductD(const vRealF &l,const vRealF &r){  
+  vRealD la,lb;
+  vRealD ra,rb;
+  Optimization::PrecisionChange::StoD(l.v,la.v,lb.v);
+  Optimization::PrecisionChange::StoD(r.v,ra.v,rb.v);
+  return innerProduct(la,ra) + innerProduct(lb,rb); 
+}
+
+  template<class l,class r,int N> inline
+  auto innerProductD (const iVector<l,N>& lhs,const iVector<r,N>& rhs) -> iScalar<decltype(innerProductD(lhs._internal[0],rhs._internal[0]))>
+  {
+    typedef decltype(innerProductD(lhs._internal[0],rhs._internal[0])) ret_t;
+    iScalar<ret_t> ret;
+    ret=zero;
+    for(int c1=0;c1<N;c1++){
+      ret._internal += innerProductD(lhs._internal[c1],rhs._internal[c1]);
     }
-    template<class l,class r,int N> inline
+    return ret;
-    auto innerProduct (const iMatrix<l,N>& lhs,const iMatrix<r,N>& rhs) -> iScalar<decltype(innerProduct(lhs._internal[0][0],rhs._internal[0][0]))>
+  }
-    {
+  template<class l,class r,int N> inline
-        typedef decltype(innerProduct(lhs._internal[0][0],rhs._internal[0][0])) ret_t;
+  auto innerProductD (const iMatrix<l,N>& lhs,const iMatrix<r,N>& rhs) -> iScalar<decltype(innerProductD(lhs._internal[0][0],rhs._internal[0][0]))>
-        iScalar<ret_t> ret;
+  {
-        iScalar<ret_t> tmp;
+    typedef decltype(innerProductD(lhs._internal[0][0],rhs._internal[0][0])) ret_t;
-	ret=zero;
+    iScalar<ret_t> ret;
-        for(int c1=0;c1<N;c1++){
+    iScalar<ret_t> tmp;
-        for(int c2=0;c2<N;c2++){
+    ret=zero;
-	  ret._internal+=innerProduct(lhs._internal[c1][c2],rhs._internal[c1][c2]);
+    for(int c1=0;c1<N;c1++){
-        }}
+    for(int c2=0;c2<N;c2++){
-        return ret;
+      ret._internal+=innerProductD(lhs._internal[c1][c2],rhs._internal[c1][c2]);
-    }
+    }}
-    template<class l,class r> inline
+    return ret;
-    auto innerProduct (const iScalar<l>& lhs,const iScalar<r>& rhs) -> iScalar<decltype(innerProduct(lhs._internal,rhs._internal))>
+  }
-    {
+  template<class l,class r> inline
-        typedef decltype(innerProduct(lhs._internal,rhs._internal)) ret_t;
+  auto innerProductD (const iScalar<l>& lhs,const iScalar<r>& rhs) -> iScalar<decltype(innerProductD(lhs._internal,rhs._internal))>
-        iScalar<ret_t> ret;
+  {
-        ret._internal = innerProduct(lhs._internal,rhs._internal);
+    typedef decltype(innerProductD(lhs._internal,rhs._internal)) ret_t;
-        return ret;
+    iScalar<ret_t> ret;
+    ret._internal = innerProductD(lhs._internal,rhs._internal);
+    return ret;
+  }
+  //////////////////////
+  // Keep same precison
+  //////////////////////
+  template<class l,class r,int N> inline
+  auto innerProduct (const iVector<l,N>& lhs,const iVector<r,N>& rhs) -> iScalar<decltype(innerProduct(lhs._internal[0],rhs._internal[0]))>
+  {
+    typedef decltype(innerProduct(lhs._internal[0],rhs._internal[0])) ret_t;
+    iScalar<ret_t> ret;
+    ret=zero;
+    for(int c1=0;c1<N;c1++){
+      ret._internal += innerProduct(lhs._internal[c1],rhs._internal[c1]);
     }
+    return ret;
+  }
+  template<class l,class r,int N> inline
+  auto innerProduct (const iMatrix<l,N>& lhs,const iMatrix<r,N>& rhs) -> iScalar<decltype(innerProduct(lhs._internal[0][0],rhs._internal[0][0]))>
+  {
+    typedef decltype(innerProduct(lhs._internal[0][0],rhs._internal[0][0])) ret_t;
+    iScalar<ret_t> ret;
+    iScalar<ret_t> tmp;
+    ret=zero;
+    for(int c1=0;c1<N;c1++){
+    for(int c2=0;c2<N;c2++){
+      ret._internal+=innerProduct(lhs._internal[c1][c2],rhs._internal[c1][c2]);
+    }}
+    return ret;
+  }
+  template<class l,class r> inline
+  auto innerProduct (const iScalar<l>& lhs,const iScalar<r>& rhs) -> iScalar<decltype(innerProduct(lhs._internal,rhs._internal))>
+  {
+    typedef decltype(innerProduct(lhs._internal,rhs._internal)) ret_t;
+    iScalar<ret_t> ret;
+    ret._internal = innerProduct(lhs._internal,rhs._internal);
+    return ret;
+  }
 
 }
 #endif

lib/tensors/Tensor_traits.h

@@ -1,29 +1,21 @@
-    /*************************************************************************************
+   /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./lib/tensors/Tensor_traits.h
-
     Copyright (C) 2015
-
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: Christopher Kelly <ckelly@phys.columbia.edu>
-
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -53,6 +45,7 @@ namespace Grid {
   public:
     typedef typename T::scalar_type scalar_type;
     typedef typename T::vector_type vector_type;
+    typedef typename T::vector_typeD vector_typeD;
     typedef typename T::tensor_reduced tensor_reduced;
     typedef typename T::scalar_object scalar_object;
     typedef typename T::Complexified Complexified;
@@ -68,6 +61,7 @@ namespace Grid {
   public:
     typedef RealF scalar_type;
     typedef RealF vector_type;
+    typedef RealD vector_typeD;
     typedef RealF tensor_reduced ;
     typedef RealF scalar_object;
     typedef ComplexF Complexified;
@@ -79,6 +73,7 @@ namespace Grid {
   public:
     typedef RealD scalar_type;
     typedef RealD vector_type;
+    typedef RealD vector_typeD;
     typedef RealD tensor_reduced;
     typedef RealD scalar_object;
     typedef ComplexD Complexified;
@@ -90,6 +85,7 @@ namespace Grid {
   public:
     typedef ComplexF scalar_type;
     typedef ComplexF vector_type;
+    typedef ComplexD vector_typeD;
     typedef ComplexF tensor_reduced;
     typedef ComplexF scalar_object;
     typedef ComplexF Complexified;
@@ -101,6 +97,7 @@ namespace Grid {
   public:
     typedef ComplexD scalar_type;
     typedef ComplexD vector_type;
+    typedef ComplexD vector_typeD;
     typedef ComplexD tensor_reduced;
     typedef ComplexD scalar_object;
     typedef ComplexD Complexified;
@@ -112,6 +109,7 @@ namespace Grid {
   public:
     typedef Integer scalar_type;
     typedef Integer vector_type;
+    typedef Integer vector_typeD;
     typedef Integer tensor_reduced;
     typedef Integer scalar_object;
     typedef void Complexified;
@@ -124,6 +122,7 @@ namespace Grid {
   public:
     typedef RealF  scalar_type;
     typedef vRealF vector_type;
+    typedef vRealD vector_typeD;
     typedef vRealF tensor_reduced;
     typedef RealF  scalar_object;
     typedef vComplexF Complexified;
@@ -135,6 +134,7 @@ namespace Grid {
   public:
     typedef RealD  scalar_type;
     typedef vRealD vector_type;
+    typedef vRealD vector_typeD;
     typedef vRealD tensor_reduced;
     typedef RealD  scalar_object;
     typedef vComplexD Complexified;
@@ -142,10 +142,23 @@ namespace Grid {
     typedef vRealD DoublePrecision;
     enum { TensorLevel = 0 };
   };
+  template<> class GridTypeMapper<vComplexH> {
+  public:
+    typedef ComplexF  scalar_type;
+    typedef vComplexH vector_type;
+    typedef vComplexD vector_typeD;
+    typedef vComplexH tensor_reduced;
+    typedef ComplexF  scalar_object;
+    typedef vComplexH Complexified;
+    typedef vRealH Realified;
+    typedef vComplexD DoublePrecision;
+    enum { TensorLevel = 0 };
+  };
   template<> class GridTypeMapper<vComplexF> {
   public:
     typedef ComplexF  scalar_type;
     typedef vComplexF vector_type;
+    typedef vComplexD vector_typeD;
     typedef vComplexF tensor_reduced;
     typedef ComplexF  scalar_object;
     typedef vComplexF Complexified;
@@ -157,6 +170,7 @@ namespace Grid {
   public:
     typedef ComplexD  scalar_type;
     typedef vComplexD vector_type;
+    typedef vComplexD vector_typeD;
     typedef vComplexD tensor_reduced;
     typedef ComplexD  scalar_object;
     typedef vComplexD Complexified;
@@ -168,6 +182,7 @@ namespace Grid {
   public:
     typedef  Integer scalar_type;
     typedef vInteger vector_type;
+    typedef vInteger vector_typeD;
     typedef vInteger tensor_reduced;
     typedef  Integer scalar_object;
     typedef void Complexified;
@@ -252,7 +267,8 @@ namespace Grid {
   template<typename T>
   class isSIMDvectorized{
     template<typename U>
-    static typename std::enable_if< !std::is_same< typename GridTypeMapper<typename getVectorType<U>::type>::scalar_type,   typename GridTypeMapper<typename getVectorType<U>::type>::vector_type>::value, char>::type test(void *);
+    static typename std::enable_if< !std::is_same< typename GridTypeMapper<typename getVectorType<U>::type>::scalar_type,   
+      typename GridTypeMapper<typename getVectorType<U>::type>::vector_type>::value, char>::type test(void *);
 
     template<typename U>
     static double test(...);
@@ -264,13 +280,15 @@ namespace Grid {
   //Get the precision of a Lattice, tensor or scalar type in units of sizeof(float)
   template<typename T>
   class getPrecision{
+  public:
     typedef typename getVectorType<T>::type vector_obj; //get the vector_obj (i.e. a grid Tensor) if its a Lattice<vobj>, do nothing otherwise (i.e. if fundamental or grid Tensor)
   
     typedef typename GridTypeMapper<vector_obj>::scalar_type scalar_type; //get the associated scalar type. Works on fundamental and tensor types
-  public:
     typedef typename GridTypeMapper<scalar_type>::Realified real_scalar_type; //remove any std::complex wrapper, should get us to the fundamental type
+
     enum { value = sizeof(real_scalar_type)/sizeof(float) };
   };
 }
 
 #endif
+

lib/util/Init.cc

@@ -311,8 +311,8 @@ void Grid_init(int *argc,char ***argv)
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"Performance:"<<std::endl;
     std::cout<<GridLogMessage<<std::endl;
-    std::cout<<GridLogMessage<<"  --comms-isend   : Asynchronous MPI calls; several dirs at a time "<<std::endl;    
+    std::cout<<GridLogMessage<<"  --comms-concurrent : Asynchronous MPI calls; several dirs at a time "<<std::endl;    
-    std::cout<<GridLogMessage<<"  --comms-sendrecv: Synchronous MPI calls; one dirs at a time "<<std::endl;    
+    std::cout<<GridLogMessage<<"  --comms-sequential : Synchronous MPI calls; one dirs at a time "<<std::endl;    
     std::cout<<GridLogMessage<<"  --comms-overlap : Overlap comms with compute "<<std::endl;    
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"  --dslash-generic: Wilson kernel for generic Nc"<<std::endl;    
@@ -464,5 +464,6 @@ void Grid_debug_handler_init(void)
 
   sigaction(SIGFPE,&sa,NULL);
   sigaction(SIGKILL,&sa,NULL);
+  sigaction(SIGILL,&sa,NULL);
 }
 }

scripts/configure.commit

@@ -16,4 +16,4 @@ else # configure.ac is modified
   printf 'commit: %s-dirty\n' `git rev-parse --short HEAD`
   printf 'branch: %s\n' `git rev-parse --abbrev-ref HEAD`
   printf 'date  : %s\n' `git log -1 --date=short --pretty=format:%cd`
- fi
+fi

scripts/grep-global

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export LANG=C
+find . -name "*.cc"  -exec grep -H $@ {} \;
+find . -name "*.h"   -exec grep -H $@ {} \;
+

tests/IO/Test_serialisation.cc

@@ -115,6 +115,7 @@ int main(int argc,char **argv)
   // test serializable class writing
   myclass              obj(1234); // non-trivial constructor
   std::vector<myclass> vec;
+  std::pair<myenum, myenum> pair;
   
   std::cout << "-- serialisable class writing to 'bother.xml'..." << std::endl;
   write(WR,"obj",obj);
@@ -122,6 +123,8 @@ int main(int argc,char **argv)
   vec.push_back(myclass(1234));
   vec.push_back(myclass(5678));
   vec.push_back(myclass(3838));
+  pair = std::make_pair(myenum::red, myenum::blue);
+
   write(WR, "objvec", vec);
   std::cout << "-- serialisable class writing to std::cout:" << std::endl;
   std::cout << obj << std::endl;
@@ -129,21 +132,30 @@ int main(int argc,char **argv)
   std::cout << "vec[0] == obj: " << ((vec[0] == obj) ? "true" : "false") << std::endl;
   std::cout << "vec[1] == obj: " << ((vec[1] == obj) ? "true" : "false") << std::endl;
   
+  write(WR, "objpair", pair);
+  std::cout << "-- pair writing to std::cout:" << std::endl;
+  std::cout << pair << std::endl;
+  
   // read tests
   std::cout << "\n==== IO self-consistency tests" << std::endl;
   //// XML
   ioTest<XmlWriter, XmlReader>("iotest.xml", obj, "XML    (object)           ");
   ioTest<XmlWriter, XmlReader>("iotest.xml", vec, "XML    (vector of objects)");
+  ioTest<XmlWriter, XmlReader>("iotest.xml", pair, "XML    (pair of objects)");
   //// binary
   ioTest<BinaryWriter, BinaryReader>("iotest.bin", obj, "binary (object)           ");
   ioTest<BinaryWriter, BinaryReader>("iotest.bin", vec, "binary (vector of objects)");
+  ioTest<BinaryWriter, BinaryReader>("iotest.bin", pair, "binary (pair of objects)");
   //// text
   ioTest<TextWriter, TextReader>("iotest.dat", obj, "text   (object)           ");
   ioTest<TextWriter, TextReader>("iotest.dat", vec, "text   (vector of objects)");
+  ioTest<TextWriter, TextReader>("iotest.dat", pair, "text   (pair of objects)");
   //// HDF5
+#undef HAVE_HDF5
 #ifdef HAVE_HDF5
   ioTest<Hdf5Writer, Hdf5Reader>("iotest.h5", obj, "HDF5   (object)           ");
   ioTest<Hdf5Writer, Hdf5Reader>("iotest.h5", vec, "HDF5   (vector of objects)");
+  ioTest<Hdf5Writer, Hdf5Reader>("iotest.h5", pair, "HDF5   (pair of objects)");
 #endif
   
   std::cout << "\n==== vector flattening/reconstruction" << std::endl;

tests/Test_simd.cc

@@ -308,18 +308,23 @@ public:
   int n;
   funcExchange(int _n) { n=_n;};
   template<class vec>    void operator()(vec &r1,vec &r2,vec &i1,vec &i2) const { exchange(r1,r2,i1,i2,n);}
-  template<class scal>   void apply(std::vector<scal> &r1,std::vector<scal> &r2,std::vector<scal> &in1,std::vector<scal> &in2)  const { 
+  template<class scal>   void apply(std::vector<scal> &r1,
+				    std::vector<scal> &r2,
+				    std::vector<scal> &in1,
+				    std::vector<scal> &in2)  const 
+  { 
     int sz=in1.size();
-
-    
     int msk = sz>>(n+1);
 
-    int j1=0;
+    for(int i=0;i<sz;i++) {
-    int j2=0;
+      int j1 = i&(~msk);
-    for(int i=0;i<sz;i++) if ( (i&msk) == 0 ) r1[j1++] = in1[ i ];
+      int j2 = i|msk;
-    for(int i=0;i<sz;i++) if ( (i&msk) == 0 ) r1[j1++] = in2[ i ];
+      if  ( (i&msk) == 0 ) { r1[i]=in1[j1];}
-    for(int i=0;i<sz;i++) if ( (i&msk)  ) r2[j2++] = in1[ i ];
+      else                 { r1[i]=in2[j1];}
-    for(int i=0;i<sz;i++) if ( (i&msk)  ) r2[j2++] = in2[ i ];
+
+      if  ( (i&msk) == 0 ) { r2[i]=in1[j2];}
+      else                 { r2[i]=in2[j2];}
+    }      
   }
   std::string name(void) const { return std::string("Exchange"); }
 };
@@ -454,8 +459,8 @@ void ExchangeTester(const functor &func)
 
   std::cout<<GridLogMessage << " " << func.name() << " " <<func.n <<std::endl;
 
-  //  for(int i=0;i<Nsimd;i++) std::cout << " i "<<i<<" "<<reference1[i]<<" "<<result1[i]<<std::endl;
+  //for(int i=0;i<Nsimd;i++) std::cout << " i "<<i<<" ref "<<reference1[i]<<" res "<<result1[i]<<std::endl;
-  //  for(int i=0;i<Nsimd;i++) std::cout << " i "<<i<<" "<<reference2[i]<<" "<<result2[i]<<std::endl;
+  //for(int i=0;i<Nsimd;i++) std::cout << " i "<<i<<" ref "<<reference2[i]<<" res "<<result2[i]<<std::endl;
 
   for(int i=0;i<Nsimd;i++){
     int found=0;
@@ -465,7 +470,7 @@ void ExchangeTester(const functor &func)
 	//	std::cout << " i "<<i<<" j "<<j<<" "<<reference1[j]<<" "<<result1[i]<<std::endl;
       }
     }
-    assert(found==1);
+    //    assert(found==1);
   }
   for(int i=0;i<Nsimd;i++){
     int found=0;
@@ -475,15 +480,24 @@ void ExchangeTester(const functor &func)
 	//	std::cout << " i "<<i<<" j "<<j<<" "<<reference2[j]<<" "<<result2[i]<<std::endl;
       }
     }
-    assert(found==1);
+    //    assert(found==1);
   }
 
+  /*
+  for(int i=0;i<Nsimd;i++){
+    std::cout << " i "<< i
+	      <<" result1  "<<result1[i]
+	      <<" result2  "<<result2[i]
+	      <<" test1  "<<test1[i]
+	      <<" test2  "<<test2[i]
+	      <<" input1 "<<input1[i]
+	      <<" input2 "<<input2[i]<<std::endl;
+  }
+  */
   for(int i=0;i<Nsimd;i++){
     assert(test1[i]==input1[i]);
     assert(test2[i]==input2[i]);
-  }//    std::cout << " i "<< i<<" test1"<<test1[i]<<" "<<input1[i]<<std::endl;
+  }
-    //    std::cout << " i "<< i<<" test2"<<test2[i]<<" "<<input2[i]<<std::endl;
-  //  }
 }
 
 
@@ -678,5 +692,69 @@ int main (int argc, char ** argv)
   IntTester(funcMinus());
   IntTester(funcTimes());
 
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing precisionChange            "<<  std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  {
+    GridSerialRNG          sRNG;
+    sRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
+    const int Ndp = 16;
+    const int Nsp = Ndp/2;
+    const int Nhp = Ndp/4;
+    std::vector<vRealH,alignedAllocator<vRealH> > H (Nhp);
+    std::vector<vRealF,alignedAllocator<vRealF> > F (Nsp);
+    std::vector<vRealF,alignedAllocator<vRealF> > FF(Nsp);
+    std::vector<vRealD,alignedAllocator<vRealD> > D (Ndp);
+    std::vector<vRealD,alignedAllocator<vRealD> > DD(Ndp);
+    for(int i=0;i<16;i++){
+      random(sRNG,D[i]);
+    }
+    // Double to Single
+    precisionChange(&F[0],&D[0],Ndp);
+    precisionChange(&DD[0],&F[0],Ndp);
+    std::cout << GridLogMessage<<"Double to single";
+    for(int i=0;i<Ndp;i++){
+      //      std::cout << "DD["<<i<<"] = "<< DD[i]<<" "<<D[i]<<" "<<DD[i]-D[i] <<std::endl; 
+      DD[i] = DD[i] - D[i];
+      decltype(innerProduct(DD[0],DD[0])) nrm;
+      nrm = innerProduct(DD[i],DD[i]);
+      auto tmp = Reduce(nrm);
+      //      std::cout << tmp << std::endl;
+      assert( tmp < 1.0e-14 ); 
+    }
+    std::cout <<" OK ! "<<std::endl;
+
+    // Double to Half
+#ifdef USE_FP16
+    std::cout << GridLogMessage<< "Double to half" ;
+    precisionChange(&H[0],&D[0],Ndp);
+    precisionChange(&DD[0],&H[0],Ndp);
+    for(int i=0;i<Ndp;i++){
+      //      std::cout << "DD["<<i<<"] = "<< DD[i]<<" "<<D[i]<<" "<<DD[i]-D[i]<<std::endl; 
+      DD[i] = DD[i] - D[i];
+      decltype(innerProduct(DD[0],DD[0])) nrm;
+      nrm = innerProduct(DD[i],DD[i]);
+      auto tmp = Reduce(nrm);
+      //      std::cout << tmp << std::endl;
+      assert( tmp < 1.0e-3 ); 
+    }
+    std::cout <<" OK ! "<<std::endl;
+
+    std::cout << GridLogMessage<< "Single to half";
+    // Single to Half
+    precisionChange(&H[0] ,&F[0],Nsp);
+    precisionChange(&FF[0],&H[0],Nsp);
+    for(int i=0;i<Nsp;i++){
+      //      std::cout << "FF["<<i<<"] = "<< FF[i]<<" "<<F[i]<<" "<<FF[i]-F[i]<<std::endl; 
+      FF[i] = FF[i] - F[i];
+      decltype(innerProduct(FF[0],FF[0])) nrm;
+      nrm = innerProduct(FF[i],FF[i]);
+      auto tmp = Reduce(nrm);
+      //      std::cout << tmp << std::endl;
+      assert( tmp < 1.0e-3 ); 
+    }
+    std::cout <<" OK ! "<<std::endl;
+#endif
+  }
   Grid_finalize();
 }

tests/core/Test_fft_gfix.cc

@@ -148,11 +148,13 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
     Complex psqMax(16.0);
     Fp =  psqMax*one/psq;
 
+    /*
     static int once;
     if ( once == 0 ) { 
       std::cout << " Fp " << Fp <<std::endl;
       once ++;
-    }
+      }*/
+
     pokeSite(TComplex(1.0),Fp,coor);
 
     dmuAmu_p  = dmuAmu_p * Fp; 

tests/core/Test_mobius_even_odd.cc

@@ -0,0 +1,287 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_dwf_even_odd.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+  d internal;
+};
+
+  Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+  };
+
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+
+  const int Ls=10;
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+
+  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+
+  LatticeFermion src   (FGrid); random(RNG5,src);
+  LatticeFermion phi   (FGrid); random(RNG5,phi);
+  LatticeFermion chi   (FGrid); random(RNG5,chi);
+  LatticeFermion result(FGrid); result=zero;
+  LatticeFermion    ref(FGrid);    ref=zero;
+  LatticeFermion    tmp(FGrid);    tmp=zero;
+  LatticeFermion    err(FGrid);    tmp=zero;
+  LatticeGaugeField Umu(UGrid); random(RNG4,Umu);
+  std::vector<LatticeColourMatrix> U(4,UGrid);
+
+  // Only one non-zero (y)
+  Umu=zero;
+  for(int nn=0;nn<Nd;nn++){
+    random(RNG4,U[nn]);
+    if ( nn>0 ) 
+      U[nn]=zero;
+    PokeIndex<LorentzIndex>(Umu,U[nn],nn);
+  }
+
+  RealD mass=0.1;
+  RealD M5  =1.8;
+  std::vector < std::complex<double>  > omegas;
+#if 0
+  for(int i=0;i<Ls;i++){
+    double imag = 0.;
+    if (i==0) imag=1.;
+    if (i==Ls-1) imag=-1.;
+    std::complex<double> temp (0.25+0.01*i, imag*0.01);
+    omegas.push_back(temp);
+  }
+#else
+  omegas.push_back( std::complex<double>(1.45806438985048,-0) );
+  omegas.push_back( std::complex<double>(1.18231318389348,-0) );
+  omegas.push_back( std::complex<double>(0.830951166685955,-0) );
+  omegas.push_back( std::complex<double>(0.542352409156791,-0) );
+  omegas.push_back( std::complex<double>(0.341985020453729,-0) );
+  omegas.push_back( std::complex<double>(0.21137902619029,-0) );
+  omegas.push_back( std::complex<double>(0.126074299502912,-0) );
+  omegas.push_back( std::complex<double>(0.0990136651962626,-0) );
+  omegas.push_back( std::complex<double>(0.0686324988446592,0.0550658530827402) );
+  omegas.push_back( std::complex<double>(0.0686324988446592,-0.0550658530827402) );
+#endif
+
+  MobiusFermionR Ddwf(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mass, M5, 0.5,0.5);
+  //  DomainWallFermionR Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+
+  LatticeFermion src_e (FrbGrid);
+  LatticeFermion src_o (FrbGrid);
+  LatticeFermion r_e   (FrbGrid);
+  LatticeFermion r_o   (FrbGrid);
+  LatticeFermion r_eo  (FGrid);
+  LatticeFermion r_eeoo(FGrid);
+
+  std::cout<<GridLogMessage<<"=========================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"= Testing that Meo + Moe + Moo + Mee = Munprec "<<std::endl;
+  std::cout<<GridLogMessage<<"=========================================================="<<std::endl;
+
+  pickCheckerboard(Even,src_e,src);
+  pickCheckerboard(Odd,src_o,src);
+
+  Ddwf.Meooe(src_e,r_o);  std::cout<<GridLogMessage<<"Applied Meo"<<std::endl;
+  Ddwf.Meooe(src_o,r_e);  std::cout<<GridLogMessage<<"Applied Moe"<<std::endl;
+  setCheckerboard(r_eo,r_o);
+  setCheckerboard(r_eo,r_e);
+
+  Ddwf.Mooee(src_e,r_e);  std::cout<<GridLogMessage<<"Applied Mee"<<std::endl;
+  Ddwf.Mooee(src_o,r_o);  std::cout<<GridLogMessage<<"Applied Moo"<<std::endl;
+  setCheckerboard(r_eeoo,r_e);
+  setCheckerboard(r_eeoo,r_o);
+
+  r_eo=r_eo+r_eeoo;
+  Ddwf.M(src,ref);  
+
+  //  std::cout<<GridLogMessage << r_eo<<std::endl;
+  //  std::cout<<GridLogMessage << ref <<std::endl;
+
+  err= ref - r_eo;
+  std::cout<<GridLogMessage << "EO norm diff   "<< norm2(err)<< " "<<norm2(ref)<< " " << norm2(r_eo) <<std::endl;
+    
+  LatticeComplex cerr(FGrid);
+  cerr = localInnerProduct(err,err);
+  //  std::cout<<GridLogMessage << cerr<<std::endl;
+
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"= Test MooeeDagger is the dagger of Mooee by requiring                "<<std::endl;
+  std::cout<<GridLogMessage<<"=  < phi | Deo | chi > * = < chi | Deo^dag| phi>  "<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+
+  LatticeFermion chi_e   (FrbGrid);
+  LatticeFermion chi_o   (FrbGrid);
+
+  LatticeFermion dchi_e  (FrbGrid);
+  LatticeFermion dchi_o  (FrbGrid);
+
+  LatticeFermion phi_e   (FrbGrid);
+  LatticeFermion phi_o   (FrbGrid);
+
+  LatticeFermion dphi_e  (FrbGrid);
+  LatticeFermion dphi_o  (FrbGrid);
+
+  pickCheckerboard(Even,chi_e,chi);
+  pickCheckerboard(Odd ,chi_o,chi);
+  pickCheckerboard(Even,phi_e,phi);
+  pickCheckerboard(Odd ,phi_o,phi);
+
+  Ddwf.Mooee(chi_e,dchi_o);
+  Ddwf.Mooee(chi_o,dchi_e);
+  Ddwf.MooeeDag(phi_e,dphi_o);
+  Ddwf.MooeeDag(phi_o,dphi_e);
+
+  ComplexD pDce = innerProduct(phi_e,dchi_e);
+  ComplexD pDco = innerProduct(phi_o,dchi_o);
+  ComplexD cDpe = innerProduct(chi_e,dphi_e);
+  ComplexD cDpo = innerProduct(chi_o,dphi_o);
+
+
+  std::cout<<GridLogMessage <<"e "<<pDce<<" "<<cDpe <<std::endl;
+  std::cout<<GridLogMessage <<"o "<<pDco<<" "<<cDpo <<std::endl;
+
+  std::cout<<GridLogMessage <<"pDce - conj(cDpo) "<< pDce-conj(cDpo) <<std::endl;
+  std::cout<<GridLogMessage <<"pDco - conj(cDpe) "<< pDco-conj(cDpe) <<std::endl;
+
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"= Test Ddagger is the dagger of D by requiring                "<<std::endl;
+  std::cout<<GridLogMessage<<"=  < phi | Deo | chi > * = < chi | Deo^dag| phi>  "<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  
+
+  pickCheckerboard(Even,chi_e,chi);
+  pickCheckerboard(Odd ,chi_o,chi);
+  pickCheckerboard(Even,phi_e,phi);
+  pickCheckerboard(Odd ,phi_o,phi);
+
+  Ddwf.Meooe(chi_e,dchi_o);
+  Ddwf.Meooe(chi_o,dchi_e);
+  Ddwf.MeooeDag(phi_e,dphi_o);
+  Ddwf.MeooeDag(phi_o,dphi_e);
+
+  pDce = innerProduct(phi_e,dchi_e);
+  pDco = innerProduct(phi_o,dchi_o);
+  cDpe = innerProduct(chi_e,dphi_e);
+  cDpo = innerProduct(chi_o,dphi_o);
+
+  std::cout<<GridLogMessage <<"e "<<pDce<<" "<<cDpe <<std::endl;
+  std::cout<<GridLogMessage <<"o "<<pDco<<" "<<cDpo <<std::endl;
+
+  std::cout<<GridLogMessage <<"pDce - conj(cDpo) "<< pDce-conj(cDpo) <<std::endl;
+  std::cout<<GridLogMessage <<"pDco - conj(cDpe) "<< pDco-conj(cDpe) <<std::endl;
+
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"= Test MeeInv Mee = 1                                         "<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+
+  pickCheckerboard(Even,chi_e,chi);
+  pickCheckerboard(Odd ,chi_o,chi);
+
+  Ddwf.Mooee(chi_e,src_e);
+  Ddwf.MooeeInv(src_e,phi_e);
+
+  Ddwf.Mooee(chi_o,src_o);
+  Ddwf.MooeeInv(src_o,phi_o);
+  
+  setCheckerboard(phi,phi_e);
+  setCheckerboard(phi,phi_o);
+
+  err = phi-chi;
+  std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<< std::endl;
+
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"= Test MeeInvDag MeeDag = 1                                   "<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+
+  pickCheckerboard(Even,chi_e,chi);
+  pickCheckerboard(Odd ,chi_o,chi);
+
+  Ddwf.MooeeDag(chi_e,src_e);
+  Ddwf.MooeeInvDag(src_e,phi_e);
+
+  Ddwf.MooeeDag(chi_o,src_o);
+  Ddwf.MooeeInvDag(src_o,phi_o);
+  
+  setCheckerboard(phi,phi_e);
+  setCheckerboard(phi,phi_o);
+
+  err = phi-chi;
+  std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<< std::endl;
+
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"= Test MpcDagMpc is Hermitian              "<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  
+  random(RNG5,phi);
+  random(RNG5,chi);
+  pickCheckerboard(Even,chi_e,chi);
+  pickCheckerboard(Odd ,chi_o,chi);
+  pickCheckerboard(Even,phi_e,phi);
+  pickCheckerboard(Odd ,phi_o,phi);
+  RealD t1,t2;
+
+
+  SchurDiagMooeeOperator<MobiusFermionR,LatticeFermion> HermOpEO(Ddwf);
+  HermOpEO.MpcDagMpc(chi_e,dchi_e,t1,t2);
+  HermOpEO.MpcDagMpc(chi_o,dchi_o,t1,t2);
+
+  HermOpEO.MpcDagMpc(phi_e,dphi_e,t1,t2);
+  HermOpEO.MpcDagMpc(phi_o,dphi_o,t1,t2);
+
+  pDce = innerProduct(phi_e,dchi_e);
+  pDco = innerProduct(phi_o,dchi_o);
+  cDpe = innerProduct(chi_e,dphi_e);
+  cDpo = innerProduct(chi_o,dphi_o);
+
+  std::cout<<GridLogMessage <<"e "<<pDce<<" "<<cDpe <<std::endl;
+  std::cout<<GridLogMessage <<"o "<<pDco<<" "<<cDpo <<std::endl;
+
+  std::cout<<GridLogMessage <<"pDce - conj(cDpo) "<< pDco-conj(cDpo) <<std::endl;
+  std::cout<<GridLogMessage <<"pDco - conj(cDpe) "<< pDce-conj(cDpe) <<std::endl;
+  
+  Grid_finalize();
+}

tests/core/Test_wilson_even_odd.cc

@@ -2,11 +2,10 @@
 
     Grid physics library, www.github.com/paboyle/Grid 
 
-    Source file: ./tests/Test_wilson_even_odd.cc
+    Source file: ./tests/Test_wilson_tm_even_odd.cc
 
     Copyright (C) 2015
 
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
@@ -89,8 +88,8 @@ int main (int argc, char ** argv)
   }
 
   RealD mass=0.1;
-  RealD mu  = 0.1;
+
-  WilsonTMFermionR Dw(Umu,Grid,RBGrid,mass,mu);
+  WilsonFermionR Dw(Umu,Grid,RBGrid,mass);
 
   LatticeFermion src_e   (&RBGrid);
   LatticeFermion src_o   (&RBGrid);
@@ -207,7 +206,7 @@ int main (int argc, char ** argv)
   pickCheckerboard(Odd ,phi_o,phi);
   RealD t1,t2;
 
-  SchurDiagMooeeOperator<WilsonTMFermionR,LatticeFermion> HermOpEO(Dw);
+  SchurDiagMooeeOperator<WilsonFermionR,LatticeFermion> HermOpEO(Dw);
   HermOpEO.MpcDagMpc(chi_e,dchi_e,t1,t2);
   HermOpEO.MpcDagMpc(chi_o,dchi_o,t1,t2);
 

tests/core/Test_wilson_twisted_mass_even_odd.cc

@@ -2,10 +2,11 @@
 
     Grid physics library, www.github.com/paboyle/Grid 
 
-    Source file: ./tests/Test_wilson_tm_even_odd.cc
+    Source file: ./tests/Test_wilson_even_odd.cc
 
     Copyright (C) 2015
 
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
@@ -88,8 +89,8 @@ int main (int argc, char ** argv)
   }
 
   RealD mass=0.1;
-
+  RealD mu  = 0.1;
-  WilsonFermionR Dw(Umu,Grid,RBGrid,mass);
+  WilsonTMFermionR Dw(Umu,Grid,RBGrid,mass,mu);
 
   LatticeFermion src_e   (&RBGrid);
   LatticeFermion src_o   (&RBGrid);
@@ -206,7 +207,7 @@ int main (int argc, char ** argv)
   pickCheckerboard(Odd ,phi_o,phi);
   RealD t1,t2;
 
-  SchurDiagMooeeOperator<WilsonFermionR,LatticeFermion> HermOpEO(Dw);
+  SchurDiagMooeeOperator<WilsonTMFermionR,LatticeFermion> HermOpEO(Dw);
   HermOpEO.MpcDagMpc(chi_e,dchi_e,t1,t2);
   HermOpEO.MpcDagMpc(chi_o,dchi_o,t1,t2);
 

tests/debug/Test_synthetic_lanczos.cc

@@ -115,8 +115,8 @@ int main (int argc, char ** argv)
   RNG.SeedFixedIntegers(seeds);
 
 
-  RealD alpha = 1.0;
+  RealD alpha = 1.2;
-  RealD beta  = 0.03;
+  RealD beta  = 0.1;
   RealD mu    = 0.0;
   int order = 11;
   ChebyshevLanczos<LatticeComplex> Cheby(alpha,beta,mu,order);
@@ -131,10 +131,9 @@ int main (int argc, char ** argv)
   const int Nit= 10000;
 
   int Nconv;
-  RealD eresid = 1.0e-8;
+  RealD eresid = 1.0e-6;
 
   ImplicitlyRestartedLanczos<LatticeComplex> IRL(HermOp,X,Nk,Nm,eresid,Nit);
-
   ImplicitlyRestartedLanczos<LatticeComplex> ChebyIRL(HermOp,Cheby,Nk,Nm,eresid,Nit);
 
   LatticeComplex src(grid); gaussian(RNG,src);
@@ -145,9 +144,9 @@ int main (int argc, char ** argv)
   }
   
   {
-    //    std::vector<RealD>          eval(Nm);
+    std::vector<RealD>          eval(Nm);
-    //    std::vector<LatticeComplex> evec(Nm,grid);
+    std::vector<LatticeComplex> evec(Nm,grid);
-    //    ChebyIRL.calc(eval,evec,src, Nconv);
+    ChebyIRL.calc(eval,evec,src, Nconv);
   }
 
   Grid_finalize();