Merge branch 'develop' into feature/doxygen

# Conflicts:
#	configure.ac

.gitignore

@@ -95,6 +95,10 @@ build.sh
 ################
 lib/Eigen/*
 
+# FFTW source #
+################
+lib/fftw/*
+
 # libtool macros #
 ##################
 m4/lt*

.travis.yml

@@ -23,6 +23,8 @@ matrix:
             - libmpfr-dev
             - libgmp-dev
             - libmpc-dev
+            - libopenmpi-dev
+            - openmpi-bin
             - binutils-dev
       env: VERSION=-4.9
     - compiler: gcc
@@ -35,6 +37,8 @@ matrix:
             - libmpfr-dev
             - libgmp-dev
             - libmpc-dev
+            - libopenmpi-dev
+            - openmpi-bin
             - binutils-dev
       env: VERSION=-5
     - compiler: clang
@@ -47,6 +51,8 @@ matrix:
             - libmpfr-dev
             - libgmp-dev
             - libmpc-dev
+            - libopenmpi-dev
+            - openmpi-bin
             - binutils-dev
       env: CLANG_LINK=http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
     - compiler: clang
@@ -59,6 +65,8 @@ matrix:
             - libmpfr-dev
             - libgmp-dev
             - libmpc-dev
+            - libopenmpi-dev
+            - openmpi-bin
             - binutils-dev
       env: CLANG_LINK=http://llvm.org/releases/3.7.0/clang+llvm-3.7.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
       
@@ -69,6 +77,7 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openmpi; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]] && [[ "$CC" == "gcc" ]]; then brew install gcc5; fi
     
 install:
@@ -92,3 +101,10 @@ script:
     - ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none
     - make -j4
     - ./benchmarks/Benchmark_dwf --threads 1
+    - echo make clean
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXXFLAGS='-DMPI_UINT32_T=MPI_UNSIGNED -DMPI_UINT64_T=MPI_UNSIGNED_LONG'; fi
+    - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto
+    - make -j4
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then mpirun -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
+

benchmarks/Benchmark_dwf.cc

@@ -86,18 +86,6 @@ int main (int argc, char ** argv)
   LatticeFermion    tmp(FGrid);
   LatticeFermion    err(FGrid);
 
-  /*  src=zero;
-  std::vector<int> origin(5,0);
-  SpinColourVector f=zero;
-  for(int sp=0;sp<4;sp++){
-  for(int co=0;co<3;co++){
-    f()(sp)(co)=Complex(1.0,0.0); 
-  }}
-  pokeSite(f,src,origin);
-  */
-
-  ColourMatrix cm = Complex(1.0,0.0);
-
   LatticeGaugeField Umu(UGrid); 
   random(RNG4,Umu);
 
@@ -144,10 +132,12 @@ int main (int argc, char ** argv)
 
   DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
   
+  std::cout<<GridLogMessage << "Naive wilson implementation "<<std::endl;
   std::cout << GridLogMessage<< "Calling Dw"<<std::endl;
   int ncall =100;
   if (1) {
 
+    Dw.ZeroCounters();
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       __SSC_START;
@@ -166,7 +156,7 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-    //    Dw.Report();
+    Dw.Report();
   }
 
   if (1)
@@ -188,8 +178,9 @@ int main (int argc, char ** argv)
       peekSite(tmp,src,site);
       pokeSite(tmp,ssrc,site);
     }}}}}
-    std::cout<<"src norms "<< norm2(src)<<" " <<norm2(ssrc)<<std::endl;
+    std::cout<<GridLogMessage<< "src norms "<< norm2(src)<<" " <<norm2(ssrc)<<std::endl;
     double t0=usecond();
+    sDw.ZeroCounters();
     for(int i=0;i<ncall;i++){
       __SSC_START;
       sDw.Dhop(ssrc,sresult,0);
@@ -199,10 +190,10 @@ int main (int argc, char ** argv)
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
     double flops=1344*volume*ncall;
 
-    std::cout<<GridLogMessage << "Called Dw sinner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NP<<std::endl;
-    //  sDw.Report();
+    sDw.Report();
   
     if(0){
       for(int i=0;i< PerformanceCounter::NumTypes(); i++ ){
@@ -215,7 +206,7 @@ int main (int argc, char ** argv)
       }
     }
 
-    std::cout<<"res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
+    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
 
 
     RealF sum=0;
@@ -235,7 +226,7 @@ int main (int argc, char ** argv)
   std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" simd   "<<simd<<std::endl;
       }
     }}}}}
-    std::cout<<" difference between normal and simd is "<<sum<<std::endl;
+    std::cout<<GridLogMessage<<" difference between normal and simd is "<<sum<<std::endl;
 
 
     if (1) {
@@ -259,6 +250,7 @@ int main (int argc, char ** argv)
       sr_e = zero;
       sr_o = zero;
 
+      sDw.ZeroCounters();
       double t0=usecond();
       for (int i = 0; i < ncall; i++) {
         sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
@@ -270,6 +262,7 @@ int main (int argc, char ** argv)
 
       std::cout<<GridLogMessage << "sDeo mflop/s =   "<< flops/(t1-t0)<<std::endl;
       std::cout<<GridLogMessage << "sDeo mflop/s per node   "<< flops/(t1-t0)/NP<<std::endl;
+      sDw.Report();
 
       sDw.DhopEO(ssrc_o,sr_e,DaggerNo);
       sDw.DhopOE(ssrc_e,sr_o,DaggerNo);
@@ -306,6 +299,7 @@ int main (int argc, char ** argv)
     ref = -0.5*ref;
   }
   Dw.Dhop(src,result,1);
+  std::cout << GridLogMessage << "Naive wilson implementation Dag" << std::endl;
   std::cout<<GridLogMessage << "Called DwDag"<<std::endl;
   std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
   std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
@@ -327,6 +321,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "src_o"<<norm2(src_o)<<std::endl;
 
   {
+    Dw.ZeroCounters();
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       Dw.DhopEO(src_o,r_e,DaggerNo);
@@ -338,6 +333,7 @@ int main (int argc, char ** argv)
 
     std::cout<<GridLogMessage << "Deo mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "Deo mflop/s per node   "<< flops/(t1-t0)/NP<<std::endl;
+    Dw.Report();
   }
   Dw.DhopEO(src_o,r_e,DaggerNo);
   Dw.DhopOE(src_e,r_o,DaggerNo);

bootstrap.sh

@@ -1,11 +1,18 @@
 #!/usr/bin/env bash
 
 EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2'
+FFTW_URL=http://www.fftw.org/fftw-3.3.4.tar.gz
 
 echo "-- deploying Eigen source..."
-wget ${EIGEN_URL}
+wget ${EIGEN_URL} --no-check-certificate
 ./scripts/update_eigen.sh `basename ${EIGEN_URL}`
 rm `basename ${EIGEN_URL}`
+
+echo "-- copying fftw prototypes..."
+wget ${FFTW_URL}
+./scripts/update_fftw.sh `basename ${FFTW_URL}`
+rm `basename ${FFTW_URL}`
+
 echo '-- generating Make.inc files...'
 ./scripts/filelist
 echo '-- generating configure script...'

configure.ac

@@ -1,18 +1,31 @@
 AC_PREREQ([2.63])
 AC_INIT([Grid], [0.5.1-dev], [https://github.com/paboyle/Grid], [Grid])
+AC_CANONICAL_BUILD
+AC_CANONICAL_HOST
+AC_CANONICAL_TARGET
 AM_INIT_AUTOMAKE(subdir-objects)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
+
 ############### Checks for programs
 AC_LANG(C++)
-: ${CXXFLAGS="-O3"}
+CXXFLAGS="-O3 $CXXFLAGS"
 AC_PROG_CXX
+AC_PROG_RANLIB
+
+############ openmp  ###############
 AC_OPENMP
+
+ac_openmp=no
+
+if test "${OPENMP_CXXFLAGS}X" != "X"; then
+ac_openmp=yes
 AM_CXXFLAGS="$OPENMP_CXXFLAGS $AM_CXXFLAGS"
-LT_INIT([disable-shared])
+AM_LDFLAGS="$OPENMP_CXXFLAGS $AM_LDFLAGS"
+fi
 
 ############### Checks for header files
 AC_CHECK_HEADERS(stdint.h)
@@ -29,20 +42,23 @@ AC_TYPE_SIZE_T
 AC_TYPE_UINT32_T
 AC_TYPE_UINT64_T
 
-############### Options
+############### GMP and MPFR #################
 AC_ARG_WITH([gmp],
     [AS_HELP_STRING([--with-gmp=prefix],
     [try this for a non-standard install prefix of the GMP library])],
     [AM_CXXFLAGS="-I$with_gmp/include $AM_CXXFLAGS"]
-    [AM_LDFLAGS="-L$with_gmp/lib" $AM_LDFLAGS])
+    [AM_LDFLAGS="-L$with_gmp/lib $AM_LDFLAGS"])
 AC_ARG_WITH([mpfr],
     [AS_HELP_STRING([--with-mpfr=prefix],
     [try this for a non-standard install prefix of the MPFR library])],
     [AM_CXXFLAGS="-I$with_mpfr/include $AM_CXXFLAGS"]
     [AM_LDFLAGS="-L$with_mpfr/lib $AM_LDFLAGS"])
+
+################## lapack ####################
 AC_ARG_ENABLE([lapack],
     [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])], 
     [ac_LAPACK=${enable_lapack}],[ac_LAPACK=no])
+
 case ${ac_LAPACK} in
     no)
         ;;
@@ -54,6 +70,13 @@ case ${ac_LAPACK} in
         AC_DEFINE([USE_LAPACK],[1],[use LAPACK])
 esac
 
+################## FFTW3 ####################
+AC_ARG_WITH([fftw],    
+            [AS_HELP_STRING([--with-fftw=prefix],
+            [try this for a non-standard install prefix of the FFTW3 library])],
+            [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
+            [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
+
 ################ Get compiler informations
 AC_LANG([C++])
 AX_CXX_COMPILE_STDCXX_11([noext],[mandatory])
@@ -67,7 +90,6 @@ AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
 ############### Checks for library functions
 CXXFLAGS_CPY=$CXXFLAGS
 LDFLAGS_CPY=$LDFLAGS
-LIBS_CPY=$LIBS
 CXXFLAGS="$AM_CXXFLAGS $CXXFLAGS"
 LDFLAGS="$AM_LDFLAGS $LDFLAGS"
 AC_CHECK_FUNCS([gettimeofday])
@@ -86,6 +108,11 @@ if test "${ac_LAPACK}x" != "nox"; then
     AC_CHECK_LIB([lapack],[LAPACKE_sbdsdc],[],
                  [AC_MSG_ERROR("LAPACK enabled but library not found")])
 fi
+AC_CHECK_LIB([fftw3],[fftw_execute],
+  [AC_DEFINE([HAVE_FFTW],[1],[Define to 1 if you have the `FFTW' library (-lfftw3).])]
+  [have_fftw=true]
+  [LIBS="$LIBS -lfftw3 -lfftw3f"],
+  [AC_MSG_WARN([**** FFTW library not found, Grid can still compile but FFT-based routines will not work ****])])
 CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
@@ -113,11 +140,14 @@ case ${ax_cv_cxx_compiler_vendor} in
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
       IMCI|KNC)
-        AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner])
+        AC_DEFINE([IMCI],[1],[IMCI intrinsics for Knights Corner])
         SIMD_FLAGS='';;
       GEN)
         AC_DEFINE([GENERIC_VEC],[1],[generic vector code])
         SIMD_FLAGS='';;
+      QPX|BGQ)
+        AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q])
+        SIMD_FLAGS='';;
       *)
         AC_MSG_ERROR(["SIMD option ${ac_SIMD} not supported by the GCC/Clang compiler"]);;
     esac;;
@@ -131,7 +161,7 @@ case ${ax_cv_cxx_compiler_vendor} in
         SIMD_FLAGS='-mavx -xavx';;
       AVXFMA4)
         AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
-        SIMD_FLAGS='-mavx -xavx -mfma';;
+        SIMD_FLAGS='-mavx -mfma';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
         SIMD_FLAGS='-march=core-avx2 -xcore-avx2';;
@@ -297,18 +327,20 @@ Summary of configuration for $PACKAGE v$VERSION
 - compiler version              : ${ax_cv_gxx_version}
 ----- BUILD OPTIONS -----------------------------------
 - SIMD                          : ${ac_SIMD}
-- communications type           : ${ac_COMMS}
-- default precision             : ${ac_PRECISION}
+- Threading                     : ${ac_openmp} 
+- Communications type           : ${ac_COMMS}
+- Default precision             : ${ac_PRECISION}
 - RNG choice                    : ${ac_RNG} 
 - GMP                           : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
 - LAPACK                        : ${ac_LAPACK}
+- FFTW                          : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
 - build DOXYGEN documentation   : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
 - CXXFLAGS:
-`echo ${AM_CXXFLAGS} ${CXXFLAGS} | sed 's/ -/\n\t-/g' | sed 's/^-/\t-/g'`
+`echo ${AM_CXXFLAGS} ${CXXFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
 - LDFLAGS:
-`echo ${AM_LDFLAGS} ${LDFLAGS} | sed 's/ -/\n\t-/g' | sed 's/^-/\t-/g'`
+`echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
 - LIBS:
-`echo ${LIBS} | sed 's/ -/\n\t-/g' | sed 's/^-/\t-/g'`
+`echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
 -------------------------------------------------------
 "

lib/FFT.h

@@ -0,0 +1,276 @@
+
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/Cshift.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef _GRID_FFT_H_
+#define _GRID_FFT_H_
+
+#ifdef HAVE_FFTW	
+#include <fftw3.h>
+#endif
+namespace Grid {
+
+  template<class scalar> struct FFTW { };
+
+#ifdef HAVE_FFTW	
+  template<> struct FFTW<ComplexD> {
+  public:
+
+    typedef fftw_complex FFTW_scalar;
+    typedef fftw_plan    FFTW_plan;
+
+    static FFTW_plan fftw_plan_many_dft(int rank, const int *n,int howmany,
+					FFTW_scalar *in, const int *inembed,		
+					int istride, int idist,		
+					FFTW_scalar *out, const int *onembed,		
+					int ostride, int odist,		
+					int sign, unsigned flags) {
+      return ::fftw_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
+    }	  
+    
+    static void fftw_flops(const FFTW_plan p,double *add, double *mul, double *fmas){
+      ::fftw_flops(p,add,mul,fmas);
+    }
+
+    inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out) {
+      ::fftw_execute_dft(p,in,out);
+    }
+    inline static void fftw_destroy_plan(const FFTW_plan p) {
+      ::fftw_destroy_plan(p);
+    }
+  };
+
+  template<> struct FFTW<ComplexF> {
+  public:
+
+    typedef fftwf_complex FFTW_scalar;
+    typedef fftwf_plan    FFTW_plan;
+
+    static FFTW_plan fftw_plan_many_dft(int rank, const int *n,int howmany,
+					FFTW_scalar *in, const int *inembed,		
+					int istride, int idist,		
+					FFTW_scalar *out, const int *onembed,		
+					int ostride, int odist,		
+					int sign, unsigned flags) {
+      return ::fftwf_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
+    }	  
+    
+    static void fftw_flops(const FFTW_plan p,double *add, double *mul, double *fmas){
+      ::fftwf_flops(p,add,mul,fmas);
+    }
+
+    inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out) {
+      ::fftwf_execute_dft(p,in,out);
+    }
+    inline static void fftw_destroy_plan(const FFTW_plan p) {
+      ::fftwf_destroy_plan(p);
+    }
+  };
+
+#endif
+
+#ifndef FFTW_FORWARD
+#define FFTW_FORWARD (-1)
+#define FFTW_BACKWARD (+1)
+#endif
+
+  class FFT { 
+  private:
+
+    GridCartesian *vgrid;
+    GridCartesian *sgrid;
+
+    int Nd;
+    double flops;
+    double flops_call;
+    uint64_t usec;
+
+    std::vector<int> dimensions;
+    std::vector<int> processors;
+    std::vector<int> processor_coor;
+
+  public:
+
+    static const int forward=FFTW_FORWARD;
+    static const int backward=FFTW_BACKWARD;
+
+    double Flops(void) {return flops;}
+    double MFlops(void) {return flops/usec;}
+
+    FFT ( GridCartesian * grid ) : 
+      vgrid(grid),
+      Nd(grid->_ndimension),
+      dimensions(grid->_fdimensions),
+      processors(grid->_processors),
+      processor_coor(grid->_processor_coor)
+    {
+      flops=0;
+      usec =0;
+      std::vector<int> layout(Nd,1);
+      sgrid = new GridCartesian(dimensions,layout,processors);
+    };
+
+    ~FFT ( void)  { 
+      delete sgrid; 
+    }
+    
+    template<class vobj>
+    void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int inverse){
+
+      conformable(result._grid,vgrid);
+      conformable(source._grid,vgrid);
+
+      int L = vgrid->_ldimensions[dim];
+      int G = vgrid->_fdimensions[dim];
+
+      std::vector<int> layout(Nd,1);
+      std::vector<int> pencil_gd(vgrid->_fdimensions);
+
+      pencil_gd[dim] = G*processors[dim];    
+
+      // Pencil global vol LxLxGxLxL per node
+      GridCartesian pencil_g(pencil_gd,layout,processors);
+
+      // Construct pencils
+      typedef typename vobj::scalar_object sobj;
+      typedef typename sobj::scalar_type   scalar;
+
+      Lattice<vobj> ssource(vgrid); ssource =source;
+      Lattice<sobj> pgsource(&pencil_g);
+      Lattice<sobj> pgresult(&pencil_g); pgresult=zero;
+
+#ifndef HAVE_FFTW	
+      assert(0);
+#else 
+      typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
+      typedef typename FFTW<scalar>::FFTW_plan   FFTW_plan;
+
+      {
+	int Ncomp = sizeof(sobj)/sizeof(scalar);
+	int Nlow  = 1;
+	for(int d=0;d<dim;d++){
+	  Nlow*=vgrid->_ldimensions[d];
+	}
+
+	int rank = 1;  /* 1d transforms */
+	int n[] = {G}; /* 1d transforms of length G */
+	int howmany = Ncomp;
+	int odist,idist,istride,ostride;
+	idist   = odist   = 1;          /* Distance between consecutive FT's */
+	istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */
+	int *inembed = n, *onembed = n;
+
+	
+	int sign = FFTW_FORWARD;
+	if (inverse) sign = FFTW_BACKWARD;
+
+	FFTW_plan p;
+	{
+	  FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[0];
+	  FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[0];
+	  p = FFTW<scalar>::fftw_plan_many_dft(rank,n,howmany,
+					       in,inembed,
+					       istride,idist,
+					       out,onembed,
+					       ostride, odist,
+					       sign,FFTW_ESTIMATE);
+	}
+
+	double add,mul,fma;
+	FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
+	flops_call = add+mul+2.0*fma;
+
+	GridStopWatch timer;
+
+	// Barrel shift and collect global pencil
+	for(int p=0;p<processors[dim];p++) { 
+
+	  for(int idx=0;idx<sgrid->lSites();idx++) { 
+
+	    std::vector<int> lcoor(Nd);
+    	    sgrid->LocalIndexToLocalCoor(idx,lcoor);
+
+	    sobj s;
+
+	    peekLocalSite(s,ssource,lcoor);
+
+	    lcoor[dim]+=p*L;
+	   
+	    pokeLocalSite(s,pgsource,lcoor);
+	  }
+
+	  ssource = Cshift(ssource,dim,L);
+	}
+	
+	// Loop over orthog coords
+	int NN=pencil_g.lSites();
+
+	GridStopWatch Timer;
+	Timer.Start();
+
+PARALLEL_FOR_LOOP
+	for(int idx=0;idx<NN;idx++) { 
+
+	  std::vector<int> lcoor(Nd);
+	  pencil_g.LocalIndexToLocalCoor(idx,lcoor);
+
+	  if ( lcoor[dim] == 0 ) {  // restricts loop to plane at lcoor[dim]==0
+	    FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[idx];
+	    FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[idx];
+	    FFTW<scalar>::fftw_execute_dft(p,in,out);
+	  }
+	}
+
+        Timer.Stop();
+	usec += Timer.useconds();
+	flops+= flops_call*NN;
+
+        int pc = processor_coor[dim];
+        for(int idx=0;idx<sgrid->lSites();idx++) { 
+	  std::vector<int> lcoor(Nd);
+	  sgrid->LocalIndexToLocalCoor(idx,lcoor);
+	  std::vector<int> gcoor = lcoor;
+	  // extract the result
+	  sobj s;
+	  gcoor[dim] = lcoor[dim]+L*pc;
+	  peekLocalSite(s,pgresult,gcoor);
+	  pokeLocalSite(s,result,lcoor);
+	}
+      	  
+	FFTW<scalar>::fftw_destroy_plan(p);
+      }
+#endif
+
+
+    }
+
+  };
+
+
+}
+
+#endif

lib/Grid.h

@@ -68,6 +68,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/Simd.h>
 #include <Grid/Threads.h>
 #include <Grid/Lexicographic.h>
+#include <Grid/Init.h>
 #include <Grid/Communicator.h> 
 #include <Grid/Cartesian.h>    
 #include <Grid/Tensors.h>      
@@ -78,7 +79,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/parallelIO/BinaryIO.h>
 #include <Grid/qcd/QCD.h>
 #include <Grid/parallelIO/NerscIO.h>
-#include <Grid/Init.h>
+
+#include <Grid/FFT.h>
 
 #include <Grid/qcd/hmc/NerscCheckpointer.h>
 #include <Grid/qcd/hmc/HmcRunner.h>

lib/Init.cc

@@ -153,6 +153,7 @@ void GridParseLayout(char **argv,int argc,
     assert(ompthreads.size()==1);
     GridThread::SetThreads(ompthreads[0]);
   }
+
   if( GridCmdOptionExists(argv,argv+argc,"--cores") ){
     std::vector<int> cores(0);
     arg= GridCmdOptionPayload(argv,argv+argc,"--cores");
@@ -203,7 +204,6 @@ void Grid_init(int *argc,char ***argv)
     GridLogConfigure(logstreams);
   }
 
-
   if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-signals") ){
     Grid_debug_handler_init();
   }

lib/Makefile.am

@@ -17,8 +17,8 @@ endif
 include Make.inc
 include Eigen.inc
 
-lib_LTLIBRARIES = libGrid.la
+lib_LIBRARIES = libGrid.a
 
-libGrid_la_SOURCES             = $(CCFILES) $(extra_sources)
-libGrid_ladir                  = $(pkgincludedir)
+libGrid_a_SOURCES              = $(CCFILES) $(extra_sources)
+libGrid_adir                   = $(pkgincludedir)
 nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) Config.h

lib/Stencil.h

@@ -106,7 +106,6 @@
  #define SERIAL_SENDS
 
        void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){
-	 comms_bytes+=2.0*bytes;
  #ifdef SEND_IMMEDIATE
 	 commtime-=usecond();
 	 _grid->SendToRecvFrom(xmit,to,rcv,from,bytes);
@@ -265,7 +264,7 @@
 	 //	 _mm_prefetch((char *)&_entries[ent],_MM_HINT_T0);
        }
        inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
-	 _mm_prefetch((char *)&_entries[ent+1],_MM_HINT_T0);
+	 //_mm_prefetch((char *)&_entries[ent+1],_MM_HINT_T0);
 	 local = _entries[ent]._is_local;
 	 perm  = _entries[ent]._permute;
 	 if (perm)  ptype = _permute_type[point]; 
@@ -301,6 +300,39 @@
        double gathermtime;
        double splicetime;
        double nosplicetime;
+       double calls;
+
+       void ZeroCounters(void) {
+         gathertime = 0.;
+         jointime = 0.;
+         commtime = 0.;
+         halogtime = 0.;
+         mergetime = 0.;
+         spintime = 0.;
+         gathermtime = 0.;
+         splicetime = 0.;
+         nosplicetime = 0.;
+         comms_bytes = 0.;
+         calls = 0.;
+       };
+
+       void Report(void) {
+#define PRINTIT(A)	\
+ std::cout << GridLogMessage << " Stencil " << #A << " "<< A/calls<<std::endl;
+	 if ( calls > 0. ) {
+ std::cout << GridLogMessage << " Stencil calls "<<calls<<std::endl;
+       PRINTIT(jointime);
+       PRINTIT(gathertime);
+       PRINTIT(commtime);
+       PRINTIT(halogtime);
+       PRINTIT(mergetime);
+       PRINTIT(spintime);
+       PRINTIT(comms_bytes);
+       PRINTIT(gathermtime);
+       PRINTIT(splicetime);
+       PRINTIT(nosplicetime);
+	 }
+       };
  #endif
 
    CartesianStencil(GridBase *grid,
@@ -310,18 +342,6 @@
 				      const std::vector<int> &distances) 
      :   _permute_type(npoints), _comm_buf_size(npoints)
      {
- #ifdef TIMING_HACK
-       gathertime=0;
-       jointime=0;
-       commtime=0;
-       halogtime=0;
-       mergetime=0;
-       spintime=0;
-       gathermtime=0;
-       splicetime=0;
-       nosplicetime=0;
-       comms_bytes=0;
- #endif
        _npoints = npoints;
        _grid    = grid;
        _directions = directions;
@@ -623,6 +643,7 @@
        template<class compressor>
        void HaloExchange(const Lattice<vobj> &source,compressor &compress) 
        {
+	 calls++;
 	 Mergers.resize(0);
          Packets.resize(0);
          HaloGather(source,compress);

lib/algorithms/iterative/ConjugateGradient.h

@@ -24,7 +24,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef GRID_CONJUGATE_GRADIENT_H
@@ -40,15 +41,17 @@ namespace Grid {
 template <class Field>
 class ConjugateGradient : public OperatorFunction<Field> {
  public:
-    bool ErrorOnNoConverge; //throw an assert when the CG fails to converge. Defaults true.
+  bool ErrorOnNoConverge;  // throw an assert when the CG fails to converge.
+                           // Defaults true.
   RealD Tolerance;
   Integer MaxIterations;
-  ConjugateGradient(RealD tol,Integer maxit, bool err_on_no_conv = true) : Tolerance(tol), MaxIterations(maxit), ErrorOnNoConverge(err_on_no_conv){ 
-    };
-
-
-    void operator() (LinearOperatorBase<Field> &Linop,const Field &src, Field &psi){
+  ConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
+      : Tolerance(tol),
+        MaxIterations(maxit),
+        ErrorOnNoConverge(err_on_no_conv){};
 
+  void operator()(LinearOperatorBase<Field> &Linop, const Field &src,
+                  Field &psi) {
     psi.checkerboard = src.checkerboard;
     conformable(psi, src);
 
@@ -62,8 +65,10 @@ public:
     RealD guess = norm2(psi);
     assert(std::isnan(guess) == 0);
 
+    
     Linop.HermOpAndNorm(psi, mmp, d, b);
     
+
     r = src - mmp;
     p = r;
 
@@ -71,12 +76,18 @@ public:
     cp = a;
     ssq = norm2(src);
 
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient: guess "<<guess<<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:   src "<<ssq  <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:    mp "<<d    <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:   mmp "<<b    <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:  cp,r "<<cp   <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:     p "<<a    <<std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient: guess " << guess << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:   src " << ssq << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:    mp " << d << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:   mmp " << b << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:  cp,r " << cp << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient:     p " << a << std::endl;
 
     RealD rsq = Tolerance * Tolerance * ssq;
 
@@ -85,7 +96,9 @@ public:
       return;
     }
 
-      std::cout<<GridLogIterative << std::setprecision(4)<< "ConjugateGradient: k=0 residual "<<cp<<" target "<<rsq<<std::endl;
+    std::cout << GridLogIterative << std::setprecision(4)
+              << "ConjugateGradient: k=0 residual " << cp << " target " << rsq
+              << std::endl;
 
     GridStopWatch LinalgTimer;
     GridStopWatch MatrixTimer;
@@ -94,7 +107,6 @@ public:
     SolverTimer.Start();
     int k;
     for (k = 1; k <= MaxIterations; k++) {
-	
       c = cp;
 
       MatrixTimer.Start();
@@ -116,11 +128,11 @@ public:
       p = p * b + r;
 
       LinalgTimer.Stop();
-	std::cout<<GridLogIterative<<"ConjugateGradient: Iteration " <<k<<" residual "<<cp<< " target "<< rsq<<std::endl;
+      std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
+                << " residual " << cp << " target " << rsq << std::endl;
 
       // Stopping condition
       if (cp <= rsq) {
-	  
         SolverTimer.Stop();
         Linop.HermOpAndNorm(psi, mmp, d, qq);
         p = mmp - src;
@@ -131,22 +143,25 @@ public:
         RealD resnorm = sqrt(norm2(p));
         RealD true_residual = resnorm / srcnorm;
 
-	  std::cout<<GridLogMessage<<"ConjugateGradient: Converged on iteration " <<k
-		   <<" computed residual "<<sqrt(cp/ssq)
-		   <<" true residual "    <<true_residual
-		   <<" target "<<Tolerance<<std::endl;
-	  std::cout<<GridLogMessage<<"Time elapsed: Total "<< SolverTimer.Elapsed() << " Matrix  "<<MatrixTimer.Elapsed() << " Linalg "<<LinalgTimer.Elapsed();
+        std::cout << GridLogMessage
+                  << "ConjugateGradient: Converged on iteration " << k << std::endl;
+        std::cout << GridLogMessage << "Computed residual " << sqrt(cp / ssq)
+                  << " true residual " << true_residual << " target "
+                  << Tolerance << std::endl;
+        std::cout << GridLogMessage << "Time elapsed: Iterations "
+                  << SolverTimer.Elapsed() << " Matrix  "
+                  << MatrixTimer.Elapsed() << " Linalg "
+                  << LinalgTimer.Elapsed();
         std::cout << std::endl;
 
-	  if(ErrorOnNoConverge)
-	    assert(true_residual/Tolerance < 1000.0);
+        if (ErrorOnNoConverge) assert(true_residual / Tolerance < 1000.0);
 
         return;
       }
     }
-      std::cout<<GridLogMessage<<"ConjugateGradient did NOT converge"<<std::endl;
-      if(ErrorOnNoConverge)	
-	assert(0);
+    std::cout << GridLogMessage << "ConjugateGradient did NOT converge"
+              << std::endl;
+    if (ErrorOnNoConverge) assert(0);
   }
 };
 }

lib/fftw/fftw3.h

@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2003, 2007-14 Matteo Frigo
+ * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
+ *
+ * The following statement of license applies *only* to this header file,
+ * and *not* to the other files distributed with FFTW or derived therefrom:
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/***************************** NOTE TO USERS *********************************
+ *
+ *                 THIS IS A HEADER FILE, NOT A MANUAL
+ *
+ *    If you want to know how to use FFTW, please read the manual,
+ *    online at http://www.fftw.org/doc/ and also included with FFTW.
+ *    For a quick start, see the manual's tutorial section.
+ *
+ *   (Reading header files to learn how to use a library is a habit
+ *    stemming from code lacking a proper manual.  Arguably, it's a
+ *    *bad* habit in most cases, because header files can contain
+ *    interfaces that are not part of the public, stable API.)
+ *
+ ****************************************************************************/
+
+#ifndef FFTW3_H
+#define FFTW3_H
+
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif /* __cplusplus */
+
+/* If <complex.h> is included, use the C99 complex type.  Otherwise
+   define a type bit-compatible with C99 complex */
+#if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I)
+#  define FFTW_DEFINE_COMPLEX(R, C) typedef R _Complex C
+#else
+#  define FFTW_DEFINE_COMPLEX(R, C) typedef R C[2]
+#endif
+
+#define FFTW_CONCAT(prefix, name) prefix ## name
+#define FFTW_MANGLE_DOUBLE(name) FFTW_CONCAT(fftw_, name)
+#define FFTW_MANGLE_FLOAT(name) FFTW_CONCAT(fftwf_, name)
+#define FFTW_MANGLE_LONG_DOUBLE(name) FFTW_CONCAT(fftwl_, name)
+#define FFTW_MANGLE_QUAD(name) FFTW_CONCAT(fftwq_, name)
+
+/* IMPORTANT: for Windows compilers, you should add a line
+        #define FFTW_DLL
+   here and in kernel/ifftw.h if you are compiling/using FFTW as a
+   DLL, in order to do the proper importing/exporting, or
+   alternatively compile with -DFFTW_DLL or the equivalent
+   command-line flag.  This is not necessary under MinGW/Cygwin, where
+   libtool does the imports/exports automatically. */
+#if defined(FFTW_DLL) && (defined(_WIN32) || defined(__WIN32__))
+   /* annoying Windows syntax for shared-library declarations */
+#  if defined(COMPILING_FFTW) /* defined in api.h when compiling FFTW */
+#    define FFTW_EXTERN extern __declspec(dllexport) 
+#  else /* user is calling FFTW; import symbol */
+#    define FFTW_EXTERN extern __declspec(dllimport) 
+#  endif
+#else
+#  define FFTW_EXTERN extern
+#endif
+
+enum fftw_r2r_kind_do_not_use_me {
+     FFTW_R2HC=0, FFTW_HC2R=1, FFTW_DHT=2,
+     FFTW_REDFT00=3, FFTW_REDFT01=4, FFTW_REDFT10=5, FFTW_REDFT11=6,
+     FFTW_RODFT00=7, FFTW_RODFT01=8, FFTW_RODFT10=9, FFTW_RODFT11=10
+};
+
+struct fftw_iodim_do_not_use_me {
+     int n;                     /* dimension size */
+     int is;			/* input stride */
+     int os;			/* output stride */
+};
+
+#include <stddef.h> /* for ptrdiff_t */
+struct fftw_iodim64_do_not_use_me {
+     ptrdiff_t n;                     /* dimension size */
+     ptrdiff_t is;			/* input stride */
+     ptrdiff_t os;			/* output stride */
+};
+
+typedef void (*fftw_write_char_func_do_not_use_me)(char c, void *);
+typedef int (*fftw_read_char_func_do_not_use_me)(void *);
+
+/*
+  huge second-order macro that defines prototypes for all API
+  functions.  We expand this macro for each supported precision
+ 
+  X: name-mangling macro
+  R: real data type
+  C: complex data type
+*/
+
+#define FFTW_DEFINE_API(X, R, C)					   \
+									   \
+FFTW_DEFINE_COMPLEX(R, C);						   \
+									   \
+typedef struct X(plan_s) *X(plan);					   \
+									   \
+typedef struct fftw_iodim_do_not_use_me X(iodim);			   \
+typedef struct fftw_iodim64_do_not_use_me X(iodim64);			   \
+									   \
+typedef enum fftw_r2r_kind_do_not_use_me X(r2r_kind);			   \
+									   \
+typedef fftw_write_char_func_do_not_use_me X(write_char_func);		   \
+typedef fftw_read_char_func_do_not_use_me X(read_char_func);		   \
+									   \
+FFTW_EXTERN void X(execute)(const X(plan) p);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft)(int rank, const int *n,			   \
+		    C *in, C *out, int sign, unsigned flags);		   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_1d)(int n, C *in, C *out, int sign,	   \
+		       unsigned flags);					   \
+FFTW_EXTERN X(plan) X(plan_dft_2d)(int n0, int n1,			   \
+		       C *in, C *out, int sign, unsigned flags);	   \
+FFTW_EXTERN X(plan) X(plan_dft_3d)(int n0, int n1, int n2,		   \
+		       C *in, C *out, int sign, unsigned flags);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_dft)(int rank, const int *n,		   \
+                         int howmany,					   \
+                         C *in, const int *inembed,			   \
+                         int istride, int idist,			   \
+                         C *out, const int *onembed,			   \
+                         int ostride, int odist,			   \
+                         int sign, unsigned flags);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_dft)(int rank, const X(iodim) *dims,	   \
+			 int howmany_rank,				   \
+			 const X(iodim) *howmany_dims,			   \
+			 C *in, C *out,					   \
+			 int sign, unsigned flags);			   \
+FFTW_EXTERN X(plan) X(plan_guru_split_dft)(int rank, const X(iodim) *dims, \
+			 int howmany_rank,				   \
+			 const X(iodim) *howmany_dims,			   \
+			 R *ri, R *ii, R *ro, R *io,			   \
+			 unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_dft)(int rank,			   \
+                         const X(iodim64) *dims,			   \
+			 int howmany_rank,				   \
+			 const X(iodim64) *howmany_dims,		   \
+			 C *in, C *out,					   \
+			 int sign, unsigned flags);			   \
+FFTW_EXTERN X(plan) X(plan_guru64_split_dft)(int rank,			   \
+                         const X(iodim64) *dims,			   \
+			 int howmany_rank,				   \
+			 const X(iodim64) *howmany_dims,		   \
+			 R *ri, R *ii, R *ro, R *io,			   \
+			 unsigned flags);				   \
+									   \
+FFTW_EXTERN void X(execute_dft)(const X(plan) p, C *in, C *out);	   \
+FFTW_EXTERN void X(execute_split_dft)(const X(plan) p, R *ri, R *ii,	   \
+                                      R *ro, R *io);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_dft_r2c)(int rank, const int *n,	   \
+                             int howmany,				   \
+                             R *in, const int *inembed,			   \
+                             int istride, int idist,			   \
+                             C *out, const int *onembed,		   \
+                             int ostride, int odist,			   \
+                             unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_r2c)(int rank, const int *n,		   \
+                        R *in, C *out, unsigned flags);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_r2c_1d)(int n,R *in,C *out,unsigned flags); \
+FFTW_EXTERN X(plan) X(plan_dft_r2c_2d)(int n0, int n1,			   \
+			   R *in, C *out, unsigned flags);		   \
+FFTW_EXTERN X(plan) X(plan_dft_r2c_3d)(int n0, int n1,			   \
+			   int n2,					   \
+			   R *in, C *out, unsigned flags);		   \
+									   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_dft_c2r)(int rank, const int *n,	   \
+			     int howmany,				   \
+			     C *in, const int *inembed,			   \
+			     int istride, int idist,			   \
+			     R *out, const int *onembed,		   \
+			     int ostride, int odist,			   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_c2r)(int rank, const int *n,		   \
+                        C *in, R *out, unsigned flags);			   \
+									   \
+FFTW_EXTERN X(plan) X(plan_dft_c2r_1d)(int n,C *in,R *out,unsigned flags); \
+FFTW_EXTERN X(plan) X(plan_dft_c2r_2d)(int n0, int n1,			   \
+			   C *in, R *out, unsigned flags);		   \
+FFTW_EXTERN X(plan) X(plan_dft_c2r_3d)(int n0, int n1,			   \
+			   int n2,					   \
+			   C *in, R *out, unsigned flags);		   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_dft_r2c)(int rank, const X(iodim) *dims,   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     R *in, C *out,				   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru_dft_c2r)(int rank, const X(iodim) *dims,   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     C *in, R *out,				   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_split_dft_r2c)(				   \
+                             int rank, const X(iodim) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     R *in, R *ro, R *io,			   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru_split_dft_c2r)(				   \
+                             int rank, const X(iodim) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim) *howmany_dims,		   \
+			     R *ri, R *ii, R *out,			   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_dft_r2c)(int rank,			   \
+                             const X(iodim64) *dims,			   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     R *in, C *out,				   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru64_dft_c2r)(int rank,			   \
+                             const X(iodim64) *dims,			   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     C *in, R *out,				   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_split_dft_r2c)(			   \
+                             int rank, const X(iodim64) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     R *in, R *ro, R *io,			   \
+			     unsigned flags);				   \
+FFTW_EXTERN X(plan) X(plan_guru64_split_dft_c2r)(			   \
+                             int rank, const X(iodim64) *dims,		   \
+			     int howmany_rank,				   \
+			     const X(iodim64) *howmany_dims,		   \
+			     R *ri, R *ii, R *out,			   \
+			     unsigned flags);				   \
+									   \
+FFTW_EXTERN void X(execute_dft_r2c)(const X(plan) p, R *in, C *out);	   \
+FFTW_EXTERN void X(execute_dft_c2r)(const X(plan) p, C *in, R *out);	   \
+									   \
+FFTW_EXTERN void X(execute_split_dft_r2c)(const X(plan) p,		   \
+                                          R *in, R *ro, R *io);		   \
+FFTW_EXTERN void X(execute_split_dft_c2r)(const X(plan) p,		   \
+                                          R *ri, R *ii, R *out);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_many_r2r)(int rank, const int *n,		   \
+                         int howmany,					   \
+                         R *in, const int *inembed,			   \
+                         int istride, int idist,			   \
+                         R *out, const int *onembed,			   \
+                         int ostride, int odist,			   \
+                         const X(r2r_kind) *kind, unsigned flags);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_r2r)(int rank, const int *n, R *in, R *out,	   \
+                    const X(r2r_kind) *kind, unsigned flags);		   \
+									   \
+FFTW_EXTERN X(plan) X(plan_r2r_1d)(int n, R *in, R *out,		   \
+                       X(r2r_kind) kind, unsigned flags);		   \
+FFTW_EXTERN X(plan) X(plan_r2r_2d)(int n0, int n1, R *in, R *out,	   \
+                       X(r2r_kind) kind0, X(r2r_kind) kind1,		   \
+                       unsigned flags);					   \
+FFTW_EXTERN X(plan) X(plan_r2r_3d)(int n0, int n1, int n2,		   \
+                       R *in, R *out, X(r2r_kind) kind0,		   \
+                       X(r2r_kind) kind1, X(r2r_kind) kind2,		   \
+                       unsigned flags);					   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru_r2r)(int rank, const X(iodim) *dims,	   \
+                         int howmany_rank,				   \
+                         const X(iodim) *howmany_dims,			   \
+                         R *in, R *out,					   \
+                         const X(r2r_kind) *kind, unsigned flags);	   \
+									   \
+FFTW_EXTERN X(plan) X(plan_guru64_r2r)(int rank, const X(iodim64) *dims,   \
+                         int howmany_rank,				   \
+                         const X(iodim64) *howmany_dims,		   \
+                         R *in, R *out,					   \
+                         const X(r2r_kind) *kind, unsigned flags);	   \
+									   \
+FFTW_EXTERN void X(execute_r2r)(const X(plan) p, R *in, R *out);	   \
+									   \
+FFTW_EXTERN void X(destroy_plan)(X(plan) p);				   \
+FFTW_EXTERN void X(forget_wisdom)(void);				   \
+FFTW_EXTERN void X(cleanup)(void);					   \
+									   \
+FFTW_EXTERN void X(set_timelimit)(double t);				   \
+									   \
+FFTW_EXTERN void X(plan_with_nthreads)(int nthreads);			   \
+FFTW_EXTERN int X(init_threads)(void);					   \
+FFTW_EXTERN void X(cleanup_threads)(void);				   \
+									   \
+FFTW_EXTERN int X(export_wisdom_to_filename)(const char *filename);	   \
+FFTW_EXTERN void X(export_wisdom_to_file)(FILE *output_file);		   \
+FFTW_EXTERN char *X(export_wisdom_to_string)(void);			   \
+FFTW_EXTERN void X(export_wisdom)(X(write_char_func) write_char,   	   \
+                                  void *data);				   \
+FFTW_EXTERN int X(import_system_wisdom)(void);				   \
+FFTW_EXTERN int X(import_wisdom_from_filename)(const char *filename);	   \
+FFTW_EXTERN int X(import_wisdom_from_file)(FILE *input_file);		   \
+FFTW_EXTERN int X(import_wisdom_from_string)(const char *input_string);	   \
+FFTW_EXTERN int X(import_wisdom)(X(read_char_func) read_char, void *data); \
+									   \
+FFTW_EXTERN void X(fprint_plan)(const X(plan) p, FILE *output_file);	   \
+FFTW_EXTERN void X(print_plan)(const X(plan) p);			   \
+FFTW_EXTERN char *X(sprint_plan)(const X(plan) p);			   \
+									   \
+FFTW_EXTERN void *X(malloc)(size_t n);					   \
+FFTW_EXTERN R *X(alloc_real)(size_t n);					   \
+FFTW_EXTERN C *X(alloc_complex)(size_t n);				   \
+FFTW_EXTERN void X(free)(void *p);					   \
+									   \
+FFTW_EXTERN void X(flops)(const X(plan) p,				   \
+                          double *add, double *mul, double *fmas);	   \
+FFTW_EXTERN double X(estimate_cost)(const X(plan) p);			   \
+FFTW_EXTERN double X(cost)(const X(plan) p);				   \
+									   \
+FFTW_EXTERN int X(alignment_of)(R *p);                                     \
+FFTW_EXTERN const char X(version)[];                                       \
+FFTW_EXTERN const char X(cc)[];						   \
+FFTW_EXTERN const char X(codelet_optim)[];
+
+
+/* end of FFTW_DEFINE_API macro */
+
+FFTW_DEFINE_API(FFTW_MANGLE_DOUBLE, double, fftw_complex)
+FFTW_DEFINE_API(FFTW_MANGLE_FLOAT, float, fftwf_complex)
+FFTW_DEFINE_API(FFTW_MANGLE_LONG_DOUBLE, long double, fftwl_complex)
+
+/* __float128 (quad precision) is a gcc extension on i386, x86_64, and ia64
+   for gcc >= 4.6 (compiled in FFTW with --enable-quad-precision) */
+#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) \
+ && !(defined(__ICC) || defined(__INTEL_COMPILER)) \
+ && (defined(__i386__) || defined(__x86_64__) || defined(__ia64__))
+#  if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I)
+/* note: __float128 is a typedef, which is not supported with the _Complex
+         keyword in gcc, so instead we use this ugly __attribute__ version.
+         However, we can't simply pass the __attribute__ version to
+         FFTW_DEFINE_API because the __attribute__ confuses gcc in pointer
+         types.  Hence redefining FFTW_DEFINE_COMPLEX.  Ugh. */
+#    undef FFTW_DEFINE_COMPLEX
+#    define FFTW_DEFINE_COMPLEX(R, C) typedef _Complex float __attribute__((mode(TC))) C
+#  endif
+FFTW_DEFINE_API(FFTW_MANGLE_QUAD, __float128, fftwq_complex)
+#endif
+
+#define FFTW_FORWARD (-1)
+#define FFTW_BACKWARD (+1)
+
+#define FFTW_NO_TIMELIMIT (-1.0)
+
+/* documented flags */
+#define FFTW_MEASURE (0U)
+#define FFTW_DESTROY_INPUT (1U << 0)
+#define FFTW_UNALIGNED (1U << 1)
+#define FFTW_CONSERVE_MEMORY (1U << 2)
+#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */
+#define FFTW_PRESERVE_INPUT (1U << 4) /* cancels FFTW_DESTROY_INPUT */
+#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */
+#define FFTW_ESTIMATE (1U << 6)
+#define FFTW_WISDOM_ONLY (1U << 21)
+
+/* undocumented beyond-guru flags */
+#define FFTW_ESTIMATE_PATIENT (1U << 7)
+#define FFTW_BELIEVE_PCOST (1U << 8)
+#define FFTW_NO_DFT_R2HC (1U << 9)
+#define FFTW_NO_NONTHREADED (1U << 10)
+#define FFTW_NO_BUFFERING (1U << 11)
+#define FFTW_NO_INDIRECT_OP (1U << 12)
+#define FFTW_ALLOW_LARGE_GENERIC (1U << 13) /* NO_LARGE_GENERIC is default */
+#define FFTW_NO_RANK_SPLITS (1U << 14)
+#define FFTW_NO_VRANK_SPLITS (1U << 15)
+#define FFTW_NO_VRECURSE (1U << 16)
+#define FFTW_NO_SIMD (1U << 17)
+#define FFTW_NO_SLOW (1U << 18)
+#define FFTW_NO_FIXED_RADIX_LARGE_N (1U << 19)
+#define FFTW_ALLOW_PRUNING (1U << 20)
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif /* __cplusplus */
+
+#endif /* FFTW3_H */

lib/lattice/Lattice_transfer.h

@@ -349,7 +349,7 @@ void localConvert(const Lattice<vobj> &in,Lattice<vvobj> &out)
     assert(ig->_ldimensions[d] == og->_ldimensions[d]);
   }
 
-PARALLEL_FOR_LOOP
+  //PARALLEL_FOR_LOOP
   for(int idx=0;idx<ig->lSites();idx++){
     std::vector<int> lcoor(ni);
     ig->LocalIndexToLocalCoor(idx,lcoor);
@@ -446,6 +446,79 @@ void ExtractSlice(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice, in
 
 }
 
+
+template<class vobj>
+void InsertSliceLocal(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice_lo,int slice_hi, int orthog)
+{
+  typedef typename vobj::scalar_object sobj;
+  sobj s;
+
+  GridBase *lg = lowDim._grid;
+  GridBase *hg = higherDim._grid;
+  int nl = lg->_ndimension;
+  int nh = hg->_ndimension;
+
+  assert(nl == nh);
+  assert(orthog<nh);
+  assert(orthog>=0);
+
+  for(int d=0;d<nh;d++){
+    assert(lg->_processors[d]  == hg->_processors[d]);
+    assert(lg->_ldimensions[d] == hg->_ldimensions[d]);
+  }
+
+  // the above should guarantee that the operations are local
+  //PARALLEL_FOR_LOOP
+  for(int idx=0;idx<lg->lSites();idx++){
+    std::vector<int> lcoor(nl);
+    std::vector<int> hcoor(nh);
+    lg->LocalIndexToLocalCoor(idx,lcoor);
+    if( lcoor[orthog] == slice_lo ) { 
+      hcoor=lcoor;
+      hcoor[orthog] = slice_hi;
+      peekLocalSite(s,lowDim,lcoor);
+      pokeLocalSite(s,higherDim,hcoor);
+    }
+  }
+}
+
+
+template<class vobj>
+void ExtractSliceLocal(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice_lo,int slice_hi, int orthog)
+{
+  typedef typename vobj::scalar_object sobj;
+  sobj s;
+
+  GridBase *lg = lowDim._grid;
+  GridBase *hg = higherDim._grid;
+  int nl = lg->_ndimension;
+  int nh = hg->_ndimension;
+
+  assert(nl == nh);
+  assert(orthog<nh);
+  assert(orthog>=0);
+
+  for(int d=0;d<nh;d++){
+    assert(lg->_processors[d]  == hg->_processors[d]);
+    assert(lg->_ldimensions[d] == hg->_ldimensions[d]);
+  }
+
+  // the above should guarantee that the operations are local
+  //PARALLEL_FOR_LOOP
+  for(int idx=0;idx<lg->lSites();idx++){
+    std::vector<int> lcoor(nl);
+    std::vector<int> hcoor(nh);
+    lg->LocalIndexToLocalCoor(idx,lcoor);
+    if( lcoor[orthog] == slice_lo ) { 
+      hcoor=lcoor;
+      hcoor[orthog] = slice_hi;
+      peekLocalSite(s,higherDim,hcoor);
+      pokeLocalSite(s,lowDim,lcoor);
+    }
+  }
+}
+
+
 template<class vobj>
 void Replicate(Lattice<vobj> &coarse,Lattice<vobj> & fine)
 {

lib/qcd/QCD.h

@@ -55,10 +55,13 @@ namespace QCD {
     //////////////////////////////////////////////////////////////////////////////
     // QCD iMatrix types
     // Index conventions:                            Lorentz x Spin x Colour
+    // note: static const int or constexpr will work for type deductions
+    //       with the intel compiler (up to version 17)
     //////////////////////////////////////////////////////////////////////////////
-    static const int ColourIndex = 2;
-    static const int SpinIndex   = 1;
-    static const int LorentzIndex= 0;
+    #define ColourIndex  2
+    #define SpinIndex    1
+    #define LorentzIndex 0
+
   
     // Also should make these a named enum type
     static const int DaggerNo=0;
@@ -490,16 +493,27 @@ namespace QCD {
 }   //namespace QCD
 } // Grid
 
+
 #include <Grid/qcd/utils/SpaceTimeGrid.h>
 #include <Grid/qcd/spin/Dirac.h>
 #include <Grid/qcd/spin/TwoSpinor.h>
 #include <Grid/qcd/utils/LinalgUtils.h>
 #include <Grid/qcd/utils/CovariantCshift.h>
+
+// Include representations 	
 #include <Grid/qcd/utils/SUn.h>
+#include <Grid/qcd/utils/SUnAdjoint.h>
+#include <Grid/qcd/utils/SUnTwoIndex.h>
+#include <Grid/qcd/representations/hmc_types.h>
+
 #include <Grid/qcd/action/Actions.h>
+
+#include <Grid/qcd/smearing/Smearing.h>
+
 #include <Grid/qcd/hmc/integrators/Integrator.h>
 #include <Grid/qcd/hmc/integrators/Integrator_algorithm.h>
 #include <Grid/qcd/hmc/HMC.h>
-#include <Grid/qcd/smearing/Smearing.h>
+
+
 
 #endif

lib/qcd/action/ActionBase.h

@@ -23,7 +23,8 @@ Author: neo <cossu@post.kek.jp>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef QCD_ACTION_BASE
@@ -33,55 +34,120 @@ namespace QCD{
 
 template <class GaugeField>
 class Action {
-
  public:
   bool is_smeared = false;
   // Boundary conditions? // Heatbath?
-  virtual void  refresh(const GaugeField &U, GridParallelRNG& pRNG) = 0;// refresh pseudofermions
+  virtual void refresh(const GaugeField& U,
+                       GridParallelRNG& pRNG) = 0;  // refresh pseudofermions
   virtual RealD S(const GaugeField& U) = 0;         // evaluate the action
-  virtual void  deriv(const GaugeField &U,GaugeField & dSdU )     = 0;  // evaluate the action derivative
+  virtual void deriv(const GaugeField& U,
+                     GaugeField& dSdU) = 0;  // evaluate the action derivative
   virtual ~Action(){};
 };
 
-// Could derive PseudoFermion action with a PF field, FermionField, and a Grid; implement refresh
+// Indexing of tuple types
+template <class T, class Tuple>
+struct Index;
+
+template <class T, class... Types>
+struct Index<T, std::tuple<T, Types...>> {
+  static const std::size_t value = 0;
+};
+
+template <class T, class U, class... Types>
+struct Index<T, std::tuple<U, Types...>> {
+  static const std::size_t value = 1 + Index<T, std::tuple<Types...>>::value;
+};
+
 /*
-template<class GaugeField, class FermionField>
-class PseudoFermionAction : public Action<GaugeField> {
+template <class GaugeField>
+struct ActionLevel {
  public:
-  FermionField Phi;
-  GridParallelRNG &pRNG;
-  GridBase &Grid;
+  typedef Action<GaugeField>*
+      ActPtr;  // now force the same colours as the rest of the code
 
-  PseudoFermionAction(GridBase &_Grid,GridParallelRNG &_pRNG) : Grid(_Grid), Phi(&_Grid), pRNG(_pRNG) {
-  };
+  //Add supported representations here
 
-  virtual void refresh(const GaugeField &gauge) {
-    gaussian(Phi,pRNG);
-  };
 
-};
-*/
-
-template<class GaugeField> struct ActionLevel{
-public:
-   
-  typedef Action<GaugeField>*  ActPtr; // now force the same colours as the rest of the code
-
-  int multiplier;
+  unsigned int multiplier;
 
   std::vector<ActPtr> actions;
 
-  ActionLevel(int mul = 1) : multiplier(mul) {
-    assert (mul > 0);
+  ActionLevel(unsigned int mul = 1) : actions(0), multiplier(mul) {
+    assert(mul >= 1);
   };
 
-  void push_back(ActPtr ptr){
-    actions.push_back(ptr);
+  void push_back(ActPtr ptr) { actions.push_back(ptr); }
+};
+*/
+
+template <class GaugeField, class Repr = NoHirep >
+struct ActionLevel {
+ public:
+  unsigned int multiplier; 
+
+  // Fundamental repr actions separated because of the smearing
+  typedef Action<GaugeField>* ActPtr;
+
+  // construct a tuple of vectors of the actions for the corresponding higher
+  // representation fields
+  typedef typename AccessTypes<Action, Repr>::VectorCollection action_collection;
+  action_collection actions_hirep;
+  typedef typename  AccessTypes<Action, Repr>::FieldTypeCollection action_hirep_types;
+
+  std::vector<ActPtr>& actions;
+
+  // Temporary conversion between ActionLevel and ActionLevelHirep
+  //ActionLevelHirep(ActionLevel<GaugeField>& AL ):actions(AL.actions), multiplier(AL.multiplier){}
+
+  ActionLevel(unsigned int mul = 1) : actions(std::get<0>(actions_hirep)), multiplier(mul) {
+    // initialize the hirep vectors to zero.
+    //apply(this->resize, actions_hirep, 0); //need a working resize
+    assert(mul >= 1);
+  };
+
+  //void push_back(ActPtr ptr) { actions.push_back(ptr); }
+
+
+
+  template < class Field >
+  void push_back(Action<Field>* ptr) {
+    // insert only in the correct vector
+    std::get< Index < Field, action_hirep_types>::value >(actions_hirep).push_back(ptr);
+  };
+
+ 
+
+  template < class ActPtr>
+  static void resize(ActPtr ap, unsigned int n){
+    ap->resize(n);
+
   }
+
+  //template <std::size_t I>
+  //auto getRepresentation(Repr& R)->decltype(std::get<I>(R).U)  {return std::get<I>(R).U;}
+
+  // Loop on tuple for a callable function
+  template <std::size_t I = 1, typename Callable, typename ...Args>
+  inline typename std::enable_if<I == std::tuple_size<action_collection>::value, void>::type apply(
+      Callable, Repr& R,Args&...) const {}
+
+  template <std::size_t I = 1, typename Callable, typename ...Args>
+  inline typename std::enable_if<I < std::tuple_size<action_collection>::value, void>::type apply(
+      Callable fn, Repr& R, Args&... arguments) const {
+    fn(std::get<I>(actions_hirep), std::get<I>(R.rep), arguments...);
+    apply<I + 1>(fn, R, arguments...);
+  }  
+
 };
 
-template<class GaugeField> using ActionSet = std::vector<ActionLevel< GaugeField > >;
 
+//template <class GaugeField>
+//using ActionSet = std::vector<ActionLevel<GaugeField> >;
 
-}}
+template <class GaugeField, class R>
+using ActionSet = std::vector<ActionLevel<GaugeField, R> >;
+
+}
+}
 #endif

lib/qcd/action/Actions.h

@@ -111,17 +111,30 @@ typedef SymanzikGaugeAction<ConjugateGimplD>        ConjugateSymanzikGaugeAction
 #define FermOp4dVecTemplateInstantiate(A) \
   template class A<WilsonImplF>;		\
   template class A<WilsonImplD>;		\
+  template class A<ZWilsonImplF>;		\
+  template class A<ZWilsonImplD>;		\
   template class A<GparityWilsonImplF>;		\
   template class A<GparityWilsonImplD>;		
 
+#define AdjointFermOpTemplateInstantiate(A) \
+  template class A<WilsonAdjImplF>; \
+  template class A<WilsonAdjImplD>; 
+
+#define TwoIndexFermOpTemplateInstantiate(A) \
+  template class A<WilsonTwoIndexSymmetricImplF>; \
+  template class A<WilsonTwoIndexSymmetricImplD>; 
+
 #define FermOp5dVecTemplateInstantiate(A) \
   template class A<DomainWallVec5dImplF>;	\
-  template class A<DomainWallVec5dImplD>;	
+  template class A<DomainWallVec5dImplD>;	\
+  template class A<ZDomainWallVec5dImplF>;	\
+  template class A<ZDomainWallVec5dImplD>;	
 
 #define FermOpTemplateInstantiate(A) \
  FermOp4dVecTemplateInstantiate(A) \
  FermOp5dVecTemplateInstantiate(A) 
 
+
 #define GparityFermOpTemplateInstantiate(A) 
 
 ////////////////////////////////////////////
@@ -138,6 +151,7 @@ typedef SymanzikGaugeAction<ConjugateGimplD>        ConjugateSymanzikGaugeAction
 #include <Grid/qcd/action/fermion/DomainWallFermion.h>
 #include <Grid/qcd/action/fermion/DomainWallFermion.h>
 #include <Grid/qcd/action/fermion/MobiusFermion.h>
+#include <Grid/qcd/action/fermion/ZMobiusFermion.h>
 #include <Grid/qcd/action/fermion/ScaledShamirFermion.h>
 #include <Grid/qcd/action/fermion/MobiusZolotarevFermion.h>
 #include <Grid/qcd/action/fermion/ShamirZolotarevFermion.h>
@@ -166,6 +180,14 @@ typedef WilsonFermion<WilsonImplR> WilsonFermionR;
 typedef WilsonFermion<WilsonImplF> WilsonFermionF;
 typedef WilsonFermion<WilsonImplD> WilsonFermionD;
 
+typedef WilsonFermion<WilsonAdjImplR> WilsonAdjFermionR;
+typedef WilsonFermion<WilsonAdjImplF> WilsonAdjFermionF;
+typedef WilsonFermion<WilsonAdjImplD> WilsonAdjFermionD;
+
+typedef WilsonFermion<WilsonTwoIndexSymmetricImplR> WilsonTwoIndexSymmetricFermionR;
+typedef WilsonFermion<WilsonTwoIndexSymmetricImplF> WilsonTwoIndexSymmetricFermionF;
+typedef WilsonFermion<WilsonTwoIndexSymmetricImplD> WilsonTwoIndexSymmetricFermionD;
+
 typedef WilsonTMFermion<WilsonImplR> WilsonTMFermionR;
 typedef WilsonTMFermion<WilsonImplF> WilsonTMFermionF;
 typedef WilsonTMFermion<WilsonImplD> WilsonTMFermionD;
@@ -176,6 +198,11 @@ typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
 typedef MobiusFermion<WilsonImplR> MobiusFermionR;
 typedef MobiusFermion<WilsonImplF> MobiusFermionF;
 typedef MobiusFermion<WilsonImplD> MobiusFermionD;
+
+typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
+typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
+typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
+
 typedef ScaledShamirFermion<WilsonImplR> ScaledShamirFermionR;
 typedef ScaledShamirFermion<WilsonImplF> ScaledShamirFermionF;
 typedef ScaledShamirFermion<WilsonImplD> ScaledShamirFermionD;

lib/qcd/action/fermion/CayleyFermion5D.cc

@@ -54,18 +54,18 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5D   (const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
-  std::vector<RealD> diag (Ls,1.0);
-  std::vector<RealD> upper(Ls,-1.0); upper[Ls-1]=mass;
-  std::vector<RealD> lower(Ls,-1.0); lower[0]   =mass;
+  std::vector<Coeff_t> diag (Ls,1.0);
+  std::vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1]=mass;
+  std::vector<Coeff_t> lower(Ls,-1.0); lower[0]   =mass;
   M5D(psi,chi,chi,lower,diag,upper);
 }
 template<class Impl>
 void CayleyFermion5D<Impl>::Meooe5D    (const FermionField &psi, FermionField &Din)
 {
   int Ls=this->Ls;
-  std::vector<RealD> diag = bs;
-  std::vector<RealD> upper= cs;
-  std::vector<RealD> lower= cs; 
+  std::vector<Coeff_t> diag = bs;
+  std::vector<Coeff_t> upper= cs;
+  std::vector<Coeff_t> lower= cs; 
   upper[Ls-1]=-mass*upper[Ls-1];
   lower[0]   =-mass*lower[0];
   M5D(psi,psi,Din,lower,diag,upper);
@@ -73,9 +73,9 @@ void CayleyFermion5D<Impl>::Meooe5D    (const FermionField &psi, FermionField &D
 template<class Impl> void CayleyFermion5D<Impl>::Meo5D     (const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
-  std::vector<RealD> diag = beo;
-  std::vector<RealD> upper(Ls);
-  std::vector<RealD> lower(Ls);
+  std::vector<Coeff_t> diag = beo;
+  std::vector<Coeff_t> upper(Ls);
+  std::vector<Coeff_t> lower(Ls);
   for(int i=0;i<Ls;i++) {
     upper[i]=-ceo[i];
     lower[i]=-ceo[i];
@@ -88,9 +88,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::Mooee       (const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
-  std::vector<RealD> diag = bee;
-  std::vector<RealD> upper(Ls);
-  std::vector<RealD> lower(Ls);
+  std::vector<Coeff_t> diag = bee;
+  std::vector<Coeff_t> upper(Ls);
+  std::vector<Coeff_t> lower(Ls);
   for(int i=0;i<Ls;i++) {
     upper[i]=-cee[i];
     lower[i]=-cee[i];
@@ -104,9 +104,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::MooeeDag    (const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
-  std::vector<RealD> diag = bee;
-  std::vector<RealD> upper(Ls);
-  std::vector<RealD> lower(Ls);
+  std::vector<Coeff_t> diag = bee;
+  std::vector<Coeff_t> upper(Ls);
+  std::vector<Coeff_t> lower(Ls);
 
   for (int s=0;s<Ls;s++){
     // Assemble the 5d matrix
@@ -129,9 +129,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5Ddag (const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
-  std::vector<RealD> diag(Ls,1.0);
-  std::vector<RealD> upper(Ls,-1.0);
-  std::vector<RealD> lower(Ls,-1.0);
+  std::vector<Coeff_t> diag(Ls,1.0);
+  std::vector<Coeff_t> upper(Ls,-1.0);
+  std::vector<Coeff_t> lower(Ls,-1.0);
   upper[Ls-1]=-mass*upper[Ls-1];
   lower[0]   =-mass*lower[0];
   M5Ddag(psi,chi,chi,lower,diag,upper);
@@ -141,9 +141,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::MeooeDag5D    (const FermionField &psi, FermionField &Din)
 {
   int Ls=this->Ls;
-  std::vector<RealD> diag =bs;
-  std::vector<RealD> upper=cs;
-  std::vector<RealD> lower=cs;
+  std::vector<Coeff_t> diag =bs;
+  std::vector<Coeff_t> upper=cs;
+  std::vector<Coeff_t> lower=cs;
   upper[Ls-1]=-mass*upper[Ls-1];
   lower[0]   =-mass*lower[0];
   M5Ddag(psi,psi,Din,lower,diag,upper);
@@ -273,11 +273,21 @@ void CayleyFermion5D<Impl>::MeoDeriv(GaugeField &mat,const FermionField &U,const
 template<class Impl>
 void CayleyFermion5D<Impl>::SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c)
 {
-  SetCoefficientsZolotarev(1.0,zdata,b,c);
+  std::vector<Coeff_t> gamma(this->Ls);
+  for(int s=0;s<this->Ls;s++) gamma[s] = zdata->gamma[s];
+  SetCoefficientsInternal(1.0,gamma,b,c);
 }
 //Zolo
 template<class Impl>
 void CayleyFermion5D<Impl>::SetCoefficientsZolotarev(RealD zolo_hi,Approx::zolotarev_data *zdata,RealD b,RealD c)
+{
+  std::vector<Coeff_t> gamma(this->Ls);
+  for(int s=0;s<this->Ls;s++) gamma[s] = zdata->gamma[s];
+  SetCoefficientsInternal(zolo_hi,gamma,b,c);
+}
+//Zolo
+template<class Impl>
+void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Coeff_t> & gamma,RealD b,RealD c)
 {
   int Ls=this->Ls;
 
@@ -315,7 +325,7 @@ void CayleyFermion5D<Impl>::SetCoefficientsZolotarev(RealD zolo_hi,Approx::zolot
   double bmc = b-c;
   for(int i=0; i < Ls; i++){
     as[i] = 1.0;
-    omega[i] = ((double)zdata->gamma[i])*zolo_hi; //NB reciprocal relative to Chroma NEF code
+    omega[i] = gamma[i]*zolo_hi; //NB reciprocal relative to Chroma NEF code
     bs[i] = 0.5*(bpc/omega[i] + bmc);
     cs[i] = 0.5*(bpc/omega[i] - bmc);
   }
@@ -377,7 +387,7 @@ void CayleyFermion5D<Impl>::SetCoefficientsZolotarev(RealD zolo_hi,Approx::zolot
   }
 	
   { 
-    double delta_d=mass*cee[Ls-1];
+    Coeff_t delta_d=mass*cee[Ls-1];
     for(int j=0;j<Ls-1;j++) delta_d *= cee[j]/bee[j];
     dee[Ls-1] += delta_d;
   }  

lib/qcd/action/fermion/CayleyFermion5D.h

@@ -62,16 +62,16 @@ namespace Grid {
       void M5D(const FermionField &psi,
 	       const FermionField &phi, 
 	       FermionField &chi,
-	       std::vector<RealD> &lower,
-	       std::vector<RealD> &diag,
-	       std::vector<RealD> &upper);
+	       std::vector<Coeff_t> &lower,
+	       std::vector<Coeff_t> &diag,
+	       std::vector<Coeff_t> &upper);
 
       void M5Ddag(const FermionField &psi,
 		  const FermionField &phi, 
 		  FermionField &chi,
-		  std::vector<RealD> &lower,
-		  std::vector<RealD> &diag,
-		  std::vector<RealD> &upper);
+		  std::vector<Coeff_t> &lower,
+		  std::vector<Coeff_t> &diag,
+		  std::vector<Coeff_t> &upper);
       void MooeeInternal(const FermionField &in, FermionField &out,int dag,int inv);
 
       virtual void   Instantiatable(void)=0;
@@ -91,23 +91,23 @@ namespace Grid {
       RealD mass;
 
       // Cayley form Moebius (tanh and zolotarev)
-      std::vector<RealD> omega; 
-      std::vector<RealD> bs;    // S dependent coeffs
-      std::vector<RealD> cs;    
-      std::vector<RealD> as;    
+      std::vector<Coeff_t> omega; 
+      std::vector<Coeff_t> bs;    // S dependent coeffs
+      std::vector<Coeff_t> cs;    
+      std::vector<Coeff_t> as;    
       // For preconditioning Cayley form
-      std::vector<RealD> bee;    
-      std::vector<RealD> cee;    
-      std::vector<RealD> aee;    
-      std::vector<RealD> beo;    
-      std::vector<RealD> ceo;    
-      std::vector<RealD> aeo;    
+      std::vector<Coeff_t> bee;    
+      std::vector<Coeff_t> cee;    
+      std::vector<Coeff_t> aee;    
+      std::vector<Coeff_t> beo;    
+      std::vector<Coeff_t> ceo;    
+      std::vector<Coeff_t> aeo;    
       // LDU factorisation of the eeoo matrix
-      std::vector<RealD> lee;    
-      std::vector<RealD> leem;    
-      std::vector<RealD> uee;    
-      std::vector<RealD> ueem;    
-      std::vector<RealD> dee;    
+      std::vector<Coeff_t> lee;    
+      std::vector<Coeff_t> leem;    
+      std::vector<Coeff_t> uee;    
+      std::vector<Coeff_t> ueem;    
+      std::vector<Coeff_t> dee;    
 
       // Constructors
       CayleyFermion5D(GaugeField &_Umu,
@@ -117,20 +117,19 @@ namespace Grid {
 		      GridRedBlackCartesian &FourDimRedBlackGrid,
 		      RealD _mass,RealD _M5,const ImplParams &p= ImplParams());
 
-
-
     protected:
       void SetCoefficientsZolotarev(RealD zolohi,Approx::zolotarev_data *zdata,RealD b,RealD c);
       void SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c);
+      void SetCoefficientsInternal(RealD zolo_hi,std::vector<Coeff_t> & gamma,RealD b,RealD c);
     };
 
   }
 }
 #define INSTANTIATE_DPERP(A)\
 template void CayleyFermion5D< A >::M5D(const FermionField &psi,const FermionField &phi,FermionField &chi,\
-					std::vector<RealD> &lower,std::vector<RealD> &diag,std::vector<RealD> &upper); \
+					std::vector<Coeff_t> &lower,std::vector<Coeff_t> &diag,std::vector<Coeff_t> &upper); \
 template void CayleyFermion5D< A >::M5Ddag(const FermionField &psi,const FermionField &phi,FermionField &chi,\
-					   std::vector<RealD> &lower,std::vector<RealD> &diag,std::vector<RealD> &upper); \
+					   std::vector<Coeff_t> &lower,std::vector<Coeff_t> &diag,std::vector<Coeff_t> &upper); \
 template void CayleyFermion5D< A >::MooeeInv    (const FermionField &psi, FermionField &chi); \
 template void CayleyFermion5D< A >::MooeeInvDag (const FermionField &psi, FermionField &chi);
 

lib/qcd/action/fermion/CayleyFermion5Dcache.cc

@@ -43,9 +43,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5D(const FermionField &psi,
 				const FermionField &phi, 
 				FermionField &chi,
-				std::vector<RealD> &lower,
-				std::vector<RealD> &diag,
-				std::vector<RealD> &upper)
+				std::vector<Coeff_t> &lower,
+				std::vector<Coeff_t> &diag,
+				std::vector<Coeff_t> &upper)
 {
   int Ls =this->Ls;
   GridBase *grid=psi._grid;
@@ -82,9 +82,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5Ddag(const FermionField &psi,
 				   const FermionField &phi, 
 				   FermionField &chi,
-				   std::vector<RealD> &lower,
-				   std::vector<RealD> &diag,
-				   std::vector<RealD> &upper)
+				   std::vector<Coeff_t> &lower,
+				   std::vector<Coeff_t> &diag,
+				   std::vector<Coeff_t> &upper)
 {
   int Ls =this->Ls;
   GridBase *grid=psi._grid;
@@ -204,6 +204,8 @@ PARALLEL_FOR_LOOP
   INSTANTIATE_DPERP(WilsonImplD);
   INSTANTIATE_DPERP(GparityWilsonImplF);
   INSTANTIATE_DPERP(GparityWilsonImplD);
+  INSTANTIATE_DPERP(ZWilsonImplF);
+  INSTANTIATE_DPERP(ZWilsonImplD);
 #endif
 
 }}

lib/qcd/action/fermion/CayleyFermion5Dssp.cc

@@ -43,9 +43,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5D(const FermionField &psi,
 				const FermionField &phi, 
 				FermionField &chi,
-				std::vector<RealD> &lower,
-				std::vector<RealD> &diag,
-				std::vector<RealD> &upper)
+				std::vector<Coeff_t> &lower,
+				std::vector<Coeff_t> &diag,
+				std::vector<Coeff_t> &upper)
 {
   int Ls=this->Ls;
   for(int s=0;s<Ls;s++){
@@ -65,9 +65,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5Ddag(const FermionField &psi,
 				   const FermionField &phi, 
 				   FermionField &chi,
-				   std::vector<RealD> &lower,
-				   std::vector<RealD> &diag,
-				   std::vector<RealD> &upper)
+				   std::vector<Coeff_t> &lower,
+				   std::vector<Coeff_t> &diag,
+				   std::vector<Coeff_t> &upper)
 {
   int Ls=this->Ls;
   for(int s=0;s<Ls;s++){

lib/qcd/action/fermion/CayleyFermion5Dvec.cc

@@ -53,9 +53,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5D(const FermionField &psi,
 				const FermionField &phi, 
 				FermionField &chi,
-				std::vector<RealD> &lower,
-				std::vector<RealD> &diag,
-				std::vector<RealD> &upper)
+				std::vector<Coeff_t> &lower,
+				std::vector<Coeff_t> &diag,
+				std::vector<Coeff_t> &upper)
 {
   GridBase *grid=psi._grid;
   int Ls   = this->Ls;
@@ -121,9 +121,9 @@ template<class Impl>
 void CayleyFermion5D<Impl>::M5Ddag(const FermionField &psi,
 				   const FermionField &phi, 
 				   FermionField &chi,
-				   std::vector<RealD> &lower,
-				   std::vector<RealD> &diag,
-				   std::vector<RealD> &upper)
+				   std::vector<Coeff_t> &lower,
+				   std::vector<Coeff_t> &diag,
+				   std::vector<Coeff_t> &upper)
 {
   GridBase *grid=psi._grid;
   int Ls   = this->Ls;
@@ -194,8 +194,8 @@ void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField
 
   chi.checkerboard=psi.checkerboard;
   
-  Eigen::MatrixXd Pplus  = Eigen::MatrixXd::Zero(Ls,Ls);
-  Eigen::MatrixXd Pminus = Eigen::MatrixXd::Zero(Ls,Ls);
+  Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
+  Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
   
   for(int s=0;s<Ls;s++){
     Pplus(s,s) = bee[s];
@@ -212,8 +212,8 @@ void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField
   Pplus (0,Ls-1) = mass*cee[0];
   Pminus(Ls-1,0) = mass*cee[Ls-1];
   
-  Eigen::MatrixXd PplusMat ;
-  Eigen::MatrixXd PminusMat;
+  Eigen::MatrixXcd PplusMat ;
+  Eigen::MatrixXcd PminusMat;
   
   if ( inv ) {
     PplusMat =Pplus.inverse();
@@ -298,8 +298,12 @@ PARALLEL_FOR_LOOP
 
 INSTANTIATE_DPERP(DomainWallVec5dImplD);
 INSTANTIATE_DPERP(DomainWallVec5dImplF);
+INSTANTIATE_DPERP(ZDomainWallVec5dImplD);
+INSTANTIATE_DPERP(ZDomainWallVec5dImplF);
 
 template void CayleyFermion5D<DomainWallVec5dImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<DomainWallVec5dImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZDomainWallVec5dImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZDomainWallVec5dImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 
 }}

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -25,7 +25,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef GRID_QCD_FERMION_OPERATOR_IMPL_H
@@ -100,7 +101,8 @@ namespace Grid {
     typedef typename Impl::SiteHalfSpinor       SiteHalfSpinor;		\
     typedef typename Impl::Compressor               Compressor;		\
     typedef typename Impl::StencilImpl             StencilImpl;		\
-    typedef typename Impl::ImplParams ImplParams;
+    typedef typename Impl::ImplParams ImplParams;			\
+    typedef typename Impl::Coeff_t       Coeff_t;
 
 #define INHERIT_IMPL_TYPES(Base) \
     INHERIT_GIMPL_TYPES(Base)	 \
@@ -109,19 +111,25 @@ namespace Grid {
     ///////
     // Single flavour four spinors with colour index
     ///////
-    template<class S,int Nrepresentation=Nc>
-    class WilsonImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S, Nrepresentation> > { 
+    template <class S, class Representation = FundamentalRepresentation,class _Coeff_t = RealD >
+    class WilsonImpl
+      : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
     public:
+      static const int Dimension = Representation::Dimension;
+      typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
+      
+      //Necessary?
+      constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
 
       const bool LsVectorised=false;
+      typedef _Coeff_t Coeff_t;
 
-      typedef PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > Gimpl;
 
       INHERIT_GIMPL_TYPES(Gimpl);
       
-      template<typename vtype> using iImplSpinor             = iScalar<iVector<iVector<vtype, Nrepresentation>, Ns> >;
-      template<typename vtype> using iImplHalfSpinor         = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhs> >;
-      template<typename vtype> using iImplDoubledGaugeField  = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds >;
+      template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
+      template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
+      template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
       
       typedef iImplSpinor<Simd>            SiteSpinor;
       typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
@@ -140,16 +148,24 @@ namespace Grid {
       
       bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
       
-      inline void multLink(SiteHalfSpinor &phi,const SiteDoubledGaugeField &U,const SiteHalfSpinor &chi,int mu,StencilEntry *SE,StencilImpl &St){
+      inline void multLink(SiteHalfSpinor &phi,
+			   const SiteDoubledGaugeField &U,
+			   const SiteHalfSpinor &chi,
+			   int mu,
+			   StencilEntry *SE,
+			   StencilImpl &St) {
 	mult(&phi(), &U(mu), &chi());
       }
       
       template <class ref>
-      inline void loadLinkElement(Simd & reg,ref &memory){
+      inline void loadLinkElement(Simd &reg,
+				  ref &memory) {
 	reg = memory;
       }
-      inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
-      {
+      
+      inline void DoubleStore(GridBase *GaugeGrid,
+			      DoubledGaugeField &Uds,
+			      const GaugeField &Umu) {
 	conformable(Uds._grid, GaugeGrid);
 	conformable(Umu._grid, GaugeGrid);
 	GaugeLinkField U(GaugeGrid);
@@ -184,20 +200,18 @@ PARALLEL_FOR_LOOP
 	PokeIndex<LorentzIndex>(mat,tmp,mu);
 	
       }
-
     };
 
-
-
     ///////
     // Single flavour four spinors with colour index, 5d redblack
     ///////
-    template<class S,int Nrepresentation=Nc>
+    template<class S,int Nrepresentation=Nc,class _Coeff_t = RealD>
     class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { 
     public:
       
+      static const int Dimension = Nrepresentation;
       const bool LsVectorised=true;
-
+      typedef _Coeff_t Coeff_t;      
       typedef PeriodicGaugeImpl<GaugeImplTypes<S, Nrepresentation> > Gimpl;
 
       INHERIT_GIMPL_TYPES(Gimpl);
@@ -213,9 +227,12 @@ PARALLEL_FOR_LOOP
       typedef Lattice<SiteSpinor> FermionField;
       
       // Make the doubled gauge field a *scalar*
-      typedef iImplDoubledGaugeField<typename Simd::scalar_type>    SiteDoubledGaugeField; // This is a scalar
-      typedef iImplGaugeField<typename Simd::scalar_type>           SiteScalarGaugeField;  // scalar
-      typedef iImplGaugeLink <typename Simd::scalar_type>           SiteScalarGaugeLink;   // scalar
+      typedef iImplDoubledGaugeField<typename Simd::scalar_type>
+      SiteDoubledGaugeField;  // This is a scalar
+      typedef iImplGaugeField<typename Simd::scalar_type>
+      SiteScalarGaugeField;  // scalar
+      typedef iImplGaugeLink<typename Simd::scalar_type>
+      SiteScalarGaugeLink;  // scalar
       
       typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
       
@@ -233,8 +250,9 @@ PARALLEL_FOR_LOOP
       inline void loadLinkElement(Simd &reg, ref &memory) {
 	vsplat(reg, memory);
       }
-      inline void multLink(SiteHalfSpinor &phi,const SiteDoubledGaugeField &U,const SiteHalfSpinor &chi,int mu,StencilEntry *SE,StencilImpl &St)
-      {
+      inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
+			   const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
+			   StencilImpl &St) {
 	SiteGaugeLink UU;
 	for (int i = 0; i < Nrepresentation; i++) {
 	  for (int j = 0; j < Nrepresentation; j++) {
@@ -244,8 +262,8 @@ PARALLEL_FOR_LOOP
 	mult(&phi(), &UU(), &chi());
       }
       
-      inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
-      {
+      inline void DoubleStore(GridBase *GaugeGrid, DoubledGaugeField &Uds,
+			      const GaugeField &Umu) {
 	SiteScalarGaugeField ScalarUmu;
 	SiteDoubledGaugeField ScalarUds;
 	
@@ -269,37 +287,45 @@ PARALLEL_FOR_LOOP
 	  
 	  pokeLocalSite(ScalarUds, Uds, lcoor);
 	}
-
       }
       
-      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A,int mu){
+      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,
+				FermionField &A, int mu) {
 	assert(0);
       }
       
-      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde,int mu){
+      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,
+				FermionField &Atilde, int mu) {
 	assert(0);
       }
-
     };
     
-
     ////////////////////////////////////////////////////////////////////////////////////////
     // Flavour doubled spinors; is Gparity the only? what about C*?
     ////////////////////////////////////////////////////////////////////////////////////////
     
-    template<class S,int Nrepresentation>
-    class GparityWilsonImpl : public ConjugateGaugeImpl< GaugeImplTypes<S,Nrepresentation> >{ 
+    template <class S, int Nrepresentation,class _Coeff_t = RealD>
+    class GparityWilsonImpl
+      : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
     public:
+      static const int Dimension = Nrepresentation;
 
       const bool LsVectorised=false;
 
+      typedef _Coeff_t Coeff_t;
       typedef ConjugateGaugeImpl< GaugeImplTypes<S,Nrepresentation> > Gimpl;
       
       INHERIT_GIMPL_TYPES(Gimpl);
       
-      template<typename vtype> using iImplSpinor             = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp >;
-      template<typename vtype> using iImplHalfSpinor         = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp >;
-      template<typename vtype> using iImplDoubledGaugeField  = iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds >, Ngp >;
+      template <typename vtype>
+      using iImplSpinor =
+      iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
+      template <typename vtype>
+      using iImplHalfSpinor =
+	iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
+      template <typename vtype>
+      using iImplDoubledGaugeField =
+	iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
       
       typedef iImplSpinor<Simd> SiteSpinor;
       typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
@@ -315,13 +341,16 @@ PARALLEL_FOR_LOOP
       
       ImplParams Params;
 
+
       GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
 
       bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
 
-      // provide the multiply by link that is differentiated between Gparity (with flavour index) and non-Gparity
-      inline void multLink(SiteHalfSpinor &phi,const SiteDoubledGaugeField &U,const SiteHalfSpinor &chi,int mu,StencilEntry *SE,StencilImpl &St){
-
+      // provide the multiply by link that is differentiated between Gparity (with
+      // flavour index) and non-Gparity
+      inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
+			   const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
+			   StencilImpl &St) {
 	typedef SiteHalfSpinor vobj;
 	typedef typename SiteHalfSpinor::scalar_object sobj;
 	
@@ -447,21 +476,23 @@ PARALLEL_FOR_LOOP
 	}
       }
       
-      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A,int mu){
       
+      inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,
+				FermionField &A, int mu) {
 	// DhopDir provides U or Uconj depending on coor/flavour.
 	GaugeLinkField link(mat._grid);
 	// use lorentz for flavour as hack.
+	auto tmp = TraceIndex<SpinIndex>(outerProduct(Btilde, A));
 	PARALLEL_FOR_LOOP
-        for(auto ss=link.begin();ss<link.end();ss++){
-	  auto ttmp = traceIndex<SpinIndex>(outerProduct(Btilde[ss],A[ss]));  
-	  link[ss]() = ttmp(0,0) + conjugate(ttmp(1,1)) ; 
+	  for (auto ss = tmp.begin(); ss < tmp.end(); ss++) {
+	    link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1));
 	  }
 	PokeIndex<LorentzIndex>(mat, link, mu);
 	return;
       }
-      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde,int mu){
       
+      inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,
+				FermionField &Atilde, int mu) {
 	int Ls = Btilde._grid->_fdimensions[0];
 	
 	GaugeLinkField tmp(mat._grid);
@@ -479,18 +510,34 @@ PARALLEL_FOR_LOOP
       }
     };
 
-    typedef WilsonImpl<vComplex ,Nc> WilsonImplR; // Real.. whichever prec
-    typedef WilsonImpl<vComplexF,Nc> WilsonImplF; // Float
-    typedef WilsonImpl<vComplexD,Nc> WilsonImplD; // Double
+    typedef WilsonImpl<vComplex,  FundamentalRepresentation > WilsonImplR;   // Real.. whichever prec
+    typedef WilsonImpl<vComplexF, FundamentalRepresentation > WilsonImplF;  // Float
+    typedef WilsonImpl<vComplexD, FundamentalRepresentation > WilsonImplD;  // Double
+
+
+    typedef WilsonImpl<vComplex,  FundamentalRepresentation, ComplexD > ZWilsonImplR; // Real.. whichever prec
+    typedef WilsonImpl<vComplexF, FundamentalRepresentation, ComplexD > ZWilsonImplF; // Float
+    typedef WilsonImpl<vComplexD, FundamentalRepresentation, ComplexD > ZWilsonImplD; // Double
+
+    typedef WilsonImpl<vComplex,  AdjointRepresentation > WilsonAdjImplR;   // Real.. whichever prec
+    typedef WilsonImpl<vComplexF, AdjointRepresentation > WilsonAdjImplF;  // Float
+    typedef WilsonImpl<vComplexD, AdjointRepresentation > WilsonAdjImplD;  // Double
+
+    typedef WilsonImpl<vComplex,  TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplR;   // Real.. whichever prec
+    typedef WilsonImpl<vComplexF, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplF;  // Float
+    typedef WilsonImpl<vComplexD, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplD;  // Double
 
     typedef DomainWallVec5dImpl<vComplex ,Nc> DomainWallVec5dImplR; // Real.. whichever prec
     typedef DomainWallVec5dImpl<vComplexF,Nc> DomainWallVec5dImplF; // Float
     typedef DomainWallVec5dImpl<vComplexD,Nc> DomainWallVec5dImplD; // Double
     
+    typedef DomainWallVec5dImpl<vComplex ,Nc,ComplexD> ZDomainWallVec5dImplR; // Real.. whichever prec
+    typedef DomainWallVec5dImpl<vComplexF,Nc,ComplexD> ZDomainWallVec5dImplF; // Float
+    typedef DomainWallVec5dImpl<vComplexD,Nc,ComplexD> ZDomainWallVec5dImplD; // Double
+
     typedef GparityWilsonImpl<vComplex, Nc>  GparityWilsonImplR;  // Real.. whichever prec
     typedef GparityWilsonImpl<vComplexF, Nc> GparityWilsonImplF;  // Float
     typedef GparityWilsonImpl<vComplexD, Nc> GparityWilsonImplD;  // Double
-
 }
 }
 #endif

lib/qcd/action/fermion/WilsonFermion.cc

@@ -25,7 +25,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #include <Grid.h>
@@ -33,8 +34,10 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 namespace QCD {
 
-  const std::vector<int> WilsonFermionStatic::directions   ({0,1,2,3, 0, 1, 2, 3});
-  const std::vector<int> WilsonFermionStatic::displacements({1,1,1,1,-1,-1,-1,-1});
+const std::vector<int> WilsonFermionStatic::directions({0, 1, 2, 3, 0, 1, 2,
+                                                        3});
+const std::vector<int> WilsonFermionStatic::displacements({1, 1, 1, 1, -1, -1,
+                                                           -1, -1});
 int WilsonFermionStatic::HandOptDslash;
 
 /////////////////////////////////
@@ -42,30 +45,29 @@ namespace QCD {
 /////////////////////////////////
 
 template <class Impl>
-  WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu,
-				     GridCartesian         &Fgrid,
-				     GridRedBlackCartesian &Hgrid, 
-				     RealD _mass,const ImplParams &p) :
-        Kernels(p),
+WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
+                                   GridRedBlackCartesian &Hgrid, RealD _mass,
+                                   const ImplParams &p)
+    : Kernels(p),
       _grid(&Fgrid),
       _cbgrid(&Hgrid),
       Stencil(&Fgrid, npoint, Even, directions, displacements),
-	StencilEven(&Hgrid,npoint,Even,directions,displacements), // source is Even
-	StencilOdd (&Hgrid,npoint,Odd ,directions,displacements), // source is Odd
+      StencilEven(&Hgrid, npoint, Even, directions,
+                  displacements),  // source is Even
+      StencilOdd(&Hgrid, npoint, Odd, directions,
+                 displacements),  // source is Odd
       mass(_mass),
       Lebesgue(_grid),
       LebesgueEvenOdd(_cbgrid),
       Umu(&Fgrid),
       UmuEven(&Hgrid),
-	UmuOdd (&Hgrid) 
-  {
+      UmuOdd(&Hgrid) {
   // Allocate the required comms buffer
   ImportGauge(_Umu);
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::ImportGauge(const GaugeField &_Umu)
-  {
+void WilsonFermion<Impl>::ImportGauge(const GaugeField &_Umu) {
   GaugeField HUmu(_Umu._grid);
   HUmu = _Umu * (-0.5);
   Impl::DoubleStore(GaugeGrid(), Umu, HUmu);
@@ -78,24 +80,21 @@ namespace QCD {
 /////////////////////////////
 
 template <class Impl>
-  RealD WilsonFermion<Impl>::M(const FermionField &in, FermionField &out) 
-  {
+RealD WilsonFermion<Impl>::M(const FermionField &in, FermionField &out) {
   out.checkerboard = in.checkerboard;
   Dhop(in, out, DaggerNo);
   return axpy_norm(out, 4 + mass, in, out);
 }
 
 template <class Impl>
-  RealD WilsonFermion<Impl>::Mdag(const FermionField &in, FermionField &out) 
-  {
+RealD WilsonFermion<Impl>::Mdag(const FermionField &in, FermionField &out) {
   out.checkerboard = in.checkerboard;
   Dhop(in, out, DaggerYes);
   return axpy_norm(out, 4 + mass, in, out);
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::Meooe(const FermionField &in, FermionField &out) 
-  {
+void WilsonFermion<Impl>::Meooe(const FermionField &in, FermionField &out) {
   if (in.checkerboard == Odd) {
     DhopEO(in, out, DaggerNo);
   } else {
@@ -103,8 +102,7 @@ namespace QCD {
   }
 }
 template <class Impl>
-  void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out) 
-  {
+void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out) {
   if (in.checkerboard == Odd) {
     DhopEO(in, out, DaggerYes);
   } else {
@@ -132,7 +130,8 @@ namespace QCD {
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in, FermionField &out) {
+void WilsonFermion<Impl>::MooeeInvDag(const FermionField &in,
+                                      FermionField &out) {
   out.checkerboard = in.checkerboard;
   MooeeInv(in, out);
 }
@@ -142,12 +141,9 @@ namespace QCD {
 ///////////////////////////////////
 
 template <class Impl>
-  void WilsonFermion<Impl>::DerivInternal(StencilImpl & st,
-					  DoubledGaugeField & U,
-					  GaugeField &mat,
-					  const FermionField &A,
+void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
+                                        GaugeField &mat, const FermionField &A,
                                         const FermionField &B, int dag) {
-	
   assert((dag == DaggerNo) || (dag == DaggerYes));
 
   Compressor compressor(dag);
@@ -159,7 +155,6 @@ namespace QCD {
   st.HaloExchange(B, compressor);
 
   for (int mu = 0; mu < Nd; mu++) {
-      
     ////////////////////////////////////////////////////////////////////////
     // Flip gamma (1+g)<->(1-g) if dag
     ////////////////////////////////////////////////////////////////////////
@@ -171,20 +166,20 @@ namespace QCD {
     ////////////////////////
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < B._grid->oSites(); sss++) {
-	  Kernels::DiracOptDhopDir(st,U,st.comm_buf,sss,sss,B,Btilde,mu,gamma);
+      Kernels::DiracOptDhopDir(st, U, st.comm_buf, sss, sss, B, Btilde, mu,
+                               gamma);
     }
 
     //////////////////////////////////////////////////
     // spin trace outer product
     //////////////////////////////////////////////////
     Impl::InsertForce4D(mat, Btilde, Atilde, mu);
-
   }
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag)
-  {
+void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U,
+                                    const FermionField &V, int dag) {
   conformable(U._grid, _grid);
   conformable(U._grid, V._grid);
   conformable(U._grid, mat._grid);
@@ -195,8 +190,8 @@ PARALLEL_FOR_LOOP
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag)
-  {
+void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U,
+                                      const FermionField &V, int dag) {
   conformable(U._grid, _cbgrid);
   conformable(U._grid, V._grid);
   conformable(U._grid, mat._grid);
@@ -209,8 +204,8 @@ PARALLEL_FOR_LOOP
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag)
-  {
+void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U,
+                                      const FermionField &V, int dag) {
   conformable(U._grid, _cbgrid);
   conformable(U._grid, V._grid);
   conformable(U._grid, mat._grid);
@@ -222,9 +217,9 @@ PARALLEL_FOR_LOOP
   DerivInternal(StencilOdd, UmuEven, mat, U, V, dag);
 }
 
-
 template <class Impl>
-  void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out,int dag) {
+void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out,
+                               int dag) {
   conformable(in._grid, _grid);  // verifies full grid
   conformable(in._grid, out._grid);
 
@@ -234,7 +229,8 @@ PARALLEL_FOR_LOOP
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out,int dag) {
+void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out,
+                                 int dag) {
   conformable(in._grid, _cbgrid);    // verifies half grid
   conformable(in._grid, out._grid);  // drops the cb check
 
@@ -245,7 +241,8 @@ PARALLEL_FOR_LOOP
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag) {
+void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,
+                                 int dag) {
   conformable(in._grid, _cbgrid);    // verifies half grid
   conformable(in._grid, out._grid);  // drops the cb check
 
@@ -256,40 +253,40 @@ PARALLEL_FOR_LOOP
 }
 
 template <class Impl>
-  void WilsonFermion<Impl>::Mdir (const FermionField &in, FermionField &out,int dir,int disp) {
+void WilsonFermion<Impl>::Mdir(const FermionField &in, FermionField &out,
+                               int dir, int disp) {
   DhopDir(in, out, dir, disp);
 }
 
-
 template <class Impl>
-  void WilsonFermion<Impl>::DhopDir(const FermionField &in, FermionField &out,int dir,int disp){
-    
+void WilsonFermion<Impl>::DhopDir(const FermionField &in, FermionField &out,
+                                  int dir, int disp) {
   int skip = (disp == 1) ? 0 : 1;
   int dirdisp = dir + skip * 4;
   int gamma = dir + (1 - skip) * 4;
 
   DhopDirDisp(in, out, dirdisp, gamma, DaggerNo);
-    
 };
 
 template <class Impl>
-  void WilsonFermion<Impl>::DhopDirDisp(const FermionField &in, FermionField &out,int dirdisp,int gamma,int dag) {
-    
+void WilsonFermion<Impl>::DhopDirDisp(const FermionField &in, FermionField &out,
+                                      int dirdisp, int gamma, int dag) {
   Compressor compressor(dag);
 
   Stencil.HaloExchange(in, compressor);
 
   PARALLEL_FOR_LOOP
   for (int sss = 0; sss < in._grid->oSites(); sss++) {
-	Kernels::DiracOptDhopDir(Stencil,Umu,Stencil.comm_buf,sss,sss,in,out,dirdisp,gamma);
+    Kernels::DiracOptDhopDir(Stencil, Umu, Stencil.comm_buf, sss, sss, in, out,
+                             dirdisp, gamma);
   }
-    
 };
 
 template <class Impl>
-  void WilsonFermion<Impl>::DhopInternal(StencilImpl & st,LebesgueOrder& lo,DoubledGaugeField & U,
-					 const FermionField &in, FermionField &out,int dag) 
-  {
+void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
+                                       DoubledGaugeField &U,
+                                       const FermionField &in,
+                                       FermionField &out, int dag) {
   assert((dag == DaggerNo) || (dag == DaggerYes));
 
   Compressor compressor(dag);
@@ -298,22 +295,21 @@ PARALLEL_FOR_LOOP
   if (dag == DaggerYes) {
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < in._grid->oSites(); sss++) {
-	Kernels::DiracOptDhopSiteDag(st,lo,U,st.comm_buf,sss,sss,1,1,in,out);
+      Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sss, sss, 1, 1, in,
+                                   out);
     }
   } else {
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < in._grid->oSites(); sss++) {
-	Kernels::DiracOptDhopSite(st,lo,U,st.comm_buf,sss,sss,1,1,in,out);
+      Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sss, sss, 1, 1, in,
+                                out);
     }
   }
 };
 
- 
 FermOpTemplateInstantiate(WilsonFermion);
+AdjointFermOpTemplateInstantiate(WilsonFermion);
+TwoIndexFermOpTemplateInstantiate(WilsonFermion);
 GparityFermOpTemplateInstantiate(WilsonFermion);
-
-
-}}
-
-
-
+}
+}

lib/qcd/action/fermion/WilsonFermion.h

@@ -24,7 +24,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef GRID_QCD_WILSON_FERMION_H
@@ -44,8 +45,7 @@ namespace Grid {
 };
 
 template <class Impl>
-    class WilsonFermion : public WilsonKernels<Impl>, public WilsonFermionStatic
-    {
+class WilsonFermion : public WilsonKernels<Impl>, public WilsonFermionStatic {
  public:
   INHERIT_IMPL_TYPES(Impl);
   typedef WilsonKernels<Impl> Kernels;
@@ -82,10 +82,12 @@ namespace Grid {
   // Derivative interface
   ////////////////////////
   // Interface calls an internal routine
-      void DhopDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-      void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-      void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag);
-
+  void DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V,
+                 int dag);
+  void DhopDerivOE(GaugeField &mat, const FermionField &U,
+                   const FermionField &V, int dag);
+  void DhopDerivEO(GaugeField &mat, const FermionField &U,
+                   const FermionField &V, int dag);
 
   ///////////////////////////////////////////////////////////////
   // non-hermitian hopping term; half cb or both
@@ -99,28 +101,22 @@ namespace Grid {
   ///////////////////////////////////////////////////////////////
   void Mdir(const FermionField &in, FermionField &out, int dir, int disp);
   void DhopDir(const FermionField &in, FermionField &out, int dir, int disp);
-      void DhopDirDisp(const FermionField &in, FermionField &out,int dirdisp,int gamma,int dag) ;
+  void DhopDirDisp(const FermionField &in, FermionField &out, int dirdisp,
+                   int gamma, int dag);
 
   ///////////////////////////////////////////////////////////////
   // Extra methods added by derived
   ///////////////////////////////////////////////////////////////
-      void DerivInternal(StencilImpl & st,
-			 DoubledGaugeField & U,
-			 GaugeField &mat,
-			 const FermionField &A,
-			 const FermionField &B,
-			 int dag);
+  void DerivInternal(StencilImpl &st, DoubledGaugeField &U, GaugeField &mat,
+                     const FermionField &A, const FermionField &B, int dag);
 
   void DhopInternal(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
                     const FermionField &in, FermionField &out, int dag);
 
   // Constructor
-      WilsonFermion(GaugeField &_Umu,
-		    GridCartesian         &Fgrid,
-		    GridRedBlackCartesian &Hgrid, 
-		    RealD _mass,
-		    const ImplParams &p= ImplParams()
-		    ) ;
+  WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
+                GridRedBlackCartesian &Hgrid, RealD _mass,
+                const ImplParams &p = ImplParams());
 
   // DoubleStore impl dependent
   void ImportGauge(const GaugeField &_Umu);
@@ -131,7 +127,6 @@ namespace Grid {
 
   //    protected:
  public:
-
   RealD mass;
 
   GridBase *_grid;
@@ -149,13 +144,12 @@ namespace Grid {
 
   LebesgueOrder Lebesgue;
   LebesgueOrder LebesgueEvenOdd;
-
-      
 };
 
 typedef WilsonFermion<WilsonImplF> WilsonFermionF;
 typedef WilsonFermion<WilsonImplD> WilsonFermionD;
 
+
 }
 }
 #endif

lib/qcd/action/fermion/WilsonFermion5D.cc

@@ -175,6 +175,73 @@ WilsonFermion5D<Impl>::WilsonFermion5D(int simd,GaugeField &_Umu,
 }  
   */
      
+template<class Impl>
+void WilsonFermion5D<Impl>::Report(void)
+{
+    std::vector<int> latt = GridDefaultLatt();          
+    RealD volume = Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt[mu];
+    RealD NP = _FourDimGrid->_Nprocessors;
+
+  if ( DhopCalls > 0 ) {
+    std::cout << GridLogMessage << "#### Dhop calls report " << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Number of Dhop Calls     : " << DhopCalls  << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " << DhopCommTime
+              << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : "
+              << DhopCommTime / DhopCalls << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : "
+              << DhopComputeTime << " us" << std::endl;
+    std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : "
+              << DhopComputeTime / DhopCalls << " us" << std::endl;
+
+    RealD mflops = 1344*volume*DhopCalls/DhopComputeTime;
+    std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
+    std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
+
+   }
+
+  if ( DerivCalls > 0 ) {
+  std::cout << GridLogMessage << "#### Deriv calls report "<< std::endl;
+  std::cout << GridLogMessage << "WilsonFermion5D Number of Deriv Calls    : " <<DerivCalls <<std::endl;
+  std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " <<DerivCommTime <<" us"<<std::endl;
+  std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls           : " <<DerivCommTime/DerivCalls<<" us" <<std::endl;
+  std::cout << GridLogMessage << "WilsonFermion5D Total Compute time       : " <<DerivComputeTime <<" us"<<std::endl;
+  std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls        : " <<DerivComputeTime/DerivCalls<<" us" <<std::endl;
+  std::cout << GridLogMessage << "WilsonFermion5D Total Dhop Compute time  : " <<DerivDhopComputeTime <<" us"<<std::endl;
+  std::cout << GridLogMessage << "WilsonFermion5D Dhop ComputeTime/Calls   : " <<DerivDhopComputeTime/DerivCalls<<" us" <<std::endl;
+
+
+
+  RealD mflops = 144*volume*DerivCalls/DerivDhopComputeTime;
+  std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
+  std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NP << std::endl;
+
+  }
+
+  if (DerivCalls > 0 || DhopCalls > 0){
+  std::cout << GridLogMessage << "WilsonFermion5D Stencil"<<std::endl;  Stencil.Report();
+  std::cout << GridLogMessage << "WilsonFermion5D StencilEven"<<std::endl;  StencilEven.Report();
+  std::cout << GridLogMessage << "WilsonFermion5D StencilOdd"<<std::endl;  StencilOdd.Report();
+  }
+}
+
+template<class Impl>
+void WilsonFermion5D<Impl>::ZeroCounters(void) {
+  DhopCalls       = 0;
+  DhopCommTime    = 0;
+  DhopComputeTime = 0;
+
+  DerivCalls       = 0;
+  DerivCommTime    = 0;
+  DerivComputeTime = 0;
+  DerivDhopComputeTime = 0;
+
+  Stencil.ZeroCounters();
+  StencilEven.ZeroCounters();
+  StencilOdd.ZeroCounters();
+}
+
+
 template<class Impl>
 void WilsonFermion5D<Impl>::ImportGauge(const GaugeField &_Umu)
 {
@@ -221,6 +288,7 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
             const FermionField &B,
             int dag)
 {
+  DerivCalls++;
   assert((dag==DaggerNo) ||(dag==DaggerYes));
 
   conformable(st._grid,A._grid);
@@ -231,12 +299,14 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
   FermionField Btilde(B._grid);
   FermionField Atilde(B._grid);
 
+  DerivCommTime-=usecond();
   st.HaloExchange(B,compressor);
+  DerivCommTime+=usecond();
 
   Atilde=A;
 
+  DerivComputeTime-=usecond();
   for (int mu = 0; mu < Nd; mu++) {
-      
     ////////////////////////////////////////////////////////////////////////
     // Flip gamma if dag
     ////////////////////////////////////////////////////////////////////////
@@ -247,6 +317,7 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
     // Call the single hop
     ////////////////////////
 
+    DerivDhopComputeTime -= usecond();
     PARALLEL_FOR_LOOP
     for (int sss = 0; sss < U._grid->oSites(); sss++) {
       for (int s = 0; s < Ls; s++) {
@@ -256,19 +327,18 @@ PARALLEL_FOR_LOOP
         assert(sF < B._grid->oSites());
         assert(sU < U._grid->oSites());
 
-	Kernels::DiracOptDhopDir(st,U,st.comm_buf,sF,sU,B,Btilde,mu,gamma);
+        Kernels::DiracOptDhopDir(st, U, st.comm_buf, sF, sU, B, Btilde, mu,
+                                 gamma);
 
         ////////////////////////////
         // spin trace outer product
         ////////////////////////////
-
       }
-
     }
-
+    DerivDhopComputeTime += usecond();
     Impl::InsertForce5D(mat, Btilde, Atilde, mu);
-
   }
+  DerivComputeTime += usecond();
 }
 
 template<class Impl>
@@ -326,29 +396,36 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
            DoubledGaugeField & U,
            const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls++;
   //  assert((dag==DaggerNo) ||(dag==DaggerYes));
   Compressor compressor(dag);
 
   int LLs = in._grid->_rdimensions[0];
   
+  DhopCommTime-=usecond();
   st.HaloExchange(in,compressor);
+  DhopCommTime+=usecond();
   
+  DhopComputeTime-=usecond();
   // Dhop takes the 4d grid from U, and makes a 5d index for fermion
   if (dag == DaggerYes) {
     PARALLEL_FOR_LOOP
     for (int ss = 0; ss < U._grid->oSites(); ss++) {
       int sU = ss;
       int sF = LLs * sU;
-	Kernels::DiracOptDhopSiteDag(st,lo,U,st.comm_buf,sF,sU,LLs,1,in,out);
+      Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in,
+                                   out);
     }
   } else {
     PARALLEL_FOR_LOOP
     for (int ss = 0; ss < U._grid->oSites(); ss++) {
       int sU = ss;
       int sF = LLs * sU;
-      Kernels::DiracOptDhopSite(st,lo,U,st.comm_buf,sF,sU,LLs,1,in,out);
+      Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in,
+                                out);
     }
   }
+  DhopComputeTime+=usecond();
 }
 
 

lib/qcd/action/fermion/WilsonFermion5D.h

@@ -61,6 +61,17 @@ namespace Grid {
      INHERIT_IMPL_TYPES(Impl);
      typedef WilsonKernels<Impl> Kernels;
 
+     void Report(void);
+     void ZeroCounters(void);
+     double DhopCalls;
+     double DhopCommTime;
+     double DhopComputeTime;
+
+     double DerivCalls;
+     double DerivCommTime;
+     double DerivComputeTime;
+     double DerivDhopComputeTime;
+
       ///////////////////////////////////////////////////////////////
       // Implement the abstract base
       ///////////////////////////////////////////////////////////////

lib/qcd/action/fermion/WilsonKernels.cc

@@ -24,7 +24,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #include <Grid.h>
@@ -37,60 +38,15 @@ namespace QCD {
 template <class Impl>
 WilsonKernels<Impl>::WilsonKernels(const ImplParams &p) : Base(p){};
 
-template<class Impl> 
-void WilsonKernels<Impl>::DiracOptDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-						  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-						  int sF,int sU,int Ls, int Ns, const FermionField &in, FermionField &out)
-{
-#ifdef AVX512
-  if ( AsmOpt ) {
-
-    WilsonKernels<Impl>::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out);
-
-  } else {
-#else
-  {  
-#endif
-    for(int site=0;site<Ns;site++) {
-      for(int s=0;s<Ls;s++) {
-	if (HandOpt) WilsonKernels<Impl>::DiracOptHandDhopSite(st,lo,U,buf,sF,sU,in,out);
-	else         WilsonKernels<Impl>::DiracOptGenericDhopSite(st,lo,U,buf,sF,sU,in,out);
-	sF++;
-      }
-      sU++;
-    }
-
-  }
-}
-
-template<class Impl> 
-void WilsonKernels<Impl>::DiracOptDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-					   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-					   int sF,int sU,int Ls, int Ns, const FermionField &in, FermionField &out)
-{
-  // No asm implementation yet.
-  //  if ( AsmOpt )     WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
-  //  else
-  for(int site=0;site<Ns;site++) {
-    for(int s=0;s<Ls;s++) {
-      if (HandOpt) WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
-      else         WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out);
-      sF++;
-    }
-    sU++;
-  }
-}
-
-
 ////////////////////////////////////////////
 // Generic implementation; move to different file?
 ////////////////////////////////////////////
 
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-					   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-					   int sF,int sU,const FermionField &in, FermionField &out)
-{
+void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(
+    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
+    int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteHalfSpinor *chi_p;
@@ -262,13 +218,12 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(StencilImpl &st,LebesgueOrd
   vstream(out._odata[sF], result);
 };
 
-
 // Need controls to do interior, exterior, or both
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptGenericDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-						  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-						  int sF,int sU,const FermionField &in, FermionField &out)
-{
+void WilsonKernels<Impl>::DiracOptGenericDhopSite(
+    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
+    int sU, const FermionField &in, FermionField &out) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteHalfSpinor *chi_p;
@@ -441,10 +396,10 @@ void WilsonKernels<Impl>::DiracOptGenericDhopSite(StencilImpl &st,LebesgueOrder
 };
 
 template <class Impl>
-void WilsonKernels<Impl>::DiracOptDhopDir(StencilImpl &st,DoubledGaugeField &U,
-					  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-					  int sF,int sU,const FermionField &in, FermionField &out,int dir,int gamma)
-{
+void WilsonKernels<Impl>::DiracOptDhopDir(
+    StencilImpl &st, DoubledGaugeField &U,
+    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf, int sF,
+    int sU, const FermionField &in, FermionField &out, int dir, int gamma) {
   SiteHalfSpinor tmp;
   SiteHalfSpinor chi;
   SiteSpinor result;
@@ -569,7 +524,9 @@ void WilsonKernels<Impl>::DiracOptDhopDir(StencilImpl &st,DoubledGaugeField &U,
   vstream(out._odata[sF], result);
 }
 
-
 FermOpTemplateInstantiate(WilsonKernels);
+AdjointFermOpTemplateInstantiate(WilsonKernels);
+TwoIndexFermOpTemplateInstantiate(WilsonKernels);
 
 }}
+

lib/qcd/action/fermion/WilsonKernels.h

@@ -24,7 +24,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef GRID_QCD_DHOP_H
@@ -53,46 +54,153 @@ namespace Grid {
      
     public:
 
-     void DiracOptDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+      template <bool EnableBool = true>
+      typename std::enable_if<Impl::Dimension == 3 && Nc == 3 &&EnableBool, void>::type
+	DiracOptDhopSite(
+			 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
 			 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
-			   int sF, int sU,int Ls, int Ns, const FermionField &in, FermionField &out);
+			 int sF, int sU, int Ls, int Ns, const FermionField &in,
+			 FermionField &out) {
+#ifdef AVX512
+	if (AsmOpt) {
+	  WilsonKernels<Impl>::DiracOptAsmDhopSite(st, lo, U, buf, sF, sU, Ls, Ns,
+						   in, out);
 
-     void DiracOptDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-			      std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-			      int sF,int sU,int Ls, int Ns, const FermionField &in,FermionField &out);
+	} else {
+#else
+	  {
+#endif
+	    for (int site = 0; site < Ns; site++) {
+	      for (int s = 0; s < Ls; s++) {
+		if (HandOpt)
+		  WilsonKernels<Impl>::DiracOptHandDhopSite(st, lo, U, buf, sF, sU,
+							    in, out);
+		else
+		  WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU,
+							       in, out);
+		sF++;
+	      }
+	      sU++;
+	    }
+	  }
+	}
 
-     void DiracOptDhopDir(StencilImpl &st,DoubledGaugeField &U,
+	template <bool EnableBool = true>
+	  typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type
+	  DiracOptDhopSite(
+			   StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
 			   std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
-			  int sF,int sU,const FermionField &in, FermionField &out,int dirdisp,int gamma);
+			   int sF, int sU, int Ls, int Ns, const FermionField &in,
+			   FermionField &out) {
+	  for (int site = 0; site < Ns; site++) {
+	    for (int s = 0; s < Ls; s++) {
+	      WilsonKernels<Impl>::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in,
+							   out);
+	      sF++;
+	    }
+	    sU++;
+	  }
+	}
+
+	template <bool EnableBool = true>
+	  typename std::enable_if<Impl::Dimension == 3 && Nc == 3 && EnableBool,
+				  void>::type
+	  DiracOptDhopSiteDag(
+			      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+			      std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+			      int sF, int sU, int Ls, int Ns, const FermionField &in,
+			      FermionField &out) {
+#ifdef AVX512
+				    if (AsmOpt) {
+				      WilsonKernels<Impl>::DiracOptAsmDhopSiteDag(st, lo, U, buf, sF, sU, Ls,
+										  Ns, in, out);
+				    } else {
+#else
+				      {
+#endif
+					for (int site = 0; site < Ns; site++) {
+					  for (int s = 0; s < Ls; s++) {
+					    if (HandOpt)
+					      WilsonKernels<Impl>::DiracOptHandDhopSiteDag(st, lo, U, buf, sF, sU,
+											   in, out);
+					    else
+					      WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF,
+											      sU, in, out);
+					    sF++;
+					  }
+					  sU++;
+					}
+				      }
+				    }
+
+				    template <bool EnableBool = true>
+				      typename std::enable_if<
+				      (Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool,
+				      void>::type
+				      DiracOptDhopSiteDag(
+							  StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+							  std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							  int sF, int sU, int Ls, int Ns, const FermionField &in,
+							  FermionField &out) {
+					for (int site = 0; site < Ns; site++) {
+					  for (int s = 0; s < Ls; s++) {
+					    WilsonKernels<Impl>::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF, sU,
+											    in, out);
+					    sF++;
+					  }
+					  sU++;
+					}
+				      }
+
+				    void DiracOptDhopDir(
+							 StencilImpl &st, DoubledGaugeField &U,
+							 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+							 int sF, int sU, const FermionField &in, FermionField &out, int dirdisp,
+							 int gamma);
 
 	private:
 				    // Specialised variants
-     void DiracOptGenericDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+				    void DiracOptGenericDhopSite(
+								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
 								 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 								 int sF, int sU, const FermionField &in, FermionField &out);
 
-     void DiracOptGenericDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+				    void DiracOptGenericDhopSiteDag(
+								    StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
 								    std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 								    int sF, int sU, const FermionField &in, FermionField &out);
 
-     void DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+				    void DiracOptAsmDhopSite(
+							     StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
 							     std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
-			      int sF,int sU,int Ls, int Ns, const FermionField &in, FermionField &out);
+							     int sF, int sU, int Ls, int Ns, const FermionField &in,
+							     FermionField &out);
 
+				    void DiracOptAsmDhopSiteDag(
+								StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+								std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
+								int sF, int sU, int Ls, int Ns, const FermionField &in,
+								FermionField &out);
 
-     void DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+				    void DiracOptHandDhopSite(
+							      StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
 							      std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 							      int sF, int sU, const FermionField &in, FermionField &out);
 
-     void DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
+				    void DiracOptHandDhopSiteDag(
+								 StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
 								 std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > &buf,
 								 int sF, int sU, const FermionField &in, FermionField &out);
+
 	public:
-
 				    WilsonKernels(const ImplParams &p = ImplParams());
-     
 				  };
     
       }
     }
+
+
+
+
+
 #endif

lib/qcd/action/fermion/WilsonKernelsAsm.cc

@@ -31,10 +31,10 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #include <Grid.h>
 
+
 namespace Grid {
   namespace QCD {
     
-
     ///////////////////////////////////////////////////////////
     // Default to no assembler implementation
     ///////////////////////////////////////////////////////////
@@ -45,6 +45,15 @@ void WilsonKernels<Impl >::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & l
     {
       assert(0);
     }
+    template<class Impl>
+      void WilsonKernels<Impl >::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+                                std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+                                int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+    {
+      assert(0);
+    }
+
+
 
 #if defined(AVX512) 
     
@@ -73,12 +82,21 @@ static int signInit = setupSigns();
 #define MAYBEPERM(A,perm) if (perm) { A ; }
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf)
 #define FX(A) WILSONASM_ ##A
+  
+#undef KERNEL_DAG
     template<>
     void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
 							 std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 							 int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
       
+#define KERNEL_DAG
+    template<>
+    void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+							    std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+							    int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
 #undef VMOVIDUP
 #undef VMOVRDUP
 #undef MAYBEPERM
@@ -89,32 +107,43 @@ void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrd
 #define VMOVIDUP(A,B,C)                                  VBCASTIDUPf(A,B,C)
 #define VMOVRDUP(A,B,C)                                  VBCASTRDUPf(A,B,C)
 #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf)
+				    
+#undef KERNEL_DAG
     template<>
     void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
 								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
 								  int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
 #include <qcd/action/fermion/WilsonKernelsAsmBody.h>
 				    
+#define KERNEL_DAG
+    template<>
+    void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
+								     std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+								     int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out)
+#include <qcd/action/fermion/WilsonKernelsAsmBody.h>
+				    
 #endif
 
-template void WilsonKernels<WilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							      int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);		
 
-template void WilsonKernels<WilsonImplD>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, 
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							       int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);		
-template void WilsonKernels<GparityWilsonImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, 
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							       int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);		
-template void WilsonKernels<GparityWilsonImplD>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, 
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							       int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);		
-template void WilsonKernels<DomainWallVec5dImplF>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, 
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							       int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);		
-template void WilsonKernels<DomainWallVec5dImplD>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, 
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							       int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);		
-}}
+#define INSTANTIATE_ASM(A)\
+template void WilsonKernels<A>::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
+                                   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+                                  int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
+template void WilsonKernels<A>::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\
+                                   std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+                                  int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\
+
+
+INSTANTIATE_ASM(WilsonImplF);
+INSTANTIATE_ASM(WilsonImplD);
+INSTANTIATE_ASM(ZWilsonImplF);
+INSTANTIATE_ASM(ZWilsonImplD);
+INSTANTIATE_ASM(GparityWilsonImplF);
+INSTANTIATE_ASM(GparityWilsonImplD);
+INSTANTIATE_ASM(DomainWallVec5dImplF);
+INSTANTIATE_ASM(DomainWallVec5dImplD);
+INSTANTIATE_ASM(ZDomainWallVec5dImplF);
+INSTANTIATE_ASM(ZDomainWallVec5dImplD);
+  }
+}
 

lib/qcd/action/fermion/WilsonKernelsAsmBody.h

@@ -30,7 +30,11 @@
   basep = st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);
+#ifdef KERNEL_DAG
+    XP_PROJMEM(base);
+#else 
     XM_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR3,perm);
   } else { 
     LOAD_CHI(base);
@@ -41,15 +45,22 @@
     MULT_2SPIN_DIR_PFXP(Xp,basep);
   }
   LOAD64(%r10,isigns);
+#ifdef KERNEL_DAG
+  XP_RECON;
+#else
   XM_RECON;
-
+#endif
   ////////////////////////////////
   // Yp
   ////////////////////////////////
   basep = st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+    YP_PROJMEM(base);
+#else
     YM_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR2,perm);
   } else { 
     LOAD_CHI(base);
@@ -60,7 +71,11 @@
     MULT_2SPIN_DIR_PFYP(Yp,basep);
   }
   LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+  YP_RECON_ACCUM;
+#else
   YM_RECON_ACCUM;
+#endif
 
   ////////////////////////////////
   // Zp
@@ -68,7 +83,11 @@
   basep = st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+    ZP_PROJMEM(base);
+#else
     ZM_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR1,perm);
   } else { 
     LOAD_CHI(base);
@@ -79,7 +98,11 @@
     MULT_2SPIN_DIR_PFZP(Zp,basep);
   }
   LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+  ZP_RECON_ACCUM;
+#else
   ZM_RECON_ACCUM;
+#endif
 
   ////////////////////////////////
   // Tp
@@ -87,7 +110,11 @@
   basep = st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+    TP_PROJMEM(base);
+#else
     TM_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR0,perm);
   } else { 
     LOAD_CHI(base);
@@ -98,7 +125,11 @@
     MULT_2SPIN_DIR_PFTP(Tp,basep);
   }
   LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+  TP_RECON_ACCUM;
+#else
   TM_RECON_ACCUM;
+#endif
 
   ////////////////////////////////
   // Xm
@@ -107,7 +138,11 @@
   //  basep= st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+    XM_PROJMEM(base);
+#else
     XP_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR3,perm);
   } else { 
     LOAD_CHI(base);
@@ -118,7 +153,11 @@
     MULT_2SPIN_DIR_PFXM(Xm,basep);
   }
   LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+  XM_RECON_ACCUM;
+#else
   XP_RECON_ACCUM;
+#endif
 
   ////////////////////////////////
   // Ym
@@ -126,7 +165,11 @@
   basep= st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+    YM_PROJMEM(base);
+#else
     YP_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR2,perm);
   } else { 
     LOAD_CHI(base);
@@ -137,7 +180,11 @@
     MULT_2SPIN_DIR_PFYM(Ym,basep);
   }
   LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+  YM_RECON_ACCUM;
+#else
   YP_RECON_ACCUM;
+#endif
 
   ////////////////////////////////
   // Zm
@@ -145,7 +192,11 @@
   basep= st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+    ZM_PROJMEM(base);
+#else
     ZP_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR1,perm);
   } else { 
     LOAD_CHI(base);
@@ -156,7 +207,11 @@
     MULT_2SPIN_DIR_PFZM(Zm,basep);
   }
   LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+  ZM_RECON_ACCUM;
+#else
   ZP_RECON_ACCUM;
+#endif
 
   ////////////////////////////////
   // Tm
@@ -164,7 +219,11 @@
   basep= st.GetPFInfo(nent,plocal); nent++;
   if ( local ) {
     LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+    TM_PROJMEM(base);
+#else
     TP_PROJMEM(base);
+#endif
     MAYBEPERM(PERMUTE_DIR0,perm);
   } else { 
     LOAD_CHI(base);
@@ -175,7 +234,11 @@
     MULT_2SPIN_DIR_PFTM(Tm,basep);
   }
   LOAD64(%r10,isigns);  // times i => shuffle and xor the real part sign bit
+#ifdef KERNEL_DAG
+  TM_RECON_ACCUM;
+#else
   TP_RECON_ACCUM;
+#endif
 
   basep= st.GetPFInfo(nent,plocal); nent++;
   SAVE_RESULT(base,basep);

lib/qcd/action/fermion/WilsonKernelsHand.cc

@@ -839,46 +839,23 @@ void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,
 ////////////// Wilson ; uses this implementation /////////////////////
 // Need Nc=3 though //
 
-template void WilsonKernels<WilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							       int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<WilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-							       int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<WilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-								  int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<WilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-								  int ss,int sU,const FermionField &in, FermionField &out);
-
-
-template void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								      std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-								      int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								      std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-								      int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<GparityWilsonImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-									 std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-									 int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<GparityWilsonImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-									 std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-									 int ss,int sU,const FermionField &in, FermionField &out);
-
-
-template void WilsonKernels<DomainWallVec5dImplF>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								      std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-								      int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<DomainWallVec5dImplD>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-								      std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-								      int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<DomainWallVec5dImplF>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-									 std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
-									 int ss,int sU,const FermionField &in, FermionField &out);
-template void WilsonKernels<DomainWallVec5dImplD>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,
-									 std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,
+#define INSTANTIATE_THEM(A) \
+template void WilsonKernels<A>::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
+							       std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
+							       int ss,int sU,const FermionField &in, FermionField &out);\
+template void WilsonKernels<A>::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\
+								  std::vector<SiteHalfSpinor,alignedAllocator<SiteHalfSpinor> >  &buf,\
 								  int ss,int sU,const FermionField &in, FermionField &out);
 
+INSTANTIATE_THEM(WilsonImplF);
+INSTANTIATE_THEM(WilsonImplD);
+INSTANTIATE_THEM(ZWilsonImplF);
+INSTANTIATE_THEM(ZWilsonImplD);
+INSTANTIATE_THEM(GparityWilsonImplF);
+INSTANTIATE_THEM(GparityWilsonImplD);
+INSTANTIATE_THEM(DomainWallVec5dImplF);
+INSTANTIATE_THEM(DomainWallVec5dImplD);
+INSTANTIATE_THEM(ZDomainWallVec5dImplF);
+INSTANTIATE_THEM(ZDomainWallVec5dImplD);
 
 }}

lib/qcd/action/fermion/ZMobiusFermion.h

@@ -0,0 +1,79 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/fermion/MobiusFermion.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef  GRID_QCD_ZMOBIUS_FERMION_H
+#define  GRID_QCD_ZMOBIUS_FERMION_H
+
+#include <Grid/Grid.h>
+
+namespace Grid {
+
+  namespace QCD {
+
+    template<class Impl>
+    class ZMobiusFermion : public CayleyFermion5D<Impl>
+    {
+    public:
+     INHERIT_IMPL_TYPES(Impl);
+    public:
+
+      virtual void   Instantiatable(void) {};
+      // Constructors
+      ZMobiusFermion(GaugeField &_Umu,
+		     GridCartesian         &FiveDimGrid,
+		     GridRedBlackCartesian &FiveDimRedBlackGrid,
+		     GridCartesian         &FourDimGrid,
+		     GridRedBlackCartesian &FourDimRedBlackGrid,
+		     RealD _mass,RealD _M5,
+		     std::vector<ComplexD> &gamma, RealD b,RealD c,const ImplParams &p= ImplParams()) : 
+      
+      CayleyFermion5D<Impl>(_Umu,
+			    FiveDimGrid,
+			    FiveDimRedBlackGrid,
+			    FourDimGrid,
+			    FourDimRedBlackGrid,_mass,_M5,p)
+
+      {
+	RealD eps = 1.0;
+	
+	std::cout<<GridLogMessage << "ZMobiusFermion (b="<<b<<",c="<<c<<") with Ls= "<<this->Ls<<" gamma passed in"<<std::endl;
+	std::vector<Coeff_t> zgamma(this->Ls);
+	for(int s=0;s<this->Ls;s++){
+	  zgamma[s] = gamma[s];
+	}
+
+	// Call base setter
+	this->SetCoefficientsInternal(1.0,zgamma,b,c);
+      }
+
+    };
+
+  }
+}
+
+#endif

lib/qcd/action/gauge/GaugeImpl.h

@@ -30,7 +30,6 @@ directory
 #define GRID_QCD_GAUGE_IMPL_H
 
 namespace Grid {
-
 namespace QCD {
 
 ////////////////////////////////////////////////////////////////////////
@@ -64,7 +63,7 @@ public:
                                                  // ugly
   typedef Lattice<SiteGaugeField> GaugeField;
 
-  // Move this elsewhere?
+  // Move this elsewhere? FIXME
   static inline void AddGaugeLink(GaugeField &U, GaugeLinkField &W,
                                   int mu) { // U[mu] += W
     PARALLEL_FOR_LOOP
@@ -174,12 +173,19 @@ typedef GaugeImplTypes<vComplex, Nc> GimplTypesR;
 typedef GaugeImplTypes<vComplexF, Nc> GimplTypesF;
 typedef GaugeImplTypes<vComplexD, Nc> GimplTypesD;
 
+typedef GaugeImplTypes<vComplex, SU<Nc>::AdjointDimension> GimplAdjointTypesR;
+typedef GaugeImplTypes<vComplexF, SU<Nc>::AdjointDimension> GimplAdjointTypesF;
+typedef GaugeImplTypes<vComplexD, SU<Nc>::AdjointDimension> GimplAdjointTypesD;
+
 typedef PeriodicGaugeImpl<GimplTypesR> PeriodicGimplR; // Real.. whichever prec
 typedef PeriodicGaugeImpl<GimplTypesF> PeriodicGimplF; // Float
 typedef PeriodicGaugeImpl<GimplTypesD> PeriodicGimplD; // Double
 
-typedef ConjugateGaugeImpl<GimplTypesR>
-    ConjugateGimplR; // Real.. whichever prec
+typedef PeriodicGaugeImpl<GimplAdjointTypesR> PeriodicGimplAdjR; // Real.. whichever prec
+typedef PeriodicGaugeImpl<GimplAdjointTypesF> PeriodicGimplAdjF; // Float
+typedef PeriodicGaugeImpl<GimplAdjointTypesD> PeriodicGimplAdjD; // Double
+
+typedef ConjugateGaugeImpl<GimplTypesR> ConjugateGimplR; // Real.. whichever prec
 typedef ConjugateGaugeImpl<GimplTypesF> ConjugateGimplF; // Float
 typedef ConjugateGaugeImpl<GimplTypesD> ConjugateGimplD; // Double
 }

lib/qcd/action/pseudofermion/TwoFlavour.h

@@ -23,7 +23,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef QCD_PSEUDOFERMION_TWO_FLAVOUR_H
@@ -41,7 +42,6 @@ namespace Grid{
   INHERIT_IMPL_TYPES(Impl);
 
  private:
-      
   FermionOperator<Impl> &FermOp;  // the basic operator
 
   OperatorFunction<FermionField> &DerivativeSolver;
@@ -56,15 +56,17 @@ namespace Grid{
   /////////////////////////////////////////////////
   TwoFlavourPseudoFermionAction(FermionOperator<Impl> &Op,
                                 OperatorFunction<FermionField> &DS,
-				  OperatorFunction<FermionField> & AS
-				  ) : FermOp(Op), DerivativeSolver(DS), ActionSolver(AS), Phi(Op.FermionGrid()) {
-      };
+                                OperatorFunction<FermionField> &AS)
+      : FermOp(Op),
+        DerivativeSolver(DS),
+        ActionSolver(AS),
+        Phi(Op.FermionGrid()){};
 
   //////////////////////////////////////////////////////////////////////////////////////
-      // Push the gauge field in to the dops. Assume any BC's and smearing already applied
+  // Push the gauge field in to the dops. Assume any BC's and smearing already
+  // applied
   //////////////////////////////////////////////////////////////////////////////////////
   virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) {
-
     // P(phi) = e^{- phi^dag (MdagM)^-1 phi}
     // Phi = Mdag eta
     // P(eta) = e^{- eta^dag eta}
@@ -76,7 +78,8 @@ namespace Grid{
     //
     // Chroma has this scale factor: two_flavor_monomial_w.h
     // IroIro: does not use this scale. It is absorbed by a change of vars
-	//         in the Phi integral, and thus is only an irrelevant prefactor for the partition function.
+    //         in the Phi integral, and thus is only an irrelevant prefactor for
+    //         the partition function.
     //
     RealD scale = std::sqrt(0.5);
     FermionField eta(FermOp.FermionGrid());
@@ -87,14 +90,12 @@ namespace Grid{
     FermOp.Mdag(eta, Phi);
 
     Phi = Phi * scale;
-	
   };
 
   //////////////////////////////////////////////////////
   // S = phi^dag (Mdag M)^-1 phi
   //////////////////////////////////////////////////////
   virtual RealD S(const GaugeField &U) {
-
     FermOp.ImportGauge(U);
 
     FermionField X(FermOp.FermionGrid());
@@ -106,19 +107,20 @@ namespace Grid{
     MdagMOp.Op(X, Y);
 
     RealD action = norm2(Y);
-	std::cout << GridLogMessage << "Pseudofermion action "<<action<<std::endl;
+    std::cout << GridLogMessage << "Pseudofermion action " << action
+              << std::endl;
     return action;
   };
 
   //////////////////////////////////////////////////////
   // dS/du = - phi^dag  (Mdag M)^-1 [ Mdag dM + dMdag M ]  (Mdag M)^-1 phi
-      //       = - phi^dag M^-1 dM (MdagM)^-1 phi -  phi^dag (MdagM)^-1 dMdag dM (Mdag)^-1 phi 
+  //       = - phi^dag M^-1 dM (MdagM)^-1 phi -  phi^dag (MdagM)^-1 dMdag dM
+  //       (Mdag)^-1 phi
   //
   //       = - Ydag dM X  - Xdag dMdag Y
   //
   //////////////////////////////////////////////////////
   virtual void deriv(const GaugeField &U, GaugeField &dSdU) {
-
     FermOp.ImportGauge(U);
 
     FermionField X(FermOp.FermionGrid());
@@ -128,21 +130,21 @@ namespace Grid{
     MdagMLinearOperator<FermionOperator<Impl>, FermionField> MdagMOp(FermOp);
 
     X = zero;
-	DerivativeSolver(MdagMOp,Phi,X);
-	MdagMOp.Op(X,Y);
+    DerivativeSolver(MdagMOp, Phi, X); // X = (MdagM)^-1 phi    
+    MdagMOp.Op(X, Y);                  // Y = M X = (Mdag)^-1 phi
 
-	// Our conventions really make this UdSdU; We do not differentiate wrt Udag here.
+    // Our conventions really make this UdSdU; We do not differentiate wrt Udag
+    // here.
     // So must take dSdU - adj(dSdU) and left multiply by mom to get dS/dt.
 
-	FermOp.MDeriv(tmp , Y, X,DaggerNo );  dSdU=tmp;
-	FermOp.MDeriv(tmp , X, Y,DaggerYes);  dSdU=dSdU+tmp;
-	
-	//dSdU = Ta(dSdU);
+    FermOp.MDeriv(tmp, Y, X, DaggerNo);
+    dSdU = tmp;
+    FermOp.MDeriv(tmp, X, Y, DaggerYes);
+    dSdU = dSdU + tmp;
 
+    // not taking here the traceless antihermitian component
   };
-
 };
-    
 }
 }
 

lib/qcd/action/pseudofermion/TwoFlavourEvenOdd.h

@@ -23,7 +23,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef QCD_PSEUDOFERMION_TWO_FLAVOUR_EVEN_ODD_H
@@ -32,20 +33,16 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 namespace QCD {
 
-
-
 ////////////////////////////////////////////////////////////////////////
 // Two flavour pseudofermion action for any EO prec dop
 ////////////////////////////////////////////////////////////////////////
 template <class Impl>
-    class TwoFlavourEvenOddPseudoFermionAction : public Action<typename Impl::GaugeField> {
-
+class TwoFlavourEvenOddPseudoFermionAction
+    : public Action<typename Impl::GaugeField> {
  public:
-
   INHERIT_IMPL_TYPES(Impl);
 
  private:
-      
   FermionOperator<Impl> &FermOp;  // the basic operator
 
   OperatorFunction<FermionField> &DerivativeSolver;
@@ -60,9 +57,8 @@ namespace Grid{
   /////////////////////////////////////////////////
   TwoFlavourEvenOddPseudoFermionAction(FermionOperator<Impl> &Op,
                                        OperatorFunction<FermionField> &DS,
-					 OperatorFunction<FermionField> & AS
-					   ) : 
-        FermOp(Op), 
+                                       OperatorFunction<FermionField> &AS)
+      : FermOp(Op),
         DerivativeSolver(DS),
         ActionSolver(AS),
         PhiEven(Op.FermionRedBlackGrid()),

lib/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h

@@ -131,9 +131,11 @@ namespace Grid{
 	Vpc.MpcDag(PhiOdd,Y);           // Y= Vdag phi
 	X=zero;
 	ActionSolver(Mpc,Y,X);          // X= (MdagM)^-1 Vdag phi
-	Mpc.Mpc(X,Y);                   // Y=  Mdag^-1 Vdag phi
+	//Mpc.Mpc(X,Y);                   // Y=  Mdag^-1 Vdag phi
+	// Multiply by Ydag
+	RealD action = real(innerProduct(Y,X));
 
-	RealD action = norm2(Y);
+	//RealD action = norm2(Y);
 
 	// The EE factorised block; normally can replace with zero if det is constant (gauge field indept)
 	// Only really clover term that creates this. Leave the EE portion as a future to do to make most

lib/qcd/hmc/HmcRunner.h

@@ -22,7 +22,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef HMC_RUNNER
@@ -31,16 +32,14 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 namespace QCD {
 
-
-template<class Gimpl>
+template <class Gimpl, class RepresentationsPolicy = NoHirep >
 class NerscHmcRunnerTemplate {
  public:
-
   INHERIT_GIMPL_TYPES(Gimpl);
 
   enum StartType_t { ColdStart, HotStart, TepidStart, CheckpointStart };
 
-  ActionSet<GaugeField> TheAction;
+  ActionSet<GaugeField, RepresentationsPolicy> TheAction;
 
   GridCartesian *UGrid;
   GridCartesian *FGrid;
@@ -49,20 +48,26 @@ public:
 
   virtual void BuildTheAction(int argc, char **argv) = 0;  // necessary?
 
-  
   void Run(int argc, char **argv) {
-
     StartType_t StartType = HotStart;
 
     std::string arg;
 
     if (GridCmdOptionExists(argv, argv + argc, "--StartType")) {
       arg = GridCmdOptionPayload(argv, argv + argc, "--StartType");
-      if ( arg == "HotStart" ) { StartType = HotStart; }
-      else if ( arg == "ColdStart" ) { StartType = ColdStart; }
-      else if ( arg == "TepidStart" ) { StartType = TepidStart; }
-      else if ( arg == "CheckpointStart" ) { StartType = CheckpointStart; }
-      else assert(0);
+      if (arg == "HotStart") {
+        StartType = HotStart;
+      } else if (arg == "ColdStart") {
+        StartType = ColdStart;
+      } else if (arg == "TepidStart") {
+        StartType = TepidStart;
+      } else if (arg == "CheckpointStart") {
+        StartType = CheckpointStart;
+      } else {
+        std::cout << GridLogError << "Unrecognized option in --StartType\n";
+        std::cout << GridLogError << "Valid [HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+        assert(0);
+      }
     }
 
     int StartTraj = 0;
@@ -89,32 +94,34 @@ public:
       NumThermalizations = ivec[0];
     }
 
-
     GridSerialRNG sRNG;
     GridParallelRNG pRNG(UGrid);
-    LatticeGaugeField  U(UGrid); // change this to an extended field (smearing class)
+    LatticeGaugeField U(UGrid);  // change this to an extended field (smearing class)?
 
     std::vector<int> SerSeed({1, 2, 3, 4, 5});
     std::vector<int> ParSeed({6, 7, 8, 9, 10});
 
-    
     // Create integrator, including the smearing policy
-    // Smearing policy
+    // Smearing policy, only defined for Nc=3
+    /*
     std::cout << GridLogDebug << " Creating the Stout class\n";
     double rho = 0.1;  // smearing parameter, now hardcoded
     int Nsmear = 1;    // number of smearing levels
     Smear_Stout<Gimpl> Stout(rho);
     std::cout << GridLogDebug << " Creating the SmearedConfiguration class\n";
-    SmearedConfiguration<Gimpl> SmearingPolicy(UGrid, Nsmear, Stout);
+    //SmearedConfiguration<Gimpl> SmearingPolicy(UGrid, Nsmear, Stout);
     std::cout << GridLogDebug << " done\n";
+    */
     //////////////
-    typedef MinimumNorm2<GaugeField, SmearedConfiguration<Gimpl> >  IntegratorType;// change here to change the algorithm
-    IntegratorParameters MDpar(20);
+    NoSmearing<Gimpl> SmearingPolicy;
+    typedef MinimumNorm2<GaugeField, NoSmearing<Gimpl>, RepresentationsPolicy >
+        IntegratorType;  // change here to change the algorithm
+    IntegratorParameters MDpar(20, 1.0);
     IntegratorType MDynamics(UGrid, MDpar, TheAction, SmearingPolicy);
 
-    
     // Checkpoint strategy
-    NerscHmcCheckpointer<Gimpl> Checkpoint(std::string("ckpoint_lat"),std::string("ckpoint_rng"),1);
+    NerscHmcCheckpointer<Gimpl> Checkpoint(std::string("ckpoint_lat"),
+                                           std::string("ckpoint_rng"), 1);
     PlaquetteLogger<Gimpl> PlaqLog(std::string("plaq"));
 
     HMCparameters HMCpar;
@@ -122,45 +129,44 @@ public:
     HMCpar.Trajectories = NumTraj;
     HMCpar.NoMetropolisUntil = NumThermalizations;
 
-
     if (StartType == HotStart) {
       // Hot start
       HMCpar.MetropolisTest = true;
       sRNG.SeedFixedIntegers(SerSeed);
       pRNG.SeedFixedIntegers(ParSeed);
-      SU3::HotConfiguration(pRNG, U);
+      SU<Nc>::HotConfiguration(pRNG, U);
     } else if (StartType == ColdStart) {
       // Cold start
       HMCpar.MetropolisTest = true;
       sRNG.SeedFixedIntegers(SerSeed);
       pRNG.SeedFixedIntegers(ParSeed);
-      SU3::ColdConfiguration(pRNG, U);
+      SU<Nc>::ColdConfiguration(pRNG, U);
     } else if (StartType == TepidStart) {
       // Tepid start
       HMCpar.MetropolisTest = true;
       sRNG.SeedFixedIntegers(SerSeed);
       pRNG.SeedFixedIntegers(ParSeed);
-      SU3::TepidConfiguration(pRNG, U);
+      SU<Nc>::TepidConfiguration(pRNG, U);
     } else if (StartType == CheckpointStart) {
       HMCpar.MetropolisTest = true;
       // CheckpointRestart
       Checkpoint.CheckpointRestore(StartTraj, U, sRNG, pRNG);
     }
 
-    // Attach the gauge field to the smearing Policy and create the fill the smeared set
+    // Attach the gauge field to the smearing Policy and create the fill the
+    // smeared set
     // notice that the unit configuration is singular in this procedure
     std::cout << GridLogMessage << "Filling the smeared set\n";
     SmearingPolicy.set_GaugeField(U);
 
-    HybridMonteCarlo<GaugeField,IntegratorType>  HMC(HMCpar, MDynamics,sRNG,pRNG,U); 
+    HybridMonteCarlo<GaugeField, IntegratorType> HMC(HMCpar, MDynamics, sRNG,
+                                                     pRNG, U);
     HMC.AddObservable(&Checkpoint);
     HMC.AddObservable(&PlaqLog);
 
     // Run it
     HMC.evolve();
-    
   }
-  
 };
 
 typedef NerscHmcRunnerTemplate<PeriodicGimplR> NerscHmcRunner;
@@ -175,5 +181,11 @@ public:
 typedef NerscHmcRunnerTemplate<ConjugateGimplF> ConjugateNerscHmcRunnerF;
 typedef NerscHmcRunnerTemplate<ConjugateGimplD> ConjugateNerscHmcRunnerD;
 
-}}
+template <class RepresentationsPolicy>
+using NerscHmcRunnerHirep = NerscHmcRunnerTemplate<PeriodicGimplR, RepresentationsPolicy>;
+
+
+
+}
+}
 #endif

lib/qcd/hmc/integrators/Integrator.h

@@ -25,7 +25,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 //--------------------------------------------------------------------
@@ -48,36 +49,29 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 namespace QCD {
 
 struct IntegratorParameters {
-
   int Nexp;
   int MDsteps;  // number of outer steps
   RealD trajL;  // trajectory length
   RealD stepsize;
 
-      IntegratorParameters(int MDsteps_, 
-      	RealD trajL_=1.0,
-      	int Nexp_=12):
-      Nexp(Nexp_),
+  IntegratorParameters(int MDsteps_, RealD trajL_ = 1.0, int Nexp_ = 12)
+      : Nexp(Nexp_),
         MDsteps(MDsteps_),
         trajL(trajL_),
-      stepsize(trajL/MDsteps)
-      {
+        stepsize(trajL / MDsteps){
             // empty body constructor
         };
-
 };
 
 /*! @brief Class for Molecular Dynamics management */
-    template<class GaugeField, class SmearingPolicy>
+template <class GaugeField, class SmearingPolicy, class RepresentationPolicy>
 class Integrator {
-
  protected:
-
   typedef IntegratorParameters ParameterType;
 
   IntegratorParameters Params;
 
-  	const ActionSet<GaugeField> as;
+  const ActionSet<GaugeField, RepresentationPolicy> as;
 
   int levels;  //
   double t_U;  // Track time passing on each level and for U and for P
@@ -87,19 +81,22 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
   SmearingPolicy& Smearer;
 
+  RepresentationPolicy Representations;
+
   // Should match any legal (SU(n)) gauge field
   // Need to use this template to match Ncol to pass to SU<N> class
-      template<int Ncol,class vec> void generate_momenta(Lattice< iVector< iScalar< iMatrix<vec,Ncol> >, Nd> > & P,GridParallelRNG& pRNG){
+  template <int Ncol, class vec>
+  void generate_momenta(Lattice<iVector<iScalar<iMatrix<vec, Ncol> >, Nd> >& P,
+                        GridParallelRNG& pRNG) {
     typedef Lattice<iScalar<iScalar<iMatrix<vec, Ncol> > > > GaugeLinkField;
     GaugeLinkField Pmu(P._grid);
     Pmu = zero;
     for (int mu = 0; mu < Nd; mu++) {
-      	SU<Ncol>::GaussianLieAlgebraMatrix(pRNG, Pmu);
+      SU<Ncol>::GaussianFundamentalLieAlgebraMatrix(pRNG, Pmu);
       PokeIndex<LorentzIndex>(P, Pmu, mu);
     }
   }
 
-
   // ObserverList observers; // not yet
   //      typedef std::vector<Observer*> ObserverList;
   //      void register_observers();
@@ -109,22 +106,50 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     t_P[level] += ep;
     update_P(P, U, level, ep);
 
-  	std::cout<<GridLogIntegrator<<"["<<level<<"] P " << " dt "<< ep <<" : t_P "<< t_P[level] <<std::endl;
+    std::cout << GridLogIntegrator << "[" << level << "] P "
+              << " dt " << ep << " : t_P " << t_P[level] << std::endl;
   }
 
+  // to be used by the actionlevel class to iterate
+  // over the representations
+  struct _updateP {
+    template <class FieldType, class GF, class Repr>
+    void operator()(std::vector<Action<FieldType>*> repr_set, Repr& Rep,
+                    GF& Mom, GF& U, double ep) {
+      for (int a = 0; a < repr_set.size(); ++a) {
+        FieldType forceR(U._grid);
+        // Implement smearing only for the fundamental representation now
+        repr_set.at(a)->deriv(Rep.U, forceR);
+        GF force =
+            Rep.RtoFundamentalProject(forceR);  // Ta for the fundamental rep
+        std::cout << GridLogIntegrator << "Hirep Force average: "
+                  << norm2(force) / (U._grid->gSites()) << std::endl;
+        Mom -= force * ep ;
+      }
+    }
+  } update_P_hireps{};
+
   void update_P(GaugeField& Mom, GaugeField& U, int level, double ep) {
-  	// input U actually not used... 
+    // input U actually not used in the fundamental case
+    // Fundamental updates, include smearing
     for (int a = 0; a < as[level].actions.size(); ++a) {
       GaugeField force(U._grid);
       GaugeField& Us = Smearer.get_U(as[level].actions.at(a)->is_smeared);
       as[level].actions.at(a)->deriv(Us, force);  // deriv should NOT include Ta
 
-	  	std::cout<< GridLogIntegrator << "Smearing (on/off): "<<as[level].actions.at(a)->is_smeared <<std::endl;
+      std::cout << GridLogIntegrator
+                << "Smearing (on/off): " << as[level].actions.at(a)->is_smeared
+                << std::endl;
       if (as[level].actions.at(a)->is_smeared) Smearer.smeared_force(force);
       force = Ta(force);
-	  	std::cout<< GridLogIntegrator << "Force average: "<< norm2(force)/(U._grid->gSites()) <<std::endl;
+      std::cout << GridLogIntegrator
+                << "Force average: " << norm2(force) / (U._grid->gSites())
+                << std::endl;
       Mom -= force * ep;
     }
+
+    // Force from the other representations
+    as[level].apply(update_P_hireps, Representations, Mom, U, ep);
   }
 
   void update_U(GaugeField& U, double ep) {
@@ -132,38 +157,37 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
     t_U += ep;
     int fl = levels - 1;
-		std::cout<< GridLogIntegrator <<"   "<<"["<<fl<<"] U " << " dt "<< ep <<" : t_U "<< t_U <<std::endl;
-
+    std::cout << GridLogIntegrator << "   "
+              << "[" << fl << "] U "
+              << " dt " << ep << " : t_U " << t_U << std::endl;
   }
   void update_U(GaugeField& Mom, GaugeField& U, double ep) {
-	//rewrite exponential to deal automatically  with the lorentz index?
-	//	GaugeLinkField Umu(U._grid);
-	//	GaugeLinkField Pmu(U._grid);
+    // rewrite exponential to deal internally with the lorentz index?
     for (int mu = 0; mu < Nd; mu++) {
       auto Umu = PeekIndex<LorentzIndex>(U, mu);
       auto Pmu = PeekIndex<LorentzIndex>(Mom, mu);
       Umu = expMat(Pmu, ep, Params.Nexp) * Umu;
-			ProjectOnGroup(Umu);
-			PokeIndex<LorentzIndex>(U, Umu, mu);
+      PokeIndex<LorentzIndex>(U, ProjectOnGroup(Umu), mu);
     }
+    
     // Update the smeared fields, can be implemented as observer
     Smearer.set_GaugeField(U);
+    // Update the higher representations fields
+    Representations.update(U);  // void functions if fundamental representation
   }
 
   virtual void step(GaugeField& U, int level, int first, int last) = 0;
 
  public:
-
-	Integrator(GridBase* grid, 
-		IntegratorParameters Par,
-		ActionSet<GaugeField> & Aset,
-		SmearingPolicy &Sm):
-	Params(Par),
+  Integrator(GridBase* grid, IntegratorParameters Par,
+             ActionSet<GaugeField, RepresentationPolicy>& Aset,
+             SmearingPolicy& Sm)
+      : Params(Par),
         as(Aset),
         P(grid),
         levels(Aset.size()),
-	Smearer(Sm)
-	{
+        Smearer(Sm),
+        Representations(grid) {
     t_P.resize(levels, 0.0);
     t_U = 0.0;
     // initialization of smearer delegated outside of Integrator
@@ -171,24 +195,71 @@ public:
 
   virtual ~Integrator() {}
 
+  // to be used by the actionlevel class to iterate
+  // over the representations
+  struct _refresh {
+    template <class FieldType, class Repr>
+    void operator()(std::vector<Action<FieldType>*> repr_set, Repr& Rep,
+                    GridParallelRNG& pRNG) {
+      for (int a = 0; a < repr_set.size(); ++a){
+        repr_set.at(a)->refresh(Rep.U, pRNG);
+      
+      std::cout << GridLogDebug << "Hirep refreshing pseudofermions" << std::endl;
+    }
+    }
+  } refresh_hireps{};
+
   // Initialization of momenta and actions
   void refresh(GaugeField& U, GridParallelRNG& pRNG) {
     std::cout << GridLogIntegrator << "Integrator refresh\n";
     generate_momenta(P, pRNG);
+
+    // Update the smeared fields, can be implemented as observer
+    // necessary to keep the fields updated even after a reject
+    // of the Metropolis
+    Smearer.set_GaugeField(U);
+    // Set the (eventual) representations gauge fields
+    Representations.update(U);
+
+    // The Smearer is attached to a pointer of the gauge field
+    // automatically gets the correct field
+    // whether or not has been accepted in the previous sweep
     for (int level = 0; level < as.size(); ++level) {
       for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
         // get gauge field from the SmearingPolicy and
         // based on the boolean is_smeared in actionID
-				GaugeField& Us = Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
+        GaugeField& Us =
+            Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
         as[level].actions.at(actionID)->refresh(Us, pRNG);
       }
+
+      // Refresh the higher representation actions
+      as[level].apply(refresh_hireps, Representations, pRNG);
     }
   }
 
+  // to be used by the actionlevel class to iterate
+  // over the representations
+  struct _S {
+    template <class FieldType, class Repr>
+    void operator()(std::vector<Action<FieldType>*> repr_set, Repr& Rep,
+                    int level, RealD& H) {
+      
+      for (int a = 0; a < repr_set.size(); ++a) {
+        RealD Hterm = repr_set.at(a)->S(Rep.U);
+        std::cout << GridLogMessage << "S Level " << level << " term " << a
+                  << " H Hirep = " << Hterm << std::endl;
+        H += Hterm;
+
+      }
+    }
+  } S_hireps{};
+
   // Calculate action
   RealD S(GaugeField& U) {  // here also U not used
 
-		LatticeComplex Hloc(U._grid);	Hloc = zero;
+    LatticeComplex Hloc(U._grid);
+    Hloc = zero;
     // Momenta
     for (int mu = 0; mu < Nd; mu++) {
       auto Pmu = PeekIndex<LorentzIndex>(P, mu);
@@ -205,18 +276,20 @@ public:
       for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
         // get gauge field from the SmearingPolicy and
         // based on the boolean is_smeared in actionID
-				GaugeField& Us = Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
+        GaugeField& Us =
+            Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
         Hterm = as[level].actions.at(actionID)->S(Us);
-				std::cout<<GridLogMessage << "S Level "<<level<<" term "<<actionID<<" H = "<<Hterm<<std::endl;
+        std::cout << GridLogMessage << "S Level " << level << " term "
+                  << actionID << " H = " << Hterm << std::endl;
         H += Hterm;
       }
+      as[level].apply(S_hireps, Representations, level, H);
     }
 
     return H;
   }
 
   void integrate(GaugeField& U) {
-
     // reset the clocks
     t_U = 0;
     for (int level = 0; level < as.size(); ++level) {
@@ -232,16 +305,14 @@ public:
     // Check the clocks all match on all levels
     for (int level = 0; level < as.size(); ++level) {
       assert(fabs(t_U - t_P[level]) < 1.0e-6);  // must be the same
-	  std::cout<<GridLogIntegrator<<" times["<<level<<"]= "<<t_P[level]<< " " << t_U <<std::endl;
+      std::cout << GridLogIntegrator << " times[" << level
+                << "]= " << t_P[level] << " " << t_U << std::endl;
     }
 
     // and that we indeed got to the end of the trajectory
     assert(fabs(t_U - Params.trajL) < 1.0e-6);
-
-
   }
 };
-
 }
 }
 #endif  // INTEGRATOR_INCLUDED

lib/qcd/hmc/integrators/Integrator_algorithm.h

@@ -91,17 +91,19 @@ namespace Grid{
     *  P 1/2                            P 1/2
     */    
 
-    template<class GaugeField, class SmearingPolicy> class LeapFrog :
-      public Integrator<GaugeField, SmearingPolicy> {
+    template<class GaugeField,
+	     class SmearingPolicy,
+	     class RepresentationPolicy = Representations< FundamentalRepresentation > > class LeapFrog :
+      public Integrator<GaugeField, SmearingPolicy, RepresentationPolicy> {
     public:
 
-      typedef LeapFrog<GaugeField, SmearingPolicy> Algorithm;
+      typedef LeapFrog<GaugeField, SmearingPolicy, RepresentationPolicy> Algorithm;
 
       LeapFrog(GridBase* grid, 
 	       IntegratorParameters Par,
-	       ActionSet<GaugeField> & Aset,
+	       ActionSet<GaugeField, RepresentationPolicy> & Aset,
 	       SmearingPolicy & Sm):
-	Integrator<GaugeField, SmearingPolicy>(grid,Par,Aset,Sm) {};
+	Integrator<GaugeField, SmearingPolicy, RepresentationPolicy>(grid,Par,Aset,Sm) {};
 
 
       void step (GaugeField& U, int level,int _first, int _last){
@@ -138,8 +140,10 @@ namespace Grid{
       }
     };
 
-    template<class GaugeField, class SmearingPolicy> class MinimumNorm2 :
-      public Integrator<GaugeField, SmearingPolicy> {
+    template<class GaugeField,
+	     class SmearingPolicy,
+	     class RepresentationPolicy = Representations < FundamentalRepresentation > > class MinimumNorm2 :
+      public Integrator<GaugeField, SmearingPolicy, RepresentationPolicy> {
     private:
       const RealD lambda = 0.1931833275037836;
 
@@ -147,9 +151,9 @@ namespace Grid{
 
       MinimumNorm2(GridBase* grid, 
 		   IntegratorParameters Par,
-		   ActionSet<GaugeField> & Aset,
+		   ActionSet<GaugeField, RepresentationPolicy> & Aset,
 		   SmearingPolicy& Sm):
-	Integrator<GaugeField, SmearingPolicy>(grid,Par,Aset,Sm) {};
+	Integrator<GaugeField, SmearingPolicy, RepresentationPolicy>(grid,Par,Aset,Sm) {};
 
       void step (GaugeField& U, int level, int _first,int _last){
 
@@ -197,8 +201,10 @@ namespace Grid{
     };
 
 
-    template<class GaugeField, class SmearingPolicy> class ForceGradient :
-      public Integrator<GaugeField, SmearingPolicy> {
+    template<class GaugeField,
+	     class SmearingPolicy,
+	     class RepresentationPolicy = Representations< FundamentalRepresentation > > class ForceGradient :
+      public Integrator<GaugeField, SmearingPolicy, RepresentationPolicy> {
     private:
       const RealD lambda = 1.0/6.0;;
       const RealD chi    = 1.0/72.0;
@@ -209,9 +215,9 @@ namespace Grid{
       // Looks like dH scales as dt^4. tested wilson/wilson 2 level.
     ForceGradient(GridBase* grid, 
 		  IntegratorParameters Par,
-		  ActionSet<GaugeField> & Aset,
+		  ActionSet<GaugeField, RepresentationPolicy> & Aset,
 		  SmearingPolicy &Sm):
-      Integrator<GaugeField, SmearingPolicy>(grid,Par,Aset, Sm) {};
+      Integrator<GaugeField, SmearingPolicy, RepresentationPolicy>(grid,Par,Aset, Sm) {};
 
 
       void FG_update_P(GaugeField&U, int level,double fg_dt,double ep){

lib/qcd/representations/adjoint.h

@@ -0,0 +1,115 @@
+/*
+ *  Policy classes for the HMC
+ *  Author: Guido Cossu
+*/
+
+#ifndef ADJOINT_H
+#define ADJOINT_H
+
+namespace Grid {
+namespace QCD {
+
+/*
+* This is an helper class for the HMC
+* Should contain only the data for the adjoint representation
+* and the facility to convert from the fundamental -> adjoint
+*/
+
+template <int ncolour>
+class AdjointRep {
+ public:
+  // typdef to be used by the Representations class in HMC to get the
+  // types for the higher representation fields
+  typedef typename SU_Adjoint<ncolour>::LatticeAdjMatrix LatticeMatrix;
+  typedef typename SU_Adjoint<ncolour>::LatticeAdjField LatticeField;
+  static const int Dimension = ncolour * ncolour - 1;
+
+  LatticeField U;
+
+  explicit AdjointRep(GridBase *grid) : U(grid) {}
+
+  void update_representation(const LatticeGaugeField &Uin) {
+    std::cout << GridLogDebug << "Updating adjoint representation\n";
+    // Uin is in the fundamental representation
+    // get the U in AdjointRep
+    // (U_adj)_B = tr[e^a U e^b U^dag]
+    // e^a = t^a/sqrt(T_F)
+    // where t^a is the generator in the fundamental
+    // T_F is 1/2 for the fundamental representation
+    conformable(U, Uin);
+    U = zero;
+    LatticeColourMatrix tmp(Uin._grid);
+
+    Vector<typename SU<ncolour>::Matrix> ta(Dimension);
+
+    // Debug lines
+    // LatticeMatrix uno(Uin._grid);
+    // uno = 1.0;
+    ////////////////
+
+    // FIXME probably not very efficient to get all the generators
+    // everytime
+    for (int a = 0; a < Dimension; a++) SU<ncolour>::generator(a, ta[a]);
+
+    for (int mu = 0; mu < Nd; mu++) {
+      auto Uin_mu = peekLorentz(Uin, mu);
+      auto U_mu = peekLorentz(U, mu);
+      for (int a = 0; a < Dimension; a++) {
+        tmp = 2.0 * adj(Uin_mu) * ta[a] * Uin_mu;
+        for (int b = 0; b < Dimension; b++)
+          pokeColour(U_mu, trace(tmp * ta[b]), a, b);
+      }
+      pokeLorentz(U, U_mu, mu);
+      // Check matrix U_mu, must be real orthogonal
+      // reality
+      /*
+      LatticeMatrix Ucheck = U_mu - conjugate(U_mu);
+      std::cout << GridLogMessage << "Reality check: " << norm2(Ucheck) <<
+      std::endl;
+
+      Ucheck = U_mu * adj(U_mu) - uno;
+      std::cout << GridLogMessage << "orthogonality check: " << norm2(Ucheck) <<
+      std::endl;
+      */
+    }
+  }
+
+  LatticeGaugeField RtoFundamentalProject(const LatticeField &in,
+                                          Real scale = 1.0) const {
+    LatticeGaugeField out(in._grid);
+    out = zero;
+
+    for (int mu = 0; mu < Nd; mu++) {
+      LatticeColourMatrix out_mu(in._grid);  // fundamental representation
+      LatticeMatrix in_mu = peekLorentz(in, mu);
+
+      out_mu = zero;
+
+      typename SU<ncolour>::LatticeAlgebraVector h(in._grid);
+      projectOnAlgebra(h, in_mu, double(Nc) * 2.0);  // factor C(r)/C(fund)
+      FundamentalLieAlgebraMatrix(h, out_mu);   // apply scale only once
+      pokeLorentz(out, out_mu, mu);
+      // Returns traceless antihermitian matrix Nc * Nc.
+      // Confirmed
+    }
+    return out;
+  }
+
+ private:
+  void projectOnAlgebra(typename SU<ncolour>::LatticeAlgebraVector &h_out,
+                        const LatticeMatrix &in, Real scale = 1.0) const {
+    SU_Adjoint<ncolour>::projectOnAlgebra(h_out, in, scale);
+  }
+
+  void FundamentalLieAlgebraMatrix(
+      typename SU<ncolour>::LatticeAlgebraVector &h,
+      typename SU<ncolour>::LatticeMatrix &out, Real scale = 1.0) const {
+    SU<ncolour>::FundamentalLieAlgebraMatrix(h, out, scale);
+  }
+};
+
+typedef AdjointRep<Nc> AdjointRepresentation;
+}
+}
+
+#endif

lib/qcd/representations/fundamental.h

@@ -0,0 +1,45 @@
+/*
+ *	Policy classes for the HMC
+ *	Author: Guido Cossu
+*/	
+
+#ifndef FUNDAMENTAL_H
+#define FUNDAMENTAL_H
+
+
+namespace Grid {
+namespace QCD {
+
+/*
+* This is an helper class for the HMC
+* Empty since HMC updates already the fundamental representation 
+*/
+
+template <int ncolour>
+class FundamentalRep {
+ public:
+  static const int Dimension = ncolour;
+
+  // typdef to be used by the Representations class in HMC to get the
+  // types for the higher representation fields
+  typedef typename SU<ncolour>::LatticeMatrix LatticeMatrix;
+  typedef LatticeGaugeField LatticeField;
+  
+  explicit FundamentalRep(GridBase* grid) {} //do nothing
+  void update_representation(const LatticeGaugeField& Uin) {} // do nothing
+
+  LatticeField RtoFundamentalProject(const LatticeField& in, Real scale = 1.0) const{
+    return (scale * in);
+  }
+
+};
+
+typedef	 FundamentalRep<Nc> FundamentalRepresentation;
+
+}
+}
+
+
+
+
+#endif

lib/qcd/representations/hmc_types.h

@@ -0,0 +1,91 @@
+#ifndef HMC_TYPES_H
+#define HMC_TYPES_H
+
+#include <Grid/qcd/representations/adjoint.h>
+#include <Grid/qcd/representations/two_index.h>
+#include <Grid/qcd/representations/fundamental.h>
+#include <tuple>
+#include <utility>
+
+namespace Grid {
+namespace QCD {
+
+// Supported types
+// enum {Fundamental, Adjoint} repr_type;
+
+// Utility to add support to the HMC for representations other than the
+// fundamental
+template <class... Reptypes>
+class Representations {
+ public:
+  typedef std::tuple<Reptypes...> Representation_type;
+
+  // Size of the tuple, known at compile time
+  static const int tuple_size = sizeof...(Reptypes);
+  // The collection of types for the gauge fields
+  typedef std::tuple<typename Reptypes::LatticeField...> Representation_Fields;
+
+  // To access the Reptypes (FundamentalRepresentation, AdjointRepresentation)
+  template <std::size_t N>
+  using repr_type = typename std::tuple_element<N, Representation_type>::type;
+  // in order to get the typename of the field use
+  // type repr_type<I>::LatticeField
+
+  Representation_type rep;
+
+  // Multiple types constructor
+  explicit Representations(GridBase* grid) : rep(Reptypes(grid)...){};
+
+  int size() { return tuple_size; }
+
+  // update the fields
+  template <std::size_t I = 0>
+  inline typename std::enable_if<(I == tuple_size), void>::type update(
+      LatticeGaugeField& U) {}
+
+  template <std::size_t I = 0>
+  inline typename std::enable_if<(I < tuple_size), void>::type update(
+      LatticeGaugeField& U) {
+    std::get<I>(rep).update_representation(U);
+    update<I + 1>(U);
+  }
+
+
+
+};
+
+typedef Representations<FundamentalRepresentation> NoHirep;
+
+// Helper classes to access the elements
+// Strips the first N parameters from the tuple
+// sequence of classes to obtain the S sequence
+// Creates a type that is a tuple of vectors of the template type A
+template <template <typename> class A, class TupleClass,
+          size_t N = TupleClass::tuple_size, size_t... S>
+struct AccessTypes : AccessTypes<A, TupleClass, N - 1, N - 1, S...> {};
+
+template <template <typename> class A, class TupleClass, size_t... S>
+struct AccessTypes<A, TupleClass, 0, S...> {
+ public:
+  typedef typename TupleClass::Representation_Fields Rfields;
+
+  template <std::size_t N>
+  using elem = typename std::tuple_element<N, Rfields>::type;  // fields types
+
+  typedef std::tuple<std::vector< A< elem<S> >* > ... > VectorCollection;
+  typedef std::tuple< elem<S> ... > FieldTypeCollection;
+
+  // Debug
+  void return_size() {
+    std::cout << GridLogMessage
+              << "Access:" << std::tuple_size<std::tuple<elem<S>...> >::value
+              << "\n";
+    std::cout << GridLogMessage
+              << "Access vectors:" << std::tuple_size<VectorCollection>::value
+              << "\n";
+  }
+};
+}
+}
+
+#endif

lib/qcd/representations/two_index.h

@@ -0,0 +1,99 @@
+/*
+ *  Policy classes for the HMC
+ *  Authors: Guido Cossu, David Preti
+*/
+
+#ifndef SUN2INDEX_H_H
+#define SUN2INDEX_H_H
+
+namespace Grid {
+namespace QCD {
+
+/*
+ * This is an helper class for the HMC
+ * Should contain only the data for the two index representations
+ * and the facility to convert from the fundamental -> two index
+ * The templated parameter TwoIndexSymmetry choses between the 
+ * symmetric and antisymmetric representations
+ * 
+ * There is an 
+ * enum TwoIndexSymmetry { Symmetric = 1, AntiSymmetric = -1 };
+ * in the SUnTwoIndex.h file
+ */
+
+template <int ncolour, TwoIndexSymmetry S>
+class TwoIndexRep {
+ public:
+  // typdef to be used by the Representations class in HMC to get the
+  // types for the higher representation fields
+  typedef typename SU_TwoIndex<ncolour, S>::LatticeTwoIndexMatrix LatticeMatrix;
+  typedef typename SU_TwoIndex<ncolour, S>::LatticeTwoIndexField LatticeField;
+  static const int Dimension = ncolour * (ncolour + S) / 2;
+
+  LatticeField U;
+
+  explicit TwoIndexRep(GridBase *grid) : U(grid) {}
+
+  void update_representation(const LatticeGaugeField &Uin) {
+    std::cout << GridLogDebug << "Updating TwoIndex representation\n";
+    // Uin is in the fundamental representation
+    // get the U in TwoIndexRep
+    // (U)_{(ij)(lk)} = tr [ adj(e^(ij)) U e^(lk) transpose(U) ]
+    conformable(U, Uin);
+    U = zero;
+    LatticeColourMatrix tmp(Uin._grid);
+
+    Vector<typename SU<ncolour>::Matrix> eij(Dimension);
+
+    for (int a = 0; a < Dimension; a++)
+      SU_TwoIndex<ncolour, S>::base(a, eij[a]);
+
+    for (int mu = 0; mu < Nd; mu++) {
+      auto Uin_mu = peekLorentz(Uin, mu);
+      auto U_mu = peekLorentz(U, mu);
+      for (int a = 0; a < Dimension; a++) {
+        tmp = transpose(Uin_mu) * adj(eij[a]) * Uin_mu;
+        for (int b = 0; b < Dimension; b++)
+          pokeColour(U_mu, trace(tmp * eij[b]), a, b);
+      }
+      pokeLorentz(U, U_mu, mu);
+    }
+  }
+
+  LatticeGaugeField RtoFundamentalProject(const LatticeField &in,
+                                          Real scale = 1.0) const {
+    LatticeGaugeField out(in._grid);
+    out = zero;
+
+    for (int mu = 0; mu < Nd; mu++) {
+      LatticeColourMatrix out_mu(in._grid);  // fundamental representation
+      LatticeMatrix in_mu = peekLorentz(in, mu);
+
+      out_mu = zero;
+
+      typename SU<ncolour>::LatticeAlgebraVector h(in._grid);
+      projectOnAlgebra(h, in_mu, double(Nc + 2 * S));  // factor T(r)/T(fund)
+      FundamentalLieAlgebraMatrix(h, out_mu);          // apply scale only once
+      pokeLorentz(out, out_mu, mu);
+    }
+    return out;
+  }
+
+ private:
+  void projectOnAlgebra(typename SU<ncolour>::LatticeAlgebraVector &h_out,
+                        const LatticeMatrix &in, Real scale = 1.0) const {
+    SU_TwoIndex<ncolour, S>::projectOnAlgebra(h_out, in, scale);
+  }
+
+  void FundamentalLieAlgebraMatrix(
+      typename SU<ncolour>::LatticeAlgebraVector &h,
+      typename SU<ncolour>::LatticeMatrix &out, Real scale = 1.0) const {
+    SU<ncolour>::FundamentalLieAlgebraMatrix(h, out, scale);
+  }
+};
+
+typedef TwoIndexRep<Nc, Symmetric> TwoIndexSymmetricRepresentation;
+typedef TwoIndexRep<Nc, AntiSymmetric> TwoIndexAntiSymmetricRepresentation;
+}
+}
+#endif

lib/qcd/smearing/GaugeConfiguration.h

@@ -10,6 +10,29 @@ namespace Grid {
 
 namespace QCD {
 
+  //trivial class for no smearing
+  template< class Gimpl >
+class NoSmearing {
+public:
+  INHERIT_GIMPL_TYPES(Gimpl);
+
+  GaugeField*
+      ThinLinks;
+
+  NoSmearing(): ThinLinks(NULL) {}
+
+  void set_GaugeField(GaugeField& U) { ThinLinks = &U; }
+
+  void smeared_force(GaugeField& SigmaTilde) const {}
+
+  GaugeField& get_SmearedU() { return *ThinLinks; }
+
+  GaugeField& get_U(bool smeared = false) {
+    return *ThinLinks;
+  }
+
+};
+
 /*!
   @brief Smeared configuration container
 
@@ -201,6 +224,8 @@ class SmearedConfiguration {
   SmearedConfiguration()
       : smearingLevels(0), StoutSmearing(), SmearedSet(), ThinLinks(NULL) {}
 
+
+  
   // attach the smeared routines to the thin links U and fill the smeared set
   void set_GaugeField(GaugeField& U) { fill_smearedSet(U); }
 

lib/qcd/smearing/StoutSmearing.h

@@ -18,14 +18,12 @@ class Smear_Stout : public Smear<Gimpl> {
   INHERIT_GIMPL_TYPES(Gimpl)
 
   Smear_Stout(Smear<Gimpl>* base) : SmearBase(base) {
-    static_assert(Nc == 3,
-                  "Stout smearing currently implemented only for Nc==3");
+    assert(Nc == 3);//                  "Stout smearing currently implemented only for Nc==3");
   }
 
   /*! Default constructor */
   Smear_Stout(double rho = 1.0) : SmearBase(new Smear_APE<Gimpl>(rho)) {
-    static_assert(Nc == 3,
-                  "Stout smearing currently implemented only for Nc==3");
+    assert(Nc == 3);//                  "Stout smearing currently implemented only for Nc==3");
   }
 
   ~Smear_Stout() {}  // delete SmearBase...

lib/qcd/utils/SUn.h

@@ -25,7 +25,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef QCD_UTIL_SUN_H
@@ -37,15 +38,21 @@ namespace Grid {
 template <int ncolour>
 class SU {
  public:
-
-  static int generators(void)   { return ncolour*ncolour-1; }
+  static const int Dimension = ncolour;
+  static const int AdjointDimension = ncolour * ncolour - 1;
   static int su2subgroups(void) { return (ncolour * (ncolour - 1)) / 2; }
 
-  template<typename vtype> using iSUnMatrix              = iScalar<iScalar<iMatrix<vtype, ncolour> > > ;
-  template<typename vtype> using iSU2Matrix              = iScalar<iScalar<iMatrix<vtype, 2> > > ;
+  template <typename vtype>
+  using iSUnMatrix = iScalar<iScalar<iMatrix<vtype, ncolour> > >;
+  template <typename vtype>
+  using iSU2Matrix = iScalar<iScalar<iMatrix<vtype, 2> > >;
+  template <typename vtype>
+  using iSUnAlgebraVector =
+      iScalar<iScalar<iVector<vtype, AdjointDimension> > >;
 
   //////////////////////////////////////////////////////////////////////////////////////////////////
-  // Types can be accessed as SU<2>::Matrix , SU<2>::vSUnMatrix, SU<2>::LatticeMatrix etc...
+  // Types can be accessed as SU<2>::Matrix , SU<2>::vSUnMatrix,
+  // SU<2>::LatticeMatrix etc...
   //////////////////////////////////////////////////////////////////////////////////////////////////
   typedef iSUnMatrix<Complex> Matrix;
   typedef iSUnMatrix<ComplexF> MatrixF;
@@ -55,10 +62,25 @@ public:
   typedef iSUnMatrix<vComplexF> vMatrixF;
   typedef iSUnMatrix<vComplexD> vMatrixD;
 
+  // For the projectors to the algebra
+  // these should be real...
+  // keeping complex for consistency with the SIMD vector types
+  typedef iSUnAlgebraVector<Complex> AlgebraVector;
+  typedef iSUnAlgebraVector<ComplexF> AlgebraVectorF;
+  typedef iSUnAlgebraVector<ComplexD> AlgebraVectorD;
+
+  typedef iSUnAlgebraVector<vComplex> vAlgebraVector;
+  typedef iSUnAlgebraVector<vComplexF> vAlgebraVectorF;
+  typedef iSUnAlgebraVector<vComplexD> vAlgebraVectorD;
+
   typedef Lattice<vMatrix> LatticeMatrix;
   typedef Lattice<vMatrixF> LatticeMatrixF;
   typedef Lattice<vMatrixD> LatticeMatrixD;
 
+  typedef Lattice<vAlgebraVector> LatticeAlgebraVector;
+  typedef Lattice<vAlgebraVectorF> LatticeAlgebraVectorF;
+  typedef Lattice<vAlgebraVectorD> LatticeAlgebraVectorD;
+
   typedef iSU2Matrix<Complex> SU2Matrix;
   typedef iSU2Matrix<ComplexF> SU2MatrixF;
   typedef iSU2Matrix<ComplexD> SU2MatrixD;
@@ -71,13 +93,13 @@ public:
   typedef Lattice<vSU2MatrixF> LatticeSU2MatrixF;
   typedef Lattice<vSU2MatrixD> LatticeSU2MatrixD;
 
-
   ////////////////////////////////////////////////////////////////////////
   // There are N^2-1 generators for SU(N).
   //
   // We take a traceless hermitian generator basis as follows
   //
-  // * Normalisation: trace ta tb = 1/2 delta_ab
+  // * Normalisation: trace ta tb = 1/2 delta_ab = T_F delta_ab
+  //   T_F = 1/2  for SU(N) groups
   //
   // * Off diagonal
   //    - pairs of rows i1,i2 behaving like pauli matrices signma_x, sigma_y
@@ -93,11 +115,14 @@ public:
   //    - There are 2 x Nc (Nc-1)/ 2 of these = Nc^2 - Nc
   //
   //    - We enumerate the row-col pairs.
-  //    - for each row col pair there is a (sigma_x) and a (sigma_y) like generator
+  //    - for each row col pair there is a (sigma_x) and a (sigma_y) like
+  //    generator
   //
   //
-  //   t^a_ij = { in 0.. Nc(Nc-1)/2 -1} =>  delta_{i,i1} delta_{j,i2} +  delta_{i,i1} delta_{j,i2}  
-  //   t^a_ij = { in Nc(Nc-1)/2 ... Nc^(Nc-1) -1} =>  i delta_{i,i1} delta_{j,i2} - i delta_{i,i1} delta_{j,i2}  
+  //   t^a_ij = { in 0.. Nc(Nc-1)/2 -1} =>  1/2(delta_{i,i1} delta_{j,i2} +
+  //   delta_{i,i1} delta_{j,i2})
+  //   t^a_ij = { in Nc(Nc-1)/2 ... Nc(Nc-1) - 1} =>  i/2( delta_{i,i1}
+  //   delta_{j,i2} - i delta_{i,i1} delta_{j,i2})
   //
   // * Diagonal; must be traceless and normalised
   //   - Sequence is
@@ -114,6 +139,7 @@ public:
   //   ( 1      )
   //   (    1   ) / sqrt(3) /2  = 1/2 lambda_8
   //   (      -2)
+  //
   ////////////////////////////////////////////////////////////////////////
   template <class cplx>
   static void generator(int lieIndex, iSUnMatrix<cplx> &ta) {
@@ -127,13 +153,16 @@ public:
       generatorDiagonal(diagIndex, ta);
       return;
     }
-    sigxy   = lieIndex&0x1;
+    sigxy = lieIndex & 0x1;  // even or odd
     su2Index = lieIndex >> 1;
-    if ( sigxy ) generatorSigmaY(su2Index,ta);
-    else         generatorSigmaX(su2Index,ta);
+    if (sigxy)
+      generatorSigmaY(su2Index, ta);
+    else
+      generatorSigmaX(su2Index, ta);
   }
+  
   template <class cplx>
-  static void generatorSigmaX(int su2Index,iSUnMatrix<cplx> &ta){
+  static void generatorSigmaY(int su2Index, iSUnMatrix<cplx> &ta) {
     ta = zero;
     int i1, i2;
     su2SubGroupIndex(i1, i2, su2Index);
@@ -142,27 +171,26 @@ public:
     ta = ta * 0.5;
   }
   template <class cplx>
-  static void generatorSigmaY(int su2Index,iSUnMatrix<cplx> &ta){
+  static void generatorSigmaX(int su2Index, iSUnMatrix<cplx> &ta) {
     ta = zero;
     cplx i(0.0, 1.0);
     int i1, i2;
     su2SubGroupIndex(i1, i2, su2Index);
-    ta()()(i1,i2)=-i;
-    ta()()(i2,i1)= i;
+    ta()()(i1, i2) = i;
+    ta()()(i2, i1) = -i;
     ta = ta * 0.5;
   }
+
   template <class cplx>
   static void generatorDiagonal(int diagIndex, iSUnMatrix<cplx> &ta) {
+    // diag ({1, 1, ..., 1}(k-times), -k, 0, 0, ...)
     ta = zero;
-    int trsq=0;
-    int last=diagIndex+1;
-    for(int i=0;i<=diagIndex;i++){
+    int k = diagIndex + 1;                  // diagIndex starts from 0
+    for (int i = 0; i <= diagIndex; i++) {  // k iterations
       ta()()(i, i) = 1.0;
-      trsq++;
     }
-    ta()()(last,last) = -last;
-    trsq+=last*last;
-    RealD nrm = 1.0/std::sqrt(2.0*trsq);
+    ta()()(k, k) = -k;  // indexing starts from 0
+    RealD nrm = 1.0 / std::sqrt(2.0 * k * (k + 1));
     ta = ta * nrm;
   }
 
@@ -170,7 +198,6 @@ public:
   // Map a su2 subgroup number to the pair of rows that are non zero
   ////////////////////////////////////////////////////////////////////////
   static void su2SubGroupIndex(int &i1, int &i2, int su2_index) {
-
     assert((su2_index >= 0) && (su2_index < (ncolour * (ncolour - 1)) / 2));
 
     int spare = su2_index;
@@ -187,8 +214,7 @@ public:
   static void su2Extract(Lattice<iSinglet<vcplx> > &Determinant,
                          Lattice<iSU2Matrix<vcplx> > &subgroup,
                          const Lattice<iSUnMatrix<vcplx> > &source,
-			     int su2_index)
-  {
+                         int su2_index) {
     GridBase *grid(source._grid);
     conformable(subgroup, source);
     conformable(subgroup, Determinant);
@@ -209,11 +235,9 @@ PARALLEL_FOR_LOOP
       subgroup._odata[ss] = Sigma;
 
       // this should be purely real
-      Determinant._odata[ss] = Sigma()()(0,0)*Sigma()()(1,1)
-   	                     - Sigma()()(0,1)*Sigma()()(1,0);
-
+      Determinant._odata[ss] =
+          Sigma()()(0, 0) * Sigma()()(1, 1) - Sigma()()(0, 1) * Sigma()()(1, 0);
     }
-
   }
 
   //////////////////////////////////////////////////////////////////////////////////////////
@@ -221,9 +245,7 @@ PARALLEL_FOR_LOOP
   //////////////////////////////////////////////////////////////////////////////////////////
   template <class vcplx>
   static void su2Insert(const Lattice<iSU2Matrix<vcplx> > &subgroup,
-			       Lattice<iSUnMatrix<vcplx> > &dest,
-			int su2_index)
-  {
+                        Lattice<iSUnMatrix<vcplx> > &dest, int su2_index) {
     GridBase *grid(dest._grid);
     conformable(subgroup, dest);
     int i0, i1;
@@ -239,24 +261,22 @@ PARALLEL_FOR_LOOP
     }
   }
 
-
   ///////////////////////////////////////////////
   // Generate e^{ Re Tr Staple Link} dlink
   //
-  // *** Note Staple should be appropriate linear compbination between all staples.
+  // *** Note Staple should be appropriate linear compbination between all
+  // staples.
   // *** If already by beta pass coefficient 1.0.
-  // *** This routine applies the additional 1/Nc factor that comes after trace in action.
+  // *** This routine applies the additional 1/Nc factor that comes after trace
+  // in action.
   //
   ///////////////////////////////////////////////
-  static void SubGroupHeatBath(      GridSerialRNG       &sRNG,
-				     GridParallelRNG     &pRNG,
+  static void SubGroupHeatBath(
+      GridSerialRNG &sRNG, GridParallelRNG &pRNG,
       RealD beta,  // coeff multiplying staple in action (with no 1/Nc)
       LatticeMatrix &link,
       const LatticeMatrix &barestaple,  // multiplied by action coeffs so th
-				     int su2_subgroup,
-				     int nheatbath,
-				     LatticeInteger &wheremask)
-  {
+      int su2_subgroup, int nheatbath, LatticeInteger &wheremask) {
     GridBase *grid = link._grid;
 
     int ntrials = 0;
@@ -267,22 +287,30 @@ PARALLEL_FOR_LOOP
 
     staple = barestaple * (beta / ncolour);
 
-    LatticeMatrix    V(grid);    V = link*staple;
+    LatticeMatrix V(grid);
+    V = link * staple;
 
     // Subgroup manipulation in the lie algebra space
-    LatticeSU2Matrix    u(grid);   // Kennedy pendleton "u" real projected normalised Sigma
+    LatticeSU2Matrix u(
+        grid);  // Kennedy pendleton "u" real projected normalised Sigma
     LatticeSU2Matrix uinv(grid);
     LatticeSU2Matrix ua(grid);  // a in pauli form
     LatticeSU2Matrix b(grid);   // rotated matrix after hb
 
     // Some handy constant fields
-    LatticeComplex ones (grid); ones = 1.0;
-    LatticeComplex zeros(grid); zeros=zero;
-    LatticeReal rones (grid); rones = 1.0;
-    LatticeReal rzeros(grid); rzeros=zero;
+    LatticeComplex ones(grid);
+    ones = 1.0;
+    LatticeComplex zeros(grid);
+    zeros = zero;
+    LatticeReal rones(grid);
+    rones = 1.0;
+    LatticeReal rzeros(grid);
+    rzeros = zero;
     LatticeComplex udet(grid);  // determinant of real(staple)
-    LatticeInteger mask_true (grid); mask_true = 1;
-    LatticeInteger mask_false(grid); mask_false= 0;
+    LatticeInteger mask_true(grid);
+    mask_true = 1;
+    LatticeInteger mask_false(grid);
+    mask_false = 0;
 
     /*
   PLB 156 P393 (1985) (Kennedy and Pendleton)
@@ -299,7 +327,8 @@ Writing Sigma = 1/Nc (beta Sigma') where sum over staples is "Sigma' "
        beta S = const - beta/Nc Re Tr h Sigma'
               = const - Re Tr h Sigma
 
-Decompose h and Sigma into (1, sigma_j) ; h_i real, h^2=1, Sigma_i complex arbitrary.
+  Decompose h and Sigma into (1, sigma_j) ; h_i real, h^2=1, Sigma_i complex
+  arbitrary.
 
       Tr h Sigma = h_i Sigma_j Tr (sigma_i sigma_j)  = h_i Sigma_j 2 delta_ij
    Re Tr h Sigma = 2 h_j Re Sigma_j
@@ -335,9 +364,12 @@ Note: Product b' xi is unvariant because scaling Sigma leaves
     LatticeSU2Matrix lident(grid);
 
     SU2Matrix ident = Complex(1.0);
-    SU2Matrix pauli1; SU<2>::generator(0,pauli1);
-    SU2Matrix pauli2; SU<2>::generator(1,pauli2);
-    SU2Matrix pauli3; SU<2>::generator(2,pauli3);
+    SU2Matrix pauli1;
+    SU<2>::generator(0, pauli1);
+    SU2Matrix pauli2;
+    SU<2>::generator(1, pauli2);
+    SU2Matrix pauli3;
+    SU<2>::generator(2, pauli3);
     pauli1 = timesI(pauli1) * 2.0;
     pauli2 = timesI(pauli2) * 2.0;
     pauli3 = timesI(pauli3) * 2.0;
@@ -352,7 +384,8 @@ Note: Product b' xi is unvariant because scaling Sigma leaves
     udet = where(adet > machine_epsilon, udet, cone);
 
     xi = 0.5 * sqrt(udet);  // 4xi^2 = Det [ Sig - Sig^dag  + 1 Tr Sigdag]
-    u   = 0.5*u*pow(xi,-1.0); //  u   = 1/2xi [ Sig - Sig^dag  + 1 Tr Sigdag]
+    u = 0.5 * u *
+        pow(xi, -1.0);  //  u   = 1/2xi [ Sig - Sig^dag  + 1 Tr Sigdag]
 
     // Debug test for sanity
     uinv = adj(u);
@@ -363,19 +396,24 @@ Note: Product b' xi is unvariant because scaling Sigma leaves
   Measure: Haar measure dh has d^4a delta(1-|a^2|)
   In polars:
     da = da0 r^2 sin theta dr dtheta dphi delta( 1 - r^2 -a0^2)
-     = da0 r^2 sin theta dr dtheta dphi delta( (sqrt(1-a0^) - r)(sqrt(1-a0^) + r) )
+       = da0 r^2 sin theta dr dtheta dphi delta( (sqrt(1-a0^) - r)(sqrt(1-a0^) +
+  r) )
        = da0 r/2 sin theta dr dtheta dphi delta( (sqrt(1-a0^) - r) )
 
-Action factor Q(h) dh  = e^-S[h]  dh =  e^{  xi Tr uh} dh    // beta enters through xi
+  Action factor Q(h) dh  = e^-S[h]  dh =  e^{  xi Tr uh} dh    // beta enters
+  through xi
                                        =  e^{2 xi (h.u)} dh
-                                     =  e^{2 xi h0u0}.e^{2 xi h1u1}.e^{2 xi h2u2}.e^{2 xi h3u3} dh
+                                       =  e^{2 xi h0u0}.e^{2 xi h1u1}.e^{2 xi
+  h2u2}.e^{2 xi h3u3} dh
 
   Therefore for each site, take xi for that site
   i) generate  |a0|<1 with dist
      (1-a0^2)^0.5 e^{2 xi a0 } da0
 
-Take alpha = 2 xi  = 2 xi [ recall 2 beta/Nc unmod staple norm]; hence 2.0/Nc factor in Chroma ]
-A. Generate two uniformly distributed pseudo-random numbers R and R', R'', R''' in the unit interval; 
+  Take alpha = 2 xi  = 2 xi [ recall 2 beta/Nc unmod staple norm]; hence 2.0/Nc
+  factor in Chroma ]
+  A. Generate two uniformly distributed pseudo-random numbers R and R', R'',
+  R''' in the unit interval;
   B. Set X = -(ln R)/alpha, X' =-(ln R')/alpha;
   C. Set C = cos^2(2pi R"), with R" another uniform random number in [0,1] ;
   D. Set A = XC;
@@ -383,7 +421,8 @@ E. Let d  = X'+A;
   F. If R'''^2 :> 1 - 0.5 d,  go back to A;
   G. Set a0 = 1 - d;
 
-Note that in step D setting B ~ X - A and using B in place of A in step E will generate a second independent a 0 value.
+  Note that in step D setting B ~ X - A and using B in place of A in step E will
+  generate a second independent a 0 value.
     */
 
     /////////////////////////////////////////////////////////
@@ -394,20 +433,22 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     rtmp = where(wheremask, rones, rzeros);
     RealD numSites = sum(rtmp);
     RealD numAccepted;
-    LatticeInteger      Accepted(grid); Accepted = zero;
+    LatticeInteger Accepted(grid);
+    Accepted = zero;
     LatticeInteger newlyAccepted(grid);
 
     std::vector<LatticeReal> xr(4, grid);
     std::vector<LatticeReal> a(4, grid);
-    LatticeReal d(grid); d=zero;
+    LatticeReal d(grid);
+    d = zero;
     LatticeReal alpha(grid);
 
     //    std::cout<<GridLogMessage<<"xi "<<xi <<std::endl;
     alpha = toReal(2.0 * xi);
 
     do {
-
-      // A. Generate two uniformly distributed pseudo-random numbers R and R', R'', R''' in the unit interval; 
+      // A. Generate two uniformly distributed pseudo-random numbers R and R',
+      // R'', R''' in the unit interval;
       random(pRNG, xr[0]);
       random(pRNG, xr[1]);
       random(pRNG, xr[2]);
@@ -430,10 +471,13 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
       d = where(Accepted, d, xr[2] + xr[1] * xr[3]);
 
       // F. If R'''^2 :> 1 - 0.5 d,  go back to A;
-      LatticeReal thresh(grid); thresh = 1.0-d*0.5;
+      LatticeReal thresh(grid);
+      thresh = 1.0 - d * 0.5;
       xrsq = xr[0] * xr[0];
-      LatticeInteger ione(grid); ione = 1;
-      LatticeInteger izero(grid); izero=zero;
+      LatticeInteger ione(grid);
+      ione = 1;
+      LatticeInteger izero(grid);
+      izero = zero;
 
       newlyAccepted = where(xrsq < thresh, ione, izero);
       Accepted = where(newlyAccepted, newlyAccepted, Accepted);
@@ -462,18 +506,18 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     LatticeReal sin_theta(grid);
     LatticeReal phi(grid);
 
-    random(pRNG,phi);            phi = phi * twopi;       // uniform in [0,2pi]
-    random(pRNG,cos_theta); cos_theta=(cos_theta*2.0)-1.0;  // uniform in [-1,1]
+    random(pRNG, phi);
+    phi = phi * twopi;  // uniform in [0,2pi]
+    random(pRNG, cos_theta);
+    cos_theta = (cos_theta * 2.0) - 1.0;  // uniform in [-1,1]
     sin_theta = sqrt(abs(1.0 - cos_theta * cos_theta));
 
     a[1] = a123mag * sin_theta * cos(phi);
     a[2] = a123mag * sin_theta * sin(phi);
     a[3] = a123mag * cos_theta;
 
-    ua = toComplex(a[0])*ident 
-       + toComplex(a[1])*pauli1 
-       + toComplex(a[2])*pauli2 
-       + toComplex(a[3])*pauli3;
+    ua = toComplex(a[0]) * ident + toComplex(a[1]) * pauli1 +
+         toComplex(a[2]) * pauli2 + toComplex(a[3]) * pauli3;
 
     b = 1.0;
     b = where(wheremask, uinv * ua, b);
@@ -508,44 +552,51 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     /////////////////////////////////
   }
 
-  static void printGenerators(void)
-  {
-    for(int gen=0;gen<generators();gen++){
+  static void printGenerators(void) {
+    for (int gen = 0; gen < AdjointDimension; gen++) {
       Matrix ta;
       generator(gen, ta);
-      std::cout<<GridLogMessage<< "Nc = "<<ncolour<<" t_"<<gen<<std::endl;
+      std::cout << GridLogMessage << "Nc = " << ncolour << " t_" << gen
+                << std::endl;
       std::cout << GridLogMessage << ta << std::endl;
     }
   }
 
+
+
   static void testGenerators(void) {
     Matrix ta;
     Matrix tb;
-    std::cout<<GridLogMessage<<"Checking trace ta tb is 0.5 delta_ab"<<std::endl;
-    for(int a=0;a<generators();a++){
-      for(int b=0;b<generators();b++){
+    std::cout << GridLogMessage
+              << "Fundamental - Checking trace ta tb is 0.5 delta_ab"
+              << std::endl;
+    for (int a = 0; a < AdjointDimension; a++) {
+      for (int b = 0; b < AdjointDimension; b++) {
         generator(a, ta);
         generator(b, tb);
         Complex tr = TensorRemove(trace(ta * tb));
-	std::cout<<GridLogMessage<<tr<<" ";
+        std::cout << GridLogMessage << "(" << a << "," << b << ") =  " << tr
+                  << std::endl;
         if (a == b) assert(abs(tr - Complex(0.5)) < 1.0e-6);
         if (a != b) assert(abs(tr) < 1.0e-6);
       }
       std::cout << GridLogMessage << std::endl;
     }
-    std::cout<<GridLogMessage<<"Checking hermitian"<<std::endl;
-    for(int a=0;a<generators();a++){
+    std::cout << GridLogMessage << "Fundamental - Checking if hermitian"
+              << std::endl;
+    for (int a = 0; a < AdjointDimension; a++) {
       generator(a, ta);
-      std::cout<<GridLogMessage<<a<<" ";
+      std::cout << GridLogMessage << a << std::endl;
       assert(norm2(ta - adj(ta)) < 1.0e-6);
     }
     std::cout << GridLogMessage << std::endl;
 
-    std::cout<<GridLogMessage<<"Checking traceless"<<std::endl;
-    for(int a=0;a<generators();a++){
+    std::cout << GridLogMessage << "Fundamental - Checking if traceless"
+              << std::endl;
+    for (int a = 0; a < AdjointDimension; a++) {
       generator(a, ta);
       Complex tr = TensorRemove(trace(ta));
-      std::cout<<GridLogMessage<<a<<" ";
+      std::cout << GridLogMessage << a << " " << std::endl;
       assert(abs(tr) < 1.0e-6);
     }
     std::cout << GridLogMessage << std::endl;
@@ -553,7 +604,8 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
 
   // reunitarise??
   template <typename LatticeMatrixType>
-  static void LieRandomize(GridParallelRNG     &pRNG,LatticeMatrixType &out,double scale=1.0){
+  static void LieRandomize(GridParallelRNG &pRNG, LatticeMatrixType &out,
+                           double scale = 1.0) {
     GridBase *grid = out._grid;
 
     typedef typename LatticeMatrixType::vector_type vector_type;
@@ -562,7 +614,8 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     typedef iSinglet<vector_type> vTComplexType;
 
     typedef Lattice<vTComplexType> LatticeComplexType;
-    typedef typename GridTypeMapper<typename LatticeMatrixType::vector_object>::scalar_object MatrixType;
+    typedef typename GridTypeMapper<
+        typename LatticeMatrixType::vector_object>::scalar_object MatrixType;
 
     LatticeComplexType ca(grid);
     LatticeMatrixType lie(grid);
@@ -572,8 +625,7 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     MatrixType ta;
 
     lie = zero;
-    for(int a=0;a<generators();a++){
-
+    for (int a = 0; a < AdjointDimension; a++) {
       random(pRNG, ca);
 
       ca = (ca + conjugate(ca)) * 0.5;
@@ -588,7 +640,9 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     taExp(lie, out);
   }
 
-  static void GaussianLieAlgebraMatrix(GridParallelRNG     &pRNG,LatticeMatrix &out,double scale=1.0){
+  static void GaussianFundamentalLieAlgebraMatrix(GridParallelRNG &pRNG,
+                                                  LatticeMatrix &out,
+                                                  Real scale = 1.0) {
     GridBase *grid = out._grid;
     LatticeReal ca(grid);
     LatticeMatrix la(grid);
@@ -596,15 +650,46 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     Matrix ta;
 
     out = zero;
-    for(int a=0;a<generators();a++){
+    for (int a = 0; a < AdjointDimension; a++) {
       gaussian(pRNG, ca);
       generator(a, ta);
-      la=toComplex(ca)*ci*ta;
+      la = toComplex(ca) * ta;
       out += la;
     }
-
+    out *= ci;
   }
 
+  static void FundamentalLieAlgebraMatrix(const LatticeAlgebraVector &h,
+                                          LatticeMatrix &out,
+                                          Real scale = 1.0) {
+    conformable(h, out);
+    GridBase *grid = out._grid;
+    LatticeMatrix la(grid);
+    Matrix ta;
+
+    out = zero;
+    for (int a = 0; a < AdjointDimension; a++) {
+      generator(a, ta);
+      la = peekColour(h, a) * timesI(ta) * scale;
+      out += la;
+    }
+  }
+
+  // Projects the algebra components a lattice matrix (of dimension ncol*ncol -1 )
+  // inverse operation: FundamentalLieAlgebraMatrix
+  static void projectOnAlgebra(LatticeAlgebraVector &h_out, const LatticeMatrix &in, Real scale = 1.0) {
+    conformable(h_out, in);
+    h_out = zero;
+    Matrix Ta;
+
+    for (int a = 0; a < AdjointDimension; a++) {
+      generator(a, Ta);
+      auto tmp = - 2.0 * (trace(timesI(Ta) * in)) * scale;// 2.0 for the normalization of the trace in the fundamental rep
+      pokeColour(h_out, tmp, a);
+    }
+  }
+
+
   template <typename GaugeField>
   static void HotConfiguration(GridParallelRNG &pRNG, GaugeField &out) {
     typedef typename GaugeField::vector_type vector_type;
@@ -617,7 +702,8 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
       PokeIndex<LorentzIndex>(out, Umu, mu);
     }
   }
-  static void TepidConfiguration(GridParallelRNG &pRNG,LatticeGaugeField &out){
+  static void TepidConfiguration(GridParallelRNG &pRNG,
+                                 LatticeGaugeField &out) {
     LatticeMatrix Umu(out._grid);
     for (int mu = 0; mu < Nd; mu++) {
       LieRandomize(pRNG, Umu, 0.01);
@@ -646,14 +732,12 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
     ex = xn + ComplexType(1.0);  // 1+x
 
     // Do a 12th order exponentiation
-    for(int i=2; i <= 12; ++i)
-    {
+    for (int i = 2; i <= 12; ++i) {
       nfac = nfac / RealD(i);  // 1/2, 1/2.3 ...
       xn = xn * x;             // x2, x3,x4....
       ex = ex + xn * nfac;     // x2/2!, x3/3!....
     }
   }
-
 };
 
 typedef SU<2> SU2;
@@ -661,6 +745,9 @@ Note that in step D setting B ~ X - A and using B in place of A in step E will g
 typedef SU<4> SU4;
 typedef SU<5> SU5;
 
+
+typedef SU<Nc> FundamentalMatrices;
+
 }
 }
 #endif

lib/qcd/utils/SUnAdjoint.h

@@ -0,0 +1,182 @@
+#ifndef QCD_UTIL_SUNADJOINT_H
+#define QCD_UTIL_SUNADJOINT_H
+
+////////////////////////////////////////////////////////////////////////
+//
+// * Adjoint representation generators
+//
+// * Normalisation for the fundamental generators: 
+//   trace ta tb = 1/2 delta_ab = T_F delta_ab
+//   T_F = 1/2  for SU(N) groups
+//
+//
+//   base for NxN hermitian traceless matrices
+//   normalized to 1:
+//
+//   (e_Adj)^a = t^a / sqrt(T_F)
+//
+//   then the real, antisymmetric generators for the adjoint representations
+//   are computed ( shortcut: e^a == (e_Adj)^a )
+//
+//   (iT_adj)^d_ba = i tr[e^a t^d e^b - t^d e^a e^b]
+//
+////////////////////////////////////////////////////////////////////////
+
+namespace Grid {
+namespace QCD {
+
+template <int ncolour>
+class SU_Adjoint : public SU<ncolour> {
+ public:
+  static const int Dimension = ncolour * ncolour - 1;
+
+  template <typename vtype>
+  using iSUnAdjointMatrix =
+      iScalar<iScalar<iMatrix<vtype, Dimension > > >;
+
+  // Actually the adjoint matrices are real...
+  // Consider this overhead... FIXME
+  typedef iSUnAdjointMatrix<Complex> AMatrix;
+  typedef iSUnAdjointMatrix<ComplexF> AMatrixF;
+  typedef iSUnAdjointMatrix<ComplexD> AMatrixD;
+
+  typedef iSUnAdjointMatrix<vComplex> vAMatrix;
+  typedef iSUnAdjointMatrix<vComplexF> vAMatrixF;
+  typedef iSUnAdjointMatrix<vComplexD> vAMatrixD;
+
+  typedef Lattice<vAMatrix>  LatticeAdjMatrix;
+  typedef Lattice<vAMatrixF> LatticeAdjMatrixF;
+  typedef Lattice<vAMatrixD> LatticeAdjMatrixD;
+
+  typedef Lattice<iVector<iScalar<iMatrix<vComplex, Dimension> >, Nd> >
+      LatticeAdjField;
+  typedef Lattice<iVector<iScalar<iMatrix<vComplexF, Dimension> >, Nd> >
+      LatticeAdjFieldF;
+  typedef Lattice<iVector<iScalar<iMatrix<vComplexD, Dimension> >, Nd> >
+      LatticeAdjFieldD;
+
+
+
+
+  template <class cplx>
+  static void generator(int Index, iSUnAdjointMatrix<cplx> &iAdjTa) {
+    // returns i(T_Adj)^index necessary for the projectors
+    // see definitions above
+    iAdjTa = zero;
+    Vector<typename SU<ncolour>::template iSUnMatrix<cplx> > ta(ncolour * ncolour - 1);
+    typename SU<ncolour>::template iSUnMatrix<cplx> tmp;
+
+    // FIXME not very efficient to get all the generators everytime
+    for (int a = 0; a < Dimension; a++) SU<ncolour>::generator(a, ta[a]);
+
+    for (int a = 0; a < Dimension; a++) {
+      tmp = ta[a] * ta[Index] - ta[Index] * ta[a];
+      for (int b = 0; b < (ncolour * ncolour - 1); b++) {
+        typename SU<ncolour>::template iSUnMatrix<cplx> tmp1 =
+            2.0 * tmp * ta[b];  // 2.0 from the normalization
+        Complex iTr = TensorRemove(timesI(trace(tmp1)));
+        //iAdjTa()()(b, a) = iTr;
+        iAdjTa()()(a, b) = iTr;
+      }
+    }
+  }
+
+  static void printGenerators(void) {
+    for (int gen = 0; gen < Dimension; gen++) {
+      AMatrix ta;
+      generator(gen, ta);
+      std::cout << GridLogMessage << "Nc = " << ncolour << " t_" << gen
+                << std::endl;
+      std::cout << GridLogMessage << ta << std::endl;
+    }
+  }
+
+  static void testGenerators(void) {
+    AMatrix adjTa;
+    std::cout << GridLogMessage << "Adjoint - Checking if real" << std::endl;
+    for (int a = 0; a < Dimension; a++) {
+      generator(a, adjTa);
+      std::cout << GridLogMessage << a << std::endl;
+      assert(norm2(adjTa - conjugate(adjTa)) < 1.0e-6);
+    }
+    std::cout << GridLogMessage << std::endl;
+
+    std::cout << GridLogMessage << "Adjoint - Checking if antisymmetric"
+              << std::endl;
+    for (int a = 0; a < Dimension; a++) {
+      generator(a, adjTa);
+      std::cout << GridLogMessage << a << std::endl;
+      assert(norm2(adjTa + transpose(adjTa)) < 1.0e-6);
+    }
+    std::cout << GridLogMessage << std::endl;
+  }
+
+  static void AdjointLieAlgebraMatrix(
+      const typename SU<ncolour>::LatticeAlgebraVector &h,
+      LatticeAdjMatrix &out, Real scale = 1.0) {
+    conformable(h, out);
+    GridBase *grid = out._grid;
+    LatticeAdjMatrix la(grid);
+    AMatrix iTa;
+
+    out = zero;
+    for (int a = 0; a < Dimension; a++) {
+      generator(a, iTa);
+      la = peekColour(h, a) * iTa;
+      out += la;
+    }
+    out *= scale;
+  }
+
+  // Projects the algebra components a lattice matrix (of dimension ncol*ncol -1 )
+  static void projectOnAlgebra(typename SU<ncolour>::LatticeAlgebraVector &h_out, const LatticeAdjMatrix &in, Real scale = 1.0) {
+    conformable(h_out, in);
+    h_out = zero;
+    AMatrix iTa;
+    Real coefficient = - 1.0/(ncolour) * scale;// 1/Nc for the normalization of the trace in the adj rep
+
+    for (int a = 0; a < Dimension; a++) {
+      generator(a, iTa);
+      auto tmp = real(trace(iTa * in)) * coefficient;
+      pokeColour(h_out, tmp, a);
+    }
+  }
+
+  // a projector that keeps the generators stored to avoid the overhead of recomputing them 
+  static void projector(typename SU<ncolour>::LatticeAlgebraVector &h_out, const LatticeAdjMatrix &in, Real scale = 1.0) {
+    conformable(h_out, in);
+    static std::vector<AMatrix> iTa(Dimension);  // to store the generators
+    h_out = zero;
+    static bool precalculated = false; 
+    if (!precalculated){
+      precalculated = true;
+        for (int a = 0; a < Dimension; a++) generator(a, iTa[a]);
+    }
+
+    Real coefficient = -1.0 / (ncolour) * scale;  // 1/Nc for the normalization of
+                                                // the trace in the adj rep
+
+    for (int a = 0; a < Dimension; a++) {
+      auto tmp = real(trace(iTa[a] * in)) * coefficient; 
+      pokeColour(h_out, tmp, a);
+    }
+  }
+
+
+};
+
+
+
+
+// Some useful type names
+
+typedef SU_Adjoint<2> SU2Adjoint;
+typedef SU_Adjoint<3> SU3Adjoint;
+typedef SU_Adjoint<4> SU4Adjoint;
+typedef SU_Adjoint<5> SU5Adjoint;
+
+typedef SU_Adjoint<Nc> AdjointMatrices;
+}
+}
+
+#endif

lib/qcd/utils/SUnTwoIndex.h

@@ -0,0 +1,276 @@
+////////////////////////////////////////////////////////////////////////
+//
+// * Two index representation generators
+//
+// * Normalisation for the fundamental generators:
+//   trace ta tb = 1/2 delta_ab = T_F delta_ab
+//   T_F = 1/2  for SU(N) groups
+//
+//
+//   base for NxN two index (anti-symmetric) matrices
+//   normalized to 1 (d_ij is the kroenecker delta)
+//
+//   (e^(ij)_{kl} = 1 / sqrt(2) (d_ik d_jl +/- d_jk d_il)
+//
+//   Then the generators are written as
+//
+//   (iT_a)^(ij)(lk) = i * ( tr[e^(ij)^dag e^(lk) T^trasp_a] +
+//   tr[e^(lk)e^(ij)^dag T_a] )  //
+//   
+//
+////////////////////////////////////////////////////////////////////////
+
+// Authors: David Preti, Guido Cossu
+
+#ifndef QCD_UTIL_SUN2INDEX_H
+#define QCD_UTIL_SUN2INDEX_H
+
+
+namespace Grid {
+namespace QCD {
+
+enum TwoIndexSymmetry { Symmetric = 1, AntiSymmetric = -1 };
+
+inline Real delta(int a, int b) { return (a == b) ? 1.0 : 0.0; }
+
+template <int ncolour, TwoIndexSymmetry S>
+class SU_TwoIndex : public SU<ncolour> {
+ public:
+  static const int Dimension = ncolour * (ncolour + S) / 2;
+  static const int NumGenerators = SU<ncolour>::AdjointDimension;
+
+  template <typename vtype>
+  using iSUnTwoIndexMatrix = iScalar<iScalar<iMatrix<vtype, Dimension> > >;
+
+  typedef iSUnTwoIndexMatrix<Complex> TIMatrix;
+  typedef iSUnTwoIndexMatrix<ComplexF> TIMatrixF;
+  typedef iSUnTwoIndexMatrix<ComplexD> TIMatrixD;
+
+  typedef iSUnTwoIndexMatrix<vComplex> vTIMatrix;
+  typedef iSUnTwoIndexMatrix<vComplexF> vTIMatrixF;
+  typedef iSUnTwoIndexMatrix<vComplexD> vTIMatrixD;
+
+  typedef Lattice<vTIMatrix> LatticeTwoIndexMatrix;
+  typedef Lattice<vTIMatrixF> LatticeTwoIndexMatrixF;
+  typedef Lattice<vTIMatrixD> LatticeTwoIndexMatrixD;
+
+  typedef Lattice<iVector<iScalar<iMatrix<vComplex, Dimension> >, Nd> >
+      LatticeTwoIndexField;
+  typedef Lattice<iVector<iScalar<iMatrix<vComplexF, Dimension> >, Nd> >
+      LatticeTwoIndexFieldF;
+  typedef Lattice<iVector<iScalar<iMatrix<vComplexD, Dimension> >, Nd> >
+      LatticeTwoIndexFieldD;
+
+  template <typename vtype>
+  using iSUnMatrix = iScalar<iScalar<iMatrix<vtype, ncolour> > >;
+
+  typedef iSUnMatrix<Complex> Matrix;
+  typedef iSUnMatrix<ComplexF> MatrixF;
+  typedef iSUnMatrix<ComplexD> MatrixD;
+
+  template <class cplx>
+  static void base(int Index, iSUnMatrix<cplx> &eij) {
+    // returns (e)^(ij)_{kl} necessary for change of base U_F -> U_R
+    assert(Index < NumGenerators);
+    eij = zero;
+
+    // for the linearisation of the 2 indexes 
+    static int a[ncolour * (ncolour - 1) / 2][2]; // store the a <-> i,j
+    static bool filled = false;
+    if (!filled) {
+      int counter = 0;
+      for (int i = 1; i < ncolour; i++) {
+        for (int j = 0; j < i; j++) {
+          a[counter][0] = i;
+          a[counter][1] = j;
+          counter++;
+        }
+      }
+      filled = true;
+    }
+
+    if (Index < ncolour * (ncolour - 1) / 2) {
+      baseOffDiagonal(a[Index][0], a[Index][1], eij);
+    } else {
+      baseDiagonal(Index, eij);
+    }
+  }
+
+  template <class cplx>
+  static void baseDiagonal(int Index, iSUnMatrix<cplx> &eij) {
+    eij = zero;
+    eij()()(Index - ncolour * (ncolour - 1) / 2,
+            Index - ncolour * (ncolour - 1) / 2) = 1.0;
+  }
+
+  template <class cplx>
+  static void baseOffDiagonal(int i, int j, iSUnMatrix<cplx> &eij) {
+    eij = zero;
+    for (int k = 0; k < ncolour; k++)
+      for (int l = 0; l < ncolour; l++)
+        eij()()(l, k) = delta(i, k) * delta(j, l) +
+                        S * delta(j, k) * delta(i, l);
+
+    RealD nrm = 1. / std::sqrt(2.0);
+    eij = eij * nrm;
+  }
+
+  static void printBase(void) {
+    for (int gen = 0; gen < Dimension; gen++) {
+      Matrix tmp;
+      base(gen, tmp);
+      std::cout << GridLogMessage << "Nc = " << ncolour << " t_" << gen
+                << std::endl;
+      std::cout << GridLogMessage << tmp << std::endl;
+    }
+  }
+
+  template <class cplx>
+  static void generator(int Index, iSUnTwoIndexMatrix<cplx> &i2indTa) {
+    Vector<typename SU<ncolour>::template iSUnMatrix<cplx> > ta(
+        ncolour * ncolour - 1);
+    Vector<typename SU<ncolour>::template iSUnMatrix<cplx> > eij(Dimension);
+    typename SU<ncolour>::template iSUnMatrix<cplx> tmp;
+    i2indTa = zero;
+    
+    for (int a = 0; a < ncolour * ncolour - 1; a++)
+      SU<ncolour>::generator(a, ta[a]);
+    
+    for (int a = 0; a < Dimension; a++) base(a, eij[a]);
+
+    for (int a = 0; a < Dimension; a++) {
+      tmp = transpose(ta[Index]) * adj(eij[a]) + adj(eij[a]) * ta[Index];
+      for (int b = 0; b < Dimension; b++) {
+        typename SU<ncolour>::template iSUnMatrix<cplx> tmp1 =
+            tmp * eij[b]; 
+        Complex iTr = TensorRemove(timesI(trace(tmp1)));
+        i2indTa()()(a, b) = iTr;
+      }
+    }
+  }
+
+  static void printGenerators(void) {
+    for (int gen = 0; gen < ncolour * ncolour - 1; gen++) {
+      TIMatrix i2indTa;
+      generator(gen, i2indTa);
+      std::cout << GridLogMessage << "Nc = " << ncolour << " t_" << gen
+                << std::endl;
+      std::cout << GridLogMessage << i2indTa << std::endl;
+    }
+  }
+
+  static void testGenerators(void) {
+    TIMatrix i2indTa, i2indTb;
+    std::cout << GridLogMessage << "2IndexRep - Checking if traceless"
+              << std::endl;
+    for (int a = 0; a < ncolour * ncolour - 1; a++) {
+      generator(a, i2indTa);
+      std::cout << GridLogMessage << a << std::endl;
+      assert(norm2(trace(i2indTa)) < 1.0e-6);
+    }
+    std::cout << GridLogMessage << std::endl;
+
+    std::cout << GridLogMessage << "2IndexRep - Checking if antihermitean"
+              << std::endl;
+    for (int a = 0; a < ncolour * ncolour - 1; a++) {
+      generator(a, i2indTa);
+      std::cout << GridLogMessage << a << std::endl;
+      assert(norm2(adj(i2indTa) + i2indTa) < 1.0e-6);
+    }
+
+    std::cout << GridLogMessage << std::endl;
+    std::cout << GridLogMessage
+              << "2IndexRep - Checking Tr[Ta*Tb]=delta(a,b)*(N +- 2)/2"
+              << std::endl;
+    for (int a = 0; a < ncolour * ncolour - 1; a++) {
+      for (int b = 0; b < ncolour * ncolour - 1; b++) {
+        generator(a, i2indTa);
+        generator(b, i2indTb);
+
+        // generator returns iTa, so we need a minus sign here
+        Complex Tr = -TensorRemove(trace(i2indTa * i2indTb));
+        std::cout << GridLogMessage << "a=" << a << "b=" << b << "Tr=" << Tr
+                  << std::endl;
+      }
+    }
+    std::cout << GridLogMessage << std::endl;
+  }
+
+  static void TwoIndexLieAlgebraMatrix(
+      const typename SU<ncolour>::LatticeAlgebraVector &h,
+      LatticeTwoIndexMatrix &out, Real scale = 1.0) {
+    conformable(h, out);
+    GridBase *grid = out._grid;
+    LatticeTwoIndexMatrix la(grid);
+    TIMatrix i2indTa;
+
+    out = zero;
+    for (int a = 0; a < ncolour * ncolour - 1; a++) {
+      generator(a, i2indTa);
+      la = peekColour(h, a) * i2indTa;
+      out += la;
+    }
+    out *= scale;
+  }
+
+  // Projects the algebra components 
+  // of a lattice matrix ( of dimension ncol*ncol -1 )
+  static void projectOnAlgebra(
+      typename SU<ncolour>::LatticeAlgebraVector &h_out,
+      const LatticeTwoIndexMatrix &in, Real scale = 1.0) {
+    conformable(h_out, in);
+    h_out = zero;
+    TIMatrix i2indTa;
+    Real coefficient = -2.0 / (ncolour + 2 * S) * scale;
+    // 2/(Nc +/- 2) for the normalization of the trace in the two index rep
+    for (int a = 0; a < ncolour * ncolour - 1; a++) {
+      generator(a, i2indTa);
+      auto tmp = real(trace(i2indTa * in)) * coefficient;
+      pokeColour(h_out, tmp, a);
+    }
+  }
+
+  // a projector that keeps the generators stored to avoid the overhead of
+  // recomputing them
+  static void projector(typename SU<ncolour>::LatticeAlgebraVector &h_out,
+                        const LatticeTwoIndexMatrix &in, Real scale = 1.0) {
+    conformable(h_out, in);
+    // to store the generators
+    static std::vector<TIMatrix> i2indTa(ncolour * ncolour -1); 
+    h_out = zero;
+    static bool precalculated = false;
+    if (!precalculated) {
+      precalculated = true;
+      for (int a = 0; a < ncolour * ncolour - 1; a++) generator(a, i2indTa[a]);
+    }
+
+    Real coefficient =
+        -2.0 / (ncolour + 2 * S) * scale;  // 2/(Nc +/- 2) for the normalization
+                                           // of the trace in the two index rep
+
+    for (int a = 0; a < ncolour * ncolour - 1; a++) {
+      auto tmp = real(trace(i2indTa[a] * in)) * coefficient;
+      pokeColour(h_out, tmp, a);
+    }
+  }
+};
+
+// Some useful type names
+typedef SU_TwoIndex<Nc, Symmetric> TwoIndexSymmMatrices;
+typedef SU_TwoIndex<Nc, AntiSymmetric> TwoIndexAntiSymmMatrices;
+
+typedef SU_TwoIndex<2, Symmetric> SU2TwoIndexSymm;
+typedef SU_TwoIndex<3, Symmetric> SU3TwoIndexSymm;
+typedef SU_TwoIndex<4, Symmetric> SU4TwoIndexSymm;
+typedef SU_TwoIndex<5, Symmetric> SU5TwoIndexSymm;
+
+typedef SU_TwoIndex<2, AntiSymmetric> SU2TwoIndexAntiSymm;
+typedef SU_TwoIndex<3, AntiSymmetric> SU3TwoIndexAntiSymm;
+typedef SU_TwoIndex<4, AntiSymmetric> SU4TwoIndexAntiSymm;
+typedef SU_TwoIndex<5, AntiSymmetric> SU5TwoIndexAntiSymm;
+
+
+}
+}
+
+#endif

lib/simd/Grid_qpx.h

@@ -1,12 +1,12 @@
-    /*************************************************************************************
+/*******************************************************************************
  
  Grid physics library, www.github.com/paboyle/Grid
  
  Source file: ./lib/simd/Grid_qpx.h
  
-    Copyright (C) 2015
+ Copyright (C) 2016
  
-Author: neo <cossu@post.kek.jp>
+ Author: Antonin Portelli <antonin.portelli@me.com>
  
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
@@ -23,96 +23,127 @@ Author: neo <cossu@post.kek.jp>
  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  
  See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-//----------------------------------------------------------------------
-/*! @file Grid_qpx.h
-  @brief Optimization libraries for QPX instructions set for BG/Q
-
-  Using intrinsics
-*/
-// Time-stamp: <2015-05-27 11:30:21 neo>
-//----------------------------------------------------------------------
-
-// lot of undefined functions
+ ******************************************************************************/
 
+namespace Grid {
 namespace Optimization {
+  typedef struct 
+  {
+    float v0,v1,v2,v3;
+  } vector4float;
+
+  inline std::ostream & operator<<(std::ostream& stream, const vector4double a)
+  {
+    stream << "{"<<vec_extract(a,0)<<","<<vec_extract(a,1)<<","<<vec_extract(a,2)<<","<<vec_extract(a,3)<<"}";
+    return stream;
+  };
+
+  inline std::ostream & operator<<(std::ostream& stream, const vector4float a)
+  {
+    stream << "{"<< a.v0 <<","<< a.v1 <<","<< a.v2 <<","<< a.v3 <<"}";
+    return stream;
+  };
   
   struct Vsplat{
     //Complex float
-    inline float operator()(float a, float b){
-      return {a,b,a,b};
+    inline vector4float operator()(float a, float b){
+      return (vector4float){a, b, a, b};
     }
     // Real float
-    inline float operator()(float a){
-      return {a,a,a,a};
+    inline vector4float operator()(float a){
+      return (vector4float){a, a, a, a};
     }
     //Complex double
     inline vector4double operator()(double a, double b){
-      return {a,b,a,b};
+      return (vector4double){a, b, a, b};
     }
     //Real double
     inline vector4double operator()(double a){
-      return {a,a,a,a};
+      return (vector4double){a, a, a, a};
     }
     //Integer
     inline int operator()(Integer a){
-#error
+      return a;
     }
   };
   
   struct Vstore{
     //Float
-    inline void operator()(float a, float* F){
-      assert(0);
+    inline void operator()(vector4double a, float *f){
+      vec_st(a, 0, f);
     }
+
+    inline void operator()(vector4double a, vector4float &f){
+      vec_st(a, 0, (float *)(&f));
+    }
+
+    inline void operator()(vector4float a, float *f){
+      f[0] = a.v0;
+      f[1] = a.v1;
+      f[2] = a.v2;
+      f[3] = a.v3;
+    }
+
     //Double
-    inline void operator()(vector4double a, double* D){
-      assert(0);
+    inline void operator()(vector4double a, double *d){
+      vec_st(a, 0, d);
     }
     //Integer
-    inline void operator()(int a, Integer* I){
-      assert(0);
+    inline void operator()(int a, Integer *i){
+      i[0] = a;
     }
-
   };
   
-
   struct Vstream{
     //Float
-    inline void operator()(float * a, float b){
-      assert(0);
-    }
-    //Double
-    inline void operator()(double * a, vector4double b){
-      assert(0);
+    inline void operator()(float *f, vector4double a){
+      vec_st(a, 0, f);
     }
 
+    inline void operator()(vector4float f, vector4double a){
+      vec_st(a, 0, (float *)(&f));
+    }
+
+    inline void operator()(float *f, vector4float a){
+      f[0] = a.v0;
+      f[1] = a.v1;
+      f[2] = a.v2;
+      f[3] = a.v3;
+    }
+
+    //Double
+    inline void operator()(double *d, vector4double a){
+      vec_st(a, 0, d);
+    }
 
   };
   
-
-
   struct Vset{
     // Complex float
-    inline float operator()(Grid::ComplexF *a){
-      return {a[0].real(),a[0].imag(),a[1].real(),a[1].imag(),a[2].real(),a[2].imag(),a[3].real(),a[3].imag()};
+    inline vector4float operator()(Grid::ComplexF *a){
+      return (vector4float){a[0].real(), a[0].imag(), a[1].real(), a[1].imag()};
     }
     // Complex double
     inline vector4double operator()(Grid::ComplexD *a){
-      return {a[0].real(),a[0].imag(),a[1].real(),a[1].imag(),a[2].real(),a[2].imag(),a[3].real(),a[3].imag()};
+      return vec_ld(0, (double *)a);
     }
+
     // Real float
-    inline float operator()(float *a){
-      return {a[0],a[1],a[2],a[3],a[4],a[5],a[6],a[7]};
+    inline vector4float operator()(float *a){
+      return (vector4float){a[0], a[1], a[2], a[3]};
     }
+
+    inline vector4double operator()(vector4float a){
+      return vec_ld(0, (float *)(&a));
+    }
+
     // Real double
     inline vector4double operator()(double *a){
-      return {a[0],a[1],a[2],a[3],a[4],a[5],a[6],a[7]};
+      return vec_ld(0, a);
     }
     // Integer
     inline int operator()(Integer *a){
-#error
+      return a[0];
     }
     
     
@@ -129,156 +160,247 @@ namespace Optimization {
     }
   };
   
-
- 
-
   /////////////////////////////////////////////////////
   // Arithmetic operations
   /////////////////////////////////////////////////////
-  struct Sum{
-    //Complex/Real float
-    inline float operator()(float a, float b){
-#error
+  #define FLOAT_WRAP_2(fn, pref)\
+  pref vector4float fn(vector4float a, vector4float b)\
+  {\
+    vector4double ad, bd, rd;\
+    vector4float  r;\
+    \
+    ad = Vset()(a);\
+    bd = Vset()(b);\
+    rd = fn(ad, bd);\
+    Vstore()(rd, r);\
+    \
+    return r;\
   }
+
+  #define FLOAT_WRAP_1(fn, pref)\
+  pref vector4float fn(vector4float a)\
+  {\
+    vector4double ad, rd;\
+    vector4float  r;\
+    \
+    ad = Vset()(a);\
+    rd = fn(ad);\
+    Vstore()(rd, r);\
+    \
+    return r;\
+  }
+
+  struct Sum{
     //Complex/Real double
     inline vector4double operator()(vector4double a, vector4double b){
       return vec_add(a, b);
     }
+
+    //Complex/Real float
+    FLOAT_WRAP_2(operator(), inline)
+
     //Integer
     inline int operator()(int a, int b){
-#error
+      return a + b;
     }
   };
   
   struct Sub{
-    //Complex/Real float
-    inline float operator()(float a, float b){
-#error
-    }
     //Complex/Real double
     inline vector4double operator()(vector4double a, vector4double b){
-#error
+      return vec_sub(a, b);
     }
+
+    //Complex/Real float
+    FLOAT_WRAP_2(operator(), inline)
+
     //Integer
-    inline floati operator()(int a, int b){
-#error
+    inline int operator()(int a, int b){
+      return a - b;
     }
   };
   
-
   struct MultComplex{
-    // Complex float
-    inline float operator()(float a, float b){
-#error
-    }
     // Complex double
     inline vector4double operator()(vector4double a, vector4double b){
-#error
+      return vec_xxnpmadd(a, b, vec_xmul(b, a));
     }
+
+    // Complex float
+    FLOAT_WRAP_2(operator(), inline)
   };
   
   struct Mult{
-    // Real float
-    inline float operator()(float a, float b){
-#error
-    }
     // Real double
     inline vector4double operator()(vector4double a, vector4double b){
-#error
+      return vec_mul(a, b);
     }
+
+    // Real float
+    FLOAT_WRAP_2(operator(), inline)
+
     // Integer
     inline int operator()(int a, int b){
-#error
+      return a*b;
     }
   };
   
-
   struct Conj{
-    // Complex single
-    inline float operator()(float in){
-      assert(0);
-    }
     // Complex double
-    inline vector4double operator()(vector4double in){
-      assert(0);
+    inline vector4double operator()(vector4double v){
+      return vec_mul(v, (vector4double){1., -1., 1., -1.});
     }
-    // do not define for integer input
+
+    // Complex float
+    FLOAT_WRAP_1(operator(), inline)
   };
   
   struct TimesMinusI{
-    //Complex single
-    inline float operator()(float in, float ret){
-      assert(0);
-    }
     //Complex double
-    inline vector4double operator()(vector4double in, vector4double ret){
-      assert(0);
+    inline vector4double operator()(vector4double v, vector4double ret){
+      return vec_xxcpnmadd(v, (vector4double){1., 1., 1., 1.},
+                               (vector4double){0., 0., 0., 0.});
     }
 
-
+    // Complex float
+    FLOAT_WRAP_2(operator(), inline)
   };
   
   struct TimesI{
-    //Complex single
-    inline float operator()(float in, float ret){
-  
-    }
     //Complex double
-    inline vector4double operator()(vector4double in, vector4double ret){
-  
+    inline vector4double operator()(vector4double v, vector4double ret){
+      return vec_xxcpnmadd(v, (vector4double){-1., -1., -1., -1.},
+                              (vector4double){0., 0., 0., 0.});
     }
 
-
+    // Complex float
+    FLOAT_WRAP_2(operator(), inline)
   };
 
+  struct Permute{
+    //Complex double
+    static inline vector4double Permute0(vector4double v){ //0123 -> 2301
+      return vec_perm(v, v, vec_gpci(02301));
+    };
+    static inline vector4double Permute1(vector4double v){ //0123 -> 1032
+      return vec_perm(v, v, vec_gpci(01032));
+    };
+    static inline vector4double Permute2(vector4double v){
+      return v;
+    };
+    static inline vector4double Permute3(vector4double v){
+      return v;
+    };
 
+    // Complex float
+    FLOAT_WRAP_1(Permute0, static inline)
+    FLOAT_WRAP_1(Permute1, static inline)
+    FLOAT_WRAP_1(Permute2, static inline)
+    FLOAT_WRAP_1(Permute3, static inline)
+  };
   
+  struct Rotate{
+    static inline vector4double rotate(vector4double v, int n){
+      switch(n){
+        case 0:
+          return v;
+          break;
+        case 1:
+          return vec_perm(v, v, vec_gpci(01230));
+          break;
+        case 2:
+          return vec_perm(v, v, vec_gpci(02301));
+          break;
+        case 3:
+          return vec_perm(v, v, vec_gpci(03012));
+          break;
+        default: assert(0);
+      }
+    }
 
+    static inline vector4float rotate(vector4float v, int n){
+      vector4double vd, rd;
+      vector4float  r;
 
-  //////////////////////////////////////////////
-  // Some Template specialization
+      vd = Vset()(v);
+      rd = rotate(vd, n);
+      Vstore()(rd, r);
+
+      return r;
+    }
+  };
   
   //Complex float Reduce
   template<>
-    inline Grid::ComplexF Reduce<Grid::ComplexF, float>::operator()(float in){
-    assert(0);
+  inline Grid::ComplexF
+  Reduce<Grid::ComplexF, vector4float>::operator()(vector4float v) { //2 complex
+    vector4float v1,v2;
+    
+    v1 = Optimization::Permute::Permute0(v);
+    v1 = Optimization::Sum()(v1, v);
+    
+    return Grid::ComplexF(v1.v0, v1.v1);
   }
   //Real float Reduce
   template<>
-    inline Grid::RealF Reduce<Grid::RealF, float>::operator()(float in){
-    assert(0);
+  inline Grid::RealF
+  Reduce<Grid::RealF, vector4float>::operator()(vector4float v){ //4 floats
+    vector4float v1,v2;
+    
+    v1 = Optimization::Permute::Permute0(v);
+    v1 = Optimization::Sum()(v1, v);
+    v2 = Optimization::Permute::Permute1(v1);
+    v1 = Optimization::Sum()(v1, v2);
+    
+    return v1.v0;
   }
   
   
   //Complex double Reduce
   template<>
-    inline Grid::ComplexD Reduce<Grid::ComplexD, vector4double>::operator()(vector4double in){
-    assert(0);
+  inline Grid::ComplexD
+  Reduce<Grid::ComplexD, vector4double>::operator()(vector4double v){ //2 complex
+    vector4double v1;
+    
+    v1 = Optimization::Permute::Permute0(v);
+    v1 = vec_add(v1, v);
+    
+    return Grid::ComplexD(vec_extract(v1, 0), vec_extract(v1, 1));
   }
   
   //Real double Reduce
   template<>
-    inline Grid::RealD Reduce<Grid::RealD, vector4double>::operator()(vector4double in){
-    assert(0);
+  inline Grid::RealD
+  Reduce<Grid::RealD, vector4double>::operator()(vector4double v){ //4 doubles
+    vector4double v1,v2;
+    
+    v1 = Optimization::Permute::Permute0(v);
+    v1 = vec_add(v1, v);
+    v2 = Optimization::Permute::Permute1(v1);
+    v1 = vec_add(v1, v2);
+
+    return vec_extract(v1, 0);
   }
   
   //Integer Reduce
   template<>
-    inline Integer Reduce<Integer, floati>::operator()(float in){
+  inline Integer Reduce<Integer, int>::operator()(int in){
+    // FIXME unimplemented
+    printf("Reduce : Missing integer implementation -> FIX\n");
     assert(0);
   }
-  
-  
 }
 
-//////////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
 // Here assign types
-namespace Grid {
-  typedef float SIMD_Ftype  __attribute__ ((vector_size (16)));         // Single precision type
+typedef Optimization::vector4float SIMD_Ftype;  // Single precision type
 typedef vector4double              SIMD_Dtype; // Double precision type
 typedef int                        SIMD_Itype; // Integer type
 
+// prefetch utilities
 inline void v_prefetch0(int size, const char *ptr){};
+inline void prefetch_HINT_T0(const char *ptr){};
+
 
 // Function name aliases
 typedef Optimization::Vsplat   VsplatSIMD;
@@ -287,7 +409,6 @@ namespace Grid {
 typedef Optimization::Vstream  VstreamSIMD;
 template <typename S, typename T> using ReduceSIMD = Optimization::Reduce<S,T>;
 
-
 // Arithmetic operations
 typedef Optimization::Sum         SumSIMD;
 typedef Optimization::Sub         SubSIMD;

lib/simd/Grid_vector_types.h

@@ -388,6 +388,12 @@ class Grid_simd {
 
 };  // end of Grid_simd class definition
 
+
+inline void permute(ComplexD &y,ComplexD b, int perm) {  y=b; }
+inline void permute(ComplexF &y,ComplexF b, int perm) {  y=b; }
+inline void permute(RealD &y,RealD b, int perm) {  y=b; }
+inline void permute(RealF &y,RealF b, int perm) {  y=b; }
+
 ////////////////////////////////////////////////////////////////////
 // General rotate
 ////////////////////////////////////////////////////////////////////

lib/simd/Grid_vector_unops.h

@@ -67,15 +67,13 @@ template <class scalar>
 struct AsinRealFunctor {
   scalar operator()(const scalar &a) const { return asin(real(a)); }
 };
-
 template <class scalar>
 struct LogRealFunctor {
   scalar operator()(const scalar &a) const { return log(real(a)); }
 };
-
 template <class scalar>
-struct ExpRealFunctor {
-  scalar operator()(const scalar &a) const { return exp(real(a)); }
+struct ExpFunctor {
+  scalar operator()(const scalar &a) const { return exp(a); }
 };
 template <class scalar>
 struct NotFunctor {
@@ -85,7 +83,6 @@ template <class scalar>
 struct AbsRealFunctor {
   scalar operator()(const scalar &a) const { return std::abs(real(a)); }
 };
-
 template <class scalar>
 struct PowRealFunctor {
   double y;
@@ -135,7 +132,6 @@ template <class Scalar>
 inline Scalar rsqrt(const Scalar &r) {
   return (RSqrtRealFunctor<Scalar>(), r);
 }
-
 template <class S, class V>
 inline Grid_simd<S, V> cos(const Grid_simd<S, V> &r) {
   return SimdApply(CosRealFunctor<S>(), r);
@@ -162,7 +158,7 @@ inline Grid_simd<S, V> abs(const Grid_simd<S, V> &r) {
 }
 template <class S, class V>
 inline Grid_simd<S, V> exp(const Grid_simd<S, V> &r) {
-  return SimdApply(ExpRealFunctor<S>(), r);
+  return SimdApply(ExpFunctor<S>(), r);
 }
 template <class S, class V>
 inline Grid_simd<S, V> Not(const Grid_simd<S, V> &r) {

lib/tensors/Tensor_exp.h

@@ -56,7 +56,7 @@ namespace Grid {
 	temp = unit + temp*arg;
       }
       
-      return ProjectOnGroup(temp);//maybe not strictly necessary
+      return temp;
       
     }
 

scripts/filelist

@@ -13,9 +13,10 @@ echo CCFILES=$CCFILES >> Make.inc
 
 # tests Make.inc
 cd $home/tests
-dirs=`find . -type d `
+dirs=`find . -type d -not -path '*/\.*'`
 for subdir in $dirs; do
     cd $home/tests/$subdir
+    pwd
     TESTS=`ls T*.cc`
     TESTLIST=`echo ${TESTS} | sed s/.cc//g `
     PREF=`[ $subdir = '.' ] && echo noinst || echo EXTRA`

scripts/reconfigure_script

@@ -0,0 +1,4 @@
+aclocal -I m4
+autoheader -f
+automake -f --add-missing
+autoconf -f

scripts/update_fftw.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+if (( $# != 1 )); then
+    echo "usage: `basename $0` <archive>" 1>&2
+    exit 1
+fi
+ARC=$1
+
+INITDIR=`pwd`
+rm -rf lib/fftw
+mkdir lib/fftw
+
+ARCDIR=`tar -tf ${ARC} | head -n1 | sed -e 's@/.*@@'`
+tar -xf ${ARC}
+cp ${ARCDIR}/api/fftw3.h lib/fftw/
+
+cd ${INITDIR}
+rm -rf ${ARCDIR}

tests/Test_simd.cc

@@ -157,10 +157,9 @@ void Tester(const functor &func)
   std::cout << GridLogMessage << " " << func.name() << std::endl;
 
   std::cout << GridLogDebug << v_input1 << std::endl;
+  std::cout << GridLogDebug << v_input2 << std::endl;
   std::cout << GridLogDebug << v_result << std::endl;
 
-
-
   int ok=0;
   for(int i=0;i<Nsimd;i++){
     if ( abs(reference[i]-result[i])>1.0e-7){

tests/core/Test_fft.cc

@@ -0,0 +1,111 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_cshift.cc
+
+    Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout( { vComplexD::Nsimd(),1,1,1});
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  int vol = 1;
+  for(int d=0;d<latt_size.size();d++){
+    vol = vol * latt_size[d];
+  }
+  GridCartesian        Fine(latt_size,simd_layout,mpi_layout);
+
+  LatticeComplexD     one(&Fine);
+  LatticeComplexD      zz(&Fine);
+  LatticeComplexD       C(&Fine);
+  LatticeComplexD  Ctilde(&Fine);
+  LatticeComplexD    coor(&Fine);
+
+  LatticeSpinMatrixD    S(&Fine);
+  LatticeSpinMatrixD    Stilde(&Fine);
+  
+  std::vector<int> p({1,2,3,2});
+
+  one = ComplexD(1.0,0.0);
+  zz  = ComplexD(0.0,0.0);
+
+  ComplexD ci(0.0,1.0);
+
+  C=zero;
+  for(int mu=0;mu<4;mu++){
+    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+    LatticeCoordinate(coor,mu);
+    C = C - (TwoPiL * p[mu]) * coor;
+  }
+
+  C = exp(C*ci);
+
+  S=zero;
+  S = S+C;
+
+  FFT theFFT(&Fine);
+
+  theFFT.FFT_dim(Ctilde,C,0,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,1,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,2,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,3,FFT::forward);  std::cout << theFFT.MFlops()<<std::endl;
+
+  //  C=zero;
+  //  Ctilde = where(abs(Ctilde)<1.0e-10,C,Ctilde);
+  TComplexD cVol;
+  cVol()()() = vol;
+
+  C=zero;
+  pokeSite(cVol,C,p);
+  C=C-Ctilde;
+  std::cout << "diff scalar "<<norm2(C) << std::endl;
+
+  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,S,3,FFT::forward);std::cout << theFFT.MFlops()<<std::endl;
+
+  SpinMatrixD Sp; 
+  Sp = zero; Sp = Sp+cVol;
+
+  S=zero;
+  pokeSite(Sp,S,p);
+
+  S= S-Stilde;
+  std::cout << "diff FT[SpinMat] "<<norm2(S) << std::endl;
+
+  Grid_finalize();
+}

tests/core/Test_fftf.cc

@@ -0,0 +1,111 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_cshift.cc
+
+    Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout( { vComplexF::Nsimd(),1,1,1});
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  int vol = 1;
+  for(int d=0;d<latt_size.size();d++){
+    vol = vol * latt_size[d];
+  }
+  GridCartesian        Fine(latt_size,simd_layout,mpi_layout);
+
+  LatticeComplexF     one(&Fine);
+  LatticeComplexF      zz(&Fine);
+  LatticeComplexF       C(&Fine);
+  LatticeComplexF  Ctilde(&Fine);
+  LatticeComplexF    coor(&Fine);
+
+  LatticeSpinMatrixF    S(&Fine);
+  LatticeSpinMatrixF    Stilde(&Fine);
+  
+  std::vector<int> p({1,2,3,2});
+
+  one = ComplexF(1.0,0.0);
+  zz  = ComplexF(0.0,0.0);
+
+  ComplexF ci(0.0,1.0);
+
+  C=zero;
+  for(int mu=0;mu<4;mu++){
+    RealD TwoPiL =  M_PI * 2.0/ latt_size[mu];
+    LatticeCoordinate(coor,mu);
+    C = C - (TwoPiL * p[mu]) * coor;
+  }
+
+  C = exp(C*ci);
+
+  S=zero;
+  S = S+C;
+
+  FFT theFFT(&Fine);
+
+  theFFT.FFT_dim(Ctilde,C,0,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,1,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,2,FFT::forward);  C=Ctilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Ctilde,C,3,FFT::forward);  std::cout << theFFT.MFlops()<<std::endl;
+
+  //  C=zero;
+  //  Ctilde = where(abs(Ctilde)<1.0e-10,C,Ctilde);
+  TComplexF cVol;
+  cVol()()() = vol;
+
+  C=zero;
+  pokeSite(cVol,C,p);
+  C=C-Ctilde;
+  std::cout << "diff scalar "<<norm2(C) << std::endl;
+
+  theFFT.FFT_dim(Stilde,S,0,FFT::forward);  S=Stilde; std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,S,1,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,S,2,FFT::forward);  S=Stilde;std::cout << theFFT.MFlops()<<std::endl;
+  theFFT.FFT_dim(Stilde,S,3,FFT::forward);std::cout << theFFT.MFlops()<<std::endl;
+
+  SpinMatrixF Sp; 
+  Sp = zero; Sp = Sp+cVol;
+
+  S=zero;
+  pokeSite(Sp,S,p);
+
+  S= S-Stilde;
+  std::cout << "diff FT[SpinMat] "<<norm2(S) << std::endl;
+
+  Grid_finalize();
+}

tests/core/Test_lie_generators.cc

@@ -8,6 +8,7 @@
 
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -23,54 +24,509 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #include <Grid/Grid.h>
 
+#include <Grid/qcd/utils/CovariantCshift.h>
+
+#include <Grid/qcd/utils/SUn.h>
+#include <Grid/qcd/utils/SUnAdjoint.h>
+#include <Grid/qcd/utils/SUnTwoIndex.h>
+
+#include <Grid/qcd/representations/adjoint.h>
+#include <Grid/qcd/representations/two_index.h>
+#include <Grid/qcd/utils/WilsonLoops.h>
 
 using namespace std;
 using namespace Grid;
 using namespace Grid::QCD;
 
-
-int main (int argc, char ** argv)
-{
+int main(int argc, char** argv) {
   Grid_init(&argc, &argv);
 
   std::vector<int> latt({4, 4, 4, 8});
-  GridCartesian * grid = SpaceTimeGrid::makeFourDimGrid(latt, 
-							GridDefaultSimd(Nd,vComplex::Nsimd()),
-							GridDefaultMpi());
+  GridCartesian* grid = SpaceTimeGrid::makeFourDimGrid(
+      latt, GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi());
 
   GridRedBlackCartesian* rbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(grid);
 
-  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
   std::cout << GridLogMessage << "* Generators for SU(2)" << std::endl;
-  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
   SU2::printGenerators();
+  std::cout << "Dimension of adjoint representation: "<< SU2Adjoint::Dimension << std::endl;
+  SU2Adjoint::printGenerators();
   SU2::testGenerators();
+  SU2Adjoint::testGenerators();
 
-  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
   std::cout << GridLogMessage << "* Generators for SU(3)" << std::endl;
-  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
   SU3::printGenerators();
+  std::cout << "Dimension of adjoint representation: "<< SU3Adjoint::Dimension << std::endl;
+  SU3Adjoint::printGenerators();
   SU3::testGenerators();
+  SU3Adjoint::testGenerators();
+
+  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
+  std::cout<<GridLogMessage<<"* Generators for SU(4)"<<std::endl;
+  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
+  SU4::printGenerators();
+  std::cout << "Dimension of adjoint representation: "<< SU4Adjoint::Dimension << std::endl;
+  SU4Adjoint::printGenerators();
+  SU4::testGenerators();
+  SU4Adjoint::testGenerators();
+
+  // Projectors 
+  GridParallelRNG gridRNG(grid);
+  gridRNG.SeedRandomDevice();
+  SU3Adjoint::LatticeAdjMatrix Gauss(grid);
+  SU3::LatticeAlgebraVector ha(grid);
+  SU3::LatticeAlgebraVector hb(grid);
+  random(gridRNG,Gauss);
+
+  std::cout << GridLogMessage << "Start projectOnAlgebra" << std::endl;
+  SU3Adjoint::projectOnAlgebra(ha, Gauss);
+  std::cout << GridLogMessage << "end projectOnAlgebra" << std::endl;
+  std::cout << GridLogMessage << "Start projector" << std::endl;
+  SU3Adjoint::projector(hb, Gauss);
+  std::cout << GridLogMessage << "end projector" << std::endl;
+
+  std::cout << GridLogMessage << "ReStart projector" << std::endl;
+  SU3Adjoint::projector(hb, Gauss);
+  std::cout << GridLogMessage << "end projector" << std::endl;
+  SU3::LatticeAlgebraVector diff = ha -hb;
+  std::cout << GridLogMessage << "Difference: " << norm2(diff) << std::endl;
+
+
+  // Testing HMC representation classes
+  AdjointRep<Nc> AdjRep(grid);
+
+  // AdjointRepresentation has the predefined number of colours Nc
+  Representations<FundamentalRepresentation, AdjointRepresentation, TwoIndexSymmetricRepresentation> RepresentationTypes(grid);  
+
+  
+  LatticeGaugeField U(grid), V(grid);
+  SU<Nc>::HotConfiguration<LatticeGaugeField>(gridRNG, U);
+  SU<Nc>::HotConfiguration<LatticeGaugeField>(gridRNG, V);
+
+  // Adjoint representation
+  // Test group structure
+  // (U_f * V_f)_r = U_r * V_r
+  LatticeGaugeField UV(grid);
+  UV = zero;
+  for (int mu = 0; mu < Nd; mu++) {
+    SU<Nc>::LatticeMatrix Umu = peekLorentz(U,mu);
+    SU<Nc>::LatticeMatrix Vmu = peekLorentz(V,mu);
+    pokeLorentz(UV,Umu*Vmu, mu);
+  }
+
+  AdjRep.update_representation(UV);
+  typename AdjointRep<Nc>::LatticeField UVr = AdjRep.U;  // (U_f * V_f)_r 
+
+
+  AdjRep.update_representation(U);
+  typename AdjointRep<Nc>::LatticeField Ur = AdjRep.U;  // U_r
+
+  AdjRep.update_representation(V);
+  typename AdjointRep<Nc>::LatticeField Vr = AdjRep.U;  // V_r
+
+  typename AdjointRep<Nc>::LatticeField UrVr(grid);
+  UrVr = zero;
+  for (int mu = 0; mu < Nd; mu++) {
+    typename AdjointRep<Nc>::LatticeMatrix Urmu = peekLorentz(Ur,mu);
+    typename AdjointRep<Nc>::LatticeMatrix Vrmu = peekLorentz(Vr,mu);
+    pokeLorentz(UrVr,Urmu*Vrmu, mu);
+  }
+
+  typename AdjointRep<Nc>::LatticeField Diff_check = UVr - UrVr;
+  std::cout << GridLogMessage << "Group structure SU("<<Nc<<") check difference (Adjoint representation) : " << norm2(Diff_check) << std::endl;
+
+  // Check correspondence of algebra and group transformations
+  // Create a random vector
+  SU<Nc>::LatticeAlgebraVector h_adj(grid);
+  typename AdjointRep<Nc>::LatticeMatrix Ar(grid);
+  random(gridRNG,h_adj);
+  h_adj = real(h_adj);
+  SU_Adjoint<Nc>::AdjointLieAlgebraMatrix(h_adj,Ar);
+
+  // Re-extract h_adj
+  SU<Nc>::LatticeAlgebraVector h_adj2(grid);
+  SU_Adjoint<Nc>::projectOnAlgebra(h_adj2, Ar);
+  SU<Nc>::LatticeAlgebraVector h_diff = h_adj - h_adj2;
+  std::cout << GridLogMessage << "Projections structure check vector difference (Adjoint representation) : " << norm2(h_diff) << std::endl;
+
+  // Exponentiate
+  typename AdjointRep<Nc>::LatticeMatrix Uadj(grid);
+  Uadj  = expMat(Ar, 1.0, 16);
+
+  typename AdjointRep<Nc>::LatticeMatrix uno(grid);
+  uno = 1.0;
+  // Check matrix Uadj, must be real orthogonal
+  typename AdjointRep<Nc>::LatticeMatrix Ucheck = Uadj - conjugate(Uadj);
+  std::cout << GridLogMessage << "Reality check: " << norm2(Ucheck)
+            << std::endl;
+
+  Ucheck = Uadj * adj(Uadj) - uno;
+  std::cout << GridLogMessage << "orthogonality check 1: " << norm2(Ucheck)
+            << std::endl;
+  Ucheck = adj(Uadj) * Uadj - uno;
+  std::cout << GridLogMessage << "orthogonality check 2: " << norm2(Ucheck)
+            << std::endl;
+      
+
+  // Construct the fundamental matrix in the group
+  SU<Nc>::LatticeMatrix Af(grid);
+  SU<Nc>::FundamentalLieAlgebraMatrix(h_adj,Af);
+  SU<Nc>::LatticeMatrix Ufund(grid);
+  Ufund  = expMat(Af, 1.0, 16);
+  // Check unitarity
+  SU<Nc>::LatticeMatrix uno_f(grid);
+  uno_f = 1.0;
+  SU<Nc>::LatticeMatrix UnitCheck(grid);
+  UnitCheck = Ufund * adj(Ufund) - uno_f;
+  std::cout << GridLogMessage << "unitarity check 1: " << norm2(UnitCheck)
+            << std::endl;
+  UnitCheck = adj(Ufund) * Ufund - uno_f;
+  std::cout << GridLogMessage << "unitarity check 2: " << norm2(UnitCheck)
+            << std::endl;
+
+
+  // Tranform to the adjoint representation
+  U = zero; // fill this with only one direction
+  pokeLorentz(U,Ufund,0); // the representation transf acts on full gauge fields
+
+  AdjRep.update_representation(U);
+  Ur = AdjRep.U;  // U_r  
+  typename AdjointRep<Nc>::LatticeMatrix Ur0 = peekLorentz(Ur,0); // this should be the same as Uadj
+
+  typename AdjointRep<Nc>::LatticeMatrix Diff_check_mat = Ur0 - Uadj;
+  std::cout << GridLogMessage << "Projections structure check group difference : " << norm2(Diff_check_mat) << std::endl;
+
+
+
+
+  // TwoIndexRep tests
+
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+
+
+  
+  std::cout << GridLogMessage << "* eS^{ij} base for SU(2)" << std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+  std::cout << GridLogMessage << "Dimension of Two Index Symmetric representation: "<< SU2TwoIndexSymm::Dimension << std::endl;
+  SU2TwoIndexSymm::printBase();
+      std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+        std::cout << GridLogMessage << "Generators of Two Index Symmetric representation: "<< SU2TwoIndexSymm::Dimension << std::endl;
+  SU2TwoIndexSymm::printGenerators();
+        std::cout << GridLogMessage << "Test of Two Index Symmetric Generators: "<< SU2TwoIndexSymm::Dimension << std::endl;
+  SU2TwoIndexSymm::testGenerators();
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+
+
+  
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+  std::cout << GridLogMessage << "* eAS^{ij} base for SU(2)" << std::endl;
+  
+  std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+  std::cout << GridLogMessage << "Dimension of Two Index anti-Symmetric representation: "<< SU2TwoIndexAntiSymm::Dimension << std::endl;
+  SU2TwoIndexAntiSymm::printBase();
+      std::cout << GridLogMessage << "*********************************************"
+            << std::endl;
+        std::cout << GridLogMessage << "Dimension of Two Index anti-Symmetric representation: "<< SU2TwoIndexAntiSymm::Dimension << std::endl;
+  SU2TwoIndexAntiSymm::printGenerators();
+  std::cout << GridLogMessage << "Test of Two Index anti-Symmetric Generators: "<< SU2TwoIndexAntiSymm::Dimension << std::endl;
+  SU2TwoIndexAntiSymm::testGenerators();
+  
+  
+  
+  std::cout << GridLogMessage << "*********************************************"
+      << std::endl;
+  std::cout << GridLogMessage << "Test for the Two Index Symmetric projectors"
+      << std::endl;
+  // Projectors 
+  SU3TwoIndexSymm::LatticeTwoIndexMatrix Gauss2(grid);
+  random(gridRNG,Gauss2);
+  
+  std::cout << GridLogMessage << "Start projectOnAlgebra" << std::endl;
+  SU3TwoIndexSymm::projectOnAlgebra(ha, Gauss2);
+  std::cout << GridLogMessage << "end projectOnAlgebra" << std::endl;
+  std::cout << GridLogMessage << "Start projector" << std::endl;
+  SU3TwoIndexSymm::projector(hb, Gauss2);
+  std::cout << GridLogMessage << "end projector" << std::endl;
+  
+  std::cout << GridLogMessage << "ReStart projector" << std::endl;
+  SU3TwoIndexSymm::projector(hb, Gauss2);
+  std::cout << GridLogMessage << "end projector" << std::endl;
+  SU3::LatticeAlgebraVector diff2 = ha - hb;
+  std::cout << GridLogMessage << "Difference: " << norm2(diff) << std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+      << std::endl;
+
+  
+    std::cout << GridLogMessage << "*********************************************"
+      << std::endl;
+  std::cout << GridLogMessage << "Test for the Two index anti-Symmetric projectors"
+      << std::endl;
+  // Projectors
+  SU3TwoIndexAntiSymm::LatticeTwoIndexMatrix Gauss2a(grid);
+  random(gridRNG,Gauss2a);
+  
+  std::cout << GridLogMessage << "Start projectOnAlgebra" << std::endl;
+  SU3TwoIndexAntiSymm::projectOnAlgebra(ha, Gauss2a);
+  std::cout << GridLogMessage << "end projectOnAlgebra" << std::endl;
+  std::cout << GridLogMessage << "Start projector" << std::endl;
+  SU3TwoIndexAntiSymm::projector(hb, Gauss2a);
+  std::cout << GridLogMessage << "end projector" << std::endl;
+  
+  std::cout << GridLogMessage << "ReStart projector" << std::endl;
+  SU3TwoIndexAntiSymm::projector(hb, Gauss2a);
+  std::cout << GridLogMessage << "end projector" << std::endl;
+  SU3::LatticeAlgebraVector diff2a = ha - hb;
+  std::cout << GridLogMessage << "Difference: " << norm2(diff2a) << std::endl;
+  std::cout << GridLogMessage << "*********************************************"
+      << std::endl;
+    
+  
+  std::cout << GridLogMessage << "Two index Symmetric: Checking Group Structure"
+      << std::endl;
+  // Testing HMC representation classes
+  TwoIndexRep< Nc, Symmetric > TIndexRep(grid);
+
+  // Test group structure
+  // (U_f * V_f)_r = U_r * V_r
+  LatticeGaugeField U2(grid), V2(grid);
+  SU<Nc>::HotConfiguration<LatticeGaugeField>(gridRNG, U2);
+  SU<Nc>::HotConfiguration<LatticeGaugeField>(gridRNG, V2);
+  
+  LatticeGaugeField UV2(grid);
+  UV2 = zero;
+  for (int mu = 0; mu < Nd; mu++) {
+    SU<Nc>::LatticeMatrix Umu2 = peekLorentz(U2,mu);
+    SU<Nc>::LatticeMatrix Vmu2 = peekLorentz(V2,mu);
+    pokeLorentz(UV2,Umu2*Vmu2, mu);
+  }
+  
+  TIndexRep.update_representation(UV2);
+  typename TwoIndexRep< Nc, Symmetric >::LatticeField UVr2 = TIndexRep.U;  // (U_f * V_f)_r
+  
+  TIndexRep.update_representation(U2);
+  typename TwoIndexRep< Nc, Symmetric >::LatticeField Ur2 = TIndexRep.U;  // U_r
+  
+  TIndexRep.update_representation(V2);
+  typename TwoIndexRep< Nc, Symmetric >::LatticeField Vr2 = TIndexRep.U;  // V_r
+  
+  typename TwoIndexRep< Nc, Symmetric >::LatticeField Ur2Vr2(grid);
+  Ur2Vr2 = zero;
+  for (int mu = 0; mu < Nd; mu++) {
+    typename TwoIndexRep< Nc, Symmetric >::LatticeMatrix Urmu2 = peekLorentz(Ur2,mu);
+    typename TwoIndexRep< Nc, Symmetric >::LatticeMatrix Vrmu2 = peekLorentz(Vr2,mu);
+    pokeLorentz(Ur2Vr2,Urmu2*Vrmu2, mu);
+  }
+  
+  typename TwoIndexRep< Nc, Symmetric >::LatticeField Diff_check2 = UVr2 - Ur2Vr2;
+  std::cout << GridLogMessage << "Group structure SU("<<Nc<<") check difference (Two Index Symmetric): " << norm2(Diff_check2) << std::endl;
+
+  
+  // Check correspondence of algebra and group transformations
+  // Create a random vector
+  SU<Nc>::LatticeAlgebraVector h_sym(grid);
+  typename TwoIndexRep< Nc, Symmetric>::LatticeMatrix Ar_sym(grid);
+  random(gridRNG,h_sym);
+  h_sym = real(h_sym);
+  SU_TwoIndex<Nc,Symmetric>::TwoIndexLieAlgebraMatrix(h_sym,Ar_sym);
+  
+  // Re-extract h_sym
+  SU<Nc>::LatticeAlgebraVector h_sym2(grid);
+  SU_TwoIndex< Nc, Symmetric>::projectOnAlgebra(h_sym2, Ar_sym);
+  SU<Nc>::LatticeAlgebraVector h_diff_sym = h_sym - h_sym2;
+  std::cout << GridLogMessage << "Projections structure check vector difference (Two Index Symmetric): " << norm2(h_diff_sym) << std::endl;
+
+  
+  // Exponentiate
+  typename TwoIndexRep< Nc, Symmetric>::LatticeMatrix U2iS(grid);
+  U2iS  = expMat(Ar_sym, 1.0, 16);
+  
+  typename TwoIndexRep< Nc, Symmetric>::LatticeMatrix uno2iS(grid);
+  uno2iS = 1.0;
+  // Check matrix U2iS, must be real orthogonal
+  typename TwoIndexRep< Nc, Symmetric>::LatticeMatrix Ucheck2iS = U2iS - conjugate(U2iS);
+  std::cout << GridLogMessage << "Reality check: " << norm2(Ucheck2iS)
+      << std::endl;
+  
+  Ucheck2iS = U2iS * adj(U2iS) - uno2iS;
+  std::cout << GridLogMessage << "orthogonality check 1: " << norm2(Ucheck2iS)
+      << std::endl;
+  Ucheck2iS = adj(U2iS) * U2iS - uno2iS;
+  std::cout << GridLogMessage << "orthogonality check 2: " << norm2(Ucheck2iS)
+      << std::endl;
+  
+  
+  
+  // Construct the fundamental matrix in the group
+  SU<Nc>::LatticeMatrix Af_sym(grid);
+  SU<Nc>::FundamentalLieAlgebraMatrix(h_sym,Af_sym);
+  SU<Nc>::LatticeMatrix Ufund2(grid);
+  Ufund2  = expMat(Af_sym, 1.0, 16);
+  SU<Nc>::LatticeMatrix UnitCheck2(grid);
+  UnitCheck2 = Ufund2 * adj(Ufund2) - uno_f;
+  std::cout << GridLogMessage << "unitarity check 1: " << norm2(UnitCheck2)
+      << std::endl;
+  UnitCheck2 = adj(Ufund2) * Ufund2 - uno_f;
+  std::cout << GridLogMessage << "unitarity check 2: " << norm2(UnitCheck2)
+      << std::endl;
+  
+
+  // Tranform to the 2Index Sym representation
+  U = zero; // fill this with only one direction
+  pokeLorentz(U,Ufund2,0); // the representation transf acts on full gauge fields
+  
+  TIndexRep.update_representation(U);
+  Ur2 = TIndexRep.U;  // U_r  
+  typename TwoIndexRep< Nc, Symmetric>::LatticeMatrix Ur02 = peekLorentz(Ur2,0); // this should be the same as U2iS
+  
+  typename TwoIndexRep< Nc, Symmetric>::LatticeMatrix Diff_check_mat2 = Ur02 - U2iS;
+  std::cout << GridLogMessage << "Projections structure check group difference (Two Index Symmetric): " << norm2(Diff_check_mat2) << std::endl;
+  
+
+
+
+  if (TwoIndexRep<Nc, AntiSymmetric >::Dimension != 1){
+
+  std::cout << GridLogMessage << "*********************************************"
+      << std::endl;
+    
+  
+  std::cout << GridLogMessage << "Two Index anti-Symmetric: Check Group Structure"
+      << std::endl;
+  // Testing HMC representation classes
+  TwoIndexRep< Nc, AntiSymmetric > TIndexRepA(grid);
+
+
+  // Test group structure
+  // (U_f * V_f)_r = U_r * V_r
+  LatticeGaugeField U2A(grid), V2A(grid);
+  SU<Nc>::HotConfiguration<LatticeGaugeField>(gridRNG, U2A);
+  SU<Nc>::HotConfiguration<LatticeGaugeField>(gridRNG, V2A);
+  
+  LatticeGaugeField UV2A(grid);
+  UV2A = zero;
+  for (int mu = 0; mu < Nd; mu++) {
+    SU<Nc>::LatticeMatrix Umu2A = peekLorentz(U2,mu);
+    SU<Nc>::LatticeMatrix Vmu2A = peekLorentz(V2,mu);
+    pokeLorentz(UV2A,Umu2A*Vmu2A, mu);
+  }
+  
+  TIndexRep.update_representation(UV2A);
+  typename TwoIndexRep< Nc, AntiSymmetric >::LatticeField UVr2A = TIndexRepA.U;  // (U_f * V_f)_r
+  
+  TIndexRep.update_representation(U2A);
+  typename TwoIndexRep< Nc, AntiSymmetric >::LatticeField Ur2A = TIndexRepA.U;  // U_r
+  
+  TIndexRep.update_representation(V2A);
+  typename TwoIndexRep< Nc, AntiSymmetric >::LatticeField Vr2A = TIndexRepA.U;  // V_r
+  
+  typename TwoIndexRep< Nc, AntiSymmetric >::LatticeField Ur2Vr2A(grid);
+  Ur2Vr2A = zero;
+  for (int mu = 0; mu < Nd; mu++) {
+    typename TwoIndexRep< Nc, AntiSymmetric >::LatticeMatrix Urmu2A = peekLorentz(Ur2A,mu);
+    typename TwoIndexRep< Nc, AntiSymmetric >::LatticeMatrix Vrmu2A = peekLorentz(Vr2A,mu);
+    pokeLorentz(Ur2Vr2A,Urmu2A*Vrmu2A, mu);
+  }
+  
+  typename TwoIndexRep< Nc, AntiSymmetric >::LatticeField Diff_check2A = UVr2A - Ur2Vr2A;
+  std::cout << GridLogMessage << "Group structure SU("<<Nc<<") check difference (Two Index anti-Symmetric): " << norm2(Diff_check2A) << std::endl;
+
+  
+  // Check correspondence of algebra and group transformations
+  // Create a random vector
+  SU<Nc>::LatticeAlgebraVector h_Asym(grid);
+  typename TwoIndexRep< Nc, AntiSymmetric>::LatticeMatrix Ar_Asym(grid);
+  random(gridRNG,h_Asym);
+  h_Asym = real(h_Asym);
+  SU_TwoIndex< Nc, AntiSymmetric>::TwoIndexLieAlgebraMatrix(h_Asym,Ar_Asym);
+  
+  // Re-extract h_sym
+  SU<Nc>::LatticeAlgebraVector h_Asym2(grid);
+  SU_TwoIndex< Nc, AntiSymmetric>::projectOnAlgebra(h_Asym2, Ar_Asym);
+  SU<Nc>::LatticeAlgebraVector h_diff_Asym = h_Asym - h_Asym2;
+  std::cout << GridLogMessage << "Projections structure check vector difference (Two Index anti-Symmetric): " << norm2(h_diff_Asym) << std::endl;
+
+  
+  // Exponentiate
+  typename TwoIndexRep< Nc, AntiSymmetric>::LatticeMatrix U2iAS(grid);
+  U2iAS  = expMat(Ar_Asym, 1.0, 16);
+  
+  typename TwoIndexRep< Nc, AntiSymmetric>::LatticeMatrix uno2iAS(grid);
+  uno2iAS = 1.0;
+  // Check matrix U2iS, must be real orthogonal
+  typename TwoIndexRep< Nc, AntiSymmetric>::LatticeMatrix Ucheck2iAS = U2iAS - conjugate(U2iAS);
+  std::cout << GridLogMessage << "Reality check: " << norm2(Ucheck2iAS)
+      << std::endl;
+  
+  Ucheck2iAS = U2iAS * adj(U2iAS) - uno2iAS;
+  std::cout << GridLogMessage << "orthogonality check 1: " << norm2(Ucheck2iAS)
+      << std::endl;
+  Ucheck2iAS = adj(U2iAS) * U2iAS - uno2iAS;
+  std::cout << GridLogMessage << "orthogonality check 2: " << norm2(Ucheck2iAS)
+      << std::endl;
+  
+  
+  
+  // Construct the fundamental matrix in the group
+  SU<Nc>::LatticeMatrix Af_Asym(grid);
+  SU<Nc>::FundamentalLieAlgebraMatrix(h_Asym,Af_Asym);
+  SU<Nc>::LatticeMatrix Ufund2A(grid);
+  Ufund2A  = expMat(Af_Asym, 1.0, 16);
+  SU<Nc>::LatticeMatrix UnitCheck2A(grid);
+  UnitCheck2A = Ufund2A * adj(Ufund2A) - uno_f;
+  std::cout << GridLogMessage << "unitarity check 1: " << norm2(UnitCheck2A)
+      << std::endl;
+  UnitCheck2A = adj(Ufund2A) * Ufund2A - uno_f;
+  std::cout << GridLogMessage << "unitarity check 2: " << norm2(UnitCheck2A)
+      << std::endl;
+  
+
+  // Tranform to the 2Index Sym representation
+  U = zero; // fill this with only one direction
+  pokeLorentz(U,Ufund2A,0); // the representation transf acts on full gauge fields
+  
+  TIndexRepA.update_representation(U);
+  Ur2A = TIndexRepA.U;  // U_r  
+  typename TwoIndexRep< Nc, AntiSymmetric>::LatticeMatrix Ur02A = peekLorentz(Ur2A,0); // this should be the same as U2iS
+  
+  typename TwoIndexRep< Nc, AntiSymmetric>::LatticeMatrix Diff_check_mat2A = Ur02A - U2iAS;
+  std::cout << GridLogMessage << "Projections structure check group difference (Two Index anti-Symmetric): " << norm2(Diff_check_mat2A) << std::endl;
+  
+} else  {
+  std::cout << GridLogMessage << "Skipping Two Index anti-Symmetric tests "
+                                 "because representation is trivial (dim = 1)"
+            << std::endl;
+}
+
+
+
+
+
+
 
-  //  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
-  //  std::cout<<GridLogMessage<<"* Generators for SU(4)"<<std::endl;
-  //  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
-  //  SU4::printGenerators();
-  //  SU4::testGenerators();
 
-  //  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
-  //  std::cout<<GridLogMessage<<"* Generators for SU(5)"<<std::endl;
-  //  std::cout<<GridLogMessage<<"*********************************************"<<std::endl;
-  //  SU5::printGenerators();
-  //  SU5::testGenerators();
 
   
   Grid_finalize();
 }
-
-

tests/debug/Test_cayley_even_odd_vec.cc

@@ -44,6 +44,7 @@ struct scal {
   };
 
 typedef DomainWallFermion<DomainWallVec5dImplR>                      DomainWallVecFermionR;
+typedef ZMobiusFermion<ZDomainWallVec5dImplR>                        ZMobiusVecFermionR;
 typedef MobiusFermion<DomainWallVec5dImplR>                          MobiusVecFermionR;
 typedef MobiusZolotarevFermion<DomainWallVec5dImplR>                 MobiusZolotarevVecFermionR;
 typedef ScaledShamirFermion<DomainWallVec5dImplR>                    ScaledShamirVecFermionR;
@@ -117,6 +118,17 @@ int main (int argc, char ** argv)
   TestWhat<MobiusFermionR>(Dmob,FGrid,FrbGrid,UGrid,mass,M5,&RNG4,&RNG5);
   TestWhat<MobiusVecFermionR>(sDmob,sFGrid,sFrbGrid,sUGrid,mass,M5,&sRNG4,&sRNG5);
 
+
+  std::cout<<GridLogMessage<<"**************************************************************"<<std::endl;
+  std::cout<<GridLogMessage <<"Z-MobiusFermion test"<<std::endl;
+  std::cout<<GridLogMessage<<"**************************************************************"<<std::endl;
+  std::vector<ComplexD> gamma(Ls,std::complex<double>(1.0,0.0));
+  ZMobiusFermionR     zDmob(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,gamma,b,c);
+  ZMobiusVecFermionR szDmob(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,mass,M5,gamma,b,c);
+  TestMoo(zDmob,szDmob);
+  TestWhat<ZMobiusFermionR>(zDmob,FGrid,FrbGrid,UGrid,mass,M5,&RNG4,&RNG5);
+  TestWhat<ZMobiusVecFermionR>(szDmob,sFGrid,sFrbGrid,sUGrid,mass,M5,&sRNG4,&sRNG5);
+
   std::cout<<GridLogMessage<<"**************************************************************"<<std::endl;
   std::cout<<GridLogMessage <<"MobiusZolotarevFermion test"<<std::endl;
   std::cout<<GridLogMessage<<"**************************************************************"<<std::endl;

tests/forces/Test_contfrac_force.cc

@@ -96,7 +96,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 

tests/forces/Test_dwf_force.cc

@@ -96,7 +96,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 

tests/forces/Test_dwf_gpforce.cc

@@ -113,7 +113,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     Hmom -= real(sum(trace(mommu*mommu)));
 

tests/forces/Test_gp_rect_force.cc

@@ -82,7 +82,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 

tests/forces/Test_gpdwf_force.cc

@@ -100,7 +100,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 
@@ -169,7 +169,7 @@ int main (int argc, char ** argv)
   //
   //	Pmu = zero;
   //	for(int mu=0;mu<Nd;mu++){
-  //	  SU<Ncol>::GaussianLieAlgebraMatrix(pRNG, Pmu);
+  //	  SU<Ncol>::GaussianFundamentalLieAlgebraMatrix(pRNG, Pmu);
   //	  PokeIndex<LorentzIndex>(P, Pmu, mu);
   //	}
   //

tests/forces/Test_gpwilson_force.cc

@@ -100,7 +100,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 

tests/forces/Test_partfrac_force.cc

@@ -96,7 +96,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 

tests/forces/Test_rect_force.cc

@@ -81,7 +81,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 

tests/forces/Test_wilson_force.cc

@@ -58,8 +58,8 @@ int main (int argc, char ** argv)
 
   LatticeGaugeField U(&Grid);
 
-  SU3::HotConfiguration(pRNG,U);
-  //  SU3::ColdConfiguration(pRNG,U);
+  //SU2::HotConfiguration(pRNG,U);
+  SU3::ColdConfiguration(pRNG,U);
   
   ////////////////////////////////////
   // Unmodified matrix element
@@ -76,6 +76,8 @@ int main (int argc, char ** argv)
 
   Dw.MDeriv(tmp , Mphi,  phi,DaggerNo );  UdSdU=tmp;
   Dw.MDeriv(tmp , phi,  Mphi,DaggerYes ); UdSdU=(UdSdU+tmp);
+  // Take the trace
+  UdSdU = Ta(UdSdU);
   
   LatticeFermion Ftmp      (&Grid);
 
@@ -93,7 +95,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     Hmom -= real(sum(trace(mommu*mommu)));
 
@@ -140,12 +142,14 @@ int main (int argc, char ** argv)
   LatticeComplex dS(&Grid); dS = zero;
   LatticeComplex dSmom(&Grid); dSmom = zero;
   LatticeComplex dSmom2(&Grid); dSmom2 = zero;
+  
   for(int mu=0;mu<Nd;mu++){
     mommu   = PeekIndex<LorentzIndex>(UdSdU,mu);
     mommu=Ta(mommu)*2.0;
     PokeIndex<LorentzIndex>(UdSdU,mommu,mu);
   }
   
+
   for(int mu=0;mu<Nd;mu++){
     mommu   = PeekIndex<LorentzIndex>(mom,mu);
     std::cout << GridLogMessage<< " Mommu  " << norm2(mommu)<<std::endl;

tests/forces/Test_wilson_force_phiMdagMphi.cc

@@ -103,7 +103,7 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
     //    Dw.DoubleStore(Dw.Umu,Uprime); // update U _and_ Udag
     Dw.DhopDirDisp(phi,Ftmp,mu,mu+4,DaggerYes); 

tests/forces/Test_wilson_force_phiMphi.cc

@@ -86,7 +86,7 @@ int main (int argc, char ** argv)
   LatticeColourMatrix Umu_save(&Grid);
   LatticeColourMatrix dU (&Grid);
   LatticeColourMatrix mom(&Grid); 
-  SU3::GaussianLieAlgebraMatrix(pRNG, mom); // Traceless antihermitian momentum; gaussian in lie alg
+  SU3::GaussianFundamentalLieAlgebraMatrix(pRNG, mom); // Traceless antihermitian momentum; gaussian in lie alg
 
 
   // check mom is as i expect

tests/hmc/Make.inc

@@ -0,0 +1,63 @@
+tests: Test_hmc_EODWFRatio Test_hmc_EODWFRatio_Gparity Test_hmc_EOWilsonFermionGauge Test_hmc_EOWilsonRatio Test_hmc_GparityIwasakiGauge Test_hmc_GparityWilsonGauge Test_hmc_IwasakiGauge Test_hmc_RectGauge Test_hmc_WilsonAdjointFermionGauge Test_hmc_WilsonFermionGauge Test_hmc_WilsonGauge Test_hmc_WilsonMixedRepresentationsFermionGauge Test_hmc_WilsonRatio Test_hmc_WilsonTwoIndexSymmetricFermionGauge Test_multishift_sqrt Test_remez Test_rhmc_EOWilson1p1 Test_rhmc_EOWilsonRatio Test_rhmc_Wilson1p1 Test_rhmc_WilsonRatio
+EXTRA_PROGRAMS = Test_hmc_EODWFRatio Test_hmc_EODWFRatio_Gparity Test_hmc_EOWilsonFermionGauge Test_hmc_EOWilsonRatio Test_hmc_GparityIwasakiGauge Test_hmc_GparityWilsonGauge Test_hmc_IwasakiGauge Test_hmc_RectGauge Test_hmc_WilsonAdjointFermionGauge Test_hmc_WilsonFermionGauge Test_hmc_WilsonGauge Test_hmc_WilsonMixedRepresentationsFermionGauge Test_hmc_WilsonRatio Test_hmc_WilsonTwoIndexSymmetricFermionGauge Test_multishift_sqrt Test_remez Test_rhmc_EOWilson1p1 Test_rhmc_EOWilsonRatio Test_rhmc_Wilson1p1 Test_rhmc_WilsonRatio
+
+Test_hmc_EODWFRatio_SOURCES=Test_hmc_EODWFRatio.cc
+Test_hmc_EODWFRatio_LDADD=-lGrid
+
+Test_hmc_EODWFRatio_Gparity_SOURCES=Test_hmc_EODWFRatio_Gparity.cc
+Test_hmc_EODWFRatio_Gparity_LDADD=-lGrid
+
+Test_hmc_EOWilsonFermionGauge_SOURCES=Test_hmc_EOWilsonFermionGauge.cc
+Test_hmc_EOWilsonFermionGauge_LDADD=-lGrid
+
+Test_hmc_EOWilsonRatio_SOURCES=Test_hmc_EOWilsonRatio.cc
+Test_hmc_EOWilsonRatio_LDADD=-lGrid
+
+Test_hmc_GparityIwasakiGauge_SOURCES=Test_hmc_GparityIwasakiGauge.cc
+Test_hmc_GparityIwasakiGauge_LDADD=-lGrid
+
+Test_hmc_GparityWilsonGauge_SOURCES=Test_hmc_GparityWilsonGauge.cc
+Test_hmc_GparityWilsonGauge_LDADD=-lGrid
+
+Test_hmc_IwasakiGauge_SOURCES=Test_hmc_IwasakiGauge.cc
+Test_hmc_IwasakiGauge_LDADD=-lGrid
+
+Test_hmc_RectGauge_SOURCES=Test_hmc_RectGauge.cc
+Test_hmc_RectGauge_LDADD=-lGrid
+
+Test_hmc_WilsonAdjointFermionGauge_SOURCES=Test_hmc_WilsonAdjointFermionGauge.cc
+Test_hmc_WilsonAdjointFermionGauge_LDADD=-lGrid
+
+Test_hmc_WilsonFermionGauge_SOURCES=Test_hmc_WilsonFermionGauge.cc
+Test_hmc_WilsonFermionGauge_LDADD=-lGrid
+
+Test_hmc_WilsonGauge_SOURCES=Test_hmc_WilsonGauge.cc
+Test_hmc_WilsonGauge_LDADD=-lGrid
+
+Test_hmc_WilsonMixedRepresentationsFermionGauge_SOURCES=Test_hmc_WilsonMixedRepresentationsFermionGauge.cc
+Test_hmc_WilsonMixedRepresentationsFermionGauge_LDADD=-lGrid
+
+Test_hmc_WilsonRatio_SOURCES=Test_hmc_WilsonRatio.cc
+Test_hmc_WilsonRatio_LDADD=-lGrid
+
+Test_hmc_WilsonTwoIndexSymmetricFermionGauge_SOURCES=Test_hmc_WilsonTwoIndexSymmetricFermionGauge.cc
+Test_hmc_WilsonTwoIndexSymmetricFermionGauge_LDADD=-lGrid
+
+Test_multishift_sqrt_SOURCES=Test_multishift_sqrt.cc
+Test_multishift_sqrt_LDADD=-lGrid
+
+Test_remez_SOURCES=Test_remez.cc
+Test_remez_LDADD=-lGrid
+
+Test_rhmc_EOWilson1p1_SOURCES=Test_rhmc_EOWilson1p1.cc
+Test_rhmc_EOWilson1p1_LDADD=-lGrid
+
+Test_rhmc_EOWilsonRatio_SOURCES=Test_rhmc_EOWilsonRatio.cc
+Test_rhmc_EOWilsonRatio_LDADD=-lGrid
+
+Test_rhmc_Wilson1p1_SOURCES=Test_rhmc_Wilson1p1.cc
+Test_rhmc_Wilson1p1_LDADD=-lGrid
+
+Test_rhmc_WilsonRatio_SOURCES=Test_rhmc_WilsonRatio.cc
+Test_rhmc_WilsonRatio_LDADD=-lGrid
+

tests/hmc/Test_hmc_EODWFRatio.cc

@@ -76,6 +76,12 @@ public:
     TheAction.push_back(Level1);
 
     Run(argc,argv);
+
+    std::cout << GridLogMessage << "Numerator report, Pauli-Villars term         : " << std::endl;
+    NumOp.Report();
+    std::cout << GridLogMessage << "Denominator report, Dw(m) term (includes CG) : " << std::endl;
+    DenOp.Report();
+    
   };
 
 };

tests/hmc/Test_hmc_EOWilsonFermionGauge.cc

@@ -67,7 +67,7 @@ public:
     TwoFlavourEvenOddPseudoFermionAction<ImplPolicy> Nf2(FermOp,CG,CG);
 
     //Set smearing (true/false), default: false
-    Nf2.is_smeared=false;
+    Nf2.is_smeared=true;
     
     //Collect actions
     ActionLevel<LatticeGaugeField> Level1(1);

tests/hmc/Test_hmc_WilsonAdjointFermionGauge.cc

@@ -0,0 +1,107 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_WilsonAdjointFermionGauge.cc
+
+Copyright (C) 2015
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include "Grid/Grid.h"
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+namespace Grid {
+namespace QCD {
+
+// Here change the allowed (higher) representations
+typedef Representations< FundamentalRepresentation, AdjointRepresentation > TheRepresentations;
+
+
+class HmcRunner : public NerscHmcRunnerHirep< TheRepresentations > {
+ public:
+  void BuildTheAction(int argc, char **argv)
+
+  {
+    typedef WilsonAdjImplR ImplPolicy; // gauge field implemetation for the pseudofermions
+    typedef WilsonAdjFermionR FermionAction; // type of lattice fermions (Wilson, DW, ...)
+    typedef typename FermionAction::FermionField FermionField;
+
+    UGrid = SpaceTimeGrid::makeFourDimGrid(
+        GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()),
+        GridDefaultMpi());
+    UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+
+    FGrid = UGrid;
+    FrbGrid = UrbGrid;
+
+    // temporarily need a gauge field
+    //LatticeGaugeField U(UGrid);
+    AdjointRepresentation::LatticeField U(UGrid);
+
+    // Gauge action
+    WilsonGaugeActionR Waction(2.25);
+
+    Real mass = -0.95;
+    FermionAction FermOp(U, *FGrid, *FrbGrid, mass);
+
+    ConjugateGradient<FermionField> CG(1.0e-8, 10000, false);
+    ConjugateResidual<FermionField> CR(1.0e-8, 10000);
+
+    // Pass two solvers: one for the force computation and one for the action
+    TwoFlavourPseudoFermionAction<ImplPolicy> Nf2(FermOp, CG, CG);
+
+    // Set smearing (true/false), default: false
+    Nf2.is_smeared = false;
+
+    // Collect actions
+    ActionLevel<LatticeGaugeField, TheRepresentations > Level1(1);
+    Level1.push_back(&Nf2);
+
+    ActionLevel<LatticeGaugeField, TheRepresentations > Level2(4);
+    Level2.push_back(&Waction);
+
+    TheAction.push_back(Level1);
+    TheAction.push_back(Level2);
+
+    Run(argc, argv);
+  };
+};
+}
+}
+
+int main(int argc, char **argv) {
+  Grid_init(&argc, &argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is setup to use " << threads
+            << " threads" << std::endl;
+
+  HmcRunner TheHMC;
+
+  TheHMC.BuildTheAction(argc, argv);
+}

tests/hmc/Test_hmc_WilsonFermionGauge.cc

@@ -37,10 +37,8 @@ using namespace Grid::QCD;
 namespace Grid {
 namespace QCD {
 
-
 class HmcRunner : public NerscHmcRunner {
  public:
-
   void BuildTheAction(int argc, char **argv)
 
   {
@@ -48,7 +46,9 @@ public:
     typedef WilsonFermionR FermionAction;
     typedef typename FermionAction::FermionField FermionField;
 
-    UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    UGrid = SpaceTimeGrid::makeFourDimGrid(
+        GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()),
+        GridDefaultMpi());
     UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
 
     FGrid = UGrid;
@@ -70,7 +70,6 @@ public:
     // Set smearing (true/false), default: false
     Nf2.is_smeared = true;
 
-
     // Collect actions
     ActionLevel<LatticeGaugeField> Level1(1);
     Level1.push_back(&Nf2);
@@ -83,22 +82,18 @@ public:
 
     Run(argc, argv);
   };
-
 };
+}
+}
 
-}}
-
-int main (int argc, char ** argv)
-{
+int main(int argc, char **argv) {
   Grid_init(&argc, &argv);
 
   int threads = GridThread::GetThreads();
-  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+  std::cout << GridLogMessage << "Grid is setup to use " << threads
+            << " threads" << std::endl;
 
   HmcRunner TheHMC;
 
   TheHMC.BuildTheAction(argc, argv);
-
 }
-
-

tests/hmc/Test_hmc_WilsonMixedRepresentationsFermionGauge.cc

@@ -0,0 +1,113 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_WilsonAdjointFermionGauge.cc
+
+Copyright (C) 2015
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include "Grid/Grid.h"
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+namespace Grid {
+namespace QCD {
+
+// Here change the allowed (higher) representations
+typedef Representations< FundamentalRepresentation, AdjointRepresentation , TwoIndexSymmetricRepresentation> TheRepresentations;
+
+class HmcRunner : public NerscHmcRunnerHirep< TheRepresentations > {
+ public:
+  void BuildTheAction(int argc, char **argv)
+
+  {
+    typedef WilsonAdjImplR AdjImplPolicy; // gauge field implemetation for the pseudofermions
+    typedef WilsonAdjFermionR AdjFermionAction; // type of lattice fermions (Wilson, DW, ...)
+    typedef WilsonTwoIndexSymmetricImplR SymmImplPolicy; 
+    typedef WilsonTwoIndexSymmetricFermionR SymmFermionAction; 
+
+ 
+    typedef typename AdjFermionAction::FermionField AdjFermionField;
+    typedef typename SymmFermionAction::FermionField SymmFermionField;
+
+    UGrid = SpaceTimeGrid::makeFourDimGrid(
+        GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()),
+        GridDefaultMpi());
+    UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+
+    FGrid = UGrid;
+    FrbGrid = UrbGrid;
+
+    // temporarily need a gauge field
+    //LatticeGaugeField U(UGrid);
+    AdjointRepresentation::LatticeField UA(UGrid);
+    TwoIndexSymmetricRepresentation::LatticeField US(UGrid);
+
+    // Gauge action
+    WilsonGaugeActionR Waction(5.6);
+
+    Real adjoint_mass = -0.1;
+    Real symm_mass = -0.5;
+    AdjFermionAction AdjFermOp(UA, *FGrid, *FrbGrid, adjoint_mass);
+    SymmFermionAction SymmFermOp(US, *FGrid, *FrbGrid, symm_mass);
+
+    ConjugateGradient<AdjFermionField> CG_adj(1.0e-8, 10000, false);
+    ConjugateGradient<SymmFermionField> CG_symm(1.0e-8, 10000, false);
+
+    // Pass two solvers: one for the force computation and one for the action
+    TwoFlavourPseudoFermionAction<AdjImplPolicy> Nf2_Adj(AdjFermOp, CG_adj, CG_adj);
+    TwoFlavourPseudoFermionAction<SymmImplPolicy> Nf2_Symm(SymmFermOp, CG_symm, CG_symm);
+
+    // Collect actions
+    ActionLevel<LatticeGaugeField, TheRepresentations > Level1(1);
+    Level1.push_back(&Nf2_Adj);
+    Level1.push_back(&Nf2_Symm);
+
+    ActionLevel<LatticeGaugeField, TheRepresentations > Level2(4);
+    Level2.push_back(&Waction);
+
+    TheAction.push_back(Level1);
+    TheAction.push_back(Level2);
+
+    Run(argc, argv);
+  };
+};
+}
+}
+
+int main(int argc, char **argv) {
+  Grid_init(&argc, &argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is setup to use " << threads
+            << " threads" << std::endl;
+
+  HmcRunner TheHMC;
+
+  TheHMC.BuildTheAction(argc, argv);
+}

tests/hmc/Test_hmc_WilsonTwoIndexSymmetricFermionGauge.cc

@@ -0,0 +1,103 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_WilsonAdjointFermionGauge.cc
+
+Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include "Grid/Grid.h"
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+namespace Grid {
+namespace QCD {
+
+// Here change the allowed (higher) representations
+typedef Representations< FundamentalRepresentation, TwoIndexSymmetricRepresentation > TheRepresentations;
+
+
+class HmcRunner : public NerscHmcRunnerHirep< TheRepresentations > {
+ public:
+  void BuildTheAction(int argc, char **argv)
+
+  {
+    typedef WilsonTwoIndexSymmetricImplR ImplPolicy; // gauge field implemetation for the pseudofermions
+    typedef WilsonTwoIndexSymmetricFermionR FermionAction; // type of lattice fermions (Wilson, DW, ...)
+    typedef typename FermionAction::FermionField FermionField;
+
+    UGrid = SpaceTimeGrid::makeFourDimGrid(
+        GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()),
+        GridDefaultMpi());
+    UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+
+    FGrid = UGrid;
+    FrbGrid = UrbGrid;
+
+    // temporarily need a gauge field
+    TwoIndexSymmetricRepresentation::LatticeField U(UGrid);
+
+    // Gauge action
+    WilsonGaugeActionR Waction(2.0);
+
+    Real mass = -0.0;
+    FermionAction FermOp(U, *FGrid, *FrbGrid, mass);
+
+    ConjugateGradient<FermionField> CG(1.0e-8, 10000, false);
+
+    // Pass two solvers: one for the force computation and one for the action
+    TwoFlavourPseudoFermionAction<ImplPolicy> Nf2(FermOp, CG, CG);
+
+    // Set smearing (true/false), default: false
+    Nf2.is_smeared = false;
+
+    // Collect actions
+    ActionLevel<LatticeGaugeField, TheRepresentations > Level1(1);
+    Level1.push_back(&Nf2);
+
+    ActionLevel<LatticeGaugeField, TheRepresentations > Level2(4);
+    Level2.push_back(&Waction);
+
+    TheAction.push_back(Level1);
+    TheAction.push_back(Level2);
+
+    Run(argc, argv);
+  };
+};
+}
+}
+
+int main(int argc, char **argv) {
+  Grid_init(&argc, &argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is setup to use " << threads
+            << " threads" << std::endl;
+
+  HmcRunner TheHMC;
+
+  TheHMC.BuildTheAction(argc, argv);
+}

tests/solver/Test_dwf_cg_prec.cc

@@ -22,7 +22,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #include <Grid/Grid.h>
@@ -36,41 +37,47 @@ struct scal {
   d internal;
 };
 
-  Gamma::GammaMatrix Gmu [] = {
-    Gamma::GammaX,
-    Gamma::GammaY,
-    Gamma::GammaZ,
-    Gamma::GammaT
-  };
+Gamma::GammaMatrix Gmu[] = {Gamma::GammaX, Gamma::GammaY, Gamma::GammaZ,
+                            Gamma::GammaT};
 
-int main (int argc, char ** argv)
-{
+int main(int argc, char** argv) {
   Grid_init(&argc, &argv);
 
-  const int Ls=8;
+  const int Ls = 16;
 
-  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
-  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian* UGrid = SpaceTimeGrid::makeFourDimGrid(
+      GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()),
+      GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid =
+      SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian* FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
-  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+  GridRedBlackCartesian* FrbGrid =
+      SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
 
   std::vector<int> seeds4({1, 2, 3, 4});
   std::vector<int> seeds5({5, 6, 7, 8});
-  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
-  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
 
-  LatticeFermion    src(FGrid); random(RNG5,src);
-  LatticeFermion result(FGrid); result=zero;
+  LatticeFermion src(FGrid);
+  random(RNG5, src);
+  LatticeFermion result(FGrid);
+  result = zero;
   LatticeGaugeField Umu(UGrid);
 
   SU3::HotConfiguration(RNG4, Umu);
 
+  std::cout << GridLogMessage << "Lattice dimensions: " << GridDefaultLatt()
+            << "   Ls: " << Ls << std::endl;
+
   std::vector<LatticeColourMatrix> U(4, UGrid);
   for (int mu = 0; mu < Nd; mu++) {
     U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
   }
 
-  RealD mass=0.1;
+  RealD mass = 0.01;
   RealD M5 = 1.8;
   DomainWallFermionR Ddwf(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mass, M5);
 
@@ -79,9 +86,20 @@ int main (int argc, char ** argv)
   pickCheckerboard(Odd, src_o, src);
   result_o = zero;
 
+  GridStopWatch CGTimer;
+
   SchurDiagMooeeOperator<DomainWallFermionR, LatticeFermion> HermOpEO(Ddwf);
-  ConjugateGradient<LatticeFermion> CG(1.0e-8,10000);
+  ConjugateGradient<LatticeFermion> CG(1.0e-8, 10000, 0);// switch off the assert
+
+  CGTimer.Start();
   CG(HermOpEO, src_o, result_o);
+  CGTimer.Stop();
+
+  std::cout << GridLogMessage << "Total CG time : " << CGTimer.Elapsed()
+            << std::endl;
+
+  std::cout << GridLogMessage << "######## Dhop calls summary" << std::endl;
+  Ddwf.Report();
 
   Grid_finalize();
 }

tests/solver/Test_wilson_cg_prec.cc

@@ -84,5 +84,6 @@ int main (int argc, char ** argv)
   ConjugateGradient<LatticeFermion> CG(1.0e-8,10000);
   CG(HermOpEO,src_o,result_o);
   
+
   Grid_finalize();
 }