Reorganise to abstract kernels that know about the lattice layout.

Move these back into grid.

.gitignore

@@ -93,6 +93,7 @@ build*/*
 *.xcodeproj/*
 build.sh
 .vscode
+*.code-workspace
 
 # Eigen source #
 ################
@@ -122,4 +123,10 @@ make-bin-BUCK.sh
 #####################
 lib/qcd/spin/gamma-gen/*.h
 lib/qcd/spin/gamma-gen/*.cc
+lib/version.h
 
+# vs code editor files #
+########################
+.vscode/
+.vscode/settings.json
+settings.json

.travis.yml

@@ -9,68 +9,6 @@ matrix:
     - os:        osx
       osx_image: xcode8.3
       compiler: clang
-    - compiler: gcc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.9
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: VERSION=-4.9
-    - compiler: gcc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-5
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: VERSION=-5
-    - compiler: clang
-      dist: trusty
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.8
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: CLANG_LINK=http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-    - compiler: clang
-      dist: trusty
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.8
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: CLANG_LINK=http://llvm.org/releases/3.7.0/clang+llvm-3.7.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
       
 before_install:
     - export GRIDDIR=`pwd`
@@ -81,6 +19,8 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc; fi
     
 install:
+    - export CWD=`pwd`
+    - echo $CWD
     - export CC=$CC$VERSION
     - export CXX=$CXX$VERSION
     - echo $PATH
@@ -98,17 +38,23 @@ script:
     - ./bootstrap.sh
     - mkdir build
     - cd build
-    - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=none
+    - mkdir lime
+    - cd lime
+    - mkdir build
+    - cd build
+    - wget http://usqcd-software.github.io/downloads/c-lime/lime-1.3.2.tar.gz
+    - tar xf lime-1.3.2.tar.gz
+    - cd lime-1.3.2
+    - ./configure --prefix=$CWD/build/lime/install
+    - make -j4
+    - make install
+    - cd $CWD/build
+    - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install
     - make -j4 
     - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - echo make clean
-    - ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none
+    - ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install
     - make -j4
     - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - make check
-    - echo make clean
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto ; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then make -j4; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
-
 

Makefile.am

@@ -5,6 +5,10 @@ include $(top_srcdir)/doxygen.inc
 
 bin_SCRIPTS=grid-config
 
+BUILT_SOURCES = version.h
+
+version.h:
+	echo "`git log -n 1 --format=format:"#define GITHASH \\"%H:%d\\"%n" HEAD`" > $(srcdir)/lib/version.h
 
 .PHONY: bench check tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
 

README.md

@@ -1,27 +1,44 @@
-# Grid
-<table>
-<tr>
-    <td>Last stable release</td>
-    <td><a href="https://travis-ci.org/paboyle/Grid">
-    <img src="https://travis-ci.org/paboyle/Grid.svg?branch=master"></a>
-    </td>
-</tr>
-<tr>
-    <td>Development branch</td>
-    <td><a href="https://travis-ci.org/paboyle/Grid">
-    <img src="https://travis-ci.org/paboyle/Grid.svg?branch=develop"></a>
-    </td>
-</tr>
-</table>
+# Grid [![Teamcity status](http://ci.cliath.ph.ed.ac.uk/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:Grid)),branch:name:develop)/statusIcon.svg)](http://ci.cliath.ph.ed.ac.uk/project.html?projectId=Grid&tab=projectOverview) [![Travis status](https://travis-ci.org/paboyle/Grid.svg?branch=develop)](https://travis-ci.org/paboyle/Grid)
 
 **Data parallel C++ mathematical object library.**
 
 License: GPL v2.
 
-Last update Nov 2016.
+Last update June 2017.
 
 _Please do not send pull requests to the `master` branch which is reserved for releases._
 
+
+
+### Description
+This library provides data parallel C++ container classes with internal memory layout
+that is transformed to map efficiently to SIMD architectures. CSHIFT facilities
+are provided, similar to HPF and cmfortran, and user control is given over the mapping of
+array indices to both MPI tasks and SIMD processing elements.
+
+* Identically shaped arrays then be processed with perfect data parallelisation.
+* Such identically shaped arrays are called conformable arrays.
+
+The transformation is based on the observation that Cartesian array processing involves
+identical processing to be performed on different regions of the Cartesian array.
+
+The library will both geometrically decompose into MPI tasks and across SIMD lanes.
+Local vector loops are parallelised with OpenMP pragmas.
+
+Data parallel array operations can then be specified with a SINGLE data parallel paradigm, but
+optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a significant simplification
+for most programmers.
+
+The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture.
+Presently SSE4, ARM NEON (128 bits) AVX, AVX2, QPX (256 bits), IMCI and AVX512 (512 bits) targets are supported.
+
+These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. 
+The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`.
+
+MPI, OpenMP, and SIMD parallelism are present in the library.
+Please see [this paper](https://arxiv.org/abs/1512.03487) for more detail.
+
+
 ### Compilers
 
 Intel ICPC v16.0.3 and later
@@ -56,35 +73,25 @@ When you file an issue, please go though the following checklist:
 6. Attach the output of `make V=1`.
 7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example.
 
+### Required libraries
+Grid requires:
 
+[GMP](https://gmplib.org/), 
 
-### Description
-This library provides data parallel C++ container classes with internal memory layout
-that is transformed to map efficiently to SIMD architectures. CSHIFT facilities
-are provided, similar to HPF and cmfortran, and user control is given over the mapping of
-array indices to both MPI tasks and SIMD processing elements.
+[MPFR](http://www.mpfr.org/) 
 
-* Identically shaped arrays then be processed with perfect data parallelisation.
-* Such identically shaped arrays are called conformable arrays.
+Bootstrapping grid downloads and uses for internal dense matrix (non-QCD operations) the Eigen library.
 
-The transformation is based on the observation that Cartesian array processing involves
-identical processing to be performed on different regions of the Cartesian array.
+Grid optionally uses:
 
-The library will both geometrically decompose into MPI tasks and across SIMD lanes.
-Local vector loops are parallelised with OpenMP pragmas.
+[HDF5](https://support.hdfgroup.org/HDF5/)  
 
-Data parallel array operations can then be specified with a SINGLE data parallel paradigm, but
-optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a significant simplification
-for most programmers.
+[LIME](http://usqcd-software.github.io/c-lime/) for ILDG and SciDAC file format support. 
 
-The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture.
-Presently SSE4 (128 bit) AVX, AVX2, QPX (256 bit), IMCI, and AVX512 (512 bit) targets are supported (ARM NEON on the way).
+[FFTW](http://www.fftw.org) either generic version or via the Intel MKL library.
 
-These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. These may be useful in themselves for other programmers.
-The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`.
+LAPACK either generic version or Intel MKL library.
 
-MPI, OpenMP, and SIMD parallelism are present in the library.
-Please see https://arxiv.org/abs/1512.03487 for more detail.
 
 ### Quick start
 First, start by cloning the repository:
@@ -155,7 +162,6 @@ The following options can be use with the `--enable-comms=` option to target dif
 | `none`         | no communications                                             |
 | `mpi[-auto]`   | MPI communications                                            |
 | `mpi3[-auto]`  | MPI communications using MPI 3 shared memory                  |
-| `mpi3l[-auto]` | MPI communications using MPI 3 shared memory and leader model |
 | `shmem `       | Cray SHMEM communications                                     |
 
 For the MPI interfaces the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names). The `-auto` suffix is not supported by the Cray environment wrapper scripts. Use the standard versions instead.  
@@ -173,17 +179,19 @@ The following options can be use with the `--enable-simd=` option to target diff
 | `AVXFMA4`   | AVX (256 bit) + FMA4                   |
 | `AVX2`      | AVX 2 (256 bit)                        |
 | `AVX512`    | AVX 512 bit                            |
-| `QPX`       | QPX (256 bit)                          |
+| `NEONv8`    | [ARM NEON](http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch07s03.html) (128 bit)                     |
+| `QPX`       | IBM QPX (256 bit)                      |
 
 Alternatively, some CPU codenames can be directly used:
 
 | `<code>`    | Description                            |
 | ----------- | -------------------------------------- |
 | `KNL`       | [Intel Xeon Phi codename Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |
+| `SKL`       | [Intel Skylake with AVX512 extensions](https://ark.intel.com/products/codename/37572/Skylake#@server) |
 | `BGQ`       | Blue Gene/Q                            |
 
 #### Notes:
-- We currently support AVX512 only for the Intel compiler. Support for GCC and clang will appear in future versions of Grid when the AVX512 support within GCC and clang will be more advanced.
+- We currently support AVX512 for the Intel compiler and GCC (KNL and SKL target). Support for clang will appear in future versions of Grid when the AVX512 support in the compiler will be more advanced.
 - For BG/Q only [bgclang](http://trac.alcf.anl.gov/projects/llvm-bgq) is supported. We do not presently plan to support more compilers for this platform.
 - BG/Q performances are currently rather poor. This is being investigated for future versions.
 - The vector size for the `GEN` target can be specified with the `configure` script option `--enable-gen-simd-width`.
@@ -195,21 +203,205 @@ The following configuration is recommended for the Intel Knights Landing platfor
 ``` bash
 ../configure --enable-precision=double\
              --enable-simd=KNL        \
-             --enable-comms=mpi-auto \
-             --with-gmp=<path>        \
-             --with-mpfr=<path>       \
+             --enable-comms=mpi-auto  \
              --enable-mkl             \
              CXX=icpc MPICXX=mpiicpc
 ```
+The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library.
 
-where `<path>` is the UNIX prefix where GMP and MPFR are installed. If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
+If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
 
 ``` bash
 ../configure --enable-precision=double\
              --enable-simd=KNL        \
              --enable-comms=mpi       \
-             --with-gmp=<path>        \
-             --with-mpfr=<path>       \
              --enable-mkl             \
              CXX=CC CC=cc
-```
+```
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+Knight's Landing with Intel Omnipath adapters with two adapters per node 
+presently performs better with use of more than one rank per node, using shared memory 
+for interior communication. This is the mpi3 communications implementation. 
+We recommend four ranks per node for best performance, but optimum is local volume dependent.
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=KNL        \
+             --enable-comms=mpi3-auto \
+             --enable-mkl             \
+             CC=icpc MPICXX=mpiicpc 
+```
+
+### Build setup for Intel Haswell Xeon platform
+
+The following configuration is recommended for the Intel Haswell platform:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX2       \
+             --enable-comms=mpi3-auto \
+             --enable-mkl             \
+             CXX=icpc MPICXX=mpiicpc
+```
+The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library.
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX2       \
+             --enable-comms=mpi3      \
+             --enable-mkl             \
+             CXX=CC CC=cc
+```
+Since Dual socket nodes are commonplace, we recommend MPI-3 as the default with the use of 
+one rank per socket. If using the Intel MPI library, threads should be pinned to NUMA domains using
+```
+        export I_MPI_PIN=1
+```
+This is the default.
+
+### Build setup for Intel Skylake Xeon platform
+
+The following configuration is recommended for the Intel Skylake platform:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX512     \
+             --enable-comms=mpi3      \
+             --enable-mkl             \
+             CXX=mpiicpc
+```
+The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library.
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX512     \
+             --enable-comms=mpi3      \
+             --enable-mkl             \
+             CXX=CC CC=cc
+```
+Since Dual socket nodes are commonplace, we recommend MPI-3 as the default with the use of 
+one rank per socket. If using the Intel MPI library, threads should be pinned to NUMA domains using
+``` 
+        export I_MPI_PIN=1
+```
+This is the default. 
+
+#### Expected Skylake Gold 6148 dual socket (single prec, single node 20+20 cores) performance using NUMA MPI mapping): 
+
+mpirun -n 2 benchmarks/Benchmark_dwf --grid 16.16.16.16 --mpi 2.1.1.1 --cacheblocking 2.2.2.2 --dslash-asm --shm 1024 --threads 18 
+
+TBA
+
+
+### Build setup for AMD EPYC / RYZEN
+
+The AMD EPYC is a multichip module comprising 32 cores spread over four distinct chips each with 8 cores.
+So, even with a single socket node there is a quad-chip module. Dual socket nodes with 64 cores total
+are common. Each chip within the module exposes a separate NUMA domain.
+There are four NUMA domains per socket and we recommend one MPI rank per NUMA domain.
+MPI-3 is recommended with the use of four ranks per socket,
+and 8 threads per rank. 
+
+The following configuration is recommended for the AMD EPYC platform.
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX2       \
+             --enable-comms=mpi3 \
+             CXX=mpicxx 
+```
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+Using MPICH and g++ v4.9.2, best performance can be obtained using explicit GOMP_CPU_AFFINITY flags for each MPI rank.
+This can be done by invoking MPI on a wrapper script omp_bind.sh to handle this. 
+
+It is recommended to run 8 MPI ranks on a single dual socket AMD EPYC, with 8 threads per rank using MPI3 and
+shared memory to communicate within this node:
+
+mpirun -np 8 ./omp_bind.sh ./Benchmark_dwf --mpi 2.2.2.1 --dslash-unroll --threads 8 --grid 16.16.16.16 --cacheblocking 4.4.4.4 
+
+Where omp_bind.sh does the following:
+```
+#!/bin/bash
+
+numanode=` expr $PMI_RANK % 8 `
+basecore=`expr $numanode \* 16`
+core0=`expr $basecore + 0 `
+core1=`expr $basecore + 2 `
+core2=`expr $basecore + 4 `
+core3=`expr $basecore + 6 `
+core4=`expr $basecore + 8 `
+core5=`expr $basecore + 10 `
+core6=`expr $basecore + 12 `
+core7=`expr $basecore + 14 `
+
+export GOMP_CPU_AFFINITY="$core0 $core1 $core2 $core3 $core4 $core5 $core6 $core7"
+echo GOMP_CUP_AFFINITY $GOMP_CPU_AFFINITY
+
+$@
+```
+
+Performance:
+
+#### Expected AMD EPYC 7601 dual socket (single prec, single node 32+32 cores) performance using NUMA MPI mapping): 
+
+mpirun  -np 8 ./omp_bind.sh ./Benchmark_dwf --threads 8 --mpi 2.2.2.1 --dslash-unroll --grid 16.16.16.16 --cacheblocking 4.4.4.4
+
+TBA
+
+### Build setup for BlueGene/Q
+
+To be written...
+
+### Build setup for ARM Neon
+
+To be written...
+
+### Build setup for laptops, other compilers, non-cluster builds
+
+Many versions of g++ and clang++ work with Grid, and involve merely replacing CXX (and MPICXX),
+and omit the enable-mkl flag. 
+
+Single node builds are enabled with 
+```
+            --enable-comms=none
+```
+
+FFTW support that is not in the default search path may then enabled with
+```
+    --with-fftw=<installpath>
+```
+
+BLAS will not be compiled in by default, and Lanczos will default to Eigen diagonalisation.
+

TODO

@@ -1,23 +1,48 @@
 TODO:
 ---------------
 
-Peter's work list:
-2)- Precision conversion and sort out localConvert      <-- 
-3)- Remove DenseVector, DenseMatrix; Use Eigen instead. <-- started 
-4)- Binary I/O speed up & x-strips
--- Profile CG, BlockCG, etc... Flop count/rate -- PARTIAL, time but no flop/s yet
--- Physical propagator interface
--- Conserved currents
--- GaugeFix into central location
--- Multigrid Wilson and DWF, compare to other Multigrid implementations
--- HDCR resume
+Code item work list
 
+a) namespaces & indentation
+ GRID_BEGIN_NAMESPACE();
+ GRID_END_NAMESPACE();
+-- delete QCD namespace
+
+b) GPU branch
+- start branch
+- Increase Macro use in core library support; prepare for change
+- Audit volume of "device" code
+- Virtual function audit
+- Start port once Nvidia box is up
+- Cut down volume of code for first port? How?
+
+Physics item work list:
+
+1)- BG/Q port and check ; Andrew says ok.
+2)- Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
+3)- Physical propagator interface
+4)- Multigrid Wilson and DWF, compare to other Multigrid implementations
+5)- HDCR resume
+
+----------------------------
 Recent DONE 
+-- RNG I/O in ILDG/SciDAC (minor) 
+-- Precision conversion and sort out localConvert      <-- partial/easy
+-- Conserved currents (Andrew)
+-- Split grid
+-- Christoph's local basis expansion Lanczos
+-- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O ; <-- DONE ; bmark cori
+-- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE
+-- GaugeFix into central location                      <-- DONE
+-- Scidac and Ildg metadata handling                   <-- DONE
+-- Binary I/O MPI2 IO                                  <-- DONE
+-- Binary I/O speed up & x-strips                      <-- DONE
 -- Cut down the exterior overhead                      <-- DONE
 -- Interior legs from SHM comms                        <-- DONE
 -- Half-precision comms                                <-- DONE
--- Merge high precision reduction into develop        
--- multiRHS DWF; benchmark on Cori/BNL for comms elimination
+-- Merge high precision reduction into develop         <-- DONE
+-- BlockCG, BCGrQ                                      <-- DONE
+-- multiRHS DWF; benchmark on Cori/BNL for comms elimination <-- DONE
    -- slice* linalg routines for multiRHS, BlockCG    
 
 -----

VERSION

@@ -1,4 +1,4 @@
-Version : 0.7.0
+Version : 0.8.0
 
 - Clang 3.5 and above, ICPC v16 and above, GCC 6.3 and above recommended
 - MPI and MPI3 comms optimisations for KNL and OPA finished

benchmarks/Benchmark_IO.cc

@@ -0,0 +1,108 @@
+#include <Grid/Grid.h>
+#ifdef HAVE_LIME
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+#define MSG cout << GridLogMessage
+#define SEP \
+"============================================================================="
+#ifndef BENCH_IO_LMAX
+#define BENCH_IO_LMAX 40
+#endif
+
+typedef function<void(const string, LatticeFermion &)> WriterFn;
+typedef function<void(LatticeFermion &, const string)> ReaderFn;
+
+string filestem(const int l)
+{
+  return "iobench_l" + to_string(l);
+}
+
+void limeWrite(const string filestem, LatticeFermion &vec)
+{
+  emptyUserRecord record;
+  ScidacWriter    binWriter(vec._grid->IsBoss());
+
+  binWriter.open(filestem + ".bin");
+  binWriter.writeScidacFieldRecord(vec, record);
+  binWriter.close();
+}
+
+void limeRead(LatticeFermion &vec, const string filestem)
+{
+  emptyUserRecord record;
+  ScidacReader    binReader;
+
+  binReader.open(filestem + ".bin");
+  binReader.readScidacFieldRecord(vec, record);
+  binReader.close();
+}
+
+void writeBenchmark(const int l, const WriterFn &write)
+{
+  auto                      mpi  = GridDefaultMpi();
+  auto                      simd = GridDefaultSimd(Nd, vComplex::Nsimd());
+  vector<int>               latt = {l*mpi[0], l*mpi[1], l*mpi[2], l*mpi[3]};
+  unique_ptr<GridCartesian> gPt(SpaceTimeGrid::makeFourDimGrid(latt, simd, mpi));
+  GridCartesian             *g = gPt.get();
+  GridParallelRNG           rng(g);
+  LatticeFermion            vec(g);
+  emptyUserRecord           record;
+  ScidacWriter              binWriter(g->IsBoss());
+
+  cout << "-- Local volume " << l << "^4" << endl;
+  random(rng, vec);
+  write(filestem(l), vec);
+}
+
+void readBenchmark(const int l, const ReaderFn &read)
+{
+  auto                      mpi  = GridDefaultMpi();
+  auto                      simd = GridDefaultSimd(Nd, vComplex::Nsimd());
+  vector<int>               latt = {l*mpi[0], l*mpi[1], l*mpi[2], l*mpi[3]};
+  unique_ptr<GridCartesian> gPt(SpaceTimeGrid::makeFourDimGrid(latt, simd, mpi));
+  GridCartesian             *g = gPt.get();
+  LatticeFermion            vec(g);
+  emptyUserRecord           record;
+  ScidacReader              binReader;
+
+  cout << "-- Local volume " << l << "^4" << endl;
+  read(vec, filestem(l));
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  auto simd = GridDefaultSimd(Nd,vComplex::Nsimd());
+  auto mpi  = GridDefaultMpi();
+
+  int64_t threads = GridThread::GetThreads();
+  MSG << "Grid is setup to use " << threads << " threads" << endl;
+  MSG << SEP << endl;
+  MSG << "Benchmark Lime write" << endl;
+  MSG << SEP << endl;
+  for (int l = 4; l <= BENCH_IO_LMAX; l += 2)
+  {
+    writeBenchmark(l, limeWrite);
+  }
+
+  MSG << "Benchmark Lime read" << endl;
+  MSG << SEP << endl;
+  for (int l = 4; l <= BENCH_IO_LMAX; l += 2)
+  {
+    readBenchmark(l, limeRead);
+  }
+
+  Grid_finalize();
+
+  return EXIT_SUCCESS;
+}
+#else
+int main (int argc, char ** argv)
+{
+  return EXIT_SUCCESS;
+}
+#endif

benchmarks/Benchmark_ITT.cc

@@ -0,0 +1,807 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./benchmarks/Benchmark_memory_bandwidth.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
+typedef WilsonFermion5D<DomainWallVec5dImplF> WilsonFermion5DF;
+typedef WilsonFermion5D<DomainWallVec5dImplD> WilsonFermion5DD;
+
+
+std::vector<int> L_list;
+std::vector<int> Ls_list;
+std::vector<double> mflop_list;
+
+double mflop_ref;
+double mflop_ref_err;
+
+int NN_global;
+
+struct time_statistics{
+  double mean;
+  double err;
+  double min;
+  double max;
+
+  void statistics(std::vector<double> v){
+      double sum = std::accumulate(v.begin(), v.end(), 0.0);
+      mean = sum / v.size();
+
+      std::vector<double> diff(v.size());
+      std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; });
+      double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+      err = std::sqrt(sq_sum / (v.size()*(v.size() - 1)));
+
+      auto result = std::minmax_element(v.begin(), v.end());
+      min = *result.first;
+      max = *result.second;
+}
+};
+
+void comms_header(){
+  std::cout <<GridLogMessage << " L  "<<"\t"<<" Ls  "<<"\t"
+            <<std::setw(11)<<"bytes"<<"MB/s uni (err/min/max)"<<"\t\t"<<"MB/s bidi (err/min/max)"<<std::endl;
+};
+
+Gamma::Algebra Gmu [] = {
+  Gamma::Algebra::GammaX,
+  Gamma::Algebra::GammaY,
+  Gamma::Algebra::GammaZ,
+  Gamma::Algebra::GammaT
+};
+struct controls {
+  int Opt;
+  int CommsOverlap;
+  Grid::CartesianCommunicator::CommunicatorPolicy_t CommsAsynch;
+  //  int HugePages;
+};
+
+class Benchmark {
+public:
+  static void Decomposition (void ) {
+
+    int threads = GridThread::GetThreads();
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Grid is setup to use "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage<<"Grid Default Decomposition patterns\n";
+    std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
+    std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
+    std::cout<<GridLogMessage<<"\tvReal          : "<<sizeof(vReal )*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vReal::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealD         : "<<sizeof(vRealD)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplex       : "<<sizeof(vComplex )*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplex::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexF      : "<<sizeof(vComplexF)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexD      : "<<sizeof(vComplexD)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  }
+
+  static void Comms(void)
+  {
+    int Nloop=200;
+    int nmu=0;
+    int maxlat=32;
+
+    std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplexD::Nsimd());
+    std::vector<int> mpi_layout  = GridDefaultMpi();
+
+    for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
+
+    std::vector<double> t_time(Nloop);
+    time_statistics timestat;
+
+    std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Benchmarking threaded STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+    std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+    comms_header();
+
+    for(int lat=4;lat<=maxlat;lat+=4){
+      for(int Ls=8;Ls<=8;Ls*=2){
+
+	std::vector<int> latt_size  ({lat*mpi_layout[0],
+	      lat*mpi_layout[1],
+	      lat*mpi_layout[2],
+	      lat*mpi_layout[3]});
+
+	GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+	RealD Nrank = Grid._Nprocessors;
+	RealD Nnode = Grid.NodeCount();
+	RealD ppn = Nrank/Nnode;
+
+	std::vector<HalfSpinColourVectorD *> xbuf(8);
+	std::vector<HalfSpinColourVectorD *> rbuf(8);
+	Grid.ShmBufferFreeAll();
+	for(int d=0;d<8;d++){
+	  xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	}
+
+	int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+	int ncomm;
+	double dbytes;
+	std::vector<double> times(Nloop);
+	for(int i=0;i<Nloop;i++){
+
+	  double start=usecond();
+
+	  dbytes=0;
+	  ncomm=0;
+#ifdef GRID_OMP
+#pragma omp parallel for num_threads(Grid::CartesianCommunicator::nCommThreads)
+#endif
+	  for(int dir=0;dir<8;dir++){
+
+	    double tbytes;
+	    int mu =dir % 4;
+
+	    if (mpi_layout[mu]>1 ) {
+	        
+	      int xmit_to_rank;
+	      int recv_from_rank;
+	      if ( dir == mu ) { 
+		int comm_proc=1;
+		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	      } else { 
+		int comm_proc = mpi_layout[mu]-1;
+		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	      }
+#ifdef GRID_OMP
+	int tid = omp_get_thread_num(); 
+#else 
+        int tid = dir;
+#endif
+	      tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
+						 (void *)&rbuf[dir][0], recv_from_rank,
+						 bytes,tid);
+	  
+#ifdef GRID_OMP
+#pragma omp atomic
+#endif
+	      ncomm++;
+
+#ifdef GRID_OMP
+#pragma omp atomic
+#endif
+	      dbytes+=tbytes;
+	    }
+	  }
+	  Grid.Barrier();
+	  double stop=usecond();
+	  t_time[i] = stop-start; // microseconds
+	}
+
+	timestat.statistics(t_time);
+	//	for(int i=0;i<t_time.size();i++){
+	//	  std::cout << i<<" "<<t_time[i]<<std::endl;
+	//	}
+
+	dbytes=dbytes*ppn;
+	double xbytes    = dbytes*0.5;
+	double rbytes    = dbytes*0.5;
+	double bidibytes = dbytes;
+
+	std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+		 <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+		 <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+		 <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+		 << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+		 << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+
+ 
+	
+	    }
+    }    
+
+    return;
+  }
+
+  static void Memory(void)
+  {
+    const int Nvec=8;
+    typedef Lattice< iVector< vReal,Nvec> > LatticeVec;
+    typedef iVector<vReal,Nvec> Vec;
+
+    std::vector<int> simd_layout = GridDefaultSimd(Nd,vReal::Nsimd());
+    std::vector<int> mpi_layout  = GridDefaultMpi();
+
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Benchmarking a*x + y bandwidth"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<< "\t\tGB/s / node"<<std::endl;
+    std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+  
+    uint64_t NP;
+    uint64_t NN;
+
+
+  uint64_t lmax=48;
+#define NLOOP (100*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
+
+    GridSerialRNG          sRNG;      sRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
+    for(int lat=8;lat<=lmax;lat+=4){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+
+      NP= Grid.RankCount();
+      NN =Grid.NodeCount();
+
+      Vec rn ; random(sRNG,rn);
+
+      LatticeVec z(&Grid); z=rn;
+      LatticeVec x(&Grid); x=rn;
+      LatticeVec y(&Grid); y=rn;
+      double a=2.0;
+
+      uint64_t Nloop=NLOOP;
+
+      double start=usecond();
+      for(int i=0;i<Nloop;i++){
+	z=a*x-y;
+        x._odata[0]=z._odata[0]; // force serial dependency to prevent optimise away
+        y._odata[4]=z._odata[4];
+      }
+      double stop=usecond();
+      double time = (stop-start)/Nloop*1000;
+     
+      double flops=vol*Nvec*2;// mul,add
+      double bytes=3.0*vol*Nvec*sizeof(Real);
+      std::cout<<GridLogMessage<<std::setprecision(3) 
+	       << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.
+	       << "\t\t"<< bytes/time/NN <<std::endl;
+
+    }
+  };
+
+  static double DWF5(int Ls,int L)
+  {
+    RealD mass=0.1;
+    RealD M5  =1.8;
+
+    double mflops;
+    double mflops_best = 0;
+    double mflops_worst= 0;
+    std::vector<double> mflops_all;
+
+    ///////////////////////////////////////////////////////
+    // Set/Get the layout & grid size
+    ///////////////////////////////////////////////////////
+    int threads = GridThread::GetThreads();
+    std::vector<int> mpi = GridDefaultMpi(); assert(mpi.size()==4);
+    std::vector<int> local({L,L,L,L});
+
+    GridCartesian         * TmpGrid   = SpaceTimeGrid::makeFourDimGrid(std::vector<int>({64,64,64,64}), 
+								       GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    uint64_t NP = TmpGrid->RankCount();
+    uint64_t NN = TmpGrid->NodeCount();
+    NN_global=NN;
+    uint64_t SHM=NP/NN;
+
+    std::vector<int> internal;
+    if      ( SHM == 1 )   internal = std::vector<int>({1,1,1,1});
+    else if ( SHM == 2 )   internal = std::vector<int>({2,1,1,1});
+    else if ( SHM == 4 )   internal = std::vector<int>({2,2,1,1});
+    else if ( SHM == 8 )   internal = std::vector<int>({2,2,2,1});
+    else assert(0);
+
+    std::vector<int> nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]});
+    std::vector<int> latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]});
+
+    ///////// Welcome message ////////////
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "Benchmark DWF Ls vec on "<<L<<"^4 local volume "<<std::endl;
+    std::cout<<GridLogMessage << "* Global volume  : "<<GridCmdVectorIntToString(latt4)<<std::endl;
+    std::cout<<GridLogMessage << "* Ls             : "<<Ls<<std::endl;
+    std::cout<<GridLogMessage << "* MPI ranks      : "<<GridCmdVectorIntToString(mpi)<<std::endl;
+    std::cout<<GridLogMessage << "* Intranode      : "<<GridCmdVectorIntToString(internal)<<std::endl;
+    std::cout<<GridLogMessage << "* nodes          : "<<GridCmdVectorIntToString(nodes)<<std::endl;
+    std::cout<<GridLogMessage << "* Using "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    ///////// Lattice Init ////////////
+    GridCartesian         * UGrid    = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    GridRedBlackCartesian * UrbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridCartesian         * sUGrid   = SpaceTimeGrid::makeFourDimDWFGrid(latt4,GridDefaultMpi());
+    GridRedBlackCartesian * sUrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(sUGrid);
+    GridCartesian         * sFGrid   = SpaceTimeGrid::makeFiveDimDWFGrid(Ls,UGrid);
+    GridRedBlackCartesian * sFrbGrid = SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(Ls,UGrid);
+
+    ///////// RNG Init ////////////
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+    GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG          RNG5(sFGrid);  RNG5.SeedFixedIntegers(seeds5);
+    std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+    ///////// Source preparation ////////////
+    LatticeFermion src   (sFGrid); random(RNG5,src);
+    LatticeFermion tmp   (sFGrid);
+
+    RealD N2 = 1.0/::sqrt(norm2(src));
+    src = src*N2;
+    
+    LatticeGaugeField Umu(UGrid);  SU3::HotConfiguration(RNG4,Umu); 
+
+    WilsonFermion5DR sDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,M5);
+    LatticeFermion src_e (sFrbGrid);
+    LatticeFermion src_o (sFrbGrid);
+    LatticeFermion r_e   (sFrbGrid);
+    LatticeFermion r_o   (sFrbGrid);
+    LatticeFermion r_eo  (sFGrid);
+    LatticeFermion err   (sFGrid);
+    {
+
+      pickCheckerboard(Even,src_e,src);
+      pickCheckerboard(Odd,src_o,src);
+
+#if defined(AVX512) 
+      const int num_cases = 6;
+      std::string fmt("A/S ; A/O ; U/S ; U/O ; G/S ; G/O ");
+#else
+      const int num_cases = 4;
+      std::string fmt("U/S ; U/O ; G/S ; G/O ");
+#endif
+      controls Cases [] = {
+#ifdef AVX512
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+#endif
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  }
+      }; 
+
+      for(int c=0;c<num_cases;c++) {
+
+	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
+	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
+	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
+
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+	if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+	int nwarm = 100;
+	uint64_t ncall = 1000;
+
+	double t0=usecond();
+	sFGrid->Barrier();
+	for(int i=0;i<nwarm;i++){
+	  sDw.DhopEO(src_o,r_e,DaggerNo);
+	}
+	sFGrid->Barrier();
+	double t1=usecond();
+
+	sDw.ZeroCounters();
+	time_statistics timestat;
+	std::vector<double> t_time(ncall);
+	for(uint64_t i=0;i<ncall;i++){
+	  t0=usecond();
+	  sDw.DhopEO(src_o,r_e,DaggerNo);
+	  t1=usecond();
+	  t_time[i] = t1-t0;
+	}
+	sFGrid->Barrier();
+	
+	double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+	double flops=(1344.0*volume)/2;
+	double mf_hi, mf_lo, mf_err;
+
+	timestat.statistics(t_time);
+	mf_hi = flops/timestat.min;
+	mf_lo = flops/timestat.max;
+	mf_err= flops/timestat.min * timestat.err/timestat.mean;
+
+	mflops = flops/timestat.mean;
+	mflops_all.push_back(mflops);
+	if ( mflops_best == 0   ) mflops_best = mflops;
+	if ( mflops_worst== 0   ) mflops_worst= mflops;
+	if ( mflops>mflops_best ) mflops_best = mflops;
+	if ( mflops<mflops_worst) mflops_worst= mflops;
+
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s =   "<< mflops << " ("<<mf_err<<") " << mf_lo<<"-"<<mf_hi <<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s per rank   "<< mflops/NP<<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s per node   "<< mflops/NN<<std::endl;
+
+	sDw.Report();
+
+      }
+      double robust = mflops_worst/mflops_best;;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
+
+      std::cout<<GridLogMessage <<std::setprecision(3)<< L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< robust <<std::endl;
+      std::cout<<GridLogMessage <<fmt << std::endl;
+      std::cout<<GridLogMessage;
+
+      for(int i=0;i<mflops_all.size();i++){
+	std::cout<<mflops_all[i]/NN<<" ; " ;
+      }
+      std::cout<<std::endl;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    }
+    return mflops_best;
+  }
+
+  static double DWF(int Ls,int L, double & robust)
+  {
+    RealD mass=0.1;
+    RealD M5  =1.8;
+
+    double mflops;
+    double mflops_best = 0;
+    double mflops_worst= 0;
+    std::vector<double> mflops_all;
+
+    ///////////////////////////////////////////////////////
+    // Set/Get the layout & grid size
+    ///////////////////////////////////////////////////////
+    int threads = GridThread::GetThreads();
+    std::vector<int> mpi = GridDefaultMpi(); assert(mpi.size()==4);
+    std::vector<int> local({L,L,L,L});
+
+    GridCartesian         * TmpGrid   = SpaceTimeGrid::makeFourDimGrid(std::vector<int>({64,64,64,64}), 
+								       GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    uint64_t NP = TmpGrid->RankCount();
+    uint64_t NN = TmpGrid->NodeCount();
+    NN_global=NN;
+    uint64_t SHM=NP/NN;
+
+    std::vector<int> internal;
+    if      ( SHM == 1 )   internal = std::vector<int>({1,1,1,1});
+    else if ( SHM == 2 )   internal = std::vector<int>({2,1,1,1});
+    else if ( SHM == 4 )   internal = std::vector<int>({2,2,1,1});
+    else if ( SHM == 8 )   internal = std::vector<int>({2,2,2,1});
+    else assert(0);
+
+    std::vector<int> nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]});
+    std::vector<int> latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]});
+
+    ///////// Welcome message ////////////
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "Benchmark DWF on "<<L<<"^4 local volume "<<std::endl;
+    std::cout<<GridLogMessage << "* Global volume  : "<<GridCmdVectorIntToString(latt4)<<std::endl;
+    std::cout<<GridLogMessage << "* Ls             : "<<Ls<<std::endl;
+    std::cout<<GridLogMessage << "* MPI ranks      : "<<GridCmdVectorIntToString(mpi)<<std::endl;
+    std::cout<<GridLogMessage << "* Intranode      : "<<GridCmdVectorIntToString(internal)<<std::endl;
+    std::cout<<GridLogMessage << "* nodes          : "<<GridCmdVectorIntToString(nodes)<<std::endl;
+    std::cout<<GridLogMessage << "* Using "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+
+    ///////// Lattice Init ////////////
+    GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+    GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+    
+    ///////// RNG Init ////////////
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+    GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+    std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+    ///////// Source preparation ////////////
+    LatticeFermion src   (FGrid); random(RNG5,src);
+    LatticeFermion ref   (FGrid);
+    LatticeFermion tmp   (FGrid);
+
+    RealD N2 = 1.0/::sqrt(norm2(src));
+    src = src*N2;
+    
+    LatticeGaugeField Umu(UGrid);  SU3::HotConfiguration(RNG4,Umu); 
+
+    DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+
+    ////////////////////////////////////
+    // Naive wilson implementation
+    ////////////////////////////////////
+    {
+      LatticeGaugeField Umu5d(FGrid); 
+      std::vector<LatticeColourMatrix> U(4,FGrid);
+      for(int ss=0;ss<Umu._grid->oSites();ss++){
+	for(int s=0;s<Ls;s++){
+	  Umu5d._odata[Ls*ss+s] = Umu._odata[ss];
+	}
+      }
+      ref = zero;
+      for(int mu=0;mu<Nd;mu++){
+	U[mu] = PeekIndex<LorentzIndex>(Umu5d,mu);
+      }
+      for(int mu=0;mu<Nd;mu++){
+	
+	tmp = U[mu]*Cshift(src,mu+1,1);
+	ref=ref + tmp - Gamma(Gmu[mu])*tmp;
+	
+	tmp =adj(U[mu])*src;
+	tmp =Cshift(tmp,mu+1,-1);
+	ref=ref + tmp + Gamma(Gmu[mu])*tmp;
+      }
+      ref = -0.5*ref;
+    }
+
+    LatticeFermion src_e (FrbGrid);
+    LatticeFermion src_o (FrbGrid);
+    LatticeFermion r_e   (FrbGrid);
+    LatticeFermion r_o   (FrbGrid);
+    LatticeFermion r_eo  (FGrid);
+    LatticeFermion err   (FGrid);
+    {
+
+      pickCheckerboard(Even,src_e,src);
+      pickCheckerboard(Odd,src_o,src);
+
+#if defined(AVX512) 
+      const int num_cases = 6;
+      std::string fmt("A/S ; A/O ; U/S ; U/O ; G/S ; G/O ");
+#else
+      const int num_cases = 4;
+      std::string fmt("U/S ; U/O ; G/S ; G/O ");
+#endif
+      controls Cases [] = {
+#ifdef AVX512
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+#endif
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  }
+      }; 
+
+      for(int c=0;c<num_cases;c++) {
+
+	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
+	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
+	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
+
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+	if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+	int nwarm = 200;
+	double t0=usecond();
+	FGrid->Barrier();
+	for(int i=0;i<nwarm;i++){
+	  Dw.DhopEO(src_o,r_e,DaggerNo);
+	}
+	FGrid->Barrier();
+	double t1=usecond();
+	//	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
+	//	if (ncall < 500) ncall = 500;
+	uint64_t ncall = 1000;
+
+	FGrid->Broadcast(0,&ncall,sizeof(ncall));
+
+	//	std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"<<std::endl;
+	Dw.ZeroCounters();
+
+	time_statistics timestat;
+	std::vector<double> t_time(ncall);
+	for(uint64_t i=0;i<ncall;i++){
+	  t0=usecond();
+	  Dw.DhopEO(src_o,r_e,DaggerNo);
+	  t1=usecond();
+	  t_time[i] = t1-t0;
+	}
+	FGrid->Barrier();
+	
+	double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+	double flops=(1344.0*volume)/2;
+	double mf_hi, mf_lo, mf_err;
+
+	timestat.statistics(t_time);
+	mf_hi = flops/timestat.min;
+	mf_lo = flops/timestat.max;
+	mf_err= flops/timestat.min * timestat.err/timestat.mean;
+
+	mflops = flops/timestat.mean;
+	mflops_all.push_back(mflops);
+	if ( mflops_best == 0   ) mflops_best = mflops;
+	if ( mflops_worst== 0   ) mflops_worst= mflops;
+	if ( mflops>mflops_best ) mflops_best = mflops;
+	if ( mflops<mflops_worst) mflops_worst= mflops;
+
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s =   "<< mflops << " ("<<mf_err<<") " << mf_lo<<"-"<<mf_hi <<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s per rank   "<< mflops/NP<<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s per node   "<< mflops/NN<<std::endl;
+
+	Dw.Report();
+
+	Dw.DhopEO(src_o,r_e,DaggerNo);
+	Dw.DhopOE(src_e,r_o,DaggerNo);
+	setCheckerboard(r_eo,r_o);
+	setCheckerboard(r_eo,r_e);
+	err = r_eo-ref; 
+	std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+	assert((norm2(err)<1.0e-4));
+
+      }
+      robust = mflops_worst/mflops_best;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << std::fixed<<std::setprecision(3)<< L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< robust  <<std::endl;
+      std::cout<<GridLogMessage <<fmt << std::endl;
+      std::cout<<GridLogMessage ;
+
+      for(int i=0;i<mflops_all.size();i++){
+	std::cout<<mflops_all[i]/NN<<" ; " ;
+      }
+      std::cout<<std::endl;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    }
+    return mflops_best;
+  }
+
+};
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  CartesianCommunicator::SetCommunicatorPolicy(CartesianCommunicator::CommunicatorPolicySequential);
+#ifdef KNL
+  LebesgueOrder::Block = std::vector<int>({8,2,2,2});
+#else
+  LebesgueOrder::Block = std::vector<int>({2,2,2,2});
+#endif
+  Benchmark::Decomposition();
+
+  int do_memory=1;
+  int do_comms =1;
+  int do_su3   =0;
+  int do_wilson=1;
+  int do_dwf   =1;
+
+  if ( do_su3 ) {
+    // empty for now
+  }
+#if 1
+  int sel=2;
+  std::vector<int> L_list({8,12,16,24});
+#else
+  int sel=1;
+  std::vector<int> L_list({8,12});
+#endif
+  int selm1=sel-1;
+  std::vector<double> robust_list;
+
+  std::vector<double> wilson;
+  std::vector<double> dwf4;
+  std::vector<double> dwf5;
+
+  if ( do_wilson ) {
+    int Ls=1;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Wilson dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    for(int l=0;l<L_list.size();l++){
+      double robust;
+      wilson.push_back(Benchmark::DWF(1,L_list[l],robust));
+    }
+  }
+
+  int Ls=16;
+  if ( do_dwf ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    for(int l=0;l<L_list.size();l++){
+      double robust;
+      double result = Benchmark::DWF(Ls,L_list[l],robust) ;
+      dwf4.push_back(result);
+      robust_list.push_back(robust);
+    }
+  }
+
+  if ( do_dwf ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    for(int l=0;l<L_list.size();l++){
+      dwf5.push_back(Benchmark::DWF5(Ls,L_list[l]));
+    }
+
+  }
+
+  if ( do_dwf ) {
+
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Summary table Ls="<<Ls <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "L \t\t Wilson \t DWF4 \t DWF5 " <<std::endl;
+  for(int l=0;l<L_list.size();l++){
+    std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]<<" \t "<<dwf4[l]<<" \t "<<dwf5[l] <<std::endl;
+  }
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  }
+
+  int NN=NN_global;
+  if ( do_memory ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Memory benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Memory();
+  }
+
+  if ( do_comms && (NN>1) ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Communications benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Comms();
+  }
+
+  if ( do_dwf ) {
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Per Node Summary table Ls="<<Ls <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " L \t\t Wilson\t\t DWF4  \t\t DWF5 " <<std::endl;
+  for(int l=0;l<L_list.size();l++){
+    std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]/NN<<" \t "<<dwf4[l]/NN<<" \t "<<dwf5[l] /NN<<std::endl;
+  }
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Comparison point     result: "  << 0.5*(dwf4[sel]+dwf4[selm1])/NN << " Mflop/s per node"<<std::endl;
+  std::cout<<GridLogMessage << " Comparison point is 0.5*("<<dwf4[sel]/NN<<"+"<<dwf4[selm1]/NN << ") "<<std::endl;
+  std::cout<<std::setprecision(3);
+  std::cout<<GridLogMessage << " Comparison point robustness: "  << robust_list[sel] <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  }
+
+
+  Grid_finalize();
+}

benchmarks/Benchmark_comms.cc

@@ -31,6 +31,32 @@ using namespace std;
 using namespace Grid;
 using namespace Grid::QCD;
 
+struct time_statistics{
+  double mean;
+  double err;
+  double min;
+  double max;
+
+  void statistics(std::vector<double> v){
+      double sum = std::accumulate(v.begin(), v.end(), 0.0);
+      mean = sum / v.size();
+
+      std::vector<double> diff(v.size());
+      std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; });
+      double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+      err = std::sqrt(sq_sum / (v.size()*(v.size() - 1)));
+
+      auto result = std::minmax_element(v.begin(), v.end());
+      min = *result.first;
+      max = *result.second;
+}
+};
+
+void header(){
+  std::cout <<GridLogMessage << " L  "<<"\t"<<" Ls  "<<"\t"
+            <<std::setw(11)<<"bytes"<<"MB/s uni (err/min/max)"<<"\t\t"<<"MB/s bidi (err/min/max)"<<std::endl;
+};
+
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
@@ -40,17 +66,21 @@ int main (int argc, char ** argv)
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
-  int Nloop=10;
+  int Nloop=100;
   int nmu=0;
+  int maxlat=32;
   for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
 
+  std::cout << GridLogMessage << "Number of iterations to average: "<< Nloop << std::endl;
+  std::vector<double> t_time(Nloop);
+  time_statistics timestat;
+
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-  int maxlat=24;
+  header();
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -58,17 +88,25 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > xbuf(8);	
+      std::vector<Vector<HalfSpinColourVectorD> > rbuf(8);
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+      for(int mu=0;mu<8;mu++){
+	xbuf[mu].resize(lat*lat*lat*Ls);
+	rbuf[mu].resize(lat*lat*lat*Ls);
+	//	std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] <<std::endl;
+      }
 
-      double start=usecond();
       for(int i=0;i<Nloop;i++){
+      double start=usecond();
 
-	std::vector<CartesianCommunicator::CommsRequest_t> requests;
+	std::vector<CommsRequest_t> requests;
 
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
@@ -79,7 +117,6 @@ int main (int argc, char ** argv)
 	    int comm_proc=1;
 	    int xmit_to_rank;
 	    int recv_from_rank;
-	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	    Grid.SendToRecvFromBegin(requests,
 				   (void *)&xbuf[mu][0],
@@ -102,18 +139,24 @@ int main (int argc, char ** argv)
 	}
 	Grid.SendToRecvFromComplete(requests);
 	Grid.Barrier();
-
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
+      timestat.statistics(t_time);
+
+      double dbytes    = bytes*ppn;
+      double xbytes    = dbytes*2.0*ncomm;
       double rbytes    = xbytes;
       double bidibytes = xbytes+rbytes;
 
-      double time = stop-start; // microseconds
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
 
@@ -121,25 +164,36 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking sequential halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-
+  header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],
+                                    lat*mpi_layout[1],
+                                    lat*mpi_layout[2],
+                                    lat*mpi_layout[3]});
 
-      std::vector<int> latt_size  ({lat,lat,lat,lat});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > xbuf(8);
+      std::vector<Vector<HalfSpinColourVectorD> > rbuf(8);
 
+      for(int mu=0;mu<8;mu++){
+	xbuf[mu].resize(lat*lat*lat*Ls);
+	rbuf[mu].resize(lat*lat*lat*Ls);
+	//	std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] <<std::endl;
+      }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
       for(int i=0;i<Nloop;i++){
+      double start=usecond();
     
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
@@ -152,7 +206,7 @@ int main (int argc, char ** argv)
 	    int recv_from_rank;
 	    
 	    {
-	      std::vector<CartesianCommunicator::CommsRequest_t> requests;
+	      std::vector<CommsRequest_t> requests;
 	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	      Grid.SendToRecvFromBegin(requests,
 				       (void *)&xbuf[mu][0],
@@ -165,7 +219,7 @@ int main (int argc, char ** argv)
 
 	    comm_proc = mpi_layout[mu]-1;
 	    {
-	      std::vector<CartesianCommunicator::CommsRequest_t> requests;
+	      std::vector<CommsRequest_t> requests;
 	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	      Grid.SendToRecvFromBegin(requests,
 				       (void *)&xbuf[mu+4][0],
@@ -178,30 +232,37 @@ int main (int argc, char ** argv)
 	  }
 	}
 	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+
       }
 
-      double stop=usecond();
+      timestat.statistics(t_time);
       
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
+      double dbytes    = bytes*ppn;
+      double xbytes    = dbytes*2.0*ncomm;
       double rbytes    = xbytes;
       double bidibytes = xbytes+rbytes;
 
-      double time = stop-start;
+    std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+      
     }
   }  
 
 
-  Nloop=10;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -209,6 +270,9 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<HalfSpinColourVectorD *> xbuf(8);
       std::vector<HalfSpinColourVectorD *> rbuf(8);
@@ -216,73 +280,86 @@ int main (int argc, char ** argv)
       for(int d=0;d<8;d++){
 	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
 	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
       }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
+      double dbytes;
       for(int i=0;i<Nloop;i++){
+	double start=usecond();
 
-	std::vector<CartesianCommunicator::CommsRequest_t> requests;
-
+	dbytes=0;
 	ncomm=0;
+
+	std::vector<CommsRequest_t> requests;
+
 	for(int mu=0;mu<4;mu++){
 	
+
 	  if (mpi_layout[mu]>1 ) {
 	  
 	    ncomm++;
 	    int comm_proc=1;
 	    int xmit_to_rank;
 	    int recv_from_rank;
-	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu][0],
-					    recv_from_rank,
-					    bytes);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu][0],
+					      recv_from_rank,
+					      bytes,mu);
 	
 	    comm_proc = mpi_layout[mu]-1;
 	  
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu+4][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu+4][0],
-					    recv_from_rank,
-					    bytes);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu+4][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu+4][0],
+					      recv_from_rank,
+					      bytes,mu+4);
 	  
 	  }
 	}
-	Grid.StencilSendToRecvFromComplete(requests);
+	Grid.StencilSendToRecvFromComplete(requests,0);
 	Grid.Barrier();
-
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+	
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
+      timestat.statistics(t_time);
+
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      double time = stop-start; // microseconds
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
 
 
-
-  Nloop=100;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking sequential STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -290,6 +367,9 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<HalfSpinColourVectorD *> xbuf(8);
       std::vector<HalfSpinColourVectorD *> rbuf(8);
@@ -297,16 +377,18 @@ int main (int argc, char ** argv)
       for(int d=0;d<8;d++){
 	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
 	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
       }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
-
-      double start=usecond();
+      double dbytes;
       for(int i=0;i<Nloop;i++){
+	double start=usecond();
 
-	std::vector<CartesianCommunicator::CommsRequest_t> requests;
-
+	std::vector<CommsRequest_t> requests;
+	dbytes=0;
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
 	
@@ -318,44 +400,147 @@ int main (int argc, char ** argv)
 	    int recv_from_rank;
 	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu][0],
-					    recv_from_rank,
-					    bytes);
-	    Grid.StencilSendToRecvFromComplete(requests);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu][0],
+					      recv_from_rank,
+					      bytes,mu);
+	    Grid.StencilSendToRecvFromComplete(requests,mu);
 	    requests.resize(0);
 
 	    comm_proc = mpi_layout[mu]-1;
 	  
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu+4][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu+4][0],
-					    recv_from_rank,
-					    bytes);
-	    Grid.StencilSendToRecvFromComplete(requests);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu+4][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu+4][0],
+					      recv_from_rank,
+					      bytes,mu+4);
+	    Grid.StencilSendToRecvFromComplete(requests,mu+4);
 	    requests.resize(0);
 	  
 	  }
 	}
 	Grid.Barrier();
-
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+	
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
+      timestat.statistics(t_time);
 
-      double time = stop-start; // microseconds
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+ 
     }
   }    
 
+
+#ifdef GRID_OMP
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking threaded STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  header();
+
+  for(int lat=4;lat<=maxlat;lat+=4){
+    for(int Ls=8;Ls<=8;Ls*=2){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],
+      				    lat*mpi_layout[1],
+      				    lat*mpi_layout[2],
+      				    lat*mpi_layout[3]});
+
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
+
+      std::vector<HalfSpinColourVectorD *> xbuf(8);
+      std::vector<HalfSpinColourVectorD *> rbuf(8);
+      Grid.ShmBufferFreeAll();
+      for(int d=0;d<8;d++){
+	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+      }
+
+      int ncomm;
+      int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+      double dbytes;
+      for(int i=0;i<Nloop;i++){
+	double start=usecond();
+
+	std::vector<CommsRequest_t> requests;
+	dbytes=0;
+	ncomm=0;
+
+#pragma omp parallel for num_threads(Grid::CartesianCommunicator::nCommThreads)
+	for(int dir=0;dir<8;dir++){
+
+	  double tbytes;
+	  int mu =dir % 4;
+
+	  if (mpi_layout[mu]>1 ) {
+	  
+	    ncomm++;
+	    int xmit_to_rank;
+	    int recv_from_rank;
+	    if ( dir == mu ) { 
+	      int comm_proc=1;
+	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    } else { 
+	      int comm_proc = mpi_layout[mu]-1;
+	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    }
+            int tid = omp_get_thread_num();
+	    tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
+					       (void *)&rbuf[dir][0], recv_from_rank, bytes,tid);
+
+#pragma omp atomic
+	    dbytes+=tbytes;
+	  }
+	}
+	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+      }
+
+      timestat.statistics(t_time);
+
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
+
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+ 
+    }
+  }    
+#endif
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= All done; Bye Bye"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+
   Grid_finalize();
 }

benchmarks/Benchmark_dwf.cc

@@ -48,10 +48,19 @@ int main (int argc, char ** argv)
 
 
   int threads = GridThread::GetThreads();
-  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
   std::vector<int> latt4 = GridDefaultLatt();
-  const int Ls=16;
+  int Ls=16;
+  for(int i=0;i<argc;i++)
+    if(std::string(argv[i]) == "-Ls"){
+      std::stringstream ss(argv[i+1]); ss >> Ls;
+    }
+
+  GridLogLayout();
+
+  long unsigned int single_site_flops = 8*QCD::Nc*(7+16*QCD::Nc);
+
+
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
@@ -165,7 +174,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
 
   DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-  int ncall =1000;
+  int ncall =500;
   if (1) {
     FGrid->Barrier();
     Dw.ZeroCounters();
@@ -181,7 +190,7 @@ int main (int argc, char ** argv)
     FGrid->Barrier();
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=1344*volume*ncall;
+    double flops=single_site_flops*volume*ncall;
 
     std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
@@ -220,7 +229,7 @@ int main (int argc, char ** argv)
     FGrid->Barrier();
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=1344*volume*ncall;
+    double flops=single_site_flops*volume*ncall;
 
     std::cout<<GridLogMessage << "Called half prec comms Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
@@ -271,7 +280,7 @@ int main (int argc, char ** argv)
     double t1=usecond();
     FGrid->Barrier();
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=1344*volume*ncall;
+    double flops=single_site_flops*volume*ncall;
 
     std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
@@ -302,6 +311,7 @@ int main (int argc, char ** argv)
       std::cout<< "sD ERR   \n " << err  <<std::endl;
     }
     assert(sum < 1.0e-4);
+
     
     if(1){
       std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
@@ -348,7 +358,7 @@ int main (int argc, char ** argv)
       //      sDw.stat.print();
 
       double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-      double flops=(1344.0*volume*ncall)/2;
+      double flops=(single_site_flops*volume*ncall)/2.0;
 
       std::cout<<GridLogMessage << "sDeo mflop/s =   "<< flops/(t1-t0)<<std::endl;
       std::cout<<GridLogMessage << "sDeo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
@@ -381,8 +391,23 @@ int main (int argc, char ** argv)
       }
       assert(error<1.0e-4);
     }
+
+  if(0){
+    std::cout << "Single cache warm call to sDw.Dhop " <<std::endl;
+    for(int i=0;i< PerformanceCounter::NumTypes(); i++ ){
+      sDw.Dhop(ssrc,sresult,0);
+      PerformanceCounter Counter(i);
+      Counter.Start();
+      sDw.Dhop(ssrc,sresult,0);
+      Counter.Stop();
+      Counter.Report();
+    }
   }
 
+  }
+
+
+
   if (1)
   { // Naive wilson dag implementation
     ref = zero;
@@ -456,7 +481,7 @@ int main (int argc, char ** argv)
     FGrid->Barrier();
     
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=(1344.0*volume*ncall)/2;
+    double flops=(single_site_flops*volume*ncall)/2.0;
 
     std::cout<<GridLogMessage << "Deo mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "Deo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
@@ -487,9 +512,9 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "norm diff even  "<< norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "norm diff odd   "<< norm2(src_o)<<std::endl;
 
-  //assert(norm2(src_e)<1.0e-4);
-  //assert(norm2(src_o)<1.0e-4);
-
+  assert(norm2(src_e)<1.0e-4);
+  assert(norm2(src_o)<1.0e-4);
   Grid_finalize();
+  exit(0);
 }
 

benchmarks/Benchmark_dwf_sweep.cc

@@ -51,6 +51,7 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
+
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
   std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
@@ -107,6 +108,7 @@ void benchDw(std::vector<int> & latt4, int Ls, int threads,int report )
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
   GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+  long unsigned int single_site_flops = 8*QCD::Nc*(7+16*QCD::Nc);
 
   std::vector<int> seeds4({1,2,3,4});
   std::vector<int> seeds5({5,6,7,8});
@@ -196,7 +198,7 @@ void benchDw(std::vector<int> & latt4, int Ls, int threads,int report )
   
   if ( ! report ) {
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=1344*volume*ncall;
+    double flops=single_site_flops*volume*ncall;
     std::cout <<"\t"<<NP<< "\t"<<flops/(t1-t0)<< "\t";
   }
   
@@ -228,7 +230,7 @@ void benchDw(std::vector<int> & latt4, int Ls, int threads,int report )
     
     if(!report){
       double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-      double flops=(1344.0*volume*ncall)/2;
+      double flops=(single_site_flops*volume*ncall)/2.0;
       std::cout<< flops/(t1-t0);
     }
   }
@@ -237,6 +239,7 @@ void benchDw(std::vector<int> & latt4, int Ls, int threads,int report )
 #define CHECK_SDW
 void benchsDw(std::vector<int> & latt4, int Ls, int threads, int report )
 {
+  long unsigned int single_site_flops = 8*QCD::Nc*(7+16*QCD::Nc);
 
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
@@ -321,7 +324,7 @@ void benchsDw(std::vector<int> & latt4, int Ls, int threads, int report )
     Counter.Report();
   } else { 
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=1344*volume*ncall;
+    double flops=single_site_flops*volume*ncall;
     std::cout<<"\t"<< flops/(t1-t0);
   }
 
@@ -358,7 +361,7 @@ void benchsDw(std::vector<int> & latt4, int Ls, int threads, int report )
     CounterSdw.Report();
   } else {
     double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
-    double flops=(1344.0*volume*ncall)/2;
+    double flops=(single_site_flops*volume*ncall)/2.0;
     std::cout<<"\t"<< flops/(t1-t0);
   }
 }

benchmarks/Benchmark_gparity.cc

@@ -0,0 +1,190 @@
+#include <Grid/Grid.h>
+#include <sstream>
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+  d internal;
+};
+
+  Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+  };
+
+typedef typename GparityDomainWallFermionF::FermionField GparityLatticeFermionF;
+typedef typename GparityDomainWallFermionD::FermionField GparityLatticeFermionD;
+
+
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int Ls=16;
+  for(int i=0;i<argc;i++)
+    if(std::string(argv[i]) == "-Ls"){
+      std::stringstream ss(argv[i+1]); ss >> Ls;
+    }
+
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+  std::cout<<GridLogMessage << "Ls = " << Ls << std::endl;
+
+  std::vector<int> latt4 = GridDefaultLatt();
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexF::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+  
+  std::cout << GridLogMessage << "Initialising 4d RNG" << std::endl;
+  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  std::cout << GridLogMessage << "Initialising 5d RNG" << std::endl;
+  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+  GparityLatticeFermionF src   (FGrid); random(RNG5,src);
+  RealD N2 = 1.0/::sqrt(norm2(src));
+  src = src*N2;
+
+  GparityLatticeFermionF result(FGrid); result=zero;
+  GparityLatticeFermionF    ref(FGrid);    ref=zero;
+  GparityLatticeFermionF    tmp(FGrid);
+  GparityLatticeFermionF    err(FGrid);
+
+  std::cout << GridLogMessage << "Drawing gauge field" << std::endl;
+  LatticeGaugeFieldF Umu(UGrid); 
+  SU3::HotConfiguration(RNG4,Umu); 
+  std::cout << GridLogMessage << "Random gauge initialised " << std::endl;
+
+  RealD mass=0.1;
+  RealD M5  =1.8;
+
+  RealD NP = UGrid->_Nprocessors;
+  RealD NN = UGrid->NodeCount();
+
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Benchmarking DomainWallFermion::Dhop                  "<<std::endl;
+  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplexF::Nsimd()<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+
+
+
+  std::cout << GridLogMessage<< "* SINGLE/SINGLE"<<std::endl;
+  GparityDomainWallFermionF Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  int ncall =1000;
+  if (1) {
+    FGrid->Barrier();
+    Dw.ZeroCounters();
+    Dw.Dhop(src,result,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      Dw.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1320*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    Dw.Report();
+  }
+
+  std::cout << GridLogMessage<< "* SINGLE/HALF"<<std::endl;
+  GparityDomainWallFermionFH DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  if (1) {
+    FGrid->Barrier();
+    DwH.ZeroCounters();
+    DwH.Dhop(src,result,0);
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwH.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1320*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called half prec comms Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    DwH.Report();
+  }
+
+  GridCartesian         * UGrid_d   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_d);
+  GridCartesian         * FGrid_d   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_d);
+  GridRedBlackCartesian * FrbGrid_d = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_d);
+
+  
+  std::cout << GridLogMessage<< "* DOUBLE/DOUBLE"<<std::endl;
+  GparityLatticeFermionD src_d(FGrid_d);
+  precisionChange(src_d,src);
+
+  LatticeGaugeFieldD Umu_d(UGrid_d); 
+  precisionChange(Umu_d,Umu);
+
+  GparityLatticeFermionD result_d(FGrid_d);
+
+  GparityDomainWallFermionD DwD(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5);
+  if (1) {
+    FGrid_d->Barrier();
+    DwD.ZeroCounters();
+    DwD.Dhop(src_d,result_d,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwD.Dhop(src_d,result_d,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid_d->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1320*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    DwD.Report();
+  }
+
+  Grid_finalize();
+}
+

benchmarks/Benchmark_memory_bandwidth.cc

@@ -55,21 +55,21 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
-  uint64_t lmax=44;
-#define NLOOP (1*lmax*lmax*lmax*lmax/vol)
-  for(int lat=4;lat<=lmax;lat+=4){
+  uint64_t lmax=64;
+#define NLOOP (10*lmax*lmax*lmax*lmax/vol)
+  for(int lat=8;lat<=lmax;lat+=8){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
       uint64_t Nloop=NLOOP;
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeVec z(&Grid); //random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      LatticeVec z(&Grid);// random(pRNG,z);
+      LatticeVec x(&Grid);// random(pRNG,x);
+      LatticeVec y(&Grid);// random(pRNG,y);
       double a=2.0;
 
 
@@ -83,7 +83,7 @@ int main (int argc, char ** argv)
       double time = (stop-start)/Nloop*1000;
       
       double flops=vol*Nvec*2;// mul,add
-      double bytes=3*vol*Nvec*sizeof(Real);
+      double bytes=3.0*vol*Nvec*sizeof(Real);
       std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
 
     }
@@ -94,17 +94,17 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
   
-  for(int lat=4;lat<=lmax;lat+=4){
+  for(int lat=8;lat<=lmax;lat+=8){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeVec z(&Grid); //random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      LatticeVec z(&Grid);// random(pRNG,z);
+      LatticeVec x(&Grid);// random(pRNG,x);
+      LatticeVec y(&Grid);// random(pRNG,y);
       double a=2.0;
 
       uint64_t Nloop=NLOOP;
@@ -119,7 +119,7 @@ int main (int argc, char ** argv)
       double time = (stop-start)/Nloop*1000;
      
       double flops=vol*Nvec*2;// mul,add
-      double bytes=3*vol*Nvec*sizeof(Real);
+      double bytes=3.0*vol*Nvec*sizeof(Real);
       std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
 
     }
@@ -129,20 +129,20 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
 
-  for(int lat=4;lat<=lmax;lat+=4){
+  for(int lat=8;lat<=lmax;lat+=8){
 
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       uint64_t Nloop=NLOOP;
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeVec z(&Grid); //random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      LatticeVec z(&Grid);// random(pRNG,z);
+      LatticeVec x(&Grid);// random(pRNG,x);
+      LatticeVec y(&Grid);// random(pRNG,y);
       RealD a=2.0;
 
 
@@ -154,7 +154,7 @@ int main (int argc, char ** argv)
       double stop=usecond();
       double time = (stop-start)/Nloop*1000;
       
-      double bytes=2*vol*Nvec*sizeof(Real);
+      double bytes=2.0*vol*Nvec*sizeof(Real);
       double flops=vol*Nvec*1;// mul
       std::cout<<GridLogMessage <<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
 
@@ -166,17 +166,17 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=4;lat<=lmax;lat+=4){
+  for(int lat=8;lat<=lmax;lat+=8){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       uint64_t Nloop=NLOOP;
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
-      LatticeVec z(&Grid); //random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
+      LatticeVec z(&Grid);// random(pRNG,z);
+      LatticeVec x(&Grid);// random(pRNG,x);
+      LatticeVec y(&Grid);// random(pRNG,y);
       RealD a=2.0;
       Real nn;      
       double start=usecond();
@@ -187,7 +187,7 @@ int main (int argc, char ** argv)
       double stop=usecond();
       double time = (stop-start)/Nloop*1000;
       
-      double bytes=vol*Nvec*sizeof(Real);
+      double bytes=1.0*vol*Nvec*sizeof(Real);
       double flops=vol*Nvec*2;// mul,add
       std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"  \t\t"<<bytes/time<<"\t\t"<<flops/time<< "\t\t"<<(stop-start)/1000./1000.<< "\t\t " <<std::endl;
 

benchmarks/Benchmark_meson_field.cc

@@ -0,0 +1,803 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./benchmarks/Benchmark_wilson.cc
+
+    Copyright (C) 2018
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+
+#include "Grid/util/Profiling.h"
+
+template<class vobj>
+void sliceInnerProductMesonField(std::vector< std::vector<ComplexD> > &mat, 
+				 const std::vector<Lattice<vobj> > &lhs,
+				 const std::vector<Lattice<vobj> > &rhs,
+				 int orthogdim) 
+{
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  int Lblock = lhs.size();
+  int Rblock = rhs.size();
+
+  GridBase *grid = lhs[0]._grid;
+  
+  const int    Nd = grid->_ndimension;
+  const int Nsimd = grid->Nsimd();
+  int Nt     = grid->GlobalDimensions()[orthogdim];
+
+  assert(mat.size()==Lblock*Rblock);
+  for(int t=0;t<mat.size();t++){
+    assert(mat[t].size()==Nt);
+  }
+
+  int fd=grid->_fdimensions[orthogdim];
+  int ld=grid->_ldimensions[orthogdim];
+  int rd=grid->_rdimensions[orthogdim];
+
+  // will locally sum vectors first
+  // sum across these down to scalars
+  // splitting the SIMD
+  std::vector<vector_type,alignedAllocator<vector_type> > lvSum(rd*Lblock*Rblock);
+  parallel_for (int r = 0; r < rd * Lblock * Rblock; r++){
+    lvSum[r] = zero;
+  }
+
+  std::vector<scalar_type > lsSum(ld*Lblock*Rblock,scalar_type(0.0));             
+
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
+  
+  std::cout << GridLogMessage << " Entering first parallel loop "<<std::endl;
+  // Parallelise over t-direction doesn't expose as much parallelism as needed for KNL
+  parallel_for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	for(int i=0;i<Lblock;i++){
+	  auto left = conjugate(lhs[i]._odata[ss]);
+	  for(int j=0;j<Rblock;j++){
+	    int idx = i+Lblock*j+Lblock*Rblock*r;
+	    auto right = rhs[j]._odata[ss];
+	    vector_type vv = left()(0)(0) * right()(0)(0)
+	      +              left()(0)(1) * right()(0)(1)
+	      +              left()(0)(2) * right()(0)(2)
+              +              left()(1)(0) * right()(1)(0)
+	      +              left()(1)(1) * right()(1)(1)
+	      +              left()(1)(2) * right()(1)(2)
+              +              left()(2)(0) * right()(2)(0)
+	      +              left()(2)(1) * right()(2)(1)
+	      +              left()(2)(2) * right()(2)(2)
+              +              left()(3)(0) * right()(3)(0)
+	      +              left()(3)(1) * right()(3)(1)
+	      +              left()(3)(2) * right()(3)(2);
+	    lvSum[idx]=lvSum[idx]+vv;
+	  }
+	}
+      }
+    }
+  }
+
+  std::cout << GridLogMessage << " Entering second parallel loop "<<std::endl;
+  // Sum across simd lanes in the plane, breaking out orthog dir.
+  parallel_for(int rt=0;rt<rd;rt++){
+
+    std::vector<int> icoor(Nd);
+
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+
+      iScalar<vector_type> temp; 
+      std::vector<iScalar<scalar_type> > extracted(Nsimd);               
+
+      temp._internal = lvSum[i+Lblock*j+Lblock*Rblock*rt];
+
+      extract(temp,extracted);
+
+      for(int idx=0;idx<Nsimd;idx++){
+
+	grid->iCoorFromIindex(icoor,idx);
+
+	int ldx =rt+icoor[orthogdim]*rd;
+      
+	int ij_dx = i+Lblock*j+Lblock*Rblock*ldx;
+	lsSum[ij_dx]=lsSum[ij_dx]+extracted[idx]._internal;
+
+      }
+    }}
+  }
+
+  std::cout << GridLogMessage << " Entering non parallel loop "<<std::endl;
+  for(int t=0;t<fd;t++)
+  {
+    int pt = t / ld; // processor plane
+    int lt = t % ld;
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+      if (pt == grid->_processor_coor[orthogdim]){
+        int ij_dx = i + Lblock * j + Lblock * Rblock * lt;
+        mat[i+j*Lblock][t] = lsSum[ij_dx];
+      }
+      else{
+        mat[i+j*Lblock][t] = scalar_type(0.0);
+      }
+    }}
+  }
+  std::cout << GridLogMessage << " Done "<<std::endl;
+  // defer sum over nodes.
+  return;
+}
+
+template<class vobj>
+void sliceInnerProductMesonFieldGamma(std::vector< std::vector<ComplexD> > &mat, 
+				      const std::vector<Lattice<vobj> > &lhs,
+				      const std::vector<Lattice<vobj> > &rhs,
+				      int orthogdim,
+				      std::vector<Gamma::Algebra> gammas) 
+{
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  int Lblock = lhs.size();
+  int Rblock = rhs.size();
+
+  GridBase *grid = lhs[0]._grid;
+  
+  const int    Nd = grid->_ndimension;
+  const int Nsimd = grid->Nsimd();
+  int Nt     = grid->GlobalDimensions()[orthogdim];
+  int Ngamma = gammas.size();
+
+  assert(mat.size()==Lblock*Rblock*Ngamma);
+  for(int t=0;t<mat.size();t++){
+    assert(mat[t].size()==Nt);
+  }
+
+  int fd=grid->_fdimensions[orthogdim];
+  int ld=grid->_ldimensions[orthogdim];
+  int rd=grid->_rdimensions[orthogdim];
+
+  // will locally sum vectors first
+  // sum across these down to scalars
+  // splitting the SIMD
+  int MFrvol = rd*Lblock*Rblock*Ngamma;
+  int MFlvol = ld*Lblock*Rblock*Ngamma;
+
+  std::vector<vector_type,alignedAllocator<vector_type> > lvSum(MFrvol);
+  parallel_for (int r = 0; r < MFrvol; r++){
+    lvSum[r] = zero;
+  }
+
+  std::vector<scalar_type > lsSum(MFlvol);             
+  parallel_for (int r = 0; r < MFlvol; r++){
+    lsSum[r]=scalar_type(0.0);
+  }
+
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
+  
+  std::cout << GridLogMessage << " Entering first parallel loop "<<std::endl;
+
+  // Parallelise over t-direction doesn't expose as much parallelism as needed for KNL
+  parallel_for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	for(int i=0;i<Lblock;i++){
+	    auto left = conjugate(lhs[i]._odata[ss]);
+	    for(int j=0;j<Rblock;j++){
+	  for(int mu=0;mu<Ngamma;mu++){
+
+        auto right = Gamma(gammas[mu])*rhs[j]._odata[ss];
+
+	      vector_type vv = left()(0)(0) * right()(0)(0)
+		+              left()(0)(1) * right()(0)(1)
+		+              left()(0)(2) * right()(0)(2)
+		+              left()(1)(0) * right()(1)(0)
+		+              left()(1)(1) * right()(1)(1)
+		+              left()(1)(2) * right()(1)(2)
+		+              left()(2)(0) * right()(2)(0)
+		+              left()(2)(1) * right()(2)(1)
+		+              left()(2)(2) * right()(2)(2)
+		+              left()(3)(0) * right()(3)(0)
+		+              left()(3)(1) * right()(3)(1)
+		+              left()(3)(2) * right()(3)(2);
+
+	      int idx = mu+i*Ngamma+Lblock*Ngamma*j+Ngamma*Lblock*Rblock*r;
+
+	      lvSum[idx]=lvSum[idx]+vv;
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  std::cout << GridLogMessage << " Entering second parallel loop "<<std::endl;
+  // Sum across simd lanes in the plane, breaking out orthog dir.
+  parallel_for(int rt=0;rt<rd;rt++){
+
+    iScalar<vector_type> temp; 
+    std::vector<int> icoor(Nd);
+    std::vector<iScalar<scalar_type> > extracted(Nsimd);               
+
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+    for(int mu=0;mu<Ngamma;mu++){
+
+      int ij_rdx = mu+i*Ngamma+Ngamma*Lblock*j+Ngamma*Lblock*Rblock*rt;
+      temp._internal = lvSum[ij_rdx];
+
+      extract(temp,extracted);
+
+      for(int idx=0;idx<Nsimd;idx++){
+
+	grid->iCoorFromIindex(icoor,idx);
+
+	int ldx =rt+icoor[orthogdim]*rd;
+      
+	int ij_ldx = mu+i*Ngamma+Ngamma*Lblock*j+Ngamma*Lblock*Rblock*ldx;
+	lsSum[ij_ldx]=lsSum[ij_ldx]+extracted[idx]._internal;
+
+      }
+    }}}
+  }
+
+  std::cout << GridLogMessage << " Entering non parallel loop "<<std::endl;
+  for(int t=0;t<fd;t++)
+  {
+    int pt = t / ld; // processor plane
+    int lt = t % ld;
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+    for(int mu=0;mu<Ngamma;mu++){
+      if (pt == grid->_processor_coor[orthogdim]){
+        int ij_dx = mu+i*Ngamma+Ngamma*Lblock*j+Ngamma*Lblock*Rblock* lt;
+        mat[mu+i*Ngamma+j*Lblock*Ngamma][t] = lsSum[ij_dx];
+      }
+      else{
+        mat[mu+i*Ngamma+j*Lblock*Ngamma][t] = scalar_type(0.0);
+      }
+    }}}
+  }
+  std::cout << GridLogMessage << " Done "<<std::endl;
+  // defer sum over nodes.
+  return;
+}
+
+
+template<class vobj>
+void sliceInnerProductMesonFieldGamma1(std::vector< std::vector<ComplexD> > &mat, 
+				      const std::vector<Lattice<vobj> > &lhs,
+				      const std::vector<Lattice<vobj> > &rhs,
+				      int orthogdim,
+				      std::vector<Gamma::Algebra> gammas) 
+{
+
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  typedef iSpinMatrix<vector_type> SpinMatrix_v;
+  typedef iSpinMatrix<scalar_type> SpinMatrix_s;
+  
+  int Lblock = lhs.size();
+  int Rblock = rhs.size();
+
+  GridBase *grid = lhs[0]._grid;
+  
+  const int    Nd = grid->_ndimension;
+  const int Nsimd = grid->Nsimd();
+  int Nt     = grid->GlobalDimensions()[orthogdim];
+  int Ngamma = gammas.size();
+
+  assert(mat.size()==Lblock*Rblock*Ngamma);
+  for(int t=0;t<mat.size();t++){
+    assert(mat[t].size()==Nt);
+  }
+
+  int fd=grid->_fdimensions[orthogdim];
+  int ld=grid->_ldimensions[orthogdim];
+  int rd=grid->_rdimensions[orthogdim];
+
+  // will locally sum vectors first
+  // sum across these down to scalars
+  // splitting the SIMD
+  int MFrvol = rd*Lblock*Rblock;
+  int MFlvol = ld*Lblock*Rblock;
+
+  Vector<SpinMatrix_v > lvSum(MFrvol);
+  parallel_for (int r = 0; r < MFrvol; r++){
+    lvSum[r] = zero;
+  }
+
+  Vector<SpinMatrix_s > lsSum(MFlvol);             
+  parallel_for (int r = 0; r < MFlvol; r++){
+    lsSum[r]=scalar_type(0.0);
+  }
+
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
+  
+  std::cout << GridLogMessage << " Entering first parallel loop "<<std::endl;
+
+  // Parallelise over t-direction doesn't expose as much parallelism as needed for KNL
+  parallel_for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	for(int i=0;i<Lblock;i++){
+
+	  auto left = conjugate(lhs[i]._odata[ss]);
+	  for(int j=0;j<Rblock;j++){
+
+	    SpinMatrix_v vv;
+	    auto right = rhs[j]._odata[ss];
+	    for(int s1=0;s1<Ns;s1++){
+	    for(int s2=0;s2<Ns;s2++){
+	     vv()(s2,s1)() = left()(s1)(0) * right()(s2)(0)
+		+             left()(s1)(1) * right()(s2)(1)
+		+             left()(s1)(2) * right()(s2)(2);
+	    }}
+
+	    int idx = i+Lblock*j+Lblock*Rblock*r;
+
+	    lvSum[idx]=lvSum[idx]+vv;
+	  
+	  }
+	  }
+	}
+      }
+    }
+
+  std::cout << GridLogMessage << " Entering second parallel loop "<<std::endl;
+  // Sum across simd lanes in the plane, breaking out orthog dir.
+  parallel_for(int rt=0;rt<rd;rt++){
+
+    std::vector<int> icoor(Nd);
+    std::vector<SpinMatrix_s> extracted(Nsimd);               
+
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+
+      int ij_rdx = i+Lblock*j+Lblock*Rblock*rt;
+
+      extract(lvSum[ij_rdx],extracted);
+
+      for(int idx=0;idx<Nsimd;idx++){
+
+	grid->iCoorFromIindex(icoor,idx);
+
+	int ldx    = rt+icoor[orthogdim]*rd;
+
+	int ij_ldx = i+Lblock*j+Lblock*Rblock*ldx;
+
+	lsSum[ij_ldx]=lsSum[ij_ldx]+extracted[idx];
+
+      }
+    }}
+  }
+
+  std::cout << GridLogMessage << " Entering third parallel loop "<<std::endl;
+  parallel_for(int t=0;t<fd;t++)
+  {
+    int pt = t / ld; // processor plane
+    int lt = t % ld;
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+      if (pt == grid->_processor_coor[orthogdim]){
+        int ij_dx = i + Lblock * j + Lblock * Rblock * lt;
+    	for(int mu=0;mu<Ngamma;mu++){
+	  mat[mu+i*Ngamma+j*Lblock*Ngamma][t] = trace(lsSum[ij_dx]*Gamma(gammas[mu]));
+	}
+      }
+      else{
+        for(int mu=0;mu<Ngamma;mu++){
+	  mat[mu+i*Ngamma+j*Lblock*Ngamma][t] = scalar_type(0.0);
+	}
+      }
+    }}
+  }
+  std::cout << GridLogMessage << " Done "<<std::endl;
+  // defer sum over nodes.
+  return;
+}
+
+template<class vobj>
+void sliceInnerProductMesonFieldGammaMom(std::vector< std::vector<ComplexD> > &mat, 
+					 const std::vector<Lattice<vobj> > &lhs,
+					 const std::vector<Lattice<vobj> > &rhs,
+					 int orthogdim,
+					 std::vector<Gamma::Algebra> gammas,
+					 const std::vector<LatticeComplex > &mom) 
+{
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  typedef iSpinMatrix<vector_type> SpinMatrix_v;
+  typedef iSpinMatrix<scalar_type> SpinMatrix_s;
+  
+  int Lblock = lhs.size();
+  int Rblock = rhs.size();
+
+  GridBase *grid = lhs[0]._grid;
+  
+  const int    Nd = grid->_ndimension;
+  const int Nsimd = grid->Nsimd();
+  int Nt     = grid->GlobalDimensions()[orthogdim];
+  int Ngamma = gammas.size();
+  int Nmom   = mom.size();
+
+  assert(mat.size()==Lblock*Rblock*Ngamma*Nmom);
+  for(int t=0;t<mat.size();t++){
+    assert(mat[t].size()==Nt);
+  }
+
+  int fd=grid->_fdimensions[orthogdim];
+  int ld=grid->_ldimensions[orthogdim];
+  int rd=grid->_rdimensions[orthogdim];
+
+  // will locally sum vectors first
+  // sum across these down to scalars
+  // splitting the SIMD
+  int MFrvol = rd*Lblock*Rblock*Nmom;
+  int MFlvol = ld*Lblock*Rblock*Nmom;
+
+  Vector<SpinMatrix_v > lvSum(MFrvol);
+  parallel_for (int r = 0; r < MFrvol; r++){
+    lvSum[r] = zero;
+  }
+
+  Vector<SpinMatrix_s > lsSum(MFlvol);             
+  parallel_for (int r = 0; r < MFlvol; r++){
+    lsSum[r]=scalar_type(0.0);
+  }
+
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
+  
+  std::cout << GridLogMessage << " Entering first parallel loop "<<std::endl;
+
+  // Parallelise over t-direction doesn't expose as much parallelism as needed for KNL
+  parallel_for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+
+
+	for(int i=0;i<Lblock;i++){
+
+	  auto left = conjugate(lhs[i]._odata[ss]);
+	  for(int j=0;j<Rblock;j++){
+
+	    SpinMatrix_v vv;
+	    auto right = rhs[j]._odata[ss];
+	    for(int s1=0;s1<Ns;s1++){
+	    for(int s2=0;s2<Ns;s2++){
+	      vv()(s1,s2)() = left()(s1)(0) * right()(s2)(0)
+		+             left()(s1)(1) * right()(s2)(1)
+		+             left()(s1)(2) * right()(s2)(2);
+	    }}
+	    
+	    // After getting the sitewise product do the mom phase loop
+	    int base = Nmom*i+Nmom*Lblock*j+Nmom*Lblock*Rblock*r;
+	    // Trigger unroll
+	    for ( int m=0;m<Nmom;m++){
+	      int idx = m+base;
+	      auto phase = mom[m]._odata[ss];
+	      mac(&lvSum[idx],&vv,&phase);
+	    }
+	  
+	  }
+	}
+      }
+    }
+  }
+
+  std::cout << GridLogMessage << " Entering second parallel loop "<<std::endl;
+  // Sum across simd lanes in the plane, breaking out orthog dir.
+  parallel_for(int rt=0;rt<rd;rt++){
+
+    std::vector<int> icoor(Nd);
+    std::vector<SpinMatrix_s> extracted(Nsimd);               
+
+
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+    for(int m=0;m<Nmom;m++){
+
+      int ij_rdx = m+Nmom*i+Nmom*Lblock*j+Nmom*Lblock*Rblock*rt;
+
+      extract(lvSum[ij_rdx],extracted);
+
+      for(int idx=0;idx<Nsimd;idx++){
+
+	grid->iCoorFromIindex(icoor,idx);
+
+	int ldx    = rt+icoor[orthogdim]*rd;
+
+	int ij_ldx = m+Nmom*i+Nmom*Lblock*j+Nmom*Lblock*Rblock*ldx;
+
+	lsSum[ij_ldx]=lsSum[ij_ldx]+extracted[idx];
+
+      }
+    }}}
+  }
+
+  std::cout << GridLogMessage << " Entering third parallel loop "<<std::endl;
+  parallel_for(int t=0;t<fd;t++)
+  {
+    int pt = t / ld; // processor plane
+    int lt = t % ld;
+    for(int i=0;i<Lblock;i++){
+    for(int j=0;j<Rblock;j++){
+      if (pt == grid->_processor_coor[orthogdim]){
+	for(int m=0;m<Nmom;m++){
+	  int ij_dx = m+Nmom*i + Nmom*Lblock * j + Nmom*Lblock * Rblock * lt;
+	  for(int mu=0;mu<Ngamma;mu++){
+	    mat[ mu
+		+m*Ngamma
+		+i*Nmom*Ngamma
+		+j*Nmom*Ngamma*Lblock][t] = trace(lsSum[ij_dx]*Gamma(gammas[mu]));
+	  }
+	}
+      }
+      else{
+	for(int mu=0;mu<Ngamma;mu++){
+	for(int m=0;m<Nmom;m++){
+	  mat[mu+m*Ngamma+i*Nmom*Ngamma+j*Nmom*Lblock*Ngamma][t] = scalar_type(0.0);
+	}}
+      }
+    }}
+  }
+  std::cout << GridLogMessage << " Done "<<std::endl;
+  // defer sum over nodes.
+  return;
+}
+
+
+
+/*
+template void sliceInnerProductMesonField<SpinColourVector>(std::vector< std::vector<ComplexD> > &mat, 
+						   const std::vector<Lattice<SpinColourVector> > &lhs,
+						   const std::vector<Lattice<SpinColourVector> > &rhs,
+						   int orthogdim) ;
+*/
+
+std::vector<Gamma::Algebra> Gmu4 ( {
+  Gamma::Algebra::GammaX,
+  Gamma::Algebra::GammaY,
+  Gamma::Algebra::GammaZ,
+  Gamma::Algebra::GammaT });
+
+std::vector<Gamma::Algebra> Gmu16 ( {
+  Gamma::Algebra::Gamma5,
+  Gamma::Algebra::GammaT,
+  Gamma::Algebra::GammaTGamma5,
+  Gamma::Algebra::GammaX,
+  Gamma::Algebra::GammaXGamma5,
+  Gamma::Algebra::GammaY,
+  Gamma::Algebra::GammaYGamma5,
+  Gamma::Algebra::GammaZ,
+  Gamma::Algebra::GammaZGamma5,
+  Gamma::Algebra::Identity,
+  Gamma::Algebra::SigmaXT,
+  Gamma::Algebra::SigmaXY,
+  Gamma::Algebra::SigmaXZ,
+  Gamma::Algebra::SigmaYT,
+  Gamma::Algebra::SigmaYZ,
+  Gamma::Algebra::SigmaZT
+});
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+  GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
+  
+  const int Nmom=7;
+  int nt = latt_size[Tp];
+  uint64_t vol = 1;
+  for(int d=0;d<Nd;d++){
+    vol = vol*latt_size[d];
+  }
+  
+  std::vector<int> seeds({1,2,3,4});
+  GridParallelRNG          pRNG(&Grid);
+  pRNG.SeedFixedIntegers(seeds);
+
+
+  int Nm = atoi(argv[1]); // number of all modes (high + low)
+
+  std::vector<LatticeFermion> v(Nm,&Grid);
+  std::vector<LatticeFermion> w(Nm,&Grid);
+  std::vector<LatticeFermion> gammaV(Nm,&Grid);
+  std::vector<LatticeComplex> phases(Nmom,&Grid);
+
+  for(int i=0;i<Nm;i++) { 
+    random(pRNG,v[i]);
+    random(pRNG,w[i]);
+  }
+
+  for(int i=0;i<Nmom;i++) { 
+    phases[i] = Complex(1.0);
+  }
+
+  double flops = vol * (11.0 * 8.0 + 6.0) * Nm*Nm;
+  double byte  = vol * (12.0 * sizeof(Complex) ) * Nm*Nm;
+
+  std::vector<ComplexD> ip(nt);
+  std::vector<std::vector<ComplexD> > MesonFields   (Nm*Nm);
+  std::vector<std::vector<ComplexD> > MesonFields4  (Nm*Nm*4);
+  std::vector<std::vector<ComplexD> > MesonFields16 (Nm*Nm*16);
+  std::vector<std::vector<ComplexD> > MesonFields161(Nm*Nm*16);
+  std::vector<std::vector<ComplexD> > MesonFields16mom (Nm*Nm*16*Nmom);
+  std::vector<std::vector<ComplexD> > MesonFieldsRef(Nm*Nm);
+
+  for(int i=0;i<MesonFields.size();i++   )  MesonFields   [i].resize(nt);
+  for(int i=0;i<MesonFieldsRef.size();i++)  MesonFieldsRef[i].resize(nt);
+  for(int i=0;i<MesonFields4.size();i++  )  MesonFields4  [i].resize(nt);
+  for(int i=0;i<MesonFields16.size();i++ )  MesonFields16 [i].resize(nt);
+  for(int i=0;i<MesonFields161.size();i++ ) MesonFields161[i].resize(nt);
+
+  for(int i=0;i<MesonFields16mom.size();i++ ) MesonFields16mom [i].resize(nt);
+
+  GridLogMessage.TimingMode(1);
+
+  std::cout<<GridLogMessage << "Running loop with sliceInnerProductVector"<<std::endl;
+  double t0 = usecond();
+  for(int i=0;i<Nm;i++) { 
+  for(int j=0;j<Nm;j++) { 
+    sliceInnerProductVector(ip, w[i],v[j],Tp);
+    for(int t=0;t<nt;t++){
+      MesonFieldsRef[i+j*Nm][t] = ip[t];
+    }
+  }}
+  double t1 = usecond();
+  std::cout<<GridLogMessage << "Done "<< (t1-t0) <<" usecond " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< flops/(t1-t0) <<" mflops " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< byte /(t1-t0) <<" MB/s " <<std::endl;
+
+  std::cout<<GridLogMessage << "Running loop with new code for Nt="<<nt<<std::endl;
+  t0 = usecond();
+  sliceInnerProductMesonField(MesonFields,w,v,Tp);
+  t1 = usecond();
+  std::cout<<GridLogMessage << "Done "<< (t1-t0) <<" usecond " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< flops/(t1-t0) <<" mflops " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< byte /(t1-t0) <<" MB/s " <<std::endl;
+
+
+  std::cout<<GridLogMessage << "Running loop with Four gammas code for Nt="<<nt<<std::endl;
+  flops = vol * (11.0 * 8.0 + 6.0) * Nm*Nm*4;
+  byte  = vol * (12.0 * sizeof(Complex) ) * Nm*Nm
+        + vol * ( 2.0 * sizeof(Complex) ) * Nm*Nm* 4;
+  t0 = usecond();
+  sliceInnerProductMesonFieldGamma(MesonFields4,w,v,Tp,Gmu4);
+  t1 = usecond();
+  std::cout<<GridLogMessage << "Done "<< (t1-t0) <<" usecond " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< flops/(t1-t0) <<" mflops " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< byte /(t1-t0) <<" MB/s " <<std::endl;
+
+  std::cout<<GridLogMessage << "Running loop with Sixteen gammas code for Nt="<<nt<<std::endl;
+  flops = vol * (11.0 * 8.0 + 6.0) * Nm*Nm*16;
+  byte  = vol * (12.0 * sizeof(Complex) ) * Nm*Nm
+        + vol * ( 2.0 * sizeof(Complex) ) * Nm*Nm* 16;
+  t0 = usecond();
+  sliceInnerProductMesonFieldGamma(MesonFields16,w,v,Tp,Gmu16);
+  t1 = usecond();
+  std::cout<<GridLogMessage << "Done "<< (t1-t0) <<" usecond " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< flops/(t1-t0) <<" mflops " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< byte /(t1-t0) <<" MB/s " <<std::endl;
+
+
+  std::cout<<GridLogMessage << "Running loop with Sixteen gammas code1 for Nt="<<nt<<std::endl;
+  flops = vol * ( 2 * 8.0 + 6.0) * Nm*Nm*16;
+  byte  = vol * (12.0 * sizeof(Complex) ) * Nm*Nm
+        + vol * ( 2.0 * sizeof(Complex) ) * Nm*Nm* 16;
+  t0 = usecond();
+  sliceInnerProductMesonFieldGamma1(MesonFields161, w, v, Tp, Gmu16);
+  t1 = usecond();
+  std::cout<<GridLogMessage << "Done "<< (t1-t0) <<" usecond " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< flops/(t1-t0) <<" mflops " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< byte /(t1-t0) <<" MB/s " <<std::endl;
+
+  std::cout<<GridLogMessage << "Running loop with Sixteen gammas "<<Nmom<<" momenta "<<std::endl;
+  flops = vol * ( 2 * 8.0 + 6.0 + 8.0*Nmom) * Nm*Nm*16;
+  byte  = vol * (12.0 * sizeof(Complex) ) * Nm*Nm
+        + vol * ( 2.0 * sizeof(Complex) *Nmom ) * Nm*Nm* 16;
+  t0 = usecond();
+  sliceInnerProductMesonFieldGammaMom(MesonFields16mom,w,v,Tp,Gmu16,phases);
+  t1 = usecond();
+  std::cout<<GridLogMessage << "Done "<< (t1-t0) <<" usecond " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< flops/(t1-t0) <<" mflops " <<std::endl;
+  std::cout<<GridLogMessage << "Done "<< byte /(t1-t0) <<" MB/s " <<std::endl;
+
+
+
+  RealD err = 0;
+  RealD err2 = 0;
+  ComplexD diff;
+  ComplexD diff2;
+
+  for(int i=0;i<Nm;i++) { 
+  for(int j=0;j<Nm;j++) { 
+    for(int t=0;t<nt;t++){
+      diff = MesonFields[i+Nm*j][t] - MesonFieldsRef[i+Nm*j][t];
+      err += real(diff*conj(diff));
+    }
+  }}
+  std::cout<<GridLogMessage << "Norm error "<< err <<std::endl;
+  
+  err = err*0.;
+  diff = diff*0.;
+
+  for (int mu = 0; mu < 16; mu++){
+    for (int k = 0; k < gammaV.size(); k++){
+      gammaV[k] = Gamma(Gmu16[mu]) * v[k];
+    }
+    for (int i = 0; i < Nm; i++){
+      for (int j = 0; j < Nm; j++){
+        sliceInnerProductVector(ip, w[i], gammaV[j], Tp);
+        for (int t = 0; t < nt; t++){
+          MesonFields[i + j * Nm][t] = ip[t];
+          diff = MesonFields16[mu+i*16+Nm*16*j][t] - MesonFields161[mu+i*16+Nm*16*j][t];
+          diff2 = MesonFields[i+j*Nm][t] - MesonFields161[mu+i*16+Nm*16*j][t];
+          err += real(diff*conj(diff));
+          err2 += real(diff2*conj(diff2));
+        }
+      }
+    }
+  }
+  std::cout << GridLogMessage << "Norm error 16 gamma1/16 gamma naive    " << err << std::endl;
+  std::cout << GridLogMessage << "Norm error 16 gamma1/sliceInnerProduct " << err2 << std::endl;
+
+  Grid_finalize();
+}
+

benchmarks/Benchmark_staggered.cc

@@ -40,7 +40,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;

benchmarks/Benchmark_su3.cc

@@ -36,13 +36,15 @@ int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 #define LMAX (32)
+#define LMIN (16)
+#define LINC (4)
 
-  int Nloop=200;
+  int64_t Nloop=2000;
 
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
 
-  int threads = GridThread::GetThreads();
+  int64_t threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
@@ -51,19 +53,19 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=LMAX;lat+=2){
+  for(int lat=LMIN;lat<=LMAX;lat+=LINC){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid);// random(pRNG,z);
-      LatticeColourMatrix x(&Grid);// random(pRNG,x);
-      LatticeColourMatrix y(&Grid);// random(pRNG,y);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
       double start=usecond();
-      for(int i=0;i<Nloop;i++){
+      for(int64_t i=0;i<Nloop;i++){
 	x=x*y;
       }
       double stop=usecond();
@@ -83,20 +85,20 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=LMAX;lat+=2){
+  for(int lat=LMIN;lat<=LMAX;lat+=LINC){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid); //random(pRNG,z);
-      LatticeColourMatrix x(&Grid); //random(pRNG,x);
-      LatticeColourMatrix y(&Grid); //random(pRNG,y);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
       double start=usecond();
-      for(int i=0;i<Nloop;i++){
+      for(int64_t i=0;i<Nloop;i++){
 	z=x*y;
       }
       double stop=usecond();
@@ -114,20 +116,20 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=LMAX;lat+=2){
+  for(int lat=LMIN;lat<=LMAX;lat+=LINC){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid); //random(pRNG,z);
-      LatticeColourMatrix x(&Grid); //random(pRNG,x);
-      LatticeColourMatrix y(&Grid); //random(pRNG,y);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
       double start=usecond();
-      for(int i=0;i<Nloop;i++){
+      for(int64_t i=0;i<Nloop;i++){
 	mult(z,x,y);
       }
       double stop=usecond();
@@ -145,30 +147,107 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=LMAX;lat+=2){
+  for(int lat=LMIN;lat<=LMAX;lat+=LINC){
+    
+    std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
+    int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+    
+    GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+    GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
+    
+    LatticeColourMatrix z(&Grid); random(pRNG,z);
+    LatticeColourMatrix x(&Grid); random(pRNG,x);
+    LatticeColourMatrix y(&Grid); random(pRNG,y);
+    
+    double start=usecond();
+    for(int64_t i=0;i<Nloop;i++){
+      mac(z,x,y);
+    }
+    double stop=usecond();
+    double time = (stop-start)/Nloop*1000.0;
+    
+    double bytes=3*vol*Nc*Nc*sizeof(Complex);
+    double flops=Nc*Nc*(6+8+8)*vol;
+    std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t" << flops/time<<std::endl;
+    
+  }
+
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking SU3xSU3  CovShiftForward(z,x,y)"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
+  std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+
+  for(int lat=LMIN;lat<=LMAX;lat+=LINC){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid); //random(pRNG,z);
-      LatticeColourMatrix x(&Grid); //random(pRNG,x);
-      LatticeColourMatrix y(&Grid); //random(pRNG,y);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
-      double start=usecond();
-      for(int i=0;i<Nloop;i++){
-	mac(z,x,y);
+      for(int mu=0;mu<4;mu++){
+	      double start=usecond();
+	      for(int64_t i=0;i<Nloop;i++){
+	        z = PeriodicBC::CovShiftForward(x,mu,y);
+	    }
+	    double stop=usecond();
+	    double time = (stop-start)/Nloop*1000.0;
+	
+	
+	    double bytes=3*vol*Nc*Nc*sizeof(Complex);
+	    double flops=Nc*Nc*(6+8+8)*vol;
+	    std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t" << flops/time<<std::endl;
       }
-      double stop=usecond();
-      double time = (stop-start)/Nloop*1000.0;
-      
-      double bytes=3*vol*Nc*Nc*sizeof(Complex);
-      double flops=Nc*Nc*(8+8+8)*vol;
-      std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t" << flops/time<<std::endl;
+  }
+#if 1
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking SU3xSU3  z= x * Cshift(y)"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
+  std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
+  for(int lat=LMIN;lat<=LMAX;lat+=LINC){
+      std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
+
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
+      LatticeColourMatrix tmp(&Grid);
+
+      for(int mu=0;mu<4;mu++){
+	double tshift=0;
+	double tmult =0;
+
+	double start=usecond();
+	for(int64_t i=0;i<Nloop;i++){
+	  tshift-=usecond();
+	  tmp = Cshift(y,mu,-1);
+	  tshift+=usecond();
+	  tmult-=usecond();
+	  z   = x*tmp;
+	  tmult+=usecond();
+	}
+	double stop=usecond();
+	double time = (stop-start)/Nloop;
+	tshift = tshift/Nloop;
+	tmult  = tmult /Nloop;
+	
+	double bytes=3*vol*Nc*Nc*sizeof(Complex);
+	double flops=Nc*Nc*(6+8+8)*vol;
+	std::cout<<GridLogMessage<<std::setprecision(3) << "total us "<<time<<" shift "<<tshift <<" mult "<<tmult<<std::endl;
+	time = time * 1000; // convert to NS for GB/s
+	std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t" << flops/time<<std::endl;
+      }
     }
-
+#endif
   Grid_finalize();
 }

benchmarks/Benchmark_wilson.cc

@@ -4,7 +4,7 @@
 
     Source file: ./benchmarks/Benchmark_wilson.cc
 
-    Copyright (C) 2015
+    Copyright (C) 2018
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
@@ -32,6 +32,9 @@ using namespace std;
 using namespace Grid;
 using namespace Grid::QCD;
 
+
+#include "Grid/util/Profiling.h"
+
 template<class d>
 struct scal {
   d internal;
@@ -45,6 +48,7 @@ struct scal {
   };
 
 bool overlapComms = false;
+bool perfProfiling = false;
 
 int main (int argc, char ** argv)
 {
@@ -53,18 +57,29 @@ int main (int argc, char ** argv)
   if( GridCmdOptionExists(argv,argv+argc,"--asynch") ){
     overlapComms = true;
   }
+  if( GridCmdOptionExists(argv,argv+argc,"--perf") ){
+    perfProfiling = true;
+  }
+
+  long unsigned int single_site_flops = 8*QCD::Nc*(7+16*QCD::Nc);
+
 
   std::vector<int> latt_size   = GridDefaultLatt();
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
-  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  GridLogLayout();
+
   std::cout<<GridLogMessage << "Grid floating point word size is REALF"<< sizeof(RealF)<<std::endl;
   std::cout<<GridLogMessage << "Grid floating point word size is REALD"<< sizeof(RealD)<<std::endl;
   std::cout<<GridLogMessage << "Grid floating point word size is REAL"<< sizeof(Real)<<std::endl;
+  std::cout<<GridLogMessage << "Grid number of colours : "<< QCD::Nc <<std::endl;
+  std::cout<<GridLogMessage << "Benchmarking Wilson operator in the fundamental representation" << std::endl;
+
 
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);
@@ -134,9 +149,25 @@ int main (int argc, char ** argv)
     Dw.Dhop(src,result,0);
   }
   double t1=usecond();
-  double flops=1344*volume*ncall;
+  double flops=single_site_flops*volume*ncall;
   
+  if (perfProfiling){
+  std::cout<<GridLogMessage << "Profiling Dw with perf"<<std::endl;
+    
+  System::profile("kernel", [&]() {
+    for(int i=0;i<ncall;i++){
+      Dw.Dhop(src,result,0);
+    }
+  });
+
+  std::cout<<GridLogMessage << "Generated kernel.data"<<std::endl;
+  std::cout<<GridLogMessage << "Use with: perf report -i kernel.data"<<std::endl;
+
+  }
+
+
   std::cout<<GridLogMessage << "Called Dw"<<std::endl;
+  std::cout<<GridLogMessage << "flops per site " << single_site_flops << std::endl;
   std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
   std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
   std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;

benchmarks/Benchmark_wilson_sweep.cc

@@ -62,6 +62,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Number of colours "<< QCD::Nc <<std::endl;
   std::cout << GridLogMessage<< "* Benchmarking WilsonFermionR::Dhop                  "<<std::endl;
   std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
   if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
@@ -69,13 +70,15 @@ int main (int argc, char ** argv)
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage << "* OpenMP threads       : "<< GridThread::GetThreads() <<std::endl;
+  std::cout << GridLogMessage << "* MPI tasks            : "<< GridCmdVectorIntToString(mpi_layout) << std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
 
-  std::cout<<GridLogMessage << "============================================================================="<< std::endl;
-  std::cout<<GridLogMessage << "= Benchmarking Wilson" << std::endl;
-  std::cout<<GridLogMessage << "============================================================================="<< std::endl;
-  std::cout<<GridLogMessage << "Volume\t\t\tWilson/MFLOPs\tWilsonDag/MFLOPs" << std::endl;
-  std::cout<<GridLogMessage << "============================================================================="<< std::endl;
+  std::cout<<GridLogMessage << "================================================================================================="<< std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking Wilson operator in the fundamental representation" << std::endl;
+  std::cout<<GridLogMessage << "================================================================================================="<< std::endl;
+  std::cout<<GridLogMessage << "Volume\t\t\tWilson/MFLOPs\tWilsonDag/MFLOPs\tWilsonEO/MFLOPs\tWilsonDagEO/MFLOPs" << std::endl;
+  std::cout<<GridLogMessage << "================================================================================================="<< std::endl;
 
   int Lmax = 32;
   int dmin = 0;
@@ -93,17 +96,24 @@ int main (int argc, char ** argv)
 	  std::cout << latt_size.back() << "\t\t";
 
 	  GridCartesian           Grid(latt_size,simd_layout,mpi_layout);
-	  GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout);
+	  GridRedBlackCartesian RBGrid(&Grid);
 
 	  GridParallelRNG  pRNG(&Grid); pRNG.SeedFixedIntegers(seeds);
 	  LatticeGaugeField Umu(&Grid); random(pRNG,Umu);
-	  LatticeFermion    src(&Grid); random(pRNG,src);
-	  LatticeFermion result(&Grid); result=zero;
+	  LatticeFermion        src(&Grid); random(pRNG,src);
+	  LatticeFermion    src_o(&RBGrid); pickCheckerboard(Odd,src_o,src);
+	  LatticeFermion     result(&Grid); result=zero;
+	  LatticeFermion result_e(&RBGrid); result_e=zero;
 
 	  double volume = std::accumulate(latt_size.begin(),latt_size.end(),1,std::multiplies<int>());
 
 	  WilsonFermionR Dw(Umu,Grid,RBGrid,mass,params);
-      
+
+    // Full operator      
+	  bench_wilson(src,result,Dw,volume,DaggerNo);
+	  bench_wilson(src,result,Dw,volume,DaggerYes);
+    std::cout << "\t";
+    // EO
 	  bench_wilson(src,result,Dw,volume,DaggerNo);
 	  bench_wilson(src,result,Dw,volume,DaggerYes);
 	  std::cout << std::endl;
@@ -122,9 +132,26 @@ void bench_wilson (
 		   int const           dag )
 {
   int ncall    = 1000;
+  long unsigned int single_site_flops = 8*QCD::Nc*(7+16*QCD::Nc);
   double t0    = usecond();
   for(int i=0; i<ncall; i++) { Dw.Dhop(src,result,dag); }
   double t1    = usecond();
-  double flops = 1344 * volume * ncall;
+  double flops = single_site_flops * volume * ncall;
+  std::cout << flops/(t1-t0) << "\t\t";
+}
+
+void bench_wilson_eo (
+		   LatticeFermion &    src,
+		   LatticeFermion & result,
+		   WilsonFermionR &     Dw,
+		   double const     volume,
+		   int const           dag )
+{
+  int ncall    = 1000;
+  long unsigned int single_site_flops = 8*QCD::Nc*(7+16*QCD::Nc);
+  double t0    = usecond();
+  for(int i=0; i<ncall; i++) { Dw.DhopEO(src,result,dag); }
+  double t1    = usecond();
+  double flops = (single_site_flops * volume * ncall)/2.0;
   std::cout << flops/(t1-t0) << "\t\t";
 }

bootstrap.sh

@@ -1,11 +1,9 @@
-]#!/usr/bin/env bash
+#!/usr/bin/env bash
 
-EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2'
+EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.3.5.tar.bz2'
 
 echo "-- deploying Eigen source..."
-wget ${EIGEN_URL} --no-check-certificate
-./scripts/update_eigen.sh `basename ${EIGEN_URL}`
-rm `basename ${EIGEN_URL}`
+wget ${EIGEN_URL} --no-check-certificate && ./scripts/update_eigen.sh `basename ${EIGEN_URL}` && rm `basename ${EIGEN_URL}`
 
 echo '-- generating Make.inc files...'
 ./scripts/filelist

configure.ac

@@ -13,6 +13,10 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 ################ Get git info
 #AC_REVISION([m4_esyscmd_s([./scripts/configure.commit])])
 
+################ Set flags
+# do not move!
+CXXFLAGS="-O3 $CXXFLAGS"
+
 ############### Checks for programs
 AC_PROG_CXX
 AC_PROG_RANLIB
@@ -27,7 +31,6 @@ AX_GXX_VERSION
 AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
       [version of g++ that will compile the code])
 
-CXXFLAGS="-O3 $CXXFLAGS"
 
 
 ############### Checks for typedefs, structures, and compiler characteristics
@@ -51,9 +54,14 @@ AC_CHECK_HEADERS(malloc/malloc.h)
 AC_CHECK_HEADERS(malloc.h)
 AC_CHECK_HEADERS(endian.h)
 AC_CHECK_HEADERS(execinfo.h)
+AC_CHECK_HEADERS(numaif.h)
 AC_CHECK_DECLS([ntohll],[], [], [[#include <arpa/inet.h>]])
 AC_CHECK_DECLS([be64toh],[], [], [[#include <arpa/inet.h>]])
 
+############## Standard libraries
+AC_CHECK_LIB([m],[cos])
+AC_CHECK_LIB([stdc++],[abort])
+
 ############### GMP and MPFR
 AC_ARG_WITH([gmp],
     [AS_HELP_STRING([--with-gmp=prefix],
@@ -184,6 +192,15 @@ AC_SEARCH_LIBS([limeCreateReader], [lime],
 In order to use ILGG file format please install or provide the correct path to your installation
 Info at: http://usqcd.jlab.org/usqcd-docs/c-lime/)])
 
+AC_SEARCH_LIBS([crc32], [z],
+               [AC_DEFINE([HAVE_ZLIB], [1], [Define to 1 if you have the `LIBZ' library])]
+               [have_zlib=true] [LIBS="${LIBS} -lz"],
+	       [AC_MSG_ERROR(zlib library was not found in your system.)])
+
+AC_SEARCH_LIBS([move_pages], [numa],
+               [AC_DEFINE([HAVE_LIBNUMA], [1], [Define to 1 if you have the `LIBNUMA' library])]
+               [have_libnuma=true] [LIBS="${LIBS} -lnuma"],
+	       [AC_MSG_WARN(libnuma library was not found in your system. Some optimisations will not apply)])
 
 AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp],
                [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])]
@@ -232,11 +249,15 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
+      SKL)
+        AC_DEFINE([AVX512],[1],[AVX512 intrinsics for SkyLake Xeon])
+        SIMD_FLAGS='-march=skylake-avx512';;
       KNC)
         AC_DEFINE([IMCI],[1],[IMCI intrinsics for Knights Corner])
         SIMD_FLAGS='';;
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
+        AC_DEFINE([KNL],[1],[Knights landing processor])
         SIMD_FLAGS='-march=knl';;
       GEN)
         AC_DEFINE([GEN],[1],[generic vector code])
@@ -244,6 +265,9 @@ case ${ax_cv_cxx_compiler_vendor} in
                            [generic SIMD vector width (in bytes)])
         SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)"
         SIMD_FLAGS='';;
+      NEONv8)
+        AC_DEFINE([NEONV8],[1],[ARMv8 NEON])
+        SIMD_FLAGS='-march=armv8-a';;
       QPX|BGQ)
         AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q])
         SIMD_FLAGS='';;
@@ -272,6 +296,7 @@ case ${ax_cv_cxx_compiler_vendor} in
         SIMD_FLAGS='';;
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing])
+        AC_DEFINE([KNL],[1],[Knights landing processor])
         SIMD_FLAGS='-xmic-avx512';;
       GEN)
         AC_DEFINE([GEN],[1],[generic vector code])
@@ -309,10 +334,47 @@ case ${ac_PRECISION} in
      double)
        AC_DEFINE([GRID_DEFAULT_PRECISION_DOUBLE],[1],[GRID_DEFAULT_PRECISION is DOUBLE] )
      ;;
+     *)
+     AC_MSG_ERROR([${ac_PRECISION} unsupported --enable-precision option]);
+     ;;
 esac
 
+######################  Shared memory allocation technique under MPI3
+AC_ARG_ENABLE([shm],[AC_HELP_STRING([--enable-shm=shmopen|shmget|hugetlbfs|shmnone],
+              [Select SHM allocation technique])],[ac_SHM=${enable_shm}],[ac_SHM=shmopen])
+
+case ${ac_SHM} in
+
+     shmopen)
+     AC_DEFINE([GRID_MPI3_SHMOPEN],[1],[GRID_MPI3_SHMOPEN] )
+     ;;
+
+     shmget)
+     AC_DEFINE([GRID_MPI3_SHMGET],[1],[GRID_MPI3_SHMGET] )
+     ;;
+
+     shmnone)
+     AC_DEFINE([GRID_MPI3_SHM_NONE],[1],[GRID_MPI3_SHM_NONE] )
+     ;;
+
+     hugetlbfs)
+     AC_DEFINE([GRID_MPI3_SHMMMAP],[1],[GRID_MPI3_SHMMMAP] )
+     ;;
+
+     *)
+     AC_MSG_ERROR([${ac_SHM} unsupported --enable-shm option]);
+     ;;
+esac
+
+######################  Shared base path for SHMMMAP
+AC_ARG_ENABLE([shmpath],[AC_HELP_STRING([--enable-shmpath=path],
+              [Select SHM mmap base path for hugetlbfs])],
+	      [ac_SHMPATH=${enable_shmpath}],
+	      [ac_SHMPATH=/var/lib/hugetlbfs/global/pagesize-2MB/])
+AC_DEFINE_UNQUOTED([GRID_SHM_PATH],["$ac_SHMPATH"],[Path to a hugetlbfs filesystem for MMAPing])
+
 ############### communication type selection
-AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi3-auto|shmem],
+AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto],
               [Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
 
 case ${ac_COMMS} in
@@ -320,22 +382,10 @@ case ${ac_COMMS} in
         AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
         comms_type='none'
      ;;
-     mpi3l*)
-       AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] )
-       comms_type='mpi3l'
-     ;;
-     mpi3*)
+     mpi*)
         AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
         comms_type='mpi3'
      ;;
-     mpi*)
-        AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
-        comms_type='mpi'
-     ;;
-     shmem)
-        AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
-        comms_type='shmem'
-     ;;
      *)
         AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]);
      ;;
@@ -355,7 +405,7 @@ esac
 AM_CONDITIONAL(BUILD_COMMS_SHMEM, [ test "${comms_type}X" == "shmemX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI,   [ test "${comms_type}X" == "mpiX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI3,  [ test "${comms_type}X" == "mpi3X" ] )
-AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] )
+AM_CONDITIONAL(BUILD_COMMS_MPIT,  [ test "${comms_type}X" == "mpitX" ] )
 AM_CONDITIONAL(BUILD_COMMS_NONE,  [ test "${comms_type}X" == "noneX" ])
 
 ############### RNG selection
@@ -430,8 +480,8 @@ GRID_LIBS=$LIBS
 GRID_SHORT_SHA=`git rev-parse --short HEAD`
 GRID_SHA=`git rev-parse HEAD`
 GRID_BRANCH=`git rev-parse --abbrev-ref HEAD`
-AM_CXXFLAGS="-I${abs_srcdir}/include $AM_CXXFLAGS"
-AM_CFLAGS="-I${abs_srcdir}/include $AM_CFLAGS"
+AM_CXXFLAGS="-I${abs_srcdir}/include -I${abs_srcdir}/Eigen/  -I${abs_srcdir}/Eigen/unsupported $AM_CXXFLAGS"
+AM_CFLAGS="-I${abs_srcdir}/include -I${abs_srcdir}/Eigen/  -I${abs_srcdir}/Eigen/unsupported $AM_CFLAGS"
 AM_LDFLAGS="-L${cwd}/lib $AM_LDFLAGS"
 AC_SUBST([AM_CFLAGS])
 AC_SUBST([AM_CXXFLAGS])
@@ -460,6 +510,8 @@ compiler version            : ${ax_cv_gxx_version}
 SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
 Threading                   : ${ac_openmp}
 Communications type         : ${comms_type}
+Shared memory allocator     : ${ac_SHM}
+Shared memory mmap path     : ${ac_SHMPATH}
 Default precision           : ${ac_PRECISION}
 Software FP16 conversion    : ${ac_SFW_FP16}
 RNG choice                  : ${ac_RNG}
@@ -493,6 +545,7 @@ AC_CONFIG_FILES(tests/forces/Makefile)
 AC_CONFIG_FILES(tests/hadrons/Makefile)
 AC_CONFIG_FILES(tests/hmc/Makefile)
 AC_CONFIG_FILES(tests/solver/Makefile)
+AC_CONFIG_FILES(tests/lanczos/Makefile)
 AC_CONFIG_FILES(tests/smearing/Makefile)
 AC_CONFIG_FILES(tests/qdpxx/Makefile)
 AC_CONFIG_FILES(tests/testu01/Makefile)

extras/Hadrons/AllToAllReduction.hpp

@@ -0,0 +1,146 @@
+#ifndef A2A_Reduction_hpp_
+#define A2A_Reduction_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Environment.hpp>
+#include <Grid/Hadrons/Solver.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+////////////////////////////////////////////
+// A2A Meson Field Inner Product
+////////////////////////////////////////////
+
+template <class FermionField>
+void sliceInnerProductMesonField(std::vector<std::vector<ComplexD>> &mat,
+                                 const std::vector<Lattice<FermionField>> &lhs,
+                                 const std::vector<Lattice<FermionField>> &rhs,
+                                 int orthogdim)
+{
+    typedef typename FermionField::scalar_type scalar_type;
+    typedef typename FermionField::vector_type vector_type;
+
+    int Lblock = lhs.size();
+    int Rblock = rhs.size();
+
+    GridBase *grid = lhs[0]._grid;
+
+    const int Nd = grid->_ndimension;
+    const int Nsimd = grid->Nsimd();
+    int Nt = grid->GlobalDimensions()[orthogdim];
+
+    assert(mat.size() == Lblock * Rblock);
+    for (int t = 0; t < mat.size(); t++)
+    {
+        assert(mat[t].size() == Nt);
+    }
+
+    int fd = grid->_fdimensions[orthogdim];
+    int ld = grid->_ldimensions[orthogdim];
+    int rd = grid->_rdimensions[orthogdim];
+
+    // will locally sum vectors first
+    // sum across these down to scalars
+    // splitting the SIMD
+    std::vector<vector_type, alignedAllocator<vector_type>> lvSum(rd * Lblock * Rblock);
+    for(int r=0;r<rd * Lblock * Rblock;r++)
+    {
+        lvSum[r]=zero;
+    }
+    std::vector<scalar_type> lsSum(ld * Lblock * Rblock, scalar_type(0.0));
+
+    int e1 = grid->_slice_nblock[orthogdim];
+    int e2 = grid->_slice_block[orthogdim];
+    int stride = grid->_slice_stride[orthogdim];
+
+    // std::cout << GridLogMessage << " Entering first parallel loop " << std::endl;
+    // Parallelise over t-direction doesn't expose as much parallelism as needed for KNL
+    parallel_for(int r = 0; r < rd; r++)
+    {
+        int so = r * grid->_ostride[orthogdim]; // base offset for start of plane
+        for (int n = 0; n < e1; n++)
+        {
+            for (int b = 0; b < e2; b++)
+            {
+                int ss = so + n * stride + b;
+                for (int i = 0; i < Lblock; i++)
+                {
+                    auto left = conjugate(lhs[i]._odata[ss]);
+                    for (int j = 0; j < Rblock; j++)
+                    {
+                        int idx = i + Lblock * j + Lblock * Rblock * r;
+                        auto right = rhs[j]._odata[ss];
+                        vector_type vv = left()(0)(0) * right()(0)(0) 
+                                       + left()(0)(1) * right()(0)(1) 
+                                       + left()(0)(2) * right()(0)(2) 
+                                       + left()(1)(0) * right()(1)(0) 
+                                       + left()(1)(1) * right()(1)(1) 
+                                       + left()(1)(2) * right()(1)(2) 
+                                       + left()(2)(0) * right()(2)(0) 
+                                       + left()(2)(1) * right()(2)(1) 
+                                       + left()(2)(2) * right()(2)(2) 
+                                       + left()(3)(0) * right()(3)(0) 
+                                       + left()(3)(1) * right()(3)(1) 
+                                       + left()(3)(2) * right()(3)(2);
+
+                        lvSum[idx] = lvSum[idx] + vv;
+                    }
+                }
+            }
+        }
+    }
+
+    // std::cout << GridLogMessage << " Entering second parallel loop " << std::endl;
+    // Sum across simd lanes in the plane, breaking out orthog dir.
+    parallel_for(int rt = 0; rt < rd; rt++)
+    {
+        std::vector<int> icoor(Nd);
+        for (int i = 0; i < Lblock; i++)
+        {
+            for (int j = 0; j < Rblock; j++)
+            {
+                iScalar<vector_type> temp;
+                std::vector<iScalar<scalar_type>> extracted(Nsimd);
+                temp._internal = lvSum[i + Lblock * j + Lblock * Rblock * rt];
+                extract(temp, extracted);
+                for (int idx = 0; idx < Nsimd; idx++)
+                {
+                    grid->iCoorFromIindex(icoor, idx);
+                    int ldx = rt + icoor[orthogdim] * rd;
+                    int ij_dx = i + Lblock * j + Lblock * Rblock * ldx;
+                    lsSum[ij_dx] = lsSum[ij_dx] + extracted[idx]._internal;
+                }
+            }
+        }
+    }
+
+    // std::cout << GridLogMessage << " Entering non parallel loop " << std::endl;
+    for (int t = 0; t < fd; t++)
+    {
+        int pt = t/ld; // processor plane
+        int lt = t%ld;
+        for (int i = 0; i < Lblock; i++)
+        {
+            for (int j = 0; j < Rblock; j++)
+            {
+                if (pt == grid->_processor_coor[orthogdim])
+                {
+                    int ij_dx = i + Lblock * j + Lblock * Rblock * lt;
+                    mat[i + j * Lblock][t] = lsSum[ij_dx];
+                }
+                else
+                {
+                    mat[i + j * Lblock][t] = scalar_type(0.0);
+                }
+                
+            }
+        }
+    }
+    // std::cout << GridLogMessage << " Done " << std::endl;
+    // defer sum over nodes.
+    return;
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // A2A_Reduction_hpp_

extras/Hadrons/AllToAllVectors.hpp

@@ -0,0 +1,210 @@
+#ifndef A2A_Vectors_hpp_
+#define A2A_Vectors_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Environment.hpp>
+#include <Grid/Hadrons/Solver.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+////////////////////////////////
+// A2A Modes
+////////////////////////////////
+
+template <class Field, class Matrix, class Solver>
+class A2AModesSchurDiagTwo
+{
+  private:
+    const std::vector<Field> *evec;
+    const std::vector<RealD> *eval;
+    Matrix &action;
+    Solver &solver;
+    std::vector<Field> w_high_5d, v_high_5d, w_high_4d, v_high_4d;
+    const int Nl, Nh;
+    const bool return_5d;
+
+  public:
+
+  int getNl (void ) {return Nl;}
+  int getNh (void ) {return Nh;}
+  int getN  (void ) {return Nh+Nl;}
+
+    A2AModesSchurDiagTwo(const std::vector<Field> *_evec, const std::vector<RealD> *_eval,
+                         Matrix &_action,
+                         Solver &_solver,
+                         std::vector<Field> _w_high_5d, std::vector<Field> _v_high_5d,
+                         std::vector<Field> _w_high_4d, std::vector<Field> _v_high_4d,
+                         const int _Nl, const int _Nh,
+                         const bool _return_5d)
+                        : evec(_evec), eval(_eval),
+                        action(_action),
+                        solver(_solver),
+                        w_high_5d(_w_high_5d), v_high_5d(_v_high_5d),
+                        w_high_4d(_w_high_4d), v_high_4d(_v_high_4d),
+                        Nl(_Nl), Nh(_Nh),
+                        return_5d(_return_5d){};
+
+    void high_modes(Field &source_5d, Field &w_source_5d, Field &source_4d, int i)
+    {
+        int i5d;
+        LOG(Message) << "A2A high modes for i = " << i << std::endl;
+        i5d = 0;
+        if (return_5d) i5d = i;
+        this->high_mode_v(action, solver, source_5d, v_high_5d[i5d], v_high_4d[i]);
+        this->high_mode_w(w_source_5d, source_4d, w_high_5d[i5d], w_high_4d[i]);
+    }
+
+    void return_v(int i, Field &vout_5d, Field &vout_4d)
+    {
+        if (i < Nl)
+        {
+            this->low_mode_v(action, evec->at(i), eval->at(i), vout_5d, vout_4d);
+        }
+        else
+        {
+            vout_4d = v_high_4d[i - Nl];
+            if (!(return_5d)) i = Nl;
+            vout_5d = v_high_5d[i - Nl];
+        }
+    }
+    void return_w(int i, Field &wout_5d, Field &wout_4d)
+    {
+        if (i < Nl)
+        {
+            this->low_mode_w(action, evec->at(i), eval->at(i), wout_5d, wout_4d);
+        }
+        else
+        {
+            wout_4d = w_high_4d[i - Nl];
+            if (!(return_5d)) i = Nl;
+            wout_5d = w_high_5d[i - Nl];
+        }
+    }
+
+    void low_mode_v(Matrix &action, const Field &evec, const RealD &eval, Field &vout_5d, Field &vout_4d)
+    {
+        GridBase *grid = action.RedBlackGrid();
+        Field src_o(grid);
+        Field sol_e(grid);
+        Field sol_o(grid);
+        Field tmp(grid);
+
+        src_o = evec;
+        src_o.checkerboard = Odd;
+        pickCheckerboard(Even, sol_e, vout_5d);
+        pickCheckerboard(Odd, sol_o, vout_5d);
+
+        /////////////////////////////////////////////////////
+        // v_ie = -(1/eval_i) * MeeInv Meo MooInv evec_i
+        /////////////////////////////////////////////////////
+        action.MooeeInv(src_o, tmp);
+        assert(tmp.checkerboard == Odd);
+        action.Meooe(tmp, sol_e);
+        assert(sol_e.checkerboard == Even);
+        action.MooeeInv(sol_e, tmp);
+        assert(tmp.checkerboard == Even);
+        sol_e = (-1.0 / eval) * tmp;
+        assert(sol_e.checkerboard == Even);
+
+        /////////////////////////////////////////////////////
+        // v_io = (1/eval_i) * MooInv evec_i
+        /////////////////////////////////////////////////////
+        action.MooeeInv(src_o, tmp);
+        assert(tmp.checkerboard == Odd);
+        sol_o = (1.0 / eval) * tmp;
+        assert(sol_o.checkerboard == Odd);
+
+        setCheckerboard(vout_5d, sol_e);
+        assert(sol_e.checkerboard == Even);
+        setCheckerboard(vout_5d, sol_o);
+        assert(sol_o.checkerboard == Odd);
+
+        action.ExportPhysicalFermionSolution(vout_5d, vout_4d);
+    }
+
+    void low_mode_w(Matrix &action, const Field &evec, const RealD &eval, Field &wout_5d, Field &wout_4d)
+    {
+        GridBase *grid = action.RedBlackGrid();
+        SchurDiagTwoOperator<Matrix, Field> _HermOpEO(action);
+
+        Field src_o(grid);
+        Field sol_e(grid);
+        Field sol_o(grid);
+        Field tmp(grid);
+
+        GridBase *fgrid = action.Grid();
+        Field tmp_wout(fgrid);
+
+        src_o = evec;
+        src_o.checkerboard = Odd;
+        pickCheckerboard(Even, sol_e, tmp_wout);
+        pickCheckerboard(Odd, sol_o, tmp_wout);
+
+        /////////////////////////////////////////////////////
+        // w_ie = - MeeInvDag MoeDag Doo evec_i
+        /////////////////////////////////////////////////////
+        _HermOpEO.Mpc(src_o, tmp);
+        assert(tmp.checkerboard == Odd);
+        action.MeooeDag(tmp, sol_e);
+        assert(sol_e.checkerboard == Even);
+        action.MooeeInvDag(sol_e, tmp);
+        assert(tmp.checkerboard == Even);
+        sol_e = (-1.0) * tmp;
+
+        /////////////////////////////////////////////////////
+        // w_io = Doo evec_i
+        /////////////////////////////////////////////////////
+        _HermOpEO.Mpc(src_o, sol_o);
+        assert(sol_o.checkerboard == Odd);
+
+        setCheckerboard(tmp_wout, sol_e);
+        assert(sol_e.checkerboard == Even);
+        setCheckerboard(tmp_wout, sol_o);
+        assert(sol_o.checkerboard == Odd);
+
+        action.DminusDag(tmp_wout, wout_5d);
+
+        action.ExportPhysicalFermionSource(wout_5d, wout_4d);
+    }
+
+    void high_mode_v(Matrix &action, Solver &solver, const Field &source, Field &vout_5d, Field &vout_4d)
+    {
+        GridBase *fgrid = action.Grid();
+        solver(vout_5d, source); // Note: solver is solver(out, in)
+        action.ExportPhysicalFermionSolution(vout_5d, vout_4d);
+    }
+
+    void high_mode_w(const Field &w_source_5d, const Field &source_4d, Field &wout_5d, Field &wout_4d)
+    {
+        wout_5d = w_source_5d;
+        wout_4d = source_4d;
+    }
+};
+
+// TODO: A2A for coarse eigenvectors
+
+// template <class FineField, class CoarseField, class Matrix, class Solver>
+// class A2ALMSchurDiagTwoCoarse : public A2AModesSchurDiagTwo<FineField, Matrix, Solver>
+// {
+//   private:
+//     const std::vector<FineField> &subspace;
+//     const std::vector<CoarseField> &evec_coarse;
+//     const std::vector<RealD> &eval_coarse;
+//     Matrix &action;
+
+//   public:
+//     A2ALMSchurDiagTwoCoarse(const std::vector<FineField> &_subspace, const std::vector<CoarseField> &_evec_coarse, const std::vector<RealD> &_eval_coarse, Matrix &_action)
+//         : subspace(_subspace), evec_coarse(_evec_coarse), eval_coarse(_eval_coarse), action(_action){};
+
+//     void operator()(int i, FineField &vout, FineField &wout)
+//     {
+//         FineField prom_evec(subspace[0]._grid);
+//         blockPromote(evec_coarse[i], prom_evec, subspace);
+//         this->low_mode_v(action, prom_evec, eval_coarse[i], vout);
+//         this->low_mode_w(action, prom_evec, eval_coarse[i], wout);
+//     }
+// };
+
+END_HADRONS_NAMESPACE
+
+#endif // A2A_Vectors_hpp_

extras/Hadrons/Application.cc

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Application.cc
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -29,6 +28,7 @@ See the full license in the file "LICENSE" in the top level distribution directo
 
 #include <Grid/Hadrons/Application.hpp>
 #include <Grid/Hadrons/GeneticScheduler.hpp>
+#include <Grid/Hadrons/Modules.hpp>
 
 using namespace Grid;
 using namespace QCD;
@@ -41,14 +41,12 @@ using namespace Hadrons;
  *                       Application implementation                           *
  ******************************************************************************/
 // constructors ////////////////////////////////////////////////////////////////
+#define MACOUT(macro)    macro              << " (" << #macro << ")"
+#define MACOUTS(macro) HADRONS_STR(macro) << " (" << #macro << ")"
+
 Application::Application(void)
 {
-    LOG(Message) << "Modules available:" << std::endl;
-    auto list = ModuleFactory::getInstance().getBuilderList();
-    for (auto &m: list)
-    {
-        LOG(Message) << "  " << m << std::endl;
-    }
+    initLogger();
     auto dim = GridDefaultLatt(), mpi = GridDefaultMpi(), loc(dim);
     locVol_ = 1;
     for (unsigned int d = 0; d < dim.size(); ++d)
@@ -56,9 +54,22 @@ Application::Application(void)
         loc[d]  /= mpi[d];
         locVol_ *= loc[d];
     }
-    LOG(Message) << "Global lattice: " << dim << std::endl;
-    LOG(Message) << "MPI partition : " << mpi << std::endl;
-    LOG(Message) << "Local lattice : " << loc << std::endl;
+    LOG(Message) << "====== HADRONS APPLICATION STARTING ======" << std::endl;
+    LOG(Message) << "** Dimensions" << std::endl;
+    LOG(Message) << "Global lattice       : " << dim << std::endl;
+    LOG(Message) << "MPI partition        : " << mpi << std::endl;
+    LOG(Message) << "Local lattice        : " << loc << std::endl;
+    LOG(Message) << std::endl;
+    LOG(Message) << "** Default parameters (and associated C macro)" << std::endl;
+    LOG(Message) << "ASCII output precision  : " << MACOUT(DEFAULT_ASCII_PREC) << std::endl;
+    LOG(Message) << "Fermion implementation  : " << MACOUTS(FIMPL) << std::endl;
+    LOG(Message) << "z-Fermion implementation: " << MACOUTS(ZFIMPL) << std::endl;
+    LOG(Message) << "Scalar implementation   : " << MACOUTS(SIMPL) << std::endl;
+    LOG(Message) << "Gauge implementation    : " << MACOUTS(GIMPL) << std::endl;
+    LOG(Message) << "Eigenvector base size   : " 
+                 << MACOUT(HADRONS_DEFAULT_LANCZOS_NBASIS) << std::endl;
+    LOG(Message) << "Schur decomposition     : " << MACOUTS(HADRONS_DEFAULT_SCHUR) << std::endl;
+    LOG(Message) << std::endl;
 }
 
 Application::Application(const Application::GlobalPar &par)
@@ -73,12 +84,6 @@ Application::Application(const std::string parameterFileName)
     parameterFileName_ = parameterFileName;
 }
 
-// environment shortcut ////////////////////////////////////////////////////////
-Environment & Application::env(void) const
-{
-    return Environment::getInstance();
-}
-
 // access //////////////////////////////////////////////////////////////////////
 void Application::setPar(const Application::GlobalPar &par)
 {
@@ -94,14 +99,13 @@ const Application::GlobalPar & Application::getPar(void)
 // execute /////////////////////////////////////////////////////////////////////
 void Application::run(void)
 {
-    if (!parameterFileName_.empty() and (env().getNModule() == 0))
+    if (!parameterFileName_.empty() and (vm().getNModule() == 0))
     {
         parseParameterFile(parameterFileName_);
     }
-    if (!scheduled_)
-    {
-        schedule();
-    }
+    vm().printContent();
+    env().printContent();
+    schedule();
     printSchedule();
     configLoop();
 }
@@ -124,12 +128,20 @@ void Application::parseParameterFile(const std::string parameterFileName)
     LOG(Message) << "Building application from '" << parameterFileName << "'..." << std::endl;
     read(reader, "parameters", par);
     setPar(par);
-    push(reader, "modules");
-    push(reader, "module");
+    if (!push(reader, "modules"))
+    {
+        HADRONS_ERROR(Parsing, "Cannot open node 'modules' in parameter file '" 
+                              + parameterFileName + "'");
+    }
+    if (!push(reader, "module"))
+    {
+        HADRONS_ERROR(Parsing, "Cannot open node 'modules/module' in parameter file '" 
+                              + parameterFileName + "'");
+    }
     do
     {
         read(reader, "id", id);
-        env().createModule(id.name, id.type, reader);
+        vm().createModule(id.name, id.type, reader);
     } while (reader.nextElement("module"));
     pop(reader);
     pop(reader);
@@ -137,140 +149,63 @@ void Application::parseParameterFile(const std::string parameterFileName)
 
 void Application::saveParameterFile(const std::string parameterFileName)
 {
-    XmlWriter          writer(parameterFileName);
-    ObjectId           id;
-    const unsigned int nMod = env().getNModule();
-    
     LOG(Message) << "Saving application to '" << parameterFileName << "'..." << std::endl;
-    write(writer, "parameters", getPar());
-    push(writer, "modules");
-    for (unsigned int i = 0; i < nMod; ++i)
+    if (env().getGrid()->IsBoss())
     {
-        push(writer, "module");
-        id.name = env().getModuleName(i);
-        id.type = env().getModule(i)->getRegisteredName();
-        write(writer, "id", id);
-        env().getModule(i)->saveParameters(writer, "options");
+        XmlWriter          writer(parameterFileName);
+        ObjectId           id;
+        const unsigned int nMod = vm().getNModule();
+
+        write(writer, "parameters", getPar());
+        push(writer, "modules");
+        for (unsigned int i = 0; i < nMod; ++i)
+        {
+            push(writer, "module");
+            id.name = vm().getModuleName(i);
+            id.type = vm().getModule(i)->getRegisteredName();
+            write(writer, "id", id);
+            vm().getModule(i)->saveParameters(writer, "options");
+            pop(writer);
+        }
+        pop(writer);
         pop(writer);
     }
-    pop(writer);
-    pop(writer);
 }
 
 // schedule computation ////////////////////////////////////////////////////////
-#define MEM_MSG(size)\
-sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
-
-#define DEFINE_MEMPEAK \
-GeneticScheduler<unsigned int>::ObjFunc memPeak = \
-[this](const std::vector<unsigned int> &program)\
-{\
-    unsigned int memPeak;\
-    bool         msg;\
-    \
-    msg = HadronsLogMessage.isActive();\
-    HadronsLogMessage.Active(false);\
-    env().dryRun(true);\
-    memPeak = env().executeProgram(program);\
-    env().dryRun(false);\
-    env().freeAll();\
-    HadronsLogMessage.Active(true);\
-    \
-    return memPeak;\
-}
-
 void Application::schedule(void)
 {
-    DEFINE_MEMPEAK;
-    
-    // build module dependency graph
-    LOG(Message) << "Building module graph..." << std::endl;
-    auto graph = env().makeModuleGraph();
-    auto con = graph.getConnectedComponents();
-    
-    // constrained topological sort using a genetic algorithm
-    LOG(Message) << "Scheduling computation..." << std::endl;
-    LOG(Message) << "               #module= " << graph.size() << std::endl;
-    LOG(Message) << "       population size= " << par_.genetic.popSize << std::endl;
-    LOG(Message) << "       max. generation= " << par_.genetic.maxGen << std::endl;
-    LOG(Message) << "  max. cst. generation= " << par_.genetic.maxCstGen << std::endl;
-    LOG(Message) << "         mutation rate= " << par_.genetic.mutationRate << std::endl;
-    
-    unsigned int                               k = 0, gen, prevPeak, nCstPeak = 0;
-    std::random_device                         rd;
-    GeneticScheduler<unsigned int>::Parameters par;
-    
-    par.popSize      = par_.genetic.popSize;
-    par.mutationRate = par_.genetic.mutationRate;
-    par.seed         = rd();
-    memPeak_         = 0;
-    CartesianCommunicator::BroadcastWorld(0, &(par.seed), sizeof(par.seed));
-    for (unsigned int i = 0; i < con.size(); ++i)
+    if (!scheduled_ and !loadedSchedule_)
     {
-        GeneticScheduler<unsigned int> scheduler(con[i], memPeak, par);
-        
-        gen = 0;
-        do
-        {
-            LOG(Debug) << "Generation " << gen << ":" << std::endl;
-            scheduler.nextGeneration();
-            if (gen != 0)
-            {
-                if (prevPeak == scheduler.getMinValue())
-                {
-                    nCstPeak++;
-                }
-                else
-                {
-                    nCstPeak = 0;
-                }
-            }
-            
-            prevPeak = scheduler.getMinValue();
-            if (gen % 10 == 0)
-            {
-                LOG(Iterative) << "Generation " << gen << ": "
-                               << MEM_MSG(scheduler.getMinValue()) << std::endl;
-            }
-            
-            gen++;
-        } while ((gen < par_.genetic.maxGen)
-                 and (nCstPeak < par_.genetic.maxCstGen));
-        auto &t = scheduler.getMinSchedule();
-        if (scheduler.getMinValue() > memPeak_)
-        {
-            memPeak_ = scheduler.getMinValue();
-        }
-        for (unsigned int j = 0; j < t.size(); ++j)
-        {
-            program_.push_back(t[j]);
-        }
+        program_   = vm().schedule(par_.genetic);
+        scheduled_ = true;
     }
-    scheduled_ = true;
 }
 
 void Application::saveSchedule(const std::string filename)
 {
-    TextWriter               writer(filename);
-    std::vector<std::string> program;
-    
-    if (!scheduled_)
-    {
-        HADRON_ERROR("Computation not scheduled");
-    }
     LOG(Message) << "Saving current schedule to '" << filename << "'..."
                  << std::endl;
-    for (auto address: program_)
+    if (env().getGrid()->IsBoss())
     {
-        program.push_back(env().getModuleName(address));
+        TextWriter               writer(filename);
+        std::vector<std::string> program;
+        
+        if (!scheduled_)
+        {
+            HADRONS_ERROR(Definition, "Computation not scheduled");
+        }
+
+        for (auto address: program_)
+        {
+            program.push_back(vm().getModuleName(address));
+        }
+        write(writer, "schedule", program);
     }
-    write(writer, "schedule", program);
 }
 
 void Application::loadSchedule(const std::string filename)
 {
-    DEFINE_MEMPEAK;
-    
     TextReader               reader(filename);
     std::vector<std::string> program;
     
@@ -280,24 +215,24 @@ void Application::loadSchedule(const std::string filename)
     program_.clear();
     for (auto &name: program)
     {
-        program_.push_back(env().getModuleAddress(name));
+        program_.push_back(vm().getModuleAddress(name));
     }
-    scheduled_ = true;
-    memPeak_   = memPeak(program_);
+    loadedSchedule_ = true;
 }
 
 void Application::printSchedule(void)
 {
     if (!scheduled_)
     {
-        HADRON_ERROR("Computation not scheduled");
+        HADRONS_ERROR(Definition, "Computation not scheduled");
     }
-    LOG(Message) << "Schedule (memory peak: " << MEM_MSG(memPeak_) << "):"
+    auto peak = vm().memoryNeeded(program_);
+    LOG(Message) << "Schedule (memory needed: " << sizeString(peak) << "):"
                  << std::endl;
     for (unsigned int i = 0; i < program_.size(); ++i)
     {
         LOG(Message) << std::setw(4) << i + 1 << ": "
-                     << env().getModuleName(program_[i]) << std::endl;
+                     << vm().getModuleName(program_[i]) << std::endl;
     }
 }
 
@@ -310,8 +245,8 @@ void Application::configLoop(void)
     {
         LOG(Message) << BIG_SEP << " Starting measurement for trajectory " << t
                      << " " << BIG_SEP << std::endl;
-        env().setTrajectory(t);
-        env().executeProgram(program_);
+        vm().setTrajectory(t);
+        vm().executeProgram(program_);
     }
     LOG(Message) << BIG_SEP << " End of measurement " << BIG_SEP << std::endl;
     env().freeAll();

extras/Hadrons/Application.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Application.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -31,9 +30,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #define Hadrons_Application_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Environment.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-#include <Grid/Hadrons/Modules.hpp>
+#include <Grid/Hadrons/VirtualMachine.hpp>
+#include <Grid/Hadrons/Module.hpp>
 
 BEGIN_HADRONS_NAMESPACE
 
@@ -51,25 +49,13 @@ public:
                                         unsigned int, end,
                                         unsigned int, step);
     };
-    class GeneticPar: Serializable
-    {
-    public:
-        GeneticPar(void):
-            popSize{20}, maxGen{1000}, maxCstGen{100}, mutationRate{.1} {};
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(GeneticPar,
-                                        unsigned int, popSize,
-                                        unsigned int, maxGen,
-                                        unsigned int, maxCstGen,
-                                        double      , mutationRate);
-    };
     class GlobalPar: Serializable
     {
     public:
         GRID_SERIALIZABLE_CLASS_MEMBERS(GlobalPar,
-                                        TrajRange,   trajCounter,
-                                        GeneticPar,  genetic,
-                                        std::string, seed);
+                                        TrajRange,                  trajCounter,
+                                        VirtualMachine::GeneticPar, genetic,
+                                        std::string,                seed);
     };
 public:
     // constructors
@@ -100,14 +86,15 @@ public:
     void configLoop(void);
 private:
     // environment shortcut
-    Environment & env(void) const;
+    DEFINE_ENV_ALIAS;
+    // virtual machine shortcut
+    DEFINE_VM_ALIAS;
 private:
-    long unsigned int         locVol_;
-    std::string               parameterFileName_{""};
-    GlobalPar                 par_;
-    std::vector<unsigned int> program_;
-    Environment::Size         memPeak_;
-    bool                      scheduled_{false};
+    long unsigned int       locVol_;
+    std::string             parameterFileName_{""};
+    GlobalPar               par_;
+    VirtualMachine::Program program_;
+    bool                    scheduled_{false}, loadedSchedule_{false};
 };
 
 /******************************************************************************
@@ -117,14 +104,16 @@ private:
 template <typename M>
 void Application::createModule(const std::string name)
 {
-    env().createModule<M>(name);
+    vm().createModule<M>(name);
+    scheduled_ = false;
 }
 
 template <typename M>
 void Application::createModule(const std::string name,
                                const typename M::Par &par)
 {
-    env().createModule<M>(name, par);
+    vm().createModule<M>(name, par);
+    scheduled_ = false;
 }
 
 END_HADRONS_NAMESPACE

extras/Hadrons/EigenPack.hpp

@@ -0,0 +1,323 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/EigenPack.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef Hadrons_EigenPack_hpp_
+#define Hadrons_EigenPack_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/algorithms/iterative/Deflation.h>
+#include <Grid/algorithms/iterative/LocalCoherenceLanczos.h>
+
+BEGIN_HADRONS_NAMESPACE
+
+// Lanczos type
+#ifndef HADRONS_DEFAULT_LANCZOS_NBASIS
+#define HADRONS_DEFAULT_LANCZOS_NBASIS 60
+#endif
+
+template <typename F>
+class EigenPack
+{
+public:
+    typedef F Field;
+    struct PackRecord
+    {
+        std::string operatorXml, solverXml;
+    };
+    struct VecRecord: Serializable
+    {
+        GRID_SERIALIZABLE_CLASS_MEMBERS(VecRecord,
+                                        unsigned int, index,
+                                        double,       eval);
+        VecRecord(void): index(0), eval(0.) {}
+    };
+public:
+    std::vector<RealD> eval;
+    std::vector<F>     evec;
+    PackRecord         record;
+public:
+    EigenPack(void)          = default;
+    virtual ~EigenPack(void) = default;
+
+    EigenPack(const size_t size, GridBase *grid)
+    {
+        resize(size, grid);
+    }
+
+    void resize(const size_t size, GridBase *grid)
+    {
+        eval.resize(size);
+        evec.resize(size, grid);
+    }
+
+    virtual void read(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        if (multiFile)
+        {
+            for(int k = 0; k < evec.size(); ++k)
+            {
+                basicReadSingle(evec[k], eval[k], evecFilename(fileStem, k, traj), k);
+            }
+        }
+        else
+        {
+            basicRead(evec, eval, evecFilename(fileStem, -1, traj), evec.size());
+        }
+    }
+
+    virtual void write(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        if (multiFile)
+        {
+            for(int k = 0; k < evec.size(); ++k)
+            {
+                basicWriteSingle(evecFilename(fileStem, k, traj), evec[k], eval[k], k);
+            }
+        }
+        else
+        {
+            basicWrite(evecFilename(fileStem, -1, traj), evec, eval, evec.size());
+        }
+    }
+protected:
+    std::string evecFilename(const std::string stem, const int vec, const int traj)
+    {
+        std::string t = (traj < 0) ? "" : ("." + std::to_string(traj));
+
+        if (vec == -1)
+        {
+            return stem + t + ".bin";
+        }
+        else
+        {
+            return stem + t + "/v" + std::to_string(vec) + ".bin";
+        }
+    }
+
+    template <typename T>
+    void basicRead(std::vector<T> &evec, std::vector<double> &eval,
+                   const std::string filename, const unsigned int size)
+    {
+        ScidacReader    binReader;
+
+        binReader.open(filename);
+        binReader.skipPastObjectRecord(SCIDAC_FILE_XML);
+        for(int k = 0; k < size; ++k) 
+        {
+            VecRecord vecRecord;
+
+            LOG(Message) << "Reading eigenvector " << k << std::endl;
+            binReader.readScidacFieldRecord(evec[k], vecRecord);
+            if (vecRecord.index != k)
+            {
+                HADRONS_ERROR(Io, "Eigenvector " + std::to_string(k) + " has a"
+                              + " wrong index (expected " + std::to_string(vecRecord.index) 
+                              + ") in file '" + filename + "'");
+            }
+            eval[k] = vecRecord.eval;
+        }
+        binReader.close();
+    }
+
+    template <typename T>
+    void basicReadSingle(T &evec, double &eval, const std::string filename, 
+                         const unsigned int index)
+    {
+        ScidacReader binReader;
+        VecRecord    vecRecord;
+
+        binReader.open(filename);
+        binReader.skipPastObjectRecord(SCIDAC_FILE_XML);
+        LOG(Message) << "Reading eigenvector " << index << std::endl;
+        binReader.readScidacFieldRecord(evec, vecRecord);
+        if (vecRecord.index != index)
+        {
+            HADRONS_ERROR(Io, "Eigenvector " + std::to_string(index) + " has a"
+                          + " wrong index (expected " + std::to_string(vecRecord.index) 
+                          + ") in file '" + filename + "'");
+        }
+        eval = vecRecord.eval;
+        binReader.close();
+    }
+
+    template <typename T>
+    void basicWrite(const std::string filename, std::vector<T> &evec, 
+                    const std::vector<double> &eval, const unsigned int size)
+    {
+        ScidacWriter binWriter(evec[0]._grid->IsBoss());
+        XmlWriter    xmlWriter("", "eigenPackPar");
+
+        makeFileDir(filename, evec[0]._grid);
+        xmlWriter.pushXmlString(record.operatorXml);
+        xmlWriter.pushXmlString(record.solverXml);
+        binWriter.open(filename);
+        binWriter.writeLimeObject(1, 1, xmlWriter, "parameters", SCIDAC_FILE_XML);
+        for(int k = 0; k < size; ++k) 
+        {
+            VecRecord vecRecord;
+
+            vecRecord.index = k;
+            vecRecord.eval  = eval[k];
+            LOG(Message) << "Writing eigenvector " << k << std::endl;
+            binWriter.writeScidacFieldRecord(evec[k], vecRecord, DEFAULT_ASCII_PREC);
+        }
+        binWriter.close();
+    }
+
+    template <typename T>
+    void basicWriteSingle(const std::string filename, T &evec, 
+                          const double eval, const unsigned int index)
+    {
+        ScidacWriter binWriter(evec._grid->IsBoss());
+        XmlWriter    xmlWriter("", "eigenPackPar");
+        VecRecord    vecRecord;
+
+        makeFileDir(filename, evec._grid);
+        xmlWriter.pushXmlString(record.operatorXml);
+        xmlWriter.pushXmlString(record.solverXml);
+        binWriter.open(filename);
+        binWriter.writeLimeObject(1, 1, xmlWriter, "parameters", SCIDAC_FILE_XML);
+        vecRecord.index = index;
+        vecRecord.eval  = eval;
+        LOG(Message) << "Writing eigenvector " << index << std::endl;
+        binWriter.writeScidacFieldRecord(evec, vecRecord, DEFAULT_ASCII_PREC);
+        binWriter.close();
+    }
+};
+
+template <typename FineF, typename CoarseF>
+class CoarseEigenPack: public EigenPack<FineF>
+{
+public:
+    typedef CoarseF CoarseField;
+public:
+    std::vector<RealD>   evalCoarse;
+    std::vector<CoarseF> evecCoarse;
+public:
+    CoarseEigenPack(void)          = default;
+    virtual ~CoarseEigenPack(void) = default;
+
+    CoarseEigenPack(const size_t sizeFine, const size_t sizeCoarse, 
+                    GridBase *gridFine, GridBase *gridCoarse)
+    {
+        resize(sizeFine, sizeCoarse, gridFine, gridCoarse);
+    }
+
+    void resize(const size_t sizeFine, const size_t sizeCoarse, 
+                GridBase *gridFine, GridBase *gridCoarse)
+    {
+        EigenPack<FineF>::resize(sizeFine, gridFine);
+        evalCoarse.resize(sizeCoarse);
+        evecCoarse.resize(sizeCoarse, gridCoarse);
+    }
+
+    void readFine(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        if (multiFile)
+        {
+            for(int k = 0; k < this->evec.size(); ++k)
+            {
+                this->basicReadSingle(this->evec[k], this->eval[k], this->evecFilename(fileStem + "_fine", k, traj), k);
+            }
+        }
+        else
+        {
+            this->basicRead(this->evec, this->eval, this->evecFilename(fileStem + "_fine", -1, traj), this->evec.size());
+        }
+    }
+
+    void readCoarse(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        if (multiFile)
+        {
+            for(int k = 0; k < evecCoarse.size(); ++k)
+            {
+                this->basicReadSingle(evecCoarse[k], evalCoarse[k], this->evecFilename(fileStem + "_coarse", k, traj), k);
+            }
+        }
+        else
+        {
+            this->basicRead(evecCoarse, evalCoarse, this->evecFilename(fileStem + "_coarse", -1, traj), evecCoarse.size());
+        }
+    }
+
+    virtual void read(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        readFine(fileStem, multiFile, traj);
+        readCoarse(fileStem, multiFile, traj);
+    }
+
+    void writeFine(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        if (multiFile)
+        {
+            for(int k = 0; k < this->evec.size(); ++k)
+            {
+                this->basicWriteSingle(this->evecFilename(fileStem + "_fine", k, traj), this->evec[k], this->eval[k], k);
+            }
+        }
+        else
+        {
+            this->basicWrite(this->evecFilename(fileStem + "_fine", -1, traj), this->evec, this->eval, this->evec.size());
+        }
+    }
+
+    void writeCoarse(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        if (multiFile)
+        {
+            for(int k = 0; k < evecCoarse.size(); ++k)
+            {
+                this->basicWriteSingle(this->evecFilename(fileStem + "_coarse", k, traj), evecCoarse[k], evalCoarse[k], k);
+            }
+        }
+        else
+        {
+            this->basicWrite(this->evecFilename(fileStem + "_coarse", -1, traj), evecCoarse, evalCoarse, evecCoarse.size());
+        }
+    }
+    
+    virtual void write(const std::string fileStem, const bool multiFile, const int traj = -1)
+    {
+        writeFine(fileStem, multiFile, traj);
+        writeCoarse(fileStem, multiFile, traj);
+    }
+};
+
+template <typename FImpl>
+using FermionEigenPack = EigenPack<typename FImpl::FermionField>;
+
+template <typename FImpl, int nBasis>
+using CoarseFermionEigenPack = CoarseEigenPack<
+    typename FImpl::FermionField,
+    typename LocalCoherenceLanczos<typename FImpl::SiteSpinor, 
+                                   typename FImpl::SiteComplex, 
+                                   nBasis>::CoarseField>;
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_EigenPack_hpp_

extras/Hadrons/Environment.cc

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Environment.cc
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -35,52 +34,33 @@ using namespace Grid;
 using namespace QCD;
 using namespace Hadrons;
 
+#define ERROR_NO_ADDRESS(address)\
+HADRONS_ERROR(Definition, "no object with address " + std::to_string(address));
+
 /******************************************************************************
  *                       Environment implementation                           *
  ******************************************************************************/
 // constructor /////////////////////////////////////////////////////////////////
 Environment::Environment(void)
 {
-    nd_ = GridDefaultLatt().size();
+    dim_ = GridDefaultLatt();
+    nd_  = dim_.size();
     grid4d_.reset(SpaceTimeGrid::makeFourDimGrid(
-        GridDefaultLatt(), GridDefaultSimd(nd_, vComplex::Nsimd()),
+        dim_, GridDefaultSimd(nd_, vComplex::Nsimd()),
         GridDefaultMpi()));
     gridRb4d_.reset(SpaceTimeGrid::makeFourDimRedBlackGrid(grid4d_.get()));
-    auto loc = getGrid()->LocalDimensions();
-    locVol_ = 1;
-    for (unsigned int d = 0; d < loc.size(); ++d)
+    vol_ = 1.;
+    for (auto d: dim_)
     {
-        locVol_ *= loc[d];
+        vol_ *= d;
     }
     rng4d_.reset(new GridParallelRNG(grid4d_.get()));
 }
 
-// dry run /////////////////////////////////////////////////////////////////////
-void Environment::dryRun(const bool isDry)
-{
-    dryRun_ = isDry;
-}
-
-bool Environment::isDryRun(void) const
-{
-    return dryRun_;
-}
-
-// trajectory number ///////////////////////////////////////////////////////////
-void Environment::setTrajectory(const unsigned int traj)
-{
-    traj_ = traj;
-}
-
-unsigned int Environment::getTrajectory(void) const
-{
-    return traj_;
-}
-
 // grids ///////////////////////////////////////////////////////////////////////
 void Environment::createGrid(const unsigned int Ls)
 {
-    if (grid5d_.find(Ls) == grid5d_.end())
+    if ((Ls > 1) and (grid5d_.find(Ls) == grid5d_.end()))
     {
         auto g = getGrid();
         
@@ -89,6 +69,49 @@ void Environment::createGrid(const unsigned int Ls)
     }
 }
 
+void Environment::createCoarseGrid(const std::vector<int> &blockSize, 
+                                   const unsigned int Ls)
+{
+    int              nd      = getNd();
+    std::vector<int> fineDim = getDim(), coarseDim;
+    unsigned int     cLs;
+    auto             key4d = blockSize, key5d = blockSize;
+
+    createGrid(Ls);
+    coarseDim.resize(nd);
+    for (int d = 0; d < coarseDim.size(); d++)
+    {
+        coarseDim[d] = fineDim[d]/blockSize[d];
+        if (coarseDim[d]*blockSize[d] != fineDim[d])
+        {
+            HADRONS_ERROR(Size, "Fine dimension " + std::to_string(d) 
+                         + " (" + std::to_string(fineDim[d]) 
+                         + ") not divisible by coarse dimension ("
+                         + std::to_string(coarseDim[d]) + ")"); 
+        }
+    }
+    if (blockSize.size() > nd)
+    {
+        cLs = Ls/blockSize[nd];
+        if (cLs*blockSize[nd] != Ls)
+        {
+            HADRONS_ERROR(Size, "Fine Ls (" + std::to_string(Ls) 
+                         + ") not divisible by coarse Ls ("
+                         + std::to_string(cLs) + ")");
+        }
+        key4d.resize(nd);
+        key5d.push_back(Ls);
+    }
+    gridCoarse4d_[key4d].reset(
+        SpaceTimeGrid::makeFourDimGrid(coarseDim, 
+            GridDefaultSimd(nd, vComplex::Nsimd()), GridDefaultMpi()));
+    if (Ls > 1)
+    {
+        gridCoarse5d_[key5d].reset(
+            SpaceTimeGrid::makeFiveDimGrid(cLs, gridCoarse4d_[key4d].get()));
+    }
+}
+
 GridCartesian * Environment::getGrid(const unsigned int Ls) const
 {
     try
@@ -104,7 +127,7 @@ GridCartesian * Environment::getGrid(const unsigned int Ls) const
     }
     catch(std::out_of_range &)
     {
-        HADRON_ERROR("no grid with Ls= " << Ls);
+        HADRONS_ERROR(Definition, "no grid with Ls= " + std::to_string(Ls));
     }
 }
 
@@ -123,7 +146,31 @@ GridRedBlackCartesian * Environment::getRbGrid(const unsigned int Ls) const
     }
     catch(std::out_of_range &)
     {
-        HADRON_ERROR("no red-black 5D grid with Ls= " << Ls);
+        HADRONS_ERROR(Definition, "no red-black grid with Ls= " + std::to_string(Ls));
+    }
+}
+
+GridCartesian * Environment::getCoarseGrid(
+    const std::vector<int> &blockSize, const unsigned int Ls) const
+{
+    auto key = blockSize;
+
+    try
+    {
+        if (Ls == 1)
+        {
+            key.resize(getNd());
+            return gridCoarse4d_.at(key).get();
+        }
+        else
+        {
+            key.push_back(Ls);
+            return gridCoarse5d_.at(key).get();
+        }
+    }
+    catch(std::out_of_range &)
+    {
+        HADRONS_ERROR(Definition, "no coarse grid with Ls= " + std::to_string(Ls));
     }
 }
 
@@ -132,6 +179,21 @@ unsigned int Environment::getNd(void) const
     return nd_;
 }
 
+std::vector<int> Environment::getDim(void) const
+{
+    return dim_;
+}
+
+int Environment::getDim(const unsigned int mu) const
+{
+    return dim_[mu];
+}
+
+double Environment::getVolume(void) const
+{
+    return vol_;
+}
+
 // random number generator /////////////////////////////////////////////////////
 void Environment::setSeed(const std::vector<int> &seed)
 {
@@ -143,276 +205,6 @@ GridParallelRNG * Environment::get4dRng(void) const
     return rng4d_.get();
 }
 
-// module management ///////////////////////////////////////////////////////////
-void Environment::pushModule(Environment::ModPt &pt)
-{
-    std::string name = pt->getName();
-    
-    if (!hasModule(name))
-    {
-        std::vector<unsigned int> inputAddress;
-        unsigned int              address;
-        ModuleInfo                m;
-        
-        m.data = std::move(pt);
-        m.type = typeIdPt(*m.data.get());
-        m.name = name;
-        auto input  = m.data->getInput();
-        for (auto &in: input)
-        {
-            if (!hasObject(in))
-            {
-                addObject(in , -1);
-            }
-            m.input.push_back(objectAddress_[in]);
-        }
-        auto output = m.data->getOutput();
-        module_.push_back(std::move(m));
-        address              = static_cast<unsigned int>(module_.size() - 1);
-        moduleAddress_[name] = address;
-        for (auto &out: output)
-        {
-            if (!hasObject(out))
-            {
-                addObject(out, address);
-            }
-            else
-            {
-                if (object_[objectAddress_[out]].module < 0)
-                {
-                    object_[objectAddress_[out]].module = address;
-                }
-                else
-                {
-                    HADRON_ERROR("object '" + out
-                                 + "' is already produced by module '"
-                                 + module_[object_[getObjectAddress(out)].module].name
-                                 + "' (while pushing module '" + name + "')");
-                }
-            }
-        }
-    }
-    else
-    {
-        HADRON_ERROR("module '" + name + "' already exists");
-    }
-}
-
-unsigned int Environment::getNModule(void) const
-{
-    return module_.size();
-}
-
-void Environment::createModule(const std::string name, const std::string type,
-                               XmlReader &reader)
-{
-    auto &factory = ModuleFactory::getInstance();
-    auto pt       = factory.create(type, name);
-    
-    pt->parseParameters(reader, "options");
-    pushModule(pt);
-}
-
-ModuleBase * Environment::getModule(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return module_[address].data.get();
-    }
-    else
-    {
-        HADRON_ERROR("no module with address " + std::to_string(address));
-    }
-}
-
-ModuleBase * Environment::getModule(const std::string name) const
-{
-    return getModule(getModuleAddress(name));
-}
-
-unsigned int Environment::getModuleAddress(const std::string name) const
-{
-    if (hasModule(name))
-    {
-        return moduleAddress_.at(name);
-    }
-    else
-    {
-        HADRON_ERROR("no module with name '" + name + "'");
-    }
-}
-
-std::string Environment::getModuleName(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return module_[address].name;
-    }
-    else
-    {
-        HADRON_ERROR("no module with address " + std::to_string(address));
-    }
-}
-
-std::string Environment::getModuleType(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return typeName(module_[address].type);
-    }
-    else
-    {
-        HADRON_ERROR("no module with address " + std::to_string(address));
-    }
-}
-
-std::string Environment::getModuleType(const std::string name) const
-{
-    return getModuleType(getModuleAddress(name));
-}
-
-bool Environment::hasModule(const unsigned int address) const
-{
-    return (address < module_.size());
-}
-
-bool Environment::hasModule(const std::string name) const
-{
-    return (moduleAddress_.find(name) != moduleAddress_.end());
-}
-
-Graph<unsigned int> Environment::makeModuleGraph(void) const
-{
-    Graph<unsigned int> moduleGraph;
-    
-    for (unsigned int i = 0; i < module_.size(); ++i)
-    {
-        moduleGraph.addVertex(i);
-        for (auto &j: module_[i].input)
-        {
-            moduleGraph.addEdge(object_[j].module, i);
-        }
-    }
-    
-    return moduleGraph;
-}
-
-#define BIG_SEP "==============="
-#define SEP     "---------------"
-#define MEM_MSG(size)\
-sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
-
-Environment::Size
-Environment::executeProgram(const std::vector<unsigned int> &p)
-{
-    Size                                memPeak = 0, sizeBefore, sizeAfter;
-    std::vector<std::set<unsigned int>> freeProg;
-    bool                                continueCollect, nothingFreed;
-    
-    // build garbage collection schedule
-    freeProg.resize(p.size());
-    for (unsigned int i = 0; i < object_.size(); ++i)
-    {
-        auto pred = [i, this](const unsigned int j)
-        {
-            auto &in = module_[j].input;
-            auto it  = std::find(in.begin(), in.end(), i);
-            
-            return (it != in.end()) or (j == object_[i].module);
-        };
-        auto it = std::find_if(p.rbegin(), p.rend(), pred);
-        if (it != p.rend())
-        {
-            freeProg[p.rend() - it - 1].insert(i);
-        }
-    }
-    
-    // program execution
-    for (unsigned int i = 0; i < p.size(); ++i)
-    {
-        // execute module
-        if (!isDryRun())
-        {
-            LOG(Message) << SEP << " Measurement step " << i+1 << "/"
-                         << p.size() << " (module '" << module_[p[i]].name
-                         << "') " << SEP << std::endl;
-        }
-        (*module_[p[i]].data)();
-        sizeBefore = getTotalSize();
-        // print used memory after execution
-        if (!isDryRun())
-        {
-            LOG(Message) << "Allocated objects: " << MEM_MSG(sizeBefore)
-                         << std::endl;
-        }
-        if (sizeBefore > memPeak)
-        {
-            memPeak = sizeBefore;
-        }
-        // garbage collection for step i
-        if (!isDryRun())
-        {
-            LOG(Message) << "Garbage collection..." << std::endl;
-        }
-        nothingFreed = true;
-        do
-        {
-            continueCollect = false;
-            auto toFree = freeProg[i];
-            for (auto &j: toFree)
-            {
-                // continue garbage collection while there are still
-                // objects without owners
-                continueCollect = continueCollect or !hasOwners(j);
-                if(freeObject(j))
-                {
-                    // if an object has been freed, remove it from
-                    // the garbage collection schedule
-                    freeProg[i].erase(j);
-                    nothingFreed = false;
-                }
-            }
-        } while (continueCollect);
-        // any remaining objects in step i garbage collection schedule
-        // is scheduled for step i + 1
-        if (i + 1 < p.size())
-        {
-            for (auto &j: freeProg[i])
-            {
-                freeProg[i + 1].insert(j);
-            }
-        }
-        // print used memory after garbage collection if necessary
-        if (!isDryRun())
-        {
-            sizeAfter = getTotalSize();
-            if (sizeBefore != sizeAfter)
-            {
-                LOG(Message) << "Allocated objects: " << MEM_MSG(sizeAfter)
-                             << std::endl;
-            }
-            else
-            {
-                LOG(Message) << "Nothing to free" << std::endl;
-            }
-        }
-    }
-    
-    return memPeak;
-}
-
-Environment::Size Environment::executeProgram(const std::vector<std::string> &p)
-{
-    std::vector<unsigned int> pAddress;
-    
-    for (auto &n: p)
-    {
-        pAddress.push_back(getModuleAddress(n));
-    }
-    
-    return executeProgram(pAddress);
-}
-
 // general memory management ///////////////////////////////////////////////////
 void Environment::addObject(const std::string name, const int moduleAddress)
 {
@@ -422,46 +214,25 @@ void Environment::addObject(const std::string name, const int moduleAddress)
         
         info.name   = name;
         info.module = moduleAddress;
+        info.data   = nullptr;
         object_.push_back(std::move(info));
         objectAddress_[name] = static_cast<unsigned int>(object_.size() - 1);
     }
     else
     {
-        HADRON_ERROR("object '" + name + "' already exists");
+        HADRONS_ERROR(Definition, "object '" + name + "' already exists");
     }
 }
 
-void Environment::registerObject(const unsigned int address,
-                                 const unsigned int size, const unsigned int Ls)
+void Environment::setObjectModule(const unsigned int objAddress,
+                                  const int modAddress)
 {
-    if (!hasRegisteredObject(address))
-    {
-        if (hasObject(address))
-        {
-            object_[address].size         = size;
-            object_[address].Ls           = Ls;
-            object_[address].isRegistered = true;
-        }
-        else
-        {
-            HADRON_ERROR("no object with address " + std::to_string(address));
-        }
-    }
-    else
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " already registered");
-    }
+    object_[objAddress].module = modAddress;
 }
 
-void Environment::registerObject(const std::string name,
-                                 const unsigned int size, const unsigned int Ls)
+unsigned int Environment::getMaxAddress(void) const
 {
-    if (!hasObject(name))
-    {
-        addObject(name);
-    }
-    registerObject(getObjectAddress(name), size, Ls);
+    return object_.size();
 }
 
 unsigned int Environment::getObjectAddress(const std::string name) const
@@ -472,7 +243,7 @@ unsigned int Environment::getObjectAddress(const std::string name) const
     }
     else
     {
-        HADRON_ERROR("no object with name '" + name + "'");
+        HADRONS_ERROR(Definition, "no object with name '" + name + "'");
     }
 }
 
@@ -484,24 +255,26 @@ std::string Environment::getObjectName(const unsigned int address) const
     }
     else
     {
-        HADRON_ERROR("no object with address " + std::to_string(address));
+        ERROR_NO_ADDRESS(address);
     }
 }
 
 std::string Environment::getObjectType(const unsigned int address) const
 {
-    if (hasRegisteredObject(address))
+    if (hasObject(address))
     {
-        return typeName(object_[address].type);
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " exists but is not registered");
+        if (object_[address].type)
+        {
+            return typeName(object_[address].type);
+        }
+        else
+        {
+            return "<no type>";
+        }
     }
     else
     {
-        HADRON_ERROR("no object with address " + std::to_string(address));
+        ERROR_NO_ADDRESS(address);
     }
 }
 
@@ -512,18 +285,13 @@ std::string Environment::getObjectType(const std::string name) const
 
 Environment::Size Environment::getObjectSize(const unsigned int address) const
 {
-    if (hasRegisteredObject(address))
+    if (hasObject(address))
     {
         return object_[address].size;
     }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " exists but is not registered");
-    }
     else
     {
-        HADRON_ERROR("no object with address " + std::to_string(address));
+        ERROR_NO_ADDRESS(address);
     }
 }
 
@@ -532,20 +300,49 @@ Environment::Size Environment::getObjectSize(const std::string name) const
     return getObjectSize(getObjectAddress(name));
 }
 
-unsigned int Environment::getObjectLs(const unsigned int address) const
+Environment::Storage Environment::getObjectStorage(const unsigned int address) const
 {
-    if (hasRegisteredObject(address))
+    if (hasObject(address))
     {
-        return object_[address].Ls;
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " exists but is not registered");
+        return object_[address].storage;
     }
     else
     {
-        HADRON_ERROR("no object with address " + std::to_string(address));
+        ERROR_NO_ADDRESS(address);
+    }
+}
+
+Environment::Storage Environment::getObjectStorage(const std::string name) const
+{
+    return getObjectStorage(getObjectAddress(name));
+}
+
+int Environment::getObjectModule(const unsigned int address) const
+{
+    if (hasObject(address))
+    {
+        return object_[address].module;
+    }
+    else
+    {
+        ERROR_NO_ADDRESS(address);
+    }
+}
+
+int Environment::getObjectModule(const std::string name) const
+{
+    return getObjectModule(getObjectAddress(name));
+}
+
+unsigned int Environment::getObjectLs(const unsigned int address) const
+{
+    if (hasCreatedObject(address))
+    {
+        return object_[address].Ls;
+    }
+    else
+    {
+        ERROR_NO_ADDRESS(address);
     }
 }
 
@@ -566,30 +363,6 @@ bool Environment::hasObject(const std::string name) const
     return ((it != objectAddress_.end()) and hasObject(it->second));
 }
 
-bool Environment::hasRegisteredObject(const unsigned int address) const
-{
-    if (hasObject(address))
-    {
-        return object_[address].isRegistered;
-    }
-    else
-    {
-        return false;
-    }
-}
-
-bool Environment::hasRegisteredObject(const std::string name) const
-{
-    if (hasObject(name))
-    {
-        return hasRegisteredObject(getObjectAddress(name));
-    }
-    else
-    {
-        return false;
-    }
-}
-
 bool Environment::hasCreatedObject(const unsigned int address) const
 {
     if (hasObject(address))
@@ -630,92 +403,27 @@ Environment::Size Environment::getTotalSize(void) const
     
     for (auto &o: object_)
     {
-        if (o.isRegistered)
-        {
-            size += o.size;
-        }
+        size += o.size;
     }
     
     return size;
 }
 
-void Environment::addOwnership(const unsigned int owner,
-                               const unsigned int property)
+void Environment::freeObject(const unsigned int address)
 {
-    if (hasObject(property))
+    if (hasCreatedObject(address))
     {
-        object_[property].owners.insert(owner);
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(property));
-    }
-    if (hasObject(owner))
-    {
-        object_[owner].properties.insert(property);
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(owner));
+        LOG(Message) << "Destroying object '" << object_[address].name
+                     << "'" << std::endl;
     }
+    object_[address].size = 0;
+    object_[address].type = nullptr;
+    object_[address].data.reset(nullptr);
 }
 
-void Environment::addOwnership(const std::string owner,
-                               const std::string property)
+void Environment::freeObject(const std::string name)
 {
-    addOwnership(getObjectAddress(owner), getObjectAddress(property));
-}
-
-bool Environment::hasOwners(const unsigned int address) const
-{
-    
-    if (hasObject(address))
-    {
-        return (!object_[address].owners.empty());
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-bool Environment::hasOwners(const std::string name) const
-{
-    return hasOwners(getObjectAddress(name));
-}
-
-bool Environment::freeObject(const unsigned int address)
-{
-    if (!hasOwners(address))
-    {
-        if (!isDryRun() and object_[address].isRegistered)
-        {
-            LOG(Message) << "Destroying object '" << object_[address].name
-                         << "'" << std::endl;
-        }
-        for (auto &p: object_[address].properties)
-        {
-            object_[p].owners.erase(address);
-        }
-        object_[address].size         = 0;
-        object_[address].Ls           = 0;
-        object_[address].isRegistered = false;
-        object_[address].type         = nullptr;
-        object_[address].owners.clear();
-        object_[address].properties.clear();
-        object_[address].data.reset(nullptr);
-        
-        return true;
-    }
-    else
-    {
-        return false;
-    }
-}
-
-bool Environment::freeObject(const std::string name)
-{
-    return freeObject(getObjectAddress(name));
+    freeObject(getObjectAddress(name));
 }
 
 void Environment::freeAll(void)
@@ -726,18 +434,24 @@ void Environment::freeAll(void)
     }
 }
 
-void Environment::printContent(void)
+void Environment::protectObjects(const bool protect)
 {
-    LOG(Message) << "Modules: " << std::endl;
-    for (unsigned int i = 0; i < module_.size(); ++i)
-    {
-        LOG(Message) << std::setw(4) << i << ": "
-                     << getModuleName(i) << std::endl;
-    }
-    LOG(Message) << "Objects: " << std::endl;
+    protect_ = protect;
+}
+
+bool Environment::objectsProtected(void) const
+{
+    return protect_;
+}
+
+// print environment content ///////////////////////////////////////////////////
+void Environment::printContent(void) const
+{
+    LOG(Debug) << "Objects: " << std::endl;
     for (unsigned int i = 0; i < object_.size(); ++i)
     {
-        LOG(Message) << std::setw(4) << i << ": "
-                     << getObjectName(i) << std::endl;
+        LOG(Debug) << std::setw(4) << i << ": "
+                   << getObjectName(i) << " ("
+                   << sizeString(getObjectSize(i)) << ")" << std::endl;
     }
 }

extras/Hadrons/Environment.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Environment.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -31,20 +30,12 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #define Hadrons_Environment_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Graph.hpp>
-
-#ifndef SITE_SIZE_TYPE
-#define SITE_SIZE_TYPE unsigned int
-#endif
 
 BEGIN_HADRONS_NAMESPACE
 
 /******************************************************************************
  *                         Global environment                                 *
  ******************************************************************************/
-// forward declaration of Module
-class ModuleBase;
-
 class Object
 {
 public:
@@ -66,149 +57,117 @@ private:
     std::unique_ptr<T> objPt_{nullptr};
 };
 
+#define DEFINE_ENV_ALIAS \
+inline Environment & env(void) const\
+{\
+    return Environment::getInstance();\
+}
+
 class Environment
 {
     SINGLETON(Environment);
 public:
     typedef SITE_SIZE_TYPE                         Size;
-    typedef std::unique_ptr<ModuleBase>            ModPt;
     typedef std::unique_ptr<GridCartesian>         GridPt;
     typedef std::unique_ptr<GridRedBlackCartesian> GridRbPt;
     typedef std::unique_ptr<GridParallelRNG>       RngPt;
-    typedef std::unique_ptr<LatticeBase>           LatticePt;
+    enum class Storage {object, cache, temporary};
 private:
-    struct ModuleInfo
-    {
-        const std::type_info      *type{nullptr};
-        std::string               name;
-        ModPt                     data{nullptr};
-        std::vector<unsigned int> input;
-    };
     struct ObjInfo
     {
         Size                    size{0};
+        Storage                 storage{Storage::object};
         unsigned int            Ls{0};
-        bool                    isRegistered{false};
-        const std::type_info    *type{nullptr};
+        const std::type_info    *type{nullptr}, *derivedType{nullptr};
         std::string             name;
         int                     module{-1};
-        std::set<unsigned int>  owners, properties;
         std::unique_ptr<Object> data{nullptr};
     };
 public:
-    // dry run
-    void                    dryRun(const bool isDry);
-    bool                    isDryRun(void) const;
-    // trajectory number
-    void                    setTrajectory(const unsigned int traj);
-    unsigned int            getTrajectory(void) const;
     // grids
     void                    createGrid(const unsigned int Ls);
+    void                    createCoarseGrid(const std::vector<int> &blockSize,
+                                             const unsigned int Ls = 1);
     GridCartesian *         getGrid(const unsigned int Ls = 1) const;
     GridRedBlackCartesian * getRbGrid(const unsigned int Ls = 1) const;
+    GridCartesian *         getCoarseGrid(const std::vector<int> &blockSize,
+                                          const unsigned int Ls = 1) const;
+    std::vector<int>        getDim(void) const;
+    int                     getDim(const unsigned int mu) const;
     unsigned int            getNd(void) const;
+    double                  getVolume(void) const;
     // random number generator
     void                    setSeed(const std::vector<int> &seed);
     GridParallelRNG *       get4dRng(void) const;
-    // module management
-    void                    pushModule(ModPt &pt);
-    template <typename M>
-    void                    createModule(const std::string name);
-    template <typename M>
-    void                    createModule(const std::string name,
-                                         const typename M::Par &par);
-    void                    createModule(const std::string name,
-                                         const std::string type,
-                                         XmlReader &reader);
-    unsigned int            getNModule(void) const;
-    ModuleBase *            getModule(const unsigned int address) const;
-    ModuleBase *            getModule(const std::string name) const;
-    template <typename M>
-    M *                     getModule(const unsigned int address) const;
-    template <typename M>
-    M *                     getModule(const std::string name) const;
-    unsigned int            getModuleAddress(const std::string name) const;
-    std::string             getModuleName(const unsigned int address) const;
-    std::string             getModuleType(const unsigned int address) const;
-    std::string             getModuleType(const std::string name) const;
-    bool                    hasModule(const unsigned int address) const;
-    bool                    hasModule(const std::string name) const;
-    Graph<unsigned int>     makeModuleGraph(void) const;
-    Size                    executeProgram(const std::vector<unsigned int> &p);
-    Size                    executeProgram(const std::vector<std::string> &p);
     // general memory management
     void                    addObject(const std::string name,
                                       const int moduleAddress = -1);
-    void                    registerObject(const unsigned int address,
-                                           const unsigned int size,
-                                           const unsigned int Ls = 1);
-    void                    registerObject(const std::string name,
-                                           const unsigned int size,
-                                           const unsigned int Ls = 1);
-    template <typename T>
-    unsigned int            lattice4dSize(void) const;
-    template <typename T>
-    void                    registerLattice(const unsigned int address,
-                                            const unsigned int Ls = 1);
-    template <typename T>
-    void                    registerLattice(const std::string name,
-                                            const unsigned int Ls = 1);
-    template <typename T>
-    void                    setObject(const unsigned int address, T *object);
-    template <typename T>
-    void                    setObject(const std::string name, T *object);
+    template <typename B, typename T, typename ... Ts>
+    void                    createDerivedObject(const std::string name,
+                                                const Environment::Storage storage,
+                                                const unsigned int Ls,
+                                                Ts && ... args);
+    template <typename T, typename ... Ts>
+    void                    createObject(const std::string name,
+                                         const Environment::Storage storage,
+                                         const unsigned int Ls,
+                                         Ts && ... args);
+    void                    setObjectModule(const unsigned int objAddress,
+                                            const int modAddress);
+    template <typename B, typename T>
+    T *                     getDerivedObject(const unsigned int address) const;
+    template <typename B, typename T>
+    T *                     getDerivedObject(const std::string name) const;
     template <typename T>
     T *                     getObject(const unsigned int address) const;
     template <typename T>
     T *                     getObject(const std::string name) const;
-    template <typename T>
-    T *                     createLattice(const unsigned int address);
-    template <typename T>
-    T *                     createLattice(const std::string name);
+    unsigned int            getMaxAddress(void) const;
     unsigned int            getObjectAddress(const std::string name) const;
     std::string             getObjectName(const unsigned int address) const;
     std::string             getObjectType(const unsigned int address) const;
     std::string             getObjectType(const std::string name) const;
     Size                    getObjectSize(const unsigned int address) const;
     Size                    getObjectSize(const std::string name) const;
+    Storage                 getObjectStorage(const unsigned int address) const;
+    Storage                 getObjectStorage(const std::string name) const;
+    int                     getObjectModule(const unsigned int address) const;
+    int                     getObjectModule(const std::string name) const;
     unsigned int            getObjectLs(const unsigned int address) const;
     unsigned int            getObjectLs(const std::string name) const;
     bool                    hasObject(const unsigned int address) const;
     bool                    hasObject(const std::string name) const;
-    bool                    hasRegisteredObject(const unsigned int address) const;
-    bool                    hasRegisteredObject(const std::string name) const;
     bool                    hasCreatedObject(const unsigned int address) const;
     bool                    hasCreatedObject(const std::string name) const;
     bool                    isObject5d(const unsigned int address) const;
     bool                    isObject5d(const std::string name) const;
+    template <typename T>
+    bool                    isObjectOfType(const unsigned int address) const;
+    template <typename T>
+    bool                    isObjectOfType(const std::string name) const;
     Environment::Size       getTotalSize(void) const;
-    void                    addOwnership(const unsigned int owner,
-                                         const unsigned int property);
-    void                    addOwnership(const std::string owner,
-                                         const std::string property);
-    bool                    hasOwners(const unsigned int address) const;
-    bool                    hasOwners(const std::string name) const;
-    bool                    freeObject(const unsigned int address);
-    bool                    freeObject(const std::string name);
+    void                    freeObject(const unsigned int address);
+    void                    freeObject(const std::string name);
     void                    freeAll(void);
-    void                    printContent(void);
+    void                    protectObjects(const bool protect);
+    bool                    objectsProtected(void) const;
+    // print environment content
+    void                    printContent(void) const;
 private:
     // general
-    bool                                   dryRun_{false};
-    unsigned int                           traj_, locVol_;
+    double                                 vol_;
+    bool                                   protect_{true};
     // grids
+    std::vector<int>                       dim_;
     GridPt                                 grid4d_;
     std::map<unsigned int, GridPt>         grid5d_;
     GridRbPt                               gridRb4d_;
     std::map<unsigned int, GridRbPt>       gridRb5d_;
+    std::map<std::vector<int>, GridPt>     gridCoarse4d_;
+    std::map<std::vector<int>, GridPt>     gridCoarse5d_;
     unsigned int                           nd_;
     // random number generator
     RngPt                                  rng4d_;
-    // module and related maps
-    std::vector<ModuleInfo>                module_;
-    std::map<std::string, unsigned int>    moduleAddress_;
-    // lattice store
-    std::map<unsigned int, LatticePt>      lattice_;
     // object store
     std::vector<ObjInfo>                   object_;
     std::map<std::string, unsigned int>    objectAddress_;
@@ -227,7 +186,7 @@ Holder<T>::Holder(T *pt)
 template <typename T>
 T & Holder<T>::get(void) const
 {
-    return &objPt_.get();
+    return *objPt_.get();
 }
 
 template <typename T>
@@ -245,117 +204,116 @@ void Holder<T>::reset(T *pt)
 /******************************************************************************
  *                     Environment template implementation                    *
  ******************************************************************************/
-// module management ///////////////////////////////////////////////////////////
-template <typename M>
-void Environment::createModule(const std::string name)
+// general memory management ///////////////////////////////////////////////////
+template <typename B, typename T, typename ... Ts>
+void Environment::createDerivedObject(const std::string name,
+                                      const Environment::Storage storage,
+                                      const unsigned int Ls,
+                                      Ts && ... args)
 {
-    ModPt pt(new M(name));
-    
-    pushModule(pt);
-}
-
-template <typename M>
-void Environment::createModule(const std::string name,
-                               const typename M::Par &par)
-{
-    ModPt pt(new M(name));
-    
-    static_cast<M *>(pt.get())->setPar(par);
-    pushModule(pt);
-}
-
-template <typename M>
-M * Environment::getModule(const unsigned int address) const
-{
-    if (auto *pt = dynamic_cast<M *>(getModule(address)))
+    if (!hasObject(name))
     {
-        return pt;
+        addObject(name);
+    }
+    
+    unsigned int address = getObjectAddress(name);
+    
+    if (!object_[address].data or !objectsProtected())
+    {
+        MemoryStats memStats;
+    
+        if (!MemoryProfiler::stats)
+        {
+            MemoryProfiler::stats = &memStats;
+        }
+        size_t initMem               = MemoryProfiler::stats->currentlyAllocated;
+        object_[address].storage     = storage;
+        object_[address].Ls          = Ls;
+        object_[address].data.reset(new Holder<B>(new T(std::forward<Ts>(args)...)));
+        object_[address].size        = MemoryProfiler::stats->maxAllocated - initMem;
+        object_[address].type        = &typeid(B);
+        object_[address].derivedType = &typeid(T);
+        if (MemoryProfiler::stats == &memStats)
+        {
+            MemoryProfiler::stats = nullptr;
+        }
+    }
+    // object already exists, no error if it is a cache, error otherwise
+    else if ((object_[address].storage     != Storage::cache) or 
+             (object_[address].storage     != storage)        or
+             (object_[address].name        != name)           or
+             (object_[address].type        != &typeid(B))     or
+             (object_[address].derivedType != &typeid(T)))
+    {
+        HADRONS_ERROR(Definition, "object '" + name + "' already allocated");
+    }
+}
+
+template <typename T, typename ... Ts>
+void Environment::createObject(const std::string name, 
+                               const Environment::Storage storage,
+                               const unsigned int Ls,
+                               Ts && ... args)
+{
+    createDerivedObject<T, T>(name, storage, Ls, std::forward<Ts>(args)...);
+}
+
+template <typename B, typename T>
+T * Environment::getDerivedObject(const unsigned int address) const
+{
+    if (hasObject(address))
+    {
+        if (hasCreatedObject(address))
+        {
+            if (auto h = dynamic_cast<Holder<B> *>(object_[address].data.get()))
+            {
+                if (&typeid(T) == &typeid(B))
+                {
+                    return dynamic_cast<T *>(h->getPt());
+                }
+                else
+                {
+                    if (auto hder = dynamic_cast<T *>(h->getPt()))
+                    {
+                        return hder;
+                    }
+                    else
+                    {
+                        HADRONS_ERROR(Definition, "object with address " + std::to_string(address) +
+                            " cannot be casted to '" + typeName(&typeid(T)) +
+                            "' (has type '" + typeName(&typeid(h->get())) + "')");
+                    }
+                }
+            }
+            else
+            {
+                HADRONS_ERROR(Definition, "object with address " + std::to_string(address) +
+                            " does not have type '" + typeName(&typeid(B)) +
+                            "' (has type '" + getObjectType(address) + "')");
+            }
+        }
+        else
+        {
+            HADRONS_ERROR(Definition, "object with address " + std::to_string(address) +
+                         " is empty");
+        }
     }
     else
     {
-        HADRON_ERROR("module '" + module_[address].name
-                     + "' does not have type " + typeid(M).name()
-                     + "(object type: " + getModuleType(address) + ")");
+        HADRONS_ERROR(Definition, "no object with address " + std::to_string(address));
     }
 }
 
-template <typename M>
-M * Environment::getModule(const std::string name) const
+template <typename B, typename T>
+T * Environment::getDerivedObject(const std::string name) const
 {
-    return getModule<M>(getModuleAddress(name));
-}
-
-template <typename T>
-unsigned int Environment::lattice4dSize(void) const
-{
-    return sizeof(typename T::vector_object)/getGrid()->Nsimd();
-}
-
-template <typename T>
-void Environment::registerLattice(const unsigned int address,
-                                  const unsigned int Ls)
-{
-    createGrid(Ls);
-    registerObject(address, Ls*lattice4dSize<T>(), Ls);
-}
-
-template <typename T>
-void Environment::registerLattice(const std::string name, const unsigned int Ls)
-{
-    createGrid(Ls);
-    registerObject(name, Ls*lattice4dSize<T>(), Ls);
-}
-
-template <typename T>
-void Environment::setObject(const unsigned int address, T *object)
-{
-    if (hasRegisteredObject(address))
-    {
-        object_[address].data.reset(new Holder<T>(object));
-        object_[address].type = &typeid(T);
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address) +
-                     " exists but is not registered");
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-template <typename T>
-void Environment::setObject(const std::string name, T *object)
-{
-    setObject(getObjectAddress(name), object);
+    return getDerivedObject<B, T>(getObjectAddress(name));
 }
 
 template <typename T>
 T * Environment::getObject(const unsigned int address) const
 {
-    if (hasRegisteredObject(address))
-    {
-        if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
-        {
-            return h->getPt();
-        }
-        else
-        {
-            HADRON_ERROR("object with address " + std::to_string(address) +
-                         " does not have type '" + typeid(T).name() +
-                         "' (has type '" + getObjectType(address) + "')");
-        }
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address) +
-                     " exists but is not registered");
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
+    return getDerivedObject<T, T>(address);
 }
 
 template <typename T>
@@ -365,19 +323,29 @@ T * Environment::getObject(const std::string name) const
 }
 
 template <typename T>
-T * Environment::createLattice(const unsigned int address)
+bool Environment::isObjectOfType(const unsigned int address) const
 {
-    GridCartesian *g = getGrid(getObjectLs(address));
-    
-    setObject(address, new T(g));
-    
-    return getObject<T>(address);
+    if (hasObject(address))
+    {
+        if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
+        {
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
+    else
+    {
+        HADRONS_ERROR(Definition, "no object with address " + std::to_string(address));
+    }
 }
 
 template <typename T>
-T * Environment::createLattice(const std::string name)
+bool Environment::isObjectOfType(const std::string name) const
 {
-    return createLattice<T>(getObjectAddress(name));
+    return isObjectOfType<T>(getObjectAddress(name));
 }
 
 END_HADRONS_NAMESPACE

extras/Hadrons/Exceptions.cc

@@ -0,0 +1,81 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Exceptions.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Exceptions.hpp>
+#include <Grid/Hadrons/VirtualMachine.hpp>
+#include <Grid/Hadrons/Module.hpp>
+
+#ifndef ERR_SUFF
+#define ERR_SUFF " (" + loc + ")"
+#endif
+
+#define CONST_EXC(name, init) \
+name::name(std::string msg, std::string loc)\
+:init\
+{}
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace Exceptions;
+
+// logic errors
+CONST_EXC(Logic, logic_error(msg + ERR_SUFF))
+CONST_EXC(Definition, Logic("definition error: " + msg, loc))
+CONST_EXC(Implementation, Logic("implementation error: " + msg, loc))
+CONST_EXC(Range, Logic("range error: " + msg, loc))
+CONST_EXC(Size, Logic("size error: " + msg, loc))
+
+// runtime errors
+CONST_EXC(Runtime, runtime_error(msg + ERR_SUFF))
+CONST_EXC(Argument, Runtime("argument error: " + msg, loc))
+CONST_EXC(Io, Runtime("IO error: " + msg, loc))
+CONST_EXC(Memory, Runtime("memory error: " + msg, loc))
+CONST_EXC(Parsing, Runtime("parsing error: " + msg, loc))
+CONST_EXC(Program, Runtime("program error: " + msg, loc))
+CONST_EXC(System, Runtime("system error: " + msg, loc))
+
+// abort functions
+void Grid::Hadrons::Exceptions::abort(const std::exception& e)
+{
+    auto &vm = VirtualMachine::getInstance();
+    int  mod = vm.getCurrentModule();
+
+    LOG(Error) << "FATAL ERROR -- Exception " << typeName(&typeid(e)) 
+               << std::endl;
+    if (mod >= 0)
+    {
+        LOG(Error) << "During execution of module '"
+                    << vm.getModuleName(mod) << "' (address " << mod << ")"
+                    << std::endl;
+    }
+    LOG(Error) << e.what() << std::endl;
+    LOG(Error) << "Aborting program" << std::endl;
+    Grid_finalize();
+
+    exit(EXIT_FAILURE);
+}

extras/Hadrons/Exceptions.hpp

@@ -0,0 +1,75 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Exceptions.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Exceptions_hpp_
+#define Hadrons_Exceptions_hpp_
+
+#include <stdexcept>
+#ifndef Hadrons_Global_hpp_
+#include <Grid/Hadrons/Global.hpp>
+#endif
+
+#define HADRONS_SRC_LOC std::string(__FUNCTION__) + " at " \
+                        + std::string(__FILE__) + ":" + std::to_string(__LINE__)
+#define HADRONS_ERROR(exc, msg)\
+throw(Exceptions::exc(msg, HADRONS_SRC_LOC));
+
+#define DECL_EXC(name, base) \
+class name: public base\
+{\
+public:\
+    name(std::string msg, std::string loc);\
+}
+
+BEGIN_HADRONS_NAMESPACE
+
+namespace Exceptions
+{
+    // logic errors
+    DECL_EXC(Logic, std::logic_error);
+    DECL_EXC(Definition, Logic);
+    DECL_EXC(Implementation, Logic);
+    DECL_EXC(Range, Logic);
+    DECL_EXC(Size, Logic);
+
+    // runtime errors
+    DECL_EXC(Runtime, std::runtime_error);
+    DECL_EXC(Argument, Runtime);
+    DECL_EXC(Io, Runtime);
+    DECL_EXC(Memory, Runtime);
+    DECL_EXC(Parsing, Runtime);
+    DECL_EXC(Program, Runtime);
+    DECL_EXC(System, Runtime);
+
+    // abort functions
+    void abort(const std::exception& e);
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Exceptions_hpp_

extras/Hadrons/Factory.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Factory.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -95,7 +94,7 @@ std::unique_ptr<T> Factory<T>::create(const std::string type,
     }
     catch (std::out_of_range &)
     {
-        HADRON_ERROR("object of type '" + type + "' unknown");
+        HADRONS_ERROR(Argument, "object of type '" + type + "' unknown");
     }
     
     return func(name);

extras/Hadrons/GeneticScheduler.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/GeneticScheduler.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -38,13 +37,13 @@ BEGIN_HADRONS_NAMESPACE
 /******************************************************************************
  *                   Scheduler based on a genetic algorithm                   *
  ******************************************************************************/
-template <typename T>
+template <typename V, typename T>
 class GeneticScheduler
 {
 public:
-    typedef std::vector<T>                   Gene;
-    typedef std::pair<Gene *, Gene *>        GenePair;
-    typedef std::function<int(const Gene &)> ObjFunc;
+    typedef std::vector<T>                 Gene;
+    typedef std::pair<Gene *, Gene *>      GenePair;
+    typedef std::function<V(const Gene &)> ObjFunc;
     struct Parameters
     {
         double       mutationRate;
@@ -58,14 +57,16 @@ public:
     virtual ~GeneticScheduler(void) = default;
     // access
     const Gene & getMinSchedule(void);
-    int          getMinValue(void);
+    V            getMinValue(void);
+    // reset population
+    void initPopulation(void);
     // breed a new generation
     void nextGeneration(void);
     // heuristic benchmarks
     void benchmarkCrossover(const unsigned int nIt);
     // print population
     friend std::ostream & operator<<(std::ostream &out,
-                                     const GeneticScheduler<T> &s)
+                                     const GeneticScheduler<V, T> &s)
     {
         out << "[";
         for (auto &p: s.population_)
@@ -77,8 +78,6 @@ public:
         return out;
     }
 private:
-    // evolution steps
-    void initPopulation(void);
     void doCrossover(void);
     void doMutation(void);
     // genetic operators
@@ -87,19 +86,19 @@ private:
     void     mutation(Gene &m, const Gene &c);
     
 private:
-    Graph<T>                 &graph_;
-    const ObjFunc            &func_;
-    const Parameters         par_;
-    std::multimap<int, Gene> population_;
-    std::mt19937             gen_;
+    Graph<T>               &graph_;
+    const ObjFunc          &func_;
+    const Parameters       par_;
+    std::multimap<V, Gene> population_;
+    std::mt19937           gen_;
 };
 
 /******************************************************************************
  *                       template implementation                              *
  ******************************************************************************/
 // constructor /////////////////////////////////////////////////////////////////
-template <typename T>
-GeneticScheduler<T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
+template <typename V, typename T>
+GeneticScheduler<V, T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
                                       const Parameters &par)
 : graph_(graph)
 , func_(func)
@@ -109,29 +108,29 @@ GeneticScheduler<T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
 }
 
 // access //////////////////////////////////////////////////////////////////////
-template <typename T>
-const typename GeneticScheduler<T>::Gene &
-GeneticScheduler<T>::getMinSchedule(void)
+template <typename V, typename T>
+const typename GeneticScheduler<V, T>::Gene &
+GeneticScheduler<V, T>::getMinSchedule(void)
 {
     return population_.begin()->second;
 }
 
-template <typename T>
-int GeneticScheduler<T>::getMinValue(void)
+template <typename V, typename T>
+V GeneticScheduler<V, T>::getMinValue(void)
 {
     return population_.begin()->first;
 }
 
 // breed a new generation //////////////////////////////////////////////////////
-template <typename T>
-void GeneticScheduler<T>::nextGeneration(void)
+template <typename V, typename T>
+void GeneticScheduler<V, T>::nextGeneration(void)
 {
     // random initialization of the population if necessary
     if (population_.size() != par_.popSize)
     {
         initPopulation();
     }
-    LOG(Debug) << "Starting population:\n" << *this << std::endl;
+    //LOG(Debug) << "Starting population:\n" << *this << std::endl;
     
     // random mutations
     //PARALLEL_FOR_LOOP
@@ -139,7 +138,7 @@ void GeneticScheduler<T>::nextGeneration(void)
     {
         doMutation();
     }
-    LOG(Debug) << "After mutations:\n" << *this << std::endl;
+    //LOG(Debug) << "After mutations:\n" << *this << std::endl;
     
     // mating
     //PARALLEL_FOR_LOOP
@@ -147,19 +146,19 @@ void GeneticScheduler<T>::nextGeneration(void)
     {
         doCrossover();
     }
-    LOG(Debug) << "After mating:\n" << *this << std::endl;
+    //LOG(Debug) << "After mating:\n" << *this << std::endl;
     
     // grim reaper
     auto it = population_.begin();
     
     std::advance(it, par_.popSize);
     population_.erase(it, population_.end());
-    LOG(Debug) << "After grim reaper:\n" << *this << std::endl;
+    //LOG(Debug) << "After grim reaper:\n" << *this << std::endl;
 }
 
 // evolution steps /////////////////////////////////////////////////////////////
-template <typename T>
-void GeneticScheduler<T>::initPopulation(void)
+template <typename V, typename T>
+void GeneticScheduler<V, T>::initPopulation(void)
 {
     population_.clear();
     for (unsigned int i = 0; i < par_.popSize; ++i)
@@ -170,8 +169,8 @@ void GeneticScheduler<T>::initPopulation(void)
     }
 }
 
-template <typename T>
-void GeneticScheduler<T>::doCrossover(void)
+template <typename V, typename T>
+void GeneticScheduler<V, T>::doCrossover(void)
 {
     auto p = selectPair();
     Gene &p1 = *(p.first), &p2 = *(p.second);
@@ -185,8 +184,8 @@ void GeneticScheduler<T>::doCrossover(void)
     }
 }
 
-template <typename T>
-void GeneticScheduler<T>::doMutation(void)
+template <typename V, typename T>
+void GeneticScheduler<V, T>::doMutation(void)
 {
     std::uniform_real_distribution<double>      mdis(0., 1.);
     std::uniform_int_distribution<unsigned int> pdis(0, population_.size() - 1);
@@ -206,40 +205,35 @@ void GeneticScheduler<T>::doMutation(void)
 }
 
 // genetic operators ///////////////////////////////////////////////////////////
-template <typename T>
-typename GeneticScheduler<T>::GenePair GeneticScheduler<T>::selectPair(void)
+template <typename V, typename T>
+typename GeneticScheduler<V, T>::GenePair GeneticScheduler<V, T>::selectPair(void)
 {
     std::vector<double> prob;
     unsigned int        ind;
     Gene                *p1, *p2;
+    const double        max = population_.rbegin()->first;
     
+
     for (auto &c: population_)
     {
-        prob.push_back(1./c.first);
-    }
-    do
-    {
-        double probCpy;
-        
-        std::discrete_distribution<unsigned int> dis1(prob.begin(), prob.end());
-        auto rIt = population_.begin();
-        ind = dis1(gen_);
-        std::advance(rIt, ind);
-        p1 = &(rIt->second);
-        probCpy   = prob[ind];
-        prob[ind] = 0.;
-        std::discrete_distribution<unsigned int> dis2(prob.begin(), prob.end());
-        rIt = population_.begin();
-        std::advance(rIt, dis2(gen_));
-        p2 = &(rIt->second);
-        prob[ind] = probCpy;
-    } while (p1 == p2);
+        prob.push_back(std::exp((c.first-1.)/max));
+    }        
+    std::discrete_distribution<unsigned int> dis1(prob.begin(), prob.end());
+    auto rIt = population_.begin();
+    ind = dis1(gen_);
+    std::advance(rIt, ind);
+    p1 = &(rIt->second);
+    prob[ind] = 0.;
+    std::discrete_distribution<unsigned int> dis2(prob.begin(), prob.end());
+    rIt = population_.begin();
+    std::advance(rIt, dis2(gen_));
+    p2 = &(rIt->second);
     
     return std::make_pair(p1, p2);
 }
 
-template <typename T>
-void GeneticScheduler<T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
+template <typename V, typename T>
+void GeneticScheduler<V, T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
                                     const Gene &p2)
 {
     Gene                                        buf;
@@ -273,8 +267,8 @@ void GeneticScheduler<T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
     }
 }
 
-template <typename T>
-void GeneticScheduler<T>::mutation(Gene &m, const Gene &c)
+template <typename V, typename T>
+void GeneticScheduler<V, T>::mutation(Gene &m, const Gene &c)
 {
     Gene                                        buf;
     std::uniform_int_distribution<unsigned int> dis(0, c.size() - 1);
@@ -303,8 +297,8 @@ void GeneticScheduler<T>::mutation(Gene &m, const Gene &c)
     }
 }
 
-template <typename T>
-void GeneticScheduler<T>::benchmarkCrossover(const unsigned int nIt)
+template <typename V, typename T>
+void GeneticScheduler<V, T>::benchmarkCrossover(const unsigned int nIt)
 {
     Gene   p1, p2, c1, c2;
     double neg = 0., eq = 0., pos = 0., total;

extras/Hadrons/Global.cc

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Global.cc
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -38,32 +37,38 @@ HadronsLogger Hadrons::HadronsLogWarning(1,"Warning");
 HadronsLogger Hadrons::HadronsLogMessage(1,"Message");
 HadronsLogger Hadrons::HadronsLogIterative(1,"Iterative");
 HadronsLogger Hadrons::HadronsLogDebug(1,"Debug");
+HadronsLogger Hadrons::HadronsLogIRL(1,"IRL");
 
-// pretty size formatting //////////////////////////////////////////////////////
-std::string Hadrons::sizeString(long unsigned int bytes)
-
+void Hadrons::initLogger(void)
 {
-    constexpr unsigned int bufSize = 256;
-    const char             *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
-    char                   buf[256];
-    long unsigned int      s     = 0;
-    double                 count = bytes;
-    
-    while (count >= 1024 && s < 7)
-    {
-        s++;
-        count /= 1024;
-    }
-    if (count - floor(count) == 0.0)
-    {
-        snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
-    }
-    else
-    {
-        snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
-    }
-    
-    return std::string(buf);
+    auto w  = std::string("Hadrons").length();
+    int  cw = 8;
+
+
+    GridLogError.setTopWidth(w);
+    GridLogWarning.setTopWidth(w);
+    GridLogMessage.setTopWidth(w);
+    GridLogIterative.setTopWidth(w);
+    GridLogDebug.setTopWidth(w);
+    GridLogIRL.setTopWidth(w);
+    GridLogError.setChanWidth(cw);
+    GridLogWarning.setChanWidth(cw);
+    GridLogMessage.setChanWidth(cw);
+    GridLogIterative.setChanWidth(cw);
+    GridLogDebug.setChanWidth(cw);
+    GridLogIRL.setChanWidth(cw);
+    HadronsLogError.Active(true);
+    HadronsLogWarning.Active(true);
+    HadronsLogMessage.Active(GridLogMessage.isActive());
+    HadronsLogIterative.Active(GridLogIterative.isActive());
+    HadronsLogDebug.Active(GridLogDebug.isActive());
+    HadronsLogIRL.Active(GridLogIRL.isActive());
+    HadronsLogError.setChanWidth(cw);
+    HadronsLogWarning.setChanWidth(cw);
+    HadronsLogMessage.setChanWidth(cw);
+    HadronsLogIterative.setChanWidth(cw);
+    HadronsLogDebug.setChanWidth(cw);
+    HadronsLogIRL.setChanWidth(cw);
 }
 
 // type utilities //////////////////////////////////////////////////////////////
@@ -80,3 +85,91 @@ std::string Hadrons::typeName(const std::type_info *info)
     
     return name;
 }
+
+// default writers/readers /////////////////////////////////////////////////////
+#ifdef HAVE_HDF5
+const std::string Hadrons::resultFileExt = "h5";
+#else
+const std::string Hadrons::resultFileExt = "xml";
+#endif
+
+// recursive mkdir /////////////////////////////////////////////////////////////
+int Hadrons::mkdir(const std::string dirName)
+{
+    if (!dirName.empty() and access(dirName.c_str(), R_OK|W_OK|X_OK))
+    {
+        mode_t mode755;
+        char   tmp[MAX_PATH_LENGTH];
+        char   *p = NULL;
+        size_t len;
+
+        mode755 = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH;
+
+        snprintf(tmp, sizeof(tmp), "%s", dirName.c_str());
+        len = strlen(tmp);
+        if(tmp[len - 1] == '/')
+        {
+            tmp[len - 1] = 0;
+        }
+        for(p = tmp + 1; *p; p++)
+        {
+            if(*p == '/')
+            {
+                *p = 0;
+                ::mkdir(tmp, mode755);
+                *p = '/';
+            }
+        }
+
+        return ::mkdir(tmp, mode755);
+    }
+    else
+    {
+        return 0;
+    }
+}
+
+std::string Hadrons::basename(const std::string &s)
+{
+    constexpr char sep = '/';
+    size_t         i   = s.rfind(sep, s.length());
+    
+    if (i != std::string::npos)
+    {
+        return s.substr(i+1, s.length() - i);
+    }
+    else
+    {
+        return s;
+    }
+}
+
+std::string Hadrons::dirname(const std::string &s)
+{
+    constexpr char sep = '/';
+    size_t         i   = s.rfind(sep, s.length());
+    
+    if (i != std::string::npos)
+    {
+        return s.substr(0, i);
+    }
+    else
+    {
+        return "";
+    }
+}
+
+void Hadrons::makeFileDir(const std::string filename, GridBase *g)
+{
+    if (g->IsBoss())
+    {
+        std::string dir    = dirname(filename);
+        int         status = mkdir(dir);
+
+        if (status)
+        {
+            HADRONS_ERROR(Io, "cannot create directory '" + dir
+                          + "' ( " + std::strerror(errno) + ")");
+        }
+    }
+}

extras/Hadrons/Global.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Global.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -35,38 +35,73 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #include <Grid/Grid.h>
 #include <cxxabi.h>
 
-#define BEGIN_HADRONS_NAMESPACE \
-namespace Grid {\
-using namespace QCD;\
-namespace Hadrons {\
-using Grid::operator<<;
-#define END_HADRONS_NAMESPACE }}
+#ifndef SITE_SIZE_TYPE
+#define SITE_SIZE_TYPE size_t
+#endif
 
-#define BEGIN_MODULE_NAMESPACE(name)\
-namespace name {\
-using Grid::operator<<;
-#define END_MODULE_NAMESPACE }
+#ifndef DEFAULT_ASCII_PREC
+#define DEFAULT_ASCII_PREC 16
+#endif
 
 /* the 'using Grid::operator<<;' statement prevents a very nasty compilation
  * error with GCC 5 (clang & GCC 6 compile fine without it).
  */
 
-// FIXME: find a way to do that in a more general fashion
+#define BEGIN_HADRONS_NAMESPACE \
+namespace Grid {\
+using namespace QCD;\
+namespace Hadrons {\
+using Grid::operator<<;\
+using Grid::operator>>;
+#define END_HADRONS_NAMESPACE }}
+
+#define BEGIN_MODULE_NAMESPACE(name)\
+namespace name {\
+using Grid::operator<<;\
+using Grid::operator>>;
+
+#define END_MODULE_NAMESPACE }
+
 #ifndef FIMPL
 #define FIMPL WilsonImplR
 #endif
+#ifndef ZFIMPL
+#define ZFIMPL ZWilsonImplR
+#endif
+#ifndef SIMPL
+#define SIMPL ScalarImplCR
+#endif
+#ifndef GIMPL
+#define GIMPL PeriodicGimplR
+#endif
 
 BEGIN_HADRONS_NAMESPACE
 
 // type aliases
-#define TYPE_ALIASES(FImpl, suffix)\
-typedef FermionOperator<FImpl>                       FMat##suffix;             \
-typedef typename FImpl::FermionField                 FermionField##suffix;     \
-typedef typename FImpl::PropagatorField              PropagatorField##suffix;  \
-typedef typename FImpl::SitePropagator               SitePropagator##suffix;   \
-typedef typename FImpl::DoubledGaugeField            DoubledGaugeField##suffix;\
-typedef std::function<void(FermionField##suffix &,                             \
-                      const FermionField##suffix &)> SolverFn##suffix;
+#define FERM_TYPE_ALIASES(FImpl, suffix)\
+typedef FermionOperator<FImpl>                        FMat##suffix;            \
+typedef typename FImpl::FermionField                  FermionField##suffix;    \
+typedef typename FImpl::PropagatorField               PropagatorField##suffix; \
+typedef typename FImpl::SitePropagator::scalar_object SitePropagator##suffix;  \
+typedef std::vector<SitePropagator##suffix>           SlicedPropagator##suffix;
+
+#define GAUGE_TYPE_ALIASES(FImpl, suffix)\
+typedef typename FImpl::DoubledGaugeField DoubledGaugeField##suffix;
+
+#define SCALAR_TYPE_ALIASES(SImpl, suffix)\
+typedef typename SImpl::Field ScalarField##suffix;\
+typedef typename SImpl::Field PropagatorField##suffix;
+
+#define SOLVER_TYPE_ALIASES(FImpl, suffix)\
+typedef Solver<FImpl> Solver##suffix;
+
+#define SINK_TYPE_ALIASES(suffix)\
+typedef std::function<SlicedPropagator##suffix\
+                      (const PropagatorField##suffix &)> SinkFn##suffix;
+
+#define FG_TYPE_ALIASES(FImpl, suffix)\
+FERM_TYPE_ALIASES(FImpl, suffix)\
+GAUGE_TYPE_ALIASES(FImpl, suffix)
 
 // logger
 class HadronsLogger: public Logger
@@ -77,18 +112,16 @@ public:
 };
 
 #define LOG(channel) std::cout << HadronsLog##channel
-#define HADRON_ERROR(msg)\
-LOG(Error) << msg << " (" << __FUNCTION__ << " at " << __FILE__ << ":"\
-           << __LINE__ << ")" << std::endl;\
-abort();
-
-#define DEBUG_VAR(var) LOG(Debug) << #var << "= " << (var) << std::endl;
+#define HADRONS_DEBUG_VAR(var) LOG(Debug) << #var << "= " << (var) << std::endl;
 
 extern HadronsLogger HadronsLogError;
 extern HadronsLogger HadronsLogWarning;
 extern HadronsLogger HadronsLogMessage;
 extern HadronsLogger HadronsLogIterative;
 extern HadronsLogger HadronsLogDebug;
+extern HadronsLogger HadronsLogIRL;
+
+void initLogger(void);
 
 // singleton pattern
 #define SINGLETON(name)\
@@ -115,9 +148,6 @@ public:\
 private:\
     name(void) = default;
 
-// pretty size formating
-std::string sizeString(long unsigned int bytes);
-
 // type utilities
 template <typename T>
 const std::type_info * typeIdPt(const T &x)
@@ -146,14 +176,43 @@ std::string typeName(void)
 }
 
 // default writers/readers
+extern const std::string resultFileExt;
+
 #ifdef HAVE_HDF5
-typedef Hdf5Reader CorrReader;
-typedef Hdf5Writer CorrWriter;
+typedef Hdf5Reader ResultReader;
+typedef Hdf5Writer ResultWriter;
 #else
-typedef XmlReader CorrReader;
-typedef XmlWriter CorrWriter;
+typedef XmlReader ResultReader;
+typedef XmlWriter ResultWriter;
 #endif
 
+#define RESULT_FILE_NAME(name) \
+name + "." + std::to_string(vm().getTrajectory()) + "." + resultFileExt
+
+// recursive mkdir
+#define MAX_PATH_LENGTH 512u
+int         mkdir(const std::string dirName);
+std::string basename(const std::string &s);
+std::string dirname(const std::string &s);
+void        makeFileDir(const std::string filename, GridBase *g);
+
+// default Schur convention
+#ifndef HADRONS_DEFAULT_SCHUR 
+#define HADRONS_DEFAULT_SCHUR DiagTwo
+#endif
+#define _HADRONS_SCHUR_OP_(conv) Schur##conv##Operator
+#define HADRONS_SCHUR_OP(conv) _HADRONS_SCHUR_OP_(conv)
+#define HADRONS_DEFAULT_SCHUR_OP HADRONS_SCHUR_OP(HADRONS_DEFAULT_SCHUR)
+#define _HADRONS_SCHUR_SOLVE_(conv) SchurRedBlack##conv##Solve
+#define HADRONS_SCHUR_SOLVE(conv) _HADRONS_SCHUR_SOLVE_(conv)
+#define HADRONS_DEFAULT_SCHUR_SOLVE HADRONS_SCHUR_SOLVE(HADRONS_DEFAULT_SCHUR)
+
+// stringify macro
+#define _HADRONS_STR(x) #x
+#define HADRONS_STR(x) _HADRONS_STR(x)
+
 END_HADRONS_NAMESPACE
 
+#include <Grid/Hadrons/Exceptions.hpp>
+
 #endif // Hadrons_Global_hpp_

extras/Hadrons/Graph.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Graph.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -185,7 +184,7 @@ void Graph<T>::removeVertex(const T &value)
     }
     else
     {
-        HADRON_ERROR("vertex " << value << " does not exists");
+        HADRONS_ERROR(Range, "vertex does not exists");
     }
 
     // remove all edges containing the vertex
@@ -214,7 +213,7 @@ void Graph<T>::removeEdge(const Edge &e)
     }
     else
     {
-        HADRON_ERROR("edge "  << e << " does not exists");
+        HADRONS_ERROR(Range, "edge does not exists");
     }
 }
 
@@ -260,7 +259,7 @@ void Graph<T>::mark(const T &value, const bool doMark)
     }
     else
     {
-        HADRON_ERROR("vertex " << value << " does not exists");
+        HADRONS_ERROR(Range, "vertex does not exists");
     }
 }
 
@@ -298,7 +297,7 @@ bool Graph<T>::isMarked(const T &value) const
     }
     else
     {
-        HADRON_ERROR("vertex " << value << " does not exists");
+        HADRONS_ERROR(Range, "vertex does not exists");
         
         return false;
     }
@@ -430,7 +429,7 @@ std::vector<T> Graph<T>::getAdjacentVertices(const T &value) const
     {
         return ((e.first == value) or (e.second == value));
     };
-    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    auto eIt = std::find_if(edgeSet_.begin(), edgeSet_.end(), pred);
     
     while (eIt != edgeSet_.end())
     {
@@ -442,7 +441,7 @@ std::vector<T> Graph<T>::getAdjacentVertices(const T &value) const
         {
             adjacentVertex.push_back((*eIt).first);
         }
-        eIt = find_if(++eIt, edgeSet_.end(), pred);
+        eIt = std::find_if(++eIt, edgeSet_.end(), pred);
     }
     
     return adjacentVertex;
@@ -458,12 +457,12 @@ std::vector<T> Graph<T>::getChildren(const T &value) const
     {
         return (e.first == value);
     };
-    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    auto eIt = std::find_if(edgeSet_.begin(), edgeSet_.end(), pred);
     
     while (eIt != edgeSet_.end())
     {
         child.push_back((*eIt).second);
-        eIt = find_if(++eIt, edgeSet_.end(), pred);
+        eIt = std::find_if(++eIt, edgeSet_.end(), pred);
     }
     
     return child;
@@ -479,12 +478,12 @@ std::vector<T> Graph<T>::getParents(const T &value) const
     {
         return (e.second == value);
     };
-    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    auto eIt = std::find_if(edgeSet_.begin(), edgeSet_.end(), pred);
     
     while (eIt != edgeSet_.end())
     {
         parent.push_back((*eIt).first);
-        eIt = find_if(++eIt, edgeSet_.end(), pred);
+        eIt = std::find_if(++eIt, edgeSet_.end(), pred);
     }
     
     return parent;
@@ -544,7 +543,7 @@ std::vector<T> Graph<T>::topoSort(void)
     {
         if (tmpMarked.at(v))
         {
-            HADRON_ERROR("cannot topologically sort a cyclic graph");
+            HADRONS_ERROR(Range, "cannot topologically sort a cyclic graph");
         }
         if (!isMarked(v))
         {
@@ -603,7 +602,7 @@ std::vector<T> Graph<T>::topoSort(Gen &gen)
     {
         if (tmpMarked.at(v))
         {
-            HADRON_ERROR("cannot topologically sort a cyclic graph");
+            HADRONS_ERROR(Range, "cannot topologically sort a cyclic graph");
         }
         if (!isMarked(v))
         {

extras/Hadrons/HadronsXmlRun.cc

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/HadronsXmlRun.cc
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -55,22 +54,23 @@ int main(int argc, char *argv[])
     
     // initialization
     Grid_init(&argc, &argv);
-    HadronsLogError.Active(GridLogError.isActive());
-    HadronsLogWarning.Active(GridLogWarning.isActive());
-    HadronsLogMessage.Active(GridLogMessage.isActive());
-    HadronsLogIterative.Active(GridLogIterative.isActive());
-    HadronsLogDebug.Active(GridLogDebug.isActive());
-    LOG(Message) << "Grid initialized" << std::endl;
     
     // execution
-    Application application(parameterFileName);
-    
-    application.parseParameterFile(parameterFileName);
-    if (!scheduleFileName.empty())
+    try
     {
-        application.loadSchedule(scheduleFileName);
+        Application application(parameterFileName);
+        
+        application.parseParameterFile(parameterFileName);
+        if (!scheduleFileName.empty())
+        {
+            application.loadSchedule(scheduleFileName);
+        }
+        application.run();
+    }
+    catch (const std::exception& e)
+    {
+        Exceptions::abort(e);
     }
-    application.run();
     
     // epilogue
     LOG(Message) << "Grid is finalizing now" << std::endl;

extras/Hadrons/HadronsXmlSchedule.cc

@@ -1,72 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/HadronsXmlSchedule.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Application.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-int main(int argc, char *argv[])
-{
-    // parse command line
-    std::string parameterFileName, scheduleFileName;
-    
-    if (argc < 3)
-    {
-        std::cerr << "usage: " << argv[0] << " <parameter file> <schedule output> [Grid options]";
-        std::cerr << std::endl;
-        std::exit(EXIT_FAILURE);
-    }
-    parameterFileName = argv[1];
-    scheduleFileName  = argv[2];
-    
-    // initialization
-    Grid_init(&argc, &argv);
-    HadronsLogError.Active(GridLogError.isActive());
-    HadronsLogWarning.Active(GridLogWarning.isActive());
-    HadronsLogMessage.Active(GridLogMessage.isActive());
-    HadronsLogIterative.Active(GridLogIterative.isActive());
-    HadronsLogDebug.Active(GridLogDebug.isActive());
-    LOG(Message) << "Grid initialized" << std::endl;
-    
-    // execution
-    Application application;
-    
-    application.parseParameterFile(parameterFileName);
-    application.schedule();
-    application.printSchedule();
-    application.saveSchedule(scheduleFileName);
-    
-    // epilogue
-    LOG(Message) << "Grid is finalizing now" << std::endl;
-    Grid_finalize();
-    
-    return EXIT_SUCCESS;
-}

extras/Hadrons/Makefile.am

@@ -1,5 +1,5 @@
 lib_LIBRARIES = libHadrons.a
-bin_PROGRAMS  = HadronsXmlRun HadronsXmlSchedule
+bin_PROGRAMS  = HadronsXmlRun
 
 include modules.inc
 
@@ -7,23 +7,28 @@ libHadrons_a_SOURCES = \
     $(modules_cc)      \
     Application.cc     \
     Environment.cc     \
+	Exceptions.cc      \
     Global.cc          \
-    Module.cc
+    Module.cc		   \
+	VirtualMachine.cc
 libHadrons_adir = $(pkgincludedir)/Hadrons
 nobase_libHadrons_a_HEADERS = \
 	$(modules_hpp)            \
+	AllToAllVectors.hpp       \
+	AllToAllReduction.hpp       \
 	Application.hpp           \
+	EigenPack.hpp             \
 	Environment.hpp           \
+	Exceptions.hpp            \
 	Factory.hpp               \
 	GeneticScheduler.hpp      \
 	Global.hpp                \
 	Graph.hpp                 \
 	Module.hpp                \
 	Modules.hpp               \
-	ModuleFactory.hpp
+	ModuleFactory.hpp         \
+	Solver.hpp                \
+	VirtualMachine.hpp
 
 HadronsXmlRun_SOURCES = HadronsXmlRun.cc
 HadronsXmlRun_LDADD   = libHadrons.a -lGrid
-
-HadronsXmlSchedule_SOURCES = HadronsXmlSchedule.cc
-HadronsXmlSchedule_LDADD   = libHadrons.a -lGrid

extras/Hadrons/Module.cc

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Module.cc
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -39,7 +38,6 @@ using namespace Hadrons;
 // constructor /////////////////////////////////////////////////////////////////
 ModuleBase::ModuleBase(const std::string name)
 : name_(name)
-, env_(Environment::getInstance())
 {}
 
 // access //////////////////////////////////////////////////////////////////////
@@ -48,15 +46,10 @@ std::string ModuleBase::getName(void) const
     return name_;
 }
 
-Environment & ModuleBase::env(void) const
-{
-    return env_;
-}
-
 // get factory registration name if available
 std::string ModuleBase::getRegisteredName(void)
 {
-    HADRON_ERROR("module '" + getName() + "' has a type not registered"
+    HADRONS_ERROR(Definition, "module '" + getName() + "' has no registered type"
                  + " in the factory");
 }
 
@@ -64,8 +57,5 @@ std::string ModuleBase::getRegisteredName(void)
 void ModuleBase::operator()(void)
 {
     setup();
-    if (!env().isDryRun())
-    {
-        execute();
-    }
+    execute();
 }

extras/Hadrons/Module.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Module.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -31,37 +30,12 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #define Hadrons_Module_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Environment.hpp>
+#include <Grid/Hadrons/VirtualMachine.hpp>
 
 BEGIN_HADRONS_NAMESPACE
 
 // module registration macros
-#define MODULE_REGISTER(mod, base)\
-class mod: public base\
-{\
-public:\
-    typedef base Base;\
-    using Base::Base;\
-    virtual std::string getRegisteredName(void)\
-    {\
-        return std::string(#mod);\
-    }\
-};\
-class mod##ModuleRegistrar\
-{\
-public:\
-    mod##ModuleRegistrar(void)\
-    {\
-        ModuleFactory &modFac = ModuleFactory::getInstance();\
-        modFac.registerBuilder(#mod, [&](const std::string name)\
-                              {\
-                                  return std::unique_ptr<mod>(new mod(name));\
-                              });\
-    }\
-};\
-static mod##ModuleRegistrar mod##ModuleRegistrarInstance;
-
-#define MODULE_REGISTER_NS(mod, base, ns)\
+#define MODULE_REGISTER(mod, base, ns)\
 class mod: public base\
 {\
 public:\
@@ -86,7 +60,74 @@ public:\
 };\
 static ns##mod##ModuleRegistrar ns##mod##ModuleRegistrarInstance;
 
+#define MODULE_REGISTER_TMP(mod, base, ns)\
+extern template class base;\
+MODULE_REGISTER(mod, ARG(base), ns);
+
 #define ARG(...) __VA_ARGS__
+#define MACRO_REDIRECT(arg1, arg2, arg3, macro, ...) macro
+
+#define envGet(type, name)\
+*env().template getObject<type>(name)
+
+#define envGetDerived(base, type, name)\
+*env().template getDerivedObject<base, type>(name)
+
+#define envGetTmp(type, var)\
+type &var = *env().template getObject<type>(getName() + "_tmp_" + #var)
+
+#define envHasType(type, name)\
+env().template isObjectOfType<type>(name)
+
+#define envCreate(type, name, Ls, ...)\
+env().template createObject<type>(name, Environment::Storage::object, Ls, __VA_ARGS__)
+
+#define envCreateDerived(base, type, name, Ls, ...)\
+env().template createDerivedObject<base, type>(name, Environment::Storage::object, Ls, __VA_ARGS__)
+
+#define envCreateLat4(type, name)\
+envCreate(type, name, 1, env().getGrid())
+
+#define envCreateLat5(type, name, Ls)\
+envCreate(type, name, Ls, env().getGrid(Ls))
+
+#define envCreateLat(...)\
+MACRO_REDIRECT(__VA_ARGS__, envCreateLat5, envCreateLat4)(__VA_ARGS__)
+
+#define envCache(type, name, Ls, ...)\
+env().template createObject<type>(name, Environment::Storage::cache, Ls, __VA_ARGS__)
+
+#define envCacheLat4(type, name)\
+envCache(type, name, 1, env().getGrid())
+
+#define envCacheLat5(type, name, Ls)\
+envCache(type, name, Ls, env().getGrid(Ls))
+
+#define envCacheLat(...)\
+MACRO_REDIRECT(__VA_ARGS__, envCacheLat5, envCacheLat4)(__VA_ARGS__)
+
+#define envTmp(type, name, Ls, ...)\
+env().template createObject<type>(getName() + "_tmp_" + name,         \
+                                  Environment::Storage::temporary, Ls, __VA_ARGS__)
+
+#define envTmpLat4(type, name)\
+envTmp(type, name, 1, env().getGrid())
+
+#define envTmpLat5(type, name, Ls)\
+envTmp(type, name, Ls, env().getGrid(Ls))
+
+#define envTmpLat(...)\
+MACRO_REDIRECT(__VA_ARGS__, envTmpLat5, envTmpLat4)(__VA_ARGS__)
+
+#define saveResult(ioStem, name, result)\
+if (env().getGrid()->IsBoss() and !ioStem.empty())\
+{\
+    makeFileDir(ioStem, env().getGrid());\
+    {\
+        ResultWriter _writer(RESULT_FILE_NAME(ioStem));\
+        write(_writer, name, result);\
+    }\
+}
 
 /******************************************************************************
  *                            Module class                                    *
@@ -101,23 +142,32 @@ public:
     virtual ~ModuleBase(void) = default;
     // access
     std::string getName(void) const;
-    Environment &env(void) const;
     // get factory registration name if available
     virtual std::string getRegisteredName(void);
     // dependencies/products
     virtual std::vector<std::string> getInput(void) = 0;
+    virtual std::vector<std::string> getReference(void)
+    {
+        return std::vector<std::string>(0);
+    };
     virtual std::vector<std::string> getOutput(void) = 0;
     // parse parameters
     virtual void parseParameters(XmlReader &reader, const std::string name) = 0;
     virtual void saveParameters(XmlWriter &writer, const std::string name) = 0;
+    // parameter string
+    virtual std::string parString(void) const = 0;
     // setup
     virtual void setup(void) {};
+    virtual void execute(void) = 0;
     // execution
     void operator()(void);
-    virtual void execute(void) = 0;
+protected:
+    // environment shortcut
+    DEFINE_ENV_ALIAS;
+    // virtual machine shortcut
+    DEFINE_VM_ALIAS;
 private:
     std::string name_;
-    Environment &env_;
 };
 
 // derived class, templating the parameter class
@@ -134,9 +184,11 @@ public:
     // parse parameters
     virtual void parseParameters(XmlReader &reader, const std::string name);
     virtual void saveParameters(XmlWriter &writer, const std::string name);
+    // parameter string
+    virtual std::string parString(void) const;
     // parameter access
-    const P & par(void) const;
-    void      setPar(const P &par);
+    const P &   par(void) const;
+    void        setPar(const P &par);
 private:
     P par_;
 };
@@ -159,6 +211,8 @@ public:
         push(writer, "options");
         pop(writer);
     };
+    // parameter string (empty)
+    virtual std::string parString(void) const {return "";};
 };
 
 /******************************************************************************
@@ -181,6 +235,16 @@ void Module<P>::saveParameters(XmlWriter &writer, const std::string name)
     write(writer, name, par_);
 }
 
+template <typename P>
+std::string Module<P>::parString(void) const
+{
+    XmlWriter writer("", "");
+
+    write(writer, par_.SerialisableClassName(), par_);
+
+    return writer.string();
+}
+
 template <typename P>
 const P & Module<P>::par(void) const
 {

extras/Hadrons/ModuleFactory.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/ModuleFactory.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 

extras/Hadrons/Modules.hpp

@@ -1,48 +1,62 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/Hadrons/Modules/MAction/DWF.hpp>
-#include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/TrKinetic.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/TimeMomProbe.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/StochFreeField.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/TwoPointNPR.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/Grad.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/TransProj.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/Div.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/TrMag.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/ShiftProbe.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/Utils.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/EMT.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/TwoPoint.hpp>
+#include <Grid/Hadrons/Modules/MScalarSUN/TrPhi.hpp>
+#include <Grid/Hadrons/Modules/MScalar/FreeProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
+#include <Grid/Hadrons/Modules/MScalar/ScalarVP.hpp>
+#include <Grid/Hadrons/Modules/MScalar/ChargedProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/VPCounterTerms.hpp>
+#include <Grid/Hadrons/Modules/MLoop/NoiseLoop.hpp>
+#include <Grid/Hadrons/Modules/MIO/LoadEigenPack.hpp>
+#include <Grid/Hadrons/Modules/MIO/LoadCoarseEigenPack.hpp>
+#include <Grid/Hadrons/Modules/MIO/LoadBinary.hpp>
+#include <Grid/Hadrons/Modules/MIO/LoadNersc.hpp>
+#include <Grid/Hadrons/Modules/MSink/Smear.hpp>
+#include <Grid/Hadrons/Modules/MSink/Point.hpp>
+#include <Grid/Hadrons/Modules/MFermion/FreeProp.hpp>
+#include <Grid/Hadrons/Modules/MFermion/GaugeProp.hpp>
+#include <Grid/Hadrons/Modules/MGauge/FundtoHirep.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
+#include <Grid/Hadrons/Modules/MGauge/StoutSmearing.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
+#include <Grid/Hadrons/Modules/MGauge/StochEm.hpp>
+#include <Grid/Hadrons/Modules/MGauge/UnitEm.hpp>
+#include <Grid/Hadrons/Modules/MUtilities/TestSeqGamma.hpp>
+#include <Grid/Hadrons/Modules/MUtilities/TestSeqConserved.hpp>
+#include <Grid/Hadrons/Modules/MSource/SeqConserved.hpp>
+#include <Grid/Hadrons/Modules/MSource/Z2.hpp>
+#include <Grid/Hadrons/Modules/MSource/Wall.hpp>
+#include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
+#include <Grid/Hadrons/Modules/MSource/Point.hpp>
+#include <Grid/Hadrons/Modules/MContraction/MesonFieldGamma.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
 #include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
-#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
-#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
+#include <Grid/Hadrons/Modules/MContraction/A2APionField.hpp>
 #include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
 #include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
-#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
-#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
 #include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Load.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
-#include <Grid/Hadrons/Modules/MLoop/NoiseLoop.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
+#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/A2AMeson.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WardIdentity.hpp>
+#include <Grid/Hadrons/Modules/MContraction/A2AMesonField.hpp>
+#include <Grid/Hadrons/Modules/MAction/WilsonClover.hpp>
+#include <Grid/Hadrons/Modules/MAction/ScaledDWF.hpp>
+#include <Grid/Hadrons/Modules/MAction/MobiusDWF.hpp>
+#include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
+#include <Grid/Hadrons/Modules/MAction/DWF.hpp>
+#include <Grid/Hadrons/Modules/MAction/ZMobiusDWF.hpp>
 #include <Grid/Hadrons/Modules/MSolver/RBPrecCG.hpp>
-#include <Grid/Hadrons/Modules/MSource/Point.hpp>
-#include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
-#include <Grid/Hadrons/Modules/MSource/Wall.hpp>
-#include <Grid/Hadrons/Modules/MSource/Z2.hpp>
-#include <Grid/Hadrons/Modules/Quark.hpp>
+#include <Grid/Hadrons/Modules/MSolver/LocalCoherenceLanczos.hpp>
+#include <Grid/Hadrons/Modules/MSolver/A2AVectors.hpp>

extras/Hadrons/Modules/MAction/DWF.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/DWF.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MAction/DWF.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MAction;
+
+template class Grid::Hadrons::MAction::TDWF<FIMPL>;
+

extras/Hadrons/Modules/MAction/DWF.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MAction/DWF.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -27,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_DWF_hpp_
-#define Hadrons_DWF_hpp_
+#ifndef Hadrons_MAction_DWF_hpp_
+#define Hadrons_MAction_DWF_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -48,29 +48,32 @@ public:
                                     std::string, gauge,
                                     unsigned int, Ls,
                                     double      , mass,
-                                    double      , M5);
+                                    double      , M5,
+                                    std::string , boundary);
 };
 
 template <typename FImpl>
 class TDWF: public Module<DWFPar>
 {
 public:
-    TYPE_ALIASES(FImpl,);
+    FG_TYPE_ALIASES(FImpl,);
 public:
     // constructor
     TDWF(const std::string name);
     // destructor
-    virtual ~TDWF(void) = default;
+    virtual ~TDWF(void) {};
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(DWF, TDWF<FIMPL>, MAction);
+extern template class TDWF<FIMPL>;
+MODULE_REGISTER_TMP(DWF, TDWF<FIMPL>, MAction);
 
 /******************************************************************************
  *                        DWF template implementation                         *
@@ -102,33 +105,32 @@ std::vector<std::string> TDWF<FImpl>::getOutput(void)
 template <typename FImpl>
 void TDWF<FImpl>::setup(void)
 {
-    unsigned int size;
-    
-    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
-    env().registerObject(getName(), size, par().Ls);
+    LOG(Message) << "Setting up domain wall fermion matrix with m= "
+                 << par().mass << ", M5= " << par().M5 << " and Ls= "
+                 << par().Ls << " using gauge field '" << par().gauge << "'"
+                 << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary
+                 << std::endl;
+                 
+    env().createGrid(par().Ls);
+    auto &U    = envGet(LatticeGaugeField, par().gauge);
+    auto &g4   = *env().getGrid();
+    auto &grb4 = *env().getRbGrid();
+    auto &g5   = *env().getGrid(par().Ls);
+    auto &grb5 = *env().getRbGrid(par().Ls);
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename DomainWallFermion<FImpl>::ImplParams implParams(boundary);
+    envCreateDerived(FMat, DomainWallFermion<FImpl>, getName(), par().Ls, U, g5,
+                     grb5, g4, grb4, par().mass, par().M5, implParams);
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TDWF<FImpl>::execute(void)
-{
-    LOG(Message) << "Setting up domain wall fermion matrix with m= "
-                 << par().mass << ", M5= " << par().M5 << " and Ls= "
-                 << par().Ls << " using gauge field '" << par().gauge << "'"
-                 << std::endl;
-    env().createGrid(par().Ls);
-    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
-    auto &g4     = *env().getGrid();
-    auto &grb4   = *env().getRbGrid();
-    auto &g5     = *env().getGrid(par().Ls);
-    auto &grb5   = *env().getRbGrid(par().Ls);
-    FMat *fMatPt = new DomainWallFermion<FImpl>(U, g5, grb5, g4, grb4,
-                                                par().mass, par().M5);
-    env().setObject(getName(), fMatPt);
-}
+{}
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_DWF_hpp_
+#endif // Hadrons_MAction_DWF_hpp_

extras/Hadrons/Modules/MAction/MobiusDWF.cc

@@ -0,0 +1,7 @@
+#include <Grid/Hadrons/Modules/MAction/MobiusDWF.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MAction;
+
+template class Grid::Hadrons::MAction::TMobiusDWF<FIMPL>;

extras/Hadrons/Modules/MAction/MobiusDWF.hpp

@@ -0,0 +1,109 @@
+#ifndef Hadrons_MAction_MobiusDWF_hpp_
+#define Hadrons_MAction_MobiusDWF_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                      Mobius domain-wall fermion action                     *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MAction)
+
+class MobiusDWFPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(MobiusDWFPar,
+                                    std::string , gauge,
+                                    unsigned int, Ls,
+                                    double      , mass,
+                                    double      , M5,
+                                    double      , b,
+                                    double      , c,
+                                    std::string , boundary);
+};
+
+template <typename FImpl>
+class TMobiusDWF: public Module<MobiusDWFPar>
+{
+public:
+    FG_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TMobiusDWF(const std::string name);
+    // destructor
+    virtual ~TMobiusDWF(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_TMP(MobiusDWF, TMobiusDWF<FIMPL>, MAction);
+
+/******************************************************************************
+ *                      TMobiusDWF implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TMobiusDWF<FImpl>::TMobiusDWF(const std::string name)
+: Module<MobiusDWFPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TMobiusDWF<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().gauge};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TMobiusDWF<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TMobiusDWF<FImpl>::setup(void)
+{
+    LOG(Message) << "Setting up Mobius domain wall fermion matrix with m= "
+                 << par().mass << ", M5= " << par().M5 << ", Ls= " << par().Ls 
+                 << ", b= " << par().b << ", c= " << par().c
+                 << " using gauge field '" << par().gauge << "'"
+                 << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary
+                 << std::endl;
+
+    env().createGrid(par().Ls);
+    auto &U    = envGet(LatticeGaugeField, par().gauge);
+    auto &g4   = *env().getGrid();
+    auto &grb4 = *env().getRbGrid();
+    auto &g5   = *env().getGrid(par().Ls);
+    auto &grb5 = *env().getRbGrid(par().Ls);
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename MobiusFermion<FImpl>::ImplParams implParams(boundary);
+    envCreateDerived(FMat, MobiusFermion<FImpl>, getName(), par().Ls, U, g5,
+                     grb5, g4, grb4, par().mass, par().M5, par().b, par().c,
+                     implParams);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TMobiusDWF<FImpl>::execute(void)
+{}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MAction_MobiusDWF_hpp_

extras/Hadrons/Modules/MAction/ScaledDWF.cc

@@ -0,0 +1,7 @@
+#include <Grid/Hadrons/Modules/MAction/ScaledDWF.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MAction;
+
+template class Grid::Hadrons::MAction::TScaledDWF<FIMPL>;

extras/Hadrons/Modules/MAction/ScaledDWF.hpp

@@ -0,0 +1,108 @@
+#ifndef Hadrons_MAction_ScaledDWF_hpp_
+#define Hadrons_MAction_ScaledDWF_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                      Scaled domain wall fermion                            *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MAction)
+
+class ScaledDWFPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(ScaledDWFPar,
+                                    std::string , gauge,
+                                    unsigned int, Ls,
+                                    double      , mass,
+                                    double      , M5,
+                                    double      , scale,
+                                    std::string , boundary);
+};
+
+template <typename FImpl>
+class TScaledDWF: public Module<ScaledDWFPar>
+{
+public:
+    FG_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TScaledDWF(const std::string name);
+    // destructor
+    virtual ~TScaledDWF(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_TMP(ScaledDWF, TScaledDWF<FIMPL>, MAction);
+
+/******************************************************************************
+ *                      TScaledDWF implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TScaledDWF<FImpl>::TScaledDWF(const std::string name)
+: Module<ScaledDWFPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TScaledDWF<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().gauge};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TScaledDWF<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TScaledDWF<FImpl>::setup(void)
+{
+    LOG(Message) << "Setting up scaled domain wall fermion matrix with m= "
+                 << par().mass << ", M5= " << par().M5 << ", Ls= " << par().Ls 
+                 << ", scale= " << par().scale
+                 << " using gauge field '" << par().gauge << "'"
+                 << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary
+                 << std::endl;
+
+    env().createGrid(par().Ls);
+    auto &U    = envGet(LatticeGaugeField, par().gauge);
+    auto &g4   = *env().getGrid();
+    auto &grb4 = *env().getRbGrid();
+    auto &g5   = *env().getGrid(par().Ls);
+    auto &grb5 = *env().getRbGrid(par().Ls);
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename MobiusFermion<FImpl>::ImplParams implParams(boundary);
+    envCreateDerived(FMat, ScaledShamirFermion<FImpl>, getName(), par().Ls, U, g5,
+                     grb5, g4, grb4, par().mass, par().M5, par().scale,
+                     implParams);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TScaledDWF<FImpl>::execute(void)
+{}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MAction_ScaledDWF_hpp_

extras/Hadrons/Modules/MAction/Wilson.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/Wilson.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MAction;
+
+template class Grid::Hadrons::MAction::TWilson<FIMPL>;
+

extras/Hadrons/Modules/MAction/Wilson.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MAction/Wilson.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -27,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_Wilson_hpp_
-#define Hadrons_Wilson_hpp_
+#ifndef Hadrons_MAction_Wilson_hpp_
+#define Hadrons_MAction_Wilson_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -46,29 +46,31 @@ class WilsonPar: Serializable
 public:
     GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonPar,
                                     std::string, gauge,
-                                    double     , mass);
+                                    double     , mass,
+                                    std::string, boundary);
 };
 
 template <typename FImpl>
 class TWilson: public Module<WilsonPar>
 {
 public:
-    TYPE_ALIASES(FImpl,);
+    FG_TYPE_ALIASES(FImpl,);
 public:
     // constructor
     TWilson(const std::string name);
     // destructor
-    virtual ~TWilson(void) = default;
+    virtual ~TWilson(void) {};
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(Wilson, TWilson<FIMPL>, MAction);
+MODULE_REGISTER_TMP(Wilson, TWilson<FIMPL>, MAction);
 
 /******************************************************************************
  *                     TWilson template implementation                        *
@@ -100,24 +102,24 @@ std::vector<std::string> TWilson<FImpl>::getOutput(void)
 template <typename FImpl>
 void TWilson<FImpl>::setup(void)
 {
-    unsigned int size;
-    
-    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
-    env().registerObject(getName(), size);
+    LOG(Message) << "Setting up Wilson fermion matrix with m= " << par().mass
+                 << " using gauge field '" << par().gauge << "'" << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary
+                 << std::endl;
+                 
+    auto &U      = envGet(LatticeGaugeField, par().gauge);
+    auto &grid   = *env().getGrid();
+    auto &gridRb = *env().getRbGrid();
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename WilsonFermion<FImpl>::ImplParams implParams(boundary);
+    envCreateDerived(FMat, WilsonFermion<FImpl>, getName(), 1, U, grid, gridRb,
+                     par().mass, implParams);
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TWilson<FImpl>::execute()
-{
-    LOG(Message) << "Setting up TWilson fermion matrix with m= " << par().mass
-                 << " using gauge field '" << par().gauge << "'" << std::endl;
-    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
-    auto &grid   = *env().getGrid();
-    auto &gridRb = *env().getRbGrid();
-    FMat *fMatPt = new WilsonFermion<FImpl>(U, grid, gridRb, par().mass);
-    env().setObject(getName(), fMatPt);
-}
+{}
 
 END_MODULE_NAMESPACE
 

extras/Hadrons/Modules/MAction/WilsonClover.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/WilsonClover.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MAction/WilsonClover.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MAction;
+
+template class Grid::Hadrons::MAction::TWilsonClover<FIMPL>;
+

extras/Hadrons/Modules/MAction/WilsonClover.hpp

@@ -0,0 +1,137 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/WilsonClover.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+Author: pretidav <david.preti@csic.es>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MAction_WilsonClover_hpp_
+#define Hadrons_MAction_WilsonClover_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         Wilson clover quark action                         *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MAction)
+
+class WilsonCloverPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonCloverPar,
+                                    std::string, gauge,
+                                    double     , mass,
+				                    double     , csw_r,
+				                    double     , csw_t,
+				                    WilsonAnisotropyCoefficients ,clover_anisotropy,
+                                    std::string, boundary
+				    );
+};
+
+template <typename FImpl>
+class TWilsonClover: public Module<WilsonCloverPar>
+{
+public:
+    FG_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TWilsonClover(const std::string name);
+    // destructor
+    virtual ~TWilsonClover(void) {};
+    // dependencies/products
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_TMP(WilsonClover, TWilsonClover<FIMPL>, MAction);
+
+/******************************************************************************
+ *                     TWilsonClover template implementation                        *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TWilsonClover<FImpl>::TWilsonClover(const std::string name)
+: Module<WilsonCloverPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TWilsonClover<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().gauge};
+
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TWilsonClover<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWilsonClover<FImpl>::setup(void)
+{
+    LOG(Message) << "Setting up Wilson clover fermion matrix with m= " << par().mass
+                 << " using gauge field '" << par().gauge << "'" << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary 
+                 << std::endl;
+    LOG(Message) << "Clover term csw_r: " << par().csw_r
+                 << " csw_t: " << par().csw_t
+                 << std::endl;
+    auto &U      = envGet(LatticeGaugeField, par().gauge);
+    auto &grid   = *env().getGrid();
+    auto &gridRb = *env().getRbGrid();
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename WilsonCloverFermion<FImpl>::ImplParams implParams(boundary);
+    envCreateDerived(FMat, WilsonCloverFermion<FImpl>, getName(), 1, U, grid, gridRb, par().mass,
+						  par().csw_r,
+						  par().csw_t,
+					      par().clover_anisotropy,
+						  implParams); 
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWilsonClover<FImpl>::execute()
+{}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WilsonClover_hpp_

extras/Hadrons/Modules/MAction/ZMobiusDWF.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/ZMobiusDWF.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MAction/ZMobiusDWF.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MAction;
+
+template class Grid::Hadrons::MAction::TZMobiusDWF<ZFIMPL>;
+

extras/Hadrons/Modules/MAction/ZMobiusDWF.hpp

@@ -0,0 +1,143 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/ZMobiusDWF.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef Hadrons_MAction_ZMobiusDWF_hpp_
+#define Hadrons_MAction_ZMobiusDWF_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                      z-Mobius domain-wall fermion action                   *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MAction)
+
+class ZMobiusDWFPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(ZMobiusDWFPar,
+                                    std::string                      , gauge,
+                                    unsigned int                     , Ls,
+                                    double                           , mass,
+                                    double                           , M5,
+                                    double                           , b,
+                                    double                           , c,
+                                    std::vector<std::complex<double>>, omega,
+                                    std::string                      , boundary);
+};
+
+template <typename FImpl>
+class TZMobiusDWF: public Module<ZMobiusDWFPar>
+{
+public:
+    FG_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TZMobiusDWF(const std::string name);
+    // destructor
+    virtual ~TZMobiusDWF(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_TMP(ZMobiusDWF, TZMobiusDWF<ZFIMPL>, MAction);
+
+/******************************************************************************
+ *                     TZMobiusDWF implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TZMobiusDWF<FImpl>::TZMobiusDWF(const std::string name)
+: Module<ZMobiusDWFPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TZMobiusDWF<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().gauge};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TZMobiusDWF<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TZMobiusDWF<FImpl>::setup(void)
+{
+    LOG(Message) << "Setting up z-Mobius domain wall fermion matrix with m= "
+                 << par().mass << ", M5= " << par().M5 << ", Ls= " << par().Ls 
+                 << ", b= " << par().b << ", c= " << par().c
+                 << " using gauge field '" << par().gauge << "'"
+                 << std::endl;
+    LOG(Message) << "Omegas: " << std::endl;
+    for (unsigned int i = 0; i < par().omega.size(); ++i)
+    {
+        LOG(Message) << "  omega[" << i << "]= " << par().omega[i] << std::endl;
+    }
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary
+                 << std::endl;
+
+    env().createGrid(par().Ls);
+    auto &U    = envGet(LatticeGaugeField, par().gauge);
+    auto &g4   = *env().getGrid();
+    auto &grb4 = *env().getRbGrid();
+    auto &g5   = *env().getGrid(par().Ls);
+    auto &grb5 = *env().getRbGrid(par().Ls);
+    auto omega = par().omega;
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename ZMobiusFermion<FImpl>::ImplParams implParams(boundary);
+    envCreateDerived(FMat, ZMobiusFermion<FImpl>, getName(), par().Ls, U, g5,
+                     grb5, g4, grb4, par().mass, par().M5, omega,
+                     par().b, par().c, implParams);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TZMobiusDWF<FImpl>::execute(void)
+{}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MAction_ZMobiusDWF_hpp_

extras/Hadrons/Modules/MContraction/A2AMeson.cc

@@ -0,0 +1,8 @@
+#include <Grid/Hadrons/Modules/MContraction/A2AMeson.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TA2AMeson<FIMPL>;
+template class Grid::Hadrons::MContraction::TA2AMeson<ZFIMPL>;

extras/Hadrons/Modules/MContraction/A2AMeson.hpp

@@ -0,0 +1,207 @@
+#ifndef Hadrons_MContraction_A2AMeson_hpp_
+#define Hadrons_MContraction_A2AMeson_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+#include <Grid/Hadrons/AllToAllVectors.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         A2AMeson                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+typedef std::pair<Gamma::Algebra, Gamma::Algebra> GammaPair;
+
+class A2AMesonPar : Serializable
+{
+  public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(A2AMesonPar,
+                                    int, Nl,
+                                    int, N,
+                                    std::string, A2A1,
+                                    std::string, A2A2,
+                                    std::string, gammas,
+                                    std::string, output);
+};
+
+template <typename FImpl>
+class TA2AMeson : public Module<A2AMesonPar>
+{
+  public:
+    FERM_TYPE_ALIASES(FImpl, );
+    SOLVER_TYPE_ALIASES(FImpl, );
+
+    typedef A2AModesSchurDiagTwo<typename FImpl::FermionField, FMat, Solver> A2ABase;
+
+    class Result : Serializable
+    {
+      public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma_snk,
+                                        Gamma::Algebra, gamma_src,
+                                        std::vector<Complex>, corr);
+    };
+
+  public:
+    // constructor
+    TA2AMeson(const std::string name);
+    // destructor
+    virtual ~TA2AMeson(void){};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    virtual void parseGammaString(std::vector<GammaPair> &gammaList);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER(A2AMeson, ARG(TA2AMeson<FIMPL>), MContraction);
+MODULE_REGISTER(ZA2AMeson, ARG(TA2AMeson<ZFIMPL>), MContraction);
+
+/******************************************************************************
+*                  TA2AMeson implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TA2AMeson<FImpl>::TA2AMeson(const std::string name)
+    : Module<A2AMesonPar>(name)
+{
+}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TA2AMeson<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().A2A1 + "_class", par().A2A2 + "_class"};
+    in.push_back(par().A2A1 + "_w_high_4d");
+    in.push_back(par().A2A2 + "_v_high_4d");
+
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TA2AMeson<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {};
+
+    return out;
+}
+
+template <typename FImpl>
+void TA2AMeson<FImpl>::parseGammaString(std::vector<GammaPair> &gammaList)
+{
+    gammaList.clear();
+    // Parse individual contractions from input string.
+    gammaList = strToVec<GammaPair>(par().gammas);
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TA2AMeson<FImpl>::setup(void)
+{
+    int nt = env().getDim(Tp);
+    int N = par().N;
+
+    int Ls_ = env().getObjectLs(par().A2A1 + "_class");
+
+    envTmp(std::vector<FermionField>, "w1", 1, N, FermionField(env().getGrid(1)));
+    envTmp(std::vector<FermionField>, "v1", 1, N, FermionField(env().getGrid(1)));
+    envTmpLat(FermionField, "tmpv_5d", Ls_);
+    envTmpLat(FermionField, "tmpw_5d", Ls_);
+
+    envTmp(std::vector<ComplexD>, "MF_x", 1, nt);
+    envTmp(std::vector<ComplexD>, "MF_y", 1, nt);
+    envTmp(std::vector<ComplexD>, "tmp", 1, nt);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TA2AMeson<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing A2A meson contractions" << std::endl;
+
+    Result result;
+    Gamma g5(Gamma::Algebra::Gamma5);
+    std::vector<GammaPair> gammaList;
+    int nt = env().getDim(Tp);
+
+    parseGammaString(gammaList);
+
+    result.gamma_snk = gammaList[0].first;
+    result.gamma_src = gammaList[0].second;
+    result.corr.resize(nt);
+
+    int Nl = par().Nl;
+    int N  = par().N;
+    LOG(Message) << "N for A2A cont: " << N << std::endl;
+
+    envGetTmp(std::vector<ComplexD>, MF_x);
+    envGetTmp(std::vector<ComplexD>, MF_y);
+    envGetTmp(std::vector<ComplexD>, tmp);
+
+    for (unsigned int t = 0; t < nt; ++t)
+    {
+        tmp[t] = TensorRemove(MF_x[t] * MF_y[t] * 0.0);
+    }
+
+    Gamma gSnk(gammaList[0].first);
+    Gamma gSrc(gammaList[0].second);
+
+    auto &a2a1_fn = envGet(A2ABase, par().A2A1 + "_class");
+
+    envGetTmp(std::vector<FermionField>, w1);
+    envGetTmp(std::vector<FermionField>, v1);
+    envGetTmp(FermionField, tmpv_5d);
+    envGetTmp(FermionField, tmpw_5d);
+
+    LOG(Message) << "Finding v and w vectors for N =  " << N << std::endl;
+    for (int i = 0; i < N; i++)
+    {
+        a2a1_fn.return_v(i, tmpv_5d, v1[i]);
+        a2a1_fn.return_w(i, tmpw_5d, w1[i]);
+    }
+    LOG(Message) << "Found v and w vectors for N =  " << N << std::endl;
+    for (unsigned int i = 0; i < N; i++)
+    {
+        v1[i] = gSnk * v1[i];
+    }
+    int ty;
+    for (unsigned int i = 0; i < N; i++)
+    {
+        for (unsigned int j = 0; j < N; j++)
+        {
+            mySliceInnerProductVector(MF_x, w1[i], v1[j], Tp);
+            mySliceInnerProductVector(MF_y, w1[j], v1[i], Tp);
+            for (unsigned int t = 0; t < nt; ++t)
+            {
+                for (unsigned int tx = 0; tx < nt; tx++)
+                {
+                    ty = (tx + t) % nt;
+                    tmp[t] += TensorRemove((MF_x[tx]) * (MF_y[ty]));
+                }
+            }
+        }
+        if (i % 10 == 0)
+        {
+            LOG(Message) << "MF for i = " << i << " of " << N << std::endl;
+        }
+    }
+    double NTinv = 1.0 / static_cast<double>(nt);
+    for (unsigned int t = 0; t < nt; ++t)
+    {
+        result.corr[t] = NTinv * tmp[t];
+    }
+
+    saveResult(par().output, "meson", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_A2AMeson_hpp_

extras/Hadrons/Modules/MContraction/A2AMesonField.cc

@@ -0,0 +1,8 @@
+#include <Grid/Hadrons/Modules/MContraction/A2AMesonField.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TA2AMesonField<FIMPL>;
+template class Grid::Hadrons::MContraction::TA2AMesonField<ZFIMPL>;

extras/Hadrons/Modules/MContraction/A2AMesonField.hpp

@@ -0,0 +1,279 @@
+#ifndef Hadrons_MContraction_A2AMesonField_hpp_
+#define Hadrons_MContraction_A2AMesonField_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+#include <Grid/Hadrons/AllToAllVectors.hpp>
+
+#include <Grid/Hadrons/Modules/MContraction/A2Autils.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         A2AMesonField                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+typedef std::pair<Gamma::Algebra, Gamma::Algebra> GammaPair;
+
+
+class A2AMesonFieldPar : Serializable
+{
+  public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(A2AMesonFieldPar,
+				    int, cacheBlock,
+				    int, schurBlock,
+				    int, Nmom,
+				    int, N,
+				    int, Nl,
+                                    std::string, A2A,
+                                    std::string, output);
+};
+
+template <typename FImpl>
+class TA2AMesonField : public Module<A2AMesonFieldPar>
+{
+  public:
+    FERM_TYPE_ALIASES(FImpl, );
+    SOLVER_TYPE_ALIASES(FImpl, );
+
+    typedef A2AModesSchurDiagTwo<typename FImpl::FermionField, FMat, Solver> A2ABase;
+
+  public:
+    // constructor
+    TA2AMesonField(const std::string name);
+    // destructor
+    virtual ~TA2AMesonField(void){};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+
+};
+
+MODULE_REGISTER(A2AMesonField, ARG(TA2AMesonField<FIMPL>), MContraction);
+MODULE_REGISTER(ZA2AMesonField, ARG(TA2AMesonField<ZFIMPL>), MContraction);
+
+/******************************************************************************
+*                  TA2AMesonField implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TA2AMesonField<FImpl>::TA2AMesonField(const std::string name)
+    : Module<A2AMesonFieldPar>(name)
+{
+}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TA2AMesonField<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().A2A + "_class"};
+    in.push_back(par().A2A + "_w_high_4d");
+    in.push_back(par().A2A + "_v_high_4d");
+
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TA2AMesonField<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {};
+
+    return out;
+}
+
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TA2AMesonField<FImpl>::setup(void)
+{
+    auto &a2a = envGet(A2ABase, par().A2A + "_class");
+    int nt = env().getDim(Tp);
+    int Nl = par().Nl;
+    int N  = par().N;
+    int Ls_ = env().getObjectLs(par().A2A + "_class");
+
+    // Four D fields
+    envTmp(std::vector<FermionField>, "w", 1, par().schurBlock, FermionField(env().getGrid(1)));
+    envTmp(std::vector<FermionField>, "v", 1, par().schurBlock, FermionField(env().getGrid(1)));
+
+    // 5D tmp
+    envTmpLat(FermionField, "tmp_5d", Ls_);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TA2AMesonField<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing A2A meson field" << std::endl;
+
+    auto &a2a = envGet(A2ABase, par().A2A + "_class");
+    
+    // 2+6+4+4 = 16 gammas
+    // Ordering defined here
+    std::vector<Gamma::Algebra> gammas ( {
+          Gamma::Algebra::Gamma5,
+	  Gamma::Algebra::Identity,    
+	  Gamma::Algebra::GammaX,
+	  Gamma::Algebra::GammaY,
+	  Gamma::Algebra::GammaZ,
+	  Gamma::Algebra::GammaT,
+	  Gamma::Algebra::GammaXGamma5,
+	  Gamma::Algebra::GammaYGamma5,
+	  Gamma::Algebra::GammaZGamma5,
+	  Gamma::Algebra::GammaTGamma5,
+	  Gamma::Algebra::SigmaXY,
+	  Gamma::Algebra::SigmaXZ,
+	  Gamma::Algebra::SigmaXT,
+	  Gamma::Algebra::SigmaYZ,
+	  Gamma::Algebra::SigmaYT,
+	  Gamma::Algebra::SigmaZT
+    });
+
+    ///////////////////////////////////////////////
+    // Square assumption for now Nl = Nr = N
+    ///////////////////////////////////////////////
+    int nt = env().getDim(Tp);
+    int nx = env().getDim(Xp);
+    int ny = env().getDim(Yp);
+    int nz = env().getDim(Zp);
+    int N  = par().N;
+    int Nl = par().Nl;
+    int ngamma = gammas.size();
+
+    int schurBlock = par().schurBlock;
+    int cacheBlock = par().cacheBlock;
+    int nmom       = par().Nmom;
+
+    ///////////////////////////////////////////////
+    // Momentum setup
+    ///////////////////////////////////////////////
+    GridBase *grid = env().getGrid(1);
+    std::vector<LatticeComplex> phases(nmom,grid);
+    for(int m=0;m<nmom;m++){
+      phases[m] = Complex(1.0);    // All zero momentum for now
+    }
+
+    Eigen::Tensor<ComplexD,5> mesonField       (nmom,ngamma,nt,N,N);    
+    LOG(Message) << "N = Nh+Nl for A2A MesonField is " << N << std::endl;
+
+    envGetTmp(std::vector<FermionField>, w);
+    envGetTmp(std::vector<FermionField>, v);
+    envGetTmp(FermionField, tmp_5d);
+
+    LOG(Message) << "Finding v and w vectors for N =  " << N << std::endl;
+
+    //////////////////////////////////////////////////////////////////////////
+    // i,j   is first  loop over SchurBlock factors reusing 5D matrices
+    // ii,jj is second loop over cacheBlock factors for high perf contractoin
+    // iii,jjj are loops within cacheBlock
+    // Total index is sum of these  i+ii+iii etc...
+    //////////////////////////////////////////////////////////////////////////
+    
+    double flops = 0.0;
+    double bytes = 0.0;
+    double vol   = nx*ny*nz*nt;
+    double t_schur=0;
+    double t_contr=0;
+    double t_int_0=0;
+    double t_int_1=0;
+    double t_int_2=0;
+    double t_int_3=0;
+
+    double t0 = usecond();
+    int N_i = N;
+    int N_j = N;
+    for(int i=0;i<N_i;i+=schurBlock){ //loop over SchurBlocking to suppress 5D matrix overhead
+    for(int j=0;j<N_j;j+=schurBlock){
+      
+
+      ///////////////////////////////////////////////////////////////
+      // Get the W and V vectors for this schurBlock^2 set of terms
+      ///////////////////////////////////////////////////////////////
+      int N_ii = MIN(N_i-i,schurBlock);
+      int N_jj = MIN(N_j-j,schurBlock);
+
+      t_schur-=usecond();
+      for(int ii =0;ii < N_ii;ii++) a2a.return_w(i+ii, tmp_5d, w[ii]);
+      for(int jj =0;jj < N_jj;jj++) a2a.return_v(j+jj, tmp_5d, v[jj]);
+      t_schur+=usecond();
+
+      LOG(Message) << "Found w vectors " << i <<" .. " << i+N_ii-1 << std::endl;
+      LOG(Message) << "Found v vectors " << j <<" .. " << j+N_jj-1 << std::endl;
+
+      ///////////////////////////////////////////////////////////////
+      // Series of cache blocked chunks of the contractions within this SchurBlock
+      /////////////////////////////////////////////////////////////// 
+      for(int ii=0;ii<N_ii;ii+=cacheBlock){
+      for(int jj=0;jj<N_jj;jj+=cacheBlock){
+
+	int N_iii = MIN(N_ii-ii,cacheBlock);
+	int N_jjj = MIN(N_jj-jj,cacheBlock);
+
+	Eigen::Tensor<ComplexD,5> mesonFieldBlocked(nmom,ngamma,nt,N_iii,N_jjj);    
+
+	t_contr-=usecond();
+	A2Autils<FImpl>::MesonField(mesonFieldBlocked, 
+				    &w[ii], 
+				    &v[jj], gammas, phases,Tp);
+	t_contr+=usecond();
+	flops += vol * ( 2 * 8.0 + 6.0 + 8.0*nmom) * N_iii*N_jjj*ngamma;
+
+	bytes  += vol * (12.0 * sizeof(Complex) ) * N_iii*N_jjj
+               +  vol * ( 2.0 * sizeof(Complex) *nmom ) * N_iii*N_jjj* ngamma;
+
+	///////////////////////////////////////////////////////////////
+	// Copy back to full meson field tensor
+	/////////////////////////////////////////////////////////////// 
+	parallel_for_nest2(int iii=0;iii< N_iii;iii++) {
+        for(int jjj=0;jjj< N_jjj;jjj++) {
+	  for(int m =0;m< nmom;m++) {
+	  for(int g =0;g< ngamma;g++) {
+          for(int t =0;t< nt;t++) {
+	    mesonField(m,g,t,i+ii+iii,j+jj+jjj) = mesonFieldBlocked(m,g,t,iii,jjj);
+	  }}}
+
+	}}
+      }}
+    }}
+
+
+    double nodes=grid->NodeCount();
+    double t1 = usecond();
+    LOG(Message) << " Contraction of MesonFields took "<<(t1-t0)/1.0e6<< " seconds "  << std::endl;
+    LOG(Message) << " Schur "<<(t_schur)/1.0e6<< " seconds "  << std::endl;
+    LOG(Message) << " Contr "<<(t_contr)/1.0e6<< " seconds "  << std::endl;
+
+    /////////////////////////////////////////////////////////////////////////
+    // Test: Build the pion correlator (two end)
+    // < PI_ij(t0) PI_ji (t0+t) >
+    /////////////////////////////////////////////////////////////////////////
+    std::vector<ComplexD> corr(nt,ComplexD(0.0));
+
+    for(int i=0;i<N;i++){
+    for(int j=0;j<N;j++){
+      int m=0; // first momentum
+      int g=0; // first gamma in above ordering is gamma5 for pion
+      for(int t0=0;t0<nt;t0++){
+      for(int t=0;t<nt;t++){
+	int tt = (t0+t)%nt;
+	corr[t] += mesonField(m,g,t0,i,j)* mesonField(m,g,tt,j,i);
+      }}
+    }}    
+    for(int t=0;t<nt;t++) corr[t] = corr[t]/ (double)nt;
+
+    for(int t=0;t<nt;t++) LOG(Message) << " " << t << " " << corr[t]<<std::endl;
+
+    //    saveResult(par().output, "meson", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_A2AMesonField_hpp_

extras/Hadrons/Modules/MContraction/A2APionField.cc

@@ -0,0 +1,8 @@
+#include <Grid/Hadrons/Modules/MContraction/A2APionField.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TA2APionField<FIMPL>;
+template class Grid::Hadrons::MContraction::TA2APionField<ZFIMPL>;

extras/Hadrons/Modules/MContraction/A2APionField.hpp

@@ -0,0 +1,502 @@
+#ifndef Hadrons_MContraction_A2APionField_hpp_
+#define Hadrons_MContraction_A2APionField_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+#include <Grid/Hadrons/AllToAllVectors.hpp>
+#include <Grid/Hadrons/Modules/MContraction/A2Autils.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         A2APionField                                       *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+typedef std::pair<Gamma::Algebra, Gamma::Algebra> GammaPair;
+
+
+class A2APionFieldPar : Serializable
+{
+  public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(A2APionFieldPar,
+				    int, cacheBlock,
+				    int, schurBlock,
+				    int, Nmom,
+                                    std::string, A2A_i,
+                                    std::string, A2A_j,
+                                    std::string, output);
+};
+
+template <typename FImpl>
+class TA2APionField : public Module<A2APionFieldPar>
+{
+public:
+  FERM_TYPE_ALIASES(FImpl, );
+  SOLVER_TYPE_ALIASES(FImpl, );
+
+  typedef typename FImpl::SiteSpinor vobj;
+
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  typedef iSpinMatrix<vector_type> SpinMatrix_v;
+  typedef iSpinMatrix<scalar_type> SpinMatrix_s;
+  typedef iSinglet<vector_type> Scalar_v;
+  typedef iSinglet<scalar_type> Scalar_s;
+
+  typedef A2AModesSchurDiagTwo<typename FImpl::FermionField, FMat, Solver> A2ABase;
+
+  public:
+    // constructor
+    TA2APionField(const std::string name);
+    // destructor
+    virtual ~TA2APionField(void){};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+
+};
+
+MODULE_REGISTER(A2APionField, ARG(TA2APionField<FIMPL>), MContraction);
+MODULE_REGISTER(ZA2APionField, ARG(TA2APionField<ZFIMPL>), MContraction);
+
+/******************************************************************************
+*                  TA2APionField implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TA2APionField<FImpl>::TA2APionField(const std::string name)
+    : Module<A2APionFieldPar>(name)
+{
+}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TA2APionField<FImpl>::getInput(void)
+{
+  std::vector<std::string> in;
+  in.push_back(par().A2A_i + "_class");
+  in.push_back(par().A2A_i + "_w_high_4d");
+  in.push_back(par().A2A_i + "_v_high_4d");
+  in.push_back(par().A2A_j + "_class");
+  in.push_back(par().A2A_j + "_w_high_4d");
+  in.push_back(par().A2A_j + "_v_high_4d");
+  
+  return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TA2APionField<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {};
+
+    return out;
+}
+
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TA2APionField<FImpl>::setup(void)
+{
+
+  // Four D fields
+  envTmp(std::vector<FermionField>, "wi", 1, par().schurBlock, FermionField(env().getGrid(1)));
+  envTmp(std::vector<FermionField>, "vi", 1, par().schurBlock, FermionField(env().getGrid(1)));
+  envTmp(std::vector<FermionField>, "wj", 1, par().schurBlock, FermionField(env().getGrid(1)));
+  envTmp(std::vector<FermionField>, "vj", 1, par().schurBlock, FermionField(env().getGrid(1)));
+
+  // 5D tmp
+  int Ls_i = env().getObjectLs(par().A2A_i + "_class");
+  envTmpLat(FermionField, "tmp_5d", Ls_i);
+  
+  int Ls_j= env().getObjectLs(par().A2A_j + "_class");
+  assert ( Ls_i == Ls_j ); 
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TA2APionField<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing A2A Pion fields" << std::endl;
+
+    auto &a2a_i = envGet(A2ABase, par().A2A_i + "_class");
+    auto &a2a_j = envGet(A2ABase, par().A2A_j + "_class");
+
+    ///////////////////////////////////////////////
+    // Square assumption for now Nl = Nr = N
+    ///////////////////////////////////////////////
+    int nt = env().getDim(Tp);
+    int nx = env().getDim(Xp);
+    int ny = env().getDim(Yp);
+    int nz = env().getDim(Zp);
+
+    //    int N_i  = a2a_i.par().N;
+    //    int N_j  = a2a_j.par().N;
+    int N_i  = a2a_i.getN();
+    int N_j  = a2a_j.getN();
+
+    int nmom=par().Nmom;
+
+    int schurBlock = par().schurBlock;
+    int cacheBlock = par().cacheBlock;
+
+
+    ///////////////////////////////////////////////
+    // Momentum setup
+    ///////////////////////////////////////////////
+    GridBase *grid = env().getGrid(1);
+    std::vector<LatticeComplex> phases(nmom,grid);
+    for(int m=0;m<nmom;m++){
+      phases[m] = Complex(1.0);    // All zero momentum for now
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+    // i and j represent different flavours, hits, with different ranks.
+    // in general non-square case.
+    ///////////////////////////////////////////////////////////////////////
+    Eigen::Tensor<ComplexD,4> pionFieldWVmom_ij     (nmom,nt,N_i,N_j);    
+    Eigen::Tensor<ComplexD,3> pionFieldWV_ij        (nt,N_i,N_j);    
+
+    Eigen::Tensor<ComplexD,4> pionFieldWVmom_ji     (nmom,nt,N_j,N_i);    
+    Eigen::Tensor<ComplexD,3> pionFieldWV_ji        (nt,N_j,N_i);    
+
+
+    LOG(Message) << "Rank for A2A PionField is " << N_i << " x "<<N_j << std::endl;
+
+    envGetTmp(std::vector<FermionField>, wi);
+    envGetTmp(std::vector<FermionField>, vi);
+
+    envGetTmp(std::vector<FermionField>, wj);
+    envGetTmp(std::vector<FermionField>, vj);
+    envGetTmp(FermionField, tmp_5d);
+
+    LOG(Message) << "Finding v and w vectors " << std::endl;
+
+    //////////////////////////////////////////////////////////////////////////
+    // i,j   is first  loop over SchurBlock factors reusing 5D matrices
+    // ii,jj is second loop over cacheBlock factors for high perf contractoin
+    // iii,jjj are loops within cacheBlock
+    // Total index is sum of these  i+ii+iii etc...
+    //////////////////////////////////////////////////////////////////////////
+    
+    double flops = 0.0;
+    double bytes = 0.0;
+    double vol   = nx*ny*nz*nt;
+    double vol3  = nx*ny*nz;
+    double t_schur=0;
+    double t_contr_vwm=0;
+    double t_contr_vw=0;
+    double t_contr_ww=0;
+    double t_contr_vv=0;
+
+    double tt0 = usecond();
+    for(int i=0;i<N_i;i+=schurBlock){ //loop over SchurBlocking to suppress 5D matrix overhead
+    for(int j=0;j<N_j;j+=schurBlock){
+
+      ///////////////////////////////////////////////////////////////
+      // Get the W and V vectors for this schurBlock^2 set of terms
+      ///////////////////////////////////////////////////////////////
+      int N_ii = MIN(N_i-i,schurBlock);
+      int N_jj = MIN(N_j-j,schurBlock);
+
+      t_schur-=usecond();
+      for(int ii =0;ii < N_ii;ii++) a2a_i.return_w(i+ii, tmp_5d, wi[ii]);
+      for(int jj =0;jj < N_jj;jj++) a2a_j.return_w(j+jj, tmp_5d, wj[jj]);
+
+      for(int ii =0;ii < N_ii;ii++) a2a_i.return_v(i+ii, tmp_5d, vi[ii]);
+      for(int jj =0;jj < N_jj;jj++) a2a_j.return_v(j+jj, tmp_5d, vj[jj]);
+      t_schur+=usecond();
+
+      LOG(Message) << "Found i w&v vectors " << i <<" .. " << i+N_ii-1 << std::endl;
+      LOG(Message) << "Found j w&v vectors " << j <<" .. " << j+N_jj-1 << std::endl;
+
+      ///////////////////////////////////////////////////////////////
+      // Series of cache blocked chunks of the contractions within this SchurBlock
+      /////////////////////////////////////////////////////////////// 
+      for(int ii=0;ii<N_ii;ii+=cacheBlock){
+      for(int jj=0;jj<N_jj;jj+=cacheBlock){
+
+	int N_iii = MIN(N_ii-ii,cacheBlock);
+	int N_jjj = MIN(N_jj-jj,cacheBlock);
+
+	Eigen::Tensor<ComplexD,4> pionFieldWVmomB_ij(nmom,nt,N_iii,N_jjj);    
+	Eigen::Tensor<ComplexD,4> pionFieldWVmomB_ji(nmom,nt,N_jjj,N_iii);    
+
+	Eigen::Tensor<ComplexD,3> pionFieldWVB_ij(nt,N_iii,N_jjj);    
+	Eigen::Tensor<ComplexD,3> pionFieldWVB_ji(nt,N_jjj,N_iii);    
+
+	t_contr_vwm-=usecond();
+	A2Autils<FImpl>::PionFieldWVmom(pionFieldWVmomB_ij, &wi[ii], &vj[jj], phases,Tp);
+	A2Autils<FImpl>::PionFieldWVmom(pionFieldWVmomB_ji, &wj[jj], &vi[ii], phases,Tp);
+	t_contr_vwm+=usecond();
+
+	t_contr_vw-=usecond();
+	A2Autils<FImpl>::PionFieldWV(pionFieldWVB_ij, &wi[ii], &vj[jj],Tp);
+	A2Autils<FImpl>::PionFieldWV(pionFieldWVB_ji, &wj[jj], &vi[ii],Tp);
+	t_contr_vw+=usecond();
+
+
+	flops += vol * ( 2 * 8.0 + 6.0 + 8.0*nmom) * N_iii*N_jjj;
+
+	bytes  += vol * (12.0 * sizeof(Complex) ) * N_iii*N_jjj
+	       +  vol * ( 2.0 * sizeof(Complex) *nmom ) * N_iii*N_jjj;
+
+	///////////////////////////////////////////////////////////////
+	// Copy back to full meson field tensor
+	/////////////////////////////////////////////////////////////// 
+	parallel_for_nest2(int iii=0;iii< N_iii;iii++) {
+        for(int jjj=0;jjj< N_jjj;jjj++) {
+
+	  for(int m =0;m< nmom;m++) {
+          for(int t =0;t< nt;t++) {
+	    pionFieldWVmom_ij(m,t,i+ii+iii,j+jj+jjj) = pionFieldWVmomB_ij(m,t,iii,jjj);
+	    pionFieldWVmom_ji(m,t,j+jj+jjj,i+ii+iii) = pionFieldWVmomB_ji(m,t,jjj,iii);
+	  }}
+
+          for(int t =0;t< nt;t++) {
+	    pionFieldWV_ij(t,i+ii+iii,j+jj+jjj) = pionFieldWVB_ij(t,iii,jjj);
+	    pionFieldWV_ji(t,j+jj+jjj,i+ii+iii) = pionFieldWVB_ji(t,jjj,iii);
+	  }
+
+	}}
+      }}
+    }}
+
+    double nodes=grid->NodeCount();
+    double tt1 = usecond();
+    LOG(Message) << " Contraction of PionFields took "<<(tt1-tt0)/1.0e6<< " seconds "  << std::endl;
+    LOG(Message) << " Schur "<<(t_schur)/1.0e6<< " seconds "  << std::endl;
+    LOG(Message) << " Contr WVmom "<<(t_contr_vwm)/1.0e6<< " seconds "  << std::endl;
+    LOG(Message) << " Contr WV    "<<(t_contr_vw)/1.0e6<< " seconds "  << std::endl;
+
+    double t_kernel = t_contr_vwm;
+    LOG(Message) << " Arith "<<flops/(t_kernel)/1.0e3/nodes<< " Gflop/s / node "  << std::endl;
+    LOG(Message) << " Arith "<<bytes/(t_kernel)/1.0e3/nodes<< " GB/s /node "  << std::endl;
+
+    /////////////////////////////////////////////////////////////////////////
+    // Test: Build the pion correlator (two end)
+    // < PI_ij(t0) PI_ji (t0+t) >
+    /////////////////////////////////////////////////////////////////////////
+    std::vector<ComplexD> corrMom(nt,ComplexD(0.0));
+
+    for(int i=0;i<N_i;i++){
+    for(int j=0;j<N_j;j++){
+      int m=0; // first momentum
+      for(int t0=0;t0<nt;t0++){
+      for(int t=0;t<nt;t++){
+	int tt = (t0+t)%nt;
+	corrMom[t] += pionFieldWVmom_ij(m,t0,i,j)* pionFieldWVmom_ji(m,tt,j,i);
+      }}
+    }}    
+    for(int t=0;t<nt;t++) corrMom[t] = corrMom[t]/ (double)nt;
+
+    for(int t=0;t<nt;t++) LOG(Message) << " C_vwm " << t << " " << corrMom[t]<<std::endl;
+
+
+    /////////////////////////////////////////////////////////////////////////
+    // Test: Build the pion correlator (two end) from zero mom contraction
+    // < PI_ij(t0) PI_ji (t0+t) >
+    /////////////////////////////////////////////////////////////////////////
+    std::vector<ComplexD> corr(nt,ComplexD(0.0));
+
+    for(int i=0;i<N_i;i++){
+    for(int j=0;j<N_j;j++){
+      for(int t0=0;t0<nt;t0++){
+      for(int t=0;t<nt;t++){
+	int tt = (t0+t)%nt;
+	corr[t] += pionFieldWV_ij(t0,i,j)* pionFieldWV_ji(tt,j,i);
+      }}
+    }}    
+    for(int t=0;t<nt;t++) corr[t] = corr[t]/ (double)nt;
+
+    for(int t=0;t<nt;t++) LOG(Message) << " C_vw " << t << " " << corr[t]<<std::endl;
+
+
+    /////////////////////////////////////////////////////////////////////////
+    // Test: Build the pion correlator from zero mom contraction with revers 
+    // charge flow
+    /////////////////////////////////////////////////////////////////////////
+    std::vector<ComplexD> corr_wwvv(nt,ComplexD(0.0));
+
+    wi.resize(N_i,grid);
+    vi.resize(N_i,grid);
+    wj.resize(N_j,grid);
+    vj.resize(N_j,grid);
+
+    for(int i =0;i < N_i;i++) a2a_i.return_v(i, tmp_5d, vi[i]);
+    for(int i =0;i < N_i;i++) a2a_i.return_w(i, tmp_5d, wi[i]);
+    for(int j =0;j < N_j;j++) a2a_j.return_v(j, tmp_5d, vj[j]);
+    for(int j =0;j < N_j;j++) a2a_j.return_w(j, tmp_5d, wj[j]);
+
+    Eigen::Tensor<ComplexD,3> pionFieldWW_ij        (nt,N_i,N_j);    
+    Eigen::Tensor<ComplexD,3> pionFieldVV_ji        (nt,N_j,N_i);    
+    Eigen::Tensor<ComplexD,3> pionFieldWW_ji        (nt,N_j,N_i);    
+    Eigen::Tensor<ComplexD,3> pionFieldVV_ij        (nt,N_i,N_j);    
+
+    A2Autils<FImpl>::PionFieldWW(pionFieldWW_ij, &wi[0], &wj[0],Tp);
+    A2Autils<FImpl>::PionFieldVV(pionFieldVV_ji, &vj[0], &vi[0],Tp);
+    A2Autils<FImpl>::PionFieldWW(pionFieldWW_ji, &wj[0], &wi[0],Tp);
+    A2Autils<FImpl>::PionFieldVV(pionFieldVV_ij, &vi[0], &vj[0],Tp);
+
+
+    for(int i=0;i<N_i;i++){
+    for(int j=0;j<N_j;j++){
+      for(int t0=0;t0<nt;t0++){
+      for(int t=0;t<nt;t++){
+	int tt = (t0+t)%nt;
+	corr_wwvv[t] += pionFieldWW_ij(t0,i,j)* pionFieldVV_ji(tt,j,i);
+	corr_wwvv[t] += pionFieldWW_ji(t0,j,i)* pionFieldVV_ij(tt,i,j);
+      }}
+    }}    
+    for(int t=0;t<nt;t++) corr_wwvv[t] = corr_wwvv[t] / vol /2.0 ; // (ij+ji noise contribs if i!=j ).
+
+    for(int t=0;t<nt;t++) LOG(Message) << " C_wwvv " << t << " " << corr_wwvv[t]<<std::endl;
+
+
+    /////////////////////////////////////////////////////////////////////////
+    // This is only correct if there are NO low modes
+    // Use the "ii" case to construct possible Z wall source one end trick
+    /////////////////////////////////////////////////////////////////////////
+    std::vector<ComplexD> corr_z2(nt,ComplexD(0.0));
+    Eigen::Tensor<ComplexD,3> pionFieldWW        (nt,N_i,N_i);    
+    Eigen::Tensor<ComplexD,3> pionFieldVV        (nt,N_i,N_i);    
+
+
+    A2Autils<FImpl>::PionFieldWW(pionFieldWW, &wi[0], &wi[0],Tp);
+    A2Autils<FImpl>::PionFieldVV(pionFieldVV, &vi[0], &vi[0],Tp);
+    for(int i=0;i<N_i;i++){
+      for(int t0=0;t0<nt;t0++){
+      for(int t=0;t<nt;t++){
+	int tt = (t0+t)%nt;
+	corr_z2[t] += pionFieldWW(t0,i,i) * pionFieldVV(tt,i,i) /vol ;
+      }}
+    }
+
+    LOG(Message) << " C_z2 WARNING only correct if Nl == 0 "<<std::endl;
+    for(int t=0;t<nt;t++) LOG(Message) << " C_z2 " << t << " " << corr_z2[t]<<std::endl;
+
+    /////////////////////////////////////////////////////////////////////////
+    // Test: Build a bag contraction
+    /////////////////////////////////////////////////////////////////////////
+    Eigen::Tensor<ComplexD,2> DeltaF2_fig8  (nt,16);
+    Eigen::Tensor<ComplexD,2> DeltaF2_trtr  (nt,16);
+    Eigen::Tensor<ComplexD,1> denom0 (nt);
+    Eigen::Tensor<ComplexD,1> denom1 (nt);
+    
+    const int dT=16;
+
+    A2Autils<FImpl>::DeltaFeq2  (dT,dT,DeltaF2_fig8,DeltaF2_trtr,
+				 denom0,denom1,
+				 pionFieldWW_ij,&vi[0],&vj[0],Tp);
+    
+    { 
+      int g=0; // O_{VV+AA}
+      for(int t=0;t<nt;t++)
+	LOG(Message) << " Bag [" << t << ","<<g<<"]  " 
+		     << (DeltaF2_fig8(t,g)+DeltaF2_trtr(t,g)) 
+	  /             ( 8.0/3.0 * denom0[t]*denom1[t])
+		     <<std::endl;
+    }
+
+    /////////////////////////////////////////////////////////////////////////
+    // Test: Build a bag contraction the Z2 way
+    // Build a wall bag comparison assuming no low modes
+    /////////////////////////////////////////////////////////////////////////
+    LOG(Message) << " Bag_z2 WARNING only correct if Nl == 0 "<<std::endl;
+
+    int t0=0;
+    int t1=dT;
+    int Nl=0;
+    LatticePropagator Qd0(grid);
+    LatticePropagator Qd1(grid);
+    LatticePropagator Qs0(grid);
+    LatticePropagator Qs1(grid);
+    for(int s=0;s<4;s++){
+      for(int c=0;c<3;c++){
+	int idx0 = Nl+t0*12+s*3+c;
+	int idx1 = Nl+t1*12+s*3+c;
+	FermToProp<FImpl>(Qd0, vi[idx0], s, c);
+	FermToProp<FImpl>(Qd1, vi[idx1], s, c);
+	FermToProp<FImpl>(Qs0, vj[idx0], s, c);
+	FermToProp<FImpl>(Qs1, vj[idx1], s, c);
+      }
+    }
+
+    std::vector<Gamma::Algebra> gammas ( {
+	  Gamma::Algebra::GammaX,
+	  Gamma::Algebra::GammaY,
+	  Gamma::Algebra::GammaZ,
+	  Gamma::Algebra::GammaT,
+	  Gamma::Algebra::GammaXGamma5,
+	  Gamma::Algebra::GammaYGamma5,
+	  Gamma::Algebra::GammaZGamma5,
+	  Gamma::Algebra::GammaTGamma5,
+	  Gamma::Algebra::Identity,    
+          Gamma::Algebra::Gamma5,
+	  Gamma::Algebra::SigmaXY,
+	  Gamma::Algebra::SigmaXZ,
+	  Gamma::Algebra::SigmaXT,
+	  Gamma::Algebra::SigmaYZ,
+	  Gamma::Algebra::SigmaYT,
+	  Gamma::Algebra::SigmaZT
+    });
+
+    auto G5 = Gamma::Algebra::Gamma5;
+    LatticePropagator anti_d0 =  adj( Gamma(G5) * Qd0 * Gamma(G5));
+    LatticePropagator anti_d1 =  adj( Gamma(G5) * Qd1 * Gamma(G5));
+    LatticeComplex TR1(grid);
+    LatticeComplex TR2(grid);
+    LatticeComplex Wick1(grid);
+    LatticeComplex Wick2(grid);
+
+    LatticePropagator PR1(grid);
+    LatticePropagator PR2(grid);
+    PR1 = Qs0 * Gamma(G5) * anti_d0;
+    PR2 = Qs1 * Gamma(G5) * anti_d1;
+
+    for(int g=0;g<Nd*Nd;g++){
+      auto g1 = gammas[g];
+      Gamma G1 (g1);
+      TR1 = trace( PR1 * G1 );
+      TR2 = trace( PR2 * G1 );
+      Wick1 = TR1*TR2;
+      Wick2 = trace( PR1* G1 * PR2 * G1 );
+      
+      std::vector<TComplex>  C1;
+      std::vector<TComplex>  C2;
+      std::vector<TComplex>  C3;
+      sliceSum(Wick1,C1, Tp);
+      sliceSum(Wick2,C2, Tp);
+      sliceSum(TR1  ,C3, Tp);
+      
+      /*
+      if(g<5){
+	for(int t=0;t<C1.size();t++){
+	  LOG(Message) << " Wick1["<<g<<","<<t<< "] "<< C1[t]<<std::endl; 
+	}
+	for(int t=0;t<C2.size();t++){
+	  LOG(Message) << " Wick2["<<g<<","<<t<< "] "<< C2[t]<<std::endl; 
+	}
+      }
+      if( (g==9) || (g==7) ){ // P and At in above ordering
+	for(int t=0;t<C3.size();t++){
+	  LOG(Message) << " <G|P>["<<g<<","<<t<< "] "<< C3[t]<<std::endl; 
+	}
+      } 
+      */
+    }
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_A2APionField_hpp_
+  

extras/Hadrons/Modules/MContraction/Baryon.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Baryon.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TBaryon<FIMPL,FIMPL,FIMPL>;
+

extras/Hadrons/Modules/MContraction/Baryon.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/Baryon.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -27,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_Baryon_hpp_
-#define Hadrons_Baryon_hpp_
+#ifndef Hadrons_MContraction_Baryon_hpp_
+#define Hadrons_MContraction_Baryon_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -55,9 +55,9 @@ template <typename FImpl1, typename FImpl2, typename FImpl3>
 class TBaryon: public Module<BaryonPar>
 {
 public:
-    TYPE_ALIASES(FImpl1, 1);
-    TYPE_ALIASES(FImpl2, 2);
-    TYPE_ALIASES(FImpl3, 3);
+    FERM_TYPE_ALIASES(FImpl1, 1);
+    FERM_TYPE_ALIASES(FImpl2, 2);
+    FERM_TYPE_ALIASES(FImpl3, 3);
     class Result: Serializable
     {
     public:
@@ -68,15 +68,18 @@ public:
     // constructor
     TBaryon(const std::string name);
     // destructor
-    virtual ~TBaryon(void) = default;
+    virtual ~TBaryon(void) {};
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
+    // setup
+    virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(Baryon, ARG(TBaryon<FIMPL, FIMPL, FIMPL>), MContraction);
+MODULE_REGISTER_TMP(Baryon, ARG(TBaryon<FIMPL, FIMPL, FIMPL>), MContraction);
 
 /******************************************************************************
  *                         TBaryon implementation                             *
@@ -99,11 +102,18 @@ std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getInput(void)
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getOutput(void)
 {
-    std::vector<std::string> out = {getName()};
+    std::vector<std::string> out = {};
     
     return out;
 }
 
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TBaryon<FImpl1, FImpl2, FImpl3>::setup(void)
+{
+    envTmpLat(LatticeComplex, "c");
+}
+
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
@@ -112,20 +122,19 @@ void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
                  << " quarks '" << par().q1 << "', '" << par().q2 << "', and '"
                  << par().q3 << "'" << std::endl;
     
-    CorrWriter             writer(par().output);
-    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
-    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
-    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q2);
-    LatticeComplex        c(env().getGrid());
-    Result                result;
+    auto       &q1 = envGet(PropagatorField1, par().q1);
+    auto       &q2 = envGet(PropagatorField2, par().q2);
+    auto       &q3 = envGet(PropagatorField3, par().q2);
+    envGetTmp(LatticeComplex, c);
+    Result     result;
     
     // FIXME: do contractions
     
-    write(writer, "meson", result);
+    // saveResult(par().output, "meson", result);
 }
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_Baryon_hpp_
+#endif // Hadrons_MContraction_Baryon_hpp_

extras/Hadrons/Modules/MContraction/DiscLoop.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/DiscLoop.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TDiscLoop<FIMPL>;
+

extras/Hadrons/Modules/MContraction/DiscLoop.hpp

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/DiscLoop.hpp
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -26,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_DiscLoop_hpp_
-#define Hadrons_DiscLoop_hpp_
+#ifndef Hadrons_MContraction_DiscLoop_hpp_
+#define Hadrons_MContraction_DiscLoop_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -52,7 +53,7 @@ public:
 template <typename FImpl>
 class TDiscLoop: public Module<DiscLoopPar>
 {
-    TYPE_ALIASES(FImpl,);
+    FERM_TYPE_ALIASES(FImpl,);
     class Result: Serializable
     {
     public:
@@ -64,17 +65,18 @@ public:
     // constructor
     TDiscLoop(const std::string name);
     // destructor
-    virtual ~TDiscLoop(void) = default;
+    virtual ~TDiscLoop(void) {};
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(DiscLoop, TDiscLoop<FIMPL>, MContraction);
+MODULE_REGISTER_TMP(DiscLoop, TDiscLoop<FIMPL>, MContraction);
 
 /******************************************************************************
  *                       TDiscLoop implementation                             *
@@ -97,7 +99,7 @@ std::vector<std::string> TDiscLoop<FImpl>::getInput(void)
 template <typename FImpl>
 std::vector<std::string> TDiscLoop<FImpl>::getOutput(void)
 {
-    std::vector<std::string> out = {getName()};
+    std::vector<std::string> out = {};
     
     return out;
 }
@@ -106,7 +108,7 @@ std::vector<std::string> TDiscLoop<FImpl>::getOutput(void)
 template <typename FImpl>
 void TDiscLoop<FImpl>::setup(void)
 {
-    
+    envTmpLat(LatticeComplex, "c");
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -117,28 +119,25 @@ void TDiscLoop<FImpl>::execute(void)
                  << "' using '" << par().q_loop << "' with " << par().gamma 
                  << " insertion." << std::endl;
 
-    CorrWriter            writer(par().output);
-    PropagatorField       &q_loop = *env().template getObject<PropagatorField>(par().q_loop);
-    LatticeComplex        c(env().getGrid());
+    auto                  &q_loop = envGet(PropagatorField, par().q_loop);
     Gamma                 gamma(par().gamma);
     std::vector<TComplex> buf;
     Result                result;
 
+    envGetTmp(LatticeComplex, c);
     c = trace(gamma*q_loop);
     sliceSum(c, buf, Tp);
-
     result.gamma = par().gamma;
     result.corr.resize(buf.size());
     for (unsigned int t = 0; t < buf.size(); ++t)
     {
         result.corr[t] = TensorRemove(buf[t]);
     }
-
-    write(writer, "disc", result);
+    saveResult(par().output, "disc", result);
 }
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_DiscLoop_hpp_
+#endif // Hadrons_MContraction_DiscLoop_hpp_

extras/Hadrons/Modules/MContraction/Gamma3pt.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Gamma3pt.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TGamma3pt<FIMPL,FIMPL,FIMPL>;
+

extras/Hadrons/Modules/MContraction/Gamma3pt.hpp

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/Gamma3pt.hpp
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -26,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_Gamma3pt_hpp_
-#define Hadrons_Gamma3pt_hpp_
+#ifndef Hadrons_MContraction_Gamma3pt_hpp_
+#define Hadrons_MContraction_Gamma3pt_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -51,6 +52,14 @@ BEGIN_HADRONS_NAMESPACE
  *                   q1
  *
  *      trace(g5*q1*adj(q2)*g5*gamma*q3)
+ * 
+ *  options:
+ *   - q1: sink smeared propagator, source at i
+ *   - q2: propagator, source at i
+ *   - q3: propagator, source at f
+ *   - gamma: gamma matrix to insert
+ *   - tSnk: sink position for propagator q1.
+ *
  */
 
 /******************************************************************************
@@ -66,15 +75,16 @@ public:
                                     std::string,    q2,
                                     std::string,    q3,
                                     Gamma::Algebra, gamma,
+                                    unsigned int,   tSnk,
                                     std::string,    output);
 };
 
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 class TGamma3pt: public Module<Gamma3ptPar>
 {
-    TYPE_ALIASES(FImpl1, 1);
-    TYPE_ALIASES(FImpl2, 2);
-    TYPE_ALIASES(FImpl3, 3);
+    FERM_TYPE_ALIASES(FImpl1, 1);
+    FERM_TYPE_ALIASES(FImpl2, 2);
+    FERM_TYPE_ALIASES(FImpl3, 3);
     class Result: Serializable
     {
     public:
@@ -86,17 +96,18 @@ public:
     // constructor
     TGamma3pt(const std::string name);
     // destructor
-    virtual ~TGamma3pt(void) = default;
+    virtual ~TGamma3pt(void) {};
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(Gamma3pt, ARG(TGamma3pt<FIMPL, FIMPL, FIMPL>), MContraction);
+MODULE_REGISTER_TMP(Gamma3pt, ARG(TGamma3pt<FIMPL, FIMPL, FIMPL>), MContraction);
 
 /******************************************************************************
  *                       TGamma3pt implementation                             *
@@ -119,7 +130,7 @@ std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getInput(void)
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getOutput(void)
 {
-    std::vector<std::string> out = {getName()};
+    std::vector<std::string> out = {};
     
     return out;
 }
@@ -128,7 +139,7 @@ std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getOutput(void)
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 void TGamma3pt<FImpl1, FImpl2, FImpl3>::setup(void)
 {
-    
+    envTmpLat(LatticeComplex, "c");
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -140,17 +151,21 @@ void TGamma3pt<FImpl1, FImpl2, FImpl3>::execute(void)
                  << par().q3 << "', with " << par().gamma << " insertion." 
                  << std::endl;
 
-    CorrWriter            writer(par().output);
-    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
-    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
-    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q3);
-    LatticeComplex        c(env().getGrid());
+    // Initialise variables. q2 and q3 are normal propagators, q1 may be 
+    // sink smeared.
+    auto                  &q1 = envGet(SlicedPropagator1, par().q1);
+    auto                  &q2 = envGet(PropagatorField2, par().q2);
+    auto                  &q3 = envGet(PropagatorField2, par().q3);
     Gamma                 g5(Gamma::Algebra::Gamma5);
     Gamma                 gamma(par().gamma);
     std::vector<TComplex> buf;
     Result                result;
-
-    c = trace(g5*q1*adj(q2)*(g5*gamma)*q3);
+    
+    // Extract relevant timeslice of sinked propagator q1, then contract &
+    // sum over all spacial positions of gamma insertion.
+    SitePropagator1 q1Snk = q1[par().tSnk];
+    envGetTmp(LatticeComplex, c);
+    c = trace(g5*q1Snk*adj(q2)*(g5*gamma)*q3);
     sliceSum(c, buf, Tp);
 
     result.gamma = par().gamma;
@@ -159,12 +174,11 @@ void TGamma3pt<FImpl1, FImpl2, FImpl3>::execute(void)
     {
         result.corr[t] = TensorRemove(buf[t]);
     }
-
-    write(writer, "gamma3pt", result);
+    saveResult(par().output, "gamma3pt", result);
 }
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_Gamma3pt_hpp_
+#endif // Hadrons_MContraction_Gamma3pt_hpp_

extras/Hadrons/Modules/MContraction/Meson.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Meson.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TMeson<FIMPL,FIMPL>;
+

extras/Hadrons/Modules/MContraction/Meson.hpp

@@ -4,12 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/Meson.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-        Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -29,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_Meson_hpp_
-#define Hadrons_Meson_hpp_
+#ifndef Hadrons_MContraction_Meson_hpp_
+#define Hadrons_MContraction_Meson_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -47,12 +45,11 @@ BEGIN_HADRONS_NAMESPACE
  - q1: input propagator 1 (string)
  - q2: input propagator 2 (string)
  - gammas: gamma products to insert at sink & source, pairs of gamma matrices 
-           (space-separated strings) in angled brackets (i.e. <g_sink g_src>),
-           in a sequence (e.g. "<Gamma5 Gamma5><Gamma5 GammaT>").
+           (space-separated strings) in round brackets (i.e. (g_sink g_src)),
+           in a sequence (e.g. "(Gamma5 Gamma5)(Gamma5 GammaT)").
 
            Special values: "all" - perform all possible contractions.
- - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0."),
-        given as multiples of (2*pi) / L.
+ - sink: module to compute the sink to use in contraction (string).
 */
 
 /******************************************************************************
@@ -69,7 +66,7 @@ public:
                                     std::string, q1,
                                     std::string, q2,
                                     std::string, gammas,
-                                    std::string, mom,
+                                    std::string, sink,
                                     std::string, output);
 };
 
@@ -77,8 +74,10 @@ template <typename FImpl1, typename FImpl2>
 class TMeson: public Module<MesonPar>
 {
 public:
-    TYPE_ALIASES(FImpl1, 1);
-    TYPE_ALIASES(FImpl2, 2);
+    FERM_TYPE_ALIASES(FImpl1, 1);
+    FERM_TYPE_ALIASES(FImpl2, 2);
+    FERM_TYPE_ALIASES(ScalarImplCR, Scalar);
+    SINK_TYPE_ALIASES(Scalar);
     class Result: Serializable
     {
     public:
@@ -91,16 +90,19 @@ public:
     // constructor
     TMeson(const std::string name);
     // destructor
-    virtual ~TMeson(void) = default;
+    virtual ~TMeson(void) {};
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
     virtual void parseGammaString(std::vector<GammaPair> &gammaList);
+protected:
+    // execution
+    virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(Meson, ARG(TMeson<FIMPL, FIMPL>), MContraction);
+MODULE_REGISTER_TMP(Meson, ARG(TMeson<FIMPL, FIMPL>), MContraction);
 
 /******************************************************************************
  *                           TMeson implementation                            *
@@ -115,7 +117,7 @@ TMeson<FImpl1, FImpl2>::TMeson(const std::string name)
 template <typename FImpl1, typename FImpl2>
 std::vector<std::string> TMeson<FImpl1, FImpl2>::getInput(void)
 {
-    std::vector<std::string> input = {par().q1, par().q2};
+    std::vector<std::string> input = {par().q1, par().q2, par().sink};
     
     return input;
 }
@@ -123,7 +125,7 @@ std::vector<std::string> TMeson<FImpl1, FImpl2>::getInput(void)
 template <typename FImpl1, typename FImpl2>
 std::vector<std::string> TMeson<FImpl1, FImpl2>::getOutput(void)
 {
-    std::vector<std::string> output = {getName()};
+    std::vector<std::string> output = {};
     
     return output;
 }
@@ -131,12 +133,11 @@ std::vector<std::string> TMeson<FImpl1, FImpl2>::getOutput(void)
 template <typename FImpl1, typename FImpl2>
 void TMeson<FImpl1, FImpl2>::parseGammaString(std::vector<GammaPair> &gammaList)
 {
+    gammaList.clear();
     // Determine gamma matrices to insert at source/sink.
     if (par().gammas.compare("all") == 0)
     {
         // Do all contractions.
-        unsigned int n_gam = Ns * Ns;
-        gammaList.resize(n_gam*n_gam);
         for (unsigned int i = 1; i < Gamma::nGamma; i += 2)
         {
             for (unsigned int j = 1; j < Gamma::nGamma; j += 2)
@@ -150,11 +151,20 @@ void TMeson<FImpl1, FImpl2>::parseGammaString(std::vector<GammaPair> &gammaList)
     {
         // Parse individual contractions from input string.
         gammaList = strToVec<GammaPair>(par().gammas);
-    }
+    } 
 }
 
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2>
+void TMeson<FImpl1, FImpl2>::setup(void)
+{
+    envTmpLat(LatticeComplex, "c");
+}
 
 // execution ///////////////////////////////////////////////////////////////////
+#define mesonConnected(q1, q2, gSnk, gSrc) \
+(g5*(gSnk))*(q1)*(adj(gSrc)*g5)*adj(q2)
+
 template <typename FImpl1, typename FImpl2>
 void TMeson<FImpl1, FImpl2>::execute(void)
 {
@@ -162,50 +172,77 @@ void TMeson<FImpl1, FImpl2>::execute(void)
                  << " quarks '" << par().q1 << "' and '" << par().q2 << "'"
                  << std::endl;
     
-    CorrWriter              writer(par().output);
-    PropagatorField1       &q1 = *env().template getObject<PropagatorField1>(par().q1);
-    PropagatorField2       &q2 = *env().template getObject<PropagatorField2>(par().q2);
-    LatticeComplex         c(env().getGrid());
-    Gamma                  g5(Gamma::Algebra::Gamma5);
-    std::vector<GammaPair> gammaList;
     std::vector<TComplex>  buf;
     std::vector<Result>    result;
-    std::vector<Real>      p;
-
-    p  = strToVec<Real>(par().mom);
-    LatticeComplex         ph(env().getGrid()), coor(env().getGrid());
-    Complex                i(0.0,1.0);
-    ph = zero;
-    for(unsigned int mu = 0; mu < env().getNd(); mu++)
-    {
-        LatticeCoordinate(coor, mu);
-        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
-    }
-    ph = exp((Real)(2*M_PI)*i*ph);
+    Gamma                  g5(Gamma::Algebra::Gamma5);
+    std::vector<GammaPair> gammaList;
+    int                    nt = env().getDim(Tp);
     
     parseGammaString(gammaList);
-
     result.resize(gammaList.size());
     for (unsigned int i = 0; i < result.size(); ++i)
     {
-        Gamma gSnk(gammaList[i].first);
-        Gamma gSrc(gammaList[i].second);
-        c = trace((g5*gSnk)*q1*(adj(gSrc)*g5)*adj(q2))*ph;
-        sliceSum(c, buf, Tp);
-
         result[i].gamma_snk = gammaList[i].first;
         result[i].gamma_src = gammaList[i].second;
-        result[i].corr.resize(buf.size());
-        for (unsigned int t = 0; t < buf.size(); ++t)
+        result[i].corr.resize(nt);
+    }
+    if (envHasType(SlicedPropagator1, par().q1) and
+        envHasType(SlicedPropagator2, par().q2))
+    {
+        auto &q1 = envGet(SlicedPropagator1, par().q1);
+        auto &q2 = envGet(SlicedPropagator2, par().q2);
+        
+        LOG(Message) << "(propagator already sinked)" << std::endl;
+        for (unsigned int i = 0; i < result.size(); ++i)
         {
-            result[i].corr[t] = TensorRemove(buf[t]);
+            Gamma gSnk(gammaList[i].first);
+            Gamma gSrc(gammaList[i].second);
+            
+            for (unsigned int t = 0; t < buf.size(); ++t)
+            {
+                result[i].corr[t] = TensorRemove(trace(mesonConnected(q1[t], q2[t], gSnk, gSrc)));
+            }
         }
     }
-    write(writer, "meson", result);
+    else
+    {
+        auto &q1 = envGet(PropagatorField1, par().q1);
+        auto &q2 = envGet(PropagatorField2, par().q2);
+        
+        envGetTmp(LatticeComplex, c);
+        LOG(Message) << "(using sink '" << par().sink << "')" << std::endl;
+        for (unsigned int i = 0; i < result.size(); ++i)
+        {
+            Gamma       gSnk(gammaList[i].first);
+            Gamma       gSrc(gammaList[i].second);
+            std::string ns;
+                
+            ns = vm().getModuleNamespace(env().getObjectModule(par().sink));
+            if (ns == "MSource")
+            {
+                PropagatorField1 &sink = envGet(PropagatorField1, par().sink);
+                
+                c = trace(mesonConnected(q1, q2, gSnk, gSrc)*sink);
+                sliceSum(c, buf, Tp);
+            }
+            else if (ns == "MSink")
+            {
+                SinkFnScalar &sink = envGet(SinkFnScalar, par().sink);
+                
+                c   = trace(mesonConnected(q1, q2, gSnk, gSrc));
+                buf = sink(c);
+            }
+            for (unsigned int t = 0; t < buf.size(); ++t)
+            {
+                result[i].corr[t] = TensorRemove(buf[t]);
+            }
+        }
+    }
+    saveResult(par().output, "meson", result);
 }
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_Meson_hpp_
+#endif // Hadrons_MContraction_Meson_hpp_

extras/Hadrons/Modules/MContraction/MesonFieldGamma.cc

@@ -0,0 +1,8 @@
+#include <Grid/Hadrons/Modules/MContraction/MesonFieldGamma.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TMesonFieldGamma<FIMPL>;
+template class Grid::Hadrons::MContraction::TMesonFieldGamma<ZFIMPL>;

extras/Hadrons/Modules/MContraction/MesonFieldGamma.hpp

@@ -0,0 +1,269 @@
+#ifndef Hadrons_MContraction_MesonFieldGamma_hpp_
+#define Hadrons_MContraction_MesonFieldGamma_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+#include <Grid/Hadrons/AllToAllVectors.hpp>
+#include <Grid/Hadrons/AllToAllReduction.hpp>
+#include <Grid/Grid_Eigen_Dense.h>
+#include <fstream>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         MesonFieldGamma                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class MesonFieldPar : Serializable
+{
+  public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(MesonFieldPar,
+                                    int, Nl,
+                                    int, N,
+                                    int, Nblock,
+                                    std::string, A2A1,
+                                    std::string, A2A2,
+                                    std::string, gammas,
+                                    std::string, output);
+};
+
+template <typename FImpl>
+class TMesonFieldGamma : public Module<MesonFieldPar>
+{
+  public:
+    FERM_TYPE_ALIASES(FImpl, );
+    SOLVER_TYPE_ALIASES(FImpl, );
+
+    typedef A2AModesSchurDiagTwo<typename FImpl::FermionField, FMat, Solver> A2ABase;
+
+    class Result : Serializable
+    {
+      public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma,
+                                        std::vector<std::vector<std::vector<ComplexD>>>, MesonField);
+    };
+
+  public:
+    // constructor
+    TMesonFieldGamma(const std::string name);
+    // destructor
+    virtual ~TMesonFieldGamma(void){};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    virtual void parseGammaString(std::vector<Gamma::Algebra> &gammaList);
+    virtual void vectorOfWs(std::vector<FermionField> &w, int i, int Nblock, FermionField &tmpw_5d, std::vector<FermionField> &vec_w);
+    virtual void vectorOfVs(std::vector<FermionField> &v, int j, int Nblock, FermionField &tmpv_5d, std::vector<FermionField> &vec_v);
+    virtual void gammaMult(std::vector<FermionField> &v, Gamma gamma);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER(MesonFieldGamma, ARG(TMesonFieldGamma<FIMPL>), MContraction);
+MODULE_REGISTER(ZMesonFieldGamma, ARG(TMesonFieldGamma<ZFIMPL>), MContraction);
+
+/******************************************************************************
+*                  TMesonFieldGamma implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TMesonFieldGamma<FImpl>::TMesonFieldGamma(const std::string name)
+    : Module<MesonFieldPar>(name)
+{
+}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TMesonFieldGamma<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().A2A1 + "_class", par().A2A2 + "_class"};
+    in.push_back(par().A2A1 + "_w_high_4d");
+    in.push_back(par().A2A2 + "_v_high_4d");
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TMesonFieldGamma<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {};
+
+    return out;
+}
+
+template <typename FImpl>
+void TMesonFieldGamma<FImpl>::parseGammaString(std::vector<Gamma::Algebra> &gammaList)
+{
+    gammaList.clear();
+    // Determine gamma matrices to insert at source/sink.
+    if (par().gammas.compare("all") == 0)
+    {
+        // Do all contractions.
+        for (unsigned int i = 1; i < Gamma::nGamma; i += 2)
+        {
+            gammaList.push_back(((Gamma::Algebra)i));
+        }
+    }
+    else
+    {
+        // Parse individual contractions from input string.
+        gammaList = strToVec<Gamma::Algebra>(par().gammas);
+    }
+}
+
+template <typename FImpl>
+void TMesonFieldGamma<FImpl>::vectorOfWs(std::vector<FermionField> &w, int i, int Nblock, FermionField &tmpw_5d, std::vector<FermionField> &vec_w)
+{
+    for (unsigned int ni = 0; ni < Nblock; ni++)
+    {
+        vec_w[ni] = w[i + ni];
+    }
+}
+
+template <typename FImpl>
+void TMesonFieldGamma<FImpl>::vectorOfVs(std::vector<FermionField> &v, int j, int Nblock, FermionField &tmpv_5d, std::vector<FermionField> &vec_v)
+{
+    for (unsigned int nj = 0; nj < Nblock; nj++)
+    {
+        vec_v[nj] = v[j+nj];
+    }
+}
+
+template <typename FImpl>
+void TMesonFieldGamma<FImpl>::gammaMult(std::vector<FermionField> &v, Gamma gamma)
+{
+    int Nblock = v.size();
+    for (unsigned int nj = 0; nj < Nblock; nj++)
+    {
+        v[nj] = gamma * v[nj];
+    }
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TMesonFieldGamma<FImpl>::setup(void)
+{
+    int nt = env().getDim(Tp);
+    int N = par().N;
+    int Nblock = par().Nblock;
+
+    int Ls_ = env().getObjectLs(par().A2A1 + "_class");
+
+    envTmpLat(FermionField, "tmpv_5d", Ls_);
+    envTmpLat(FermionField, "tmpw_5d", Ls_);
+
+    envTmp(std::vector<FermionField>, "w", 1, N, FermionField(env().getGrid(1)));
+    envTmp(std::vector<FermionField>, "v", 1, N, FermionField(env().getGrid(1)));
+
+    envTmp(Eigen::MatrixXcd, "MF", 1, Eigen::MatrixXcd::Zero(nt, N * N));
+
+    envTmp(std::vector<FermionField>, "w_block", 1, Nblock, FermionField(env().getGrid(1)));
+    envTmp(std::vector<FermionField>, "v_block", 1, Nblock, FermionField(env().getGrid(1)));
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TMesonFieldGamma<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing A2A meson field for gamma = " << par().gammas << ", taking w from " << par().A2A1 << " and v from " << par().A2A2 << std::endl;
+
+    int N = par().N;
+    int nt = env().getDim(Tp);
+    int Nblock = par().Nblock;
+
+    std::vector<Result> result;
+    std::vector<Gamma::Algebra> gammaResultList;
+    std::vector<Gamma> gammaList;
+
+    parseGammaString(gammaResultList);
+    result.resize(gammaResultList.size());
+
+    Gamma g5(Gamma::Algebra::Gamma5);
+    gammaList.resize(gammaResultList.size(), g5);
+
+    for (unsigned int i = 0; i < result.size(); ++i)
+    {
+        result[i].gamma = gammaResultList[i];
+        result[i].MesonField.resize(N, std::vector<std::vector<ComplexD>>(N, std::vector<ComplexD>(nt)));
+
+        Gamma gamma(gammaResultList[i]);
+        gammaList[i] = gamma;
+    }
+
+    auto &a2a1 = envGet(A2ABase, par().A2A1 + "_class");
+    auto &a2a2 = envGet(A2ABase, par().A2A2 + "_class");
+
+    envGetTmp(FermionField, tmpv_5d);
+    envGetTmp(FermionField, tmpw_5d);
+
+    envGetTmp(std::vector<FermionField>, v);
+    envGetTmp(std::vector<FermionField>, w);
+    LOG(Message) << "Finding v and w vectors for N =  " << N << std::endl;
+    for (int i = 0; i < N; i++)
+    {
+        a2a2.return_v(i, tmpv_5d, v[i]);
+        a2a1.return_w(i, tmpw_5d, w[i]);
+    }
+    LOG(Message) << "Found v and w vectors for N =  " << N << std::endl;
+
+    std::vector<std::vector<ComplexD>> MesonField_ij;
+    LOG(Message) << "Before blocked MFs, Nblock = " << Nblock << std::endl;
+    envGetTmp(std::vector<FermionField>, v_block);
+    envGetTmp(std::vector<FermionField>, w_block);
+    MesonField_ij.resize(Nblock * Nblock, std::vector<ComplexD>(nt));
+
+    envGetTmp(Eigen::MatrixXcd, MF);
+
+    LOG(Message) << "Before blocked MFs, Nblock = " << Nblock << std::endl;
+    for (unsigned int i = 0; i < N; i += Nblock)
+    {
+        vectorOfWs(w, i, Nblock, tmpw_5d, w_block);
+        for (unsigned int j = 0; j < N; j += Nblock)
+        {
+            vectorOfVs(v, j, Nblock, tmpv_5d, v_block);
+            for (unsigned int k = 0; k < result.size(); k++)
+            {
+                gammaMult(v_block, gammaList[k]);
+                sliceInnerProductMesonField(MesonField_ij, w_block, v_block, Tp);
+                for (unsigned int nj = 0; nj < Nblock; nj++)
+                {
+                    for (unsigned int ni = 0; ni < Nblock; ni++)
+                    {
+                        MF.col((i + ni) + (j + nj) * N) = Eigen::VectorXcd::Map(&MesonField_ij[nj * Nblock + ni][0], MesonField_ij[nj * Nblock + ni].size());
+                    }
+                }
+            }
+        }
+        if (i % 10 == 0)
+        {
+            LOG(Message) << "MF for i = " << i << " of " << N << std::endl;
+        }
+    }
+    LOG(Message) << "Before Global sum, Nblock = " << Nblock << std::endl;
+    v_block[0]._grid->GlobalSumVector(MF.data(), MF.size());
+    LOG(Message) << "After Global sum, Nblock = " << Nblock << std::endl;
+    for (unsigned int i = 0; i < N; i++)
+    {
+        for (unsigned int j = 0; j < N; j++)
+        {
+            for (unsigned int k = 0; k < result.size(); k++)
+            {
+                for (unsigned int t = 0; t < nt; t++)
+                {
+                    result[k].MesonField[i][j][t] = MF.col(i + N * j)[t];
+                }
+            }
+        }
+    }
+    saveResult(par().output, "meson", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_MesonFieldGm_hpp_

extras/Hadrons/Modules/MContraction/WardIdentity.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WardIdentity.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MContraction/WardIdentity.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+template class Grid::Hadrons::MContraction::TWardIdentity<FIMPL>;
+

extras/Hadrons/Modules/MContraction/WardIdentity.hpp

@@ -0,0 +1,224 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WardIdentity.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_WardIdentity_hpp_
+#define Hadrons_MContraction_WardIdentity_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+  Ward Identity contractions
+ -----------------------------
+ 
+ * options:
+ - q:          propagator, 5D if available (string)
+ - action:     action module used for propagator solution (string)
+ - mass:       mass of quark (double)
+ - test_axial: whether or not to test PCAC relation.
+*/
+
+/******************************************************************************
+ *                              WardIdentity                                  *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class WardIdentityPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WardIdentityPar,
+                                    std::string, q,
+                                    std::string, action,
+                                    double,      mass,
+                                    bool,        test_axial);
+};
+
+template <typename FImpl>
+class TWardIdentity: public Module<WardIdentityPar>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TWardIdentity(const std::string name);
+    // destructor
+    virtual ~TWardIdentity(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+protected:
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+private:
+    unsigned int Ls_;
+};
+
+MODULE_REGISTER_TMP(WardIdentity, TWardIdentity<FIMPL>, MContraction);
+
+/******************************************************************************
+ *                     TWardIdentity implementation                           *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TWardIdentity<FImpl>::TWardIdentity(const std::string name)
+: Module<WardIdentityPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TWardIdentity<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q, par().action};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TWardIdentity<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWardIdentity<FImpl>::setup(void)
+{
+    Ls_ = env().getObjectLs(par().q);
+    if (Ls_ != env().getObjectLs(par().action))
+    {
+        HADRONS_ERROR(Size, "Ls mismatch between quark action and propagator");
+    }
+    envTmpLat(PropagatorField, "tmp");
+    envTmpLat(PropagatorField, "vector_WI");
+    if (par().test_axial)
+    {
+        envTmpLat(PropagatorField, "psi");
+        envTmpLat(LatticeComplex,  "PP");
+        envTmpLat(LatticeComplex,  "axial_defect");
+        envTmpLat(LatticeComplex,  "PJ5q");
+    }
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWardIdentity<FImpl>::execute(void)
+{
+    LOG(Message) << "Performing Ward Identity checks for quark '" << par().q
+                 << "'." << std::endl;
+
+    auto  &q   = envGet(PropagatorField, par().q);
+    auto  &act = envGet(FMat, par().action);
+    Gamma g5(Gamma::Algebra::Gamma5);
+
+    // Compute D_mu V_mu, D here is backward derivative.
+    envGetTmp(PropagatorField, tmp);
+    envGetTmp(PropagatorField, vector_WI);
+    vector_WI    = zero;
+    for (unsigned int mu = 0; mu < Nd; ++mu)
+    {
+        act.ContractConservedCurrent(q, q, tmp, Current::Vector, mu);
+        tmp -= Cshift(tmp, mu, -1);
+        vector_WI += tmp;
+    }
+
+    // Test ward identity D_mu V_mu = 0;
+    LOG(Message) << "Vector Ward Identity check Delta_mu V_mu = " 
+                 << norm2(vector_WI) << std::endl;
+
+    if (par().test_axial)
+    {
+        envGetTmp(PropagatorField, psi);
+        envGetTmp(LatticeComplex, PP);
+        envGetTmp(LatticeComplex, axial_defect);
+        envGetTmp(LatticeComplex, PJ5q);
+        std::vector<TComplex> axial_buf;
+
+        // Compute <P|D_mu A_mu>, D is backwards derivative.
+        axial_defect = zero;
+        for (unsigned int mu = 0; mu < Nd; ++mu)
+        {
+            act.ContractConservedCurrent(q, q, tmp, Current::Axial, mu);
+            tmp -= Cshift(tmp, mu, -1);
+            axial_defect += trace(g5*tmp);
+        }
+
+        // Get <P|J5q> for 5D (zero for 4D) and <P|P>.
+        PJ5q = zero;
+        if (Ls_ > 1)
+        {
+            // <P|P>
+            ExtractSlice(tmp, q, 0, 0);
+            psi  = 0.5 * (tmp - g5*tmp);
+            ExtractSlice(tmp, q, Ls_ - 1, 0);
+            psi += 0.5 * (tmp + g5*tmp);
+            PP = trace(adj(psi)*psi);
+
+            // <P|5Jq>
+            ExtractSlice(tmp, q, Ls_/2 - 1, 0);
+            psi  = 0.5 * (tmp + g5*tmp);
+            ExtractSlice(tmp, q, Ls_/2, 0);
+            psi += 0.5 * (tmp - g5*tmp);
+            PJ5q = trace(adj(psi)*psi);
+        }
+        else
+        {
+            PP = trace(adj(q)*q);
+        }
+
+        // Test ward identity <P|D_mu A_mu> = 2m<P|P> + 2<P|J5q>
+        LOG(Message) << "|D_mu A_mu|^2 = " << norm2(axial_defect) << std::endl;
+        LOG(Message) << "|PP|^2        = " << norm2(PP) << std::endl;
+        LOG(Message) << "|PJ5q|^2      = " << norm2(PJ5q) << std::endl;
+        LOG(Message) << "Axial Ward Identity defect Delta_mu A_mu = "
+                     << norm2(axial_defect) << std::endl;
+    
+        // Axial defect by timeslice.
+        axial_defect -= 2.*(par().mass*PP + PJ5q);
+        LOG(Message) << "Check Axial defect by timeslice" << std::endl;
+        sliceSum(axial_defect, axial_buf, Tp);
+        for (int t = 0; t < axial_buf.size(); ++t)
+        {
+            LOG(Message) << "t = " << t << ": " 
+                         << TensorRemove(axial_buf[t]) << std::endl;
+        }
+    }
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WardIdentity_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -26,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_WeakHamiltonian_hpp_
-#define Hadrons_WeakHamiltonian_hpp_
+#ifndef Hadrons_MContraction_WeakHamiltonian_hpp_
+#define Hadrons_MContraction_WeakHamiltonian_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -76,6 +77,7 @@ public:
                                     std::string, q2,
                                     std::string, q3,
                                     std::string, q4,
+                                    unsigned int, tSnk,
                                     std::string, output);
 };
 
@@ -83,7 +85,7 @@ public:
 class T##modname: public Module<WeakHamiltonianPar>\
 {\
 public:\
-    TYPE_ALIASES(FIMPL,)\
+    FERM_TYPE_ALIASES(FIMPL,)\
     class Result: Serializable\
     {\
     public:\
@@ -95,20 +97,22 @@ public:\
     /* constructor */ \
     T##modname(const std::string name);\
     /* destructor */ \
-    virtual ~T##modname(void) = default;\
+    virtual ~T##modname(void) {};\
     /* dependency relation */ \
     virtual std::vector<std::string> getInput(void);\
     virtual std::vector<std::string> getOutput(void);\
+public:\
+    std::vector<std::string> VA_label = {"V", "A"};\
+protected:\
     /* setup */ \
     virtual void setup(void);\
     /* execution */ \
     virtual void execute(void);\
-    std::vector<std::string> VA_label = {"V", "A"};\
 };\
-MODULE_REGISTER_NS(modname, T##modname, MContraction);
+MODULE_REGISTER(modname, T##modname, MContraction);
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_WeakHamiltonian_hpp_
+#endif // Hadrons_MContraction_WeakHamiltonian_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -54,6 +55,8 @@ using namespace MContraction;
  * 
  * S: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1]*q4*gL[mu][p_2])
  * E: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1])*trace(q4*gL[mu][p_2])
+ * 
+ * Note q1 must be sink smeared.
  */
 
 /******************************************************************************
@@ -74,7 +77,7 @@ std::vector<std::string> TWeakHamiltonianEye::getInput(void)
 
 std::vector<std::string> TWeakHamiltonianEye::getOutput(void)
 {
-    std::vector<std::string> out = {getName()};
+    std::vector<std::string> out = {};
     
     return out;
 }
@@ -82,7 +85,15 @@ std::vector<std::string> TWeakHamiltonianEye::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TWeakHamiltonianEye::setup(void)
 {
+    unsigned int ndim = env().getNd();
 
+    envTmpLat(LatticeComplex,  "expbuf");
+    envTmpLat(PropagatorField, "tmp1");
+    envTmpLat(LatticeComplex,  "tmp2");
+    envTmp(std::vector<PropagatorField>, "S_body", 1, ndim, PropagatorField(env().getGrid()));
+    envTmp(std::vector<PropagatorField>, "S_loop", 1, ndim, PropagatorField(env().getGrid()));
+    envTmp(std::vector<LatticeComplex>,  "E_body", 1, ndim, LatticeComplex(env().getGrid()));
+    envTmp(std::vector<LatticeComplex>,  "E_loop", 1, ndim, LatticeComplex(env().getGrid()));
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -93,28 +104,30 @@ void TWeakHamiltonianEye::execute(void)
                  << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
                  << "'." << std::endl;
 
-    CorrWriter             writer(par().output);
-    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
-    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
-    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
-    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
-    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
-    LatticeComplex        expbuf(env().getGrid());
-    std::vector<TComplex> corrbuf;
-    std::vector<Result>   result(n_eye_diag);
-    unsigned int ndim   = env().getNd();
+    auto                   &q1 = envGet(SlicedPropagator, par().q1);
+    auto                   &q2 = envGet(PropagatorField, par().q2);
+    auto                   &q3 = envGet(PropagatorField, par().q3);
+    auto                   &q4 = envGet(PropagatorField, par().q4);
+    Gamma                  g5  = Gamma(Gamma::Algebra::Gamma5);
+    std::vector<TComplex>  corrbuf;
+    std::vector<Result>    result(n_eye_diag);
+    unsigned int ndim    = env().getNd();
 
-    PropagatorField              tmp1(env().getGrid());
-    LatticeComplex               tmp2(env().getGrid());
-    std::vector<PropagatorField> S_body(ndim, tmp1);
-    std::vector<PropagatorField> S_loop(ndim, tmp1);
-    std::vector<LatticeComplex>  E_body(ndim, tmp2);
-    std::vector<LatticeComplex>  E_loop(ndim, tmp2);
+    envGetTmp(LatticeComplex,               expbuf); 
+    envGetTmp(PropagatorField,              tmp1);
+    envGetTmp(LatticeComplex,               tmp2);
+    envGetTmp(std::vector<PropagatorField>, S_body);
+    envGetTmp(std::vector<PropagatorField>, S_loop);
+    envGetTmp(std::vector<LatticeComplex>,  E_body);
+    envGetTmp(std::vector<LatticeComplex>,  E_loop);
+
+    // Get sink timeslice of q1.
+    SitePropagator q1Snk = q1[par().tSnk];
 
     // Setup for S-type contractions.
     for (int mu = 0; mu < ndim; ++mu)
     {
-        S_body[mu] = MAKE_SE_BODY(q1, q2, q3, GammaL(Gamma::gmu[mu]));
+        S_body[mu] = MAKE_SE_BODY(q1Snk, q2, q3, GammaL(Gamma::gmu[mu]));
         S_loop[mu] = MAKE_SE_LOOP(q4, GammaL(Gamma::gmu[mu]));
     }
 
@@ -133,5 +146,6 @@ void TWeakHamiltonianEye::execute(void)
     SUM_MU(expbuf, E_body[mu]*E_loop[mu])
     MAKE_DIAG(expbuf, corrbuf, result[E_diag], "HW_E")
 
-    write(writer, "HW_Eye", result);
+    // IO
+    saveResult(par().output, "HW_Eye", result);
 }

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -26,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_WeakHamiltonianEye_hpp_
-#define Hadrons_WeakHamiltonianEye_hpp_
+#ifndef Hadrons_MContraction_WeakHamiltonianEye_hpp_
+#define Hadrons_MContraction_WeakHamiltonianEye_hpp_
 
 #include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
 
@@ -55,4 +56,4 @@ END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_WeakHamiltonianEye_hpp_
+#endif // Hadrons_MContraction_WeakHamiltonianEye_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -76,7 +77,7 @@ std::vector<std::string> TWeakHamiltonianNonEye::getInput(void)
 
 std::vector<std::string> TWeakHamiltonianNonEye::getOutput(void)
 {
-    std::vector<std::string> out = {getName()};
+    std::vector<std::string> out = {};
     
     return out;
 }
@@ -84,7 +85,15 @@ std::vector<std::string> TWeakHamiltonianNonEye::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TWeakHamiltonianNonEye::setup(void)
 {
+    unsigned int ndim = env().getNd();
 
+    envTmpLat(LatticeComplex,  "expbuf");
+    envTmpLat(PropagatorField, "tmp1");
+    envTmpLat(LatticeComplex,  "tmp2");
+    envTmp(std::vector<PropagatorField>, "C_i_side_loop", 1, ndim, PropagatorField(env().getGrid()));
+    envTmp(std::vector<PropagatorField>, "C_f_side_loop", 1, ndim, PropagatorField(env().getGrid()));
+    envTmp(std::vector<LatticeComplex>,  "W_i_side_loop", 1, ndim, LatticeComplex(env().getGrid()));
+    envTmp(std::vector<LatticeComplex>,  "W_f_side_loop", 1, ndim, LatticeComplex(env().getGrid()));
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -95,23 +104,22 @@ void TWeakHamiltonianNonEye::execute(void)
                  << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
                  << "'." << std::endl;
     
-    CorrWriter             writer(par().output);
-    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
-    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
-    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
-    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
-    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
-    LatticeComplex        expbuf(env().getGrid());
+    auto                  &q1 = envGet(PropagatorField, par().q1);
+    auto                  &q2 = envGet(PropagatorField, par().q2);
+    auto                  &q3 = envGet(PropagatorField, par().q3);
+    auto                  &q4 = envGet(PropagatorField, par().q4);
+    Gamma                 g5  = Gamma(Gamma::Algebra::Gamma5);
     std::vector<TComplex> corrbuf;
     std::vector<Result>   result(n_noneye_diag); 
-    unsigned int ndim   = env().getNd();
+    unsigned int          ndim = env().getNd();
 
-    PropagatorField              tmp1(env().getGrid());
-    LatticeComplex               tmp2(env().getGrid());
-    std::vector<PropagatorField> C_i_side_loop(ndim, tmp1);
-    std::vector<PropagatorField> C_f_side_loop(ndim, tmp1);
-    std::vector<LatticeComplex>  W_i_side_loop(ndim, tmp2);
-    std::vector<LatticeComplex>  W_f_side_loop(ndim, tmp2);
+    envGetTmp(LatticeComplex,               expbuf); 
+    envGetTmp(PropagatorField,              tmp1);
+    envGetTmp(LatticeComplex,               tmp2);
+    envGetTmp(std::vector<PropagatorField>, C_i_side_loop);
+    envGetTmp(std::vector<PropagatorField>, C_f_side_loop);
+    envGetTmp(std::vector<LatticeComplex>,  W_i_side_loop);
+    envGetTmp(std::vector<LatticeComplex>,  W_f_side_loop);
 
     // Setup for C-type contractions.
     for (int mu = 0; mu < ndim; ++mu)
@@ -135,5 +143,6 @@ void TWeakHamiltonianNonEye::execute(void)
     SUM_MU(expbuf, W_i_side_loop[mu]*W_f_side_loop[mu])
     MAKE_DIAG(expbuf, corrbuf, result[W_diag], "HW_W")
 
-    write(writer, "HW_NonEye", result);
+    // IO
+    saveResult(par().output, "HW_NonEye", result);
 }

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -26,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_WeakHamiltonianNonEye_hpp_
-#define Hadrons_WeakHamiltonianNonEye_hpp_
+#ifndef Hadrons_MContraction_WeakHamiltonianNonEye_hpp_
+#define Hadrons_MContraction_WeakHamiltonianNonEye_hpp_
 
 #include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
 
@@ -54,4 +55,4 @@ END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_WeakHamiltonianNonEye_hpp_
+#endif // Hadrons_MContraction_WeakHamiltonianNonEye_hpp_

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -78,7 +79,7 @@ std::vector<std::string> TWeakNeutral4ptDisc::getInput(void)
 
 std::vector<std::string> TWeakNeutral4ptDisc::getOutput(void)
 {
-    std::vector<std::string> out = {getName()};
+    std::vector<std::string> out = {};
     
     return out;
 }
@@ -86,7 +87,13 @@ std::vector<std::string> TWeakNeutral4ptDisc::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TWeakNeutral4ptDisc::setup(void)
 {
+    unsigned int ndim = env().getNd();
 
+    envTmpLat(LatticeComplex,  "expbuf");
+    envTmpLat(PropagatorField, "tmp");
+    envTmpLat(LatticeComplex,  "curr");
+    envTmp(std::vector<PropagatorField>, "meson", 1, ndim, PropagatorField(env().getGrid()));
+    envTmp(std::vector<PropagatorField>, "loop", 1, ndim,  PropagatorField(env().getGrid()));
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -97,21 +104,20 @@ void TWeakNeutral4ptDisc::execute(void)
                  << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
                  << "'." << std::endl;
 
-    CorrWriter             writer(par().output);
-    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
-    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
-    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
-    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
-    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
-    LatticeComplex        expbuf(env().getGrid());
+    auto                  &q1 = envGet(PropagatorField, par().q1);
+    auto                  &q2 = envGet(PropagatorField, par().q2);
+    auto                  &q3 = envGet(PropagatorField, par().q3);
+    auto                  &q4 = envGet(PropagatorField, par().q4);
+    Gamma                 g5  = Gamma(Gamma::Algebra::Gamma5);
     std::vector<TComplex> corrbuf;
     std::vector<Result>   result(n_neut_disc_diag);
-    unsigned int ndim   = env().getNd();
+    unsigned int          ndim = env().getNd();
 
-    PropagatorField              tmp(env().getGrid());
-    std::vector<PropagatorField> meson(ndim, tmp);
-    std::vector<PropagatorField> loop(ndim, tmp);
-    LatticeComplex               curr(env().getGrid());
+    envGetTmp(LatticeComplex,               expbuf); 
+    envGetTmp(PropagatorField,              tmp);
+    envGetTmp(LatticeComplex,               curr);
+    envGetTmp(std::vector<PropagatorField>, meson);
+    envGetTmp(std::vector<PropagatorField>, loop);
 
     // Setup for type 1 contractions.
     for (int mu = 0; mu < ndim; ++mu)
@@ -131,5 +137,6 @@ void TWeakNeutral4ptDisc::execute(void)
     expbuf *= curr;
     MAKE_DIAG(expbuf, corrbuf, result[neut_disc_2_diag], "HW_disc0_2")
 
-    write(writer, "HW_disc0", result);
+    // IO
+    saveResult(par().output, "HW_disc0", result);
 }

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp

@@ -4,9 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp
 
-Copyright (C) 2017
+Copyright (C) 2015-2018
 
-Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -26,8 +27,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_WeakNeutral4ptDisc_hpp_
-#define Hadrons_WeakNeutral4ptDisc_hpp_
+#ifndef Hadrons_MContraction_WeakNeutral4ptDisc_hpp_
+#define Hadrons_MContraction_WeakNeutral4ptDisc_hpp_
 
 #include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
 
@@ -56,4 +57,4 @@ END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_WeakNeutral4ptDisc_hpp_
+#endif // Hadrons_MContraction_WeakNeutral4ptDisc_hpp_

extras/Hadrons/Modules/MFermion/FreeProp.cc

@@ -0,0 +1,36 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MFermion/FreeProp.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Vera Guelpers <V.M.Guelpers@soton.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MFermion/FreeProp.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MFermion;
+
+template class Grid::Hadrons::MFermion::TFreeProp<FIMPL>;
+

extras/Hadrons/Modules/MFermion/FreeProp.hpp

@@ -0,0 +1,187 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MFermion/FreeProp.hpp
+
+Copyright (C) 2015-2018
+
+Author: Vera Guelpers <V.M.Guelpers@soton.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+
+#ifndef Hadrons_MFermion_FreeProp_hpp_
+#define Hadrons_MFermion_FreeProp_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         FreeProp                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MFermion)
+
+class FreePropPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(FreePropPar,
+                                    std::string, source,
+				    std::string,  action,
+				    double, mass,
+				    std::string,  twist);
+};
+
+template <typename FImpl>
+class TFreeProp: public Module<FreePropPar>
+{
+public:
+    FG_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TFreeProp(const std::string name);
+    // destructor
+    virtual ~TFreeProp(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+protected:
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+private:
+    unsigned int Ls_;
+};
+
+MODULE_REGISTER_TMP(FreeProp, TFreeProp<FIMPL>, MFermion);
+
+/******************************************************************************
+ *                 TFreeProp implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TFreeProp<FImpl>::TFreeProp(const std::string name)
+: Module<FreePropPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TFreeProp<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().source, par().action};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TFreeProp<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName(), getName() + "_5d"};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TFreeProp<FImpl>::setup(void)
+{
+    Ls_ = env().getObjectLs(par().action);
+    envCreateLat(PropagatorField, getName());
+    envTmpLat(FermionField, "source", Ls_);
+    envTmpLat(FermionField, "sol", Ls_);
+    envTmpLat(FermionField, "tmp");
+    if (Ls_ > 1)
+    {
+        envCreateLat(PropagatorField, getName() + "_5d", Ls_);
+    }    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TFreeProp<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing free fermion propagator '" << getName() << "'"
+                 << std::endl;
+    
+    std::string propName = (Ls_ == 1) ? getName() : (getName() + "_5d");
+    auto        &prop    = envGet(PropagatorField, propName);
+    auto        &fullSrc = envGet(PropagatorField, par().source);
+    auto        &mat = envGet(FMat, par().action);
+    RealD mass = par().mass;
+    
+    envGetTmp(FermionField, source);
+    envGetTmp(FermionField, sol);
+    envGetTmp(FermionField, tmp);
+    LOG(Message) << "Calculating a free Propagator with mass " << mass 
+		 << " using the action '" << par().action
+                 << "' on source '" << par().source << "'" << std::endl;
+    for (unsigned int s = 0; s < Ns; ++s)
+      for (unsigned int c = 0; c < FImpl::Dimension; ++c)
+    {
+        LOG(Message) << "Calculation for spin= " << s << ", color= " << c
+                     << std::endl;
+        // source conversion for 4D sources
+        if (!env().isObject5d(par().source))
+        {
+            if (Ls_ == 1)
+            {
+               PropToFerm<FImpl>(source, fullSrc, s, c);
+            }
+            else
+            {
+                PropToFerm<FImpl>(tmp, fullSrc, s, c);
+                mat.ImportPhysicalFermionSource(tmp, source);
+            }
+        }
+        // source conversion for 5D sources
+        else
+        {
+            if (Ls_ != env().getObjectLs(par().source))
+            {
+                HADRONS_ERROR(Size, "Ls mismatch between quark action and source");
+            }
+            else
+            {
+                PropToFerm<FImpl>(source, fullSrc, s, c);
+            }
+        }
+        sol = zero;
+	std::vector<Real> twist = strToVec<Real>(par().twist);
+	if(twist.size() != Nd) HADRONS_ERROR(Size, "number of twist angles does not match number of dimensions");
+	mat.FreePropagator(source,sol,mass,twist);
+        FermToProp<FImpl>(prop, sol, s, c);
+        // create 4D propagators from 5D one if necessary
+        if (Ls_ > 1)
+        {
+            PropagatorField &p4d = envGet(PropagatorField, getName());
+            mat.ExportPhysicalFermionSolution(sol, tmp);
+            FermToProp<FImpl>(p4d, tmp, s, c);
+        }
+    }
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MFermion_FreeProp_hpp_

extras/Hadrons/Modules/MFermion/GaugeProp.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MFermion/GaugeProp.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MFermion/GaugeProp.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MFermion;
+
+template class Grid::Hadrons::MFermion::TGaugeProp<FIMPL>;
+template class Grid::Hadrons::MFermion::TGaugeProp<ZFIMPL>;

extras/Hadrons/Modules/MFermion/GaugeProp.hpp

@@ -2,12 +2,14 @@
 
 Grid physics library, www.github.com/paboyle/Grid 
 
-Source file: extras/Hadrons/Modules/Quark.hpp
+Source file: extras/Hadrons/Modules/MFermion/GaugeProp.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+Author: Lanny91 <andrew.lawson@gmail.com>
+Author: pretidav <david.preti@csic.es>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -27,62 +29,68 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_Quark_hpp_
-#define Hadrons_Quark_hpp_
+#ifndef Hadrons_MFermion_GaugeProp_hpp_
+#define Hadrons_MFermion_GaugeProp_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
 #include <Grid/Hadrons/ModuleFactory.hpp>
+#include <Grid/Hadrons/Solver.hpp>
 
 BEGIN_HADRONS_NAMESPACE
 
 /******************************************************************************
- *                               TQuark                                       *
+ *                                GaugeProp                                   *
  ******************************************************************************/
-class QuarkPar: Serializable
+BEGIN_MODULE_NAMESPACE(MFermion)
+
+class GaugePropPar: Serializable
 {
 public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(QuarkPar,
+    GRID_SERIALIZABLE_CLASS_MEMBERS(GaugePropPar,
                                     std::string, source,
                                     std::string, solver);
 };
 
 template <typename FImpl>
-class TQuark: public Module<QuarkPar>
+class TGaugeProp: public Module<GaugePropPar>
 {
 public:
-    TYPE_ALIASES(FImpl,);
+    FG_TYPE_ALIASES(FImpl,);
+    SOLVER_TYPE_ALIASES(FImpl,);
 public:
     // constructor
-    TQuark(const std::string name);
+    TGaugeProp(const std::string name);
     // destructor
-    virtual ~TQuark(void) = default;
-    // dependencies/products
+    virtual ~TGaugeProp(void) {};
+    // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 private:
     unsigned int Ls_;
-    SolverFn     *solver_{nullptr};
+    Solver       *solver_{nullptr};
 };
 
-MODULE_REGISTER(Quark, TQuark<FIMPL>);
+MODULE_REGISTER_TMP(GaugeProp, TGaugeProp<FIMPL>, MFermion);
+MODULE_REGISTER_TMP(ZGaugeProp, TGaugeProp<ZFIMPL>, MFermion);
 
 /******************************************************************************
- *                          TQuark implementation                             *
+ *                      TGaugeProp implementation                             *
  ******************************************************************************/
 // constructor /////////////////////////////////////////////////////////////////
 template <typename FImpl>
-TQuark<FImpl>::TQuark(const std::string name)
-: Module(name)
+TGaugeProp<FImpl>::TGaugeProp(const std::string name)
+: Module<GaugePropPar>(name)
 {}
 
 // dependencies/products ///////////////////////////////////////////////////////
 template <typename FImpl>
-std::vector<std::string> TQuark<FImpl>::getInput(void)
+std::vector<std::string> TGaugeProp<FImpl>::getInput(void)
 {
     std::vector<std::string> in = {par().source, par().solver};
     
@@ -90,7 +98,7 @@ std::vector<std::string> TQuark<FImpl>::getInput(void)
 }
 
 template <typename FImpl>
-std::vector<std::string> TQuark<FImpl>::getOutput(void)
+std::vector<std::string> TGaugeProp<FImpl>::getOutput(void)
 {
     std::vector<std::string> out = {getName(), getName() + "_5d"};
     
@@ -99,56 +107,54 @@ std::vector<std::string> TQuark<FImpl>::getOutput(void)
 
 // setup ///////////////////////////////////////////////////////////////////////
 template <typename FImpl>
-void TQuark<FImpl>::setup(void)
+void TGaugeProp<FImpl>::setup(void)
 {
     Ls_ = env().getObjectLs(par().solver);
-    env().template registerLattice<PropagatorField>(getName());
+    envCreateLat(PropagatorField, getName());
+    envTmpLat(FermionField, "source", Ls_);
+    envTmpLat(FermionField, "sol", Ls_);
+    envTmpLat(FermionField, "tmp");
     if (Ls_ > 1)
     {
-        env().template registerLattice<PropagatorField>(getName() + "_5d", Ls_);
+        envCreateLat(PropagatorField, getName() + "_5d", Ls_);
     }
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
-void TQuark<FImpl>::execute(void)
+void TGaugeProp<FImpl>::execute(void)
 {
     LOG(Message) << "Computing quark propagator '" << getName() << "'"
                  << std::endl;
     
-    FermionField    source(env().getGrid(Ls_)), sol(env().getGrid(Ls_)),
-                    tmp(env().getGrid());
-    std::string     propName = (Ls_ == 1) ? getName() : (getName() + "_5d");
-    PropagatorField &prop    = *env().template createLattice<PropagatorField>(propName);
-    PropagatorField &fullSrc = *env().template getObject<PropagatorField>(par().source);
-    SolverFn        &solver  = *env().template getObject<SolverFn>(par().solver);
-    if (Ls_ > 1)
-    {
-        env().template createLattice<PropagatorField>(getName());
-    }
+    std::string propName = (Ls_ == 1) ? getName() : (getName() + "_5d");
+    auto        &prop    = envGet(PropagatorField, propName);
+    auto        &fullSrc = envGet(PropagatorField, par().source);
+    auto        &solver  = envGet(Solver, par().solver);
+    auto        &mat     = solver.getFMat();
     
+    envGetTmp(FermionField, source);
+    envGetTmp(FermionField, sol);
+    envGetTmp(FermionField, tmp);
     LOG(Message) << "Inverting using solver '" << par().solver
                  << "' on source '" << par().source << "'" << std::endl;
     for (unsigned int s = 0; s < Ns; ++s)
-    for (unsigned int c = 0; c < Nc; ++c)
+    for (unsigned int c = 0; c < FImpl::Dimension; ++c)
     {
         LOG(Message) << "Inversion for spin= " << s << ", color= " << c
-        << std::endl;
+                     << std::endl;
         // source conversion for 4D sources
+        LOG(Message) << "Import source" << std::endl;
         if (!env().isObject5d(par().source))
         {
             if (Ls_ == 1)
             {
-                PropToFerm(source, fullSrc, s, c);
+               PropToFerm<FImpl>(source, fullSrc, s, c);
             }
             else
             {
-                source = zero;
-                PropToFerm(tmp, fullSrc, s, c);
-                InsertSlice(tmp, source, 0, 0);
-                InsertSlice(tmp, source, Ls_-1, 0);
-                axpby_ssp_pplus(source, 0., source, 1., source, 0, 0);
-                axpby_ssp_pminus(source, 0., source, 1., source, Ls_-1, Ls_-1);
+                PropToFerm<FImpl>(tmp, fullSrc, s, c);
+                mat.ImportPhysicalFermionSource(tmp, source);
             }
         }
         // source conversion for 5D sources
@@ -156,30 +162,30 @@ void TQuark<FImpl>::execute(void)
         {
             if (Ls_ != env().getObjectLs(par().source))
             {
-                HADRON_ERROR("Ls mismatch between quark action and source");
+                HADRONS_ERROR(Size, "Ls mismatch between quark action and source");
             }
             else
             {
-                PropToFerm(source, fullSrc, s, c);
+                PropToFerm<FImpl>(source, fullSrc, s, c);
             }
         }
+        LOG(Message) << "Solve" << std::endl;
         sol = zero;
         solver(sol, source);
-        FermToProp(prop, sol, s, c);
+        LOG(Message) << "Export solution" << std::endl;
+        FermToProp<FImpl>(prop, sol, s, c);
         // create 4D propagators from 5D one if necessary
         if (Ls_ > 1)
         {
-            PropagatorField &p4d =
-                *env().template getObject<PropagatorField>(getName());
-            
-            axpby_ssp_pminus(sol, 0., sol, 1., sol, 0, 0);
-            axpby_ssp_pplus(sol, 1., sol, 1., sol, 0, Ls_-1);
-            ExtractSlice(tmp, sol, 0, 0);
-            FermToProp(p4d, tmp, s, c);
+            PropagatorField &p4d = envGet(PropagatorField, getName());
+            mat.ExportPhysicalFermionSolution(sol, tmp);
+            FermToProp<FImpl>(p4d, tmp, s, c);
         }
     }
 }
 
+END_MODULE_NAMESPACE
+
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_Quark_hpp_
+#endif // Hadrons_MFermion_GaugeProp_hpp_

extras/Hadrons/Modules/MGauge/FundtoHirep.cc

@@ -0,0 +1,79 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/FundtoHirep.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+Author: pretidav <david.preti@csic.es>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MGauge/FundtoHirep.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+// constructor /////////////////////////////////////////////////////////////////
+template <class Rep>
+TFundtoHirep<Rep>::TFundtoHirep(const std::string name)
+: Module<FundtoHirepPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <class Rep>
+std::vector<std::string> TFundtoHirep<Rep>::getInput(void)
+{
+    std::vector<std::string> in = {par().gaugeconf};
+
+    return in;
+}
+
+template <class Rep>
+std::vector<std::string> TFundtoHirep<Rep>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename Rep>
+void TFundtoHirep<Rep>::setup(void)
+{
+    envCreateLat(Rep::LatticeField, getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <class Rep>
+void TFundtoHirep<Rep>::execute(void)
+{
+    LOG(Message) << "Transforming Representation" << std::endl;
+
+    auto &U    = envGet(LatticeGaugeField, par().gaugeconf);
+    auto &URep = envGet(Rep::LatticeField, getName());
+
+    Rep TargetRepresentation(U._grid);
+    TargetRepresentation.update_representation(U);
+    URep = TargetRepresentation.U;
+}

extras/Hadrons/Modules/MGauge/FundtoHirep.hpp

@@ -0,0 +1,76 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/FundtoHirep.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: pretidav <david.preti@csic.es>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MGauge_FundtoHirep_hpp_
+#define Hadrons_MGauge_FundtoHirep_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         Load a NERSC configuration                         *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class FundtoHirepPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(FundtoHirepPar,
+                                    std::string, gaugeconf);
+};
+
+template <class Rep>
+class TFundtoHirep: public Module<FundtoHirepPar>
+{
+public:
+    // constructor
+    TFundtoHirep(const std::string name);
+    // destructor
+    virtual ~TFundtoHirep(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    void setup(void);
+    // execution
+    void execute(void);
+};
+
+//MODULE_REGISTER_TMP(FundtoAdjoint,   TFundtoHirep<AdjointRepresentation>, MGauge);
+//MODULE_REGISTER_TMP(FundtoTwoIndexSym, TFundtoHirep<TwoIndexSymmetricRepresentation>, MGauge);
+//MODULE_REGISTER_TMP(FundtoTwoIndexAsym, TFundtoHirep<TwoIndexAntiSymmetricRepresentation>, MGauge);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_FundtoHirep_hpp_

extras/Hadrons/Modules/MGauge/Random.cc

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Random.cc
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -44,7 +43,9 @@ TRandom::TRandom(const std::string name)
 // dependencies/products ///////////////////////////////////////////////////////
 std::vector<std::string> TRandom::getInput(void)
 {
-    return std::vector<std::string>();
+    std::vector<std::string> in;
+    
+    return in;
 }
 
 std::vector<std::string> TRandom::getOutput(void)
@@ -57,13 +58,14 @@ std::vector<std::string> TRandom::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TRandom::setup(void)
 {
-    env().registerLattice<LatticeGaugeField>(getName());
+    envCreateLat(LatticeGaugeField, getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 void TRandom::execute(void)
 {
     LOG(Message) << "Generating random gauge configuration" << std::endl;
-    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
+    
+    auto &U = envGet(LatticeGaugeField, getName());
     SU3::HotConfiguration(*env().get4dRng(), U);
 }

extras/Hadrons/Modules/MGauge/Random.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Random.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -27,8 +26,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_Random_hpp_
-#define Hadrons_Random_hpp_
+#ifndef Hadrons_MGauge_Random_hpp_
+#define Hadrons_MGauge_Random_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -47,20 +46,21 @@ public:
     // constructor
     TRandom(const std::string name);
     // destructor
-    virtual ~TRandom(void) = default;
+    virtual ~TRandom(void) {};
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(Random, TRandom, MGauge);
+MODULE_REGISTER(Random, TRandom, MGauge);
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_Random_hpp_
+#endif // Hadrons_MGauge_Random_hpp_

extras/Hadrons/Modules/MGauge/StochEm.cc

@@ -0,0 +1,85 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/StochEm.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: James Harrison <j.harrison@soton.ac.uk>
+Author: Vera Guelpers <vmg1n14@soton.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MGauge/StochEm.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+/******************************************************************************
+*                  TStochEm implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TStochEm::TStochEm(const std::string name)
+: Module<StochEmPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TStochEm::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+std::vector<std::string> TStochEm::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TStochEm::setup(void)
+{
+    weightDone_ = env().hasCreatedObject("_" + getName() + "_weight");
+    envCacheLat(EmComp, "_" + getName() + "_weight");
+    envCreateLat(EmField, getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TStochEm::execute(void)
+{
+    LOG(Message) << "Generating stochastic EM potential..." << std::endl;
+
+    std::vector<Real> improvements = strToVec<Real>(par().improvement);
+    PhotonR photon(par().gauge, par().zmScheme, improvements, par().G0_qedInf);
+    auto    &a = envGet(EmField, getName());
+    auto    &w = envGet(EmComp, "_" + getName() + "_weight");
+    
+    if (!weightDone_)
+    {
+        LOG(Message) << "Caching stochastic EM potential weight (gauge: "
+                     << par().gauge << ", zero-mode scheme: "
+                     << par().zmScheme << ")..." << std::endl;
+        photon.StochasticWeight(w);
+    }
+    photon.StochasticField(a, *env().get4dRng(), w);
+}

extras/Hadrons/Modules/MGauge/StochEm.hpp

@@ -0,0 +1,82 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/StochEm.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: James Harrison <j.harrison@soton.ac.uk>
+Author: Vera Guelpers <vmg1n14@soton.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef Hadrons_MGauge_StochEm_hpp_
+#define Hadrons_MGauge_StochEm_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         StochEm                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class StochEmPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(StochEmPar,
+                                    PhotonR::Gauge,    gauge,
+                                    PhotonR::ZmScheme, zmScheme,
+                                    std::string,       improvement,
+                                    Real,              G0_qedInf);
+};
+
+class TStochEm: public Module<StochEmPar>
+{
+public:
+    typedef PhotonR::GaugeField     EmField;
+    typedef PhotonR::GaugeLinkField EmComp;
+public:
+    // constructor
+    TStochEm(const std::string name);
+    // destructor
+    virtual ~TStochEm(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+protected:
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+private:
+    bool    weightDone_;
+};
+
+MODULE_REGISTER(StochEm, TStochEm, MGauge);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_StochEm_hpp_

extras/Hadrons/Modules/MGauge/StoutSmearing.cc

@@ -0,0 +1,7 @@
+#include <Grid/Hadrons/Modules/MGauge/StoutSmearing.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+template class Grid::Hadrons::MGauge::TStoutSmearing<GIMPL>;

extras/Hadrons/Modules/MGauge/StoutSmearing.hpp

@@ -0,0 +1,104 @@
+#ifndef Hadrons_MGauge_StoutSmearing_hpp_
+#define Hadrons_MGauge_StoutSmearing_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                            Stout smearing                                  *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class StoutSmearingPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(StoutSmearingPar,
+                                    std::string, gauge,
+                                    unsigned int, steps,
+                                    double, rho);
+};
+
+template <typename GImpl>
+class TStoutSmearing: public Module<StoutSmearingPar>
+{
+public:
+    typedef typename GImpl::Field GaugeField;
+public:
+    // constructor
+    TStoutSmearing(const std::string name);
+    // destructor
+    virtual ~TStoutSmearing(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_TMP(StoutSmearing, TStoutSmearing<GIMPL>, MGauge);
+
+/******************************************************************************
+ *                     TStoutSmearing implementation                          *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename GImpl>
+TStoutSmearing<GImpl>::TStoutSmearing(const std::string name)
+: Module<StoutSmearingPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename GImpl>
+std::vector<std::string> TStoutSmearing<GImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().gauge};
+    
+    return in;
+}
+
+template <typename GImpl>
+std::vector<std::string> TStoutSmearing<GImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename GImpl>
+void TStoutSmearing<GImpl>::setup(void)
+{
+    envCreateLat(GaugeField, getName());
+    envTmpLat(GaugeField, "buf");
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename GImpl>
+void TStoutSmearing<GImpl>::execute(void)
+{
+    LOG(Message) << "Smearing '" << par().gauge << "' with " << par().steps
+                 << " step" << ((par().steps > 1) ? "s" : "") 
+                 << " of stout smearing and rho= " << par().rho << std::endl;
+
+    Smear_Stout<GImpl> smearer(par().rho);
+    auto               &U    = envGet(GaugeField, par().gauge);
+    auto               &Usmr = envGet(GaugeField, getName());
+
+    envGetTmp(GaugeField, buf);
+    buf = U;
+    for (unsigned int n = 0; n < par().steps; ++n)
+    {
+        smearer.smear(Usmr, buf);
+        buf = Usmr;
+    }
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_StoutSmearing_hpp_

extras/Hadrons/Modules/MGauge/Unit.cc

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Unit.cc
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -57,13 +56,14 @@ std::vector<std::string> TUnit::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TUnit::setup(void)
 {
-    env().registerLattice<LatticeGaugeField>(getName());
+    envCreateLat(LatticeGaugeField, getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 void TUnit::execute(void)
 {
     LOG(Message) << "Creating unit gauge configuration" << std::endl;
-    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
+    
+    auto &U = envGet(LatticeGaugeField, getName());
     SU3::ColdConfiguration(*env().get4dRng(), U);
 }

extras/Hadrons/Modules/MGauge/Unit.hpp

@@ -4,8 +4,7 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Unit.hpp
 
-Copyright (C) 2015
-Copyright (C) 2016
+Copyright (C) 2015-2018
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -27,8 +26,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 *************************************************************************************/
 /*  END LEGAL */
 
-#ifndef Hadrons_Unit_hpp_
-#define Hadrons_Unit_hpp_
+#ifndef Hadrons_MGauge_Unit_hpp_
+#define Hadrons_MGauge_Unit_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -47,20 +46,21 @@ public:
     // constructor
     TUnit(const std::string name);
     // destructor
-    virtual ~TUnit(void) = default;
+    virtual ~TUnit(void) {};
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(Unit, TUnit, MGauge);
+MODULE_REGISTER(Unit, TUnit, MGauge);
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_Unit_hpp_
+#endif // Hadrons_MGauge_Unit_hpp_

extras/Hadrons/Modules/MGauge/UnitEm.cc

@@ -0,0 +1,69 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/StochEm.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: James Harrison <j.harrison@soton.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MGauge/UnitEm.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+/******************************************************************************
+*                  TStochEm implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TUnitEm::TUnitEm(const std::string name)
+: Module<NoPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TUnitEm::getInput(void)
+{
+    return std::vector<std::string>();
+}
+
+std::vector<std::string> TUnitEm::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TUnitEm::setup(void)
+{
+    envCreateLat(EmField, getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TUnitEm::execute(void)
+{
+    PhotonR photon(0, 0); // Just chose arbitrary input values here
+    auto    &a = envGet(EmField, getName());
+    LOG(Message) << "Generating unit EM potential..." << std::endl;
+    photon.UnitField(a);
+}

extras/Hadrons/Modules/MGauge/UnitEm.hpp

@@ -0,0 +1,69 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/StochEm.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: James Harrison <j.harrison@soton.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef Hadrons_MGauge_UnitEm_hpp_
+#define Hadrons_MGauge_UnitEm_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         StochEm                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class TUnitEm: public Module<NoPar>
+{
+public:
+    typedef PhotonR::GaugeField     EmField;
+    typedef PhotonR::GaugeLinkField EmComp;
+public:
+    // constructor
+    TUnitEm(const std::string name);
+    // destructor
+    virtual ~TUnitEm(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+protected:
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER(UnitEm, TUnitEm, MGauge);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_UnitEm_hpp_

extras/Hadrons/Modules/MIO/LoadBinary.cc

@@ -0,0 +1,40 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MIO/LoadBinary.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MIO/LoadBinary.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MIO;
+
+template class Grid::Hadrons::MIO::TLoadBinary<GIMPL>;
+template class Grid::Hadrons::MIO::TLoadBinary<ScalarNxNAdjImplR<2>>;
+template class Grid::Hadrons::MIO::TLoadBinary<ScalarNxNAdjImplR<3>>;
+template class Grid::Hadrons::MIO::TLoadBinary<ScalarNxNAdjImplR<4>>;
+template class Grid::Hadrons::MIO::TLoadBinary<ScalarNxNAdjImplR<5>>;
+template class Grid::Hadrons::MIO::TLoadBinary<ScalarNxNAdjImplR<6>>;
+

extras/Hadrons/Modules/MIO/LoadBinary.hpp

@@ -0,0 +1,140 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MIO/LoadBinary.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef Hadrons_MIO_LoadBinary_hpp_
+#define Hadrons_MIO_LoadBinary_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                       Load a binary configurations                         *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MIO)
+
+class LoadBinaryPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(LoadBinaryPar,
+                                    std::string, file,
+                                    std::string, format);
+};
+
+template <typename Impl>
+class TLoadBinary: public Module<LoadBinaryPar>
+{
+public:
+    typedef typename Impl::Field                  Field;
+    typedef typename Impl::Simd                   Simd;
+    typedef typename Field::vector_object         vobj;
+    typedef typename vobj::scalar_object          sobj;
+    typedef typename sobj::DoublePrecision        sobj_double;
+    typedef BinarySimpleMunger<sobj_double, sobj> Munger;
+public:
+    // constructor
+    TLoadBinary(const std::string name);
+    // destructor
+    virtual ~TLoadBinary(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_TMP(LoadBinary, TLoadBinary<GIMPL>, MIO);
+MODULE_REGISTER_TMP(LoadBinaryScalarSU2, TLoadBinary<ScalarNxNAdjImplR<2>>, MIO);
+MODULE_REGISTER_TMP(LoadBinaryScalarSU3, TLoadBinary<ScalarNxNAdjImplR<3>>, MIO);
+MODULE_REGISTER_TMP(LoadBinaryScalarSU4, TLoadBinary<ScalarNxNAdjImplR<4>>, MIO);
+MODULE_REGISTER_TMP(LoadBinaryScalarSU5, TLoadBinary<ScalarNxNAdjImplR<5>>, MIO);
+MODULE_REGISTER_TMP(LoadBinaryScalarSU6, TLoadBinary<ScalarNxNAdjImplR<6>>, MIO);
+
+/******************************************************************************
+ *                         TLoadBinary implementation                         *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename Impl>
+TLoadBinary<Impl>::TLoadBinary(const std::string name)
+: Module<LoadBinaryPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename Impl>
+std::vector<std::string> TLoadBinary<Impl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename Impl>
+std::vector<std::string> TLoadBinary<Impl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename Impl>
+void TLoadBinary<Impl>::setup(void)
+{
+    envCreateLat(Field, getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename Impl>
+void TLoadBinary<Impl>::execute(void)
+{
+    Munger      munge;
+    uint32_t    nersc_csum, scidac_csuma, scidac_csumb;
+    auto        &U = envGet(Field, getName());
+    std::string filename = par().file + "."
+                           + std::to_string(vm().getTrajectory());
+
+    LOG(Message) << "Loading " << par().format 
+                 << " binary configuration from file '" << filename
+                 << "'" << std::endl;
+    BinaryIO::readLatticeObject<vobj, sobj_double>(U, filename, munge, 0, 
+                                                   par().format, nersc_csum,
+                                                   scidac_csuma, scidac_csumb);
+    LOG(Message) << "Checksums:" << std::endl;
+    LOG(Message) << "  NERSC    " << nersc_csum << std::endl;
+    LOG(Message) << "  SciDAC A " << scidac_csuma << std::endl;
+    LOG(Message) << "  SciDAC B " << scidac_csumb << std::endl;
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MIO_LoadBinary_hpp_

extras/Hadrons/Modules/MIO/LoadCoarseEigenPack.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MIO/LoadCoarseEigenPack.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MIO/LoadCoarseEigenPack.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MIO;
+
+template class Grid::Hadrons::MIO::TLoadCoarseEigenPack<CoarseFermionEigenPack<FIMPL,HADRONS_DEFAULT_LANCZOS_NBASIS>>;
+

extras/Hadrons/Modules/MIO/LoadCoarseEigenPack.hpp

@@ -0,0 +1,135 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MIO/LoadCoarseEigenPack.hpp
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef Hadrons_MIO_LoadCoarseEigenPack_hpp_
+#define Hadrons_MIO_LoadCoarseEigenPack_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+#include <Grid/Hadrons/EigenPack.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *              Load local coherence eigen vectors/values package             *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MIO)
+
+class LoadCoarseEigenPackPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(LoadCoarseEigenPackPar,
+                                    std::string, filestem,
+                                    bool,         multiFile,
+                                    unsigned int, sizeFine,
+                                    unsigned int, sizeCoarse,
+                                    unsigned int, Ls,
+                                    std::vector<int>, blockSize);
+};
+
+template <typename Pack>
+class TLoadCoarseEigenPack: public Module<LoadCoarseEigenPackPar>
+{
+public:
+    typedef CoarseEigenPack<typename Pack::Field, typename Pack::CoarseField> BasePack;
+    template <typename vtype> 
+    using iImplScalar = iScalar<iScalar<iScalar<vtype>>>;
+    typedef iImplScalar<typename Pack::Field::vector_type> SiteComplex;
+public:
+    // constructor
+    TLoadCoarseEigenPack(const std::string name);
+    // destructor
+    virtual ~TLoadCoarseEigenPack(void) {};
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_TMP(LoadCoarseFermionEigenPack, ARG(TLoadCoarseEigenPack<CoarseFermionEigenPack<FIMPL, HADRONS_DEFAULT_LANCZOS_NBASIS>>), MIO);
+
+/******************************************************************************
+ *                 TLoadCoarseEigenPack implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename Pack>
+TLoadCoarseEigenPack<Pack>::TLoadCoarseEigenPack(const std::string name)
+: Module<LoadCoarseEigenPackPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename Pack>
+std::vector<std::string> TLoadCoarseEigenPack<Pack>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename Pack>
+std::vector<std::string> TLoadCoarseEigenPack<Pack>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename Pack>
+void TLoadCoarseEigenPack<Pack>::setup(void)
+{
+    env().createGrid(par().Ls);
+    env().createCoarseGrid(par().blockSize, par().Ls);
+    envCreateDerived(BasePack, Pack, getName(), par().Ls, par().sizeFine,
+                     par().sizeCoarse, env().getRbGrid(par().Ls), 
+                     env().getCoarseGrid(par().blockSize, par().Ls));
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename Pack>
+void TLoadCoarseEigenPack<Pack>::execute(void)
+{
+    auto                 cg     = env().getCoarseGrid(par().blockSize, par().Ls);
+    auto                 &epack = envGetDerived(BasePack, Pack, getName());
+    Lattice<SiteComplex> dummy(cg);
+
+    epack.read(par().filestem, par().multiFile, vm().getTrajectory());
+    LOG(Message) << "Block Gramm-Schmidt pass 1"<< std::endl;
+    blockOrthogonalise(dummy, epack.evec);
+    LOG(Message) << "Block Gramm-Schmidt pass 2"<< std::endl;
+    blockOrthogonalise(dummy, epack.evec);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MIO_LoadCoarseEigenPack_hpp_

extras/Hadrons/Modules/MIO/LoadEigenPack.cc

@@ -0,0 +1,35 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MIO/LoadEigenPack.cc
+
+Copyright (C) 2015-2018
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MIO/LoadEigenPack.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MIO;
+
+template class Grid::Hadrons::MIO::TLoadEigenPack<FermionEigenPack<FIMPL>>;
+