Merge branch 'develop' of https://github.com/paboyle/Grid into feature/staggering

.gitignore

@@ -9,6 +9,7 @@
 ################
 *~
 *#
+*.sublime-*
 
 # Precompiled Headers #
 #######################
@@ -91,6 +92,7 @@ build*/*
 #####################
 *.xcodeproj/*
 build.sh
+.vscode
 
 # Eigen source #
 ################
@@ -104,3 +106,20 @@ lib/fftw/*
 ##################
 m4/lt*
 m4/libtool.m4
+
+# github pages #
+################
+gh-pages/
+
+# Buck files #
+##############
+.buck*
+buck-out
+BUCK
+make-bin-BUCK.sh
+
+# generated sources #
+#####################
+lib/qcd/spin/gamma-gen/*.h
+lib/qcd/spin/gamma-gen/*.cc
+

.travis.yml

@@ -7,64 +7,8 @@ cache:
 matrix:
   include:
     - os:        osx
-      osx_image: xcode7.2
+      osx_image: xcode8.3
       compiler: clang
-    - compiler: gcc
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.9
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: VERSION=-4.9
-    - compiler: gcc
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-5
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: VERSION=-5
-    - compiler: clang
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.8
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: CLANG_LINK=http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-    - compiler: clang
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.8
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: CLANG_LINK=http://llvm.org/releases/3.7.0/clang+llvm-3.7.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
       
 before_install:
     - export GRIDDIR=`pwd`
@@ -73,13 +17,15 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openmpi; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]] && [[ "$CC" == "gcc" ]]; then brew install gcc5; fi
     
 install:
     - export CC=$CC$VERSION
     - export CXX=$CXX$VERSION
     - echo $PATH
+    - which autoconf
+    - autoconf  --version
+    - which automake
+    - automake  --version
     - which $CC
     - $CC  --version
     - which $CXX
@@ -92,15 +38,9 @@ script:
     - cd build
     - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=none
     - make -j4 
-    - ./benchmarks/Benchmark_dwf --threads 1
+    - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - echo make clean
     - ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none
     - make -j4
-    - ./benchmarks/Benchmark_dwf --threads 1
+    - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
-    - echo make clean
+    - make check
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXXFLAGS='-DMPI_UINT32_T=MPI_UNSIGNED -DMPI_UINT64_T=MPI_UNSIGNED_LONG'; fi
-    - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto
-    - make -j4
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then mpirun -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
-

Makefile.am

@@ -1,12 +1,17 @@
 # additional include paths necessary to compile the C++ library
-SUBDIRS = lib benchmarks tests
+SUBDIRS = lib benchmarks tests extras
 
 include $(top_srcdir)/doxygen.inc
 
-tests: all
+bin_SCRIPTS=grid-config
-	$(MAKE) -C tests tests
 
-.PHONY: tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
+
+.PHONY: bench check tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
+
+tests-local: all
+bench-local: all
+check-local: all
 
 AM_CXXFLAGS += -I$(top_builddir)/include
+
 ACLOCAL_AMFLAGS = -I m4

README.md

@@ -1,41 +1,13 @@
-# Grid
+# Grid [![Teamcity status](http://ci.cliath.ph.ed.ac.uk/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:Grid)),branch:name:develop)/statusIcon.svg)](http://ci.cliath.ph.ed.ac.uk/project.html?projectId=Grid&tab=projectOverview) [![Travis status](https://travis-ci.org/paboyle/Grid.svg?branch=develop)](https://travis-ci.org/paboyle/Grid)
-<table>
-<tr>
-    <td>Last stable release</td>
-    <td><a href="https://travis-ci.org/paboyle/Grid">
-    <img src="https://travis-ci.org/paboyle/Grid.svg?branch=master"></a>
-    </td>
-</tr>
-<tr>
-    <td>Development branch</td>
-    <td><a href="https://travis-ci.org/paboyle/Grid">
-    <img src="https://travis-ci.org/paboyle/Grid.svg?branch=develop"></a>
-    </td>
-</tr>
-</table>
 
 **Data parallel C++ mathematical object library.**
 
 License: GPL v2.
 
-Last update Nov 2016.
+Last update June 2017.
 
 _Please do not send pull requests to the `master` branch which is reserved for releases._
 
-### Bug report
-
-_To help us tracking and solving more efficiently issues with Grid, please report problems using the issue system of GitHub rather than sending emails to Grid developers._
-
-When you file an issue, please go though the following checklist:
-
-1. Check that the code is pointing to the `HEAD` of `develop` or any commit in `master` which is tagged with a version number. 
-2. Give a description of the target platform (CPU, network, compiler). Please give the full CPU part description, using for example `cat /proc/cpuinfo | grep 'model name' | uniq` (Linux) or `sysctl machdep.cpu.brand_string` (macOS) and the full output the `--version` option of your compiler.
-3. Give the exact `configure` command used.
-4. Attach `config.log`.
-5. Attach `config.summary`.
-6. Attach the output of `make V=1`.
-7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example.
-
 
 
 ### Description
@@ -58,13 +30,68 @@ optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a signifi
 for most programmers.
 
 The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture.
-Presently SSE4 (128 bit) AVX, AVX2, QPX (256 bit), IMCI, and AVX512 (512 bit) targets are supported (ARM NEON on the way).
+Presently SSE4, ARM NEON (128 bits) AVX, AVX2, QPX (256 bits), IMCI and AVX512 (512 bits) targets are supported.
 
-These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. These may be useful in themselves for other programmers.
+These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. 
 The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`.
 
 MPI, OpenMP, and SIMD parallelism are present in the library.
-Please see https://arxiv.org/abs/1512.03487 for more detail.
+Please see [this paper](https://arxiv.org/abs/1512.03487) for more detail.
+
+
+### Compilers
+
+Intel ICPC v16.0.3 and later
+
+Clang v3.5 and later (need 3.8 and later for OpenMP)
+
+GCC   v4.9.x (recommended)
+
+GCC   v6.3 and later
+
+### Important: 
+
+Some versions of GCC appear to have a bug under high optimisation (-O2, -O3).
+
+The safety of these compiler versions cannot be guaranteed at this time. Follow Issue 100 for details and updates.
+
+GCC   v5.x
+
+GCC   v6.1, v6.2
+
+### Bug report
+
+_To help us tracking and solving more efficiently issues with Grid, please report problems using the issue system of GitHub rather than sending emails to Grid developers._
+
+When you file an issue, please go though the following checklist:
+
+1. Check that the code is pointing to the `HEAD` of `develop` or any commit in `master` which is tagged with a version number. 
+2. Give a description of the target platform (CPU, network, compiler). Please give the full CPU part description, using for example `cat /proc/cpuinfo | grep 'model name' | uniq` (Linux) or `sysctl machdep.cpu.brand_string` (macOS) and the full output the `--version` option of your compiler.
+3. Give the exact `configure` command used.
+4. Attach `config.log`.
+5. Attach `grid.config.summary`.
+6. Attach the output of `make V=1`.
+7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example.
+
+### Required libraries
+Grid requires:
+
+[GMP](https://gmplib.org/), 
+
+[MPFR](http://www.mpfr.org/) 
+
+Bootstrapping grid downloads and uses for internal dense matrix (non-QCD operations) the Eigen library.
+
+Grid optionally uses:
+
+[HDF5](https://support.hdfgroup.org/HDF5/)  
+
+[LIME](http://usqcd-software.github.io/c-lime/) for ILDG and SciDAC file format support. 
+
+[FFTW](http://www.fftw.org) either generic version or via the Intel MKL library.
+
+LAPACK either generic version or Intel MKL library.
+
 
 ### Quick start
 First, start by cloning the repository:
@@ -95,10 +122,10 @@ install Grid. Other options are detailed in the next section, you can also use `
 `CXX`, `CXXFLAGS`, `LDFLAGS`, ... environment variables can be modified to
 customise the build.
 
-Finally, you can build and install Grid:
+Finally, you can build, check, and install Grid:
 
 ``` bash
-make; make install
+make; make check; make install
 ```
 
 To minimise the build time, only the tests at the root of the `tests` directory are built by default. If you want to build tests in the sub-directory `<subdir>` you can execute:
@@ -121,7 +148,7 @@ If you want to build all the tests at once just use `make tests`.
 - `--enable-gen-simd-width=<size>`: select the size (in bytes) of the generic SIMD vector type (default: 32 bytes).
 - `--enable-precision={single|double}`: set the default precision (default: `double`).
 - `--enable-precision=<comm>`: Use `<comm>` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below.
-- `--enable-rng={ranlux48|mt19937}`: choose the RNG (default: `ranlux48 `).
+- `--enable-rng={sitmo|ranlux48|mt19937}`: choose the RNG (default: `sitmo `).
 - `--disable-timers`: disable system dependent high-resolution timers.
 - `--enable-chroma`: enable Chroma regression tests.
 - `--enable-doxygen-doc`: enable the Doxygen documentation generation (build with `make doxygen-doc`)
@@ -135,7 +162,6 @@ The following options can be use with the `--enable-comms=` option to target dif
 | `none`         | no communications                                             |
 | `mpi[-auto]`   | MPI communications                                            |
 | `mpi3[-auto]`  | MPI communications using MPI 3 shared memory                  |
-| `mpi3l[-auto]` | MPI communications using MPI 3 shared memory and leader model |
 | `shmem `       | Cray SHMEM communications                                     |
 
 For the MPI interfaces the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names). The `-auto` suffix is not supported by the Cray environment wrapper scripts. Use the standard versions instead.  
@@ -153,13 +179,13 @@ The following options can be use with the `--enable-simd=` option to target diff
 | `AVXFMA4`   | AVX (256 bit) + FMA4                   |
 | `AVX2`      | AVX 2 (256 bit)                        |
 | `AVX512`    | AVX 512 bit                            |
-| `QPX`       | QPX (256 bit)                          |
+| `NEONv8`    | [ARM NEON](http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch07s03.html) (128 bit)                     |
+| `QPX`       | IBM QPX (256 bit)                      |
 
 Alternatively, some CPU codenames can be directly used:
 
 | `<code>`    | Description                            |
 | ----------- | -------------------------------------- |
-| `KNC`       | [Intel Xeon Phi codename Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) |
 | `KNL`       | [Intel Xeon Phi codename Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |
 | `BGQ`       | Blue Gene/Q                            |
 
@@ -176,21 +202,205 @@ The following configuration is recommended for the Intel Knights Landing platfor
 ``` bash
 ../configure --enable-precision=double\
              --enable-simd=KNL        \
-             --enable-comms=mpi-auto \
+             --enable-comms=mpi-auto  \
-             --with-gmp=<path>        \
-             --with-mpfr=<path>       \
              --enable-mkl             \
              CXX=icpc MPICXX=mpiicpc
 ```
+The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library.
 
-where `<path>` is the UNIX prefix where GMP and MPFR are installed. If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
+If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
 
 ``` bash
 ../configure --enable-precision=double\
              --enable-simd=KNL        \
              --enable-comms=mpi       \
-             --with-gmp=<path>        \
-             --with-mpfr=<path>       \
              --enable-mkl             \
              CXX=CC CC=cc
 ```
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+Knight's Landing with Intel Omnipath adapters with two adapters per node 
+presently performs better with use of more than one rank per node, using shared memory 
+for interior communication. This is the mpi3 communications implementation. 
+We recommend four ranks per node for best performance, but optimum is local volume dependent.
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=KNL        \
+             --enable-comms=mpi3-auto \
+             --enable-mkl             \
+             CC=icpc MPICXX=mpiicpc 
+```
+
+### Build setup for Intel Haswell Xeon platform
+
+The following configuration is recommended for the Intel Haswell platform:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX2       \
+             --enable-comms=mpi3-auto \
+             --enable-mkl             \
+             CXX=icpc MPICXX=mpiicpc
+```
+The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library.
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX2       \
+             --enable-comms=mpi3      \
+             --enable-mkl             \
+             CXX=CC CC=cc
+```
+Since Dual socket nodes are commonplace, we recommend MPI-3 as the default with the use of 
+one rank per socket. If using the Intel MPI library, threads should be pinned to NUMA domains using
+```
+        export I_MPI_PIN=1
+```
+This is the default.
+
+### Build setup for Intel Skylake Xeon platform
+
+The following configuration is recommended for the Intel Skylake platform:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX512     \
+             --enable-comms=mpi3      \
+             --enable-mkl             \
+             CXX=mpiicpc
+```
+The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library.
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use:
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX512     \
+             --enable-comms=mpi3      \
+             --enable-mkl             \
+             CXX=CC CC=cc
+```
+Since Dual socket nodes are commonplace, we recommend MPI-3 as the default with the use of 
+one rank per socket. If using the Intel MPI library, threads should be pinned to NUMA domains using
+``` 
+        export I_MPI_PIN=1
+```
+This is the default. 
+
+#### Expected Skylake Gold 6148 dual socket (single prec, single node 20+20 cores) performance using NUMA MPI mapping): 
+
+mpirun -n 2 benchmarks/Benchmark_dwf --grid 16.16.16.16 --mpi 2.1.1.1 --cacheblocking 2.2.2.2 --dslash-asm --shm 1024 --threads 18 
+
+TBA
+
+
+### Build setup for AMD EPYC / RYZEN
+
+The AMD EPYC is a multichip module comprising 32 cores spread over four distinct chips each with 8 cores.
+So, even with a single socket node there is a quad-chip module. Dual socket nodes with 64 cores total
+are common. Each chip within the module exposes a separate NUMA domain.
+There are four NUMA domains per socket and we recommend one MPI rank per NUMA domain.
+MPI-3 is recommended with the use of four ranks per socket,
+and 8 threads per rank. 
+
+The following configuration is recommended for the AMD EPYC platform.
+
+``` bash
+../configure --enable-precision=double\
+             --enable-simd=AVX2       \
+             --enable-comms=mpi3 \
+             CXX=mpicxx 
+```
+
+If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed:
+``` bash
+               --with-gmp=<path>        \
+               --with-mpfr=<path>       \
+```
+where `<path>` is the UNIX prefix where GMP and MPFR are installed. 
+
+Using MPICH and g++ v4.9.2, best performance can be obtained using explicit GOMP_CPU_AFFINITY flags for each MPI rank.
+This can be done by invoking MPI on a wrapper script omp_bind.sh to handle this. 
+
+It is recommended to run 8 MPI ranks on a single dual socket AMD EPYC, with 8 threads per rank using MPI3 and
+shared memory to communicate within this node:
+
+mpirun -np 8 ./omp_bind.sh ./Benchmark_dwf --mpi 2.2.2.1 --dslash-unroll --threads 8 --grid 16.16.16.16 --cacheblocking 4.4.4.4 
+
+Where omp_bind.sh does the following:
+```
+#!/bin/bash
+
+numanode=` expr $PMI_RANK % 8 `
+basecore=`expr $numanode \* 16`
+core0=`expr $basecore + 0 `
+core1=`expr $basecore + 2 `
+core2=`expr $basecore + 4 `
+core3=`expr $basecore + 6 `
+core4=`expr $basecore + 8 `
+core5=`expr $basecore + 10 `
+core6=`expr $basecore + 12 `
+core7=`expr $basecore + 14 `
+
+export GOMP_CPU_AFFINITY="$core0 $core1 $core2 $core3 $core4 $core5 $core6 $core7"
+echo GOMP_CUP_AFFINITY $GOMP_CPU_AFFINITY
+
+$@
+```
+
+Performance:
+
+#### Expected AMD EPYC 7601 dual socket (single prec, single node 32+32 cores) performance using NUMA MPI mapping): 
+
+mpirun  -np 8 ./omp_bind.sh ./Benchmark_dwf --threads 8 --mpi 2.2.2.1 --dslash-unroll --grid 16.16.16.16 --cacheblocking 4.4.4.4
+
+TBA
+
+### Build setup for BlueGene/Q
+
+To be written...
+
+### Build setup for ARM Neon
+
+To be written...
+
+### Build setup for laptops, other compilers, non-cluster builds
+
+Many versions of g++ and clang++ work with Grid, and involve merely replacing CXX (and MPICXX),
+and omit the enable-mkl flag. 
+
+Single node builds are enabled with 
+```
+            --enable-comms=none
+```
+
+FFTW support that is not in the default search path may then enabled with
+```
+    --with-fftw=<installpath>
+```
+
+BLAS will not be compiled in by default, and Lanczos will default to Eigen diagonalisation.
+

TODO

@@ -1,6 +1,35 @@
 TODO:
 ---------------
 
+Large item work list:
+
+1)- BG/Q port and check
+2)- Christoph's local basis expansion Lanczos
+3)- Precision conversion and sort out localConvert      <-- partial
+
+  - Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
+4)- Physical propagator interface
+5)- Conserved currents
+6)- Multigrid Wilson and DWF, compare to other Multigrid implementations
+7)- HDCR resume
+
+Recent DONE 
+
+-- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O.  <--- DONE
+-- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE
+-- GaugeFix into central location                      <-- DONE
+-- Scidac and Ildg metadata handling                   <-- DONE
+-- Binary I/O MPI2 IO                                  <-- DONE
+-- Binary I/O speed up & x-strips                      <-- DONE
+-- Cut down the exterior overhead                      <-- DONE
+-- Interior legs from SHM comms                        <-- DONE
+-- Half-precision comms                                <-- DONE
+-- Merge high precision reduction into develop         <-- DONE
+-- BlockCG, BCGrQ                                      <-- DONE
+-- multiRHS DWF; benchmark on Cori/BNL for comms elimination <-- DONE
+   -- slice* linalg routines for multiRHS, BlockCG    
+
+-----
 * Forces; the UdSdU  term in gauge force term is half of what I think it should
   be. This is a consequence of taking ONLY the first term in:
 
@@ -21,16 +50,8 @@ TODO:
   This means we must double the force in the Test_xxx_force routines, and is the origin of the factor of two.
   This 2x is applied by hand in the fermion routines and in the Test_rect_force routine.
 
-
-Policies:
-
-* Link smearing/boundary conds; Policy class based implementation ; framework more in place
-
 * Support different boundary conditions (finite temp, chem. potential ... )
 
-* Support different fermion representations? 
-  - contained entirely within the integrator presently
-
 - Sign of force term.
 
 - Reversibility test.
@@ -41,11 +62,6 @@ Policies:
 
 - Audit oIndex usage for cb behaviour
 
-- Rectangle gauge actions.
-  Iwasaki,
-  Symanzik,
-  ... etc...
-
 - Prepare multigrid for HMC. - Alternate setup schemes.
 
 - Support for ILDG --- ugly, not done
@@ -55,9 +71,11 @@ Policies:
 - FFTnD ?
 
 - Gparity; hand opt use template specialisation elegance to enable the optimised paths ?
+
 - Gparity force term; Gparity (R)HMC.
-- Random number state save restore
+
 - Mobius implementation clean up to rmove #if 0 stale code sequences
+
 - CG -- profile carefully, kernel fusion, whole CG performance measurements.
 
 ================================================================
@@ -90,6 +108,7 @@ Insert/Extract
 Not sure of status of this -- reverify. Things are working nicely now though.
 
 * Make the Tensor types and Complex etc... play more nicely.
+
   - TensorRemove is a hack, come up with a long term rationalised approach to Complex vs. Scalar<Scalar<Scalar<Complex > > >
     QDP forces use of "toDouble" to get back to non tensor scalar. This role is presently taken TensorRemove, but I
     want to introduce a syntax that does not require this.
@@ -112,6 +131,8 @@ Not sure of status of this -- reverify. Things are working nicely now though.
 RECENT
 ---------------
 
+  - Support different fermion representations? -- DONE
+  - contained entirely within the integrator presently
   - Clean up HMC                                                             -- DONE
   - LorentzScalar<GaugeField> gets Gauge link type (cleaner).                -- DONE
   - Simplified the integrators a bit.                                        -- DONE
@@ -123,6 +144,26 @@ RECENT
   - Parallel io improvements                                  -- DONE
   - Plaquette and link trace checks into nersc reader from the Grid_nersc_io.cc test. -- DONE
 
+
+DONE:
+- MultiArray -- MultiRHS done
+- ConjugateGradientMultiShift -- DONE
+- MCR                         -- DONE
+- Remez -- Mike or Boost?     -- DONE
+- Proto (ET)                  -- DONE
+- uBlas                       -- DONE ; Eigen
+- Potentially Useful Boost libraries -- DONE ; Eigen
+- Aligned allocator; memory pool -- DONE
+- Multiprecision              -- DONE
+- Serialization               -- DONE
+- Regex -- Not needed
+- Tokenize -- Why?
+
+- Random number state save restore -- DONE
+- Rectangle gauge actions. -- DONE
+  Iwasaki,
+  Symanzik,
+  ... etc...
 Done: Cayley, Partial , ContFrac force terms.
 
 DONE
@@ -207,6 +248,7 @@ Done
 FUNCTIONALITY: it pleases me to keep track of things I have done (keeps me arguably sane)
 ======================================================================================================
 
+* Link smearing/boundary conds; Policy class based implementation ; framework more in place -- DONE
 * Command line args for geometry, simd, etc. layout. Is it necessary to have -- DONE
   user pass these? Is this a QCD specific?
 

VERSION

@@ -1,6 +1,5 @@
-Version : 0.6.0
+Version : 0.7.0
 
-- AVX512, AVX2, AVX, SSE good
+- Clang 3.5 and above, ICPC v16 and above, GCC 6.3 and above recommended
-- Clang 3.5 and above, ICPC v16 and above, GCC 4.9 and above
+- MPI and MPI3 comms optimisations for KNL and OPA finished
-- MPI and MPI3
+- Half precision comms
-- HiRep, Smearing, Generic gauge group

benchmarks/Benchmark_ITT.cc

@@ -0,0 +1,800 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./benchmarks/Benchmark_memory_bandwidth.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
+typedef WilsonFermion5D<DomainWallVec5dImplF> WilsonFermion5DF;
+typedef WilsonFermion5D<DomainWallVec5dImplD> WilsonFermion5DD;
+
+
+std::vector<int> L_list;
+std::vector<int> Ls_list;
+std::vector<double> mflop_list;
+
+double mflop_ref;
+double mflop_ref_err;
+
+int NN_global;
+
+struct time_statistics{
+  double mean;
+  double err;
+  double min;
+  double max;
+
+  void statistics(std::vector<double> v){
+      double sum = std::accumulate(v.begin(), v.end(), 0.0);
+      mean = sum / v.size();
+
+      std::vector<double> diff(v.size());
+      std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; });
+      double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+      err = std::sqrt(sq_sum / (v.size()*(v.size() - 1)));
+
+      auto result = std::minmax_element(v.begin(), v.end());
+      min = *result.first;
+      max = *result.second;
+}
+};
+
+void comms_header(){
+  std::cout <<GridLogMessage << " L  "<<"\t"<<" Ls  "<<"\t"
+            <<std::setw(11)<<"bytes"<<"MB/s uni (err/min/max)"<<"\t\t"<<"MB/s bidi (err/min/max)"<<std::endl;
+};
+
+Gamma::Algebra Gmu [] = {
+  Gamma::Algebra::GammaX,
+  Gamma::Algebra::GammaY,
+  Gamma::Algebra::GammaZ,
+  Gamma::Algebra::GammaT
+};
+struct controls {
+  int Opt;
+  int CommsOverlap;
+  Grid::CartesianCommunicator::CommunicatorPolicy_t CommsAsynch;
+  //  int HugePages;
+};
+
+class Benchmark {
+public:
+  static void Decomposition (void ) {
+
+    int threads = GridThread::GetThreads();
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Grid is setup to use "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage<<"Grid Default Decomposition patterns\n";
+    std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
+    std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
+    std::cout<<GridLogMessage<<"\tvReal          : "<<sizeof(vReal )*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vReal::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealD         : "<<sizeof(vRealD)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplex       : "<<sizeof(vComplex )*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplex::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexF      : "<<sizeof(vComplexF)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexD      : "<<sizeof(vComplexD)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  }
+
+  static void Comms(void)
+  {
+    int Nloop=200;
+    int nmu=0;
+    int maxlat=32;
+
+    std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplexD::Nsimd());
+    std::vector<int> mpi_layout  = GridDefaultMpi();
+
+    for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
+
+    std::vector<double> t_time(Nloop);
+    time_statistics timestat;
+
+    std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Benchmarking threaded STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+    std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+    comms_header();
+
+    for(int lat=4;lat<=maxlat;lat+=4){
+      for(int Ls=8;Ls<=8;Ls*=2){
+
+	std::vector<int> latt_size  ({lat*mpi_layout[0],
+	      lat*mpi_layout[1],
+	      lat*mpi_layout[2],
+	      lat*mpi_layout[3]});
+
+	GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+	RealD Nrank = Grid._Nprocessors;
+	RealD Nnode = Grid.NodeCount();
+	RealD ppn = Nrank/Nnode;
+
+	std::vector<HalfSpinColourVectorD *> xbuf(8);
+	std::vector<HalfSpinColourVectorD *> rbuf(8);
+	Grid.ShmBufferFreeAll();
+	for(int d=0;d<8;d++){
+	  xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	}
+
+	int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+	int ncomm;
+	double dbytes;
+	std::vector<double> times(Nloop);
+	for(int i=0;i<Nloop;i++){
+
+	  double start=usecond();
+
+	  dbytes=0;
+	  ncomm=0;
+
+	  parallel_for(int dir=0;dir<8;dir++){
+
+	    double tbytes;
+	    int mu =dir % 4;
+
+	    if (mpi_layout[mu]>1 ) {
+	        
+	      int xmit_to_rank;
+	      int recv_from_rank;
+	      if ( dir == mu ) { 
+		int comm_proc=1;
+		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	      } else { 
+		int comm_proc = mpi_layout[mu]-1;
+		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	      }
+	      tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
+						 (void *)&rbuf[dir][0], recv_from_rank,
+						 bytes,dir);
+	  
+#ifdef GRID_OMP
+#pragma omp atomic
+#endif
+	      ncomm++;
+
+#ifdef GRID_OMP
+#pragma omp atomic
+#endif
+	      dbytes+=tbytes;
+	    }
+	  }
+	  Grid.Barrier();
+	  double stop=usecond();
+	  t_time[i] = stop-start; // microseconds
+	}
+
+	timestat.statistics(t_time);
+	//	for(int i=0;i<t_time.size();i++){
+	//	  std::cout << i<<" "<<t_time[i]<<std::endl;
+	//	}
+
+	dbytes=dbytes*ppn;
+	double xbytes    = dbytes*0.5;
+	double rbytes    = dbytes*0.5;
+	double bidibytes = dbytes;
+
+	std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+		 <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+		 <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+		 <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+		 << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+		 << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+
+ 
+	
+	    }
+    }    
+
+    return;
+  }
+
+  static void Memory(void)
+  {
+    const int Nvec=8;
+    typedef Lattice< iVector< vReal,Nvec> > LatticeVec;
+    typedef iVector<vReal,Nvec> Vec;
+
+    std::vector<int> simd_layout = GridDefaultSimd(Nd,vReal::Nsimd());
+    std::vector<int> mpi_layout  = GridDefaultMpi();
+
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Benchmarking a*x + y bandwidth"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<< "\t\tGB/s / node"<<std::endl;
+    std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+  
+    uint64_t NP;
+    uint64_t NN;
+
+
+  uint64_t lmax=48;
+#define NLOOP (100*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
+
+    GridSerialRNG          sRNG;      sRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
+    for(int lat=8;lat<=lmax;lat+=4){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+
+      NP= Grid.RankCount();
+      NN =Grid.NodeCount();
+
+      Vec rn ; random(sRNG,rn);
+
+      LatticeVec z(&Grid); z=rn;
+      LatticeVec x(&Grid); x=rn;
+      LatticeVec y(&Grid); y=rn;
+      double a=2.0;
+
+      uint64_t Nloop=NLOOP;
+
+      double start=usecond();
+      for(int i=0;i<Nloop;i++){
+	z=a*x-y;
+        x._odata[0]=z._odata[0]; // force serial dependency to prevent optimise away
+        y._odata[4]=z._odata[4];
+      }
+      double stop=usecond();
+      double time = (stop-start)/Nloop*1000;
+     
+      double flops=vol*Nvec*2;// mul,add
+      double bytes=3.0*vol*Nvec*sizeof(Real);
+      std::cout<<GridLogMessage<<std::setprecision(3) 
+	       << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.
+	       << "\t\t"<< bytes/time/NN <<std::endl;
+
+    }
+  };
+
+  static double DWF5(int Ls,int L)
+  {
+    RealD mass=0.1;
+    RealD M5  =1.8;
+
+    double mflops;
+    double mflops_best = 0;
+    double mflops_worst= 0;
+    std::vector<double> mflops_all;
+
+    ///////////////////////////////////////////////////////
+    // Set/Get the layout & grid size
+    ///////////////////////////////////////////////////////
+    int threads = GridThread::GetThreads();
+    std::vector<int> mpi = GridDefaultMpi(); assert(mpi.size()==4);
+    std::vector<int> local({L,L,L,L});
+
+    GridCartesian         * TmpGrid   = SpaceTimeGrid::makeFourDimGrid(std::vector<int>({64,64,64,64}), 
+								       GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    uint64_t NP = TmpGrid->RankCount();
+    uint64_t NN = TmpGrid->NodeCount();
+    NN_global=NN;
+    uint64_t SHM=NP/NN;
+
+    std::vector<int> internal;
+    if      ( SHM == 1 )   internal = std::vector<int>({1,1,1,1});
+    else if ( SHM == 2 )   internal = std::vector<int>({2,1,1,1});
+    else if ( SHM == 4 )   internal = std::vector<int>({2,2,1,1});
+    else if ( SHM == 8 )   internal = std::vector<int>({2,2,2,1});
+    else assert(0);
+
+    std::vector<int> nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]});
+    std::vector<int> latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]});
+
+    ///////// Welcome message ////////////
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "Benchmark DWF Ls vec on "<<L<<"^4 local volume "<<std::endl;
+    std::cout<<GridLogMessage << "* Global volume  : "<<GridCmdVectorIntToString(latt4)<<std::endl;
+    std::cout<<GridLogMessage << "* Ls             : "<<Ls<<std::endl;
+    std::cout<<GridLogMessage << "* MPI ranks      : "<<GridCmdVectorIntToString(mpi)<<std::endl;
+    std::cout<<GridLogMessage << "* Intranode      : "<<GridCmdVectorIntToString(internal)<<std::endl;
+    std::cout<<GridLogMessage << "* nodes          : "<<GridCmdVectorIntToString(nodes)<<std::endl;
+    std::cout<<GridLogMessage << "* Using "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    ///////// Lattice Init ////////////
+    GridCartesian         * UGrid    = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    GridRedBlackCartesian * UrbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridCartesian         * sUGrid   = SpaceTimeGrid::makeFourDimDWFGrid(latt4,GridDefaultMpi());
+    GridRedBlackCartesian * sUrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(sUGrid);
+    GridCartesian         * sFGrid   = SpaceTimeGrid::makeFiveDimDWFGrid(Ls,UGrid);
+    GridRedBlackCartesian * sFrbGrid = SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(Ls,UGrid);
+
+    ///////// RNG Init ////////////
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+    GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG          RNG5(sFGrid);  RNG5.SeedFixedIntegers(seeds5);
+    std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+    ///////// Source preparation ////////////
+    LatticeFermion src   (sFGrid); random(RNG5,src);
+    LatticeFermion tmp   (sFGrid);
+
+    RealD N2 = 1.0/::sqrt(norm2(src));
+    src = src*N2;
+    
+    LatticeGaugeField Umu(UGrid);  SU3::HotConfiguration(RNG4,Umu); 
+
+    WilsonFermion5DR sDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,M5);
+    LatticeFermion src_e (sFrbGrid);
+    LatticeFermion src_o (sFrbGrid);
+    LatticeFermion r_e   (sFrbGrid);
+    LatticeFermion r_o   (sFrbGrid);
+    LatticeFermion r_eo  (sFGrid);
+    LatticeFermion err   (sFGrid);
+    {
+
+      pickCheckerboard(Even,src_e,src);
+      pickCheckerboard(Odd,src_o,src);
+
+#if defined(AVX512) 
+      const int num_cases = 6;
+      std::string fmt("A/S ; A/O ; U/S ; U/O ; G/S ; G/O ");
+#else
+      const int num_cases = 4;
+      std::string fmt("U/S ; U/O ; G/S ; G/O ");
+#endif
+      controls Cases [] = {
+#ifdef AVX512
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+#endif
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  }
+      }; 
+
+      for(int c=0;c<num_cases;c++) {
+
+	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
+	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
+	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
+
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+	if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+	int nwarm = 100;
+	uint64_t ncall = 1000;
+
+	double t0=usecond();
+	sFGrid->Barrier();
+	for(int i=0;i<nwarm;i++){
+	  sDw.DhopEO(src_o,r_e,DaggerNo);
+	}
+	sFGrid->Barrier();
+	double t1=usecond();
+
+	sDw.ZeroCounters();
+	time_statistics timestat;
+	std::vector<double> t_time(ncall);
+	for(uint64_t i=0;i<ncall;i++){
+	  t0=usecond();
+	  sDw.DhopEO(src_o,r_e,DaggerNo);
+	  t1=usecond();
+	  t_time[i] = t1-t0;
+	}
+	sFGrid->Barrier();
+	
+	double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+	double flops=(1344.0*volume)/2;
+	double mf_hi, mf_lo, mf_err;
+
+	timestat.statistics(t_time);
+	mf_hi = flops/timestat.min;
+	mf_lo = flops/timestat.max;
+	mf_err= flops/timestat.min * timestat.err/timestat.mean;
+
+	mflops = flops/timestat.mean;
+	mflops_all.push_back(mflops);
+	if ( mflops_best == 0   ) mflops_best = mflops;
+	if ( mflops_worst== 0   ) mflops_worst= mflops;
+	if ( mflops>mflops_best ) mflops_best = mflops;
+	if ( mflops<mflops_worst) mflops_worst= mflops;
+
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s =   "<< mflops << " ("<<mf_err<<") " << mf_lo<<"-"<<mf_hi <<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s per rank   "<< mflops/NP<<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s per node   "<< mflops/NN<<std::endl;
+
+	sDw.Report();
+
+      }
+      double robust = mflops_worst/mflops_best;;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
+
+      std::cout<<GridLogMessage <<std::setprecision(3)<< L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< robust <<std::endl;
+      std::cout<<GridLogMessage <<fmt << std::endl;
+      std::cout<<GridLogMessage;
+
+      for(int i=0;i<mflops_all.size();i++){
+	std::cout<<mflops_all[i]/NN<<" ; " ;
+      }
+      std::cout<<std::endl;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    }
+    return mflops_best;
+  }
+
+  static double DWF(int Ls,int L, double & robust)
+  {
+    RealD mass=0.1;
+    RealD M5  =1.8;
+
+    double mflops;
+    double mflops_best = 0;
+    double mflops_worst= 0;
+    std::vector<double> mflops_all;
+
+    ///////////////////////////////////////////////////////
+    // Set/Get the layout & grid size
+    ///////////////////////////////////////////////////////
+    int threads = GridThread::GetThreads();
+    std::vector<int> mpi = GridDefaultMpi(); assert(mpi.size()==4);
+    std::vector<int> local({L,L,L,L});
+
+    GridCartesian         * TmpGrid   = SpaceTimeGrid::makeFourDimGrid(std::vector<int>({64,64,64,64}), 
+								       GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    uint64_t NP = TmpGrid->RankCount();
+    uint64_t NN = TmpGrid->NodeCount();
+    NN_global=NN;
+    uint64_t SHM=NP/NN;
+
+    std::vector<int> internal;
+    if      ( SHM == 1 )   internal = std::vector<int>({1,1,1,1});
+    else if ( SHM == 2 )   internal = std::vector<int>({2,1,1,1});
+    else if ( SHM == 4 )   internal = std::vector<int>({2,2,1,1});
+    else if ( SHM == 8 )   internal = std::vector<int>({2,2,2,1});
+    else assert(0);
+
+    std::vector<int> nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]});
+    std::vector<int> latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]});
+
+    ///////// Welcome message ////////////
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "Benchmark DWF on "<<L<<"^4 local volume "<<std::endl;
+    std::cout<<GridLogMessage << "* Global volume  : "<<GridCmdVectorIntToString(latt4)<<std::endl;
+    std::cout<<GridLogMessage << "* Ls             : "<<Ls<<std::endl;
+    std::cout<<GridLogMessage << "* MPI ranks      : "<<GridCmdVectorIntToString(mpi)<<std::endl;
+    std::cout<<GridLogMessage << "* Intranode      : "<<GridCmdVectorIntToString(internal)<<std::endl;
+    std::cout<<GridLogMessage << "* nodes          : "<<GridCmdVectorIntToString(nodes)<<std::endl;
+    std::cout<<GridLogMessage << "* Using "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+
+    ///////// Lattice Init ////////////
+    GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+    GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+    
+    ///////// RNG Init ////////////
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+    GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+    std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+    ///////// Source preparation ////////////
+    LatticeFermion src   (FGrid); random(RNG5,src);
+    LatticeFermion ref   (FGrid);
+    LatticeFermion tmp   (FGrid);
+
+    RealD N2 = 1.0/::sqrt(norm2(src));
+    src = src*N2;
+    
+    LatticeGaugeField Umu(UGrid);  SU3::HotConfiguration(RNG4,Umu); 
+
+    DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+
+    ////////////////////////////////////
+    // Naive wilson implementation
+    ////////////////////////////////////
+    {
+      LatticeGaugeField Umu5d(FGrid); 
+      std::vector<LatticeColourMatrix> U(4,FGrid);
+      for(int ss=0;ss<Umu._grid->oSites();ss++){
+	for(int s=0;s<Ls;s++){
+	  Umu5d._odata[Ls*ss+s] = Umu._odata[ss];
+	}
+      }
+      ref = zero;
+      for(int mu=0;mu<Nd;mu++){
+	U[mu] = PeekIndex<LorentzIndex>(Umu5d,mu);
+      }
+      for(int mu=0;mu<Nd;mu++){
+	
+	tmp = U[mu]*Cshift(src,mu+1,1);
+	ref=ref + tmp - Gamma(Gmu[mu])*tmp;
+	
+	tmp =adj(U[mu])*src;
+	tmp =Cshift(tmp,mu+1,-1);
+	ref=ref + tmp + Gamma(Gmu[mu])*tmp;
+      }
+      ref = -0.5*ref;
+    }
+
+    LatticeFermion src_e (FrbGrid);
+    LatticeFermion src_o (FrbGrid);
+    LatticeFermion r_e   (FrbGrid);
+    LatticeFermion r_o   (FrbGrid);
+    LatticeFermion r_eo  (FGrid);
+    LatticeFermion err   (FGrid);
+    {
+
+      pickCheckerboard(Even,src_e,src);
+      pickCheckerboard(Odd,src_o,src);
+
+#if defined(AVX512) 
+      const int num_cases = 6;
+      std::string fmt("A/S ; A/O ; U/S ; U/O ; G/S ; G/O ");
+#else
+      const int num_cases = 4;
+      std::string fmt("U/S ; U/O ; G/S ; G/O ");
+#endif
+      controls Cases [] = {
+#ifdef AVX512
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+#endif
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  }
+      }; 
+
+      for(int c=0;c<num_cases;c++) {
+
+	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
+	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
+	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
+
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+	if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+	int nwarm = 200;
+	double t0=usecond();
+	FGrid->Barrier();
+	for(int i=0;i<nwarm;i++){
+	  Dw.DhopEO(src_o,r_e,DaggerNo);
+	}
+	FGrid->Barrier();
+	double t1=usecond();
+	//	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
+	//	if (ncall < 500) ncall = 500;
+	uint64_t ncall = 1000;
+
+	FGrid->Broadcast(0,&ncall,sizeof(ncall));
+
+	//	std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"<<std::endl;
+	Dw.ZeroCounters();
+
+	time_statistics timestat;
+	std::vector<double> t_time(ncall);
+	for(uint64_t i=0;i<ncall;i++){
+	  t0=usecond();
+	  Dw.DhopEO(src_o,r_e,DaggerNo);
+	  t1=usecond();
+	  t_time[i] = t1-t0;
+	}
+	FGrid->Barrier();
+	
+	double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+	double flops=(1344.0*volume)/2;
+	double mf_hi, mf_lo, mf_err;
+
+	timestat.statistics(t_time);
+	mf_hi = flops/timestat.min;
+	mf_lo = flops/timestat.max;
+	mf_err= flops/timestat.min * timestat.err/timestat.mean;
+
+	mflops = flops/timestat.mean;
+	mflops_all.push_back(mflops);
+	if ( mflops_best == 0   ) mflops_best = mflops;
+	if ( mflops_worst== 0   ) mflops_worst= mflops;
+	if ( mflops>mflops_best ) mflops_best = mflops;
+	if ( mflops<mflops_worst) mflops_worst= mflops;
+
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s =   "<< mflops << " ("<<mf_err<<") " << mf_lo<<"-"<<mf_hi <<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s per rank   "<< mflops/NP<<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s per node   "<< mflops/NN<<std::endl;
+
+	Dw.Report();
+
+	Dw.DhopEO(src_o,r_e,DaggerNo);
+	Dw.DhopOE(src_e,r_o,DaggerNo);
+	setCheckerboard(r_eo,r_o);
+	setCheckerboard(r_eo,r_e);
+	err = r_eo-ref; 
+	std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+	assert((norm2(err)<1.0e-4));
+
+      }
+      robust = mflops_worst/mflops_best;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << std::fixed<<std::setprecision(3)<< L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< robust  <<std::endl;
+      std::cout<<GridLogMessage <<fmt << std::endl;
+      std::cout<<GridLogMessage ;
+
+      for(int i=0;i<mflops_all.size();i++){
+	std::cout<<mflops_all[i]/NN<<" ; " ;
+      }
+      std::cout<<std::endl;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    }
+    return mflops_best;
+  }
+
+};
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  CartesianCommunicator::SetCommunicatorPolicy(CartesianCommunicator::CommunicatorPolicySequential);
+#ifdef KNL
+  LebesgueOrder::Block = std::vector<int>({8,2,2,2});
+#else
+  LebesgueOrder::Block = std::vector<int>({2,2,2,2});
+#endif
+  Benchmark::Decomposition();
+
+  int do_memory=1;
+  int do_comms =1;
+  int do_su3   =0;
+  int do_wilson=1;
+  int do_dwf   =1;
+
+  if ( do_su3 ) {
+    // empty for now
+  }
+#if 1
+  int sel=2;
+  std::vector<int> L_list({8,12,16,24});
+#else
+  int sel=1;
+  std::vector<int> L_list({8,12});
+#endif
+  int selm1=sel-1;
+  std::vector<double> robust_list;
+
+  std::vector<double> wilson;
+  std::vector<double> dwf4;
+  std::vector<double> dwf5;
+
+  if ( do_wilson ) {
+    int Ls=1;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Wilson dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    for(int l=0;l<L_list.size();l++){
+      double robust;
+      wilson.push_back(Benchmark::DWF(1,L_list[l],robust));
+    }
+  }
+
+  int Ls=16;
+  if ( do_dwf ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    for(int l=0;l<L_list.size();l++){
+      double robust;
+      double result = Benchmark::DWF(Ls,L_list[l],robust) ;
+      dwf4.push_back(result);
+      robust_list.push_back(robust);
+    }
+  }
+
+  if ( do_dwf ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    for(int l=0;l<L_list.size();l++){
+      dwf5.push_back(Benchmark::DWF5(Ls,L_list[l]));
+    }
+
+  }
+
+  if ( do_dwf ) {
+
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Summary table Ls="<<Ls <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "L \t\t Wilson \t DWF4 \t DWF5 " <<std::endl;
+  for(int l=0;l<L_list.size();l++){
+    std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]<<" \t "<<dwf4[l]<<" \t "<<dwf5[l] <<std::endl;
+  }
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  }
+
+  int NN=NN_global;
+  if ( do_memory ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Memory benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Memory();
+  }
+
+  if ( do_comms && (NN>1) ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Communications benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Comms();
+  }
+
+  if ( do_dwf ) {
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Per Node Summary table Ls="<<Ls <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " L \t\t Wilson\t\t DWF4  \t\t DWF5 " <<std::endl;
+  for(int l=0;l<L_list.size();l++){
+    std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]/NN<<" \t "<<dwf4[l]/NN<<" \t "<<dwf5[l] /NN<<std::endl;
+  }
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Comparison point     result: "  << 0.5*(dwf4[sel]+dwf4[selm1])/NN << " Mflop/s per node"<<std::endl;
+  std::cout<<GridLogMessage << " Comparison point is 0.5*("<<dwf4[sel]/NN<<"+"<<dwf4[selm1]/NN << ") "<<std::endl;
+  std::cout<<std::setprecision(3);
+  std::cout<<GridLogMessage << " Comparison point robustness: "  << robust_list[sel] <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  }
+
+
+  Grid_finalize();
+}

benchmarks/Benchmark_comms.cc

@@ -31,6 +31,32 @@ using namespace std;
 using namespace Grid;
 using namespace Grid::QCD;
 
+struct time_statistics{
+  double mean;
+  double err;
+  double min;
+  double max;
+
+  void statistics(std::vector<double> v){
+      double sum = std::accumulate(v.begin(), v.end(), 0.0);
+      mean = sum / v.size();
+
+      std::vector<double> diff(v.size());
+      std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; });
+      double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+      err = std::sqrt(sq_sum / (v.size()*(v.size() - 1)));
+
+      auto result = std::minmax_element(v.begin(), v.end());
+      min = *result.first;
+      max = *result.second;
+}
+};
+
+void header(){
+  std::cout <<GridLogMessage << " L  "<<"\t"<<" Ls  "<<"\t"
+            <<std::setw(11)<<"bytes"<<"MB/s uni (err/min/max)"<<"\t\t"<<"MB/s bidi (err/min/max)"<<std::endl;
+};
+
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
@@ -40,17 +66,21 @@ int main (int argc, char ** argv)
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
-  int Nloop=10;
+  int Nloop=100;
   int nmu=0;
+  int maxlat=32;
   for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
 
+  std::cout << GridLogMessage << "Number of iterations to average: "<< Nloop << std::endl;
+  std::vector<double> t_time(Nloop);
+  time_statistics timestat;
+
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
-  int maxlat=16;
+  for(int lat=4;lat<=maxlat;lat+=4){
-  for(int lat=4;lat<=maxlat;lat+=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
-    for(int Ls=1;Ls<=16;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -58,15 +88,23 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > xbuf(8);	
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > rbuf(8);
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+      for(int mu=0;mu<8;mu++){
+	xbuf[mu].resize(lat*lat*lat*Ls);
+	rbuf[mu].resize(lat*lat*lat*Ls);
+	//	std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] <<std::endl;
+      }
 
-      double start=usecond();
       for(int i=0;i<Nloop;i++){
+      double start=usecond();
 
 	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 
@@ -79,7 +117,6 @@ int main (int argc, char ** argv)
 	    int comm_proc=1;
 	    int xmit_to_rank;
 	    int recv_from_rank;
-	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	    Grid.SendToRecvFromBegin(requests,
 				   (void *)&xbuf[mu][0],
@@ -102,18 +139,24 @@ int main (int argc, char ** argv)
 	}
 	Grid.SendToRecvFromComplete(requests);
 	Grid.Barrier();
-
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
+      timestat.statistics(t_time);
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
+
+      double dbytes    = bytes*ppn;
+      double xbytes    = dbytes*2.0*ncomm;
       double rbytes    = xbytes;
       double bidibytes = xbytes+rbytes;
 
-      double time = stop-start; // microseconds
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
 
@@ -121,25 +164,32 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking sequential halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
 
-
+  for(int lat=4;lat<=maxlat;lat+=4){
-  for(int lat=4;lat<=maxlat;lat+=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
-    for(int Ls=1;Ls<=16;Ls*=2){
 
       std::vector<int> latt_size  ({lat,lat,lat,lat});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > xbuf(8);
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > rbuf(8);
 
+      for(int mu=0;mu<8;mu++){
+	xbuf[mu].resize(lat*lat*lat*Ls);
+	rbuf[mu].resize(lat*lat*lat*Ls);
+	//	std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] <<std::endl;
+      }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
       for(int i=0;i<Nloop;i++){
+      double start=usecond();
     
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
@@ -178,30 +228,37 @@ int main (int argc, char ** argv)
 	  }
 	}
 	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+
       }
 
-      double stop=usecond();
+      timestat.statistics(t_time);
       
-      double dbytes    = bytes;
+      double dbytes    = bytes*ppn;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
+      double xbytes    = dbytes*2.0*ncomm;
       double rbytes    = xbytes;
       double bidibytes = xbytes+rbytes;
 
-      double time = stop-start;
+    std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+
       
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }  
 
 
-  Nloop=100;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
 
-  for(int lat=4;lat<=maxlat;lat+=2){
+  for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=1;Ls<=16;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -209,6 +266,9 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<HalfSpinColourVectorD *> xbuf(8);
       std::vector<HalfSpinColourVectorD *> rbuf(8);
@@ -216,73 +276,86 @@ int main (int argc, char ** argv)
       for(int d=0;d<8;d++){
 	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
 	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
       }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
+      double dbytes;
       for(int i=0;i<Nloop;i++){
+	double start=usecond();
+
+	dbytes=0;
+	ncomm=0;
 
 	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 
-	ncomm=0;
 	for(int mu=0;mu<4;mu++){
 	
+
 	  if (mpi_layout[mu]>1 ) {
 	  
 	    ncomm++;
 	    int comm_proc=1;
 	    int xmit_to_rank;
 	    int recv_from_rank;
-	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
+	    dbytes+=
-					    (void *)&xbuf[mu][0],
+	      Grid.StencilSendToRecvFromBegin(requests,
-					    xmit_to_rank,
+					      (void *)&xbuf[mu][0],
-					    (void *)&rbuf[mu][0],
+					      xmit_to_rank,
-					    recv_from_rank,
+					      (void *)&rbuf[mu][0],
-					    bytes);
+					      recv_from_rank,
+					      bytes,mu);
 	
 	    comm_proc = mpi_layout[mu]-1;
 	  
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
+	    dbytes+=
-					    (void *)&xbuf[mu+4][0],
+	      Grid.StencilSendToRecvFromBegin(requests,
-					    xmit_to_rank,
+					      (void *)&xbuf[mu+4][0],
-					    (void *)&rbuf[mu+4][0],
+					      xmit_to_rank,
-					    recv_from_rank,
+					      (void *)&rbuf[mu+4][0],
-					    bytes);
+					      recv_from_rank,
+					      bytes,mu+4);
 	  
 	  }
 	}
-	Grid.StencilSendToRecvFromComplete(requests);
+	Grid.StencilSendToRecvFromComplete(requests,0);
 	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
 	
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
+      timestat.statistics(t_time);
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
+
-      double rbytes    = xbytes;
+      dbytes=dbytes*ppn;
-      double bidibytes = xbytes+rbytes;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      double time = stop-start; // microseconds
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
 
 
-
-  Nloop=100;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking sequential STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
 
-  for(int lat=4;lat<=maxlat;lat+=2){
+  for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=1;Ls<=16;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -290,6 +363,9 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<HalfSpinColourVectorD *> xbuf(8);
       std::vector<HalfSpinColourVectorD *> rbuf(8);
@@ -297,16 +373,18 @@ int main (int argc, char ** argv)
       for(int d=0;d<8;d++){
 	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
 	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
       }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
-
+      double dbytes;
-      double start=usecond();
       for(int i=0;i<Nloop;i++){
+	double start=usecond();
 
 	std::vector<CartesianCommunicator::CommsRequest_t> requests;
-
+	dbytes=0;
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
 	
@@ -318,44 +396,146 @@ int main (int argc, char ** argv)
 	    int recv_from_rank;
 	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
+	    dbytes+=
-					    (void *)&xbuf[mu][0],
+	      Grid.StencilSendToRecvFromBegin(requests,
-					    xmit_to_rank,
+					      (void *)&xbuf[mu][0],
-					    (void *)&rbuf[mu][0],
+					      xmit_to_rank,
-					    recv_from_rank,
+					      (void *)&rbuf[mu][0],
-					    bytes);
+					      recv_from_rank,
-	    //	    Grid.StencilSendToRecvFromComplete(requests);
+					      bytes,mu);
-	    //	    requests.resize(0);
+	    Grid.StencilSendToRecvFromComplete(requests,mu);
+	    requests.resize(0);
 
 	    comm_proc = mpi_layout[mu]-1;
 	  
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
+	    dbytes+=
-					    (void *)&xbuf[mu+4][0],
+	      Grid.StencilSendToRecvFromBegin(requests,
-					    xmit_to_rank,
+					      (void *)&xbuf[mu+4][0],
-					    (void *)&rbuf[mu+4][0],
+					      xmit_to_rank,
-					    recv_from_rank,
+					      (void *)&rbuf[mu+4][0],
-					    bytes);
+					      recv_from_rank,
-	    Grid.StencilSendToRecvFromComplete(requests);
+					      bytes,mu+4);
+	    Grid.StencilSendToRecvFromComplete(requests,mu+4);
 	    requests.resize(0);
 	  
 	  }
 	}
 	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
 	
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
+      timestat.statistics(t_time);
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
 
-      double time = stop-start; // microseconds
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
+
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
  
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
 
+
+
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking threaded STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  header();
+
+  for(int lat=4;lat<=maxlat;lat+=4){
+    for(int Ls=8;Ls<=8;Ls*=2){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],
+      				    lat*mpi_layout[1],
+      				    lat*mpi_layout[2],
+      				    lat*mpi_layout[3]});
+
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
+
+      std::vector<HalfSpinColourVectorD *> xbuf(8);
+      std::vector<HalfSpinColourVectorD *> rbuf(8);
+      Grid.ShmBufferFreeAll();
+      for(int d=0;d<8;d++){
+	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+      }
+
+      int ncomm;
+      int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+      double dbytes;
+      for(int i=0;i<Nloop;i++){
+	double start=usecond();
+
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
+	dbytes=0;
+	ncomm=0;
+
+	parallel_for(int dir=0;dir<8;dir++){
+
+	  double tbytes;
+	  int mu =dir % 4;
+
+	  if (mpi_layout[mu]>1 ) {
+	  
+	    ncomm++;
+	    int xmit_to_rank;
+	    int recv_from_rank;
+	    if ( dir == mu ) { 
+	      int comm_proc=1;
+	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    } else { 
+	      int comm_proc = mpi_layout[mu]-1;
+	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    }
+
+	    tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
+					       (void *)&rbuf[dir][0], recv_from_rank, bytes,dir);
+
+#pragma omp atomic
+	    dbytes+=tbytes;
+	  }
+	}
+	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+      }
+
+      timestat.statistics(t_time);
+
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
+
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+ 
+    }
+  }    
+
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= All done; Bye Bye"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+
   Grid_finalize();
 }

benchmarks/Benchmark_dwf.cc

@@ -1,28 +1,22 @@
-    /*************************************************************************************
+ /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./benchmarks/Benchmark_dwf.cc
-
     Copyright (C) 2015
 
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -37,27 +31,33 @@ struct scal {
   d internal;
 };
 
-  Gamma::GammaMatrix Gmu [] = {
+  Gamma::Algebra Gmu [] = {
-    Gamma::GammaX,
+    Gamma::Algebra::GammaX,
-    Gamma::GammaY,
+    Gamma::Algebra::GammaY,
-    Gamma::GammaZ,
+    Gamma::Algebra::GammaZ,
-    Gamma::GammaT
+    Gamma::Algebra::GammaT
   };
 
 typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
 typedef WilsonFermion5D<DomainWallVec5dImplF> WilsonFermion5DF;
 typedef WilsonFermion5D<DomainWallVec5dImplD> WilsonFermion5DD;
 
-
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
 
+
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
   std::vector<int> latt4 = GridDefaultLatt();
-  const int Ls=8;
+  int Ls=16;
+  for(int i=0;i<argc;i++)
+    if(std::string(argv[i]) == "-Ls"){
+      std::stringstream ss(argv[i+1]); ss >> Ls;
+    }
+
+
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
@@ -72,34 +72,65 @@ int main (int argc, char ** argv)
   std::vector<int> seeds4({1,2,3,4});
   std::vector<int> seeds5({5,6,7,8});
   
+  std::cout << GridLogMessage << "Initialising 4d RNG" << std::endl;
   GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  std::cout << GridLogMessage << "Initialising 5d RNG" << std::endl;
   GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
 
   LatticeFermion src   (FGrid); random(RNG5,src);
+#if 0
+  src = zero;
+  {
+    std::vector<int> origin({0,0,0,latt4[2]-1,0});
+    SpinColourVectorF tmp;
+    tmp=zero;
+    tmp()(0)(0)=Complex(-2.0,0.0);
+    std::cout << " source site 0 " << tmp<<std::endl;
+    pokeSite(tmp,src,origin);
+  }
+#else
+  RealD N2 = 1.0/::sqrt(norm2(src));
+  src = src*N2;
+#endif
+
+
   LatticeFermion result(FGrid); result=zero;
   LatticeFermion    ref(FGrid);    ref=zero;
   LatticeFermion    tmp(FGrid);
   LatticeFermion    err(FGrid);
 
+  std::cout << GridLogMessage << "Drawing gauge field" << std::endl;
   LatticeGaugeField Umu(UGrid); 
-  random(RNG4,Umu);
+  SU3::HotConfiguration(RNG4,Umu); 
-
+  std::cout << GridLogMessage << "Random gauge initialised " << std::endl;
-  LatticeGaugeField Umu5d(FGrid); 
+#if 0
+  Umu=1.0;
+  for(int mu=0;mu<Nd;mu++){
+    LatticeColourMatrix ttmp(UGrid);
+    ttmp = PeekIndex<LorentzIndex>(Umu,mu);
+    //    if (mu !=2 ) ttmp = 0;
+    //    ttmp = ttmp* pow(10.0,mu);
+    PokeIndex<LorentzIndex>(Umu,ttmp,mu);
+  }
+  std::cout << GridLogMessage << "Forced to diagonal " << std::endl;
+#endif
 
+  ////////////////////////////////////
+  // Naive wilson implementation
+  ////////////////////////////////////
   // replicate across fifth dimension
+  LatticeGaugeField Umu5d(FGrid); 
+  std::vector<LatticeColourMatrix> U(4,FGrid);
   for(int ss=0;ss<Umu._grid->oSites();ss++){
     for(int s=0;s<Ls;s++){
       Umu5d._odata[Ls*ss+s] = Umu._odata[ss];
     }
   }
-
-  ////////////////////////////////////
-  // Naive wilson implementation
-  ////////////////////////////////////
-  std::vector<LatticeColourMatrix> U(4,FGrid);
   for(int mu=0;mu<Nd;mu++){
     U[mu] = PeekIndex<LorentzIndex>(Umu5d,mu);
   }
+  std::cout << GridLogMessage << "Setting up Cshift based reference " << std::endl;
 
   if (1)
   {
@@ -120,8 +151,7 @@ int main (int argc, char ** argv)
   RealD M5  =1.8;
 
   RealD NP = UGrid->_Nprocessors;
-
+  RealD NN = UGrid->NodeCount();
-  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
 
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
   std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
@@ -131,15 +161,22 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
   if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
   if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
 
-  int ncall =100;
+  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  int ncall =500;
   if (1) {
     FGrid->Barrier();
     Dw.ZeroCounters();
+    Dw.Dhop(src,result,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       __SSC_START;
@@ -153,16 +190,55 @@ int main (int argc, char ** argv)
     double flops=1344*volume*ncall;
 
     std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
-    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
-    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+
+    /*
+    if(( norm2(err)>1.0e-4) ) { 
+      std::cout << "RESULT\n " << result<<std::endl;
+      std::cout << "REF   \n " << ref   <<std::endl;
+      std::cout << "ERR   \n " << err   <<std::endl;
+      FGrid->Barrier();
+      exit(-1);
+    }
+    */
     assert (norm2(err)< 1.0e-4 );
     Dw.Report();
   }
 
+  DomainWallFermionRL DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  if (1) {
+    FGrid->Barrier();
+    DwH.ZeroCounters();
+    DwH.Dhop(src,result,0);
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwH.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called half prec comms Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    err = ref-result; 
+    std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+
+    assert (norm2(err)< 1.0e-3 );
+    DwH.Report();
+  }
+
   if (1)
   {
 
@@ -171,6 +247,10 @@ int main (int argc, char ** argv)
     std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
     if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
     if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
@@ -183,20 +263,12 @@ int main (int argc, char ** argv)
 
     WilsonFermion5DR sDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,M5);
 
-    for(int x=0;x<latt4[0];x++){
+    localConvert(src,ssrc);
-    for(int y=0;y<latt4[1];y++){
-    for(int z=0;z<latt4[2];z++){
-    for(int t=0;t<latt4[3];t++){
-    for(int s=0;s<Ls;s++){
-      std::vector<int> site({s,x,y,z,t});
-      SpinColourVector tmp;
-      peekSite(tmp,src,site);
-      pokeSite(tmp,ssrc,site);
-    }}}}}
     std::cout<<GridLogMessage<< "src norms "<< norm2(src)<<" " <<norm2(ssrc)<<std::endl;
     FGrid->Barrier();
-    double t0=usecond();
+    sDw.Dhop(ssrc,sresult,0);
     sDw.ZeroCounters();
+    double t0=usecond();
     for(int i=0;i<ncall;i++){
       __SSC_START;
       sDw.Dhop(ssrc,sresult,0);
@@ -210,46 +282,53 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    //    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
     sDw.Report();
-  
-    if(0){
-      for(int i=0;i< PerformanceCounter::NumTypes(); i++ ){
-	sDw.Dhop(ssrc,sresult,0);
-	PerformanceCounter Counter(i);
-	Counter.Start();
-	sDw.Dhop(ssrc,sresult,0);
-	Counter.Stop();
-	Counter.Report();
-      }
-    }
-
-    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
-
     RealD sum=0;
-    for(int x=0;x<latt4[0];x++){
+
-    for(int y=0;y<latt4[1];y++){
+    err=zero;
-    for(int z=0;z<latt4[2];z++){
+    localConvert(sresult,err);
-    for(int t=0;t<latt4[3];t++){
+    err = err - ref;
-    for(int s=0;s<Ls;s++){
+    sum = norm2(err);
-      std::vector<int> site({s,x,y,z,t});
+    std::cout<<GridLogMessage<<" difference between normal ref and simd is "<<sum<<std::endl;
-      SpinColourVector normal, simd;
+    if(sum > 1.0e-4 ){
-      peekSite(normal,result,site);
+      std::cout<< "sD REF\n " <<ref << std::endl;
-      peekSite(simd,sresult,site);
+      std::cout<< "sD ERR   \n " <<err  <<std::endl;
-      sum=sum+norm2(normal-simd);
+    }
-      if (norm2(normal-simd) > 1.0e-6 ) {
+    //    assert(sum < 1.0e-4);
-	std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" "<<norm2(normal-simd)<<std::endl;
+
-	std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" normal "<<normal<<std::endl;
+    err=zero;
-	std::cout << "site "<<x<<","<<y<<","<<z<<","<<t<<","<<s<<" simd   "<<simd<<std::endl;
+    localConvert(sresult,err);
-      }
+    err = err - result;
-    }}}}}
+    sum = norm2(err);
-    std::cout<<GridLogMessage<<" difference between normal and simd is "<<sum<<std::endl;
+    std::cout<<GridLogMessage<<" difference between normal result and simd is "<<sum<<std::endl;
-    assert (sum< 1.0e-4 );
+    if(sum > 1.0e-4 ){
+      std::cout<< "sD REF\n " <<result << std::endl;
+      std::cout<< "sD ERR   \n " << err  <<std::endl;
+    }
+    assert(sum < 1.0e-4);
 
     
-    if (1) {
+    if(1){
+      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
+      std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::DhopEO "<<std::endl;
+      std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
+      if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+      if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
+      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) 
+	std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) 
+	std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) 
+	std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
 
       LatticeFermion sr_eo(sFGrid);
-
       LatticeFermion ssrc_e (sFrbGrid);
       LatticeFermion ssrc_o (sFrbGrid);
       LatticeFermion sr_e   (sFrbGrid);
@@ -257,39 +336,30 @@ int main (int argc, char ** argv)
 
       pickCheckerboard(Even,ssrc_e,ssrc);
       pickCheckerboard(Odd,ssrc_o,ssrc);
-
+      //      setCheckerboard(sr_eo,ssrc_o);
-      setCheckerboard(sr_eo,ssrc_o);
+      //      setCheckerboard(sr_eo,ssrc_e);
-      setCheckerboard(sr_eo,ssrc_e);
 
       sr_e = zero;
       sr_o = zero;
 
-      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-      std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::DhopEO "<<std::endl;
-      std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
-      if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
-      if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
-      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
-      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
-      if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
-      std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
-
       FGrid->Barrier();
+      sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
       sDw.ZeroCounters();
-      sDw.stat.init("DhopEO");
+      //      sDw.stat.init("DhopEO");
       double t0=usecond();
       for (int i = 0; i < ncall; i++) {
         sDw.DhopEO(ssrc_o, sr_e, DaggerNo);
       }
       double t1=usecond();
       FGrid->Barrier();
-      sDw.stat.print();
+      //      sDw.stat.print();
 
       double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
       double flops=(1344.0*volume*ncall)/2;
 
       std::cout<<GridLogMessage << "sDeo mflop/s =   "<< flops/(t1-t0)<<std::endl;
       std::cout<<GridLogMessage << "sDeo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+      std::cout<<GridLogMessage << "sDeo mflop/s per node   "<< flops/(t1-t0)/NN<<std::endl;
       sDw.Report();
 
       sDw.DhopEO(ssrc_o,sr_e,DaggerNo);
@@ -298,51 +368,75 @@ int main (int argc, char ** argv)
 
       pickCheckerboard(Even,ssrc_e,sresult);
       pickCheckerboard(Odd ,ssrc_o,sresult);
+
       ssrc_e = ssrc_e - sr_e;
       RealD error = norm2(ssrc_e);
-
       std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm"<<norm2(sr_e) <<std::endl;
-      ssrc_o = ssrc_o - sr_o;
 
+      ssrc_o = ssrc_o - sr_o;
       error+= norm2(ssrc_o);
       std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm"<<norm2(sr_o) <<std::endl;
-      if(error>1.0e-4) { 
+
+      if(( error>1.0e-4) ) { 
 	setCheckerboard(ssrc,ssrc_o);
 	setCheckerboard(ssrc,ssrc_e);
-	std::cout<< ssrc << std::endl;
+	std::cout<< "DIFF\n " <<ssrc << std::endl;
+	setCheckerboard(ssrc,sr_o);
+	setCheckerboard(ssrc,sr_e);
+	std::cout<< "CBRESULT\n " <<ssrc << std::endl;
+	std::cout<< "RESULT\n " <<sresult<< std::endl;
       }
+      assert(error<1.0e-4);
     }
 
+  if(0){
+    std::cout << "Single cache warm call to sDw.Dhop " <<std::endl;
+    for(int i=0;i< PerformanceCounter::NumTypes(); i++ ){
+      sDw.Dhop(ssrc,sresult,0);
+      PerformanceCounter Counter(i);
+      Counter.Start();
+      sDw.Dhop(ssrc,sresult,0);
+      Counter.Stop();
+      Counter.Report();
+    }
+  }
 
   }
 
+
+
   if (1)
   { // Naive wilson dag implementation
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
 
-      //    ref =  src - Gamma(Gamma::GammaX)* src ; // 1+gamma_x
+      //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu+1,1);
       for(int i=0;i<ref._odata.size();i++){
-  ref._odata[i]+= tmp._odata[i] + Gamma(Gmu[mu])*tmp._odata[i]; ;
+	ref._odata[i]+= tmp._odata[i] + Gamma(Gmu[mu])*tmp._odata[i]; ;
       }
 
       tmp =adj(U[mu])*src;
       tmp =Cshift(tmp,mu+1,-1);
       for(int i=0;i<ref._odata.size();i++){
-  ref._odata[i]+= tmp._odata[i] - Gamma(Gmu[mu])*tmp._odata[i]; ;
+	ref._odata[i]+= tmp._odata[i] - Gamma(Gmu[mu])*tmp._odata[i]; ;
       }
     }
     ref = -0.5*ref;
   }
+  //  dump=1;
   Dw.Dhop(src,result,1);
   std::cout << GridLogMessage << "Compare to naive wilson implementation Dag to verify correctness" << std::endl;
   std::cout<<GridLogMessage << "Called DwDag"<<std::endl;
-  std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+  std::cout<<GridLogMessage << "norm dag result "<< norm2(result)<<std::endl;
-  std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+  std::cout<<GridLogMessage << "norm dag ref    "<< norm2(ref)<<std::endl;
   err = ref-result; 
-  std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+  std::cout<<GridLogMessage << "norm dag diff   "<< norm2(err)<<std::endl;
-  assert(norm2(err)<1.0e-4);
+  if((norm2(err)>1.0e-4)){
+	std::cout<< "DAG RESULT\n "  <<ref     << std::endl;
+	std::cout<< "DAG sRESULT\n " <<result  << std::endl;
+	std::cout<< "DAG ERR   \n "  << err    <<std::endl;
+  }
   LatticeFermion src_e (FrbGrid);
   LatticeFermion src_o (FrbGrid);
   LatticeFermion r_e   (FrbGrid);
@@ -350,18 +444,24 @@ int main (int argc, char ** argv)
   LatticeFermion r_eo  (FGrid);
 
 
-  std::cout<<GridLogMessage << "Calling Deo and Doe and assert Deo+Doe == Dunprec"<<std::endl;
+  std::cout<<GridLogMessage << "Calling Deo and Doe and //assert Deo+Doe == Dunprec"<<std::endl;
   pickCheckerboard(Even,src_e,src);
   pickCheckerboard(Odd,src_o,src);
 
   std::cout<<GridLogMessage << "src_e"<<norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "src_o"<<norm2(src_o)<<std::endl;
 
+
+  // S-direction is INNERMOST and takes no part in the parity.
   std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
   std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::DhopEO                "<<std::endl;
   std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
   if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
   if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
@@ -369,6 +469,7 @@ int main (int argc, char ** argv)
   {
     Dw.ZeroCounters();
     FGrid->Barrier();
+    Dw.DhopEO(src_o,r_e,DaggerNo);
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       Dw.DhopEO(src_o,r_e,DaggerNo);
@@ -381,6 +482,7 @@ int main (int argc, char ** argv)
 
     std::cout<<GridLogMessage << "Deo mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "Deo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "Deo mflop/s per node   "<< flops/(t1-t0)/NN<<std::endl;
     Dw.Report();
   }
   Dw.DhopEO(src_o,r_e,DaggerNo);
@@ -396,14 +498,20 @@ int main (int argc, char ** argv)
 
   err = r_eo-result; 
   std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
-  assert(norm2(err)<1.0e-4);
+  if((norm2(err)>1.0e-4)){
+	std::cout<< "Deo RESULT\n " <<r_eo << std::endl;
+	std::cout<< "Deo REF\n " <<result  << std::endl;
+	std::cout<< "Deo ERR   \n " << err <<std::endl;
+  }
 
   pickCheckerboard(Even,src_e,err);
   pickCheckerboard(Odd,src_o,err);
   std::cout<<GridLogMessage << "norm diff even  "<< norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "norm diff odd   "<< norm2(src_o)<<std::endl;
+
   assert(norm2(src_e)<1.0e-4);
   assert(norm2(src_o)<1.0e-4);
-
   Grid_finalize();
+  exit(0);
 }
+

benchmarks/Benchmark_dwf_sweep.cc

@@ -37,11 +37,11 @@ struct scal {
   d internal;
 };
 
-  Gamma::GammaMatrix Gmu [] = {
+  Gamma::Algebra Gmu [] = {
-    Gamma::GammaX,
+    Gamma::Algebra::GammaX,
-    Gamma::GammaY,
+    Gamma::Algebra::GammaY,
-    Gamma::GammaZ,
+    Gamma::Algebra::GammaZ,
-    Gamma::GammaT
+    Gamma::Algebra::GammaT
   };
 
 void benchDw(std::vector<int> & L, int Ls, int threads, int report =0 );

benchmarks/Benchmark_gparity.cc

@@ -0,0 +1,190 @@
+#include <Grid/Grid.h>
+#include <sstream>
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+  d internal;
+};
+
+  Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+  };
+
+typedef typename GparityDomainWallFermionF::FermionField GparityLatticeFermionF;
+typedef typename GparityDomainWallFermionD::FermionField GparityLatticeFermionD;
+
+
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int Ls=16;
+  for(int i=0;i<argc;i++)
+    if(std::string(argv[i]) == "-Ls"){
+      std::stringstream ss(argv[i+1]); ss >> Ls;
+    }
+
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+  std::cout<<GridLogMessage << "Ls = " << Ls << std::endl;
+
+  std::vector<int> latt4 = GridDefaultLatt();
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexF::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+  
+  std::cout << GridLogMessage << "Initialising 4d RNG" << std::endl;
+  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  std::cout << GridLogMessage << "Initialising 5d RNG" << std::endl;
+  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+  GparityLatticeFermionF src   (FGrid); random(RNG5,src);
+  RealD N2 = 1.0/::sqrt(norm2(src));
+  src = src*N2;
+
+  GparityLatticeFermionF result(FGrid); result=zero;
+  GparityLatticeFermionF    ref(FGrid);    ref=zero;
+  GparityLatticeFermionF    tmp(FGrid);
+  GparityLatticeFermionF    err(FGrid);
+
+  std::cout << GridLogMessage << "Drawing gauge field" << std::endl;
+  LatticeGaugeFieldF Umu(UGrid); 
+  SU3::HotConfiguration(RNG4,Umu); 
+  std::cout << GridLogMessage << "Random gauge initialised " << std::endl;
+
+  RealD mass=0.1;
+  RealD M5  =1.8;
+
+  RealD NP = UGrid->_Nprocessors;
+  RealD NN = UGrid->NodeCount();
+
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Benchmarking DomainWallFermion::Dhop                  "<<std::endl;
+  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplexF::Nsimd()<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+
+
+
+  std::cout << GridLogMessage<< "* SINGLE/SINGLE"<<std::endl;
+  GparityDomainWallFermionF Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  int ncall =1000;
+  if (1) {
+    FGrid->Barrier();
+    Dw.ZeroCounters();
+    Dw.Dhop(src,result,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      Dw.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    Dw.Report();
+  }
+
+  std::cout << GridLogMessage<< "* SINGLE/HALF"<<std::endl;
+  GparityDomainWallFermionFH DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  if (1) {
+    FGrid->Barrier();
+    DwH.ZeroCounters();
+    DwH.Dhop(src,result,0);
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwH.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called half prec comms Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    DwH.Report();
+  }
+
+  GridCartesian         * UGrid_d   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_d);
+  GridCartesian         * FGrid_d   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_d);
+  GridRedBlackCartesian * FrbGrid_d = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_d);
+
+  
+  std::cout << GridLogMessage<< "* DOUBLE/DOUBLE"<<std::endl;
+  GparityLatticeFermionD src_d(FGrid_d);
+  precisionChange(src_d,src);
+
+  LatticeGaugeFieldD Umu_d(UGrid_d); 
+  precisionChange(Umu_d,Umu);
+
+  GparityLatticeFermionD result_d(FGrid_d);
+
+  GparityDomainWallFermionD DwD(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5);
+  if (1) {
+    FGrid_d->Barrier();
+    DwD.ZeroCounters();
+    DwD.Dhop(src_d,result_d,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwD.Dhop(src_d,result_d,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid_d->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    DwD.Report();
+  }
+
+  Grid_finalize();
+}
+

benchmarks/Benchmark_memory_asynch.cc

@@ -66,7 +66,8 @@ int main (int argc, char ** argv)
 
     Vec tsum; tsum = zero;
 
-    GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+    GridParallelRNG          pRNG(&Grid);      
+    pRNG.SeedFixedIntegers(std::vector<int>({56,17,89,101}));
 
     std::vector<double> stop(threads);
     Vector<Vec> sum(threads);
@@ -77,8 +78,7 @@ int main (int argc, char ** argv)
     }
 
     double start=usecond();
-PARALLEL_FOR_LOOP
+    parallel_for(int t=0;t<threads;t++){
-    for(int t=0;t<threads;t++){
 
       sum[t] = x[t]._odata[0];
       for(int i=0;i<Nloop;i++){

benchmarks/Benchmark_memory_bandwidth.cc

@@ -55,21 +55,21 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
-  uint64_t lmax=44;
+  uint64_t lmax=96;
-#define NLOOP (1*lmax*lmax*lmax*lmax/vol)
+#define NLOOP (10*lmax*lmax*lmax*lmax/vol)
-  for(int lat=4;lat<=lmax;lat+=4){
+  for(int lat=8;lat<=lmax;lat+=8){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
       uint64_t Nloop=NLOOP;
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeVec z(&Grid); //random(pRNG,z);
+      LatticeVec z(&Grid);// random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
+      LatticeVec x(&Grid);// random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      LatticeVec y(&Grid);// random(pRNG,y);
       double a=2.0;
 
 
@@ -83,7 +83,7 @@ int main (int argc, char ** argv)
       double time = (stop-start)/Nloop*1000;
       
       double flops=vol*Nvec*2;// mul,add
-      double bytes=3*vol*Nvec*sizeof(Real);
+      double bytes=3.0*vol*Nvec*sizeof(Real);
       std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
 
     }
@@ -94,17 +94,17 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
   
-  for(int lat=4;lat<=lmax;lat+=4){
+  for(int lat=8;lat<=lmax;lat+=8){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeVec z(&Grid); //random(pRNG,z);
+      LatticeVec z(&Grid);// random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
+      LatticeVec x(&Grid);// random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      LatticeVec y(&Grid);// random(pRNG,y);
       double a=2.0;
 
       uint64_t Nloop=NLOOP;
@@ -119,7 +119,7 @@ int main (int argc, char ** argv)
       double time = (stop-start)/Nloop*1000;
      
       double flops=vol*Nvec*2;// mul,add
-      double bytes=3*vol*Nvec*sizeof(Real);
+      double bytes=3.0*vol*Nvec*sizeof(Real);
       std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
 
     }
@@ -129,20 +129,20 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
 
-  for(int lat=4;lat<=lmax;lat+=4){
+  for(int lat=8;lat<=lmax;lat+=8){
 
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       uint64_t Nloop=NLOOP;
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeVec z(&Grid); //random(pRNG,z);
+      LatticeVec z(&Grid);// random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
+      LatticeVec x(&Grid);// random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      LatticeVec y(&Grid);// random(pRNG,y);
       RealD a=2.0;
 
 
@@ -154,7 +154,7 @@ int main (int argc, char ** argv)
       double stop=usecond();
       double time = (stop-start)/Nloop*1000;
       
-      double bytes=2*vol*Nvec*sizeof(Real);
+      double bytes=2.0*vol*Nvec*sizeof(Real);
       double flops=vol*Nvec*1;// mul
       std::cout<<GridLogMessage <<std::setprecision(3) << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
 
@@ -166,17 +166,17 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=4;lat<=lmax;lat+=4){
+  for(int lat=8;lat<=lmax;lat+=8){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       uint64_t Nloop=NLOOP;
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
-      LatticeVec z(&Grid); //random(pRNG,z);
+      LatticeVec z(&Grid);// random(pRNG,z);
-      LatticeVec x(&Grid); //random(pRNG,x);
+      LatticeVec x(&Grid);// random(pRNG,x);
-      LatticeVec y(&Grid); //random(pRNG,y);
+      LatticeVec y(&Grid);// random(pRNG,y);
       RealD a=2.0;
       Real nn;      
       double start=usecond();
@@ -187,7 +187,7 @@ int main (int argc, char ** argv)
       double stop=usecond();
       double time = (stop-start)/Nloop*1000;
       
-      double bytes=vol*Nvec*sizeof(Real);
+      double bytes=1.0*vol*Nvec*sizeof(Real);
       double flops=vol*Nvec*2;// mul,add
       std::cout<<GridLogMessage<<std::setprecision(3) << lat<<"\t\t"<<bytes<<"  \t\t"<<bytes/time<<"\t\t"<<flops/time<< "\t\t"<<(stop-start)/1000./1000.<< "\t\t " <<std::endl;
 

benchmarks/Benchmark_mooee.cc

@@ -113,6 +113,36 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "Called " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
     std::cout<<GridLogMessage << "******************"<<std::endl;
 
+#define BENCH_ZDW(A,in,out)			\
+    zDw.CayleyZeroCounters();			\
+    zDw. A (in,out);				\
+    FGrid->Barrier();				\
+    t0=usecond();				\
+    for(int i=0;i<ncall;i++){			\
+      zDw. A (in,out);				\
+    }						\
+    t1=usecond();				\
+    FGrid->Barrier();				\
+    zDw.CayleyReport();							\
+    std::cout<<GridLogMessage << "Called ZDw " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
+    std::cout<<GridLogMessage << "******************"<<std::endl;
+
+#define BENCH_DW_SSC(A,in,out)			\
+    Dw.CayleyZeroCounters();			\
+    Dw. A (in,out);				\
+    FGrid->Barrier();				\
+    t0=usecond();				\
+    for(int i=0;i<ncall;i++){			\
+      __SSC_START ;				\
+      Dw. A (in,out);				\
+      __SSC_STOP ;				\
+    }						\
+    t1=usecond();				\
+    FGrid->Barrier();				\
+    Dw.CayleyReport();					\
+    std::cout<<GridLogMessage << "Called " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
+    std::cout<<GridLogMessage << "******************"<<std::endl;
+
 #define BENCH_DW_MEO(A,in,out)			\
     Dw.CayleyZeroCounters();			\
     Dw. A (in,out,0);				\
@@ -148,9 +178,15 @@ int main (int argc, char ** argv)
     LatticeFermion sref(sFGrid);
     LatticeFermion result(sFGrid);
 
+
     std::cout<<GridLogMessage << "Constructing Vec5D Dw "<<std::endl;
     DomainWallFermionVec5dR Dw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,mass,M5);
 
+    RealD b=1.5;// Scale factor b+c=2, b-c=1
+    RealD c=0.5;
+    std::vector<ComplexD> gamma(Ls,std::complex<double>(1.0,0.0));
+    ZMobiusFermionVec5dR zDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,mass,M5,gamma,b,c);
+
     std::cout<<GridLogMessage << "Calling Dhop "<<std::endl;
     FGrid->Barrier();
 
@@ -173,10 +209,13 @@ int main (int argc, char ** argv)
 
     BENCH_DW_MEO(Dhop    ,src,result);
     BENCH_DW_MEO(DhopEO  ,src_o,r_e);
-    BENCH_DW(Meooe   ,src_o,r_e);
+    BENCH_DW_SSC(Meooe   ,src_o,r_e);
     BENCH_DW(Mooee   ,src_o,r_o);
     BENCH_DW(MooeeInv,src_o,r_o);
 
+    BENCH_ZDW(Mooee   ,src_o,r_o);
+    BENCH_ZDW(MooeeInv,src_o,r_o);
+
   }
 
   Grid_finalize();

benchmarks/Benchmark_staggered.cc

@@ -51,7 +51,7 @@ int main (int argc, char ** argv)
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);
   pRNG.SeedFixedIntegers(seeds);
-  //  pRNG.SeedRandomDevice();
+  //  pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
 
   typedef typename ImprovedStaggeredFermionR::FermionField FermionField; 
   typename ImprovedStaggeredFermionR::ImplParams params; 

benchmarks/Benchmark_su3.cc

@@ -35,13 +35,14 @@ using namespace Grid::QCD;
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
+#define LMAX (64)
 
-  int Nloop=1000;
+  int64_t Nloop=20;
 
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
 
-  int threads = GridThread::GetThreads();
+  int64_t threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
@@ -50,19 +51,19 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid);// random(pRNG,z);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
-      LatticeColourMatrix x(&Grid);// random(pRNG,x);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
-      LatticeColourMatrix y(&Grid);// random(pRNG,y);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
       double start=usecond();
-      for(int i=0;i<Nloop;i++){
+      for(int64_t i=0;i<Nloop;i++){
 	x=x*y;
       }
       double stop=usecond();
@@ -82,20 +83,20 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid); //random(pRNG,z);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
-      LatticeColourMatrix x(&Grid); //random(pRNG,x);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
-      LatticeColourMatrix y(&Grid); //random(pRNG,y);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
       double start=usecond();
-      for(int i=0;i<Nloop;i++){
+      for(int64_t i=0;i<Nloop;i++){
 	z=x*y;
       }
       double stop=usecond();
@@ -113,20 +114,20 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid); //random(pRNG,z);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
-      LatticeColourMatrix x(&Grid); //random(pRNG,x);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
-      LatticeColourMatrix y(&Grid); //random(pRNG,y);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
       double start=usecond();
-      for(int i=0;i<Nloop;i++){
+      for(int64_t i=0;i<Nloop;i++){
 	mult(z,x,y);
       }
       double stop=usecond();
@@ -144,20 +145,20 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
-      int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
-      //      GridParallelRNG          pRNG(&Grid);      pRNG.SeedRandomDevice();
+      GridParallelRNG          pRNG(&Grid);      pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
-      LatticeColourMatrix z(&Grid); //random(pRNG,z);
+      LatticeColourMatrix z(&Grid); random(pRNG,z);
-      LatticeColourMatrix x(&Grid); //random(pRNG,x);
+      LatticeColourMatrix x(&Grid); random(pRNG,x);
-      LatticeColourMatrix y(&Grid); //random(pRNG,y);
+      LatticeColourMatrix y(&Grid); random(pRNG,y);
 
       double start=usecond();
-      for(int i=0;i<Nloop;i++){
+      for(int64_t i=0;i<Nloop;i++){
 	mac(z,x,y);
       }
       double stop=usecond();

benchmarks/Benchmark_wilson.cc

@@ -37,11 +37,11 @@ struct scal {
   d internal;
 };
 
-  Gamma::GammaMatrix Gmu [] = {
+  Gamma::Algebra Gmu [] = {
-    Gamma::GammaX,
+    Gamma::Algebra::GammaX,
-    Gamma::GammaY,
+    Gamma::Algebra::GammaY,
-    Gamma::GammaZ,
+    Gamma::Algebra::GammaZ,
-    Gamma::GammaT
+    Gamma::Algebra::GammaT
   };
 
 bool overlapComms = false;
@@ -69,7 +69,7 @@ int main (int argc, char ** argv)
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);
   pRNG.SeedFixedIntegers(seeds);
-  //  pRNG.SeedRandomDevice();
+  //  pRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
 
   LatticeFermion src   (&Grid); random(pRNG,src);
   LatticeFermion result(&Grid); result=zero;
@@ -106,7 +106,7 @@ int main (int argc, char ** argv)
   { // Naive wilson implementation
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
-      //    ref =  src + Gamma(Gamma::GammaX)* src ; // 1-gamma_x
+      //    ref =  src + Gamma(Gamma::Algebra::GammaX)* src ; // 1-gamma_x
       tmp = U[mu]*Cshift(src,mu,1);
       for(int i=0;i<ref._odata.size();i++){
 	ref._odata[i]+= tmp._odata[i] - Gamma(Gmu[mu])*tmp._odata[i]; ;
@@ -159,7 +159,7 @@ int main (int argc, char ** argv)
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
 
-      //    ref =  src - Gamma(Gamma::GammaX)* src ; // 1+gamma_x
+      //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu,1);
       for(int i=0;i<ref._odata.size();i++){
 	ref._odata[i]+= tmp._odata[i] + Gamma(Gmu[mu])*tmp._odata[i]; ;

benchmarks/Benchmark_wilson_sweep.cc

@@ -30,11 +30,11 @@ struct scal {
   d internal;
 };
 
-Gamma::GammaMatrix Gmu [] = {
+Gamma::Algebra Gmu [] = {
-  Gamma::GammaX,
+    Gamma::Algebra::GammaX,
-  Gamma::GammaY,
+    Gamma::Algebra::GammaY,
-  Gamma::GammaZ,
+    Gamma::Algebra::GammaZ,
-  Gamma::GammaT
+    Gamma::Algebra::GammaT
 };
 
 bool overlapComms = false;

benchmarks/Makefile.am

@@ -1,11 +1,7 @@
 include Make.inc
 
-simple: simple_su3_test.o simple_su3_expr.o simple_simd_test.o
+bench-local: all
-
+	./Benchmark_su3
-EXTRA_LIBRARIES = libsimple_su3_test.a libsimple_su3_expr.a libsimple_simd_test.a
+	./Benchmark_memory_bandwidth
-
+	./Benchmark_wilson
-libsimple_su3_test_a_SOURCES = simple_su3_test.cc
+	./Benchmark_dwf --dslash-unroll
-
-libsimple_su3_expr_a_SOURCES = simple_su3_expr.cc
-
-libsimple_simd_test_a_SOURCES = simple_simd_test.cc

bootstrap.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2'
+EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2'
 
 echo "-- deploying Eigen source..."
 wget ${EIGEN_URL} --no-check-certificate

configure.ac

@@ -1,16 +1,23 @@
 AC_PREREQ([2.63])
-AC_INIT([Grid], [0.6.0], [https://github.com/paboyle/Grid], [Grid])
+AC_INIT([Grid], [0.7.0], [https://github.com/paboyle/Grid], [Grid])
 AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
-AM_INIT_AUTOMAKE(subdir-objects)
+AM_INIT_AUTOMAKE([subdir-objects 1.13])
+AM_EXTRA_RECURSIVE_TARGETS([tests bench])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([lib/Grid.h])
-AC_CONFIG_HEADERS([lib/Config.h])
+AC_CONFIG_HEADERS([lib/Config.h],[sed -i 's|PACKAGE_|GRID_|' lib/Config.h])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
-############### Checks for programs
+################ Get git info
+#AC_REVISION([m4_esyscmd_s([./scripts/configure.commit])])
+
+################ Set flags
+# do not move!
 CXXFLAGS="-O3 $CXXFLAGS"
+
+############### Checks for programs
 AC_PROG_CXX
 AC_PROG_RANLIB
 
@@ -24,6 +31,8 @@ AX_GXX_VERSION
 AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
       [version of g++ that will compile the code])
 
+
+
 ############### Checks for typedefs, structures, and compiler characteristics
 AC_TYPE_SIZE_T
 AC_TYPE_UINT32_T
@@ -45,9 +54,14 @@ AC_CHECK_HEADERS(malloc/malloc.h)
 AC_CHECK_HEADERS(malloc.h)
 AC_CHECK_HEADERS(endian.h)
 AC_CHECK_HEADERS(execinfo.h)
+AC_CHECK_HEADERS(numaif.h)
 AC_CHECK_DECLS([ntohll],[], [], [[#include <arpa/inet.h>]])
 AC_CHECK_DECLS([be64toh],[], [], [[#include <arpa/inet.h>]])
 
+############## Standard libraries
+AC_CHECK_LIB([m],[cos])
+AC_CHECK_LIB([stdc++],[abort])
+
 ############### GMP and MPFR
 AC_ARG_WITH([gmp],
     [AS_HELP_STRING([--with-gmp=prefix],
@@ -67,6 +81,13 @@ AC_ARG_WITH([fftw],
             [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
             [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
 
+############### LIME
+AC_ARG_WITH([lime],
+            [AS_HELP_STRING([--with-lime=prefix],
+            [try this for a non-standard install prefix of the LIME library])],
+            [AM_CXXFLAGS="-I$with_lime/include $AM_CXXFLAGS"]
+            [AM_LDFLAGS="-L$with_lime/lib $AM_LDFLAGS"])
+
 ############### lapack
 AC_ARG_ENABLE([lapack],
     [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])],
@@ -83,6 +104,18 @@ case ${ac_LAPACK} in
         AC_DEFINE([USE_LAPACK],[1],[use LAPACK]);;
 esac
 
+############### FP16 conversions
+AC_ARG_ENABLE([sfw-fp16],
+    [AC_HELP_STRING([--enable-sfw-fp16=yes|no], [enable software fp16 comms])],
+    [ac_SFW_FP16=${enable_sfw_fp16}], [ac_SFW_FP16=yes])
+case ${ac_SFW_FP16} in
+    yes)
+      AC_DEFINE([SFW_FP16],[1],[software conversion to fp16]);;
+    no);;
+    *)
+      AC_MSG_ERROR(["SFW FP16 option not supported ${ac_SFW_FP16}"]);;
+esac
+
 ############### MKL
 AC_ARG_ENABLE([mkl],
     [AC_HELP_STRING([--enable-mkl=yes|no|prefix], [enable Intel MKL for LAPACK & FFTW])],
@@ -99,6 +132,13 @@ case ${ac_MKL} in
         AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
 esac
 
+############### HDF5
+AC_ARG_WITH([hdf5],
+    [AS_HELP_STRING([--with-hdf5=prefix],
+    [try this for a non-standard install prefix of the HDF5 library])],
+    [AM_CXXFLAGS="-I$with_hdf5/include $AM_CXXFLAGS"]
+    [AM_LDFLAGS="-L$with_hdf5/lib $AM_LDFLAGS"])
+
 ############### first-touch
 AC_ARG_ENABLE([numa],
     [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])],
@@ -145,6 +185,29 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
                [have_fftw=true])
 
+AC_SEARCH_LIBS([limeCreateReader], [lime],
+               [AC_DEFINE([HAVE_LIME], [1], [Define to 1 if you have the `LIME' library])]
+               [have_lime=true],
+	       [AC_MSG_WARN(C-LIME library was not found in your system.
+In order to use ILGG file format please install or provide the correct path to your installation
+Info at: http://usqcd.jlab.org/usqcd-docs/c-lime/)])
+
+AC_SEARCH_LIBS([crc32], [z],
+               [AC_DEFINE([HAVE_ZLIB], [1], [Define to 1 if you have the `LIBZ' library])]
+               [have_zlib=true] [LIBS="${LIBS} -lz"],
+	       [AC_MSG_ERROR(zlib library was not found in your system.)])
+
+AC_SEARCH_LIBS([move_pages], [numa],
+               [AC_DEFINE([HAVE_LIBNUMA], [1], [Define to 1 if you have the `LIBNUMA' library])]
+               [have_libnuma=true] [LIBS="${LIBS} -lnuma"],
+	       [AC_MSG_WARN(libnuma library was not found in your system. Some optimisations will not apply)])
+
+AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp],
+               [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])]
+               [have_hdf5=true]
+               [LIBS="${LIBS} -lhdf5"], [], [-lhdf5])
+AM_CONDITIONAL(BUILD_HDF5, [ test "${have_hdf5}X" == "trueX" ])
+
 CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
@@ -163,19 +226,26 @@ case ${ax_cv_cxx_compiler_vendor} in
     case ${ac_SIMD} in
       SSE4)
         AC_DEFINE([SSE4],[1],[SSE4 intrinsics])
-        SIMD_FLAGS='-msse4.2';;
+	case ${ac_SFW_FP16} in
+	  yes)
+	  SIMD_FLAGS='-msse4.2';;
+	  no)
+	  SIMD_FLAGS='-msse4.2 -mf16c';;
+	  *)
+          AC_MSG_ERROR(["SFW_FP16 must be either yes or no value ${ac_SFW_FP16} "]);;
+	esac;;
       AVX)
         AC_DEFINE([AVX1],[1],[AVX intrinsics])
-        SIMD_FLAGS='-mavx';;
+        SIMD_FLAGS='-mavx -mf16c';;
       AVXFMA4)
         AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
-        SIMD_FLAGS='-mavx -mfma4';;
+        SIMD_FLAGS='-mavx -mfma4 -mf16c';;
       AVXFMA)
         AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA3])
-        SIMD_FLAGS='-mavx -mfma';;
+        SIMD_FLAGS='-mavx -mfma -mf16c';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
-        SIMD_FLAGS='-mavx2 -mfma';;
+        SIMD_FLAGS='-mavx2 -mfma -mf16c';;
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
@@ -184,6 +254,7 @@ case ${ax_cv_cxx_compiler_vendor} in
         SIMD_FLAGS='';;
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
+        AC_DEFINE([KNL],[1],[Knights landing processor])
         SIMD_FLAGS='-march=knl';;
       GEN)
         AC_DEFINE([GEN],[1],[generic vector code])
@@ -191,6 +262,9 @@ case ${ax_cv_cxx_compiler_vendor} in
                            [generic SIMD vector width (in bytes)])
         SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)"
         SIMD_FLAGS='';;
+      NEONv8)
+        AC_DEFINE([NEONV8],[1],[ARMv8 NEON])
+        SIMD_FLAGS='-march=armv8-a';;
       QPX|BGQ)
         AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q])
         SIMD_FLAGS='';;
@@ -219,6 +293,7 @@ case ${ax_cv_cxx_compiler_vendor} in
         SIMD_FLAGS='';;
       KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing])
+        AC_DEFINE([KNL],[1],[Knights landing processor])
         SIMD_FLAGS='-xmic-avx512';;
       GEN)
         AC_DEFINE([GEN],[1],[generic vector code])
@@ -256,8 +331,41 @@ case ${ac_PRECISION} in
      double)
        AC_DEFINE([GRID_DEFAULT_PRECISION_DOUBLE],[1],[GRID_DEFAULT_PRECISION is DOUBLE] )
      ;;
+     *)
+     AC_MSG_ERROR([${ac_PRECISION} unsupported --enable-precision option]);
+     ;;
 esac
 
+######################  Shared memory allocation technique under MPI3
+AC_ARG_ENABLE([shm],[AC_HELP_STRING([--enable-shm=shmget|shmopen|hugetlbfs],
+              [Select SHM allocation technique])],[ac_SHM=${enable_shm}],[ac_SHM=shmopen])
+
+case ${ac_SHM} in
+
+     shmget)
+     AC_DEFINE([GRID_MPI3_SHMGET],[1],[GRID_MPI3_SHMGET] )
+     ;;
+
+     shmopen)
+     AC_DEFINE([GRID_MPI3_SHMOPEN],[1],[GRID_MPI3_SHMOPEN] )
+     ;;
+
+     hugetlbfs)
+     AC_DEFINE([GRID_MPI3_SHMMMAP],[1],[GRID_MPI3_SHMMMAP] )
+     ;;
+
+     *)
+     AC_MSG_ERROR([${ac_SHM} unsupported --enable-shm option]);
+     ;;
+esac
+
+######################  Shared base path for SHMMMAP
+AC_ARG_ENABLE([shmpath],[AC_HELP_STRING([--enable-shmpath=path],
+              [Select SHM mmap base path for hugetlbfs])],
+	      [ac_SHMPATH=${enable_shmpath}],
+	      [ac_SHMPATH=/var/lib/hugetlbfs/pagesize-2MB/])
+AC_DEFINE_UNQUOTED([GRID_SHM_PATH],["$ac_SHMPATH"],[Path to a hugetlbfs filesystem for MMAPing])
+
 ############### communication type selection
 AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi3-auto|shmem],
               [Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
@@ -267,14 +375,14 @@ case ${ac_COMMS} in
         AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
         comms_type='none'
      ;;
-     mpi3l*)
-       AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] )
-       comms_type='mpi3l'
-     ;;
      mpi3*)
         AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
         comms_type='mpi3'
      ;;
+     mpit)
+        AC_DEFINE([GRID_COMMS_MPIT],[1],[GRID_COMMS_MPIT] )
+        comms_type='mpit'
+     ;;
      mpi*)
         AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
         comms_type='mpi'
@@ -302,13 +410,13 @@ esac
 AM_CONDITIONAL(BUILD_COMMS_SHMEM, [ test "${comms_type}X" == "shmemX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI,   [ test "${comms_type}X" == "mpiX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI3,  [ test "${comms_type}X" == "mpi3X" ] )
-AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] )
+AM_CONDITIONAL(BUILD_COMMS_MPIT,  [ test "${comms_type}X" == "mpitX" ] )
 AM_CONDITIONAL(BUILD_COMMS_NONE,  [ test "${comms_type}X" == "noneX" ])
 
 ############### RNG selection
-AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937],\
+AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937|sitmo],\
 	            [Select Random Number Generator to be used])],\
-	            [ac_RNG=${enable_rng}],[ac_RNG=ranlux48])
+	            [ac_RNG=${enable_rng}],[ac_RNG=sitmo])
 
 case ${ac_RNG} in
      ranlux48)
@@ -317,6 +425,9 @@ case ${ac_RNG} in
      mt19937)
       AC_DEFINE([RNG_MT19937],[1],[RNG_MT19937] )
      ;;
+     sitmo)
+      AC_DEFINE([RNG_SITMO],[1],[RNG_SITMO] )
+     ;;
      *)
       AC_MSG_ERROR([${ac_RNG} unsupported --enable-rng option]);
      ;;
@@ -368,29 +479,31 @@ DX_INIT_DOXYGEN([$PACKAGE_NAME], [doxygen.cfg])
 
 ############### Ouput
 cwd=`pwd -P`; cd ${srcdir}; abs_srcdir=`pwd -P`; cd ${cwd}
+GRID_CXXFLAGS="$AM_CXXFLAGS $CXXFLAGS"
+GRID_LDFLAGS="$AM_LDFLAGS $LDFLAGS"
+GRID_LIBS=$LIBS
+GRID_SHORT_SHA=`git rev-parse --short HEAD`
+GRID_SHA=`git rev-parse HEAD`
+GRID_BRANCH=`git rev-parse --abbrev-ref HEAD`
 AM_CXXFLAGS="-I${abs_srcdir}/include $AM_CXXFLAGS"
 AM_CFLAGS="-I${abs_srcdir}/include $AM_CFLAGS"
 AM_LDFLAGS="-L${cwd}/lib $AM_LDFLAGS"
 AC_SUBST([AM_CFLAGS])
 AC_SUBST([AM_CXXFLAGS])
 AC_SUBST([AM_LDFLAGS])
-AC_CONFIG_FILES(Makefile)
+AC_SUBST([GRID_CXXFLAGS])
-AC_CONFIG_FILES(lib/Makefile)
+AC_SUBST([GRID_LDFLAGS])
-AC_CONFIG_FILES(tests/Makefile)
+AC_SUBST([GRID_LIBS])
-AC_CONFIG_FILES(tests/IO/Makefile)
+AC_SUBST([GRID_SHA])
-AC_CONFIG_FILES(tests/core/Makefile)
+AC_SUBST([GRID_BRANCH])
-AC_CONFIG_FILES(tests/debug/Makefile)
+
-AC_CONFIG_FILES(tests/forces/Makefile)
+git_commit=`cd $srcdir && ./scripts/configure.commit`
-AC_CONFIG_FILES(tests/hmc/Makefile)
-AC_CONFIG_FILES(tests/solver/Makefile)
-AC_CONFIG_FILES(tests/qdpxx/Makefile)
-AC_CONFIG_FILES(benchmarks/Makefile)
-AC_OUTPUT
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Summary of configuration for $PACKAGE v$VERSION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
+----- GIT VERSION -------------------------------------
+$git_commit
 ----- PLATFORM ----------------------------------------
 architecture (build)        : $build_cpu
 os (build)                  : $build_os
@@ -402,11 +515,16 @@ compiler version            : ${ax_cv_gxx_version}
 SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
 Threading                   : ${ac_openmp}
 Communications type         : ${comms_type}
+Shared memory allocator     : ${ac_SHM}
+Shared memory mmap path     : ${ac_SHMPATH}
 Default precision           : ${ac_PRECISION}
+Software FP16 conversion    : ${ac_SFW_FP16}
 RNG choice                  : ${ac_RNG}
 GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
 LAPACK                      : ${ac_LAPACK}
 FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
+LIME (ILDG support)         : `if test "x$have_lime" = xtrue; then echo yes; else echo no; fi`
+HDF5                        : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi`
 build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
 CXXFLAGS:
@@ -415,7 +533,32 @@ LDFLAGS:
 `echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
 LIBS:
 `echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
--------------------------------------------------------" > config.summary
+-------------------------------------------------------" > grid.configure.summary
+
+GRID_SUMMARY="`cat grid.configure.summary`"
+AM_SUBST_NOTMAKE([GRID_SUMMARY])
+AC_SUBST([GRID_SUMMARY])
+
+AC_CONFIG_FILES([grid-config], [chmod +x grid-config])
+AC_CONFIG_FILES(Makefile)
+AC_CONFIG_FILES(lib/Makefile)
+AC_CONFIG_FILES(tests/Makefile)
+AC_CONFIG_FILES(tests/IO/Makefile)
+AC_CONFIG_FILES(tests/core/Makefile)
+AC_CONFIG_FILES(tests/debug/Makefile)
+AC_CONFIG_FILES(tests/forces/Makefile)
+AC_CONFIG_FILES(tests/hadrons/Makefile)
+AC_CONFIG_FILES(tests/hmc/Makefile)
+AC_CONFIG_FILES(tests/solver/Makefile)
+AC_CONFIG_FILES(tests/smearing/Makefile)
+AC_CONFIG_FILES(tests/qdpxx/Makefile)
+AC_CONFIG_FILES(tests/testu01/Makefile)
+AC_CONFIG_FILES(benchmarks/Makefile)
+AC_CONFIG_FILES(extras/Makefile)
+AC_CONFIG_FILES(extras/Hadrons/Makefile)
+AC_OUTPUT
+
 echo ""
-cat config.summary
+cat grid.configure.summary
 echo ""
+

extras/Hadrons/Application.cc

@@ -0,0 +1,318 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Application.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Application.hpp>
+#include <Grid/Hadrons/GeneticScheduler.hpp>
+
+using namespace Grid;
+using namespace QCD;
+using namespace Hadrons;
+
+#define BIG_SEP "==============="
+#define SEP     "---------------"
+
+/******************************************************************************
+ *                       Application implementation                           *
+ ******************************************************************************/
+// constructors ////////////////////////////////////////////////////////////////
+Application::Application(void)
+{
+    LOG(Message) << "Modules available:" << std::endl;
+    auto list = ModuleFactory::getInstance().getBuilderList();
+    for (auto &m: list)
+    {
+        LOG(Message) << "  " << m << std::endl;
+    }
+    auto dim = GridDefaultLatt(), mpi = GridDefaultMpi(), loc(dim);
+    locVol_ = 1;
+    for (unsigned int d = 0; d < dim.size(); ++d)
+    {
+        loc[d]  /= mpi[d];
+        locVol_ *= loc[d];
+    }
+    LOG(Message) << "Global lattice: " << dim << std::endl;
+    LOG(Message) << "MPI partition : " << mpi << std::endl;
+    LOG(Message) << "Local lattice : " << loc << std::endl;
+}
+
+Application::Application(const Application::GlobalPar &par)
+: Application()
+{
+    setPar(par);
+}
+
+Application::Application(const std::string parameterFileName)
+: Application()
+{
+    parameterFileName_ = parameterFileName;
+}
+
+// environment shortcut ////////////////////////////////////////////////////////
+Environment & Application::env(void) const
+{
+    return Environment::getInstance();
+}
+
+// access //////////////////////////////////////////////////////////////////////
+void Application::setPar(const Application::GlobalPar &par)
+{
+    par_ = par;
+    env().setSeed(strToVec<int>(par_.seed));
+}
+
+const Application::GlobalPar & Application::getPar(void)
+{
+    return par_;
+}
+
+// execute /////////////////////////////////////////////////////////////////////
+void Application::run(void)
+{
+    if (!parameterFileName_.empty() and (env().getNModule() == 0))
+    {
+        parseParameterFile(parameterFileName_);
+    }
+    if (!scheduled_)
+    {
+        schedule();
+    }
+    printSchedule();
+    configLoop();
+}
+
+// parse parameter file ////////////////////////////////////////////////////////
+class ObjectId: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(ObjectId,
+                                    std::string, name,
+                                    std::string, type);
+};
+
+void Application::parseParameterFile(const std::string parameterFileName)
+{
+    XmlReader reader(parameterFileName);
+    GlobalPar par;
+    ObjectId  id;
+    
+    LOG(Message) << "Building application from '" << parameterFileName << "'..." << std::endl;
+    read(reader, "parameters", par);
+    setPar(par);
+    push(reader, "modules");
+    push(reader, "module");
+    do
+    {
+        read(reader, "id", id);
+        env().createModule(id.name, id.type, reader);
+    } while (reader.nextElement("module"));
+    pop(reader);
+    pop(reader);
+}
+
+void Application::saveParameterFile(const std::string parameterFileName)
+{
+    XmlWriter          writer(parameterFileName);
+    ObjectId           id;
+    const unsigned int nMod = env().getNModule();
+    
+    LOG(Message) << "Saving application to '" << parameterFileName << "'..." << std::endl;
+    write(writer, "parameters", getPar());
+    push(writer, "modules");
+    for (unsigned int i = 0; i < nMod; ++i)
+    {
+        push(writer, "module");
+        id.name = env().getModuleName(i);
+        id.type = env().getModule(i)->getRegisteredName();
+        write(writer, "id", id);
+        env().getModule(i)->saveParameters(writer, "options");
+        pop(writer);
+    }
+    pop(writer);
+    pop(writer);
+}
+
+// schedule computation ////////////////////////////////////////////////////////
+#define MEM_MSG(size)\
+sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
+
+#define DEFINE_MEMPEAK \
+GeneticScheduler<unsigned int>::ObjFunc memPeak = \
+[this](const std::vector<unsigned int> &program)\
+{\
+    unsigned int memPeak;\
+    bool         msg;\
+    \
+    msg = HadronsLogMessage.isActive();\
+    HadronsLogMessage.Active(false);\
+    env().dryRun(true);\
+    memPeak = env().executeProgram(program);\
+    env().dryRun(false);\
+    env().freeAll();\
+    HadronsLogMessage.Active(true);\
+    \
+    return memPeak;\
+}
+
+void Application::schedule(void)
+{
+    DEFINE_MEMPEAK;
+    
+    // build module dependency graph
+    LOG(Message) << "Building module graph..." << std::endl;
+    auto graph = env().makeModuleGraph();
+    auto con = graph.getConnectedComponents();
+    
+    // constrained topological sort using a genetic algorithm
+    LOG(Message) << "Scheduling computation..." << std::endl;
+    LOG(Message) << "               #module= " << graph.size() << std::endl;
+    LOG(Message) << "       population size= " << par_.genetic.popSize << std::endl;
+    LOG(Message) << "       max. generation= " << par_.genetic.maxGen << std::endl;
+    LOG(Message) << "  max. cst. generation= " << par_.genetic.maxCstGen << std::endl;
+    LOG(Message) << "         mutation rate= " << par_.genetic.mutationRate << std::endl;
+    
+    unsigned int                               k = 0, gen, prevPeak, nCstPeak = 0;
+    std::random_device                         rd;
+    GeneticScheduler<unsigned int>::Parameters par;
+    
+    par.popSize      = par_.genetic.popSize;
+    par.mutationRate = par_.genetic.mutationRate;
+    par.seed         = rd();
+    memPeak_         = 0;
+    CartesianCommunicator::BroadcastWorld(0, &(par.seed), sizeof(par.seed));
+    for (unsigned int i = 0; i < con.size(); ++i)
+    {
+        GeneticScheduler<unsigned int> scheduler(con[i], memPeak, par);
+        
+        gen = 0;
+        do
+        {
+            LOG(Debug) << "Generation " << gen << ":" << std::endl;
+            scheduler.nextGeneration();
+            if (gen != 0)
+            {
+                if (prevPeak == scheduler.getMinValue())
+                {
+                    nCstPeak++;
+                }
+                else
+                {
+                    nCstPeak = 0;
+                }
+            }
+            
+            prevPeak = scheduler.getMinValue();
+            if (gen % 10 == 0)
+            {
+                LOG(Iterative) << "Generation " << gen << ": "
+                               << MEM_MSG(scheduler.getMinValue()) << std::endl;
+            }
+            
+            gen++;
+        } while ((gen < par_.genetic.maxGen)
+                 and (nCstPeak < par_.genetic.maxCstGen));
+        auto &t = scheduler.getMinSchedule();
+        if (scheduler.getMinValue() > memPeak_)
+        {
+            memPeak_ = scheduler.getMinValue();
+        }
+        for (unsigned int j = 0; j < t.size(); ++j)
+        {
+            program_.push_back(t[j]);
+        }
+    }
+    scheduled_ = true;
+}
+
+void Application::saveSchedule(const std::string filename)
+{
+    TextWriter               writer(filename);
+    std::vector<std::string> program;
+    
+    if (!scheduled_)
+    {
+        HADRON_ERROR("Computation not scheduled");
+    }
+    LOG(Message) << "Saving current schedule to '" << filename << "'..."
+                 << std::endl;
+    for (auto address: program_)
+    {
+        program.push_back(env().getModuleName(address));
+    }
+    write(writer, "schedule", program);
+}
+
+void Application::loadSchedule(const std::string filename)
+{
+    DEFINE_MEMPEAK;
+    
+    TextReader               reader(filename);
+    std::vector<std::string> program;
+    
+    LOG(Message) << "Loading schedule from '" << filename << "'..."
+                 << std::endl;
+    read(reader, "schedule", program);
+    program_.clear();
+    for (auto &name: program)
+    {
+        program_.push_back(env().getModuleAddress(name));
+    }
+    scheduled_ = true;
+    memPeak_   = memPeak(program_);
+}
+
+void Application::printSchedule(void)
+{
+    if (!scheduled_)
+    {
+        HADRON_ERROR("Computation not scheduled");
+    }
+    LOG(Message) << "Schedule (memory peak: " << MEM_MSG(memPeak_) << "):"
+                 << std::endl;
+    for (unsigned int i = 0; i < program_.size(); ++i)
+    {
+        LOG(Message) << std::setw(4) << i + 1 << ": "
+                     << env().getModuleName(program_[i]) << std::endl;
+    }
+}
+
+// loop on configurations //////////////////////////////////////////////////////
+void Application::configLoop(void)
+{
+    auto range = par_.trajCounter;
+    
+    for (unsigned int t = range.start; t < range.end; t += range.step)
+    {
+        LOG(Message) << BIG_SEP << " Starting measurement for trajectory " << t
+                     << " " << BIG_SEP << std::endl;
+        env().setTrajectory(t);
+        env().executeProgram(program_);
+    }
+    LOG(Message) << BIG_SEP << " End of measurement " << BIG_SEP << std::endl;
+    env().freeAll();
+}

extras/Hadrons/Application.hpp

@@ -0,0 +1,132 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Application.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Application_hpp_
+#define Hadrons_Application_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Environment.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+#include <Grid/Hadrons/Modules.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         Main program manager                               *
+ ******************************************************************************/
+class Application
+{
+public:
+    class TrajRange: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(TrajRange,
+                                        unsigned int, start,
+                                        unsigned int, end,
+                                        unsigned int, step);
+    };
+    class GeneticPar: Serializable
+    {
+    public:
+        GeneticPar(void):
+            popSize{20}, maxGen{1000}, maxCstGen{100}, mutationRate{.1} {};
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(GeneticPar,
+                                        unsigned int, popSize,
+                                        unsigned int, maxGen,
+                                        unsigned int, maxCstGen,
+                                        double      , mutationRate);
+    };
+    class GlobalPar: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(GlobalPar,
+                                        TrajRange,   trajCounter,
+                                        GeneticPar,  genetic,
+                                        std::string, seed);
+    };
+public:
+    // constructors
+    Application(void);
+    Application(const GlobalPar &par);
+    Application(const std::string parameterFileName);
+    // destructor
+    virtual ~Application(void) = default;
+    // access
+    void              setPar(const GlobalPar &par);
+    const GlobalPar & getPar(void);
+    // module creation
+    template <typename M>
+    void createModule(const std::string name);
+    template <typename M>
+    void createModule(const std::string name, const typename M::Par &par);
+    // execute
+    void run(void);
+    // XML parameter file I/O
+    void parseParameterFile(const std::string parameterFileName);
+    void saveParameterFile(const std::string parameterFileName);
+    // schedule computation
+    void schedule(void);
+    void saveSchedule(const std::string filename);
+    void loadSchedule(const std::string filename);
+    void printSchedule(void);
+    // loop on configurations
+    void configLoop(void);
+private:
+    // environment shortcut
+    Environment & env(void) const;
+private:
+    long unsigned int         locVol_;
+    std::string               parameterFileName_{""};
+    GlobalPar                 par_;
+    std::vector<unsigned int> program_;
+    Environment::Size         memPeak_;
+    bool                      scheduled_{false};
+};
+
+/******************************************************************************
+ *                     Application template implementation                    *
+ ******************************************************************************/
+// module creation /////////////////////////////////////////////////////////////
+template <typename M>
+void Application::createModule(const std::string name)
+{
+    env().createModule<M>(name);
+}
+
+template <typename M>
+void Application::createModule(const std::string name,
+                               const typename M::Par &par)
+{
+    env().createModule<M>(name, par);
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Application_hpp_

extras/Hadrons/Environment.cc

@@ -0,0 +1,793 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Environment.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Environment.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+using namespace Grid;
+using namespace QCD;
+using namespace Hadrons;
+
+/******************************************************************************
+ *                       Environment implementation                           *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+Environment::Environment(void)
+{
+    dim_ = GridDefaultLatt();
+    nd_  = dim_.size();
+    grid4d_.reset(SpaceTimeGrid::makeFourDimGrid(
+        dim_, GridDefaultSimd(nd_, vComplex::Nsimd()),
+        GridDefaultMpi()));
+    gridRb4d_.reset(SpaceTimeGrid::makeFourDimRedBlackGrid(grid4d_.get()));
+    auto loc = getGrid()->LocalDimensions();
+    locVol_ = 1;
+    for (unsigned int d = 0; d < loc.size(); ++d)
+    {
+        locVol_ *= loc[d];
+    }
+    rng4d_.reset(new GridParallelRNG(grid4d_.get()));
+}
+
+// dry run /////////////////////////////////////////////////////////////////////
+void Environment::dryRun(const bool isDry)
+{
+    dryRun_ = isDry;
+}
+
+bool Environment::isDryRun(void) const
+{
+    return dryRun_;
+}
+
+// trajectory number ///////////////////////////////////////////////////////////
+void Environment::setTrajectory(const unsigned int traj)
+{
+    traj_ = traj;
+}
+
+unsigned int Environment::getTrajectory(void) const
+{
+    return traj_;
+}
+
+// grids ///////////////////////////////////////////////////////////////////////
+void Environment::createGrid(const unsigned int Ls)
+{
+    if (grid5d_.find(Ls) == grid5d_.end())
+    {
+        auto g = getGrid();
+        
+        grid5d_[Ls].reset(SpaceTimeGrid::makeFiveDimGrid(Ls, g));
+        gridRb5d_[Ls].reset(SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, g));
+    }
+}
+
+GridCartesian * Environment::getGrid(const unsigned int Ls) const
+{
+    try
+    {
+        if (Ls == 1)
+        {
+            return grid4d_.get();
+        }
+        else
+        {
+            return grid5d_.at(Ls).get();
+        }
+    }
+    catch(std::out_of_range &)
+    {
+        HADRON_ERROR("no grid with Ls= " << Ls);
+    }
+}
+
+GridRedBlackCartesian * Environment::getRbGrid(const unsigned int Ls) const
+{
+    try
+    {
+        if (Ls == 1)
+        {
+            return gridRb4d_.get();
+        }
+        else
+        {
+            return gridRb5d_.at(Ls).get();
+        }
+    }
+    catch(std::out_of_range &)
+    {
+        HADRON_ERROR("no red-black 5D grid with Ls= " << Ls);
+    }
+}
+
+unsigned int Environment::getNd(void) const
+{
+    return nd_;
+}
+
+std::vector<int> Environment::getDim(void) const
+{
+    return dim_;
+}
+
+int Environment::getDim(const unsigned int mu) const
+{
+    return dim_[mu];
+}
+
+// random number generator /////////////////////////////////////////////////////
+void Environment::setSeed(const std::vector<int> &seed)
+{
+    rng4d_->SeedFixedIntegers(seed);
+}
+
+GridParallelRNG * Environment::get4dRng(void) const
+{
+    return rng4d_.get();
+}
+
+// module management ///////////////////////////////////////////////////////////
+void Environment::pushModule(Environment::ModPt &pt)
+{
+    std::string name = pt->getName();
+    
+    if (!hasModule(name))
+    {
+        std::vector<unsigned int> inputAddress;
+        unsigned int              address;
+        ModuleInfo                m;
+        
+        m.data = std::move(pt);
+        m.type = typeIdPt(*m.data.get());
+        m.name = name;
+        auto input  = m.data->getInput();
+        for (auto &in: input)
+        {
+            if (!hasObject(in))
+            {
+                addObject(in , -1);
+            }
+            m.input.push_back(objectAddress_[in]);
+        }
+        auto output = m.data->getOutput();
+        module_.push_back(std::move(m));
+        address              = static_cast<unsigned int>(module_.size() - 1);
+        moduleAddress_[name] = address;
+        for (auto &out: output)
+        {
+            if (!hasObject(out))
+            {
+                addObject(out, address);
+            }
+            else
+            {
+                if (object_[objectAddress_[out]].module < 0)
+                {
+                    object_[objectAddress_[out]].module = address;
+                }
+                else
+                {
+                    HADRON_ERROR("object '" + out
+                                 + "' is already produced by module '"
+                                 + module_[object_[getObjectAddress(out)].module].name
+                                 + "' (while pushing module '" + name + "')");
+                }
+            }
+        }
+    }
+    else
+    {
+        HADRON_ERROR("module '" + name + "' already exists");
+    }
+}
+
+unsigned int Environment::getNModule(void) const
+{
+    return module_.size();
+}
+
+void Environment::createModule(const std::string name, const std::string type,
+                               XmlReader &reader)
+{
+    auto &factory = ModuleFactory::getInstance();
+    auto pt       = factory.create(type, name);
+    
+    pt->parseParameters(reader, "options");
+    pushModule(pt);
+}
+
+ModuleBase * Environment::getModule(const unsigned int address) const
+{
+    if (hasModule(address))
+    {
+        return module_[address].data.get();
+    }
+    else
+    {
+        HADRON_ERROR("no module with address " + std::to_string(address));
+    }
+}
+
+ModuleBase * Environment::getModule(const std::string name) const
+{
+    return getModule(getModuleAddress(name));
+}
+
+unsigned int Environment::getModuleAddress(const std::string name) const
+{
+    if (hasModule(name))
+    {
+        return moduleAddress_.at(name);
+    }
+    else
+    {
+        HADRON_ERROR("no module with name '" + name + "'");
+    }
+}
+
+std::string Environment::getModuleName(const unsigned int address) const
+{
+    if (hasModule(address))
+    {
+        return module_[address].name;
+    }
+    else
+    {
+        HADRON_ERROR("no module with address " + std::to_string(address));
+    }
+}
+
+std::string Environment::getModuleType(const unsigned int address) const
+{
+    if (hasModule(address))
+    {
+        return typeName(module_[address].type);
+    }
+    else
+    {
+        HADRON_ERROR("no module with address " + std::to_string(address));
+    }
+}
+
+std::string Environment::getModuleType(const std::string name) const
+{
+    return getModuleType(getModuleAddress(name));
+}
+
+std::string Environment::getModuleNamespace(const unsigned int address) const
+{
+    std::string type = getModuleType(address), ns;
+    
+    auto pos2 = type.rfind("::");
+    auto pos1 = type.rfind("::", pos2 - 2);
+    
+    return type.substr(pos1 + 2, pos2 - pos1 - 2);
+}
+
+std::string Environment::getModuleNamespace(const std::string name) const
+{
+    return getModuleNamespace(getModuleAddress(name));
+}
+
+bool Environment::hasModule(const unsigned int address) const
+{
+    return (address < module_.size());
+}
+
+bool Environment::hasModule(const std::string name) const
+{
+    return (moduleAddress_.find(name) != moduleAddress_.end());
+}
+
+Graph<unsigned int> Environment::makeModuleGraph(void) const
+{
+    Graph<unsigned int> moduleGraph;
+    
+    for (unsigned int i = 0; i < module_.size(); ++i)
+    {
+        moduleGraph.addVertex(i);
+        for (auto &j: module_[i].input)
+        {
+            moduleGraph.addEdge(object_[j].module, i);
+        }
+    }
+    
+    return moduleGraph;
+}
+
+#define BIG_SEP "==============="
+#define SEP     "---------------"
+#define MEM_MSG(size)\
+sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
+
+Environment::Size
+Environment::executeProgram(const std::vector<unsigned int> &p)
+{
+    Size                                memPeak = 0, sizeBefore, sizeAfter;
+    std::vector<std::set<unsigned int>> freeProg;
+    bool                                continueCollect, nothingFreed;
+    
+    // build garbage collection schedule
+    freeProg.resize(p.size());
+    for (unsigned int i = 0; i < object_.size(); ++i)
+    {
+        auto pred = [i, this](const unsigned int j)
+        {
+            auto &in = module_[j].input;
+            auto it  = std::find(in.begin(), in.end(), i);
+            
+            return (it != in.end()) or (j == object_[i].module);
+        };
+        auto it = std::find_if(p.rbegin(), p.rend(), pred);
+        if (it != p.rend())
+        {
+            freeProg[p.rend() - it - 1].insert(i);
+        }
+    }
+    
+    // program execution
+    for (unsigned int i = 0; i < p.size(); ++i)
+    {
+        // execute module
+        if (!isDryRun())
+        {
+            LOG(Message) << SEP << " Measurement step " << i+1 << "/"
+                         << p.size() << " (module '" << module_[p[i]].name
+                         << "') " << SEP << std::endl;
+        }
+        (*module_[p[i]].data)();
+        sizeBefore = getTotalSize();
+        // print used memory after execution
+        if (!isDryRun())
+        {
+            LOG(Message) << "Allocated objects: " << MEM_MSG(sizeBefore)
+                         << std::endl;
+        }
+        if (sizeBefore > memPeak)
+        {
+            memPeak = sizeBefore;
+        }
+        // garbage collection for step i
+        if (!isDryRun())
+        {
+            LOG(Message) << "Garbage collection..." << std::endl;
+        }
+        nothingFreed = true;
+        do
+        {
+            continueCollect = false;
+            auto toFree = freeProg[i];
+            for (auto &j: toFree)
+            {
+                // continue garbage collection while there are still
+                // objects without owners
+                continueCollect = continueCollect or !hasOwners(j);
+                if(freeObject(j))
+                {
+                    // if an object has been freed, remove it from
+                    // the garbage collection schedule
+                    freeProg[i].erase(j);
+                    nothingFreed = false;
+                }
+            }
+        } while (continueCollect);
+        // any remaining objects in step i garbage collection schedule
+        // is scheduled for step i + 1
+        if (i + 1 < p.size())
+        {
+            for (auto &j: freeProg[i])
+            {
+                freeProg[i + 1].insert(j);
+            }
+        }
+        // print used memory after garbage collection if necessary
+        if (!isDryRun())
+        {
+            sizeAfter = getTotalSize();
+            if (sizeBefore != sizeAfter)
+            {
+                LOG(Message) << "Allocated objects: " << MEM_MSG(sizeAfter)
+                             << std::endl;
+            }
+            else
+            {
+                LOG(Message) << "Nothing to free" << std::endl;
+            }
+        }
+    }
+    
+    return memPeak;
+}
+
+Environment::Size Environment::executeProgram(const std::vector<std::string> &p)
+{
+    std::vector<unsigned int> pAddress;
+    
+    for (auto &n: p)
+    {
+        pAddress.push_back(getModuleAddress(n));
+    }
+    
+    return executeProgram(pAddress);
+}
+
+// general memory management ///////////////////////////////////////////////////
+void Environment::addObject(const std::string name, const int moduleAddress)
+{
+    if (!hasObject(name))
+    {
+        ObjInfo info;
+        
+        info.name   = name;
+        info.module = moduleAddress;
+        object_.push_back(std::move(info));
+        objectAddress_[name] = static_cast<unsigned int>(object_.size() - 1);
+    }
+    else
+    {
+        HADRON_ERROR("object '" + name + "' already exists");
+    }
+}
+
+void Environment::registerObject(const unsigned int address,
+                                 const unsigned int size, const unsigned int Ls)
+{
+    if (!hasRegisteredObject(address))
+    {
+        if (hasObject(address))
+        {
+            object_[address].size         = size;
+            object_[address].Ls           = Ls;
+            object_[address].isRegistered = true;
+        }
+        else
+        {
+            HADRON_ERROR("no object with address " + std::to_string(address));
+        }
+    }
+    else
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " already registered");
+    }
+}
+
+void Environment::registerObject(const std::string name,
+                                 const unsigned int size, const unsigned int Ls)
+{
+    if (!hasObject(name))
+    {
+        addObject(name);
+    }
+    registerObject(getObjectAddress(name), size, Ls);
+}
+
+unsigned int Environment::getObjectAddress(const std::string name) const
+{
+    if (hasObject(name))
+    {
+        return objectAddress_.at(name);
+    }
+    else
+    {
+        HADRON_ERROR("no object with name '" + name + "'");
+    }
+}
+
+std::string Environment::getObjectName(const unsigned int address) const
+{
+    if (hasObject(address))
+    {
+        return object_[address].name;
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+std::string Environment::getObjectType(const unsigned int address) const
+{
+    if (hasRegisteredObject(address))
+    {
+        if (object_[address].type)
+        {
+            return typeName(object_[address].type);
+        }
+        else
+        {
+            return "<no type>";
+        }
+    }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " exists but is not registered");
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+std::string Environment::getObjectType(const std::string name) const
+{
+    return getObjectType(getObjectAddress(name));
+}
+
+Environment::Size Environment::getObjectSize(const unsigned int address) const
+{
+    if (hasRegisteredObject(address))
+    {
+        return object_[address].size;
+    }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " exists but is not registered");
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+Environment::Size Environment::getObjectSize(const std::string name) const
+{
+    return getObjectSize(getObjectAddress(name));
+}
+
+unsigned int Environment::getObjectModule(const unsigned int address) const
+{
+    if (hasObject(address))
+    {
+        return object_[address].module;
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+unsigned int Environment::getObjectModule(const std::string name) const
+{
+    return getObjectModule(getObjectAddress(name));
+}
+
+unsigned int Environment::getObjectLs(const unsigned int address) const
+{
+    if (hasRegisteredObject(address))
+    {
+        return object_[address].Ls;
+    }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " exists but is not registered");
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+unsigned int Environment::getObjectLs(const std::string name) const
+{
+    return getObjectLs(getObjectAddress(name));
+}
+
+bool Environment::hasObject(const unsigned int address) const
+{
+    return (address < object_.size());
+}
+
+bool Environment::hasObject(const std::string name) const
+{
+    auto it = objectAddress_.find(name);
+    
+    return ((it != objectAddress_.end()) and hasObject(it->second));
+}
+
+bool Environment::hasRegisteredObject(const unsigned int address) const
+{
+    if (hasObject(address))
+    {
+        return object_[address].isRegistered;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool Environment::hasRegisteredObject(const std::string name) const
+{
+    if (hasObject(name))
+    {
+        return hasRegisteredObject(getObjectAddress(name));
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool Environment::hasCreatedObject(const unsigned int address) const
+{
+    if (hasObject(address))
+    {
+        return (object_[address].data != nullptr);
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool Environment::hasCreatedObject(const std::string name) const
+{
+    if (hasObject(name))
+    {
+        return hasCreatedObject(getObjectAddress(name));
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool Environment::isObject5d(const unsigned int address) const
+{
+    return (getObjectLs(address) > 1);
+}
+
+bool Environment::isObject5d(const std::string name) const
+{
+    return (getObjectLs(name) > 1);
+}
+
+Environment::Size Environment::getTotalSize(void) const
+{
+    Environment::Size size = 0;
+    
+    for (auto &o: object_)
+    {
+        if (o.isRegistered)
+        {
+            size += o.size;
+        }
+    }
+    
+    return size;
+}
+
+void Environment::addOwnership(const unsigned int owner,
+                               const unsigned int property)
+{
+    if (hasObject(property))
+    {
+        object_[property].owners.insert(owner);
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(property));
+    }
+    if (hasObject(owner))
+    {
+        object_[owner].properties.insert(property);
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(owner));
+    }
+}
+
+void Environment::addOwnership(const std::string owner,
+                               const std::string property)
+{
+    addOwnership(getObjectAddress(owner), getObjectAddress(property));
+}
+
+bool Environment::hasOwners(const unsigned int address) const
+{
+    
+    if (hasObject(address))
+    {
+        return (!object_[address].owners.empty());
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+bool Environment::hasOwners(const std::string name) const
+{
+    return hasOwners(getObjectAddress(name));
+}
+
+bool Environment::freeObject(const unsigned int address)
+{
+    if (!hasOwners(address))
+    {
+        if (!isDryRun() and object_[address].isRegistered)
+        {
+            LOG(Message) << "Destroying object '" << object_[address].name
+                         << "'" << std::endl;
+        }
+        for (auto &p: object_[address].properties)
+        {
+            object_[p].owners.erase(address);
+        }
+        object_[address].size         = 0;
+        object_[address].Ls           = 0;
+        object_[address].isRegistered = false;
+        object_[address].type         = nullptr;
+        object_[address].owners.clear();
+        object_[address].properties.clear();
+        object_[address].data.reset(nullptr);
+        
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool Environment::freeObject(const std::string name)
+{
+    return freeObject(getObjectAddress(name));
+}
+
+void Environment::freeAll(void)
+{
+    for (unsigned int i = 0; i < object_.size(); ++i)
+    {
+        freeObject(i);
+    }
+}
+
+void Environment::printContent(void)
+{
+    LOG(Message) << "Modules: " << std::endl;
+    for (unsigned int i = 0; i < module_.size(); ++i)
+    {
+        LOG(Message) << std::setw(4) << i << ": "
+                     << getModuleName(i) << std::endl;
+    }
+    LOG(Message) << "Objects: " << std::endl;
+    for (unsigned int i = 0; i < object_.size(); ++i)
+    {
+        LOG(Message) << std::setw(4) << i << ": "
+                     << getObjectName(i) << std::endl;
+    }
+}

extras/Hadrons/Environment.hpp

@@ -0,0 +1,427 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Environment.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Environment_hpp_
+#define Hadrons_Environment_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Graph.hpp>
+
+#ifndef SITE_SIZE_TYPE
+#define SITE_SIZE_TYPE unsigned int
+#endif
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         Global environment                                 *
+ ******************************************************************************/
+// forward declaration of Module
+class ModuleBase;
+
+class Object
+{
+public:
+    Object(void) = default;
+    virtual ~Object(void) = default;
+};
+
+template <typename T>
+class Holder: public Object
+{
+public:
+    Holder(void) = default;
+    Holder(T *pt);
+    virtual ~Holder(void) = default;
+    T &       get(void) const;
+    T *       getPt(void) const;
+    void      reset(T *pt);
+private:
+    std::unique_ptr<T> objPt_{nullptr};
+};
+
+class Environment
+{
+    SINGLETON(Environment);
+public:
+    typedef SITE_SIZE_TYPE                         Size;
+    typedef std::unique_ptr<ModuleBase>            ModPt;
+    typedef std::unique_ptr<GridCartesian>         GridPt;
+    typedef std::unique_ptr<GridRedBlackCartesian> GridRbPt;
+    typedef std::unique_ptr<GridParallelRNG>       RngPt;
+    typedef std::unique_ptr<LatticeBase>           LatticePt;
+private:
+    struct ModuleInfo
+    {
+        const std::type_info      *type{nullptr};
+        std::string               name;
+        ModPt                     data{nullptr};
+        std::vector<unsigned int> input;
+    };
+    struct ObjInfo
+    {
+        Size                    size{0};
+        unsigned int            Ls{0};
+        bool                    isRegistered{false};
+        const std::type_info    *type{nullptr};
+        std::string             name;
+        int                     module{-1};
+        std::set<unsigned int>  owners, properties;
+        std::unique_ptr<Object> data{nullptr};
+    };
+public:
+    // dry run
+    void                    dryRun(const bool isDry);
+    bool                    isDryRun(void) const;
+    // trajectory number
+    void                    setTrajectory(const unsigned int traj);
+    unsigned int            getTrajectory(void) const;
+    // grids
+    void                    createGrid(const unsigned int Ls);
+    GridCartesian *         getGrid(const unsigned int Ls = 1) const;
+    GridRedBlackCartesian * getRbGrid(const unsigned int Ls = 1) const;
+    std::vector<int>        getDim(void) const;
+    int                     getDim(const unsigned int mu) const;
+    unsigned int            getNd(void) const;
+    // random number generator
+    void                    setSeed(const std::vector<int> &seed);
+    GridParallelRNG *       get4dRng(void) const;
+    // module management
+    void                    pushModule(ModPt &pt);
+    template <typename M>
+    void                    createModule(const std::string name);
+    template <typename M>
+    void                    createModule(const std::string name,
+                                         const typename M::Par &par);
+    void                    createModule(const std::string name,
+                                         const std::string type,
+                                         XmlReader &reader);
+    unsigned int            getNModule(void) const;
+    ModuleBase *            getModule(const unsigned int address) const;
+    ModuleBase *            getModule(const std::string name) const;
+    template <typename M>
+    M *                     getModule(const unsigned int address) const;
+    template <typename M>
+    M *                     getModule(const std::string name) const;
+    unsigned int            getModuleAddress(const std::string name) const;
+    std::string             getModuleName(const unsigned int address) const;
+    std::string             getModuleType(const unsigned int address) const;
+    std::string             getModuleType(const std::string name) const;
+    std::string             getModuleNamespace(const unsigned int address) const;
+    std::string             getModuleNamespace(const std::string name) const;
+    bool                    hasModule(const unsigned int address) const;
+    bool                    hasModule(const std::string name) const;
+    Graph<unsigned int>     makeModuleGraph(void) const;
+    Size                    executeProgram(const std::vector<unsigned int> &p);
+    Size                    executeProgram(const std::vector<std::string> &p);
+    // general memory management
+    void                    addObject(const std::string name,
+                                      const int moduleAddress = -1);
+    void                    registerObject(const unsigned int address,
+                                           const unsigned int size,
+                                           const unsigned int Ls = 1);
+    void                    registerObject(const std::string name,
+                                           const unsigned int size,
+                                           const unsigned int Ls = 1);
+    template <typename T>
+    unsigned int            lattice4dSize(void) const;
+    template <typename T>
+    void                    registerLattice(const unsigned int address,
+                                            const unsigned int Ls = 1);
+    template <typename T>
+    void                    registerLattice(const std::string name,
+                                            const unsigned int Ls = 1);
+    template <typename T>
+    void                    setObject(const unsigned int address, T *object);
+    template <typename T>
+    void                    setObject(const std::string name, T *object);
+    template <typename T>
+    T *                     getObject(const unsigned int address) const;
+    template <typename T>
+    T *                     getObject(const std::string name) const;
+    template <typename T>
+    T *                     createLattice(const unsigned int address);
+    template <typename T>
+    T *                     createLattice(const std::string name);
+    unsigned int            getObjectAddress(const std::string name) const;
+    std::string             getObjectName(const unsigned int address) const;
+    std::string             getObjectType(const unsigned int address) const;
+    std::string             getObjectType(const std::string name) const;
+    Size                    getObjectSize(const unsigned int address) const;
+    Size                    getObjectSize(const std::string name) const;
+    unsigned int            getObjectModule(const unsigned int address) const;
+    unsigned int            getObjectModule(const std::string name) const;
+    unsigned int            getObjectLs(const unsigned int address) const;
+    unsigned int            getObjectLs(const std::string name) const;
+    bool                    hasObject(const unsigned int address) const;
+    bool                    hasObject(const std::string name) const;
+    bool                    hasRegisteredObject(const unsigned int address) const;
+    bool                    hasRegisteredObject(const std::string name) const;
+    bool                    hasCreatedObject(const unsigned int address) const;
+    bool                    hasCreatedObject(const std::string name) const;
+    bool                    isObject5d(const unsigned int address) const;
+    bool                    isObject5d(const std::string name) const;
+    template <typename T>
+    bool                    isObjectOfType(const unsigned int address) const;
+    template <typename T>
+    bool                    isObjectOfType(const std::string name) const;
+    Environment::Size       getTotalSize(void) const;
+    void                    addOwnership(const unsigned int owner,
+                                         const unsigned int property);
+    void                    addOwnership(const std::string owner,
+                                         const std::string property);
+    bool                    hasOwners(const unsigned int address) const;
+    bool                    hasOwners(const std::string name) const;
+    bool                    freeObject(const unsigned int address);
+    bool                    freeObject(const std::string name);
+    void                    freeAll(void);
+    void                    printContent(void);
+private:
+    // general
+    bool                                   dryRun_{false};
+    unsigned int                           traj_, locVol_;
+    // grids
+    std::vector<int>                       dim_;
+    GridPt                                 grid4d_;
+    std::map<unsigned int, GridPt>         grid5d_;
+    GridRbPt                               gridRb4d_;
+    std::map<unsigned int, GridRbPt>       gridRb5d_;
+    unsigned int                           nd_;
+    // random number generator
+    RngPt                                  rng4d_;
+    // module and related maps
+    std::vector<ModuleInfo>                module_;
+    std::map<std::string, unsigned int>    moduleAddress_;
+    // lattice store
+    std::map<unsigned int, LatticePt>      lattice_;
+    // object store
+    std::vector<ObjInfo>                   object_;
+    std::map<std::string, unsigned int>    objectAddress_;
+};
+
+/******************************************************************************
+ *                       Holder template implementation                       *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename T>
+Holder<T>::Holder(T *pt)
+: objPt_(pt)
+{}
+
+// access //////////////////////////////////////////////////////////////////////
+template <typename T>
+T & Holder<T>::get(void) const
+{
+    return &objPt_.get();
+}
+
+template <typename T>
+T * Holder<T>::getPt(void) const
+{
+    return objPt_.get();
+}
+
+template <typename T>
+void Holder<T>::reset(T *pt)
+{
+    objPt_.reset(pt);
+}
+
+/******************************************************************************
+ *                     Environment template implementation                    *
+ ******************************************************************************/
+// module management ///////////////////////////////////////////////////////////
+template <typename M>
+void Environment::createModule(const std::string name)
+{
+    ModPt pt(new M(name));
+    
+    pushModule(pt);
+}
+
+template <typename M>
+void Environment::createModule(const std::string name,
+                               const typename M::Par &par)
+{
+    ModPt pt(new M(name));
+    
+    static_cast<M *>(pt.get())->setPar(par);
+    pushModule(pt);
+}
+
+template <typename M>
+M * Environment::getModule(const unsigned int address) const
+{
+    if (auto *pt = dynamic_cast<M *>(getModule(address)))
+    {
+        return pt;
+    }
+    else
+    {
+        HADRON_ERROR("module '" + module_[address].name
+                     + "' does not have type " + typeid(M).name()
+                     + "(object type: " + getModuleType(address) + ")");
+    }
+}
+
+template <typename M>
+M * Environment::getModule(const std::string name) const
+{
+    return getModule<M>(getModuleAddress(name));
+}
+
+template <typename T>
+unsigned int Environment::lattice4dSize(void) const
+{
+    return sizeof(typename T::vector_object)/getGrid()->Nsimd();
+}
+
+template <typename T>
+void Environment::registerLattice(const unsigned int address,
+                                  const unsigned int Ls)
+{
+    createGrid(Ls);
+    registerObject(address, Ls*lattice4dSize<T>(), Ls);
+}
+
+template <typename T>
+void Environment::registerLattice(const std::string name, const unsigned int Ls)
+{
+    createGrid(Ls);
+    registerObject(name, Ls*lattice4dSize<T>(), Ls);
+}
+
+template <typename T>
+void Environment::setObject(const unsigned int address, T *object)
+{
+    if (hasRegisteredObject(address))
+    {
+        object_[address].data.reset(new Holder<T>(object));
+        object_[address].type = &typeid(T);
+    }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address) +
+                     " exists but is not registered");
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+template <typename T>
+void Environment::setObject(const std::string name, T *object)
+{
+    setObject(getObjectAddress(name), object);
+}
+
+template <typename T>
+T * Environment::getObject(const unsigned int address) const
+{
+    if (hasRegisteredObject(address))
+    {
+        if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
+        {
+            return h->getPt();
+        }
+        else
+        {
+            HADRON_ERROR("object with address " + std::to_string(address) +
+                         " does not have type '" + typeName(&typeid(T)) +
+                         "' (has type '" + getObjectType(address) + "')");
+        }
+    }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address) +
+                     " exists but is not registered");
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+template <typename T>
+T * Environment::getObject(const std::string name) const
+{
+    return getObject<T>(getObjectAddress(name));
+}
+
+template <typename T>
+T * Environment::createLattice(const unsigned int address)
+{
+    GridCartesian *g = getGrid(getObjectLs(address));
+    
+    setObject(address, new T(g));
+    
+    return getObject<T>(address);
+}
+
+template <typename T>
+T * Environment::createLattice(const std::string name)
+{
+    return createLattice<T>(getObjectAddress(name));
+}
+
+template <typename T>
+bool Environment::isObjectOfType(const unsigned int address) const
+{
+    if (hasRegisteredObject(address))
+    {
+        if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
+        {
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address) +
+                     " exists but is not registered");
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+template <typename T>
+bool Environment::isObjectOfType(const std::string name) const
+{
+    return isObjectOfType<T>(getObjectAddress(name));
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Environment_hpp_

extras/Hadrons/Factory.hpp

@@ -0,0 +1,106 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Factory.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Factory_hpp_
+#define Hadrons_Factory_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                        abstract factory class                              *
+ ******************************************************************************/
+template <typename T>
+class Factory
+{
+public:
+    typedef std::function<std::unique_ptr<T>(const std::string)> Func;
+public:
+    // constructor
+    Factory(void) = default;
+    // destructor
+    virtual ~Factory(void) = default;
+    // registration
+    void registerBuilder(const std::string type, const Func &f);
+    // get builder list
+    std::vector<std::string> getBuilderList(void) const;
+    // factory
+    std::unique_ptr<T> create(const std::string type,
+                              const std::string name) const;
+private:
+    std::map<std::string, Func> builder_;
+};
+
+/******************************************************************************
+ *                         template implementation                            *
+ ******************************************************************************/
+// registration ////////////////////////////////////////////////////////////////
+template <typename T>
+void Factory<T>::registerBuilder(const std::string type, const Func &f)
+{
+    builder_[type] = f;
+}
+
+// get module list /////////////////////////////////////////////////////////////
+template <typename T>
+std::vector<std::string> Factory<T>::getBuilderList(void) const
+{
+    std::vector<std::string> list;
+    
+    for (auto &b: builder_)
+    {
+        list.push_back(b.first);
+    }
+    
+    return list;
+}
+
+// factory /////////////////////////////////////////////////////////////////////
+template <typename T>
+std::unique_ptr<T> Factory<T>::create(const std::string type,
+                                      const std::string name) const
+{
+    Func func;
+    
+    try
+    {
+        func = builder_.at(type);
+    }
+    catch (std::out_of_range &)
+    {
+        HADRON_ERROR("object of type '" + type + "' unknown");
+    }
+    
+    return func(name);
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Factory_hpp_

extras/Hadrons/GeneticScheduler.hpp

@@ -0,0 +1,329 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/GeneticScheduler.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_GeneticScheduler_hpp_
+#define Hadrons_GeneticScheduler_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Graph.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                   Scheduler based on a genetic algorithm                   *
+ ******************************************************************************/
+template <typename T>
+class GeneticScheduler
+{
+public:
+    typedef std::vector<T>                   Gene;
+    typedef std::pair<Gene *, Gene *>        GenePair;
+    typedef std::function<int(const Gene &)> ObjFunc;
+    struct Parameters
+    {
+        double       mutationRate;
+        unsigned int popSize, seed;
+    };
+public:
+    // constructor
+    GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
+                     const Parameters &par);
+    // destructor
+    virtual ~GeneticScheduler(void) = default;
+    // access
+    const Gene & getMinSchedule(void);
+    int          getMinValue(void);
+    // breed a new generation
+    void nextGeneration(void);
+    // heuristic benchmarks
+    void benchmarkCrossover(const unsigned int nIt);
+    // print population
+    friend std::ostream & operator<<(std::ostream &out,
+                                     const GeneticScheduler<T> &s)
+    {
+        out << "[";
+        for (auto &p: s.population_)
+        {
+            out << p.first << ", ";
+        }
+        out << "\b\b]";
+        
+        return out;
+    }
+private:
+    // evolution steps
+    void initPopulation(void);
+    void doCrossover(void);
+    void doMutation(void);
+    // genetic operators
+    GenePair selectPair(void);
+    void     crossover(Gene &c1, Gene &c2, const Gene &p1, const Gene &p2);
+    void     mutation(Gene &m, const Gene &c);
+    
+private:
+    Graph<T>                 &graph_;
+    const ObjFunc            &func_;
+    const Parameters         par_;
+    std::multimap<int, Gene> population_;
+    std::mt19937             gen_;
+};
+
+/******************************************************************************
+ *                       template implementation                              *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename T>
+GeneticScheduler<T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
+                                      const Parameters &par)
+: graph_(graph)
+, func_(func)
+, par_(par)
+{
+    gen_.seed(par_.seed);
+}
+
+// access //////////////////////////////////////////////////////////////////////
+template <typename T>
+const typename GeneticScheduler<T>::Gene &
+GeneticScheduler<T>::getMinSchedule(void)
+{
+    return population_.begin()->second;
+}
+
+template <typename T>
+int GeneticScheduler<T>::getMinValue(void)
+{
+    return population_.begin()->first;
+}
+
+// breed a new generation //////////////////////////////////////////////////////
+template <typename T>
+void GeneticScheduler<T>::nextGeneration(void)
+{
+    // random initialization of the population if necessary
+    if (population_.size() != par_.popSize)
+    {
+        initPopulation();
+    }
+    LOG(Debug) << "Starting population:\n" << *this << std::endl;
+    
+    // random mutations
+    //PARALLEL_FOR_LOOP
+    for (unsigned int i = 0; i < par_.popSize; ++i)
+    {
+        doMutation();
+    }
+    LOG(Debug) << "After mutations:\n" << *this << std::endl;
+    
+    // mating
+    //PARALLEL_FOR_LOOP
+    for (unsigned int i = 0; i < par_.popSize/2; ++i)
+    {
+        doCrossover();
+    }
+    LOG(Debug) << "After mating:\n" << *this << std::endl;
+    
+    // grim reaper
+    auto it = population_.begin();
+    
+    std::advance(it, par_.popSize);
+    population_.erase(it, population_.end());
+    LOG(Debug) << "After grim reaper:\n" << *this << std::endl;
+}
+
+// evolution steps /////////////////////////////////////////////////////////////
+template <typename T>
+void GeneticScheduler<T>::initPopulation(void)
+{
+    population_.clear();
+    for (unsigned int i = 0; i < par_.popSize; ++i)
+    {
+        auto p = graph_.topoSort(gen_);
+        
+        population_.insert(std::make_pair(func_(p), p));
+    }
+}
+
+template <typename T>
+void GeneticScheduler<T>::doCrossover(void)
+{
+    auto p = selectPair();
+    Gene &p1 = *(p.first), &p2 = *(p.second);
+    Gene c1, c2;
+    
+    crossover(c1, c2, p1, p2);
+    PARALLEL_CRITICAL
+    {
+        population_.insert(std::make_pair(func_(c1), c1));
+        population_.insert(std::make_pair(func_(c2), c2));
+    }
+}
+
+template <typename T>
+void GeneticScheduler<T>::doMutation(void)
+{
+    std::uniform_real_distribution<double>      mdis(0., 1.);
+    std::uniform_int_distribution<unsigned int> pdis(0, population_.size() - 1);
+    
+    if (mdis(gen_) < par_.mutationRate)
+    {
+        Gene m;
+        auto it = population_.begin();
+        
+        std::advance(it, pdis(gen_));
+        mutation(m, it->second);
+        PARALLEL_CRITICAL
+        {
+            population_.insert(std::make_pair(func_(m), m));
+        }
+    }
+}
+
+// genetic operators ///////////////////////////////////////////////////////////
+template <typename T>
+typename GeneticScheduler<T>::GenePair GeneticScheduler<T>::selectPair(void)
+{
+    std::vector<double> prob;
+    unsigned int        ind;
+    Gene                *p1, *p2;
+    
+    for (auto &c: population_)
+    {
+        prob.push_back(1./c.first);
+    }
+    do
+    {
+        double probCpy;
+        
+        std::discrete_distribution<unsigned int> dis1(prob.begin(), prob.end());
+        auto rIt = population_.begin();
+        ind = dis1(gen_);
+        std::advance(rIt, ind);
+        p1 = &(rIt->second);
+        probCpy   = prob[ind];
+        prob[ind] = 0.;
+        std::discrete_distribution<unsigned int> dis2(prob.begin(), prob.end());
+        rIt = population_.begin();
+        std::advance(rIt, dis2(gen_));
+        p2 = &(rIt->second);
+        prob[ind] = probCpy;
+    } while (p1 == p2);
+    
+    return std::make_pair(p1, p2);
+}
+
+template <typename T>
+void GeneticScheduler<T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
+                                    const Gene &p2)
+{
+    Gene                                        buf;
+    std::uniform_int_distribution<unsigned int> dis(0, p1.size() - 1);
+    unsigned int                                cut = dis(gen_);
+    
+    c1.clear();
+    buf = p2;
+    for (unsigned int i = 0; i < cut; ++i)
+    {
+        c1.push_back(p1[i]);
+        buf.erase(std::find(buf.begin(), buf.end(), p1[i]));
+    }
+    for (unsigned int i = 0; i < buf.size(); ++i)
+    {
+        c1.push_back(buf[i]);
+    }
+    c2.clear();
+    buf = p2;
+    for (unsigned int i = cut; i < p1.size(); ++i)
+    {
+        buf.erase(std::find(buf.begin(), buf.end(), p1[i]));
+    }
+    for (unsigned int i = 0; i < buf.size(); ++i)
+    {
+        c2.push_back(buf[i]);
+    }
+    for (unsigned int i = cut; i < p1.size(); ++i)
+    {
+        c2.push_back(p1[i]);
+    }
+}
+
+template <typename T>
+void GeneticScheduler<T>::mutation(Gene &m, const Gene &c)
+{
+    Gene                                        buf;
+    std::uniform_int_distribution<unsigned int> dis(0, c.size() - 1);
+    unsigned int                                cut = dis(gen_);
+    Graph<T>                                    g1 = graph_, g2 = graph_;
+    
+    for (unsigned int i = 0; i < cut; ++i)
+    {
+        g1.removeVertex(c[i]);
+    }
+    for (unsigned int i = cut; i < c.size(); ++i)
+    {
+        g2.removeVertex(c[i]);
+    }
+    if (g1.size() > 0)
+    {
+        buf = g1.topoSort(gen_);
+    }
+    if (g2.size() > 0)
+    {
+        m = g2.topoSort(gen_);
+    }
+    for (unsigned int i = cut; i < c.size(); ++i)
+    {
+        m.push_back(buf[i - cut]);
+    }
+}
+
+template <typename T>
+void GeneticScheduler<T>::benchmarkCrossover(const unsigned int nIt)
+{
+    Gene   p1, p2, c1, c2;
+    double neg = 0., eq = 0., pos = 0., total;
+    int    improvement;
+    
+    LOG(Message) << "Benchmarking crossover..." << std::endl;
+    for (unsigned int i = 0; i < nIt; ++i)
+    {
+        p1 = graph_.topoSort(gen_);
+        p2 = graph_.topoSort(gen_);
+        crossover(c1, c2, p1, p2);
+        improvement = (func_(c1) + func_(c2) - func_(p1) - func_(p2))/2;
+        if (improvement < 0) neg++; else if (improvement == 0) eq++; else pos++;
+    }
+    total = neg + eq + pos;
+    LOG(Message) << "  -: " << neg/total << "  =: " << eq/total
+                 << "  +: " << pos/total << std::endl;
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_GeneticScheduler_hpp_

extras/Hadrons/Global.cc

@@ -0,0 +1,82 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Global.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Global.hpp>
+
+using namespace Grid;
+using namespace QCD;
+using namespace Hadrons;
+
+HadronsLogger Hadrons::HadronsLogError(1,"Error");
+HadronsLogger Hadrons::HadronsLogWarning(1,"Warning");
+HadronsLogger Hadrons::HadronsLogMessage(1,"Message");
+HadronsLogger Hadrons::HadronsLogIterative(1,"Iterative");
+HadronsLogger Hadrons::HadronsLogDebug(1,"Debug");
+
+// pretty size formatting //////////////////////////////////////////////////////
+std::string Hadrons::sizeString(long unsigned int bytes)
+
+{
+    constexpr unsigned int bufSize = 256;
+    const char             *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
+    char                   buf[256];
+    long unsigned int      s     = 0;
+    double                 count = bytes;
+    
+    while (count >= 1024 && s < 7)
+    {
+        s++;
+        count /= 1024;
+    }
+    if (count - floor(count) == 0.0)
+    {
+        snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
+    }
+    else
+    {
+        snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
+    }
+    
+    return std::string(buf);
+}
+
+// type utilities //////////////////////////////////////////////////////////////
+constexpr unsigned int maxNameSize = 1024u;
+
+std::string Hadrons::typeName(const std::type_info *info)
+{
+    char        *buf;
+    std::string name;
+    
+    buf  = abi::__cxa_demangle(info->name(), nullptr, nullptr, nullptr);
+    name = buf;
+    free(buf);
+    
+    return name;
+}

extras/Hadrons/Global.hpp

@@ -0,0 +1,179 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Global.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Global_hpp_
+#define Hadrons_Global_hpp_
+
+#include <set>
+#include <stack>
+#include <Grid/Grid.h>
+#include <cxxabi.h>
+
+#define BEGIN_HADRONS_NAMESPACE \
+namespace Grid {\
+using namespace QCD;\
+namespace Hadrons {\
+using Grid::operator<<;
+#define END_HADRONS_NAMESPACE }}
+
+#define BEGIN_MODULE_NAMESPACE(name)\
+namespace name {\
+using Grid::operator<<;
+#define END_MODULE_NAMESPACE }
+
+/* the 'using Grid::operator<<;' statement prevents a very nasty compilation
+ * error with GCC 5 (clang & GCC 6 compile fine without it).
+ */
+
+#ifndef FIMPL
+#define FIMPL WilsonImplR
+#endif
+#ifndef SIMPL
+#define SIMPL ScalarImplCR
+#endif
+
+BEGIN_HADRONS_NAMESPACE
+
+// type aliases
+#define FERM_TYPE_ALIASES(FImpl, suffix)\
+typedef FermionOperator<FImpl>                       FMat##suffix;             \
+typedef typename FImpl::FermionField                 FermionField##suffix;     \
+typedef typename FImpl::PropagatorField              PropagatorField##suffix;  \
+typedef typename FImpl::SitePropagator               SitePropagator##suffix;   \
+typedef std::vector<typename FImpl::SitePropagator::scalar_object>             \
+                                                     SlicedPropagator##suffix;
+
+#define GAUGE_TYPE_ALIASES(FImpl, suffix)\
+typedef typename FImpl::DoubledGaugeField DoubledGaugeField##suffix;
+
+#define SCALAR_TYPE_ALIASES(SImpl, suffix)\
+typedef typename SImpl::Field ScalarField##suffix;\
+typedef typename SImpl::Field PropagatorField##suffix;
+
+#define SOLVER_TYPE_ALIASES(FImpl, suffix)\
+typedef std::function<void(FermionField##suffix &,\
+                      const FermionField##suffix &)> SolverFn##suffix;
+
+#define SINK_TYPE_ALIASES(suffix)\
+typedef std::function<SlicedPropagator##suffix(const PropagatorField##suffix &)> SinkFn##suffix;
+
+#define FGS_TYPE_ALIASES(FImpl, suffix)\
+FERM_TYPE_ALIASES(FImpl, suffix)\
+GAUGE_TYPE_ALIASES(FImpl, suffix)\
+SOLVER_TYPE_ALIASES(FImpl, suffix)
+
+// logger
+class HadronsLogger: public Logger
+{
+public:
+    HadronsLogger(int on, std::string nm): Logger("Hadrons", on, nm,
+                                                  GridLogColours, "BLACK"){};
+};
+
+#define LOG(channel) std::cout << HadronsLog##channel
+#define HADRON_ERROR(msg)\
+LOG(Error) << msg << " (" << __FUNCTION__ << " at " << __FILE__ << ":"\
+           << __LINE__ << ")" << std::endl;\
+abort();
+
+#define DEBUG_VAR(var) LOG(Debug) << #var << "= " << (var) << std::endl;
+
+extern HadronsLogger HadronsLogError;
+extern HadronsLogger HadronsLogWarning;
+extern HadronsLogger HadronsLogMessage;
+extern HadronsLogger HadronsLogIterative;
+extern HadronsLogger HadronsLogDebug;
+
+// singleton pattern
+#define SINGLETON(name)\
+public:\
+    name(const name &e) = delete;\
+    void operator=(const name &e) = delete;\
+    static name & getInstance(void)\
+    {\
+        static name e;\
+        return e;\
+    }\
+private:\
+    name(void);
+
+#define SINGLETON_DEFCTOR(name)\
+public:\
+    name(const name &e) = delete;\
+    void operator=(const name &e) = delete;\
+    static name & getInstance(void)\
+    {\
+        static name e;\
+        return e;\
+    }\
+private:\
+    name(void) = default;
+
+// pretty size formating
+std::string sizeString(long unsigned int bytes);
+
+// type utilities
+template <typename T>
+const std::type_info * typeIdPt(const T &x)
+{
+    return &typeid(x);
+}
+
+std::string typeName(const std::type_info *info);
+
+template <typename T>
+const std::type_info * typeIdPt(void)
+{
+    return &typeid(T);
+}
+
+template <typename T>
+std::string typeName(const T &x)
+{
+    return typeName(typeIdPt(x));
+}
+
+template <typename T>
+std::string typeName(void)
+{
+    return typeName(typeIdPt<T>());
+}
+
+// default writers/readers
+#ifdef HAVE_HDF5
+typedef Hdf5Reader CorrReader;
+typedef Hdf5Writer CorrWriter;
+#else
+typedef XmlReader CorrReader;
+typedef XmlWriter CorrWriter;
+#endif
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Global_hpp_

extras/Hadrons/Graph.hpp

@@ -0,0 +1,760 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Graph.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Graph_hpp_
+#define Hadrons_Graph_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                          Oriented graph class                              *
+ ******************************************************************************/
+// I/O for edges
+template <typename T>
+std::ostream & operator<<(std::ostream &out, const std::pair<T, T> &e)
+{
+    out << "\""  << e.first << "\" -> \"" << e.second << "\"";
+    
+    return out;
+}
+
+// main class
+template <typename T>
+class Graph
+{
+public:
+    typedef std::pair<T, T> Edge;
+public:
+    // constructor
+    Graph(void);
+    // destructor
+    virtual ~Graph(void) = default;
+    // access
+    void           addVertex(const T &value);
+    void           addEdge(const Edge &e);
+    void           addEdge(const T &start, const T &end);
+    std::vector<T> getVertices(void) const;
+    void           removeVertex(const T &value);
+    void           removeEdge(const Edge &e);
+    void           removeEdge(const T &start, const T &end);
+    unsigned int   size(void) const;
+    // tests
+    bool gotValue(const T &value) const;
+    // graph topological manipulations
+    std::vector<T>              getAdjacentVertices(const T &value) const;
+    std::vector<T>              getChildren(const T &value) const;
+    std::vector<T>              getParents(const T &value) const;
+    std::vector<T>              getRoots(void) const;
+    std::vector<Graph<T>>       getConnectedComponents(void) const;
+    std::vector<T>              topoSort(void);
+    template <typename Gen>
+    std::vector<T>              topoSort(Gen &gen);
+    std::vector<std::vector<T>> allTopoSort(void);
+    // I/O
+    friend std::ostream & operator<<(std::ostream &out, const Graph<T> &g)
+    {
+        out << "{";
+        for (auto &e: g.edgeSet_)
+        {
+            out << e << ", ";
+        }
+        if (g.edgeSet_.size() != 0)
+        {
+            out << "\b\b";
+        }
+        out << "}";
+        
+        return out;
+    }
+private:
+    // vertex marking
+    void      mark(const T &value, const bool doMark = true);
+    void      markAll(const bool doMark = true);
+    void      unmark(const T &value);
+    void      unmarkAll(void);
+    bool      isMarked(const T &value) const;
+    const T * getFirstMarked(const bool isMarked = true) const;
+    template <typename Gen>
+    const T * getRandomMarked(const bool isMarked, Gen &gen);
+    const T * getFirstUnmarked(void) const;
+    template <typename Gen>
+    const T * getRandomUnmarked(Gen &gen);
+    // prune marked/unmarked vertices
+    void removeMarked(const bool isMarked = true);
+    void removeUnmarked(void);
+    // depth-first search marking
+    void depthFirstSearch(void);
+    void depthFirstSearch(const T &root);
+private:
+    std::map<T, bool>  isMarked_;
+    std::set<Edge>     edgeSet_;
+};
+
+// build depedency matrix from topological sorts
+template <typename T>
+std::map<T, std::map<T, bool>>
+makeDependencyMatrix(const std::vector<std::vector<T>> &topSort);
+
+/******************************************************************************
+ *                       template implementation                              *
+ ******************************************************************************
+ * in all the following V is the number of vertex and E is the number of edge
+ * in the worst case E = V^2
+ */
+
+// constructor /////////////////////////////////////////////////////////////////
+template <typename T>
+Graph<T>::Graph(void)
+{}
+
+// access //////////////////////////////////////////////////////////////////////
+// complexity: log(V)
+template <typename T>
+void Graph<T>::addVertex(const T &value)
+{
+    isMarked_[value] = false;
+}
+
+// complexity: O(log(V))
+template <typename T>
+void Graph<T>::addEdge(const Edge &e)
+{
+    addVertex(e.first);
+    addVertex(e.second);
+    edgeSet_.insert(e);
+}
+
+// complexity: O(log(V))
+template <typename T>
+void Graph<T>::addEdge(const T &start, const T &end)
+{
+    addEdge(Edge(start, end));
+}
+
+template <typename T>
+std::vector<T> Graph<T>::getVertices(void) const
+{
+    std::vector<T> vertex;
+    
+    for (auto &v: isMarked_)
+    {
+        vertex.push_back(v.first);
+    }
+    
+    return vertex;
+}
+
+// complexity: O(V*log(V))
+template <typename T>
+void Graph<T>::removeVertex(const T &value)
+{
+    // remove vertex from the mark table
+    auto vIt = isMarked_.find(value);
+    
+    if (vIt != isMarked_.end())
+    {
+        isMarked_.erase(vIt);
+    }
+    else
+    {
+        HADRON_ERROR("vertex " << value << " does not exists");
+    }
+
+    // remove all edges containing the vertex
+    auto pred = [&value](const Edge &e)
+    {
+        return ((e.first == value) or (e.second == value));
+    };
+    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    
+    while (eIt != edgeSet_.end())
+    {
+        edgeSet_.erase(eIt);
+        eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    }
+}
+
+// complexity: O(log(V))
+template <typename T>
+void Graph<T>::removeEdge(const Edge &e)
+{
+    auto eIt = edgeSet_.find(e);
+    
+    if (eIt != edgeSet_.end())
+    {
+        edgeSet_.erase(eIt);
+    }
+    else
+    {
+        HADRON_ERROR("edge "  << e << " does not exists");
+    }
+}
+
+// complexity: O(log(V))
+template <typename T>
+void Graph<T>::removeEdge(const T &start, const T &end)
+{
+    removeEdge(Edge(start, end));
+}
+
+// complexity: O(1)
+template <typename T>
+unsigned int Graph<T>::size(void) const
+{
+    return isMarked_.size();
+}
+
+// tests ///////////////////////////////////////////////////////////////////////
+// complexity: O(log(V))
+template <typename T>
+bool Graph<T>::gotValue(const T &value) const
+{
+    auto it = isMarked_.find(value);
+    
+    if (it == isMarked_.end())
+    {
+        return false;
+    }
+    else
+    {
+        return true;
+    }
+}
+
+// vertex marking //////////////////////////////////////////////////////////////
+// complexity: O(log(V))
+template <typename T>
+void Graph<T>::mark(const T &value, const bool doMark)
+{
+    if (gotValue(value))
+    {
+        isMarked_[value] = doMark;
+    }
+    else
+    {
+        HADRON_ERROR("vertex " << value << " does not exists");
+    }
+}
+
+// complexity: O(V*log(V))
+template <typename T>
+void Graph<T>::markAll(const bool doMark)
+{
+    for (auto &v: isMarked_)
+    {
+        mark(v.first, doMark);
+    }
+}
+
+// complexity: O(log(V))
+template <typename T>
+void Graph<T>::unmark(const T &value)
+{
+    mark(value, false);
+}
+
+// complexity: O(V*log(V))
+template <typename T>
+void Graph<T>::unmarkAll(void)
+{
+    markAll(false);
+}
+
+// complexity: O(log(V))
+template <typename T>
+bool Graph<T>::isMarked(const T &value) const
+{
+    if (gotValue(value))
+    {
+        return isMarked_.at(value);
+    }
+    else
+    {
+        HADRON_ERROR("vertex " << value << " does not exists");
+        
+        return false;
+    }
+}
+
+// complexity: O(log(V))
+template <typename T>
+const T * Graph<T>::getFirstMarked(const bool isMarked) const
+{
+    auto pred = [&isMarked](const std::pair<T, bool> &v)
+    {
+        return (v.second == isMarked);
+    };
+    auto vIt = std::find_if(isMarked_.begin(), isMarked_.end(), pred);
+    
+    if (vIt != isMarked_.end())
+    {
+        return &(vIt->first);
+    }
+    else
+    {
+        return nullptr;
+    }
+}
+
+// complexity: O(log(V))
+template <typename T>
+template <typename Gen>
+const T * Graph<T>::getRandomMarked(const bool isMarked, Gen &gen)
+{
+    auto pred = [&isMarked](const std::pair<T, bool> &v)
+    {
+        return (v.second == isMarked);
+    };
+    std::uniform_int_distribution<unsigned int> dis(0, size() - 1);
+    auto                                        rIt = isMarked_.begin();
+    
+    std::advance(rIt, dis(gen));
+    auto vIt = std::find_if(rIt, isMarked_.end(), pred);
+    if (vIt != isMarked_.end())
+    {
+        return &(vIt->first);
+    }
+    else
+    {
+        vIt = std::find_if(isMarked_.begin(), rIt, pred);
+        if (vIt != rIt)
+        {
+            return &(vIt->first);
+        }
+        else
+        {
+            return nullptr;
+        }
+    }
+}
+
+// complexity: O(log(V))
+template <typename T>
+const T * Graph<T>::getFirstUnmarked(void) const
+{
+    return getFirstMarked(false);
+}
+
+// complexity: O(log(V))
+template <typename T>
+template <typename Gen>
+const T * Graph<T>::getRandomUnmarked(Gen &gen)
+{
+    return getRandomMarked(false, gen);
+}
+
+// prune marked/unmarked vertices //////////////////////////////////////////////
+// complexity: O(V^2*log(V))
+template <typename T>
+void Graph<T>::removeMarked(const bool isMarked)
+{
+    auto isMarkedCopy = isMarked_;
+    
+    for (auto &v: isMarkedCopy)
+    {
+        if (v.second == isMarked)
+        {
+            removeVertex(v.first);
+        }
+    }
+}
+
+// complexity: O(V^2*log(V))
+template <typename T>
+void Graph<T>::removeUnmarked(void)
+{
+    removeMarked(false);
+}
+
+// depth-first search marking //////////////////////////////////////////////////
+// complexity: O(V*log(V))
+template <typename T>
+void Graph<T>::depthFirstSearch(void)
+{
+    depthFirstSearch(isMarked_.begin()->first);
+}
+
+// complexity: O(V*log(V))
+template <typename T>
+void Graph<T>::depthFirstSearch(const T &root)
+{
+    std::vector<T> adjacentVertex;
+    
+    mark(root);
+    adjacentVertex = getAdjacentVertices(root);
+    for (auto &v: adjacentVertex)
+    {
+        if (!isMarked(v))
+        {
+            depthFirstSearch(v);
+        }
+    }
+}
+
+// graph topological manipulations /////////////////////////////////////////////
+// complexity: O(V*log(V))
+template <typename T>
+std::vector<T> Graph<T>::getAdjacentVertices(const T &value) const
+{
+    std::vector<T> adjacentVertex;
+    
+    auto pred = [&value](const Edge &e)
+    {
+        return ((e.first == value) or (e.second == value));
+    };
+    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    
+    while (eIt != edgeSet_.end())
+    {
+        if (eIt->first == value)
+        {
+            adjacentVertex.push_back((*eIt).second);
+        }
+        else if (eIt->second == value)
+        {
+            adjacentVertex.push_back((*eIt).first);
+        }
+        eIt = find_if(++eIt, edgeSet_.end(), pred);
+    }
+    
+    return adjacentVertex;
+}
+
+// complexity: O(V*log(V))
+template <typename T>
+std::vector<T> Graph<T>::getChildren(const T &value) const
+{
+    std::vector<T> child;
+    
+    auto pred = [&value](const Edge &e)
+    {
+        return (e.first == value);
+    };
+    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    
+    while (eIt != edgeSet_.end())
+    {
+        child.push_back((*eIt).second);
+        eIt = find_if(++eIt, edgeSet_.end(), pred);
+    }
+    
+    return child;
+}
+
+// complexity: O(V*log(V))
+template <typename T>
+std::vector<T> Graph<T>::getParents(const T &value) const
+{
+    std::vector<T> parent;
+    
+    auto pred = [&value](const Edge &e)
+    {
+        return (e.second == value);
+    };
+    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    
+    while (eIt != edgeSet_.end())
+    {
+        parent.push_back((*eIt).first);
+        eIt = find_if(++eIt, edgeSet_.end(), pred);
+    }
+    
+    return parent;
+}
+
+// complexity: O(V^2*log(V))
+template <typename T>
+std::vector<T> Graph<T>::getRoots(void) const
+{
+    std::vector<T> root;
+    
+    for (auto &v: isMarked_)
+    {
+        auto parent = getParents(v.first);
+        
+        if (parent.size() == 0)
+        {
+            root.push_back(v.first);
+        }
+    }
+    
+    return root;
+}
+
+// complexity: O(V^2*log(V))
+template <typename T>
+std::vector<Graph<T>> Graph<T>::getConnectedComponents(void) const
+{
+    std::vector<Graph<T>> res;
+    Graph<T>              copy(*this);
+    
+    while (copy.size() > 0)
+    {
+        copy.depthFirstSearch();
+        res.push_back(copy);
+        res.back().removeUnmarked();
+        res.back().unmarkAll();
+        copy.removeMarked();
+        copy.unmarkAll();
+    }
+    
+    return res;
+}
+
+// topological sort using a directed DFS algorithm
+// complexity: O(V*log(V))
+template <typename T>
+std::vector<T> Graph<T>::topoSort(void)
+{
+    std::stack<T>     buf;
+    std::vector<T>    res;
+    const T           *vPt;
+    std::map<T, bool> tmpMarked(isMarked_);
+
+    // visit function
+    std::function<void(const T &)> visit = [&](const T &v)
+    {
+        if (tmpMarked.at(v))
+        {
+            HADRON_ERROR("cannot topologically sort a cyclic graph");
+        }
+        if (!isMarked(v))
+        {
+            std::vector<T> child = getChildren(v);
+
+            tmpMarked[v] = true;
+            for (auto &c: child)
+            {
+                visit(c);
+            }
+            mark(v);
+            tmpMarked[v] = false;
+            buf.push(v);
+        }
+    };
+    
+    // reset temporary marks
+    for (auto &v: tmpMarked)
+    {
+        tmpMarked.at(v.first) = false;
+    }
+    
+    // loop on unmarked vertices
+    unmarkAll();
+    vPt = getFirstUnmarked();
+    while (vPt)
+    {
+        visit(*vPt);
+        vPt = getFirstUnmarked();
+    }
+    unmarkAll();
+    
+    // create result vector
+    while (!buf.empty())
+    {
+        res.push_back(buf.top());
+        buf.pop();
+    }
+    
+    return res;
+}
+
+// random version of the topological sort
+// complexity: O(V*log(V))
+template <typename T>
+template <typename Gen>
+std::vector<T> Graph<T>::topoSort(Gen &gen)
+{
+    std::stack<T>     buf;
+    std::vector<T>    res;
+    const T           *vPt;
+    std::map<T, bool> tmpMarked(isMarked_);
+    
+    // visit function
+    std::function<void(const T &)> visit = [&](const T &v)
+    {
+        if (tmpMarked.at(v))
+        {
+            HADRON_ERROR("cannot topologically sort a cyclic graph");
+        }
+        if (!isMarked(v))
+        {
+            std::vector<T> child = getChildren(v);
+            
+            tmpMarked[v] = true;
+            std::shuffle(child.begin(), child.end(), gen);
+            for (auto &c: child)
+            {
+                visit(c);
+            }
+            mark(v);
+            tmpMarked[v] = false;
+            buf.push(v);
+        }
+    };
+    
+    // reset temporary marks
+    for (auto &v: tmpMarked)
+    {
+        tmpMarked.at(v.first) = false;
+    }
+    
+    // loop on unmarked vertices
+    unmarkAll();
+    vPt = getRandomUnmarked(gen);
+    while (vPt)
+    {
+        visit(*vPt);
+        vPt = getRandomUnmarked(gen);
+    }
+    unmarkAll();
+    
+    // create result vector
+    while (!buf.empty())
+    {
+        res.push_back(buf.top());
+        buf.pop();
+    }
+    
+    return res;
+}
+
+// generate all possible topological sorts
+// Y. L. Varol & D. Rotem, Comput. J. 24(1), pp. 83–84, 1981
+// http://comjnl.oupjournals.org/cgi/doi/10.1093/comjnl/24.1.83
+// complexity: O(V*log(V)) (from the paper, but really ?)
+template <typename T>
+std::vector<std::vector<T>> Graph<T>::allTopoSort(void)
+{
+    std::vector<std::vector<T>>    res;
+    std::map<T, std::map<T, bool>> iMat;
+    
+    // create incidence matrix
+    for (auto &v1: isMarked_)
+    for (auto &v2: isMarked_)
+    {
+        iMat[v1.first][v2.first] = false;
+    }
+    for (auto &v: isMarked_)
+    {
+        auto cVec = getChildren(v.first);
+        
+        for (auto &c: cVec)
+        {
+            iMat[v.first][c] = true;
+        }
+    }
+    
+    // generate initial topological sort
+    res.push_back(topoSort());
+    
+    // generate all other topological sorts by permutation
+    std::vector<T>            p = res[0];
+    const unsigned int        n = size();
+    std::vector<unsigned int> loc(n);
+    unsigned int              i, k, k1;
+    T                         obj_k, obj_k1;
+    bool                      isFinal;
+    
+    for (unsigned int j = 0; j < n; ++j)
+    {
+        loc[j] = j;
+    }
+    i = 0;
+    while (i < n-1)
+    {
+        k      = loc[i];
+        k1     = k + 1;
+        obj_k  = p[k];
+        if (k1 >= n)
+        {
+            isFinal = true;
+            obj_k1  = obj_k;
+        }
+        else
+        {
+            isFinal = false;
+            obj_k1  = p[k1];
+        }
+        if (iMat[res[0][i]][obj_k1] or isFinal)
+        {
+            for (unsigned int l = k; l >= i + 1; --l)
+            {
+                p[l]   = p[l-1];
+            }
+            p[i]   = obj_k;
+            loc[i] = i;
+            i++;
+        }
+        else
+        {
+            p[k]   = obj_k1;
+            p[k1]  = obj_k;
+            loc[i] = k1;
+            i      = 0;
+            res.push_back(p);
+        }
+    }
+    
+    return res;
+}
+
+// build depedency matrix from topological sorts ///////////////////////////////
+// complexity: something like O(V^2*log(V!))
+template <typename T>
+std::map<T, std::map<T, bool>>
+makeDependencyMatrix(const std::vector<std::vector<T>> &topSort)
+{
+    std::map<T, std::map<T, bool>> m;
+    const std::vector<T>           &vList = topSort[0];
+    
+    for (auto &v1: vList)
+    for (auto &v2: vList)
+    {
+        bool dep = true;
+        
+        for (auto &t: topSort)
+        {
+            auto i1 = std::find(t.begin(), t.end(), v1);
+            auto i2 = std::find(t.begin(), t.end(), v2);
+            
+            dep = dep and (i1 - i2 > 0);
+            if (!dep) break;
+        }
+        m[v1][v2] = dep;
+    }
+    
+    return m;
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Graph_hpp_

extras/Hadrons/HadronsXmlRun.cc

@@ -0,0 +1,80 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/HadronsXmlRun.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Application.hpp>
+
+using namespace Grid;
+using namespace QCD;
+using namespace Hadrons;
+
+int main(int argc, char *argv[])
+{
+    // parse command line
+    std::string parameterFileName, scheduleFileName = "";
+    
+    if (argc < 2)
+    {
+        std::cerr << "usage: " << argv[0] << " <parameter file> [<precomputed schedule>] [Grid options]";
+        std::cerr << std::endl;
+        std::exit(EXIT_FAILURE);
+    }
+    parameterFileName = argv[1];
+    if (argc > 2)
+    {
+        if (argv[2][0] != '-')
+        {
+            scheduleFileName = argv[2];
+        }
+    }
+    
+    // initialization
+    Grid_init(&argc, &argv);
+    HadronsLogError.Active(GridLogError.isActive());
+    HadronsLogWarning.Active(GridLogWarning.isActive());
+    HadronsLogMessage.Active(GridLogMessage.isActive());
+    HadronsLogIterative.Active(GridLogIterative.isActive());
+    HadronsLogDebug.Active(GridLogDebug.isActive());
+    LOG(Message) << "Grid initialized" << std::endl;
+    
+    // execution
+    Application application(parameterFileName);
+    
+    application.parseParameterFile(parameterFileName);
+    if (!scheduleFileName.empty())
+    {
+        application.loadSchedule(scheduleFileName);
+    }
+    application.run();
+    
+    // epilogue
+    LOG(Message) << "Grid is finalizing now" << std::endl;
+    Grid_finalize();
+    
+    return EXIT_SUCCESS;
+}

extras/Hadrons/HadronsXmlSchedule.cc

@@ -0,0 +1,72 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/HadronsXmlSchedule.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Application.hpp>
+
+using namespace Grid;
+using namespace QCD;
+using namespace Hadrons;
+
+int main(int argc, char *argv[])
+{
+    // parse command line
+    std::string parameterFileName, scheduleFileName;
+    
+    if (argc < 3)
+    {
+        std::cerr << "usage: " << argv[0] << " <parameter file> <schedule output> [Grid options]";
+        std::cerr << std::endl;
+        std::exit(EXIT_FAILURE);
+    }
+    parameterFileName = argv[1];
+    scheduleFileName  = argv[2];
+    
+    // initialization
+    Grid_init(&argc, &argv);
+    HadronsLogError.Active(GridLogError.isActive());
+    HadronsLogWarning.Active(GridLogWarning.isActive());
+    HadronsLogMessage.Active(GridLogMessage.isActive());
+    HadronsLogIterative.Active(GridLogIterative.isActive());
+    HadronsLogDebug.Active(GridLogDebug.isActive());
+    LOG(Message) << "Grid initialized" << std::endl;
+    
+    // execution
+    Application application;
+    
+    application.parseParameterFile(parameterFileName);
+    application.schedule();
+    application.printSchedule();
+    application.saveSchedule(scheduleFileName);
+    
+    // epilogue
+    LOG(Message) << "Grid is finalizing now" << std::endl;
+    Grid_finalize();
+    
+    return EXIT_SUCCESS;
+}

extras/Hadrons/Makefile.am

@@ -0,0 +1,29 @@
+lib_LIBRARIES = libHadrons.a
+bin_PROGRAMS  = HadronsXmlRun HadronsXmlSchedule
+
+include modules.inc
+
+libHadrons_a_SOURCES = \
+    $(modules_cc)      \
+    Application.cc     \
+    Environment.cc     \
+    Global.cc          \
+    Module.cc
+libHadrons_adir = $(pkgincludedir)/Hadrons
+nobase_libHadrons_a_HEADERS = \
+	$(modules_hpp)            \
+	Application.hpp           \
+	Environment.hpp           \
+	Factory.hpp               \
+	GeneticScheduler.hpp      \
+	Global.hpp                \
+	Graph.hpp                 \
+	Module.hpp                \
+	Modules.hpp               \
+	ModuleFactory.hpp
+
+HadronsXmlRun_SOURCES = HadronsXmlRun.cc
+HadronsXmlRun_LDADD   = libHadrons.a -lGrid
+
+HadronsXmlSchedule_SOURCES = HadronsXmlSchedule.cc
+HadronsXmlSchedule_LDADD   = libHadrons.a -lGrid

extras/Hadrons/Module.cc

@@ -0,0 +1,71 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Module.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Module.hpp>
+
+using namespace Grid;
+using namespace QCD;
+using namespace Hadrons;
+
+/******************************************************************************
+ *                       ModuleBase implementation                            *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+ModuleBase::ModuleBase(const std::string name)
+: name_(name)
+, env_(Environment::getInstance())
+{}
+
+// access //////////////////////////////////////////////////////////////////////
+std::string ModuleBase::getName(void) const
+{
+    return name_;
+}
+
+Environment & ModuleBase::env(void) const
+{
+    return env_;
+}
+
+// get factory registration name if available
+std::string ModuleBase::getRegisteredName(void)
+{
+    HADRON_ERROR("module '" + getName() + "' has a type not registered"
+                 + " in the factory");
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void ModuleBase::operator()(void)
+{
+    setup();
+    if (!env().isDryRun())
+    {
+        execute();
+    }
+}

extras/Hadrons/Module.hpp

@@ -0,0 +1,198 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Module.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Module_hpp_
+#define Hadrons_Module_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Environment.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+// module registration macros
+#define MODULE_REGISTER(mod, base)\
+class mod: public base\
+{\
+public:\
+    typedef base Base;\
+    using Base::Base;\
+    virtual std::string getRegisteredName(void)\
+    {\
+        return std::string(#mod);\
+    }\
+};\
+class mod##ModuleRegistrar\
+{\
+public:\
+    mod##ModuleRegistrar(void)\
+    {\
+        ModuleFactory &modFac = ModuleFactory::getInstance();\
+        modFac.registerBuilder(#mod, [&](const std::string name)\
+                              {\
+                                  return std::unique_ptr<mod>(new mod(name));\
+                              });\
+    }\
+};\
+static mod##ModuleRegistrar mod##ModuleRegistrarInstance;
+
+#define MODULE_REGISTER_NS(mod, base, ns)\
+class mod: public base\
+{\
+public:\
+    typedef base Base;\
+    using Base::Base;\
+    virtual std::string getRegisteredName(void)\
+    {\
+        return std::string(#ns "::" #mod);\
+    }\
+};\
+class ns##mod##ModuleRegistrar\
+{\
+public:\
+    ns##mod##ModuleRegistrar(void)\
+    {\
+        ModuleFactory &modFac = ModuleFactory::getInstance();\
+        modFac.registerBuilder(#ns "::" #mod, [&](const std::string name)\
+                              {\
+                                  return std::unique_ptr<ns::mod>(new ns::mod(name));\
+                              });\
+    }\
+};\
+static ns##mod##ModuleRegistrar ns##mod##ModuleRegistrarInstance;
+
+#define ARG(...) __VA_ARGS__
+
+/******************************************************************************
+ *                            Module class                                    *
+ ******************************************************************************/
+// base class
+class ModuleBase
+{
+public:
+    // constructor
+    ModuleBase(const std::string name);
+    // destructor
+    virtual ~ModuleBase(void) = default;
+    // access
+    std::string getName(void) const;
+    Environment &env(void) const;
+    // get factory registration name if available
+    virtual std::string getRegisteredName(void);
+    // dependencies/products
+    virtual std::vector<std::string> getInput(void) = 0;
+    virtual std::vector<std::string> getOutput(void) = 0;
+    // parse parameters
+    virtual void parseParameters(XmlReader &reader, const std::string name) = 0;
+    virtual void saveParameters(XmlWriter &writer, const std::string name) = 0;
+    // setup
+    virtual void setup(void) {};
+    // execution
+    void operator()(void);
+    virtual void execute(void) = 0;
+private:
+    std::string name_;
+    Environment &env_;
+};
+
+// derived class, templating the parameter class
+template <typename P>
+class Module: public ModuleBase
+{
+public:
+    typedef P Par;
+public:
+    // constructor
+    Module(const std::string name);
+    // destructor
+    virtual ~Module(void) = default;
+    // parse parameters
+    virtual void parseParameters(XmlReader &reader, const std::string name);
+    virtual void saveParameters(XmlWriter &writer, const std::string name);
+    // parameter access
+    const P & par(void) const;
+    void      setPar(const P &par);
+private:
+    P par_;
+};
+
+// no parameter type
+class NoPar {};
+
+template <>
+class Module<NoPar>: public ModuleBase
+{
+public:
+    // constructor
+    Module(const std::string name): ModuleBase(name) {};
+    // destructor
+    virtual ~Module(void) = default;
+    // parse parameters (do nothing)
+    virtual void parseParameters(XmlReader &reader, const std::string name) {};
+    virtual void saveParameters(XmlWriter &writer, const std::string name)
+    {
+        push(writer, "options");
+        pop(writer);
+    };
+};
+
+/******************************************************************************
+ *                           Template implementation                          *
+ ******************************************************************************/
+template <typename P>
+Module<P>::Module(const std::string name)
+: ModuleBase(name)
+{}
+
+template <typename P>
+void Module<P>::parseParameters(XmlReader &reader, const std::string name)
+{
+    read(reader, name, par_);
+}
+
+template <typename P>
+void Module<P>::saveParameters(XmlWriter &writer, const std::string name)
+{
+    write(writer, name, par_);
+}
+
+template <typename P>
+const P & Module<P>::par(void) const
+{
+    return par_;
+}
+
+template <typename P>
+void Module<P>::setPar(const P &par)
+{
+    par_ = par;
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Module_hpp_

extras/Hadrons/ModuleFactory.hpp

@@ -0,0 +1,49 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/ModuleFactory.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_ModuleFactory_hpp_
+#define Hadrons_ModuleFactory_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Factory.hpp>
+#include <Grid/Hadrons/Module.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                            ModuleFactory                                   *
+ ******************************************************************************/
+class ModuleFactory: public Factory<ModuleBase>
+{
+    SINGLETON_DEFCTOR(ModuleFactory)
+};
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_ModuleFactory_hpp_

extras/Hadrons/Modules.hpp

@@ -0,0 +1,25 @@
+#include <Grid/Hadrons/Modules/MAction/DWF.hpp>
+#include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
+#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
+#include <Grid/Hadrons/Modules/MFermion/GaugeProp.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Load.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
+#include <Grid/Hadrons/Modules/MGauge/StochEm.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
+#include <Grid/Hadrons/Modules/MLoop/NoiseLoop.hpp>
+#include <Grid/Hadrons/Modules/MScalar/ChargedProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/FreeProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
+#include <Grid/Hadrons/Modules/MSink/Point.hpp>
+#include <Grid/Hadrons/Modules/MSolver/RBPrecCG.hpp>
+#include <Grid/Hadrons/Modules/MSource/Point.hpp>
+#include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
+#include <Grid/Hadrons/Modules/MSource/Wall.hpp>
+#include <Grid/Hadrons/Modules/MSource/Z2.hpp>

extras/Hadrons/Modules/MAction/DWF.hpp

@@ -0,0 +1,140 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/DWF.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MAction_DWF_hpp_
+#define Hadrons_MAction_DWF_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                     Domain wall quark action                               *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MAction)
+
+class DWFPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(DWFPar,
+                                    std::string, gauge,
+                                    unsigned int, Ls,
+                                    double      , mass,
+                                    double      , M5,
+                                    std::string , boundary);
+};
+
+template <typename FImpl>
+class TDWF: public Module<DWFPar>
+{
+public:
+    FGS_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TDWF(const std::string name);
+    // destructor
+    virtual ~TDWF(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(DWF, TDWF<FIMPL>, MAction);
+
+/******************************************************************************
+ *                        DWF template implementation                         *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TDWF<FImpl>::TDWF(const std::string name)
+: Module<DWFPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TDWF<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().gauge};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TDWF<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDWF<FImpl>::setup(void)
+{
+    unsigned int size;
+    
+    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
+    env().registerObject(getName(), size, par().Ls);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDWF<FImpl>::execute(void)
+{
+    LOG(Message) << "Setting up domain wall fermion matrix with m= "
+                 << par().mass << ", M5= " << par().M5 << " and Ls= "
+                 << par().Ls << " using gauge field '" << par().gauge << "'"
+                 << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary 
+                 << std::endl;
+    env().createGrid(par().Ls);
+    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
+    auto &g4     = *env().getGrid();
+    auto &grb4   = *env().getRbGrid();
+    auto &g5     = *env().getGrid(par().Ls);
+    auto &grb5   = *env().getRbGrid(par().Ls);
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename DomainWallFermion<FImpl>::ImplParams implParams(boundary);
+    FMat *fMatPt = new DomainWallFermion<FImpl>(U, g5, grb5, g4, grb4,
+                                                par().mass, par().M5,
+                                                implParams);
+    env().setObject(getName(), fMatPt);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MAction_DWF_hpp_

extras/Hadrons/Modules/MAction/Wilson.hpp

@@ -0,0 +1,132 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MAction/Wilson.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MAction_Wilson_hpp_
+#define Hadrons_MAction_Wilson_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                            TWilson quark action                            *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MAction)
+
+class WilsonPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonPar,
+                                    std::string, gauge,
+                                    double     , mass,
+                                    std::string, boundary);
+};
+
+template <typename FImpl>
+class TWilson: public Module<WilsonPar>
+{
+public:
+    FGS_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TWilson(const std::string name);
+    // destructor
+    virtual ~TWilson(void) = default;
+    // dependencies/products
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Wilson, TWilson<FIMPL>, MAction);
+
+/******************************************************************************
+ *                     TWilson template implementation                        *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TWilson<FImpl>::TWilson(const std::string name)
+: Module<WilsonPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TWilson<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().gauge};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TWilson<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWilson<FImpl>::setup(void)
+{
+    unsigned int size;
+    
+    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
+    env().registerObject(getName(), size);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWilson<FImpl>::execute()
+{
+    LOG(Message) << "Setting up TWilson fermion matrix with m= " << par().mass
+                 << " using gauge field '" << par().gauge << "'" << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary 
+                 << std::endl;
+    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
+    auto &grid   = *env().getGrid();
+    auto &gridRb = *env().getRbGrid();
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename WilsonFermion<FImpl>::ImplParams implParams(boundary);
+    FMat *fMatPt = new WilsonFermion<FImpl>(U, grid, gridRb, par().mass,
+                                            implParams);
+    env().setObject(getName(), fMatPt);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Wilson_hpp_

extras/Hadrons/Modules/MContraction/Baryon.hpp

@@ -0,0 +1,131 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Baryon.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_Baryon_hpp_
+#define Hadrons_MContraction_Baryon_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                               Baryon                                       *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class BaryonPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(BaryonPar,
+                                    std::string, q1,
+                                    std::string, q2,
+                                    std::string, q3,
+                                    std::string, output);
+};
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+class TBaryon: public Module<BaryonPar>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl1, 1);
+    FERM_TYPE_ALIASES(FImpl2, 2);
+    FERM_TYPE_ALIASES(FImpl3, 3);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        std::vector<std::vector<std::vector<Complex>>>, corr);
+    };
+public:
+    // constructor
+    TBaryon(const std::string name);
+    // destructor
+    virtual ~TBaryon(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Baryon, ARG(TBaryon<FIMPL, FIMPL, FIMPL>), MContraction);
+
+/******************************************************************************
+ *                         TBaryon implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+TBaryon<FImpl1, FImpl2, FImpl3>::TBaryon(const std::string name)
+: Module<BaryonPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getInput(void)
+{
+    std::vector<std::string> input = {par().q1, par().q2, par().q3};
+    
+    return input;
+}
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
+{
+    LOG(Message) << "Computing baryon contractions '" << getName() << "' using"
+                 << " quarks '" << par().q1 << "', '" << par().q2 << "', and '"
+                 << par().q3 << "'" << std::endl;
+    
+    CorrWriter             writer(par().output);
+    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q2);
+    LatticeComplex        c(env().getGrid());
+    Result                result;
+    
+    // FIXME: do contractions
+    
+    // write(writer, "meson", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_Baryon_hpp_

extras/Hadrons/Modules/MContraction/DiscLoop.hpp

@@ -0,0 +1,144 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/DiscLoop.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_DiscLoop_hpp_
+#define Hadrons_MContraction_DiscLoop_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                                DiscLoop                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class DiscLoopPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(DiscLoopPar,
+                                    std::string,    q_loop,
+                                    Gamma::Algebra, gamma,
+                                    std::string,    output);
+};
+
+template <typename FImpl>
+class TDiscLoop: public Module<DiscLoopPar>
+{
+    FERM_TYPE_ALIASES(FImpl,);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma,
+                                        std::vector<Complex>, corr);
+    };
+public:
+    // constructor
+    TDiscLoop(const std::string name);
+    // destructor
+    virtual ~TDiscLoop(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(DiscLoop, TDiscLoop<FIMPL>, MContraction);
+
+/******************************************************************************
+ *                       TDiscLoop implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TDiscLoop<FImpl>::TDiscLoop(const std::string name)
+: Module<DiscLoopPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TDiscLoop<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q_loop};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TDiscLoop<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDiscLoop<FImpl>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDiscLoop<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing disconnected loop contraction '" << getName() 
+                 << "' using '" << par().q_loop << "' with " << par().gamma 
+                 << " insertion." << std::endl;
+
+    CorrWriter            writer(par().output);
+    PropagatorField       &q_loop = *env().template getObject<PropagatorField>(par().q_loop);
+    LatticeComplex        c(env().getGrid());
+    Gamma                 gamma(par().gamma);
+    std::vector<TComplex> buf;
+    Result                result;
+
+    c = trace(gamma*q_loop);
+    sliceSum(c, buf, Tp);
+
+    result.gamma = par().gamma;
+    result.corr.resize(buf.size());
+    for (unsigned int t = 0; t < buf.size(); ++t)
+    {
+        result.corr[t] = TensorRemove(buf[t]);
+    }
+
+    write(writer, "disc", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_DiscLoop_hpp_

extras/Hadrons/Modules/MContraction/Gamma3pt.hpp

@@ -0,0 +1,170 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Gamma3pt.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_Gamma3pt_hpp_
+#define Hadrons_MContraction_Gamma3pt_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ * 3pt contraction with gamma matrix insertion.
+ *
+ * Schematic:
+ *
+ *             q2           q3
+ *        /----<------*------<----¬
+ *       /          gamma          \
+ *      /                           \
+ *   i *                            * f
+ *      \                          /
+ *       \                        /
+ *        \----------->----------/
+ *                   q1
+ *
+ *      trace(g5*q1*adj(q2)*g5*gamma*q3)
+ */
+
+/******************************************************************************
+ *                               Gamma3pt                                     *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class Gamma3ptPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(Gamma3ptPar,
+                                    std::string,    q1,
+                                    std::string,    q2,
+                                    std::string,    q3,
+                                    Gamma::Algebra, gamma,
+                                    std::string,    output);
+};
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+class TGamma3pt: public Module<Gamma3ptPar>
+{
+    FERM_TYPE_ALIASES(FImpl1, 1);
+    FERM_TYPE_ALIASES(FImpl2, 2);
+    FERM_TYPE_ALIASES(FImpl3, 3);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma,
+                                        std::vector<Complex>, corr);
+    };
+public:
+    // constructor
+    TGamma3pt(const std::string name);
+    // destructor
+    virtual ~TGamma3pt(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Gamma3pt, ARG(TGamma3pt<FIMPL, FIMPL, FIMPL>), MContraction);
+
+/******************************************************************************
+ *                       TGamma3pt implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+TGamma3pt<FImpl1, FImpl2, FImpl3>::TGamma3pt(const std::string name)
+: Module<Gamma3ptPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3};
+    
+    return in;
+}
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TGamma3pt<FImpl1, FImpl2, FImpl3>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TGamma3pt<FImpl1, FImpl2, FImpl3>::execute(void)
+{
+    LOG(Message) << "Computing 3pt contractions '" << getName() << "' using"
+                 << " quarks '" << par().q1 << "', '" << par().q2 << "' and '"
+                 << par().q3 << "', with " << par().gamma << " insertion." 
+                 << std::endl;
+
+    CorrWriter            writer(par().output);
+    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q3);
+    LatticeComplex        c(env().getGrid());
+    Gamma                 g5(Gamma::Algebra::Gamma5);
+    Gamma                 gamma(par().gamma);
+    std::vector<TComplex> buf;
+    Result                result;
+
+    c = trace(g5*q1*adj(q2)*(g5*gamma)*q3);
+    sliceSum(c, buf, Tp);
+
+    result.gamma = par().gamma;
+    result.corr.resize(buf.size());
+    for (unsigned int t = 0; t < buf.size(); ++t)
+    {
+        result.corr[t] = TensorRemove(buf[t]);
+    }
+
+    write(writer, "gamma3pt", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_Gamma3pt_hpp_

extras/Hadrons/Modules/MContraction/Meson.hpp

@@ -0,0 +1,244 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Meson.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+Copyright (C) 2017
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+        Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_Meson_hpp_
+#define Hadrons_MContraction_Meson_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Meson contractions
+ -----------------------------
+ 
+ * options:
+ - q1: input propagator 1 (string)
+ - q2: input propagator 2 (string)
+ - gammas: gamma products to insert at sink & source, pairs of gamma matrices 
+           (space-separated strings) in angled brackets (i.e. <g_sink g_src>),
+           in a sequence (e.g. "<Gamma5 Gamma5><Gamma5 GammaT>").
+
+           Special values: "all" - perform all possible contractions.
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0."),
+        given as multiples of (2*pi) / L.
+*/
+
+/******************************************************************************
+ *                                TMeson                                       *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+typedef std::pair<Gamma::Algebra, Gamma::Algebra> GammaPair;
+
+class MesonPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(MesonPar,
+                                    std::string, q1,
+                                    std::string, q2,
+                                    std::string, gammas,
+                                    std::string, sink,
+                                    std::string, output);
+};
+
+template <typename FImpl1, typename FImpl2>
+class TMeson: public Module<MesonPar>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl1, 1);
+    FERM_TYPE_ALIASES(FImpl2, 2);
+    FERM_TYPE_ALIASES(ScalarImplCR, Scalar);
+    SINK_TYPE_ALIASES(Scalar);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma_snk,
+                                        Gamma::Algebra, gamma_src,
+                                        std::vector<Complex>, corr);
+    };
+public:
+    // constructor
+    TMeson(const std::string name);
+    // destructor
+    virtual ~TMeson(void) = default;
+    // dependencies/products
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    virtual void parseGammaString(std::vector<GammaPair> &gammaList);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Meson, ARG(TMeson<FIMPL, FIMPL>), MContraction);
+
+/******************************************************************************
+ *                           TMeson implementation                            *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2>
+TMeson<FImpl1, FImpl2>::TMeson(const std::string name)
+: Module<MesonPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2>
+std::vector<std::string> TMeson<FImpl1, FImpl2>::getInput(void)
+{
+    std::vector<std::string> input = {par().q1, par().q2, par().sink};
+    
+    return input;
+}
+
+template <typename FImpl1, typename FImpl2>
+std::vector<std::string> TMeson<FImpl1, FImpl2>::getOutput(void)
+{
+    std::vector<std::string> output = {getName()};
+    
+    return output;
+}
+
+template <typename FImpl1, typename FImpl2>
+void TMeson<FImpl1, FImpl2>::parseGammaString(std::vector<GammaPair> &gammaList)
+{
+    gammaList.clear();
+    // Determine gamma matrices to insert at source/sink.
+    if (par().gammas.compare("all") == 0)
+    {
+        // Do all contractions.
+        for (unsigned int i = 1; i < Gamma::nGamma; i += 2)
+        {
+            for (unsigned int j = 1; j < Gamma::nGamma; j += 2)
+            {
+                gammaList.push_back(std::make_pair((Gamma::Algebra)i, 
+                                                   (Gamma::Algebra)j));
+            }
+        }
+    }
+    else
+    {
+        // Parse individual contractions from input string.
+        gammaList = strToVec<GammaPair>(par().gammas);
+    }
+}
+
+
+// execution ///////////////////////////////////////////////////////////////////
+#define mesonConnected(q1, q2, gSnk, gSrc) \
+(g5*(gSnk))*(q1)*(adj(gSrc)*g5)*adj(q2)
+
+template <typename FImpl1, typename FImpl2>
+void TMeson<FImpl1, FImpl2>::execute(void)
+{
+    LOG(Message) << "Computing meson contractions '" << getName() << "' using"
+                 << " quarks '" << par().q1 << "' and '" << par().q2 << "'"
+                 << std::endl;
+    
+    CorrWriter             writer(par().output);
+    std::vector<TComplex>  buf;
+    std::vector<Result>    result;
+    Gamma                  g5(Gamma::Algebra::Gamma5);
+    std::vector<GammaPair> gammaList;
+    int                    nt = env().getDim(Tp);
+    
+    parseGammaString(gammaList);
+    result.resize(gammaList.size());
+    for (unsigned int i = 0; i < result.size(); ++i)
+    {
+        result[i].gamma_snk = gammaList[i].first;
+        result[i].gamma_src = gammaList[i].second;
+        result[i].corr.resize(nt);
+    }
+    if (env().template isObjectOfType<SlicedPropagator1>(par().q1) and
+        env().template isObjectOfType<SlicedPropagator2>(par().q2))
+    {
+        SlicedPropagator1 &q1 = *env().template getObject<SlicedPropagator1>(par().q1);
+        SlicedPropagator2 &q2 = *env().template getObject<SlicedPropagator2>(par().q2);
+        
+        LOG(Message) << "(propagator already sinked)" << std::endl;
+        for (unsigned int i = 0; i < result.size(); ++i)
+        {
+            Gamma gSnk(gammaList[i].first);
+            Gamma gSrc(gammaList[i].second);
+            
+            for (unsigned int t = 0; t < buf.size(); ++t)
+            {
+                result[i].corr[t] = TensorRemove(trace(mesonConnected(q1[t], q2[t], gSnk, gSrc)));
+            }
+        }
+    }
+    else
+    {
+        PropagatorField1 &q1   = *env().template getObject<PropagatorField1>(par().q1);
+        PropagatorField2 &q2   = *env().template getObject<PropagatorField2>(par().q2);
+        LatticeComplex   c(env().getGrid());
+        
+        LOG(Message) << "(using sink '" << par().sink << "')" << std::endl;
+        for (unsigned int i = 0; i < result.size(); ++i)
+        {
+            Gamma       gSnk(gammaList[i].first);
+            Gamma       gSrc(gammaList[i].second);
+            std::string ns;
+                
+            ns = env().getModuleNamespace(env().getObjectModule(par().sink));
+            if (ns == "MSource")
+            {
+                PropagatorField1 &sink =
+                    *env().template getObject<PropagatorField1>(par().sink);
+                
+                c = trace(mesonConnected(q1, q2, gSnk, gSrc)*sink);
+                sliceSum(c, buf, Tp);
+            }
+            else if (ns == "MSink")
+            {
+                SinkFnScalar &sink = *env().template getObject<SinkFnScalar>(par().sink);
+                
+                c   = trace(mesonConnected(q1, q2, gSnk, gSrc));
+                buf = sink(c);
+            }
+            for (unsigned int t = 0; t < buf.size(); ++t)
+            {
+                result[i].corr[t] = TensorRemove(buf[t]);
+            }
+        }
+    }
+    write(writer, "meson", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_Meson_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp

@@ -0,0 +1,114 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_WeakHamiltonian_hpp_
+#define Hadrons_MContraction_WeakHamiltonian_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonian                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+/*******************************************************************************
+ * Utilities for contractions involving the Weak Hamiltonian.
+ ******************************************************************************/
+//// Sum and store correlator.
+#define MAKE_DIAG(exp, buf, res, n)\
+sliceSum(exp, buf, Tp);\
+res.name = (n);\
+res.corr.resize(buf.size());\
+for (unsigned int t = 0; t < buf.size(); ++t)\
+{\
+    res.corr[t] = TensorRemove(buf[t]);\
+}
+
+//// Contraction of mu index: use 'mu' variable in exp.
+#define SUM_MU(buf,exp)\
+buf = zero;\
+for (unsigned int mu = 0; mu < ndim; ++mu)\
+{\
+    buf += exp;\
+}
+
+enum 
+{
+  i_V = 0,
+  i_A = 1,
+  n_i = 2
+};
+
+class WeakHamiltonianPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WeakHamiltonianPar,
+                                    std::string, q1,
+                                    std::string, q2,
+                                    std::string, q3,
+                                    std::string, q4,
+                                    std::string, output);
+};
+
+#define MAKE_WEAK_MODULE(modname)\
+class T##modname: public Module<WeakHamiltonianPar>\
+{\
+public:\
+    FERM_TYPE_ALIASES(FIMPL,)\
+    class Result: Serializable\
+    {\
+    public:\
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,\
+                                        std::string, name,\
+                                        std::vector<Complex>, corr);\
+    };\
+public:\
+    /* constructor */ \
+    T##modname(const std::string name);\
+    /* destructor */ \
+    virtual ~T##modname(void) = default;\
+    /* dependency relation */ \
+    virtual std::vector<std::string> getInput(void);\
+    virtual std::vector<std::string> getOutput(void);\
+    /* setup */ \
+    virtual void setup(void);\
+    /* execution */ \
+    virtual void execute(void);\
+    std::vector<std::string> VA_label = {"V", "A"};\
+};\
+MODULE_REGISTER_NS(modname, T##modname, MContraction);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_WeakHamiltonian_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc

@@ -0,0 +1,137 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian current-current contractions, Eye-type.
+ * 
+ * These contractions are generated by the Q1 and Q2 operators in the physical
+ * basis (see e.g. Fig 3 of arXiv:1507.03094).
+ * 
+ * Schematics:        q4                 |                  
+ *                  /-<-¬                |                             
+ *                 /     \               |             q2           q3
+ *                 \     /               |        /----<------*------<----¬                        
+ *            q2    \   /    q3          |       /          /-*-¬          \
+ *       /-----<-----* *-----<----¬      |      /          /     \          \
+ *    i *            H_W           * f   |   i *           \     /  q4      * f
+ *       \                        /      |      \           \->-/          /   
+ *        \                      /       |       \                        /       
+ *         \---------->---------/        |        \----------->----------/        
+ *                   q1                  |                   q1                  
+ *                                       |
+ *                Saucer (S)             |                  Eye (E)
+ * 
+ * S: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1]*q4*gL[mu][p_2])
+ * E: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1])*trace(q4*gL[mu][p_2])
+ */
+
+/******************************************************************************
+ *                  TWeakHamiltonianEye implementation                        *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakHamiltonianEye::TWeakHamiltonianEye(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakHamiltonianEye::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakHamiltonianEye::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakHamiltonianEye::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakHamiltonianEye::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian (Eye type) contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_eye_diag);
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> S_body(ndim, tmp1);
+    std::vector<PropagatorField> S_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  E_body(ndim, tmp2);
+    std::vector<LatticeComplex>  E_loop(ndim, tmp2);
+
+    // Setup for S-type contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        S_body[mu] = MAKE_SE_BODY(q1, q2, q3, GammaL(Gamma::gmu[mu]));
+        S_loop[mu] = MAKE_SE_LOOP(q4, GammaL(Gamma::gmu[mu]));
+    }
+
+    // Perform S-type contractions.    
+    SUM_MU(expbuf, trace(S_body[mu]*S_loop[mu]))
+    MAKE_DIAG(expbuf, corrbuf, result[S_diag], "HW_S")
+
+    // Recycle sub-expressions for E-type contractions.
+    for (unsigned int mu = 0; mu < ndim; ++mu)
+    {
+        E_body[mu] = trace(S_body[mu]);
+        E_loop[mu] = trace(S_loop[mu]);
+    }
+
+    // Perform E-type contractions.
+    SUM_MU(expbuf, E_body[mu]*E_loop[mu])
+    MAKE_DIAG(expbuf, corrbuf, result[E_diag], "HW_E")
+
+    write(writer, "HW_Eye", result);
+}

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp

@@ -0,0 +1,58 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_WeakHamiltonianEye_hpp_
+#define Hadrons_MContraction_WeakHamiltonianEye_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonianEye                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    S_diag = 0,
+    E_diag = 1,
+    n_eye_diag = 2
+};
+
+// Saucer and Eye subdiagram contractions.
+#define MAKE_SE_BODY(Q_1, Q_2, Q_3, gamma) (Q_3*g5*Q_1*adj(Q_2)*g5*gamma)
+#define MAKE_SE_LOOP(Q_loop, gamma) (Q_loop*gamma)
+
+MAKE_WEAK_MODULE(WeakHamiltonianEye)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_WeakHamiltonianEye_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc

@@ -0,0 +1,139 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian current-current contractions, Non-Eye-type.
+ * 
+ * These contractions are generated by the Q1 and Q2 operators in the physical
+ * basis (see e.g. Fig 3 of arXiv:1507.03094).
+ * 
+ * Schematic:     
+ *            q2             q3          |           q2              q3
+ *          /--<--¬       /--<--¬        |        /--<--¬         /--<--¬       
+ *         /       \     /       \       |       /       \       /       \      
+ *        /         \   /         \      |      /         \     /         \     
+ *       /           \ /           \     |     /           \   /           \    
+ *    i *             * H_W         *  f |  i *             * * H_W         * f 
+ *      \             *             |    |     \           /   \           /
+ *       \           / \           /     |      \         /     \         /    
+ *        \         /   \         /      |       \       /       \       /  
+ *         \       /     \       /       |        \-->--/         \-->--/      
+ *          \-->--/       \-->--/        |          q1               q4 
+ *            q1             q4          |
+ *                Connected (C)          |                 Wing (W)
+ *
+ * C: trace(q1*adj(q2)*g5*gL[mu]*q3*adj(q4)*g5*gL[mu])
+ * W: trace(q1*adj(q2)*g5*gL[mu])*trace(q3*adj(q4)*g5*gL[mu])
+ * 
+ */
+
+/******************************************************************************
+ *                  TWeakHamiltonianNonEye implementation                     *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakHamiltonianNonEye::TWeakHamiltonianNonEye(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakHamiltonianNonEye::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakHamiltonianNonEye::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakHamiltonianNonEye::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakHamiltonianNonEye::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian (Non-Eye type) contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+    
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_noneye_diag); 
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> C_i_side_loop(ndim, tmp1);
+    std::vector<PropagatorField> C_f_side_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  W_i_side_loop(ndim, tmp2);
+    std::vector<LatticeComplex>  W_f_side_loop(ndim, tmp2);
+
+    // Setup for C-type contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        C_i_side_loop[mu] = MAKE_CW_SUBDIAG(q1, q2, GammaL(Gamma::gmu[mu]));
+        C_f_side_loop[mu] = MAKE_CW_SUBDIAG(q3, q4, GammaL(Gamma::gmu[mu]));
+    }
+
+    // Perform C-type contractions.    
+    SUM_MU(expbuf, trace(C_i_side_loop[mu]*C_f_side_loop[mu]))
+    MAKE_DIAG(expbuf, corrbuf, result[C_diag], "HW_C")
+
+    // Recycle sub-expressions for W-type contractions.
+    for (unsigned int mu = 0; mu < ndim; ++mu)
+    {
+        W_i_side_loop[mu] = trace(C_i_side_loop[mu]);
+        W_f_side_loop[mu] = trace(C_f_side_loop[mu]);
+    }
+
+    // Perform W-type contractions.
+    SUM_MU(expbuf, W_i_side_loop[mu]*W_f_side_loop[mu])
+    MAKE_DIAG(expbuf, corrbuf, result[W_diag], "HW_W")
+
+    write(writer, "HW_NonEye", result);
+}

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp

@@ -0,0 +1,57 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_WeakHamiltonianNonEye_hpp_
+#define Hadrons_MContraction_WeakHamiltonianNonEye_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonianNonEye                              *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    W_diag = 0,
+    C_diag = 1,
+    n_noneye_diag = 2
+};
+
+// Wing and Connected subdiagram contractions
+#define MAKE_CW_SUBDIAG(Q_1, Q_2, gamma) (Q_1*adj(Q_2)*g5*gamma)
+
+MAKE_WEAK_MODULE(WeakHamiltonianNonEye)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_WeakHamiltonianNonEye_hpp_

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc

@@ -0,0 +1,135 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian + current contractions, disconnected topology for neutral 
+ * mesons.
+ * 
+ * These contractions are generated by operators Q_1,...,10 of the dS=1 Weak
+ * Hamiltonian in the physical basis and an additional current J (see e.g. 
+ * Fig 11 of arXiv:1507.03094).
+ * 
+ * Schematic:
+ *                        
+ *           q2          q4             q3
+ *       /--<--¬     /---<--¬       /---<--¬
+ *     /         \ /         \     /        \
+ *  i *           * H_W      |  J *          * f
+ *     \         / \         /     \        /
+ *      \--->---/   \-------/       \------/
+ *          q1 
+ * 
+ * options
+ * - q1: input propagator 1 (string)
+ * - q2: input propagator 2 (string)
+ * - q3: input propagator 3 (string), assumed to be sequential propagator 
+ * - q4: input propagator 4 (string), assumed to be a loop
+ * 
+ * type 1: trace(q1*adj(q2)*g5*gL[mu])*trace(loop*gL[mu])*trace(q3*g5)
+ * type 2: trace(q1*adj(q2)*g5*gL[mu]*loop*gL[mu])*trace(q3*g5)
+ */
+
+/*******************************************************************************
+ *                  TWeakNeutral4ptDisc implementation                         *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakNeutral4ptDisc::TWeakNeutral4ptDisc(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakNeutral4ptDisc::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakNeutral4ptDisc::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakNeutral4ptDisc::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakNeutral4ptDisc::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian neutral disconnected contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_neut_disc_diag);
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp(env().getGrid());
+    std::vector<PropagatorField> meson(ndim, tmp);
+    std::vector<PropagatorField> loop(ndim, tmp);
+    LatticeComplex               curr(env().getGrid());
+
+    // Setup for type 1 contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        meson[mu] = MAKE_DISC_MESON(q1, q2, GammaL(Gamma::gmu[mu]));
+        loop[mu] = MAKE_DISC_LOOP(q4, GammaL(Gamma::gmu[mu]));
+    }
+    curr = MAKE_DISC_CURR(q3, GammaL(Gamma::Algebra::Gamma5));
+
+    // Perform type 1 contractions.    
+    SUM_MU(expbuf, trace(meson[mu]*loop[mu]))
+    expbuf *= curr;
+    MAKE_DIAG(expbuf, corrbuf, result[neut_disc_1_diag], "HW_disc0_1")
+
+    // Perform type 2 contractions.
+    SUM_MU(expbuf, trace(meson[mu])*trace(loop[mu]))
+    expbuf *= curr;
+    MAKE_DIAG(expbuf, corrbuf, result[neut_disc_2_diag], "HW_disc0_2")
+
+    write(writer, "HW_disc0", result);
+}

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp

@@ -0,0 +1,59 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MContraction_WeakNeutral4ptDisc_hpp_
+#define Hadrons_MContraction_WeakNeutral4ptDisc_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakNeutral4ptDisc                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    neut_disc_1_diag = 0,
+    neut_disc_2_diag = 1,
+    n_neut_disc_diag = 2
+};
+
+// Neutral 4pt disconnected subdiagram contractions.
+#define MAKE_DISC_MESON(Q_1, Q_2, gamma) (Q_1*adj(Q_2)*g5*gamma)
+#define MAKE_DISC_LOOP(Q_LOOP, gamma) (Q_LOOP*gamma)
+#define MAKE_DISC_CURR(Q_c, gamma) (trace(Q_c*gamma))
+
+MAKE_WEAK_MODULE(WeakNeutral4ptDisc)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MContraction_WeakNeutral4ptDisc_hpp_

extras/Hadrons/Modules/MFermion/GaugeProp.hpp

@@ -0,0 +1,160 @@
+#ifndef Hadrons_MFermion_GaugeProp_hpp_
+#define Hadrons_MFermion_GaugeProp_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                                GaugeProp                                   *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MFermion)
+
+class GaugePropPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(GaugePropPar,
+                                    std::string, source,
+                                    std::string, solver);
+};
+
+template <typename FImpl>
+class TGaugeProp: public Module<GaugePropPar>
+{
+public:
+    FGS_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TGaugeProp(const std::string name);
+    // destructor
+    virtual ~TGaugeProp(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+private:
+    unsigned int Ls_;
+    SolverFn     *solver_{nullptr};
+};
+
+MODULE_REGISTER_NS(GaugeProp, TGaugeProp<FIMPL>, MFermion);
+
+/******************************************************************************
+ *                      TGaugeProp implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TGaugeProp<FImpl>::TGaugeProp(const std::string name)
+: Module<GaugePropPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TGaugeProp<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().source, par().solver};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TGaugeProp<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName(), getName() + "_5d"};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TGaugeProp<FImpl>::setup(void)
+{
+    Ls_ = env().getObjectLs(par().solver);
+    env().template registerLattice<PropagatorField>(getName());
+    if (Ls_ > 1)
+    {
+        env().template registerLattice<PropagatorField>(getName() + "_5d", Ls_);
+    }
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TGaugeProp<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing quark propagator '" << getName() << "'"
+    << std::endl;
+    
+    FermionField    source(env().getGrid(Ls_)), sol(env().getGrid(Ls_)),
+    tmp(env().getGrid());
+    std::string     propName = (Ls_ == 1) ? getName() : (getName() + "_5d");
+    PropagatorField &prop    = *env().template createLattice<PropagatorField>(propName);
+    PropagatorField &fullSrc = *env().template getObject<PropagatorField>(par().source);
+    SolverFn        &solver  = *env().template getObject<SolverFn>(par().solver);
+    if (Ls_ > 1)
+    {
+        env().template createLattice<PropagatorField>(getName());
+    }
+    
+    LOG(Message) << "Inverting using solver '" << par().solver
+    << "' on source '" << par().source << "'" << std::endl;
+    for (unsigned int s = 0; s < Ns; ++s)
+    for (unsigned int c = 0; c < Nc; ++c)
+    {
+        LOG(Message) << "Inversion for spin= " << s << ", color= " << c
+        << std::endl;
+        // source conversion for 4D sources
+        if (!env().isObject5d(par().source))
+        {
+            if (Ls_ == 1)
+            {
+                PropToFerm(source, fullSrc, s, c);
+            }
+            else
+            {
+                source = zero;
+                PropToFerm(tmp, fullSrc, s, c);
+                InsertSlice(tmp, source, 0, 0);
+                InsertSlice(tmp, source, Ls_-1, 0);
+                axpby_ssp_pplus(source, 0., source, 1., source, 0, 0);
+                axpby_ssp_pminus(source, 0., source, 1., source, Ls_-1, Ls_-1);
+            }
+        }
+        // source conversion for 5D sources
+        else
+        {
+            if (Ls_ != env().getObjectLs(par().source))
+            {
+                HADRON_ERROR("Ls mismatch between quark action and source");
+            }
+            else
+            {
+                PropToFerm(source, fullSrc, s, c);
+            }
+        }
+        sol = zero;
+        solver(sol, source);
+        FermToProp(prop, sol, s, c);
+        // create 4D propagators from 5D one if necessary
+        if (Ls_ > 1)
+        {
+            PropagatorField &p4d =
+            *env().template getObject<PropagatorField>(getName());
+            
+            axpby_ssp_pminus(sol, 0., sol, 1., sol, 0, 0);
+            axpby_ssp_pplus(sol, 1., sol, 1., sol, 0, Ls_-1);
+            ExtractSlice(tmp, sol, 0, 0);
+            FermToProp(p4d, tmp, s, c);
+        }
+    }
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MFermion_GaugeProp_hpp_

extras/Hadrons/Modules/MGauge/Load.cc

@@ -0,0 +1,78 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/Load.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MGauge/Load.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+/******************************************************************************
+*                           TLoad implementation                               *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TLoad::TLoad(const std::string name)
+: Module<LoadPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TLoad::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+std::vector<std::string> TLoad::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TLoad::setup(void)
+{
+    env().registerLattice<LatticeGaugeField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TLoad::execute(void)
+{
+    FieldMetaData  header;
+    std::string fileName = par().file + "."
+                           + std::to_string(env().getTrajectory());
+    
+    LOG(Message) << "Loading NERSC configuration from file '" << fileName
+                 << "'" << std::endl;
+    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
+    NerscIO::readConfiguration(U, header, fileName);
+    LOG(Message) << "NERSC header:" << std::endl;
+    dump_meta_data(header, LOG(Message));
+}

extras/Hadrons/Modules/MGauge/Load.hpp

@@ -0,0 +1,73 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/Load.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MGauge_Load_hpp_
+#define Hadrons_MGauge_Load_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         Load a NERSC configuration                         *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class LoadPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(LoadPar,
+                                    std::string, file);
+};
+
+class TLoad: public Module<LoadPar>
+{
+public:
+    // constructor
+    TLoad(const std::string name);
+    // destructor
+    virtual ~TLoad(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Load, TLoad, MGauge);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_Load_hpp_

extras/Hadrons/Modules/MGauge/Random.cc

@@ -0,0 +1,69 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/Random.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+/******************************************************************************
+*                           TRandom implementation                            *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TRandom::TRandom(const std::string name)
+: Module<NoPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TRandom::getInput(void)
+{
+    return std::vector<std::string>();
+}
+
+std::vector<std::string> TRandom::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TRandom::setup(void)
+{
+    env().registerLattice<LatticeGaugeField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TRandom::execute(void)
+{
+    LOG(Message) << "Generating random gauge configuration" << std::endl;
+    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
+    SU3::HotConfiguration(*env().get4dRng(), U);
+}

extras/Hadrons/Modules/MGauge/Random.hpp

@@ -0,0 +1,66 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/Random.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MGauge_Random_hpp_
+#define Hadrons_MGauge_Random_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                             Random gauge                                   *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class TRandom: public Module<NoPar>
+{
+public:
+    // constructor
+    TRandom(const std::string name);
+    // destructor
+    virtual ~TRandom(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Random, TRandom, MGauge);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_Random_hpp_

extras/Hadrons/Modules/MGauge/StochEm.cc

@@ -0,0 +1,88 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/StochEm.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Hadrons/Modules/MGauge/StochEm.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+/******************************************************************************
+*                  TStochEm implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TStochEm::TStochEm(const std::string name)
+: Module<StochEmPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TStochEm::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+std::vector<std::string> TStochEm::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TStochEm::setup(void)
+{
+    if (!env().hasRegisteredObject("_" + getName() + "_weight"))
+    {
+        env().registerLattice<EmComp>("_" + getName() + "_weight");
+    }
+    env().registerLattice<EmField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TStochEm::execute(void)
+{
+    PhotonR photon(par().gauge, par().zmScheme);
+    EmField &a = *env().createLattice<EmField>(getName());
+    EmComp  *w;
+    
+    if (!env().hasCreatedObject("_" + getName() + "_weight"))
+    {
+        LOG(Message) << "Caching stochatic EM potential weight (gauge: "
+                     << par().gauge << ", zero-mode scheme: "
+                     << par().zmScheme << ")..." << std::endl;
+        w = env().createLattice<EmComp>("_" + getName() + "_weight");
+        photon.StochasticWeight(*w);
+    }
+    else
+    {
+        w = env().getObject<EmComp>("_" + getName() + "_weight");
+    }
+    LOG(Message) << "Generating stochatic EM potential..." << std::endl;
+    photon.StochasticField(a, *env().get4dRng(), *w);
+}

extras/Hadrons/Modules/MGauge/StochEm.hpp

@@ -0,0 +1,75 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/StochEm.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef Hadrons_MGauge_StochEm_hpp_
+#define Hadrons_MGauge_StochEm_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         StochEm                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class StochEmPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(StochEmPar,
+                                    PhotonR::Gauge,    gauge,
+                                    PhotonR::ZmScheme, zmScheme);
+};
+
+class TStochEm: public Module<StochEmPar>
+{
+public:
+    typedef PhotonR::GaugeField     EmField;
+    typedef PhotonR::GaugeLinkField EmComp;
+public:
+    // constructor
+    TStochEm(const std::string name);
+    // destructor
+    virtual ~TStochEm(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(StochEm, TStochEm, MGauge);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_StochEm_hpp_

extras/Hadrons/Modules/MGauge/Unit.cc

@@ -0,0 +1,69 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/Unit.cc
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MGauge;
+
+/******************************************************************************
+*                            TUnit implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TUnit::TUnit(const std::string name)
+: Module<NoPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TUnit::getInput(void)
+{
+    return std::vector<std::string>();
+}
+
+std::vector<std::string> TUnit::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TUnit::setup(void)
+{
+    env().registerLattice<LatticeGaugeField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TUnit::execute(void)
+{
+    LOG(Message) << "Creating unit gauge configuration" << std::endl;
+    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
+    SU3::ColdConfiguration(*env().get4dRng(), U);
+}

extras/Hadrons/Modules/MGauge/Unit.hpp

@@ -0,0 +1,66 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MGauge/Unit.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MGauge_Unit_hpp_
+#define Hadrons_MGauge_Unit_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                              Unit gauge                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MGauge)
+
+class TUnit: public Module<NoPar>
+{
+public:
+    // constructor
+    TUnit(const std::string name);
+    // destructor
+    virtual ~TUnit(void) = default;
+    // dependencies/products
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Unit, TUnit, MGauge);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MGauge_Unit_hpp_

extras/Hadrons/Modules/MLoop/NoiseLoop.hpp

@@ -0,0 +1,132 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MLoop/NoiseLoop.hpp
+
+Copyright (C) 2016
+
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MLoop_NoiseLoop_hpp_
+#define Hadrons_MLoop_NoiseLoop_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Noise loop propagator
+ -----------------------------
+ * loop_x = q_x * adj(eta_x)
+ 
+ * options:
+ - q = Result of inversion on noise source.
+ - eta = noise source.
+
+ */
+
+
+/******************************************************************************
+ *                         NoiseLoop                                          *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MLoop)
+
+class NoiseLoopPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(NoiseLoopPar,
+                                    std::string, q,
+                                    std::string, eta);
+};
+
+template <typename FImpl>
+class TNoiseLoop: public Module<NoiseLoopPar>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TNoiseLoop(const std::string name);
+    // destructor
+    virtual ~TNoiseLoop(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(NoiseLoop, TNoiseLoop<FIMPL>, MLoop);
+
+/******************************************************************************
+ *                 TNoiseLoop implementation                                  *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TNoiseLoop<FImpl>::TNoiseLoop(const std::string name)
+: Module<NoiseLoopPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TNoiseLoop<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q, par().eta};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TNoiseLoop<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TNoiseLoop<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TNoiseLoop<FImpl>::execute(void)
+{
+    PropagatorField &loop = *env().template createLattice<PropagatorField>(getName());
+    PropagatorField &q    = *env().template getObject<PropagatorField>(par().q);
+    PropagatorField &eta  = *env().template getObject<PropagatorField>(par().eta);
+    loop = q*adj(eta);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MLoop_NoiseLoop_hpp_

extras/Hadrons/Modules/MScalar/ChargedProp.cc

@@ -0,0 +1,226 @@
+#include <Grid/Hadrons/Modules/MScalar/ChargedProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MScalar;
+
+/******************************************************************************
+*                     TChargedProp implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TChargedProp::TChargedProp(const std::string name)
+: Module<ChargedPropPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TChargedProp::getInput(void)
+{
+    std::vector<std::string> in = {par().source, par().emField};
+    
+    return in;
+}
+
+std::vector<std::string> TChargedProp::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TChargedProp::setup(void)
+{
+    freeMomPropName_ = FREEMOMPROP(par().mass);
+    phaseName_.clear();
+    for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+    {
+        phaseName_.push_back("_shiftphase_" + std::to_string(mu));
+    }
+    GFSrcName_ = "_" + getName() + "_DinvSrc";
+    if (!env().hasRegisteredObject(freeMomPropName_))
+    {
+        env().registerLattice<ScalarField>(freeMomPropName_);
+    }
+    if (!env().hasRegisteredObject(phaseName_[0]))
+    {
+        for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+        {
+            env().registerLattice<ScalarField>(phaseName_[mu]);
+        }
+    }
+    if (!env().hasRegisteredObject(GFSrcName_))
+    {
+        env().registerLattice<ScalarField>(GFSrcName_);
+    }
+    env().registerLattice<ScalarField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TChargedProp::execute(void)
+{
+    // CACHING ANALYTIC EXPRESSIONS
+    ScalarField &source = *env().getObject<ScalarField>(par().source);
+    Complex     ci(0.0,1.0);
+    FFT         fft(env().getGrid());
+    
+    // cache free scalar propagator
+    if (!env().hasCreatedObject(freeMomPropName_))
+    {
+        LOG(Message) << "Caching momentum space free scalar propagator"
+                     << " (mass= " << par().mass << ")..." << std::endl;
+        freeMomProp_ = env().createLattice<ScalarField>(freeMomPropName_);
+        SIMPL::MomentumSpacePropagator(*freeMomProp_, par().mass);
+    }
+    else
+    {
+        freeMomProp_ = env().getObject<ScalarField>(freeMomPropName_);
+    }
+    // cache G*F*src
+    if (!env().hasCreatedObject(GFSrcName_))
+        
+    {
+        GFSrc_ = env().createLattice<ScalarField>(GFSrcName_);
+        fft.FFT_all_dim(*GFSrc_, source, FFT::forward);
+        *GFSrc_ = (*freeMomProp_)*(*GFSrc_);
+    }
+    else
+    {
+        GFSrc_ = env().getObject<ScalarField>(GFSrcName_);
+    }
+    // cache phases
+    if (!env().hasCreatedObject(phaseName_[0]))
+    {
+        std::vector<int> &l = env().getGrid()->_fdimensions;
+        
+        LOG(Message) << "Caching shift phases..." << std::endl;
+        for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+        {
+            Real    twoPiL = M_PI*2./l[mu];
+            
+            phase_.push_back(env().createLattice<ScalarField>(phaseName_[mu]));
+            LatticeCoordinate(*(phase_[mu]), mu);
+            *(phase_[mu]) = exp(ci*twoPiL*(*(phase_[mu])));
+        }
+    }
+    else
+    {
+        for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+        {
+            phase_.push_back(env().getObject<ScalarField>(phaseName_[mu]));
+        }
+    }
+
+    // PROPAGATOR CALCULATION
+    LOG(Message) << "Computing charged scalar propagator"
+                 << " (mass= " << par().mass
+                 << ", charge= " << par().charge << ")..." << std::endl;
+    
+    ScalarField &prop   = *env().createLattice<ScalarField>(getName());
+    ScalarField buf(env().getGrid());
+    ScalarField &GFSrc = *GFSrc_, &G = *freeMomProp_;
+    double      q = par().charge;
+    
+    // G*F*Src
+    prop = GFSrc;
+
+    // - q*G*momD1*G*F*Src (momD1 = F*D1*Finv)
+    buf = GFSrc;
+    momD1(buf, fft);
+    buf = G*buf;
+    prop = prop - q*buf;
+
+    // + q^2*G*momD1*G*momD1*G*F*Src (here buf = G*momD1*G*F*Src)
+    momD1(buf, fft);
+    prop = prop + q*q*G*buf;
+
+    // - q^2*G*momD2*G*F*Src (momD2 = F*D2*Finv)
+    buf = GFSrc;
+    momD2(buf, fft);
+    prop = prop - q*q*G*buf;
+
+    // final FT
+    fft.FFT_all_dim(prop, prop, FFT::backward);
+    
+    // OUTPUT IF NECESSARY
+    if (!par().output.empty())
+    {
+        std::string           filename = par().output + "." +
+                                         std::to_string(env().getTrajectory());
+        
+        LOG(Message) << "Saving zero-momentum projection to '"
+                     << filename << "'..." << std::endl;
+        
+        CorrWriter            writer(filename);
+        std::vector<TComplex> vecBuf;
+        std::vector<Complex>  result;
+        
+        sliceSum(prop, vecBuf, Tp);
+        result.resize(vecBuf.size());
+        for (unsigned int t = 0; t < vecBuf.size(); ++t)
+        {
+            result[t] = TensorRemove(vecBuf[t]);
+        }
+        write(writer, "charge", q);
+        write(writer, "prop", result);
+    }
+}
+
+void TChargedProp::momD1(ScalarField &s, FFT &fft)
+{
+    EmField     &A = *env().getObject<EmField>(par().emField);
+    ScalarField buf(env().getGrid()), result(env().getGrid()),
+                Amu(env().getGrid());
+    Complex     ci(0.0,1.0);
+
+    result = zero;
+
+    for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+    {
+        Amu = peekLorentz(A, mu);
+        buf = (*phase_[mu])*s;
+        fft.FFT_all_dim(buf, buf, FFT::backward);
+        buf = Amu*buf;
+        fft.FFT_all_dim(buf, buf, FFT::forward);
+        result = result - ci*buf;
+    }
+    fft.FFT_all_dim(s, s, FFT::backward);
+    for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+    {
+        Amu = peekLorentz(A, mu);
+        buf = Amu*s;
+        fft.FFT_all_dim(buf, buf, FFT::forward);
+        result = result + ci*adj(*phase_[mu])*buf;
+    }
+
+    s = result;
+}
+
+void TChargedProp::momD2(ScalarField &s, FFT &fft)
+{
+    EmField     &A = *env().getObject<EmField>(par().emField);
+    ScalarField buf(env().getGrid()), result(env().getGrid()),
+                Amu(env().getGrid());
+
+    result = zero;
+    
+    for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+    {
+        Amu = peekLorentz(A, mu);
+        buf = (*phase_[mu])*s;
+        fft.FFT_all_dim(buf, buf, FFT::backward);
+        buf = Amu*Amu*buf;
+        fft.FFT_all_dim(buf, buf, FFT::forward);
+        result = result + .5*buf;
+    }
+    fft.FFT_all_dim(s, s, FFT::backward);
+    for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+    {
+        Amu = peekLorentz(A, mu);        
+        buf = Amu*Amu*s;
+        fft.FFT_all_dim(buf, buf, FFT::forward);
+        result = result + .5*adj(*phase_[mu])*buf;
+    }
+
+    s = result;
+}

extras/Hadrons/Modules/MScalar/ChargedProp.hpp

@@ -0,0 +1,61 @@
+#ifndef Hadrons_MScalar_ChargedProp_hpp_
+#define Hadrons_MScalar_ChargedProp_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                       Charged scalar propagator                            *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MScalar)
+
+class ChargedPropPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(ChargedPropPar,
+                                    std::string, emField,
+                                    std::string, source,
+                                    double,      mass,
+                                    double,      charge,
+                                    std::string, output);
+};
+
+class TChargedProp: public Module<ChargedPropPar>
+{
+public:
+    SCALAR_TYPE_ALIASES(SIMPL,);
+    typedef PhotonR::GaugeField     EmField;
+    typedef PhotonR::GaugeLinkField EmComp;
+public:
+    // constructor
+    TChargedProp(const std::string name);
+    // destructor
+    virtual ~TChargedProp(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+private:
+    void momD1(ScalarField &s, FFT &fft);
+    void momD2(ScalarField &s, FFT &fft);
+private:
+    std::string                freeMomPropName_, GFSrcName_;
+    std::vector<std::string>   phaseName_;
+    ScalarField                *freeMomProp_, *GFSrc_;
+    std::vector<ScalarField *> phase_;
+    EmField                    *A;
+};
+
+MODULE_REGISTER_NS(ChargedProp, TChargedProp, MScalar);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MScalar_ChargedProp_hpp_

extras/Hadrons/Modules/MScalar/FreeProp.cc

@@ -0,0 +1,79 @@
+#include <Grid/Hadrons/Modules/MScalar/FreeProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MScalar;
+
+/******************************************************************************
+*                        TFreeProp implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TFreeProp::TFreeProp(const std::string name)
+: Module<FreePropPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TFreeProp::getInput(void)
+{
+    std::vector<std::string> in = {par().source};
+    
+    return in;
+}
+
+std::vector<std::string> TFreeProp::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TFreeProp::setup(void)
+{
+    freeMomPropName_ = FREEMOMPROP(par().mass);
+    
+    if (!env().hasRegisteredObject(freeMomPropName_))
+    {
+        env().registerLattice<ScalarField>(freeMomPropName_);
+    }
+    env().registerLattice<ScalarField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TFreeProp::execute(void)
+{
+    ScalarField &prop   = *env().createLattice<ScalarField>(getName());
+    ScalarField &source = *env().getObject<ScalarField>(par().source);
+    ScalarField *freeMomProp;
+
+    if (!env().hasCreatedObject(freeMomPropName_))
+    {
+        LOG(Message) << "Caching momentum space free scalar propagator"
+                     << " (mass= " << par().mass << ")..." << std::endl;
+        freeMomProp = env().createLattice<ScalarField>(freeMomPropName_);
+        SIMPL::MomentumSpacePropagator(*freeMomProp, par().mass);
+    }
+    else
+    {
+        freeMomProp = env().getObject<ScalarField>(freeMomPropName_);
+    }
+    LOG(Message) << "Computing free scalar propagator..." << std::endl;
+    SIMPL::FreePropagator(source, prop, *freeMomProp);
+    
+    if (!par().output.empty())
+    {
+        TextWriter            writer(par().output + "." +
+                                     std::to_string(env().getTrajectory()));
+        std::vector<TComplex> buf;
+        std::vector<Complex>  result;
+        
+        sliceSum(prop, buf, Tp);
+        result.resize(buf.size());
+        for (unsigned int t = 0; t < buf.size(); ++t)
+        {
+            result[t] = TensorRemove(buf[t]);
+        }
+        write(writer, "prop", result);
+    }
+}

extras/Hadrons/Modules/MScalar/FreeProp.hpp

@@ -0,0 +1,50 @@
+#ifndef Hadrons_MScalar_FreeProp_hpp_
+#define Hadrons_MScalar_FreeProp_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                               FreeProp                                     *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MScalar)
+
+class FreePropPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(FreePropPar,
+                                    std::string, source,
+                                    double,      mass,
+                                    std::string, output);
+};
+
+class TFreeProp: public Module<FreePropPar>
+{
+public:
+    SCALAR_TYPE_ALIASES(SIMPL,);
+public:
+    // constructor
+    TFreeProp(const std::string name);
+    // destructor
+    virtual ~TFreeProp(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+private:
+    std::string freeMomPropName_;
+};
+
+MODULE_REGISTER_NS(FreeProp, TFreeProp, MScalar);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MScalar_FreeProp_hpp_

extras/Hadrons/Modules/MScalar/Scalar.hpp

@@ -0,0 +1,6 @@
+#ifndef Hadrons_Scalar_hpp_
+#define Hadrons_Scalar_hpp_
+
+#define FREEMOMPROP(m) "_scalar_mom_prop_" + std::to_string(m)
+
+#endif // Hadrons_Scalar_hpp_

extras/Hadrons/Modules/MSink/Point.hpp

@@ -0,0 +1,114 @@
+#ifndef Hadrons_MSink_Point_hpp_
+#define Hadrons_MSink_Point_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                                   Point                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSink)
+
+class PointPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(PointPar,
+                                    std::string, mom);
+};
+
+template <typename FImpl>
+class TPoint: public Module<PointPar>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl,);
+    SINK_TYPE_ALIASES();
+public:
+    // constructor
+    TPoint(const std::string name);
+    // destructor
+    virtual ~TPoint(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Point,       TPoint<FIMPL>,        MSink);
+MODULE_REGISTER_NS(ScalarPoint, TPoint<ScalarImplCR>, MSink);
+
+/******************************************************************************
+ *                          TPoint implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TPoint<FImpl>::TPoint(const std::string name)
+: Module<PointPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TPoint<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TPoint<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TPoint<FImpl>::setup(void)
+{
+    unsigned int size;
+    
+    size = env().template lattice4dSize<LatticeComplex>();
+    env().registerObject(getName(), size);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TPoint<FImpl>::execute(void)
+{
+    std::vector<Real> p = strToVec<Real>(par().mom);
+    LatticeComplex    ph(env().getGrid()), coor(env().getGrid());
+    Complex           i(0.0,1.0);
+    
+    LOG(Message) << "Setting up point sink function for momentum ["
+                 << par().mom << "]" << std::endl;
+    ph = zero;
+    for(unsigned int mu = 0; mu < env().getNd(); mu++)
+    {
+        LatticeCoordinate(coor, mu);
+        ph = ph + (p[mu]/env().getGrid()->_fdimensions[mu])*coor;
+    }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    auto sink = [ph](const PropagatorField &field)
+    {
+        SlicedPropagator res;
+        PropagatorField  tmp = ph*field;
+        
+        sliceSum(tmp, res, Tp);
+        
+        return res;
+    };
+    env().setObject(getName(), new SinkFn(sink));
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MSink_Point_hpp_

extras/Hadrons/Modules/MSolver/RBPrecCG.hpp

@@ -0,0 +1,132 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MSolver/RBPrecCG.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MSolver_RBPrecCG_hpp_
+#define Hadrons_MSolver_RBPrecCG_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                     Schur red-black preconditioned CG                      *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSolver)
+
+class RBPrecCGPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(RBPrecCGPar,
+                                    std::string, action,
+                                    double     , residual);
+};
+
+template <typename FImpl>
+class TRBPrecCG: public Module<RBPrecCGPar>
+{
+public:
+    FGS_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TRBPrecCG(const std::string name);
+    // destructor
+    virtual ~TRBPrecCG(void) = default;
+    // dependencies/products
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(RBPrecCG, TRBPrecCG<FIMPL>, MSolver);
+
+/******************************************************************************
+ *                      TRBPrecCG template implementation                     *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TRBPrecCG<FImpl>::TRBPrecCG(const std::string name)
+: Module(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TRBPrecCG<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().action};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TRBPrecCG<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TRBPrecCG<FImpl>::setup(void)
+{
+    auto Ls = env().getObjectLs(par().action);
+    
+    env().registerObject(getName(), 0, Ls);
+    env().addOwnership(getName(), par().action);
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TRBPrecCG<FImpl>::execute(void)
+{
+    auto &mat   = *(env().template getObject<FMat>(par().action));
+    auto solver = [&mat, this](FermionField &sol, const FermionField &source)
+    {
+        ConjugateGradient<FermionField>           cg(par().residual, 10000);
+        SchurRedBlackDiagMooeeSolve<FermionField> schurSolver(cg);
+        
+        schurSolver(mat, source, sol);
+    };
+    
+    LOG(Message) << "setting up Schur red-black preconditioned CG for"
+                 << " action '" << par().action << "' with residual "
+                 << par().residual << std::endl;
+    env().setObject(getName(), new SolverFn(solver));
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MSolver_RBPrecCG_hpp_

extras/Hadrons/Modules/MSource/Point.hpp

@@ -0,0 +1,136 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MSource/Point.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MSource_Point_hpp_
+#define Hadrons_MSource_Point_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Point source
+ ------------
+ * src_x = delta_x,position
+ 
+ * options:
+ - position: space-separated integer sequence (e.g. "0 1 1 0")
+ 
+ */
+
+/******************************************************************************
+ *                                  TPoint                                     *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSource)
+
+class PointPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(PointPar,
+                                    std::string, position);
+};
+
+template <typename FImpl>
+class TPoint: public Module<PointPar>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TPoint(const std::string name);
+    // destructor
+    virtual ~TPoint(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Point,       TPoint<FIMPL>,        MSource);
+MODULE_REGISTER_NS(ScalarPoint, TPoint<ScalarImplCR>, MSource);
+
+/******************************************************************************
+ *                       TPoint template implementation                       *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TPoint<FImpl>::TPoint(const std::string name)
+: Module<PointPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TPoint<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TPoint<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TPoint<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TPoint<FImpl>::execute(void)
+{
+    std::vector<int> position = strToVec<int>(par().position);
+    typename SitePropagator::scalar_object id;
+    
+    LOG(Message) << "Creating point source at position [" << par().position
+                 << "]" << std::endl;
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    id  = 1.;
+    src = zero;
+    pokeSite(id, src, position);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MSource_Point_hpp_

extras/Hadrons/Modules/MSource/SeqGamma.hpp

@@ -0,0 +1,164 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MSource/SeqGamma.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+Copyright (C) 2017
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MSource_SeqGamma_hpp_
+#define Hadrons_MSource_SeqGamma_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Sequential source
+ -----------------------------
+ * src_x = q_x * theta(x_3 - tA) * theta(tB - x_3) * gamma * exp(i x.mom)
+ 
+ * options:
+ - q: input propagator (string)
+ - tA: begin timeslice (integer)
+ - tB: end timesilce (integer)
+ - gamma: gamma product to insert (integer)
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0.")
+ 
+ */
+
+/******************************************************************************
+ *                         SeqGamma                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSource)
+
+class SeqGammaPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(SeqGammaPar,
+                                    std::string,    q,
+                                    unsigned int,   tA,
+                                    unsigned int,   tB,
+                                    Gamma::Algebra, gamma,
+                                    std::string,    mom);
+};
+
+template <typename FImpl>
+class TSeqGamma: public Module<SeqGammaPar>
+{
+public:
+    FGS_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TSeqGamma(const std::string name);
+    // destructor
+    virtual ~TSeqGamma(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(SeqGamma, TSeqGamma<FIMPL>, MSource);
+
+/******************************************************************************
+ *                         TSeqGamma implementation                           *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TSeqGamma<FImpl>::TSeqGamma(const std::string name)
+: Module<SeqGammaPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TSeqGamma<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TSeqGamma<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TSeqGamma<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TSeqGamma<FImpl>::execute(void)
+{
+    if (par().tA == par().tB)
+    {
+        LOG(Message) << "Generating gamma_" << par().gamma
+                     << " sequential source at t= " << par().tA << std::endl;
+    }
+    else
+    {
+        LOG(Message) << "Generating gamma_" << par().gamma
+                     << " sequential source for "
+                     << par().tA << " <= t <= " << par().tB << std::endl;
+    }
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    PropagatorField &q   = *env().template getObject<PropagatorField>(par().q);
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             ph(env().getGrid()), coor(env().getGrid());
+    Gamma                      g(par().gamma);
+    std::vector<Real>          p;
+    Complex                    i(0.0,1.0);
+    
+    p  = strToVec<Real>(par().mom);
+    ph = zero;
+    for(unsigned int mu = 0; mu < env().getNd(); mu++)
+    {
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
+    }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    LatticeCoordinate(t, Tp);
+    src = where((t >= par().tA) and (t <= par().tB), ph*(g*q), 0.*q);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MSource_SeqGamma_hpp_

extras/Hadrons/Modules/MSource/Wall.hpp

@@ -0,0 +1,147 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MSource/Wall.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MSource_WallSource_hpp_
+#define Hadrons_MSource_WallSource_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Wall source
+ -----------------------------
+ * src_x = delta(x_3 - tW) * exp(i x.mom)
+ 
+ * options:
+ - tW: source timeslice (integer)
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0.")
+ 
+ */
+
+/******************************************************************************
+ *                         Wall                                               *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSource)
+
+class WallPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WallPar,
+                                    unsigned int, tW,
+                                    std::string, mom);
+};
+
+template <typename FImpl>
+class TWall: public Module<WallPar>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TWall(const std::string name);
+    // destructor
+    virtual ~TWall(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Wall, TWall<FIMPL>, MSource);
+
+/******************************************************************************
+ *                 TWall implementation                                       *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TWall<FImpl>::TWall(const std::string name)
+: Module<WallPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TWall<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TWall<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWall<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWall<FImpl>::execute(void)
+{    
+    LOG(Message) << "Generating wall source at t = " << par().tW 
+                 << " with momentum " << par().mom << std::endl;
+    
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             ph(env().getGrid()), coor(env().getGrid());
+    std::vector<Real>          p;
+    Complex                    i(0.0,1.0);
+    
+    p  = strToVec<Real>(par().mom);
+    ph = zero;
+    for(unsigned int mu = 0; mu < Nd; mu++)
+    {
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
+    }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    LatticeCoordinate(t, Tp);
+    src = 1.;
+    src = where((t == par().tW), src*ph, 0.*src);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MSource_WallSource_hpp_

extras/Hadrons/Modules/MSource/Z2.hpp

@@ -0,0 +1,152 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MSource/Z2.hpp
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_MSource_Z2_hpp_
+#define Hadrons_MSource_Z2_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Z_2 stochastic source
+ -----------------------------
+ * src_x = eta_x * theta(x_3 - tA) * theta(tB - x_3)
+ 
+ the eta_x are independent uniform random numbers in {+/- 1 +/- i}
+ 
+ * options:
+ - tA: begin timeslice (integer)
+ - tB: end timesilce (integer)
+ 
+ */
+ 
+/******************************************************************************
+ *                          Z2 stochastic source                              *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSource)
+
+class Z2Par: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(Z2Par,
+                                    unsigned int, tA,
+                                    unsigned int, tB);
+};
+
+template <typename FImpl>
+class TZ2: public Module<Z2Par>
+{
+public:
+    FERM_TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TZ2(const std::string name);
+    // destructor
+    virtual ~TZ2(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Z2,       TZ2<FIMPL>,        MSource);
+MODULE_REGISTER_NS(ScalarZ2, TZ2<ScalarImplCR>, MSource);
+
+/******************************************************************************
+ *                       TZ2 template implementation                          *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TZ2<FImpl>::TZ2(const std::string name)
+: Module<Z2Par>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TZ2<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TZ2<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TZ2<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TZ2<FImpl>::execute(void)
+{
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             eta(env().getGrid());
+    Complex                    shift(1., 1.);
+    
+    if (par().tA == par().tB)
+    {
+        LOG(Message) << "Generating Z_2 wall source at t= " << par().tA
+        << std::endl;
+    }
+    else
+    {
+        LOG(Message) << "Generating Z_2 band for " << par().tA << " <= t <= "
+        << par().tB << std::endl;
+    }
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    LatticeCoordinate(t, Tp);
+    bernoulli(*env().get4dRng(), eta);
+    eta = (2.*eta - shift)*(1./::sqrt(2.));
+    eta = where((t >= par().tA) and (t <= par().tB), eta, 0.*eta);
+    src = 1.;
+    src = src*eta;
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_MSource_Z2_hpp_

extras/Hadrons/Modules/templates/Module.cc.template

@@ -0,0 +1,39 @@
+#include <Grid/Hadrons/Modules/___FILEBASENAME___.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+
+/******************************************************************************
+*                  T___FILEBASENAME___ implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+T___FILEBASENAME___::T___FILEBASENAME___(const std::string name)
+: Module<___FILEBASENAME___Par>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> T___FILEBASENAME___::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+std::vector<std::string> T___FILEBASENAME___::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void T___FILEBASENAME___::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void T___FILEBASENAME___::execute(void)
+{
+
+}

extras/Hadrons/Modules/templates/Module.hpp.template

@@ -0,0 +1,40 @@
+#ifndef Hadrons____FILEBASENAME____hpp_
+#define Hadrons____FILEBASENAME____hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         ___FILEBASENAME___                                 *
+ ******************************************************************************/
+class ___FILEBASENAME___Par: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par, 
+                                    unsigned int, i);
+};
+
+class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
+{
+public:
+    // constructor
+    T___FILEBASENAME___(const std::string name);
+    // destructor
+    virtual ~T___FILEBASENAME___(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER(___FILEBASENAME___, T___FILEBASENAME___);
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons____FILEBASENAME____hpp_

extras/Hadrons/Modules/templates/Module_in_NS.cc.template

@@ -0,0 +1,40 @@
+#include <Grid/Hadrons/Modules/___NAMESPACE___/___FILEBASENAME___.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace ___NAMESPACE___;
+
+/******************************************************************************
+*                  T___FILEBASENAME___ implementation                             *
+******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+T___FILEBASENAME___::T___FILEBASENAME___(const std::string name)
+: Module<___FILEBASENAME___Par>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> T___FILEBASENAME___::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+std::vector<std::string> T___FILEBASENAME___::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void T___FILEBASENAME___::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void T___FILEBASENAME___::execute(void)
+{
+
+}

extras/Hadrons/Modules/templates/Module_in_NS.hpp.template

@@ -0,0 +1,44 @@
+#ifndef Hadrons____NAMESPACE_______FILEBASENAME____hpp_
+#define Hadrons____NAMESPACE_______FILEBASENAME____hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         ___FILEBASENAME___                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(___NAMESPACE___)
+
+class ___FILEBASENAME___Par: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par,
+                                    unsigned int, i);
+};
+
+class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
+{
+public:
+    // constructor
+    T___FILEBASENAME___(const std::string name);
+    // destructor
+    virtual ~T___FILEBASENAME___(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(___FILEBASENAME___, T___FILEBASENAME___, ___NAMESPACE___);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons____NAMESPACE_______FILEBASENAME____hpp_

extras/Hadrons/Modules/templates/Module_tmp.hpp.template

@@ -0,0 +1,81 @@
+#ifndef Hadrons____FILEBASENAME____hpp_
+#define Hadrons____FILEBASENAME____hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         ___FILEBASENAME___                                 *
+ ******************************************************************************/
+class ___FILEBASENAME___Par: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par,
+                                    unsigned int, i);
+};
+
+template <typename FImpl>
+class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
+{
+public:
+    // constructor
+    T___FILEBASENAME___(const std::string name);
+    // destructor
+    virtual ~T___FILEBASENAME___(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER(___FILEBASENAME___, T___FILEBASENAME___<FIMPL>);
+
+/******************************************************************************
+ *                 T___FILEBASENAME___ implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+T___FILEBASENAME___<FImpl>::T___FILEBASENAME___(const std::string name)
+: Module<___FILEBASENAME___Par>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> T___FILEBASENAME___<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> T___FILEBASENAME___<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void T___FILEBASENAME___<FImpl>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void T___FILEBASENAME___<FImpl>::execute(void)
+{
+    
+}
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons____FILEBASENAME____hpp_

extras/Hadrons/Modules/templates/Module_tmp_in_NS.hpp.template

@@ -0,0 +1,85 @@
+#ifndef Hadrons____NAMESPACE_______FILEBASENAME____hpp_
+#define Hadrons____NAMESPACE_______FILEBASENAME____hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         ___FILEBASENAME___                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(___NAMESPACE___)
+
+class ___FILEBASENAME___Par: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par,
+                                    unsigned int, i);
+};
+
+template <typename FImpl>
+class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
+{
+public:
+    // constructor
+    T___FILEBASENAME___(const std::string name);
+    // destructor
+    virtual ~T___FILEBASENAME___(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(___FILEBASENAME___, T___FILEBASENAME___<FIMPL>, ___NAMESPACE___);
+
+/******************************************************************************
+ *                 T___FILEBASENAME___ implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+T___FILEBASENAME___<FImpl>::T___FILEBASENAME___(const std::string name)
+: Module<___FILEBASENAME___Par>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> T___FILEBASENAME___<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> T___FILEBASENAME___<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void T___FILEBASENAME___<FImpl>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void T___FILEBASENAME___<FImpl>::execute(void)
+{
+    
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons____NAMESPACE_______FILEBASENAME____hpp_

extras/Hadrons/add_module.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+if (( $# != 1 && $# != 2)); then
+    echo "usage: `basename $0` <module name> [<namespace>]" 1>&2
+    exit 1
+fi
+NAME=$1
+NS=$2
+
+if (( $# == 1 )); then
+	if [ -e "Modules/${NAME}.cc" ] || [ -e "Modules/${NAME}.hpp" ]; then
+	    echo "error: files Modules/${NAME}.* already exists" 1>&2
+	    exit 1
+	fi
+	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module.cc.template > Modules/${NAME}.cc
+	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module.hpp.template > Modules/${NAME}.hpp
+elif (( $# == 2 )); then
+	mkdir -p Modules/${NS}
+	if [ -e "Modules/${NS}/${NAME}.cc" ] || [ -e "Modules/${NS}/${NAME}.hpp" ]; then
+	    echo "error: files Modules/${NS}/${NAME}.* already exists" 1>&2
+	    exit 1
+	fi
+	TMPCC=".${NS}.${NAME}.tmp.cc"
+	TMPHPP=".${NS}.${NAME}.tmp.hpp"
+	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_in_NS.cc.template  > ${TMPCC}
+	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_in_NS.hpp.template > ${TMPHPP}
+	sed "s/___NAMESPACE___/${NS}/g" ${TMPCC}  > Modules/${NS}/${NAME}.cc
+	sed "s/___NAMESPACE___/${NS}/g" ${TMPHPP} > Modules/${NS}/${NAME}.hpp
+	rm -f ${TMPCC} ${TMPHPP}
+fi
+./make_module_list.sh

extras/Hadrons/add_module_template.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+if (( $# != 1 && $# != 2)); then
+    echo "usage: `basename $0` <module name> [<namespace>]" 1>&2
+    exit 1
+fi
+NAME=$1
+NS=$2
+
+if (( $# == 1 )); then
+	if [ -e "Modules/${NAME}.cc" ] || [ -e "Modules/${NAME}.hpp" ]; then
+	    echo "error: files Modules/${NAME}.* already exists" 1>&2
+	    exit 1
+	fi
+	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_tmp.hpp.template > Modules/${NAME}.hpp
+elif (( $# == 2 )); then
+	mkdir -p Modules/${NS}
+	if [ -e "Modules/${NS}/${NAME}.cc" ] || [ -e "Modules/${NS}/${NAME}.hpp" ]; then
+	    echo "error: files Modules/${NS}/${NAME}.* already exists" 1>&2
+	    exit 1
+	fi
+	TMPCC=".${NS}.${NAME}.tmp.cc"
+	TMPHPP=".${NS}.${NAME}.tmp.hpp"
+	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_tmp_in_NS.hpp.template > ${TMPHPP}
+	sed "s/___NAMESPACE___/${NS}/g" ${TMPHPP} > Modules/${NS}/${NAME}.hpp
+	rm -f ${TMPCC} ${TMPHPP}
+fi
+./make_module_list.sh

extras/Hadrons/make_module_list.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+echo 'modules_cc =\' > modules.inc
+find Modules -name '*.cc' -type f -print | sed 's/^/  /;$q;s/$/ \\/' >> modules.inc
+echo '' >> modules.inc
+echo 'modules_hpp =\' >> modules.inc
+find Modules -name '*.hpp' -type f -print | sed 's/^/  /;$q;s/$/ \\/' >> modules.inc
+echo '' >> modules.inc
+rm -f Modules.hpp
+for f in `find Modules -name '*.hpp'`; do
+	echo "#include <Grid/Hadrons/${f}>" >> Modules.hpp
+done

extras/Hadrons/modules.inc

@@ -0,0 +1,38 @@
+modules_cc =\
+  Modules/MContraction/WeakHamiltonianEye.cc \
+  Modules/MContraction/WeakHamiltonianNonEye.cc \
+  Modules/MContraction/WeakNeutral4ptDisc.cc \
+  Modules/MGauge/Load.cc \
+  Modules/MGauge/Random.cc \
+  Modules/MGauge/StochEm.cc \
+  Modules/MGauge/Unit.cc \
+  Modules/MScalar/ChargedProp.cc \
+  Modules/MScalar/FreeProp.cc
+
+modules_hpp =\
+  Modules/MAction/DWF.hpp \
+  Modules/MAction/Wilson.hpp \
+  Modules/MContraction/Baryon.hpp \
+  Modules/MContraction/DiscLoop.hpp \
+  Modules/MContraction/Gamma3pt.hpp \
+  Modules/MContraction/Meson.hpp \
+  Modules/MContraction/WeakHamiltonian.hpp \
+  Modules/MContraction/WeakHamiltonianEye.hpp \
+  Modules/MContraction/WeakHamiltonianNonEye.hpp \
+  Modules/MContraction/WeakNeutral4ptDisc.hpp \
+  Modules/MFermion/GaugeProp.hpp \
+  Modules/MGauge/Load.hpp \
+  Modules/MGauge/Random.hpp \
+  Modules/MGauge/StochEm.hpp \
+  Modules/MGauge/Unit.hpp \
+  Modules/MLoop/NoiseLoop.hpp \
+  Modules/MScalar/ChargedProp.hpp \
+  Modules/MScalar/FreeProp.hpp \
+  Modules/MScalar/Scalar.hpp \
+  Modules/MSink/Point.hpp \
+  Modules/MSolver/RBPrecCG.hpp \
+  Modules/MSource/Point.hpp \
+  Modules/MSource/SeqGamma.hpp \
+  Modules/MSource/Wall.hpp \
+  Modules/MSource/Z2.hpp
+

extras/Makefile.am

@@ -0,0 +1 @@
+SUBDIRS = Hadrons

extras/qed-fvol/Global.cc

@@ -0,0 +1,11 @@
+#include <qed-fvol/Global.hpp>
+
+using namespace Grid;
+using namespace QCD;
+using namespace QedFVol;
+
+QedFVolLogger QedFVol::QedFVolLogError(1,"Error");
+QedFVolLogger QedFVol::QedFVolLogWarning(1,"Warning");
+QedFVolLogger QedFVol::QedFVolLogMessage(1,"Message");
+QedFVolLogger QedFVol::QedFVolLogIterative(1,"Iterative");
+QedFVolLogger QedFVol::QedFVolLogDebug(1,"Debug");

extras/qed-fvol/Global.hpp

@@ -0,0 +1,42 @@
+#ifndef QedFVol_Global_hpp_
+#define QedFVol_Global_hpp_
+
+#include <Grid/Grid.h>
+
+#define BEGIN_QEDFVOL_NAMESPACE \
+namespace Grid {\
+using namespace QCD;\
+namespace QedFVol {\
+using Grid::operator<<;
+#define END_QEDFVOL_NAMESPACE }}
+
+/* the 'using Grid::operator<<;' statement prevents a very nasty compilation
+ * error with GCC (clang compiles fine without it).
+ */
+
+BEGIN_QEDFVOL_NAMESPACE
+
+class QedFVolLogger: public Logger
+{
+public:
+    QedFVolLogger(int on, std::string nm): Logger("QedFVol", on, nm,
+                                                  GridLogColours, "BLACK"){};
+};
+
+#define LOG(channel) std::cout << QedFVolLog##channel
+#define QEDFVOL_ERROR(msg)\
+LOG(Error) << msg << " (" << __FUNCTION__ << " at " << __FILE__ << ":"\
+           << __LINE__ << ")" << std::endl;\
+abort();
+
+#define DEBUG_VAR(var) LOG(Debug) << #var << "= " << (var) << std::endl;
+
+extern QedFVolLogger QedFVolLogError;
+extern QedFVolLogger QedFVolLogWarning;
+extern QedFVolLogger QedFVolLogMessage;
+extern QedFVolLogger QedFVolLogIterative;
+extern QedFVolLogger QedFVolLogDebug;
+
+END_QEDFVOL_NAMESPACE
+
+#endif // QedFVol_Global_hpp_

extras/qed-fvol/Makefile.am

@@ -0,0 +1,9 @@
+AM_CXXFLAGS += -I$(top_srcdir)/extras
+
+bin_PROGRAMS = qed-fvol
+
+qed_fvol_SOURCES =   \
+    qed-fvol.cc      \
+    Global.cc
+
+qed_fvol_LDADD   = -lGrid

extras/qed-fvol/WilsonLoops.h

@@ -0,0 +1,265 @@
+#ifndef QEDFVOL_WILSONLOOPS_H
+#define QEDFVOL_WILSONLOOPS_H
+
+#include <Global.hpp>
+
+BEGIN_QEDFVOL_NAMESPACE
+
+template <class Gimpl> class NewWilsonLoops : public Gimpl {
+public:
+  INHERIT_GIMPL_TYPES(Gimpl);
+
+  typedef typename Gimpl::GaugeLinkField GaugeMat;
+  typedef typename Gimpl::GaugeField GaugeLorentz;
+
+  //////////////////////////////////////////////////
+  // directed plaquette oriented in mu,nu plane
+  //////////////////////////////////////////////////
+  static void dirPlaquette(GaugeMat &plaq, const std::vector<GaugeMat> &U,
+                           const int mu, const int nu) {
+    // Annoyingly, must use either scope resolution to find dependent base
+    // class,
+    // or this-> ; there is no "this" in a static method. This forces explicit
+    // Gimpl scope
+    // resolution throughout the usage in this file, and rather defeats the
+    // purpose of deriving
+    // from Gimpl.
+    plaq = Gimpl::CovShiftBackward(
+        U[mu], mu, Gimpl::CovShiftBackward(
+                       U[nu], nu, Gimpl::CovShiftForward(U[mu], mu, U[nu])));
+  }
+  //////////////////////////////////////////////////
+  // trace of directed plaquette oriented in mu,nu plane
+  //////////////////////////////////////////////////
+  static void traceDirPlaquette(LatticeComplex &plaq,
+                                const std::vector<GaugeMat> &U, const int mu,
+                                const int nu) {
+    GaugeMat sp(U[0]._grid);
+    dirPlaquette(sp, U, mu, nu);
+    plaq = trace(sp);
+  }
+  //////////////////////////////////////////////////
+  // sum over all planes of plaquette
+  //////////////////////////////////////////////////
+  static void sitePlaquette(LatticeComplex &Plaq,
+                            const std::vector<GaugeMat> &U) {
+    LatticeComplex sitePlaq(U[0]._grid);
+    Plaq = zero;
+    for (int mu = 1; mu < U[0]._grid->_ndimension; mu++) {
+      for (int nu = 0; nu < mu; nu++) {
+        traceDirPlaquette(sitePlaq, U, mu, nu);
+        Plaq = Plaq + sitePlaq;
+      }
+    }
+  }
+  //////////////////////////////////////////////////
+  // sum over all x,y,z,t and over all planes of plaquette
+  //////////////////////////////////////////////////
+  static Real sumPlaquette(const GaugeLorentz &Umu) {
+    std::vector<GaugeMat> U(4, Umu._grid);
+
+    for (int mu = 0; mu < Umu._grid->_ndimension; mu++) {
+      U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
+    }
+
+    LatticeComplex Plaq(Umu._grid);
+
+    sitePlaquette(Plaq, U);
+
+    TComplex Tp = sum(Plaq);
+    Complex p = TensorRemove(Tp);
+    return p.real();
+  }
+  //////////////////////////////////////////////////
+  // average over all x,y,z,t and over all planes of plaquette
+  //////////////////////////////////////////////////
+  static Real avgPlaquette(const GaugeLorentz &Umu) {
+    int ndim = Umu._grid->_ndimension;
+    Real sumplaq = sumPlaquette(Umu);
+    Real vol = Umu._grid->gSites();
+    Real faces = (1.0 * ndim * (ndim - 1)) / 2.0;
+    return sumplaq / vol / faces / Nc; // Nc dependent... FIXME
+  }
+
+  //////////////////////////////////////////////////
+  // Wilson loop of size (R1, R2), oriented in mu,nu plane
+  //////////////////////////////////////////////////
+  static void wilsonLoop(GaugeMat &wl, const std::vector<GaugeMat> &U,
+                           const int Rmu, const int Rnu,
+                           const int mu, const int nu) {
+    wl = U[nu];
+
+    for(int i = 0; i < Rnu-1; i++){
+      wl = Gimpl::CovShiftForward(U[nu], nu, wl);
+    }
+
+    for(int i = 0; i < Rmu; i++){
+      wl = Gimpl::CovShiftForward(U[mu], mu, wl);
+    }
+
+    for(int i = 0; i < Rnu; i++){
+      wl = Gimpl::CovShiftBackward(U[nu], nu, wl);
+    }
+
+    for(int i = 0; i < Rmu; i++){
+      wl = Gimpl::CovShiftBackward(U[mu], mu, wl);
+    }
+  }
+  //////////////////////////////////////////////////
+  // trace of Wilson Loop oriented in mu,nu plane
+  //////////////////////////////////////////////////
+  static void traceWilsonLoop(LatticeComplex &wl,
+                                const std::vector<GaugeMat> &U,
+                                const int Rmu, const int Rnu,
+                                const int mu, const int nu) {
+    GaugeMat sp(U[0]._grid);
+    wilsonLoop(sp, U, Rmu, Rnu, mu, nu);
+    wl = trace(sp);
+  }
+  //////////////////////////////////////////////////
+  // sum over all planes of Wilson loop
+  //////////////////////////////////////////////////
+  static void siteWilsonLoop(LatticeComplex &Wl,
+                            const std::vector<GaugeMat> &U,
+                            const int R1, const int R2) {
+    LatticeComplex siteWl(U[0]._grid);
+    Wl = zero;
+    for (int mu = 1; mu < U[0]._grid->_ndimension; mu++) {
+      for (int nu = 0; nu < mu; nu++) {
+        traceWilsonLoop(siteWl, U, R1, R2, mu, nu);
+        Wl = Wl + siteWl;
+        traceWilsonLoop(siteWl, U, R2, R1, mu, nu);
+        Wl = Wl + siteWl;
+      }
+    }
+  }
+  //////////////////////////////////////////////////
+  // sum over planes of Wilson loop with length R1
+  // in the time direction
+  //////////////////////////////////////////////////
+  static void siteTimelikeWilsonLoop(LatticeComplex &Wl,
+                            const std::vector<GaugeMat> &U,
+                            const int R1, const int R2) {
+    LatticeComplex siteWl(U[0]._grid);
+
+    int ndim = U[0]._grid->_ndimension;
+
+    Wl = zero;
+    for (int nu = 0; nu < ndim - 1; nu++) {
+      traceWilsonLoop(siteWl, U, R1, R2, ndim-1, nu);
+      Wl = Wl + siteWl;
+    }
+  }
+  //////////////////////////////////////////////////
+  // sum Wilson loop over all planes orthogonal to the time direction
+  //////////////////////////////////////////////////
+  static void siteSpatialWilsonLoop(LatticeComplex &Wl,
+                            const std::vector<GaugeMat> &U,
+                            const int R1, const int R2) {
+    LatticeComplex siteWl(U[0]._grid);
+
+    Wl = zero;
+    for (int mu = 1; mu < U[0]._grid->_ndimension - 1; mu++) {
+      for (int nu = 0; nu < mu; nu++) {
+        traceWilsonLoop(siteWl, U, R1, R2, mu, nu);
+        Wl = Wl + siteWl;
+        traceWilsonLoop(siteWl, U, R2, R1, mu, nu);
+        Wl = Wl + siteWl;
+      }
+    }
+  }
+  //////////////////////////////////////////////////
+  // sum over all x,y,z,t and over all planes of Wilson loop
+  //////////////////////////////////////////////////
+  static Real sumWilsonLoop(const GaugeLorentz &Umu,
+                            const int R1, const int R2) {
+    std::vector<GaugeMat> U(4, Umu._grid);
+
+    for (int mu = 0; mu < Umu._grid->_ndimension; mu++) {
+      U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
+    }
+
+    LatticeComplex Wl(Umu._grid);
+
+    siteWilsonLoop(Wl, U, R1, R2);
+
+    TComplex Tp = sum(Wl);
+    Complex p = TensorRemove(Tp);
+    return p.real();
+  }
+  //////////////////////////////////////////////////
+  // sum over all x,y,z,t and over all planes of timelike Wilson loop
+  //////////////////////////////////////////////////
+  static Real sumTimelikeWilsonLoop(const GaugeLorentz &Umu,
+                            const int R1, const int R2) {
+    std::vector<GaugeMat> U(4, Umu._grid);
+
+    for (int mu = 0; mu < Umu._grid->_ndimension; mu++) {
+      U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
+    }
+
+    LatticeComplex Wl(Umu._grid);
+
+    siteTimelikeWilsonLoop(Wl, U, R1, R2);
+
+    TComplex Tp = sum(Wl);
+    Complex p = TensorRemove(Tp);
+    return p.real();
+  }
+  //////////////////////////////////////////////////
+  // sum over all x,y,z,t and over all planes of spatial Wilson loop
+  //////////////////////////////////////////////////
+  static Real sumSpatialWilsonLoop(const GaugeLorentz &Umu,
+                            const int R1, const int R2) {
+    std::vector<GaugeMat> U(4, Umu._grid);
+
+    for (int mu = 0; mu < Umu._grid->_ndimension; mu++) {
+      U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
+    }
+
+    LatticeComplex Wl(Umu._grid);
+
+    siteSpatialWilsonLoop(Wl, U, R1, R2);
+
+    TComplex Tp = sum(Wl);
+    Complex p = TensorRemove(Tp);
+    return p.real();
+  }
+  //////////////////////////////////////////////////
+  // average over all x,y,z,t and over all planes of Wilson loop
+  //////////////////////////////////////////////////
+  static Real avgWilsonLoop(const GaugeLorentz &Umu,
+                            const int R1, const int R2) {
+    int ndim = Umu._grid->_ndimension;
+    Real sumWl = sumWilsonLoop(Umu, R1, R2);
+    Real vol = Umu._grid->gSites();
+    Real faces = 1.0 * ndim * (ndim - 1);
+    return sumWl / vol / faces / Nc; // Nc dependent... FIXME
+  }
+  //////////////////////////////////////////////////
+  // average over all x,y,z,t and over all planes of timelike Wilson loop
+  //////////////////////////////////////////////////
+  static Real avgTimelikeWilsonLoop(const GaugeLorentz &Umu,
+                            const int R1, const int R2) {
+    int ndim = Umu._grid->_ndimension;
+    Real sumWl = sumTimelikeWilsonLoop(Umu, R1, R2);
+    Real vol = Umu._grid->gSites();
+    Real faces = 1.0 * (ndim - 1);
+    return sumWl / vol / faces / Nc; // Nc dependent... FIXME
+  }
+  //////////////////////////////////////////////////
+  // average over all x,y,z,t and over all planes of spatial Wilson loop
+  //////////////////////////////////////////////////
+  static Real avgSpatialWilsonLoop(const GaugeLorentz &Umu,
+                            const int R1, const int R2) {
+    int ndim = Umu._grid->_ndimension;
+    Real sumWl = sumSpatialWilsonLoop(Umu, R1, R2);
+    Real vol = Umu._grid->gSites();
+    Real faces = 1.0 * (ndim - 1) * (ndim - 2);
+    return sumWl / vol / faces / Nc; // Nc dependent... FIXME
+  }
+};
+
+END_QEDFVOL_NAMESPACE
+
+#endif // QEDFVOL_WILSONLOOPS_H

extras/qed-fvol/qed-fvol.cc

@@ -0,0 +1,88 @@
+#include <Global.hpp>
+#include <WilsonLoops.h>
+
+using namespace Grid;
+using namespace QCD;
+using namespace QedFVol;
+
+typedef PeriodicGaugeImpl<QedGimplR>    QedPeriodicGimplR;
+typedef PhotonR::GaugeField             EmField;
+typedef PhotonR::GaugeLinkField         EmComp;
+
+const int NCONFIGS = 10;
+const int NWILSON = 10;
+
+int main(int argc, char *argv[])
+{
+    // parse command line
+    std::string parameterFileName;
+    
+    if (argc < 2)
+    {
+        std::cerr << "usage: " << argv[0] << " <parameter file> [Grid options]";
+        std::cerr << std::endl;
+        std::exit(EXIT_FAILURE);
+    }
+    parameterFileName = argv[1];
+    
+    // initialization
+    Grid_init(&argc, &argv);
+    QedFVolLogError.Active(GridLogError.isActive());
+    QedFVolLogWarning.Active(GridLogWarning.isActive());
+    QedFVolLogMessage.Active(GridLogMessage.isActive());
+    QedFVolLogIterative.Active(GridLogIterative.isActive());
+    QedFVolLogDebug.Active(GridLogDebug.isActive());
+    LOG(Message) << "Grid initialized" << std::endl;
+    
+    // QED stuff
+    std::vector<int> latt_size   = GridDefaultLatt();
+    std::vector<int> simd_layout = GridDefaultSimd(4, vComplex::Nsimd());
+    std::vector<int> mpi_layout  = GridDefaultMpi();
+    GridCartesian    grid(latt_size,simd_layout,mpi_layout);
+    GridParallelRNG  pRNG(&grid);
+    PhotonR          photon(PhotonR::Gauge::feynman,
+                            PhotonR::ZmScheme::qedL);
+    EmField          a(&grid);
+    EmField          expA(&grid);
+
+    Complex imag_unit(0, 1);
+
+    Real wlA;
+    std::vector<Real> logWlAvg(NWILSON, 0.0), logWlTime(NWILSON, 0.0), logWlSpace(NWILSON, 0.0);
+
+    pRNG.SeedRandomDevice();
+
+    LOG(Message) << "Wilson loop calculation beginning" << std::endl;
+    for(int ic = 0; ic < NCONFIGS; ic++){
+        LOG(Message) << "Configuration " << ic <<std::endl;
+        photon.StochasticField(a, pRNG);
+
+        // Exponentiate photon field
+        expA = exp(imag_unit*a);
+
+        // Calculate Wilson loops
+        for(int iw=1; iw<=NWILSON; iw++){
+            wlA = NewWilsonLoops<QedPeriodicGimplR>::avgWilsonLoop(expA, iw, iw) * 3;
+            logWlAvg[iw-1] -= 2*log(wlA);
+            wlA = NewWilsonLoops<QedPeriodicGimplR>::avgTimelikeWilsonLoop(expA, iw, iw) * 3;
+            logWlTime[iw-1] -= 2*log(wlA);
+            wlA = NewWilsonLoops<QedPeriodicGimplR>::avgSpatialWilsonLoop(expA, iw, iw) * 3;
+            logWlSpace[iw-1] -= 2*log(wlA);
+        }
+    }
+    LOG(Message) << "Wilson loop calculation completed" << std::endl;
+    
+    // Calculate Wilson loops
+    for(int iw=1; iw<=10; iw++){
+        LOG(Message) << iw << 'x' << iw << " Wilson loop" << std::endl;
+        LOG(Message) << "-2log(W) average: " << logWlAvg[iw-1]/NCONFIGS << std::endl;
+        LOG(Message) << "-2log(W) timelike: " << logWlTime[iw-1]/NCONFIGS << std::endl;
+        LOG(Message) << "-2log(W) spatial: " << logWlSpace[iw-1]/NCONFIGS << std::endl;
+    }
+
+    // epilogue
+    LOG(Message) << "Grid is finalizing now" << std::endl;
+    Grid_finalize();
+    
+    return EXIT_SUCCESS;
+}

gcc-bug-report/README

@@ -21,3 +21,16 @@ problem. The test case works with icpc and with clang++, but fails consistently
 current variants.
 
 Peter
+
+
+************
+
+Second GCC bug reported, see Issue 100.
+
+https://wandbox.org/permlink/tzssJza6R9XnqANw
+https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80652
+
+Getting Travis fails under gcc-5 for Test_simd, now that I added more comprehensive testing to the
+CI test suite. The limitations of Travis runtime limits & weak cores are being shown.
+
+Travis uses 5.4.1 for g++-5.

grid-config.in

@@ -0,0 +1,86 @@
+#! /bin/sh
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+includedir=@includedir@
+
+usage()
+{
+  cat <<EOF
+Usage: grid-config [OPTION]
+
+Known values for OPTION are:
+
+  --prefix     show Grid installation prefix
+  --cxxflags   print pre-processor and compiler flags
+  --ldflags    print library linking flags
+  --libs       print library linking information
+  --summary    print full build summary
+  --help       display this help and exit
+  --version    output version information
+  --git        print git revision
+
+EOF
+  
+  exit $1
+}
+
+if test $# -eq 0; then
+  usage 1
+fi
+
+cflags=false
+libs=false
+
+while test $# -gt 0; do
+  case "$1" in
+    -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+    *) optarg= ;;
+  esac
+  
+  case "$1" in
+    --prefix)
+      echo $prefix
+    ;;
+    
+    --version)
+      echo @VERSION@
+      exit 0
+    ;;
+    
+    --git)
+      echo "@GRID_BRANCH@ @GRID_SHA@"
+      exit 0
+    ;;
+    
+    --help)
+      usage 0
+    ;;
+    
+    --cxxflags)
+      echo @GRID_CXXFLAGS@
+    ;;
+    
+    --ldflags)
+      echo @GRID_LDFLAGS@
+    ;;
+    
+    --libs)
+      echo @GRID_LIBS@
+    ;;
+    
+    --summary)
+      echo ""
+      echo "@GRID_SUMMARY@"
+      echo ""
+    ;;
+    
+    *)
+      usage
+      exit 1
+    ;;
+  esac
+  shift
+done
+
+exit 0

lib/DisableWarnings.h

@@ -0,0 +1,37 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/DisableWarnings.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef DISABLE_WARNINGS_H
+#define DISABLE_WARNINGS_H
+
+ //disables and intel compiler specific warning (in json.hpp)
+#pragma warning disable 488  
+
+
+#endif

lib/Grid.h

@@ -38,52 +38,12 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_H
 #define GRID_H
 
-///////////////////
+#include <Grid/GridCore.h>
-// Std C++ dependencies
+#include <Grid/GridQCDcore.h>
-///////////////////
+#include <Grid/qcd/action/Action.h>
-#include <cassert>
+#include <Grid/qcd/utils/GaugeFix.h>
-#include <complex>
+#include <Grid/qcd/smearing/Smearing.h>
-#include <vector>
+#include <Grid/parallelIO/MetaData.h>
-#include <iostream>
+#include <Grid/qcd/hmc/HMC_aggregate.h>
-#include <iomanip>
-#include <random>
-#include <functional>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <signal.h>
-#include <ctime>
-#include <sys/time.h>
-#include <chrono>
-
-///////////////////
-// Grid headers
-///////////////////
-#include <Grid/serialisation/Serialisation.h>
-#include "Config.h"
-#include <Grid/Timer.h>
-#include <Grid/PerfCount.h>
-#include <Grid/Log.h>
-#include <Grid/AlignedAllocator.h>
-#include <Grid/Simd.h>
-#include <Grid/Threads.h>
-#include <Grid/Lexicographic.h>
-#include <Grid/Init.h>
-#include <Grid/Communicator.h> 
-#include <Grid/Cartesian.h>    
-#include <Grid/Tensors.h>      
-#include <Grid/Lattice.h>      
-#include <Grid/Cshift.h>       
-#include <Grid/Stencil.h>      
-#include <Grid/Algorithms.h>   
-#include <Grid/parallelIO/BinaryIO.h>
-#include <Grid/FFT.h>
-
-#include <Grid/qcd/QCD.h>
-#include <Grid/parallelIO/NerscIO.h>
-#include <Grid/qcd/hmc/NerscCheckpointer.h>
-#include <Grid/qcd/hmc/HmcRunner.h>
-
-
 
 #endif

lib/GridCore.h

@@ -2,11 +2,13 @@
 
     Grid physics library, www.github.com/paboyle/Grid 
 
-    Source file: ./lib/algorithms/iterative/MatrixUtils.h
+    Source file: ./lib/Grid.h
 
     Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: azusayamaguchi <ayamaguc@YAMAKAZE.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -25,51 +27,34 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#ifndef GRID_MATRIX_UTILS_H
+//
-#define GRID_MATRIX_UTILS_H
+//  Grid.h
+//  simd
+//
+//  Created by Peter Boyle on 09/05/2014.
+//  Copyright (c) 2014 University of Edinburgh. All rights reserved.
+//
 
-namespace Grid {
+#ifndef GRID_BASE_H
+#define GRID_BASE_H
 
-  namespace MatrixUtils { 
+#include <Grid/GridStd.h>
 
-    template<class T> inline void Size(Matrix<T>& A,int &N,int &M){
+#include <Grid/perfmon/Timer.h>
-      N=A.size(); assert(N>0);
+#include <Grid/perfmon/PerfCount.h>
-      M=A[0].size();
+#include <Grid/log/Log.h>
-      for(int i=0;i<N;i++){
+#include <Grid/allocator/AlignedAllocator.h>
-	assert(A[i].size()==M);
+#include <Grid/simd/Simd.h>
-      }
+#include <Grid/serialisation/Serialisation.h>
-    }
+#include <Grid/threads/Threads.h>
+#include <Grid/util/Util.h>
+#include <Grid/communicator/Communicator.h> 
+#include <Grid/cartesian/Cartesian.h>    
+#include <Grid/tensors/Tensors.h>      
+#include <Grid/lattice/Lattice.h>      
+#include <Grid/cshift/Cshift.h>       
+#include <Grid/stencil/Stencil.h>      
+#include <Grid/parallelIO/BinaryIO.h>
+#include <Grid/algorithms/Algorithms.h>   
 
-    template<class T> inline void SizeSquare(Matrix<T>& A,int &N)
-    {
-      int M;
-      Size(A,N,M);
-      assert(N==M);
-    }
-
-    template<class T> inline void Fill(Matrix<T>& A,T & val)
-    { 
-      int N,M;
-      Size(A,N,M);
-      for(int i=0;i<N;i++){
-      for(int j=0;j<M;j++){
-	A[i][j]=val;
-      }}
-    }
-    template<class T> inline void Diagonal(Matrix<T>& A,T & val)
-    { 
-      int N;
-      SizeSquare(A,N);
-      for(int i=0;i<N;i++){
-	A[i][i]=val;
-      }
-    }
-    template<class T> inline void Identity(Matrix<T>& A)
-    {
-      Fill(A,0.0);
-      Diagonal(A,1.0);
-    }
-
-  };
-}
 #endif

lib/GridQCDcore.h

@@ -2,12 +2,12 @@
 
     Grid physics library, www.github.com/paboyle/Grid 
 
-    Source file: ./lib/qcd/hmc/HMC.cc
+    Source file: ./lib/Grid.h
 
     Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: neo <cossu@post.kek.jp>
+Author: azusayamaguchi <ayamaguc@YAMAKAZE.local>
 Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
@@ -27,10 +27,16 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid.h>
+#ifndef GRID_QCD_CORE_H
+#define GRID_QCD_CORE_H
 
-namespace Grid{
+/////////////////////////
-  namespace QCD{
+// Core Grid QCD headers
+/////////////////////////
+#include <Grid/GridCore.h>
+#include <Grid/qcd/QCD.h>
+#include <Grid/qcd/spin/Spin.h>
+#include <Grid/qcd/utils/Utils.h>
+#include <Grid/qcd/representations/Representations.h>
 
-  }
+#endif
-}

lib/GridStd.h

@@ -0,0 +1,29 @@
+#ifndef GRID_STD_H
+#define GRID_STD_H
+
+///////////////////
+// Std C++ dependencies
+///////////////////
+#include <cassert>
+#include <complex>
+#include <vector>
+#include <string>
+#include <iostream>
+#include <iomanip>
+#include <random>
+#include <functional>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <ctime>
+#include <sys/time.h>
+#include <chrono>
+#include <zlib.h>
+
+///////////////////
+// Grid config
+///////////////////
+#include "Config.h"
+
+#endif /* GRID_STD_H */

lib/Grid_Eigen_Dense.h

@@ -0,0 +1,9 @@
+#pragma once
+#if defined __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+#include <Grid/Eigen/Dense>
+#if defined __GNUC__
+#pragma GCC diagnostic pop
+#endif

lib/Hadrons

@@ -0,0 +1 @@
+../extras/Hadrons

lib/Makefile.am

@@ -1,4 +1,5 @@
 extra_sources=
+extra_headers=
 if BUILD_COMMS_MPI
   extra_sources+=communicator/Communicator_mpi.cc
   extra_sources+=communicator/Communicator_base.cc
@@ -9,8 +10,8 @@ if BUILD_COMMS_MPI3
   extra_sources+=communicator/Communicator_base.cc
 endif
 
-if BUILD_COMMS_MPI3L
+if BUILD_COMMS_MPIT
-  extra_sources+=communicator/Communicator_mpi3_leader.cc
+  extra_sources+=communicator/Communicator_mpit.cc
   extra_sources+=communicator/Communicator_base.cc
 endif
 
@@ -24,6 +25,12 @@ if BUILD_COMMS_NONE
   extra_sources+=communicator/Communicator_base.cc
 endif
 
+if BUILD_HDF5
+  extra_sources+=serialisation/Hdf5IO.cc 
+  extra_headers+=serialisation/Hdf5IO.h
+  extra_headers+=serialisation/Hdf5Type.h
+endif
+
 #
 # Libraries
 #
@@ -32,6 +39,9 @@ include Eigen.inc
 
 lib_LIBRARIES = libGrid.a
 
-libGrid_a_SOURCES              = $(CCFILES) $(extra_sources)
+CCFILES += $(extra_sources)
+HFILES  += $(extra_headers)
+
+libGrid_a_SOURCES              = $(CCFILES)
 libGrid_adir                   = $(pkgincludedir)
 nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) Config.h

lib/Old/Tensor_peek.h

@@ -1,154 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/Old/Tensor_peek.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef GRID_MATH_PEEK_H
-#define GRID_MATH_PEEK_H
-namespace Grid {
-
-//////////////////////////////////////////////////////////////////////////////
-// Peek on a specific index; returns a scalar in that index, tensor inherits rest
-//////////////////////////////////////////////////////////////////////////////
-// If we hit the right index, return scalar with no further recursion
-
-//template<int Level> inline ComplexF peekIndex(const ComplexF arg) { return arg;}
-//template<int Level> inline ComplexD peekIndex(const ComplexD arg) { return arg;}
-//template<int Level> inline RealF peekIndex(const RealF arg) { return arg;}
-//template<int Level> inline RealD peekIndex(const RealD arg) { return arg;}
-#if 0
-// Scalar peek, no indices
-template<int Level,class vtype,typename std::enable_if< iScalar<vtype>::TensorLevel == Level >::type * =nullptr> inline 
-  auto peekIndex(const iScalar<vtype> &arg) ->  iScalar<vtype> 
-{
-  return arg;
-}
-// Vector peek, one index
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel == Level >::type * =nullptr> inline 
-  auto peekIndex(const iVector<vtype,N> &arg,int i) -> iScalar<vtype> // Index matches
-{
-  iScalar<vtype> ret;                              // return scalar
-  ret._internal = arg._internal[i];
-  return ret;
-}
-// Matrix peek, two indices
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel == Level >::type * =nullptr> inline 
-  auto peekIndex(const iMatrix<vtype,N> &arg,int i,int j) ->  iScalar<vtype>
-{
-  iScalar<vtype> ret;                              // return scalar
-  ret._internal = arg._internal[i][j];
-  return ret;
-}
-
-/////////////
-// No match peek for scalar,vector,matrix must forward on either 0,1,2 args. Must have 9 routines with notvalue
-/////////////
-// scalar
-template<int Level,class vtype,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  auto peekIndex(const iScalar<vtype> &arg) -> iScalar<decltype(peekIndex<Level>(arg._internal))>
-{
-  iScalar<decltype(peekIndex<Level>(arg._internal))> ret;
-  ret._internal= peekIndex<Level>(arg._internal);
-  return ret;
-}
-template<int Level,class vtype, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  auto peekIndex(const iScalar<vtype> &arg,int i) ->  iScalar<decltype(peekIndex<Level>(arg._internal,i))> 
-{
-  iScalar<decltype(peekIndex<Level>(arg._internal,i))> ret;
-  ret._internal=peekIndex<Level>(arg._internal,i);
-  return ret;
-}
-template<int Level,class vtype, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  auto peekIndex(const iScalar<vtype> &arg,int i,int j) ->  iScalar<decltype(peekIndex<Level>(arg._internal,i,j))>
-{
-  iScalar<decltype(peekIndex<Level>(arg._internal,i,j))> ret;
-  ret._internal=peekIndex<Level>(arg._internal,i,j);
-  return ret;
-}
-// vector
-template<int Level,class vtype,int N, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-auto peekIndex(const iVector<vtype,N> &arg) ->   iVector<decltype(peekIndex<Level>(arg._internal[0])),N>
-{
-  iVector<decltype(peekIndex<Level>(arg._internal[0])),N> ret;
-  for(int ii=0;ii<N;ii++){
-    ret._internal[ii]=peekIndex<Level>(arg._internal[ii]);
-  }
-  return ret;
-}
-template<int Level,class vtype,int N, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  auto peekIndex(const iVector<vtype,N> &arg,int i) ->  iVector<decltype(peekIndex<Level>(arg._internal[0],i)),N>
-{
-  iVector<decltype(peekIndex<Level>(arg._internal[0],i)),N> ret;
-  for(int ii=0;ii<N;ii++){
-    ret._internal[ii]=peekIndex<Level>(arg._internal[ii],i);
-  }
-  return ret;
-}
-template<int Level,class vtype,int N, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  auto peekIndex(const iVector<vtype,N> &arg,int i,int j) ->  iVector<decltype(peekIndex<Level>(arg._internal[0],i,j)),N> 
-{
-  iVector<decltype(peekIndex<Level>(arg._internal[0],i,j)),N> ret;
-  for(int ii=0;ii<N;ii++){
-    ret._internal[ii]=peekIndex<Level>(arg._internal[ii],i,j);
-  }
-  return ret;
-}
-
-// matrix
-template<int Level,class vtype,int N, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-auto peekIndex(const iMatrix<vtype,N> &arg) ->   iMatrix<decltype(peekIndex<Level>(arg._internal[0][0])),N> 
-{
-  iMatrix<decltype(peekIndex<Level>(arg._internal[0][0])),N> ret;
-  for(int ii=0;ii<N;ii++){
-  for(int jj=0;jj<N;jj++){
-    ret._internal[ii][jj]=peekIndex<Level>(arg._internal[ii][jj]);// Could avoid this because peeking a scalar is dumb
-  }}
-  return ret;
-}
-template<int Level,class vtype,int N, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  auto peekIndex(const iMatrix<vtype,N> &arg,int i) ->   iMatrix<decltype(peekIndex<Level>(arg._internal[0][0],i)),N>
-{
-  iMatrix<decltype(peekIndex<Level>(arg._internal[0][0],i)),N> ret;
-  for(int ii=0;ii<N;ii++){
-  for(int jj=0;jj<N;jj++){
-    ret._internal[ii][jj]=peekIndex<Level>(arg._internal[ii][jj],i);
-  }}
-  return ret;
-}
-template<int Level,class vtype,int N, typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  auto peekIndex(const iMatrix<vtype,N> &arg,int i,int j) ->   iMatrix<decltype(peekIndex<Level>(arg._internal[0][0],i,j)),N>
-{
-  iMatrix<decltype(peekIndex<Level>(arg._internal[0][0],i,j)),N> ret;
-  for(int ii=0;ii<N;ii++){
-  for(int jj=0;jj<N;jj++){
-    ret._internal[ii][jj]=peekIndex<Level>(arg._internal[ii][jj],i,j);
-  }}
-  return ret;
-}
-#endif
-
-
-}
-#endif

lib/Old/Tensor_poke.h

@@ -1,127 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/Old/Tensor_poke.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef GRID_MATH_POKE_H
-#define GRID_MATH_POKE_H
-namespace Grid {
-
-//////////////////////////////////////////////////////////////////////////////
-// Poke a specific index; 
-//////////////////////////////////////////////////////////////////////////////
-#if 0
-// Scalar poke
-template<int Level,class vtype,typename std::enable_if< iScalar<vtype>::TensorLevel == Level >::type * =nullptr> inline 
-  void pokeIndex(iScalar<vtype> &ret, const iScalar<vtype> &arg)
-{
-  ret._internal = arg._internal;
-}
-// Vector poke, one index
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel == Level >::type * =nullptr> inline 
-  void pokeIndex(iVector<vtype,N> &ret, const iScalar<vtype> &arg,int i)
-{
-  ret._internal[i] = arg._internal;
-}
-//Matrix poke, two indices
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel == Level >::type * =nullptr> inline 
-  void pokeIndex(iMatrix<vtype,N> &ret, const iScalar<vtype> &arg,int i,int j)
-{
-  ret._internal[i][j] = arg._internal;
-}
-
-/////////////
-// No match poke for scalar,vector,matrix must forward on either 0,1,2 args. Must have 9 routines with notvalue
-/////////////
-// scalar
-template<int Level,class vtype,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-void pokeIndex(iScalar<vtype> &ret, const iScalar<decltype(peekIndex<Level>(ret._internal))>  &arg)
-{
-  pokeIndex<Level>(ret._internal,arg._internal);
-}
-template<int Level,class vtype,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iScalar<vtype> &ret, const iScalar<decltype(peekIndex<Level>(ret._internal,0))> &arg, int i)
-		 
-{
-  pokeIndex<Level>(ret._internal,arg._internal,i);
-}
-template<int Level,class vtype,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iScalar<vtype> &ret, const iScalar<decltype(peekIndex<Level>(ret._internal,0,0))> &arg,int i,int j)
-{
-  pokeIndex<Level>(ret._internal,arg._internal,i,j);
-}
-
-// Vector
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iVector<vtype,N> &ret, iVector<decltype(peekIndex<Level>(ret._internal)),N>  &arg)
-{
-  for(int ii=0;ii<N;ii++){
-    pokeIndex<Level>(ret._internal[ii],arg._internal[ii]);
-  }
-}
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iVector<vtype,N> &ret, const iVector<decltype(peekIndex<Level>(ret._internal,0)),N> &arg,int i)
-{
-  for(int ii=0;ii<N;ii++){
-    pokeIndex<Level>(ret._internal[ii],arg._internal[ii],i);
-  }
-}
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iVector<vtype,N> &ret, const iVector<decltype(peekIndex<Level>(ret._internal,0,0)),N> &arg,int i,int j)
-{
-  for(int ii=0;ii<N;ii++){
-    pokeIndex<Level>(ret._internal[ii],arg._internal[ii],i,j);
-  }
-}
-
-// Matrix
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iMatrix<vtype,N> &ret, const iMatrix<decltype(peekIndex<Level>(ret._internal)),N> &arg)		 
-{
-  for(int ii=0;ii<N;ii++){
-  for(int jj=0;jj<N;jj++){
-    pokeIndex<Level>(ret._internal[ii][jj],arg._internal[ii][jj]);
-  }}
-}
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iMatrix<vtype,N> &ret, const iMatrix<decltype(peekIndex<Level>(ret._internal,0)),N> &arg,int i)
-{
-  for(int ii=0;ii<N;ii++){
-  for(int jj=0;jj<N;jj++){
-    pokeIndex<Level>(ret._internal[ii][jj],arg._internal[ii][jj],i);
-  }}
-}
-template<int Level,class vtype,int N,typename std::enable_if< iScalar<vtype>::TensorLevel != Level >::type * =nullptr> inline 
-  void pokeIndex(iMatrix<vtype,N> &ret, const iMatrix<decltype(peekIndex<Level>(ret._internal,0,0)),N> &arg, int i,int j)
-{
-  for(int ii=0;ii<N;ii++){
-  for(int jj=0;jj<N;jj++){
-    pokeIndex<Level>(ret._internal[ii][jj],arg._internal[ii][jj],i,j);
-  }}
-}
-#endif
-
-}
-#endif