Update to enable multiple records per file more consistent with SciDAC.

open, close, write records...

.gitignore

@@ -92,6 +92,7 @@ build*/*
 #####################
 *.xcodeproj/*
 build.sh
+.vscode
 
 # Eigen source #
 ################
@@ -106,6 +107,10 @@ lib/fftw/*
 m4/lt*
 m4/libtool.m4
 
+# github pages #
+################
+gh-pages/
+
 # Buck files #
 ##############
 .buck*
@@ -116,4 +121,5 @@ make-bin-BUCK.sh
 # generated sources #
 #####################
 lib/qcd/spin/gamma-gen/*.h
-lib/qcd/spin/gamma-gen/*.cc
+lib/qcd/spin/gamma-gen/*.cc
+

.travis.yml

@@ -7,9 +7,11 @@ cache:
 matrix:
   include:
     - os:        osx
-      osx_image: xcode7.2
+      osx_image: xcode8.3
       compiler: clang
     - compiler: gcc
+      dist: trusty
+      sudo: required
       addons:
         apt:
           sources:
@@ -24,6 +26,8 @@ matrix:
             - binutils-dev
       env: VERSION=-4.9
     - compiler: gcc
+      dist: trusty
+      sudo: required
       addons:
         apt:
           sources:
@@ -38,6 +42,7 @@ matrix:
             - binutils-dev
       env: VERSION=-5
     - compiler: clang
+      dist: trusty
       addons:
         apt:
           sources:
@@ -52,6 +57,7 @@ matrix:
             - binutils-dev
       env: CLANG_LINK=http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
     - compiler: clang
+      dist: trusty
       addons:
         apt:
           sources:
@@ -73,13 +79,15 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openmpi; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]] && [[ "$CC" == "gcc" ]]; then brew install gcc5; fi
     
 install:
     - export CC=$CC$VERSION
     - export CXX=$CXX$VERSION
     - echo $PATH
+    - which autoconf
+    - autoconf  --version
+    - which automake
+    - automake  --version
     - which $CC
     - $CC  --version
     - which $CXX
@@ -92,15 +100,15 @@ script:
     - cd build
     - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=none
     - make -j4 
-    - ./benchmarks/Benchmark_dwf --threads 1
+    - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - echo make clean
     - ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none
     - make -j4
-    - ./benchmarks/Benchmark_dwf --threads 1
+    - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
+    - make check
     - echo make clean
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXXFLAGS='-DMPI_UINT32_T=MPI_UNSIGNED -DMPI_UINT64_T=MPI_UNSIGNED_LONG'; fi
-    - ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto
-    - make -j4
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto ; fi
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then make -j4; fi
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
 
 

Makefile.am

@@ -3,10 +3,15 @@ SUBDIRS = lib benchmarks tests extras
 
 include $(top_srcdir)/doxygen.inc
 
-tests: all
-	$(MAKE) -C tests tests
+bin_SCRIPTS=grid-config
 
-.PHONY: tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
+
+.PHONY: bench check tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
+
+tests-local: all
+bench-local: all
+check-local: all
 
 AM_CXXFLAGS += -I$(top_builddir)/include
+
 ACLOCAL_AMFLAGS = -I m4

README.md

@@ -22,6 +22,26 @@ Last update Nov 2016.
 
 _Please do not send pull requests to the `master` branch which is reserved for releases._
 
+### Compilers
+
+Intel ICPC v16.0.3 and later
+
+Clang v3.5 and later (need 3.8 and later for OpenMP)
+
+GCC   v4.9.x (recommended)
+
+GCC   v6.3 and later
+
+### Important: 
+
+Some versions of GCC appear to have a bug under high optimisation (-O2, -O3).
+
+The safety of these compiler versions cannot be guaranteed at this time. Follow Issue 100 for details and updates.
+
+GCC   v5.x
+
+GCC   v6.1, v6.2
+
 ### Bug report
 
 _To help us tracking and solving more efficiently issues with Grid, please report problems using the issue system of GitHub rather than sending emails to Grid developers._
@@ -32,7 +52,7 @@ When you file an issue, please go though the following checklist:
 2. Give a description of the target platform (CPU, network, compiler). Please give the full CPU part description, using for example `cat /proc/cpuinfo | grep 'model name' | uniq` (Linux) or `sysctl machdep.cpu.brand_string` (macOS) and the full output the `--version` option of your compiler.
 3. Give the exact `configure` command used.
 4. Attach `config.log`.
-5. Attach `config.summary`.
+5. Attach `grid.config.summary`.
 6. Attach the output of `make V=1`.
 7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example.
 
@@ -95,10 +115,10 @@ install Grid. Other options are detailed in the next section, you can also use `
 `CXX`, `CXXFLAGS`, `LDFLAGS`, ... environment variables can be modified to
 customise the build.
 
-Finally, you can build and install Grid:
+Finally, you can build, check, and install Grid:
 
 ``` bash
-make; make install
+make; make check; make install
 ```
 
 To minimise the build time, only the tests at the root of the `tests` directory are built by default. If you want to build tests in the sub-directory `<subdir>` you can execute:
@@ -121,7 +141,7 @@ If you want to build all the tests at once just use `make tests`.
 - `--enable-gen-simd-width=<size>`: select the size (in bytes) of the generic SIMD vector type (default: 32 bytes).
 - `--enable-precision={single|double}`: set the default precision (default: `double`).
 - `--enable-precision=<comm>`: Use `<comm>` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below.
-- `--enable-rng={ranlux48|mt19937}`: choose the RNG (default: `ranlux48 `).
+- `--enable-rng={sitmo|ranlux48|mt19937}`: choose the RNG (default: `sitmo `).
 - `--disable-timers`: disable system dependent high-resolution timers.
 - `--enable-chroma`: enable Chroma regression tests.
 - `--enable-doxygen-doc`: enable the Doxygen documentation generation (build with `make doxygen-doc`)
@@ -159,7 +179,6 @@ Alternatively, some CPU codenames can be directly used:
 
 | `<code>`    | Description                            |
 | ----------- | -------------------------------------- |
-| `KNC`       | [Intel Xeon Phi codename Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) |
 | `KNL`       | [Intel Xeon Phi codename Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |
 | `BGQ`       | Blue Gene/Q                            |
 

TODO

@@ -1,6 +1,27 @@
 TODO:
 ---------------
 
+Peter's work list:
+1)- Precision conversion and sort out localConvert      <-- 
+2)- Remove DenseVector, DenseMatrix; Use Eigen instead. <-- 
+
+-- Profile CG, BlockCG, etc... Flop count/rate -- PARTIAL, time but no flop/s yet
+-- Physical propagator interface
+-- Conserved currents
+-- GaugeFix into central location
+-- Multigrid Wilson and DWF, compare to other Multigrid implementations
+-- HDCR resume
+
+Recent DONE 
+-- Binary I/O speed up & x-strips                      <-- DONE
+-- Cut down the exterior overhead                      <-- DONE
+-- Interior legs from SHM comms                        <-- DONE
+-- Half-precision comms                                <-- DONE
+-- Merge high precision reduction into develop        
+-- multiRHS DWF; benchmark on Cori/BNL for comms elimination
+   -- slice* linalg routines for multiRHS, BlockCG    
+
+-----
 * Forces; the UdSdU  term in gauge force term is half of what I think it should
   be. This is a consequence of taking ONLY the first term in:
 
@@ -21,16 +42,8 @@ TODO:
   This means we must double the force in the Test_xxx_force routines, and is the origin of the factor of two.
   This 2x is applied by hand in the fermion routines and in the Test_rect_force routine.
 
-
-Policies:
-
-* Link smearing/boundary conds; Policy class based implementation ; framework more in place
-
 * Support different boundary conditions (finite temp, chem. potential ... )
 
-* Support different fermion representations? 
-  - contained entirely within the integrator presently
-
 - Sign of force term.
 
 - Reversibility test.
@@ -41,11 +54,6 @@ Policies:
 
 - Audit oIndex usage for cb behaviour
 
-- Rectangle gauge actions.
-  Iwasaki,
-  Symanzik,
-  ... etc...
-
 - Prepare multigrid for HMC. - Alternate setup schemes.
 
 - Support for ILDG --- ugly, not done
@@ -55,9 +63,11 @@ Policies:
 - FFTnD ?
 
 - Gparity; hand opt use template specialisation elegance to enable the optimised paths ?
+
 - Gparity force term; Gparity (R)HMC.
-- Random number state save restore
+
 - Mobius implementation clean up to rmove #if 0 stale code sequences
+
 - CG -- profile carefully, kernel fusion, whole CG performance measurements.
 
 ================================================================
@@ -90,6 +100,7 @@ Insert/Extract
 Not sure of status of this -- reverify. Things are working nicely now though.
 
 * Make the Tensor types and Complex etc... play more nicely.
+
   - TensorRemove is a hack, come up with a long term rationalised approach to Complex vs. Scalar<Scalar<Scalar<Complex > > >
     QDP forces use of "toDouble" to get back to non tensor scalar. This role is presently taken TensorRemove, but I
     want to introduce a syntax that does not require this.
@@ -112,6 +123,8 @@ Not sure of status of this -- reverify. Things are working nicely now though.
 RECENT
 ---------------
 
+  - Support different fermion representations? -- DONE
+  - contained entirely within the integrator presently
   - Clean up HMC                                                             -- DONE
   - LorentzScalar<GaugeField> gets Gauge link type (cleaner).                -- DONE
   - Simplified the integrators a bit.                                        -- DONE
@@ -123,6 +136,26 @@ RECENT
   - Parallel io improvements                                  -- DONE
   - Plaquette and link trace checks into nersc reader from the Grid_nersc_io.cc test. -- DONE
 
+
+DONE:
+- MultiArray -- MultiRHS done
+- ConjugateGradientMultiShift -- DONE
+- MCR                         -- DONE
+- Remez -- Mike or Boost?     -- DONE
+- Proto (ET)                  -- DONE
+- uBlas                       -- DONE ; Eigen
+- Potentially Useful Boost libraries -- DONE ; Eigen
+- Aligned allocator; memory pool -- DONE
+- Multiprecision              -- DONE
+- Serialization               -- DONE
+- Regex -- Not needed
+- Tokenize -- Why?
+
+- Random number state save restore -- DONE
+- Rectangle gauge actions. -- DONE
+  Iwasaki,
+  Symanzik,
+  ... etc...
 Done: Cayley, Partial , ContFrac force terms.
 
 DONE
@@ -207,6 +240,7 @@ Done
 FUNCTIONALITY: it pleases me to keep track of things I have done (keeps me arguably sane)
 ======================================================================================================
 
+* Link smearing/boundary conds; Policy class based implementation ; framework more in place -- DONE
 * Command line args for geometry, simd, etc. layout. Is it necessary to have -- DONE
   user pass these? Is this a QCD specific?
 

VERSION

@@ -1,6 +1,5 @@
-Version : 0.6.0
+Version : 0.7.0
 
-- AVX512, AVX2, AVX, SSE good
-- Clang 3.5 and above, ICPC v16 and above, GCC 4.9 and above
-- MPI and MPI3
-- HiRep, Smearing, Generic gauge group
+- Clang 3.5 and above, ICPC v16 and above, GCC 6.3 and above recommended
+- MPI and MPI3 comms optimisations for KNL and OPA finished
+- Half precision comms

benchmarks/Benchmark_comms.cc

@@ -31,6 +31,32 @@ using namespace std;
 using namespace Grid;
 using namespace Grid::QCD;
 
+struct time_statistics{
+  double mean;
+  double err;
+  double min;
+  double max;
+
+  void statistics(std::vector<double> v){
+      double sum = std::accumulate(v.begin(), v.end(), 0.0);
+      mean = sum / v.size();
+
+      std::vector<double> diff(v.size());
+      std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; });
+      double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+      err = std::sqrt(sq_sum / (v.size()*(v.size() - 1)));
+
+      auto result = std::minmax_element(v.begin(), v.end());
+      min = *result.first;
+      max = *result.second;
+}
+};
+
+void header(){
+  std::cout <<GridLogMessage << " L  "<<"\t"<<" Ls  "<<"\t"
+            <<std::setw(11)<<"bytes"<<"MB/s uni (err/min/max)"<<"\t\t"<<"MB/s bidi (err/min/max)"<<std::endl;
+};
+
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
@@ -40,15 +66,19 @@ int main (int argc, char ** argv)
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
-  int Nloop=10;
+  int Nloop=100;
   int nmu=0;
+  int maxlat=24;
   for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
 
+  std::cout << GridLogMessage << "Number of iterations to average: "<< Nloop << std::endl;
+  std::vector<double> t_time(Nloop);
+  time_statistics timestat;
+
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-  int maxlat=24;
+  header();
   for(int lat=4;lat<=maxlat;lat+=4){
     for(int Ls=8;Ls<=32;Ls*=2){
 
@@ -58,6 +88,9 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
       std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
@@ -65,8 +98,8 @@ int main (int argc, char ** argv)
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
       for(int i=0;i<Nloop;i++){
+      double start=usecond();
 
 	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 
@@ -102,18 +135,24 @@ int main (int argc, char ** argv)
 	}
 	Grid.SendToRecvFromComplete(requests);
 	Grid.Barrier();
-
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
+      timestat.statistics(t_time);
+
+      double dbytes    = bytes*ppn;
+      double xbytes    = dbytes*2.0*ncomm;
       double rbytes    = xbytes;
       double bidibytes = xbytes+rbytes;
 
-      double time = stop-start; // microseconds
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
 
@@ -121,8 +160,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking sequential halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
-
+  header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
     for(int Ls=8;Ls<=32;Ls*=2){
@@ -130,6 +168,9 @@ int main (int argc, char ** argv)
       std::vector<int> latt_size  ({lat,lat,lat,lat});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
       std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
@@ -138,8 +179,8 @@ int main (int argc, char ** argv)
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
       for(int i=0;i<Nloop;i++){
+      double start=usecond();
     
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
@@ -178,27 +219,34 @@ int main (int argc, char ** argv)
 	  }
 	}
 	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+
       }
 
-      double stop=usecond();
+      timestat.statistics(t_time);
       
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
+      double dbytes    = bytes*ppn;
+      double xbytes    = dbytes*2.0*ncomm;
       double rbytes    = xbytes;
       double bidibytes = xbytes+rbytes;
 
-      double time = stop-start;
+    std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+      
     }
   }  
 
 
-  Nloop=10;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking concurrent STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
     for(int Ls=8;Ls<=32;Ls*=2){
@@ -209,6 +257,9 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<HalfSpinColourVectorD *> xbuf(8);
       std::vector<HalfSpinColourVectorD *> rbuf(8);
@@ -216,70 +267,83 @@ int main (int argc, char ** argv)
       for(int d=0;d<8;d++){
 	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
 	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
       }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
 
-      double start=usecond();
+      double dbytes;
       for(int i=0;i<Nloop;i++){
+	double start=usecond();
+
+	dbytes=0;
+	ncomm=0;
 
 	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 
-	ncomm=0;
 	for(int mu=0;mu<4;mu++){
 	
+
 	  if (mpi_layout[mu]>1 ) {
 	  
 	    ncomm++;
 	    int comm_proc=1;
 	    int xmit_to_rank;
 	    int recv_from_rank;
-	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu][0],
-					    recv_from_rank,
-					    bytes);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu][0],
+					      recv_from_rank,
+					      bytes);
 	
 	    comm_proc = mpi_layout[mu]-1;
 	  
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu+4][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu+4][0],
-					    recv_from_rank,
-					    bytes);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu+4][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu+4][0],
+					      recv_from_rank,
+					      bytes);
 	  
 	  }
 	}
 	Grid.StencilSendToRecvFromComplete(requests);
 	Grid.Barrier();
-
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+	
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
+      timestat.statistics(t_time);
+
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
-      double time = stop-start; // microseconds
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
     }
   }    
 
 
-
-  Nloop=100;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking sequential STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "  L  "<<"\t\t"<<" Ls  "<<"\t\t"<<"bytes"<<"\t\t"<<"MB/s uni"<<"\t\t"<<"MB/s bidi"<<std::endl;
+  header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
     for(int Ls=8;Ls<=32;Ls*=2){
@@ -290,6 +354,9 @@ int main (int argc, char ** argv)
       				    lat*mpi_layout[3]});
 
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
 
       std::vector<HalfSpinColourVectorD *> xbuf(8);
       std::vector<HalfSpinColourVectorD *> rbuf(8);
@@ -297,16 +364,18 @@ int main (int argc, char ** argv)
       for(int d=0;d<8;d++){
 	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
 	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
       }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
-
-      double start=usecond();
+      double dbytes;
       for(int i=0;i<Nloop;i++){
+	double start=usecond();
 
 	std::vector<CartesianCommunicator::CommsRequest_t> requests;
-
+	dbytes=0;
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
 	
@@ -318,42 +387,52 @@ int main (int argc, char ** argv)
 	    int recv_from_rank;
 	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu][0],
-					    recv_from_rank,
-					    bytes);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu][0],
+					      recv_from_rank,
+					      bytes);
 	    Grid.StencilSendToRecvFromComplete(requests);
 	    requests.resize(0);
 
 	    comm_proc = mpi_layout[mu]-1;
 	  
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	    Grid.StencilSendToRecvFromBegin(requests,
-					    (void *)&xbuf[mu+4][0],
-					    xmit_to_rank,
-					    (void *)&rbuf[mu+4][0],
-					    recv_from_rank,
-					    bytes);
+	    dbytes+=
+	      Grid.StencilSendToRecvFromBegin(requests,
+					      (void *)&xbuf[mu+4][0],
+					      xmit_to_rank,
+					      (void *)&rbuf[mu+4][0],
+					      recv_from_rank,
+					      bytes);
 	    Grid.StencilSendToRecvFromComplete(requests);
 	    requests.resize(0);
 	  
 	  }
 	}
 	Grid.Barrier();
-
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+	
       }
-      double stop=usecond();
 
-      double dbytes    = bytes;
-      double xbytes    = Nloop*dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
+      timestat.statistics(t_time);
 
-      double time = stop-start; // microseconds
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
 
-      std::cout<<GridLogMessage << lat<<"\t\t"<<Ls<<"\t\t"<<bytes<<"\t\t"<<xbytes/time<<"\t\t"<<bidibytes/time<<std::endl;
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+ 
     }
   }    
 

benchmarks/Benchmark_dwf.cc

@@ -1,28 +1,22 @@
-    /*************************************************************************************
-
+ /*************************************************************************************
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./benchmarks/Benchmark_dwf.cc
-
     Copyright (C) 2015
 
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+    Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -151,9 +145,7 @@ int main (int argc, char ** argv)
   RealD M5  =1.8;
 
   RealD NP = UGrid->_Nprocessors;
-
-  std::cout << GridLogMessage << "Creating action operator " << std::endl;
-  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  RealD NN = UGrid->NodeCount();
 
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
   std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
@@ -163,16 +155,22 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
   if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
   if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
 
+  DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
   int ncall =1000;
   if (1) {
     FGrid->Barrier();
     Dw.ZeroCounters();
     Dw.Dhop(src,result,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
     double t0=usecond();
     for(int i=0;i<ncall;i++){
       __SSC_START;
@@ -190,6 +188,7 @@ int main (int argc, char ** argv)
     //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
     err = ref-result; 
     std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
 
@@ -206,6 +205,34 @@ int main (int argc, char ** argv)
     Dw.Report();
   }
 
+  DomainWallFermionRL DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  if (1) {
+    FGrid->Barrier();
+    DwH.ZeroCounters();
+    DwH.Dhop(src,result,0);
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwH.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called half prec comms Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    err = ref-result; 
+    std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+
+    assert (norm2(err)< 1.0e-3 );
+    DwH.Report();
+  }
+
   if (1)
   {
 
@@ -214,6 +241,10 @@ int main (int argc, char ** argv)
     std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
     if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
     if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
     if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
@@ -245,6 +276,7 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
     //    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
     sDw.Report();
     RealD sum=0;
@@ -277,6 +309,10 @@ int main (int argc, char ** argv)
       std::cout << GridLogMessage<< "* Vectorising fifth dimension by "<<vComplex::Nsimd()<<std::endl;
       if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
       if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
       if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) 
 	std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
       if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) 
@@ -316,6 +352,7 @@ int main (int argc, char ** argv)
 
       std::cout<<GridLogMessage << "sDeo mflop/s =   "<< flops/(t1-t0)<<std::endl;
       std::cout<<GridLogMessage << "sDeo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+      std::cout<<GridLogMessage << "sDeo mflop/s per node   "<< flops/(t1-t0)/NN<<std::endl;
       sDw.Report();
 
       sDw.DhopEO(ssrc_o,sr_e,DaggerNo);
@@ -394,14 +431,15 @@ int main (int argc, char ** argv)
 
 
   // S-direction is INNERMOST and takes no part in the parity.
-  static int Opt;  // these are a temporary hack
-  static int Comms;  // these are a temporary hack
-
   std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
   std::cout << GridLogMessage<< "* Benchmarking DomainWallFermionR::DhopEO                "<<std::endl;
   std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplex::Nsimd()<<std::endl;
   if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
   if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
   if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
@@ -422,6 +460,7 @@ int main (int argc, char ** argv)
 
     std::cout<<GridLogMessage << "Deo mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "Deo mflop/s per rank   "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "Deo mflop/s per node   "<< flops/(t1-t0)/NN<<std::endl;
     Dw.Report();
   }
   Dw.DhopEO(src_o,r_e,DaggerNo);
@@ -453,3 +492,4 @@ int main (int argc, char ** argv)
 
   Grid_finalize();
 }
+

benchmarks/Benchmark_memory_bandwidth.cc

@@ -55,8 +55,8 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
-  uint64_t lmax=44;
-#define NLOOP (1*lmax*lmax*lmax*lmax/vol)
+  uint64_t lmax=64;
+#define NLOOP (100*lmax*lmax*lmax*lmax/vol)
   for(int lat=4;lat<=lmax;lat+=4){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});

benchmarks/Benchmark_su3.cc

@@ -35,8 +35,9 @@ using namespace Grid::QCD;
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
+#define LMAX (64)
 
-  int Nloop=1000;
+  int Nloop=20;
 
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
@@ -50,7 +51,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
@@ -82,7 +83,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
@@ -113,7 +114,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
@@ -144,7 +145,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];

benchmarks/Makefile.am

@@ -1,11 +1,7 @@
 include Make.inc
 
-simple: simple_su3_test.o simple_su3_expr.o simple_simd_test.o
-
-EXTRA_LIBRARIES = libsimple_su3_test.a libsimple_su3_expr.a libsimple_simd_test.a
-
-libsimple_su3_test_a_SOURCES = simple_su3_test.cc
-
-libsimple_su3_expr_a_SOURCES = simple_su3_expr.cc
-
-libsimple_simd_test_a_SOURCES = simple_simd_test.cc
+bench-local: all
+	./Benchmark_su3
+	./Benchmark_memory_bandwidth
+	./Benchmark_wilson
+	./Benchmark_dwf --dslash-unroll

bootstrap.sh

@@ -1,6 +1,6 @@
-#!/usr/bin/env bash
+]#!/usr/bin/env bash
 
-EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2'
+EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2'
 
 echo "-- deploying Eigen source..."
 wget ${EIGEN_URL} --no-check-certificate

configure.ac

@@ -1,16 +1,19 @@
 AC_PREREQ([2.63])
-AC_INIT([Grid], [0.6.0], [https://github.com/paboyle/Grid], [Grid])
+AC_INIT([Grid], [0.7.0], [https://github.com/paboyle/Grid], [Grid])
 AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
-AM_INIT_AUTOMAKE(subdir-objects)
+AM_INIT_AUTOMAKE([subdir-objects 1.13])
+AM_EXTRA_RECURSIVE_TARGETS([tests bench])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h],[sed -i 's|PACKAGE_|GRID_|' lib/Config.h])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
+################ Get git info
+#AC_REVISION([m4_esyscmd_s([./scripts/configure.commit])])
+
 ############### Checks for programs
-CXXFLAGS="-O3 $CXXFLAGS"
 AC_PROG_CXX
 AC_PROG_RANLIB
 
@@ -24,12 +27,15 @@ AX_GXX_VERSION
 AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
       [version of g++ that will compile the code])
 
+CXXFLAGS="-g $CXXFLAGS"
+
+
 ############### Checks for typedefs, structures, and compiler characteristics
 AC_TYPE_SIZE_T
 AC_TYPE_UINT32_T
 AC_TYPE_UINT64_T
 
-############### OpenMP 
+############### OpenMP
 AC_OPENMP
 ac_openmp=no
 if test "${OPENMP_CXXFLAGS}X" != "X"; then
@@ -60,16 +66,23 @@ AC_ARG_WITH([mpfr],
     [AM_CXXFLAGS="-I$with_mpfr/include $AM_CXXFLAGS"]
     [AM_LDFLAGS="-L$with_mpfr/lib $AM_LDFLAGS"])
 
-############### FFTW3 
-AC_ARG_WITH([fftw],    
+############### FFTW3
+AC_ARG_WITH([fftw],
             [AS_HELP_STRING([--with-fftw=prefix],
             [try this for a non-standard install prefix of the FFTW3 library])],
             [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
             [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
 
-############### lapack 
+############### LIME
+AC_ARG_WITH([lime],
+            [AS_HELP_STRING([--with-lime=prefix],
+            [try this for a non-standard install prefix of the LIME library])],
+            [AM_CXXFLAGS="-I$with_lime/include $AM_CXXFLAGS"]
+            [AM_LDFLAGS="-L$with_lime/lib $AM_LDFLAGS"])
+
+############### lapack
 AC_ARG_ENABLE([lapack],
-    [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])], 
+    [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])],
     [ac_LAPACK=${enable_lapack}], [ac_LAPACK=no])
 
 case ${ac_LAPACK} in
@@ -83,6 +96,18 @@ case ${ac_LAPACK} in
         AC_DEFINE([USE_LAPACK],[1],[use LAPACK]);;
 esac
 
+############### FP16 conversions
+AC_ARG_ENABLE([sfw-fp16],
+    [AC_HELP_STRING([--enable-sfw-fp16=yes|no], [enable software fp16 comms])],
+    [ac_SFW_FP16=${enable_sfw_fp16}], [ac_SFW_FP16=yes])
+case ${ac_SFW_FP16} in
+    yes)
+      AC_DEFINE([SFW_FP16],[1],[software conversion to fp16]);;
+    no);;
+    *)
+      AC_MSG_ERROR(["SFW FP16 option not supported ${ac_SFW_FP16}"]);;
+esac
+
 ############### MKL
 AC_ARG_ENABLE([mkl],
     [AC_HELP_STRING([--enable-mkl=yes|no|prefix], [enable Intel MKL for LAPACK & FFTW])],
@@ -108,7 +133,7 @@ AC_ARG_WITH([hdf5],
 
 ############### first-touch
 AC_ARG_ENABLE([numa],
-    [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
+    [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])],
     [ac_NUMA=${enable_NUMA}],[ac_NUMA=no])
 
 case ${ac_NUMA} in
@@ -134,8 +159,8 @@ if test "${ac_MKL}x" != "nox"; then
 fi
 
 AC_SEARCH_LIBS([__gmpf_init], [gmp],
-               [AC_SEARCH_LIBS([mpfr_init], [mpfr], 
-                               [AC_DEFINE([HAVE_LIBMPFR], [1], 
+               [AC_SEARCH_LIBS([mpfr_init], [mpfr],
+                               [AC_DEFINE([HAVE_LIBMPFR], [1],
                                           [Define to 1 if you have the `MPFR' library])]
                                [have_mpfr=true], [AC_MSG_ERROR([MPFR library not found])])]
                [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library])]
@@ -144,7 +169,7 @@ AC_SEARCH_LIBS([__gmpf_init], [gmp],
 if test "${ac_LAPACK}x" != "nox"; then
     AC_SEARCH_LIBS([LAPACKE_sbdsdc], [lapack], [],
                    [AC_MSG_ERROR("LAPACK enabled but library not found")])
-fi   
+fi
 
 AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_SEARCH_LIBS([fftwf_execute], [fftw3f], [],
@@ -152,6 +177,18 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
                [have_fftw=true])
 
+AC_SEARCH_LIBS([limeCreateReader], [lime],
+               [AC_DEFINE([HAVE_LIME], [1], [Define to 1 if you have the `LIME' library])]
+               [have_lime=true],
+	       [AC_MSG_WARN(C-LIME library was not found in your system.
+In order to use ILGG file format please install or provide the correct path to your installation
+Info at: http://usqcd.jlab.org/usqcd-docs/c-lime/)])
+
+AC_SEARCH_LIBS([crc32], [z],
+               [AC_DEFINE([HAVE_ZLIB], [1], [Define to 1 if you have the `LIBZ' library])]
+               [have_zlib=true],
+	       [AC_MSG_ERROR(zlib library was not found in your system.)])
+
 AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp],
                [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])]
                [have_hdf5=true]
@@ -176,19 +213,26 @@ case ${ax_cv_cxx_compiler_vendor} in
     case ${ac_SIMD} in
       SSE4)
         AC_DEFINE([SSE4],[1],[SSE4 intrinsics])
-        SIMD_FLAGS='-msse4.2';;
+	case ${ac_SFW_FP16} in
+	  yes)
+	  SIMD_FLAGS='-msse4.2';;
+	  no)
+	  SIMD_FLAGS='-msse4.2 -mf16c';;
+	  *)
+          AC_MSG_ERROR(["SFW_FP16 must be either yes or no value ${ac_SFW_FP16} "]);;
+	esac;;
       AVX)
         AC_DEFINE([AVX1],[1],[AVX intrinsics])
-        SIMD_FLAGS='-mavx';;
+        SIMD_FLAGS='-mavx -mf16c';;
       AVXFMA4)
         AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4])
-        SIMD_FLAGS='-mavx -mfma4';;
+        SIMD_FLAGS='-mavx -mfma4 -mf16c';;
       AVXFMA)
         AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA3])
-        SIMD_FLAGS='-mavx -mfma';;
+        SIMD_FLAGS='-mavx -mfma -mf16c';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
-        SIMD_FLAGS='-mavx2 -mfma';;
+        SIMD_FLAGS='-mavx2 -mfma -mf16c';;
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
@@ -297,7 +341,7 @@ case ${ac_COMMS} in
         comms_type='shmem'
      ;;
      *)
-        AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); 
+        AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]);
      ;;
 esac
 case ${ac_COMMS} in
@@ -334,7 +378,7 @@ case ${ac_RNG} in
       AC_DEFINE([RNG_SITMO],[1],[RNG_SITMO] )
      ;;
      *)
-      AC_MSG_ERROR([${ac_RNG} unsupported --enable-rng option]); 
+      AC_MSG_ERROR([${ac_RNG} unsupported --enable-rng option]);
      ;;
 esac
 
@@ -351,7 +395,7 @@ case ${ac_TIMERS} in
       AC_DEFINE([TIMERS_OFF],[1],[TIMERS_OFF] )
      ;;
      *)
-      AC_MSG_ERROR([${ac_TIMERS} unsupported --enable-timers option]); 
+      AC_MSG_ERROR([${ac_TIMERS} unsupported --enable-timers option]);
      ;;
 esac
 
@@ -363,7 +407,7 @@ case ${ac_CHROMA} in
      yes|no)
      ;;
      *)
-       AC_MSG_ERROR([${ac_CHROMA} unsupported --enable-chroma option]); 
+       AC_MSG_ERROR([${ac_CHROMA} unsupported --enable-chroma option]);
      ;;
 esac
 
@@ -384,12 +428,65 @@ DX_INIT_DOXYGEN([$PACKAGE_NAME], [doxygen.cfg])
 
 ############### Ouput
 cwd=`pwd -P`; cd ${srcdir}; abs_srcdir=`pwd -P`; cd ${cwd}
+GRID_CXXFLAGS="$AM_CXXFLAGS $CXXFLAGS"
+GRID_LDFLAGS="$AM_LDFLAGS $LDFLAGS"
+GRID_LIBS=$LIBS
+GRID_SHORT_SHA=`git rev-parse --short HEAD`
+GRID_SHA=`git rev-parse HEAD`
+GRID_BRANCH=`git rev-parse --abbrev-ref HEAD`
 AM_CXXFLAGS="-I${abs_srcdir}/include $AM_CXXFLAGS"
 AM_CFLAGS="-I${abs_srcdir}/include $AM_CFLAGS"
 AM_LDFLAGS="-L${cwd}/lib $AM_LDFLAGS"
 AC_SUBST([AM_CFLAGS])
 AC_SUBST([AM_CXXFLAGS])
 AC_SUBST([AM_LDFLAGS])
+AC_SUBST([GRID_CXXFLAGS])
+AC_SUBST([GRID_LDFLAGS])
+AC_SUBST([GRID_LIBS])
+AC_SUBST([GRID_SHA])
+AC_SUBST([GRID_BRANCH])
+
+git_commit=`cd $srcdir && ./scripts/configure.commit`
+
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Summary of configuration for $PACKAGE v$VERSION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----- GIT VERSION -------------------------------------
+$git_commit
+----- PLATFORM ----------------------------------------
+architecture (build)        : $build_cpu
+os (build)                  : $build_os
+architecture (target)       : $target_cpu
+os (target)                 : $target_os
+compiler vendor             : ${ax_cv_cxx_compiler_vendor}
+compiler version            : ${ax_cv_gxx_version}
+----- BUILD OPTIONS -----------------------------------
+SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
+Threading                   : ${ac_openmp}
+Communications type         : ${comms_type}
+Default precision           : ${ac_PRECISION}
+Software FP16 conversion    : ${ac_SFW_FP16}
+RNG choice                  : ${ac_RNG}
+GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
+LAPACK                      : ${ac_LAPACK}
+FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
+LIME (ILDG support)         : `if test "x$have_lime" = xtrue; then echo yes; else echo no; fi`
+HDF5                        : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi`
+build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
+----- BUILD FLAGS -------------------------------------
+CXXFLAGS:
+`echo ${AM_CXXFLAGS} ${CXXFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
+LDFLAGS:
+`echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
+LIBS:
+`echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
+-------------------------------------------------------" > grid.configure.summary
+
+GRID_SUMMARY="`cat grid.configure.summary`"
+AM_SUBST_NOTMAKE([GRID_SUMMARY])
+AC_SUBST([GRID_SUMMARY])
+
+AC_CONFIG_FILES([grid-config], [chmod +x grid-config])
 AC_CONFIG_FILES(Makefile)
 AC_CONFIG_FILES(lib/Makefile)
 AC_CONFIG_FILES(tests/Makefile)
@@ -400,6 +497,7 @@ AC_CONFIG_FILES(tests/forces/Makefile)
 AC_CONFIG_FILES(tests/hadrons/Makefile)
 AC_CONFIG_FILES(tests/hmc/Makefile)
 AC_CONFIG_FILES(tests/solver/Makefile)
+AC_CONFIG_FILES(tests/smearing/Makefile)
 AC_CONFIG_FILES(tests/qdpxx/Makefile)
 AC_CONFIG_FILES(tests/testu01/Makefile)
 AC_CONFIG_FILES(benchmarks/Makefile)
@@ -407,36 +505,7 @@ AC_CONFIG_FILES(extras/Makefile)
 AC_CONFIG_FILES(extras/Hadrons/Makefile)
 AC_OUTPUT
 
-echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Summary of configuration for $PACKAGE v$VERSION
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+echo ""
+cat grid.configure.summary
+echo ""
 
------ PLATFORM ----------------------------------------
-architecture (build)        : $build_cpu
-os (build)                  : $build_os
-architecture (target)       : $target_cpu
-os (target)                 : $target_os
-compiler vendor             : ${ax_cv_cxx_compiler_vendor}
-compiler version            : ${ax_cv_gxx_version}
------ BUILD OPTIONS -----------------------------------
-SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
-Threading                   : ${ac_openmp} 
-Communications type         : ${comms_type}
-Default precision           : ${ac_PRECISION}
-RNG choice                  : ${ac_RNG} 
-GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
-LAPACK                      : ${ac_LAPACK}
-FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
-HDF5                        : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi`
-build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
------ BUILD FLAGS -------------------------------------
-CXXFLAGS:
-`echo ${AM_CXXFLAGS} ${CXXFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
-LDFLAGS:
-`echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
-LIBS:
-`echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
--------------------------------------------------------" > config.summary
-echo ""
-cat config.summary
-echo ""

extras/Hadrons/Application.cc

@@ -162,7 +162,8 @@ void Application::saveParameterFile(const std::string parameterFileName)
 sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
 
 #define DEFINE_MEMPEAK \
-auto memPeak = [this](const std::vector<unsigned int> &program)\
+GeneticScheduler<unsigned int>::ObjFunc memPeak = \
+[this](const std::vector<unsigned int> &program)\
 {\
     unsigned int memPeak;\
     bool         msg;\

extras/Hadrons/Global.hpp

@@ -145,6 +145,15 @@ std::string typeName(void)
     return typeName(typeIdPt<T>());
 }
 
+// default writers/readers
+#ifdef HAVE_HDF5
+typedef Hdf5Reader CorrReader;
+typedef Hdf5Writer CorrWriter;
+#else
+typedef XmlReader CorrReader;
+typedef XmlWriter CorrWriter;
+#endif
+
 END_HADRONS_NAMESPACE
 
 #endif // Hadrons_Global_hpp_

extras/Hadrons/Modules.hpp

@@ -29,12 +29,20 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #include <Grid/Hadrons/Modules/MAction/DWF.hpp>
 #include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
 #include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
+#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
 #include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
 #include <Grid/Hadrons/Modules/MGauge/Load.hpp>
 #include <Grid/Hadrons/Modules/MGauge/Random.hpp>
 #include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
+#include <Grid/Hadrons/Modules/MLoop/NoiseLoop.hpp>
 #include <Grid/Hadrons/Modules/MSolver/RBPrecCG.hpp>
 #include <Grid/Hadrons/Modules/MSource/Point.hpp>
 #include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
+#include <Grid/Hadrons/Modules/MSource/Wall.hpp>
 #include <Grid/Hadrons/Modules/MSource/Z2.hpp>
 #include <Grid/Hadrons/Modules/Quark.hpp>

extras/Hadrons/Modules/MContraction/Baryon.hpp

@@ -112,7 +112,7 @@ void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
                  << " quarks '" << par().q1 << "', '" << par().q2 << "', and '"
                  << par().q3 << "'" << std::endl;
     
-    XmlWriter             writer(par().output);
+    CorrWriter             writer(par().output);
     PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
     PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
     PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q2);

extras/Hadrons/Modules/MContraction/DiscLoop.hpp

@@ -0,0 +1,144 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/DiscLoop.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_DiscLoop_hpp_
+#define Hadrons_DiscLoop_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                                DiscLoop                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class DiscLoopPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(DiscLoopPar,
+                                    std::string,    q_loop,
+                                    Gamma::Algebra, gamma,
+                                    std::string,    output);
+};
+
+template <typename FImpl>
+class TDiscLoop: public Module<DiscLoopPar>
+{
+    TYPE_ALIASES(FImpl,);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma,
+                                        std::vector<Complex>, corr);
+    };
+public:
+    // constructor
+    TDiscLoop(const std::string name);
+    // destructor
+    virtual ~TDiscLoop(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(DiscLoop, TDiscLoop<FIMPL>, MContraction);
+
+/******************************************************************************
+ *                       TDiscLoop implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TDiscLoop<FImpl>::TDiscLoop(const std::string name)
+: Module<DiscLoopPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TDiscLoop<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q_loop};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TDiscLoop<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDiscLoop<FImpl>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TDiscLoop<FImpl>::execute(void)
+{
+    LOG(Message) << "Computing disconnected loop contraction '" << getName() 
+                 << "' using '" << par().q_loop << "' with " << par().gamma 
+                 << " insertion." << std::endl;
+
+    CorrWriter            writer(par().output);
+    PropagatorField       &q_loop = *env().template getObject<PropagatorField>(par().q_loop);
+    LatticeComplex        c(env().getGrid());
+    Gamma                 gamma(par().gamma);
+    std::vector<TComplex> buf;
+    Result                result;
+
+    c = trace(gamma*q_loop);
+    sliceSum(c, buf, Tp);
+
+    result.gamma = par().gamma;
+    result.corr.resize(buf.size());
+    for (unsigned int t = 0; t < buf.size(); ++t)
+    {
+        result.corr[t] = TensorRemove(buf[t]);
+    }
+
+    write(writer, "disc", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_DiscLoop_hpp_

extras/Hadrons/Modules/MContraction/Gamma3pt.hpp

@@ -0,0 +1,170 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/Gamma3pt.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_Gamma3pt_hpp_
+#define Hadrons_Gamma3pt_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ * 3pt contraction with gamma matrix insertion.
+ *
+ * Schematic:
+ *
+ *             q2           q3
+ *        /----<------*------<----¬
+ *       /          gamma          \
+ *      /                           \
+ *   i *                            * f
+ *      \                          /
+ *       \                        /
+ *        \----------->----------/
+ *                   q1
+ *
+ *      trace(g5*q1*adj(q2)*g5*gamma*q3)
+ */
+
+/******************************************************************************
+ *                               Gamma3pt                                     *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+class Gamma3ptPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(Gamma3ptPar,
+                                    std::string,    q1,
+                                    std::string,    q2,
+                                    std::string,    q3,
+                                    Gamma::Algebra, gamma,
+                                    std::string,    output);
+};
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+class TGamma3pt: public Module<Gamma3ptPar>
+{
+    TYPE_ALIASES(FImpl1, 1);
+    TYPE_ALIASES(FImpl2, 2);
+    TYPE_ALIASES(FImpl3, 3);
+    class Result: Serializable
+    {
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma,
+                                        std::vector<Complex>, corr);
+    };
+public:
+    // constructor
+    TGamma3pt(const std::string name);
+    // destructor
+    virtual ~TGamma3pt(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Gamma3pt, ARG(TGamma3pt<FIMPL, FIMPL, FIMPL>), MContraction);
+
+/******************************************************************************
+ *                       TGamma3pt implementation                             *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+TGamma3pt<FImpl1, FImpl2, FImpl3>::TGamma3pt(const std::string name)
+: Module<Gamma3ptPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3};
+    
+    return in;
+}
+
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TGamma3pt<FImpl1, FImpl2, FImpl3>::setup(void)
+{
+    
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl1, typename FImpl2, typename FImpl3>
+void TGamma3pt<FImpl1, FImpl2, FImpl3>::execute(void)
+{
+    LOG(Message) << "Computing 3pt contractions '" << getName() << "' using"
+                 << " quarks '" << par().q1 << "', '" << par().q2 << "' and '"
+                 << par().q3 << "', with " << par().gamma << " insertion." 
+                 << std::endl;
+
+    CorrWriter            writer(par().output);
+    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q3);
+    LatticeComplex        c(env().getGrid());
+    Gamma                 g5(Gamma::Algebra::Gamma5);
+    Gamma                 gamma(par().gamma);
+    std::vector<TComplex> buf;
+    Result                result;
+
+    c = trace(g5*q1*adj(q2)*(g5*gamma)*q3);
+    sliceSum(c, buf, Tp);
+
+    result.gamma = par().gamma;
+    result.corr.resize(buf.size());
+    for (unsigned int t = 0; t < buf.size(); ++t)
+    {
+        result.corr[t] = TensorRemove(buf[t]);
+    }
+
+    write(writer, "gamma3pt", result);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_Gamma3pt_hpp_

extras/Hadrons/Modules/MContraction/Meson.hpp

@@ -6,8 +6,10 @@ Source file: extras/Hadrons/Modules/MContraction/Meson.hpp
 
 Copyright (C) 2015
 Copyright (C) 2016
+Copyright (C) 2017
 
 Author: Antonin Portelli <antonin.portelli@me.com>
+        Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -36,20 +38,39 @@ See the full license in the file "LICENSE" in the top level distribution directo
 
 BEGIN_HADRONS_NAMESPACE
 
+/*
+ 
+ Meson contractions
+ -----------------------------
+ 
+ * options:
+ - q1: input propagator 1 (string)
+ - q2: input propagator 2 (string)
+ - gammas: gamma products to insert at sink & source, pairs of gamma matrices 
+           (space-separated strings) in angled brackets (i.e. <g_sink g_src>),
+           in a sequence (e.g. "<Gamma5 Gamma5><Gamma5 GammaT>").
+
+           Special values: "all" - perform all possible contractions.
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0."),
+        given as multiples of (2*pi) / L.
+*/
+
 /******************************************************************************
  *                                TMeson                                       *
  ******************************************************************************/
 BEGIN_MODULE_NAMESPACE(MContraction)
 
+typedef std::pair<Gamma::Algebra, Gamma::Algebra> GammaPair;
+
 class MesonPar: Serializable
 {
 public:
     GRID_SERIALIZABLE_CLASS_MEMBERS(MesonPar,
-                                    std::string,    q1,
-                                    std::string,    q2,
-                                    std::string,    output,
-                                    Gamma::Algebra, gammaSource,
-                                    Gamma::Algebra, gammaSink);
+                                    std::string, q1,
+                                    std::string, q2,
+                                    std::string, gammas,
+                                    std::string, mom,
+                                    std::string, output);
 };
 
 template <typename FImpl1, typename FImpl2>
@@ -61,7 +82,10 @@ public:
     class Result: Serializable
     {
     public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(Result, std::vector<Complex>, corr);
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
+                                        Gamma::Algebra, gamma_snk,
+                                        Gamma::Algebra, gamma_src,
+                                        std::vector<Complex>, corr);
     };
 public:
     // constructor
@@ -71,6 +95,7 @@ public:
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
+    virtual void parseGammaString(std::vector<GammaPair> &gammaList);
     // execution
     virtual void execute(void);
 };
@@ -103,6 +128,32 @@ std::vector<std::string> TMeson<FImpl1, FImpl2>::getOutput(void)
     return output;
 }
 
+template <typename FImpl1, typename FImpl2>
+void TMeson<FImpl1, FImpl2>::parseGammaString(std::vector<GammaPair> &gammaList)
+{
+    // Determine gamma matrices to insert at source/sink.
+    if (par().gammas.compare("all") == 0)
+    {
+        // Do all contractions.
+        unsigned int n_gam = Ns * Ns;
+        gammaList.resize(n_gam*n_gam);
+        for (unsigned int i = 1; i < Gamma::nGamma; i += 2)
+        {
+            for (unsigned int j = 1; j < Gamma::nGamma; j += 2)
+            {
+                gammaList.push_back(std::make_pair((Gamma::Algebra)i, 
+                                                   (Gamma::Algebra)j));
+            }
+        }
+    }
+    else
+    {
+        // Parse individual contractions from input string.
+        gammaList = strToVec<GammaPair>(par().gammas);
+    }
+}
+
+
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl1, typename FImpl2>
 void TMeson<FImpl1, FImpl2>::execute(void)
@@ -111,21 +162,44 @@ void TMeson<FImpl1, FImpl2>::execute(void)
                  << " quarks '" << par().q1 << "' and '" << par().q2 << "'"
                  << std::endl;
     
-    XmlWriter             writer(par().output);
-    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
-    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
-    LatticeComplex        c(env().getGrid());
-    Gamma                 gSrc(par().gammaSource), gSnk(par().gammaSink);
-    Gamma                 g5(Gamma::Algebra::Gamma5);
-    std::vector<TComplex> buf;
-    Result                result;
-    
-    c = trace(gSnk*q1*adj(gSrc)*g5*adj(q2)*g5);
-    sliceSum(c, buf, Tp);
-    result.corr.resize(buf.size());
-    for (unsigned int t = 0; t < buf.size(); ++t)
+    CorrWriter              writer(par().output);
+    PropagatorField1       &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField2       &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    LatticeComplex         c(env().getGrid());
+    Gamma                  g5(Gamma::Algebra::Gamma5);
+    std::vector<GammaPair> gammaList;
+    std::vector<TComplex>  buf;
+    std::vector<Result>    result;
+    std::vector<Real>      p;
+
+    p  = strToVec<Real>(par().mom);
+    LatticeComplex         ph(env().getGrid()), coor(env().getGrid());
+    Complex                i(0.0,1.0);
+    ph = zero;
+    for(unsigned int mu = 0; mu < env().getNd(); mu++)
     {
-        result.corr[t] = TensorRemove(buf[t]);
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
+    }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    
+    parseGammaString(gammaList);
+
+    result.resize(gammaList.size());
+    for (unsigned int i = 0; i < result.size(); ++i)
+    {
+        Gamma gSnk(gammaList[i].first);
+        Gamma gSrc(gammaList[i].second);
+        c = trace((g5*gSnk)*q1*(adj(gSrc)*g5)*adj(q2))*ph;
+        sliceSum(c, buf, Tp);
+
+        result[i].gamma_snk = gammaList[i].first;
+        result[i].gamma_src = gammaList[i].second;
+        result[i].corr.resize(buf.size());
+        for (unsigned int t = 0; t < buf.size(); ++t)
+        {
+            result[i].corr[t] = TensorRemove(buf[t]);
+        }
     }
     write(writer, "meson", result);
 }

extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp

@@ -0,0 +1,114 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakHamiltonian_hpp_
+#define Hadrons_WeakHamiltonian_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonian                                    *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+/*******************************************************************************
+ * Utilities for contractions involving the Weak Hamiltonian.
+ ******************************************************************************/
+//// Sum and store correlator.
+#define MAKE_DIAG(exp, buf, res, n)\
+sliceSum(exp, buf, Tp);\
+res.name = (n);\
+res.corr.resize(buf.size());\
+for (unsigned int t = 0; t < buf.size(); ++t)\
+{\
+    res.corr[t] = TensorRemove(buf[t]);\
+}
+
+//// Contraction of mu index: use 'mu' variable in exp.
+#define SUM_MU(buf,exp)\
+buf = zero;\
+for (unsigned int mu = 0; mu < ndim; ++mu)\
+{\
+    buf += exp;\
+}
+
+enum 
+{
+  i_V = 0,
+  i_A = 1,
+  n_i = 2
+};
+
+class WeakHamiltonianPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WeakHamiltonianPar,
+                                    std::string, q1,
+                                    std::string, q2,
+                                    std::string, q3,
+                                    std::string, q4,
+                                    std::string, output);
+};
+
+#define MAKE_WEAK_MODULE(modname)\
+class T##modname: public Module<WeakHamiltonianPar>\
+{\
+public:\
+    TYPE_ALIASES(FIMPL,)\
+    class Result: Serializable\
+    {\
+    public:\
+        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,\
+                                        std::string, name,\
+                                        std::vector<Complex>, corr);\
+    };\
+public:\
+    /* constructor */ \
+    T##modname(const std::string name);\
+    /* destructor */ \
+    virtual ~T##modname(void) = default;\
+    /* dependency relation */ \
+    virtual std::vector<std::string> getInput(void);\
+    virtual std::vector<std::string> getOutput(void);\
+    /* setup */ \
+    virtual void setup(void);\
+    /* execution */ \
+    virtual void execute(void);\
+    std::vector<std::string> VA_label = {"V", "A"};\
+};\
+MODULE_REGISTER_NS(modname, T##modname, MContraction);
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakHamiltonian_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc

@@ -0,0 +1,137 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian current-current contractions, Eye-type.
+ * 
+ * These contractions are generated by the Q1 and Q2 operators in the physical
+ * basis (see e.g. Fig 3 of arXiv:1507.03094).
+ * 
+ * Schematics:        q4                 |                  
+ *                  /-<-¬                |                             
+ *                 /     \               |             q2           q3
+ *                 \     /               |        /----<------*------<----¬                        
+ *            q2    \   /    q3          |       /          /-*-¬          \
+ *       /-----<-----* *-----<----¬      |      /          /     \          \
+ *    i *            H_W           * f   |   i *           \     /  q4      * f
+ *       \                        /      |      \           \->-/          /   
+ *        \                      /       |       \                        /       
+ *         \---------->---------/        |        \----------->----------/        
+ *                   q1                  |                   q1                  
+ *                                       |
+ *                Saucer (S)             |                  Eye (E)
+ * 
+ * S: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1]*q4*gL[mu][p_2])
+ * E: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1])*trace(q4*gL[mu][p_2])
+ */
+
+/******************************************************************************
+ *                  TWeakHamiltonianEye implementation                        *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakHamiltonianEye::TWeakHamiltonianEye(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakHamiltonianEye::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakHamiltonianEye::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakHamiltonianEye::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakHamiltonianEye::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian (Eye type) contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_eye_diag);
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> S_body(ndim, tmp1);
+    std::vector<PropagatorField> S_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  E_body(ndim, tmp2);
+    std::vector<LatticeComplex>  E_loop(ndim, tmp2);
+
+    // Setup for S-type contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        S_body[mu] = MAKE_SE_BODY(q1, q2, q3, GammaL(Gamma::gmu[mu]));
+        S_loop[mu] = MAKE_SE_LOOP(q4, GammaL(Gamma::gmu[mu]));
+    }
+
+    // Perform S-type contractions.    
+    SUM_MU(expbuf, trace(S_body[mu]*S_loop[mu]))
+    MAKE_DIAG(expbuf, corrbuf, result[S_diag], "HW_S")
+
+    // Recycle sub-expressions for E-type contractions.
+    for (unsigned int mu = 0; mu < ndim; ++mu)
+    {
+        E_body[mu] = trace(S_body[mu]);
+        E_loop[mu] = trace(S_loop[mu]);
+    }
+
+    // Perform E-type contractions.
+    SUM_MU(expbuf, E_body[mu]*E_loop[mu])
+    MAKE_DIAG(expbuf, corrbuf, result[E_diag], "HW_E")
+
+    write(writer, "HW_Eye", result);
+}

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp

@@ -0,0 +1,58 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakHamiltonianEye_hpp_
+#define Hadrons_WeakHamiltonianEye_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonianEye                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    S_diag = 0,
+    E_diag = 1,
+    n_eye_diag = 2
+};
+
+// Saucer and Eye subdiagram contractions.
+#define MAKE_SE_BODY(Q_1, Q_2, Q_3, gamma) (Q_3*g5*Q_1*adj(Q_2)*g5*gamma)
+#define MAKE_SE_LOOP(Q_loop, gamma) (Q_loop*gamma)
+
+MAKE_WEAK_MODULE(WeakHamiltonianEye)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakHamiltonianEye_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc

@@ -0,0 +1,139 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian current-current contractions, Non-Eye-type.
+ * 
+ * These contractions are generated by the Q1 and Q2 operators in the physical
+ * basis (see e.g. Fig 3 of arXiv:1507.03094).
+ * 
+ * Schematic:     
+ *            q2             q3          |           q2              q3
+ *          /--<--¬       /--<--¬        |        /--<--¬         /--<--¬       
+ *         /       \     /       \       |       /       \       /       \      
+ *        /         \   /         \      |      /         \     /         \     
+ *       /           \ /           \     |     /           \   /           \    
+ *    i *             * H_W         *  f |  i *             * * H_W         * f 
+ *      \             *             |    |     \           /   \           /
+ *       \           / \           /     |      \         /     \         /    
+ *        \         /   \         /      |       \       /       \       /  
+ *         \       /     \       /       |        \-->--/         \-->--/      
+ *          \-->--/       \-->--/        |          q1               q4 
+ *            q1             q4          |
+ *                Connected (C)          |                 Wing (W)
+ *
+ * C: trace(q1*adj(q2)*g5*gL[mu]*q3*adj(q4)*g5*gL[mu])
+ * W: trace(q1*adj(q2)*g5*gL[mu])*trace(q3*adj(q4)*g5*gL[mu])
+ * 
+ */
+
+/******************************************************************************
+ *                  TWeakHamiltonianNonEye implementation                     *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakHamiltonianNonEye::TWeakHamiltonianNonEye(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakHamiltonianNonEye::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakHamiltonianNonEye::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakHamiltonianNonEye::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakHamiltonianNonEye::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian (Non-Eye type) contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+    
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_noneye_diag); 
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> C_i_side_loop(ndim, tmp1);
+    std::vector<PropagatorField> C_f_side_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  W_i_side_loop(ndim, tmp2);
+    std::vector<LatticeComplex>  W_f_side_loop(ndim, tmp2);
+
+    // Setup for C-type contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        C_i_side_loop[mu] = MAKE_CW_SUBDIAG(q1, q2, GammaL(Gamma::gmu[mu]));
+        C_f_side_loop[mu] = MAKE_CW_SUBDIAG(q3, q4, GammaL(Gamma::gmu[mu]));
+    }
+
+    // Perform C-type contractions.    
+    SUM_MU(expbuf, trace(C_i_side_loop[mu]*C_f_side_loop[mu]))
+    MAKE_DIAG(expbuf, corrbuf, result[C_diag], "HW_C")
+
+    // Recycle sub-expressions for W-type contractions.
+    for (unsigned int mu = 0; mu < ndim; ++mu)
+    {
+        W_i_side_loop[mu] = trace(C_i_side_loop[mu]);
+        W_f_side_loop[mu] = trace(C_f_side_loop[mu]);
+    }
+
+    // Perform W-type contractions.
+    SUM_MU(expbuf, W_i_side_loop[mu]*W_f_side_loop[mu])
+    MAKE_DIAG(expbuf, corrbuf, result[W_diag], "HW_W")
+
+    write(writer, "HW_NonEye", result);
+}

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp

@@ -0,0 +1,57 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakHamiltonianNonEye_hpp_
+#define Hadrons_WeakHamiltonianNonEye_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakHamiltonianNonEye                              *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    W_diag = 0,
+    C_diag = 1,
+    n_noneye_diag = 2
+};
+
+// Wing and Connected subdiagram contractions
+#define MAKE_CW_SUBDIAG(Q_1, Q_2, gamma) (Q_1*adj(Q_2)*g5*gamma)
+
+MAKE_WEAK_MODULE(WeakHamiltonianNonEye)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakHamiltonianNonEye_hpp_

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc

@@ -0,0 +1,135 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
+
+using namespace Grid;
+using namespace Hadrons;
+using namespace MContraction;
+
+/*
+ * Weak Hamiltonian + current contractions, disconnected topology for neutral 
+ * mesons.
+ * 
+ * These contractions are generated by operators Q_1,...,10 of the dS=1 Weak
+ * Hamiltonian in the physical basis and an additional current J (see e.g. 
+ * Fig 11 of arXiv:1507.03094).
+ * 
+ * Schematic:
+ *                        
+ *           q2          q4             q3
+ *       /--<--¬     /---<--¬       /---<--¬
+ *     /         \ /         \     /        \
+ *  i *           * H_W      |  J *          * f
+ *     \         / \         /     \        /
+ *      \--->---/   \-------/       \------/
+ *          q1 
+ * 
+ * options
+ * - q1: input propagator 1 (string)
+ * - q2: input propagator 2 (string)
+ * - q3: input propagator 3 (string), assumed to be sequential propagator 
+ * - q4: input propagator 4 (string), assumed to be a loop
+ * 
+ * type 1: trace(q1*adj(q2)*g5*gL[mu])*trace(loop*gL[mu])*trace(q3*g5)
+ * type 2: trace(q1*adj(q2)*g5*gL[mu]*loop*gL[mu])*trace(q3*g5)
+ */
+
+/*******************************************************************************
+ *                  TWeakNeutral4ptDisc implementation                         *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+TWeakNeutral4ptDisc::TWeakNeutral4ptDisc(const std::string name)
+: Module<WeakHamiltonianPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+std::vector<std::string> TWeakNeutral4ptDisc::getInput(void)
+{
+    std::vector<std::string> in = {par().q1, par().q2, par().q3, par().q4};
+    
+    return in;
+}
+
+std::vector<std::string> TWeakNeutral4ptDisc::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+void TWeakNeutral4ptDisc::setup(void)
+{
+
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+void TWeakNeutral4ptDisc::execute(void)
+{
+    LOG(Message) << "Computing Weak Hamiltonian neutral disconnected contractions '" 
+                 << getName() << "' using quarks '" << par().q1 << "', '" 
+                 << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
+                 << "'." << std::endl;
+
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_neut_disc_diag);
+    unsigned int ndim   = env().getNd();
+
+    PropagatorField              tmp(env().getGrid());
+    std::vector<PropagatorField> meson(ndim, tmp);
+    std::vector<PropagatorField> loop(ndim, tmp);
+    LatticeComplex               curr(env().getGrid());
+
+    // Setup for type 1 contractions.
+    for (int mu = 0; mu < ndim; ++mu)
+    {
+        meson[mu] = MAKE_DISC_MESON(q1, q2, GammaL(Gamma::gmu[mu]));
+        loop[mu] = MAKE_DISC_LOOP(q4, GammaL(Gamma::gmu[mu]));
+    }
+    curr = MAKE_DISC_CURR(q3, GammaL(Gamma::Algebra::Gamma5));
+
+    // Perform type 1 contractions.    
+    SUM_MU(expbuf, trace(meson[mu]*loop[mu]))
+    expbuf *= curr;
+    MAKE_DIAG(expbuf, corrbuf, result[neut_disc_1_diag], "HW_disc0_1")
+
+    // Perform type 2 contractions.
+    SUM_MU(expbuf, trace(meson[mu])*trace(loop[mu]))
+    expbuf *= curr;
+    MAKE_DIAG(expbuf, corrbuf, result[neut_disc_2_diag], "HW_disc0_2")
+
+    write(writer, "HW_disc0", result);
+}

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp

@@ -0,0 +1,59 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WeakNeutral4ptDisc_hpp_
+#define Hadrons_WeakNeutral4ptDisc_hpp_
+
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/******************************************************************************
+ *                         WeakNeutral4ptDisc                                 *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MContraction)
+
+enum
+{
+    neut_disc_1_diag = 0,
+    neut_disc_2_diag = 1,
+    n_neut_disc_diag = 2
+};
+
+// Neutral 4pt disconnected subdiagram contractions.
+#define MAKE_DISC_MESON(Q_1, Q_2, gamma) (Q_1*adj(Q_2)*g5*gamma)
+#define MAKE_DISC_LOOP(Q_LOOP, gamma) (Q_LOOP*gamma)
+#define MAKE_DISC_CURR(Q_c, gamma) (trace(Q_c*gamma))
+
+MAKE_WEAK_MODULE(WeakNeutral4ptDisc)
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WeakNeutral4ptDisc_hpp_

extras/Hadrons/Modules/MGauge/Load.cc

@@ -65,7 +65,7 @@ void TLoad::setup(void)
 // execution ///////////////////////////////////////////////////////////////////
 void TLoad::execute(void)
 {
-    NerscField  header;
+    FieldMetaData  header;
     std::string fileName = par().file + "."
                            + std::to_string(env().getTrajectory());
     
@@ -74,5 +74,5 @@ void TLoad::execute(void)
     LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
     NerscIO::readConfiguration(U, header, fileName);
     LOG(Message) << "NERSC header:" << std::endl;
-    dump_nersc_header(header, LOG(Message));
+    dump_meta_data(header, LOG(Message));
 }

extras/Hadrons/Modules/MLoop/NoiseLoop.hpp

@@ -0,0 +1,132 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MLoop/NoiseLoop.hpp
+
+Copyright (C) 2016
+
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_NoiseLoop_hpp_
+#define Hadrons_NoiseLoop_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Noise loop propagator
+ -----------------------------
+ * loop_x = q_x * adj(eta_x)
+ 
+ * options:
+ - q = Result of inversion on noise source.
+ - eta = noise source.
+
+ */
+
+
+/******************************************************************************
+ *                         NoiseLoop                                          *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MLoop)
+
+class NoiseLoopPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(NoiseLoopPar,
+                                    std::string, q,
+                                    std::string, eta);
+};
+
+template <typename FImpl>
+class TNoiseLoop: public Module<NoiseLoopPar>
+{
+public:
+    TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TNoiseLoop(const std::string name);
+    // destructor
+    virtual ~TNoiseLoop(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(NoiseLoop, TNoiseLoop<FIMPL>, MLoop);
+
+/******************************************************************************
+ *                 TNoiseLoop implementation                                  *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TNoiseLoop<FImpl>::TNoiseLoop(const std::string name)
+: Module<NoiseLoopPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TNoiseLoop<FImpl>::getInput(void)
+{
+    std::vector<std::string> in = {par().q, par().eta};
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TNoiseLoop<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TNoiseLoop<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TNoiseLoop<FImpl>::execute(void)
+{
+    PropagatorField &loop = *env().template createLattice<PropagatorField>(getName());
+    PropagatorField &q    = *env().template getObject<PropagatorField>(par().q);
+    PropagatorField &eta  = *env().template getObject<PropagatorField>(par().eta);
+    loop = q*adj(eta);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_NoiseLoop_hpp_

extras/Hadrons/Modules/MSource/SeqGamma.hpp

@@ -6,6 +6,7 @@ Source file: extras/Hadrons/Modules/MSource/SeqGamma.hpp
 
 Copyright (C) 2015
 Copyright (C) 2016
+Copyright (C) 2017
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -149,9 +150,9 @@ void TSeqGamma<FImpl>::execute(void)
     for(unsigned int mu = 0; mu < env().getNd(); mu++)
     {
         LatticeCoordinate(coor, mu);
-        ph = ph + p[mu]*coor;
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
     }
-    ph = exp(i*ph);
+    ph = exp((Real)(2*M_PI)*i*ph);
     LatticeCoordinate(t, Tp);
     src = where((t >= par().tA) and (t <= par().tB), ph*(g*q), 0.*q);
 }

extras/Hadrons/Modules/MSource/Wall.hpp

@@ -0,0 +1,147 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid 
+
+Source file: extras/Hadrons/Modules/MSource/Wall.hpp
+
+Copyright (C) 2017
+
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Hadrons_WallSource_hpp_
+#define Hadrons_WallSource_hpp_
+
+#include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Module.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
+
+BEGIN_HADRONS_NAMESPACE
+
+/*
+ 
+ Wall source
+ -----------------------------
+ * src_x = delta(x_3 - tW) * exp(i x.mom)
+ 
+ * options:
+ - tW: source timeslice (integer)
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0.")
+ 
+ */
+
+/******************************************************************************
+ *                         Wall                                               *
+ ******************************************************************************/
+BEGIN_MODULE_NAMESPACE(MSource)
+
+class WallPar: Serializable
+{
+public:
+    GRID_SERIALIZABLE_CLASS_MEMBERS(WallPar,
+                                    unsigned int, tW,
+                                    std::string, mom);
+};
+
+template <typename FImpl>
+class TWall: public Module<WallPar>
+{
+public:
+    TYPE_ALIASES(FImpl,);
+public:
+    // constructor
+    TWall(const std::string name);
+    // destructor
+    virtual ~TWall(void) = default;
+    // dependency relation
+    virtual std::vector<std::string> getInput(void);
+    virtual std::vector<std::string> getOutput(void);
+    // setup
+    virtual void setup(void);
+    // execution
+    virtual void execute(void);
+};
+
+MODULE_REGISTER_NS(Wall, TWall<FIMPL>, MSource);
+
+/******************************************************************************
+ *                 TWall implementation                                       *
+ ******************************************************************************/
+// constructor /////////////////////////////////////////////////////////////////
+template <typename FImpl>
+TWall<FImpl>::TWall(const std::string name)
+: Module<WallPar>(name)
+{}
+
+// dependencies/products ///////////////////////////////////////////////////////
+template <typename FImpl>
+std::vector<std::string> TWall<FImpl>::getInput(void)
+{
+    std::vector<std::string> in;
+    
+    return in;
+}
+
+template <typename FImpl>
+std::vector<std::string> TWall<FImpl>::getOutput(void)
+{
+    std::vector<std::string> out = {getName()};
+    
+    return out;
+}
+
+// setup ///////////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWall<FImpl>::setup(void)
+{
+    env().template registerLattice<PropagatorField>(getName());
+}
+
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TWall<FImpl>::execute(void)
+{    
+    LOG(Message) << "Generating wall source at t = " << par().tW 
+                 << " with momentum " << par().mom << std::endl;
+    
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             ph(env().getGrid()), coor(env().getGrid());
+    std::vector<Real>          p;
+    Complex                    i(0.0,1.0);
+    
+    p  = strToVec<Real>(par().mom);
+    ph = zero;
+    for(unsigned int mu = 0; mu < Nd; mu++)
+    {
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
+    }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    LatticeCoordinate(t, Tp);
+    src = 1.;
+    src = where((t == par().tW), src*ph, 0.*src);
+}
+
+END_MODULE_NAMESPACE
+
+END_HADRONS_NAMESPACE
+
+#endif // Hadrons_WallSource_hpp_

extras/Hadrons/Modules/Quark.hpp

@@ -173,7 +173,7 @@ void TQuark<FImpl>::execute(void)
                 *env().template getObject<PropagatorField>(getName());
             
             axpby_ssp_pminus(sol, 0., sol, 1., sol, 0, 0);
-            axpby_ssp_pplus(sol, 0., sol, 1., sol, 0, Ls_-1);
+            axpby_ssp_pplus(sol, 1., sol, 1., sol, 0, Ls_-1);
             ExtractSlice(tmp, sol, 0, 0);
             FermToProp(p4d, tmp, s, c);
         }

extras/Hadrons/modules.inc

@@ -1,4 +1,7 @@
 modules_cc =\
+  Modules/MContraction/WeakHamiltonianEye.cc \
+  Modules/MContraction/WeakHamiltonianNonEye.cc \
+  Modules/MContraction/WeakNeutral4ptDisc.cc \
   Modules/MGauge/Load.cc \
   Modules/MGauge/Random.cc \
   Modules/MGauge/Unit.cc
@@ -7,13 +10,21 @@ modules_hpp =\
   Modules/MAction/DWF.hpp \
   Modules/MAction/Wilson.hpp \
   Modules/MContraction/Baryon.hpp \
+  Modules/MContraction/DiscLoop.hpp \
+  Modules/MContraction/Gamma3pt.hpp \
   Modules/MContraction/Meson.hpp \
+  Modules/MContraction/WeakHamiltonian.hpp \
+  Modules/MContraction/WeakHamiltonianEye.hpp \
+  Modules/MContraction/WeakHamiltonianNonEye.hpp \
+  Modules/MContraction/WeakNeutral4ptDisc.hpp \
   Modules/MGauge/Load.hpp \
   Modules/MGauge/Random.hpp \
   Modules/MGauge/Unit.hpp \
+  Modules/MLoop/NoiseLoop.hpp \
   Modules/MSolver/RBPrecCG.hpp \
   Modules/MSource/Point.hpp \
   Modules/MSource/SeqGamma.hpp \
+  Modules/MSource/Wall.hpp \
   Modules/MSource/Z2.hpp \
   Modules/Quark.hpp
 

gcc-bug-report/README

@@ -20,4 +20,17 @@ The simple testcase in this directory is the submitted bug report that encapsula
 problem. The test case works with icpc and with clang++, but fails consistently on g++
 current variants.
 
-Peter
+Peter
+
+
+************
+
+Second GCC bug reported, see Issue 100.
+
+https://wandbox.org/permlink/tzssJza6R9XnqANw
+https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80652
+
+Getting Travis fails under gcc-5 for Test_simd, now that I added more comprehensive testing to the
+CI test suite. The limitations of Travis runtime limits & weak cores are being shown.
+
+Travis uses 5.4.1 for g++-5.

grid-config.in

@@ -0,0 +1,86 @@
+#! /bin/sh
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+includedir=@includedir@
+
+usage()
+{
+  cat <<EOF
+Usage: grid-config [OPTION]
+
+Known values for OPTION are:
+
+  --prefix     show Grid installation prefix
+  --cxxflags   print pre-processor and compiler flags
+  --ldflags    print library linking flags
+  --libs       print library linking information
+  --summary    print full build summary
+  --help       display this help and exit
+  --version    output version information
+  --git        print git revision
+
+EOF
+  
+  exit $1
+}
+
+if test $# -eq 0; then
+  usage 1
+fi
+
+cflags=false
+libs=false
+
+while test $# -gt 0; do
+  case "$1" in
+    -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+    *) optarg= ;;
+  esac
+  
+  case "$1" in
+    --prefix)
+      echo $prefix
+    ;;
+    
+    --version)
+      echo @VERSION@
+      exit 0
+    ;;
+    
+    --git)
+      echo "@GRID_BRANCH@ @GRID_SHA@"
+      exit 0
+    ;;
+    
+    --help)
+      usage 0
+    ;;
+    
+    --cxxflags)
+      echo @GRID_CXXFLAGS@
+    ;;
+    
+    --ldflags)
+      echo @GRID_LDFLAGS@
+    ;;
+    
+    --libs)
+      echo @GRID_LIBS@
+    ;;
+    
+    --summary)
+      echo ""
+      echo "@GRID_SUMMARY@"
+      echo ""
+    ;;
+    
+    *)
+      usage
+      exit 1
+    ;;
+  esac
+  shift
+done
+
+exit 0

lib/DisableWarnings.h

@@ -0,0 +1,37 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/DisableWarnings.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef DISABLE_WARNINGS_H
+#define DISABLE_WARNINGS_H
+
+ //disables and intel compiler specific warning (in json.hpp)
+#pragma warning disable 488  
+
+
+#endif

lib/Grid.h

@@ -42,6 +42,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/GridQCDcore.h>
 #include <Grid/qcd/action/Action.h>
 #include <Grid/qcd/smearing/Smearing.h>
+#include <Grid/parallelIO/MetaData.h>
 #include <Grid/qcd/hmc/HMC_aggregate.h>
 
 #endif

lib/GridCore.h

@@ -38,28 +38,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_BASE_H
 #define GRID_BASE_H
 
-///////////////////
-// Std C++ dependencies
-///////////////////
-#include <cassert>
-#include <complex>
-#include <vector>
-#include <iostream>
-#include <iomanip>
-#include <random>
-#include <functional>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <signal.h>
-#include <ctime>
-#include <sys/time.h>
-#include <chrono>
-
-///////////////////
-// Grid headers
-///////////////////
-#include "Config.h"
+#include <Grid/GridStd.h>
 
 #include <Grid/perfmon/Timer.h>
 #include <Grid/perfmon/PerfCount.h>

lib/GridStd.h

@@ -0,0 +1,29 @@
+#ifndef GRID_STD_H
+#define GRID_STD_H
+
+///////////////////
+// Std C++ dependencies
+///////////////////
+#include <cassert>
+#include <complex>
+#include <vector>
+#include <string>
+#include <iostream>
+#include <iomanip>
+#include <random>
+#include <functional>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <ctime>
+#include <sys/time.h>
+#include <chrono>
+#include <zlib.h>
+
+///////////////////
+// Grid config
+///////////////////
+#include "Config.h"
+
+#endif /* GRID_STD_H */

lib/Grid_Eigen_Dense.h

@@ -0,0 +1,9 @@
+#pragma once
+#if defined __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+#include <Grid/Eigen/Dense>
+#if defined __GNUC__
+#pragma GCC diagnostic pop
+#endif

lib/algorithms/Algorithms.h

@@ -46,7 +46,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/iterative/ConjugateGradientMixedPrec.h>
 
 // Lanczos support
-#include <Grid/algorithms/iterative/MatrixUtils.h>
+//#include <Grid/algorithms/iterative/MatrixUtils.h>
 #include <Grid/algorithms/iterative/ImplicitlyRestartedLanczos.h>
 #include <Grid/algorithms/CoarsenedMatrix.h>
 #include <Grid/algorithms/FFT.h>

lib/algorithms/LinearOperator.h

@@ -235,7 +235,7 @@ namespace Grid {
 	Field tmp(in._grid);
 
 	_Mat.MeooeDag(in,tmp);
-	_Mat.MooeeInvDag(tmp,out);
+        _Mat.MooeeInvDag(tmp,out);
 	_Mat.MeooeDag(out,tmp);
 
 	_Mat.MooeeDag(in,out);

lib/algorithms/approx/Chebyshev.h

@@ -197,8 +197,9 @@ namespace Grid {
     void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
 
       GridBase *grid=in._grid;
-//std::cout << "Chevyshef(): in._grid="<<in._grid<<std::endl;
-//<<" Linop.Grid()="<<Linop.Grid()<<"Linop.RedBlackGrid()="<<Linop.RedBlackGrid()<<std::endl;
+
+      // std::cout << "Chevyshef(): in._grid="<<in._grid<<std::endl;
+      //std::cout <<" Linop.Grid()="<<Linop.Grid()<<"Linop.RedBlackGrid()="<<Linop.RedBlackGrid()<<std::endl;
 
       int vol=grid->gSites();
 

lib/algorithms/approx/Remez.h

@@ -16,7 +16,7 @@
 #define INCLUDED_ALG_REMEZ_H
 
 #include <stddef.h>
-#include <Config.h>
+#include <Grid/GridStd.h>
 
 #ifdef HAVE_LIBGMP
 #include "bigfloat.h"

lib/algorithms/iterative/BlockConjugateGradient.h

@@ -0,0 +1,366 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/algorithms/iterative/BlockConjugateGradient.h
+
+Copyright (C) 2017
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_BLOCK_CONJUGATE_GRADIENT_H
+#define GRID_BLOCK_CONJUGATE_GRADIENT_H
+
+
+namespace Grid {
+
+//////////////////////////////////////////////////////////////////////////
+// Block conjugate gradient. Dimension zero should be the block direction
+//////////////////////////////////////////////////////////////////////////
+template <class Field>
+class BlockConjugateGradient : public OperatorFunction<Field> {
+ public:
+
+  typedef typename Field::scalar_type scomplex;
+
+  const int blockDim = 0;
+
+  int Nblock;
+  bool ErrorOnNoConverge;  // throw an assert when the CG fails to converge.
+                           // Defaults true.
+  RealD Tolerance;
+  Integer MaxIterations;
+  Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
+  
+  BlockConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
+    : Tolerance(tol),
+    MaxIterations(maxit),
+    ErrorOnNoConverge(err_on_no_conv){};
+
+void operator()(LinearOperatorBase<Field> &Linop, const Field &Src, Field &Psi) 
+{
+  int Orthog = 0; // First dimension is block dim
+  Nblock = Src._grid->_fdimensions[Orthog];
+
+  std::cout<<GridLogMessage<<" Block Conjugate Gradient : Orthog "<<Orthog<<" Nblock "<<Nblock<<std::endl;
+
+  Psi.checkerboard = Src.checkerboard;
+  conformable(Psi, Src);
+
+  Field P(Src);
+  Field AP(Src);
+  Field R(Src);
+  
+  Eigen::MatrixXcd m_pAp    = Eigen::MatrixXcd::Identity(Nblock,Nblock);
+  Eigen::MatrixXcd m_pAp_inv= Eigen::MatrixXcd::Identity(Nblock,Nblock);
+  Eigen::MatrixXcd m_rr     = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+  Eigen::MatrixXcd m_rr_inv = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+
+  Eigen::MatrixXcd m_alpha      = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+  Eigen::MatrixXcd m_beta   = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+
+  // Initial residual computation & set up
+  std::vector<RealD> residuals(Nblock);
+  std::vector<RealD> ssq(Nblock);
+
+  sliceNorm(ssq,Src,Orthog);
+  RealD sssum=0;
+  for(int b=0;b<Nblock;b++) sssum+=ssq[b];
+
+  sliceNorm(residuals,Src,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  sliceNorm(residuals,Psi,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  // Initial search dir is guess
+  Linop.HermOp(Psi, AP);
+  
+
+  /************************************************************************
+   * Block conjugate gradient (Stephen Pickles, thesis 1995, pp 71, O Leary 1980)
+   ************************************************************************
+   * O'Leary : R = B - A X
+   * O'Leary : P = M R ; preconditioner M = 1
+   * O'Leary : alpha = PAP^{-1} RMR
+   * O'Leary : beta  = RMR^{-1}_old RMR_new
+   * O'Leary : X=X+Palpha
+   * O'Leary : R_new=R_old-AP alpha
+   * O'Leary : P=MR_new+P beta
+   */
+
+  R = Src - AP;  
+  P = R;
+  sliceInnerProductMatrix(m_rr,R,R,Orthog);
+
+  GridStopWatch sliceInnerTimer;
+  GridStopWatch sliceMaddTimer;
+  GridStopWatch MatrixTimer;
+  GridStopWatch SolverTimer;
+  SolverTimer.Start();
+
+  int k;
+  for (k = 1; k <= MaxIterations; k++) {
+
+    RealD rrsum=0;
+    for(int b=0;b<Nblock;b++) rrsum+=real(m_rr(b,b));
+
+    std::cout << GridLogIterative << "\titeration "<<k<<" rr_sum "<<rrsum<<" ssq_sum "<< sssum
+	      <<" / "<<std::sqrt(rrsum/sssum) <<std::endl;
+
+    MatrixTimer.Start();
+    Linop.HermOp(P, AP);
+    MatrixTimer.Stop();
+
+    // Alpha
+    sliceInnerTimer.Start();
+    sliceInnerProductMatrix(m_pAp,P,AP,Orthog);
+    sliceInnerTimer.Stop();
+    m_pAp_inv = m_pAp.inverse();
+    m_alpha   = m_pAp_inv * m_rr ;
+
+    // Psi, R update
+    sliceMaddTimer.Start();
+    sliceMaddMatrix(Psi,m_alpha, P,Psi,Orthog);     // add alpha *  P to psi
+    sliceMaddMatrix(R  ,m_alpha,AP,  R,Orthog,-1.0);// sub alpha * AP to resid
+    sliceMaddTimer.Stop();
+
+    // Beta
+    m_rr_inv = m_rr.inverse();
+    sliceInnerTimer.Start();
+    sliceInnerProductMatrix(m_rr,R,R,Orthog);
+    sliceInnerTimer.Stop();
+    m_beta = m_rr_inv *m_rr;
+
+    // Search update
+    sliceMaddTimer.Start();
+    sliceMaddMatrix(AP,m_beta,P,R,Orthog);
+    sliceMaddTimer.Stop();
+    P= AP;
+
+    /*********************
+     * convergence monitor
+     *********************
+     */
+    RealD max_resid=0;
+    for(int b=0;b<Nblock;b++){
+      RealD rr = real(m_rr(b,b))/ssq[b];
+      if ( rr > max_resid ) max_resid = rr;
+    }
+    
+    if ( max_resid < Tolerance*Tolerance ) { 
+
+      SolverTimer.Stop();
+
+      std::cout << GridLogMessage<<"BlockCG converged in "<<k<<" iterations"<<std::endl;
+      for(int b=0;b<Nblock;b++){
+	std::cout << GridLogMessage<< "\t\tblock "<<b<<" resid "<< std::sqrt(real(m_rr(b,b))/ssq[b])<<std::endl;
+      }
+      std::cout << GridLogMessage<<"\tMax residual is "<<std::sqrt(max_resid)<<std::endl;
+
+      Linop.HermOp(Psi, AP);
+      AP = AP-Src;
+      std::cout << GridLogMessage <<"\tTrue residual is " << std::sqrt(norm2(AP)/norm2(Src)) <<std::endl;
+
+      std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
+      std::cout << GridLogMessage << "\tElapsed    " << SolverTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tMatrix     " << MatrixTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tInnerProd  " << sliceInnerTimer.Elapsed() <<std::endl;
+      std::cout << GridLogMessage << "\tMaddMatrix " << sliceMaddTimer.Elapsed()  <<std::endl;
+	    
+      IterationsToComplete = k;
+      return;
+    }
+
+  }
+  std::cout << GridLogMessage << "BlockConjugateGradient did NOT converge" << std::endl;
+
+  if (ErrorOnNoConverge) assert(0);
+  IterationsToComplete = k;
+}
+};
+
+
+//////////////////////////////////////////////////////////////////////////
+// multiRHS conjugate gradient. Dimension zero should be the block direction
+//////////////////////////////////////////////////////////////////////////
+template <class Field>
+class MultiRHSConjugateGradient : public OperatorFunction<Field> {
+ public:
+
+  typedef typename Field::scalar_type scomplex;
+
+  const int blockDim = 0;
+
+  int Nblock;
+  bool ErrorOnNoConverge;  // throw an assert when the CG fails to converge.
+                           // Defaults true.
+  RealD Tolerance;
+  Integer MaxIterations;
+  Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
+  
+   MultiRHSConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
+    : Tolerance(tol),
+    MaxIterations(maxit),
+    ErrorOnNoConverge(err_on_no_conv){};
+
+void operator()(LinearOperatorBase<Field> &Linop, const Field &Src, Field &Psi) 
+{
+  int Orthog = 0; // First dimension is block dim
+  Nblock = Src._grid->_fdimensions[Orthog];
+
+  std::cout<<GridLogMessage<<"MultiRHS Conjugate Gradient : Orthog "<<Orthog<<" Nblock "<<Nblock<<std::endl;
+
+  Psi.checkerboard = Src.checkerboard;
+  conformable(Psi, Src);
+
+  Field P(Src);
+  Field AP(Src);
+  Field R(Src);
+  
+  std::vector<ComplexD> v_pAp(Nblock);
+  std::vector<RealD> v_rr (Nblock);
+  std::vector<RealD> v_rr_inv(Nblock);
+  std::vector<RealD> v_alpha(Nblock);
+  std::vector<RealD> v_beta(Nblock);
+
+  // Initial residual computation & set up
+  std::vector<RealD> residuals(Nblock);
+  std::vector<RealD> ssq(Nblock);
+
+  sliceNorm(ssq,Src,Orthog);
+  RealD sssum=0;
+  for(int b=0;b<Nblock;b++) sssum+=ssq[b];
+
+  sliceNorm(residuals,Src,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  sliceNorm(residuals,Psi,Orthog);
+  for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
+
+  // Initial search dir is guess
+  Linop.HermOp(Psi, AP);
+
+  R = Src - AP;  
+  P = R;
+  sliceNorm(v_rr,R,Orthog);
+
+  GridStopWatch sliceInnerTimer;
+  GridStopWatch sliceMaddTimer;
+  GridStopWatch sliceNormTimer;
+  GridStopWatch MatrixTimer;
+  GridStopWatch SolverTimer;
+
+  SolverTimer.Start();
+  int k;
+  for (k = 1; k <= MaxIterations; k++) {
+
+    RealD rrsum=0;
+    for(int b=0;b<Nblock;b++) rrsum+=real(v_rr[b]);
+
+    std::cout << GridLogIterative << "\titeration "<<k<<" rr_sum "<<rrsum<<" ssq_sum "<< sssum
+	      <<" / "<<std::sqrt(rrsum/sssum) <<std::endl;
+
+    MatrixTimer.Start();
+    Linop.HermOp(P, AP);
+    MatrixTimer.Stop();
+
+    // Alpha
+    //    sliceInnerProductVectorTest(v_pAp_test,P,AP,Orthog);
+    sliceInnerTimer.Start();
+    sliceInnerProductVector(v_pAp,P,AP,Orthog);
+    sliceInnerTimer.Stop();
+    for(int b=0;b<Nblock;b++){
+      //      std::cout << " "<< v_pAp[b]<<" "<< v_pAp_test[b]<<std::endl;
+      v_alpha[b] = v_rr[b]/real(v_pAp[b]);
+    }
+
+    // Psi, R update
+    sliceMaddTimer.Start();
+    sliceMaddVector(Psi,v_alpha, P,Psi,Orthog);     // add alpha *  P to psi
+    sliceMaddVector(R  ,v_alpha,AP,  R,Orthog,-1.0);// sub alpha * AP to resid
+    sliceMaddTimer.Stop();
+
+    // Beta
+    for(int b=0;b<Nblock;b++){
+      v_rr_inv[b] = 1.0/v_rr[b];
+    }
+    sliceNormTimer.Start();
+    sliceNorm(v_rr,R,Orthog);
+    sliceNormTimer.Stop();
+    for(int b=0;b<Nblock;b++){
+      v_beta[b] = v_rr_inv[b] *v_rr[b];
+    }
+
+    // Search update
+    sliceMaddTimer.Start();
+    sliceMaddVector(P,v_beta,P,R,Orthog);
+    sliceMaddTimer.Stop();
+
+    /*********************
+     * convergence monitor
+     *********************
+     */
+    RealD max_resid=0;
+    for(int b=0;b<Nblock;b++){
+      RealD rr = v_rr[b]/ssq[b];
+      if ( rr > max_resid ) max_resid = rr;
+    }
+    
+    if ( max_resid < Tolerance*Tolerance ) { 
+
+      SolverTimer.Stop();
+
+      std::cout << GridLogMessage<<"MultiRHS solver converged in " <<k<<" iterations"<<std::endl;
+      for(int b=0;b<Nblock;b++){
+	std::cout << GridLogMessage<< "\t\tBlock "<<b<<" resid "<< std::sqrt(v_rr[b]/ssq[b])<<std::endl;
+      }
+      std::cout << GridLogMessage<<"\tMax residual is "<<std::sqrt(max_resid)<<std::endl;
+
+      Linop.HermOp(Psi, AP);
+      AP = AP-Src;
+      std::cout <<GridLogMessage << "\tTrue residual is " << std::sqrt(norm2(AP)/norm2(Src)) <<std::endl;
+
+      std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
+      std::cout << GridLogMessage << "\tElapsed    " << SolverTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tMatrix     " << MatrixTimer.Elapsed()     <<std::endl;
+      std::cout << GridLogMessage << "\tInnerProd  " << sliceInnerTimer.Elapsed() <<std::endl;
+      std::cout << GridLogMessage << "\tNorm       " << sliceNormTimer.Elapsed() <<std::endl;
+      std::cout << GridLogMessage << "\tMaddMatrix " << sliceMaddTimer.Elapsed()  <<std::endl;
+
+
+      IterationsToComplete = k;
+      return;
+    }
+
+  }
+  std::cout << GridLogMessage << "MultiRHSConjugateGradient did NOT converge" << std::endl;
+
+  if (ErrorOnNoConverge) assert(0);
+  IterationsToComplete = k;
+}
+};
+
+
+
+}
+#endif

lib/algorithms/iterative/ConjugateGradient.h

@@ -78,18 +78,12 @@ class ConjugateGradient : public OperatorFunction<Field> {
     cp = a;
     ssq = norm2(src);
 
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient: guess " << guess << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:   src " << ssq << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:    mp " << d << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:   mmp " << b << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:  cp,r " << cp << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:     p " << a << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient: guess " << guess << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:   src " << ssq << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:    mp " << d << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:   mmp " << b << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:  cp,r " << cp << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:     p " << a << std::endl;
 
     RealD rsq = Tolerance * Tolerance * ssq;
 
@@ -99,8 +93,7 @@ class ConjugateGradient : public OperatorFunction<Field> {
     }
 
     std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient: k=0 residual " << cp << " target " << rsq
-              << std::endl;
+              << "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
 
     GridStopWatch LinalgTimer;
     GridStopWatch MatrixTimer;
@@ -130,8 +123,11 @@ class ConjugateGradient : public OperatorFunction<Field> {
       p = p * b + r;
 
       LinalgTimer.Stop();
+
       std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
                 << " residual " << cp << " target " << rsq << std::endl;
+      std::cout << GridLogDebug << "a = "<< a << " b_pred = "<< b_pred << "  b = "<< b << std::endl;
+      std::cout << GridLogDebug << "qq = "<< qq << " d = "<< d << "  c = "<< c << std::endl;
 
       // Stopping condition
       if (cp <= rsq) {
@@ -139,32 +135,33 @@ class ConjugateGradient : public OperatorFunction<Field> {
         Linop.HermOpAndNorm(psi, mmp, d, qq);
         p = mmp - src;
 
-        RealD mmpnorm = sqrt(norm2(mmp));
-        RealD psinorm = sqrt(norm2(psi));
         RealD srcnorm = sqrt(norm2(src));
         RealD resnorm = sqrt(norm2(p));
         RealD true_residual = resnorm / srcnorm;
 
-        std::cout << GridLogMessage
-                  << "ConjugateGradient: Converged on iteration " << k << std::endl;
-        std::cout << GridLogMessage << "Computed residual " << sqrt(cp / ssq)
-                  << " true residual " << true_residual << " target "
-                  << Tolerance << std::endl;
-        std::cout << GridLogMessage << "Time elapsed: Iterations "
-                  << SolverTimer.Elapsed() << " Matrix  "
-                  << MatrixTimer.Elapsed() << " Linalg "
-                  << LinalgTimer.Elapsed();
-        std::cout << std::endl;
+        std::cout << GridLogMessage << "ConjugateGradient Converged on iteration " << k << std::endl;
+        std::cout << GridLogMessage << "\tComputed residual " << sqrt(cp / ssq)<<std::endl;
+	std::cout << GridLogMessage << "\tTrue residual " << true_residual<<std::endl;
+	std::cout << GridLogMessage << "\tTarget " << Tolerance << std::endl;
+
+        std::cout << GridLogMessage << "Time breakdown "<<std::endl;
+	std::cout << GridLogMessage << "\tElapsed    " << SolverTimer.Elapsed() <<std::endl;
+	std::cout << GridLogMessage << "\tMatrix     " << MatrixTimer.Elapsed() <<std::endl;
+	std::cout << GridLogMessage << "\tLinalg     " << LinalgTimer.Elapsed() <<std::endl;
 
         if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
+
 	IterationsToComplete = k;	
+
         return;
       }
     }
     std::cout << GridLogMessage << "ConjugateGradient did NOT converge"
               << std::endl;
+
     if (ErrorOnNoConverge) assert(0);
     IterationsToComplete = k;
+
   }
 };
 }

lib/algorithms/iterative/ImplicitlyRestartedLanczos.h

@@ -30,6 +30,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define GRID_IRL_H
 
 #include <string.h> //memset
+
 #ifdef USE_LAPACK
 void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
                    double *vl, double *vu, int *il, int *iu, double *abstol,
@@ -37,8 +38,9 @@ void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
                    double *work, int *lwork, int *iwork, int *liwork,
                    int *info);
 #endif
-#include "DenseMatrix.h"
-#include "EigenSort.h"
+
+#include <Grid/algorithms/densematrix/DenseMatrix.h>
+#include <Grid/algorithms/iterative/EigenSort.h>
 
 namespace Grid {
 
@@ -1088,8 +1090,6 @@ static void Lock(DenseMatrix<T> &H, 	// Hess mtx
 		 int dfg,
 		 bool herm)
 {	
-
-
   //ForceTridiagonal(H);
 
   int M = H.dim;
@@ -1121,7 +1121,6 @@ static void Lock(DenseMatrix<T> &H, 	// Hess mtx
 
   AH = Hermitian(QQ)*AH;
   AH = AH*QQ;
-	
 
   for(int i=con;i<M;i++){
     for(int j=con;j<M;j++){

lib/algorithms/iterative/Matrix.h

@@ -1,453 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/iterative/Matrix.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef MATRIX_H
-#define MATRIX_H
-
-#include <cstdlib>
-#include <string>
-#include <cmath>
-#include <vector>
-#include <iostream>
-#include <iomanip>
-#include <complex>
-#include <typeinfo>
-#include <Grid/Grid.h>
-
-
-/** Sign function **/
-template <class T> T sign(T p){return ( p/abs(p) );}
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////////
-///////////////////// Hijack STL containers for our wicked means /////////////////////////////////////////
-/////////////////////////////////////////////////////////////////////////////////////////////////////////
-template<class T> using Vector = Vector<T>;
-template<class T> using Matrix = Vector<Vector<T> >;
-
-template<class T> void Resize(Vector<T > & vec, int N) { vec.resize(N); }
-
-template<class T> void Resize(Matrix<T > & mat, int N, int M) { 
-  mat.resize(N);
-  for(int i=0;i<N;i++){
-    mat[i].resize(M);
-  }
-}
-template<class T> void Size(Vector<T> & vec, int &N) 
-{ 
-  N= vec.size();
-}
-template<class T> void Size(Matrix<T> & mat, int &N,int &M) 
-{ 
-  N= mat.size();
-  M= mat[0].size();
-}
-template<class T> void SizeSquare(Matrix<T> & mat, int &N) 
-{ 
-  int M; Size(mat,N,M);
-  assert(N==M);
-}
-template<class T> void SizeSame(Matrix<T> & mat1,Matrix<T> &mat2, int &N1,int &M1) 
-{ 
-  int N2,M2;
-  Size(mat1,N1,M1);
-  Size(mat2,N2,M2);
-  assert(N1==N2);
-  assert(M1==M2);
-}
-
-//*****************************************
-//*	(Complex) Vector operations	*
-//*****************************************
-
-/**Conj of a Vector **/
-template <class T> Vector<T> conj(Vector<T> p){
-	Vector<T> q(p.size());
-	for(int i=0;i<p.size();i++){q[i] = conj(p[i]);}
-	return q;
-}
-
-/** Norm of a Vector**/
-template <class T> T norm(Vector<T> p){
-	T sum = 0;
-	for(int i=0;i<p.size();i++){sum = sum + p[i]*conj(p[i]);}
-	return abs(sqrt(sum));
-}
-
-/** Norm squared of a Vector **/
-template <class T> T norm2(Vector<T> p){
-	T sum = 0;
-	for(int i=0;i<p.size();i++){sum = sum + p[i]*conj(p[i]);}
-	return abs((sum));
-}
-
-/** Sum elements of a Vector **/
-template <class T> T trace(Vector<T> p){
-	T sum = 0;
-	for(int i=0;i<p.size();i++){sum = sum + p[i];}
-	return sum;
-}
-
-/** Fill a Vector with constant c **/
-template <class T> void Fill(Vector<T> &p, T c){
-	for(int i=0;i<p.size();i++){p[i] = c;}
-}
-/** Normalize a Vector **/
-template <class T> void normalize(Vector<T> &p){
-	T m = norm(p);
-	if( abs(m) > 0.0) for(int i=0;i<p.size();i++){p[i] /= m;}
-}
-/** Vector by scalar **/
-template <class T, class U> Vector<T> times(Vector<T> p, U s){
-	for(int i=0;i<p.size();i++){p[i] *= s;}
-	return p;
-}
-template <class T, class U> Vector<T> times(U s, Vector<T> p){
-	for(int i=0;i<p.size();i++){p[i] *= s;}
-	return p;
-}
-/** inner product of a and b = conj(a) . b **/
-template <class T> T inner(Vector<T> a, Vector<T> b){
-	T m = 0.;
-	for(int i=0;i<a.size();i++){m = m + conj(a[i])*b[i];}
-	return m;
-}
-/** sum of a and b = a + b **/
-template <class T> Vector<T> add(Vector<T> a, Vector<T> b){
-	Vector<T> m(a.size());
-	for(int i=0;i<a.size();i++){m[i] = a[i] + b[i];}
-	return m;
-}
-/** sum of a and b = a - b **/
-template <class T> Vector<T> sub(Vector<T> a, Vector<T> b){
-	Vector<T> m(a.size());
-	for(int i=0;i<a.size();i++){m[i] = a[i] - b[i];}
-	return m;
-}
-
-/** 
- *********************************
- *	Matrices	         *
- *********************************
- **/
-
-template<class T> void Fill(Matrix<T> & mat, T&val) { 
-  int N,M;
-  Size(mat,N,M);
-  for(int i=0;i<N;i++){
-  for(int j=0;j<M;j++){
-    mat[i][j] = val;
-  }}
-}
-
-/** Transpose of a matrix **/
-Matrix<T> Transpose(Matrix<T> & mat){
-  int N,M;
-  Size(mat,N,M);
-  Matrix C; Resize(C,M,N);
-  for(int i=0;i<M;i++){
-  for(int j=0;j<N;j++){
-    C[i][j] = mat[j][i];
-  }} 
-  return C;
-}
-/** Set Matrix to unit matrix **/
-template<class T> void Unity(Matrix<T> &mat){
-  int N;  SizeSquare(mat,N);
-  for(int i=0;i<N;i++){
-    for(int j=0;j<N;j++){
-      if ( i==j ) A[i][j] = 1;
-      else        A[i][j] = 0;
-    } 
-  } 
-}
-/** Add C * I to matrix **/
-template<class T>
-void PlusUnit(Matrix<T> & A,T c){
-  int dim;  SizeSquare(A,dim);
-  for(int i=0;i<dim;i++){A[i][i] = A[i][i] + c;} 
-}
-
-/** return the Hermitian conjugate of matrix **/
-Matrix<T> HermitianConj(Matrix<T> &mat){
-
-  int dim; SizeSquare(mat,dim);
-
-  Matrix<T> C; Resize(C,dim,dim);
-
-  for(int i=0;i<dim;i++){
-    for(int j=0;j<dim;j++){
-      C[i][j] = conj(mat[j][i]);
-    } 
-  } 
-  return C;
-}
-
-/** return diagonal entries as a Vector **/
-Vector<T> diag(Matrix<T> &A)
-{
-  int dim; SizeSquare(A,dim);
-  Vector<T> d; Resize(d,dim);
-
-  for(int i=0;i<dim;i++){
-    d[i] = A[i][i];
-  }
-  return d;
-}
-
-/** Left multiply by a Vector **/
-Vector<T> operator *(Vector<T> &B,Matrix<T> &A)
-{
-  int K,M,N; 
-  Size(B,K);
-  Size(A,M,N);
-  assert(K==M);
-  
-  Vector<T> C; Resize(C,N);
-
-  for(int j=0;j<N;j++){
-    T sum = 0.0;
-    for(int i=0;i<M;i++){
-      sum += B[i] * A[i][j];
-    }
-    C[j] =  sum;
-  }
-  return C; 
-}
-
-/** return 1/diagonal entries as a Vector **/
-Vector<T> inv_diag(Matrix<T> & A){
-  int dim; SizeSquare(A,dim);
-  Vector<T> d; Resize(d,dim);
-  for(int i=0;i<dim;i++){
-    d[i] = 1.0/A[i][i];
-  }
-  return d;
-}
-/** Matrix Addition **/
-inline Matrix<T> operator + (Matrix<T> &A,Matrix<T> &B)
-{
-  int N,M  ; SizeSame(A,B,N,M);
-  Matrix C; Resize(C,N,M);
-  for(int i=0;i<N;i++){
-    for(int j=0;j<M;j++){
-      C[i][j] = A[i][j] +  B[i][j];
-    } 
-  } 
-  return C;
-} 
-/** Matrix Subtraction **/
-inline Matrix<T> operator- (Matrix<T> & A,Matrix<T> &B){
-  int N,M  ; SizeSame(A,B,N,M);
-  Matrix C; Resize(C,N,M);
-  for(int i=0;i<N;i++){
-  for(int j=0;j<M;j++){
-    C[i][j] = A[i][j] -  B[i][j];
-  }}
-  return C;
-} 
-
-/** Matrix scalar multiplication **/
-inline Matrix<T> operator* (Matrix<T> & A,T c){
-  int N,M; Size(A,N,M);
-  Matrix C; Resize(C,N,M);
-  for(int i=0;i<N;i++){
-  for(int j=0;j<M;j++){
-    C[i][j] = A[i][j]*c;
-  }} 
-  return C;
-} 
-/** Matrix Matrix multiplication **/
-inline Matrix<T> operator* (Matrix<T> &A,Matrix<T> &B){
-  int K,L,N,M;
-  Size(A,K,L);
-  Size(B,N,M); assert(L==N);
-  Matrix C; Resize(C,K,M);
-
-  for(int i=0;i<K;i++){
-    for(int j=0;j<M;j++){
-      T sum = 0.0;
-      for(int k=0;k<N;k++) sum += A[i][k]*B[k][j];
-      C[i][j] =sum;
-    }
-  }
-  return C; 
-} 
-/** Matrix Vector multiplication **/
-inline Vector<T> operator* (Matrix<T> &A,Vector<T> &B){
-  int M,N,K;
-  Size(A,N,M);
-  Size(B,K); assert(K==M);
-  Vector<T> C; Resize(C,N);
-  for(int i=0;i<N;i++){
-    T sum = 0.0;
-    for(int j=0;j<M;j++) sum += A[i][j]*B[j];
-    C[i] =  sum;
-  }
-  return C; 
-} 
-
-/** Some version of Matrix norm **/
-/*
-inline T Norm(){ // this is not a usual L2 norm
-    T norm = 0;
-    for(int i=0;i<dim;i++){
-      for(int j=0;j<dim;j++){
-	norm += abs(A[i][j]);
-    }}
-    return norm;
-  }
-*/
-
-/** Some version of Matrix norm **/
-template<class T> T LargestDiag(Matrix<T> &A)
-{
-  int dim ; SizeSquare(A,dim); 
-
-  T ld = abs(A[0][0]);
-  for(int i=1;i<dim;i++){
-    T cf = abs(A[i][i]);
-    if(abs(cf) > abs(ld) ){ld = cf;}
-  }
-  return ld;
-}
-
-/** Look for entries on the leading subdiagonal that are smaller than 'small' **/
-template <class T,class U> int Chop_subdiag(Matrix<T> &A,T norm, int offset, U small)
-{
-  int dim; SizeSquare(A,dim);
-  for(int l = dim - 1 - offset; l >= 1; l--) {             		
-    if((U)abs(A[l][l - 1]) < (U)small) {
-      A[l][l-1]=(U)0.0;
-      return l;
-    }
-  }
-  return 0;
-}
-
-/** Look for entries on the leading subdiagonal that are smaller than 'small' **/
-template <class T,class U> int Chop_symm_subdiag(Matrix<T> & A,T norm, int offset, U small) 
-{
-  int dim; SizeSquare(A,dim);
-  for(int l = dim - 1 - offset; l >= 1; l--) {
-    if((U)abs(A[l][l - 1]) < (U)small) {
-      A[l][l - 1] = (U)0.0;
-      A[l - 1][l] = (U)0.0;
-      return l;
-    }
-  }
-  return 0;
-}
-/**Assign a submatrix to a larger one**/
-template<class T>
-void AssignSubMtx(Matrix<T> & A,int row_st, int row_end, int col_st, int col_end, Matrix<T> &S)
-{
-  for(int i = row_st; i<row_end; i++){
-    for(int j = col_st; j<col_end; j++){
-      A[i][j] = S[i - row_st][j - col_st];
-    }
-  }
-}
-
-/**Get a square submatrix**/
-template <class T>
-Matrix<T> GetSubMtx(Matrix<T> &A,int row_st, int row_end, int col_st, int col_end)
-{
-  Matrix<T> H; Resize(row_end - row_st,col_end-col_st);
-
-  for(int i = row_st; i<row_end; i++){
-  for(int j = col_st; j<col_end; j++){
-    H[i-row_st][j-col_st]=A[i][j];
-  }}
-  return H;
-}
-  
- /**Assign a submatrix to a larger one NB remember Vector Vectors are transposes of the matricies they represent**/
-template<class T>
-void AssignSubMtx(Matrix<T> & A,int row_st, int row_end, int col_st, int col_end, Matrix<T> &S)
-{
-  for(int i = row_st; i<row_end; i++){
-  for(int j = col_st; j<col_end; j++){
-    A[i][j] = S[i - row_st][j - col_st];
-  }}
-}
-  
-/** compute b_i A_ij b_j **/ // surprised no Conj
-template<class T> T proj(Matrix<T> A, Vector<T> B){
-  int dim; SizeSquare(A,dim);
-  int dimB; Size(B,dimB);
-  assert(dimB==dim);
-  T C = 0;
-  for(int i=0;i<dim;i++){
-    T sum = 0.0;
-    for(int j=0;j<dim;j++){
-      sum += A[i][j]*B[j];
-    }
-    C +=  B[i]*sum; // No conj?
-  }
-  return C; 
-}
-
-
-/*
- *************************************************************
- *
- * Matrix Vector products
- *
- *************************************************************
- */
-// Instead make a linop and call my CG;
-
-/// q -> q Q
-template <class T,class Fermion> void times(Vector<Fermion> &q, Matrix<T> &Q)
-{
-  int M; SizeSquare(Q,M);
-  int N; Size(q,N); 
-  assert(M==N);
-
-  times(q,Q,N);
-}
-
-/// q -> q Q
-template <class T> void times(multi1d<LatticeFermion> &q, Matrix<T> &Q, int N)
-{
-  GridBase *grid = q[0]._grid;
-  int M; SizeSquare(Q,M);
-  int K; Size(q,K); 
-  assert(N<M);
-  assert(N<K);
-  Vector<Fermion> S(N,grid );
-  for(int j=0;j<N;j++){
-    S[j] = zero;
-    for(int k=0;k<N;k++){
-      S[j] = S[j] +  q[k]* Q[k][j]; 
-    }
-  }
-  for(int j=0;j<q.size();j++){
-    q[j] = S[j];
-  }
-}
-#endif

lib/algorithms/iterative/MatrixUtils.h

@@ -1,75 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/iterative/MatrixUtils.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef GRID_MATRIX_UTILS_H
-#define GRID_MATRIX_UTILS_H
-
-namespace Grid {
-
-  namespace MatrixUtils { 
-
-    template<class T> inline void Size(Matrix<T>& A,int &N,int &M){
-      N=A.size(); assert(N>0);
-      M=A[0].size();
-      for(int i=0;i<N;i++){
-	assert(A[i].size()==M);
-      }
-    }
-
-    template<class T> inline void SizeSquare(Matrix<T>& A,int &N)
-    {
-      int M;
-      Size(A,N,M);
-      assert(N==M);
-    }
-
-    template<class T> inline void Fill(Matrix<T>& A,T & val)
-    { 
-      int N,M;
-      Size(A,N,M);
-      for(int i=0;i<N;i++){
-      for(int j=0;j<M;j++){
-	A[i][j]=val;
-      }}
-    }
-    template<class T> inline void Diagonal(Matrix<T>& A,T & val)
-    { 
-      int N;
-      SizeSquare(A,N);
-      for(int i=0;i<N;i++){
-	A[i][i]=val;
-      }
-    }
-    template<class T> inline void Identity(Matrix<T>& A)
-    {
-      Fill(A,0.0);
-      Diagonal(A,1.0);
-    }
-
-  };
-}
-#endif

lib/algorithms/iterative/TODO

@@ -1,15 +0,0 @@
-- ConjugateGradientMultiShift
-- MCR
-
-- Potentially Useful Boost libraries
-
-- MultiArray
-- Aligned allocator; memory pool
-- Remez -- Mike or Boost?
-- Multiprecision
-- quaternians
-- Tokenize
-- Serialization
-- Regex
-- Proto (ET)
-- uBlas

lib/algorithms/iterative/bisec.c

@@ -1,122 +0,0 @@
-#include <math.h>
-#include <stdlib.h>
-#include <vector>
-
-struct Bisection {
-
-static void get_eig2(int row_num,std::vector<RealD> &ALPHA,std::vector<RealD> &BETA, std::vector<RealD> & eig)
-{
-  int i,j;
-  std::vector<RealD> evec1(row_num+3);
-  std::vector<RealD> evec2(row_num+3);
-  RealD eps2;
-  ALPHA[1]=0.;
-  BETHA[1]=0.;
-  for(i=0;i<row_num-1;i++) {
-    ALPHA[i+1] = A[i*(row_num+1)].real();
-    BETHA[i+2] = A[i*(row_num+1)+1].real();
-  }
-  ALPHA[row_num] = A[(row_num-1)*(row_num+1)].real();
-  bisec(ALPHA,BETHA,row_num,1,row_num,1e-10,1e-10,evec1,eps2);
-  bisec(ALPHA,BETHA,row_num,1,row_num,1e-16,1e-16,evec2,eps2);
-
-  // Do we really need to sort here?
-  int begin=1;
-  int end = row_num;
-  int swapped=1;
-  while(swapped) {
-    swapped=0;
-    for(i=begin;i<end;i++){
-      if(mag(evec2[i])>mag(evec2[i+1]))	{
-	swap(evec2+i,evec2+i+1);
-	swapped=1;
-      }
-    }
-    end--;
-    for(i=end-1;i>=begin;i--){
-      if(mag(evec2[i])>mag(evec2[i+1]))	{
-	swap(evec2+i,evec2+i+1);
-	swapped=1;
-      }
-    }
-    begin++;
-  }
-
-  for(i=0;i<row_num;i++){
-    for(j=0;j<row_num;j++) {
-      if(i==j) H[i*row_num+j]=evec2[i+1];
-      else H[i*row_num+j]=0.;
-    }
-  }
-}
-
-static void bisec(std::vector<RealD> &c,   
-		  std::vector<RealD> &b,
-		  int n,
-		  int m1,
-		  int m2,
-		  RealD eps1,
-		  RealD relfeh,
-		  std::vector<RealD> &x,
-		  RealD &eps2)
-{
-  std::vector<RealD> wu(n+2);
-
-  RealD h,q,x1,xu,x0,xmin,xmax; 
-  int i,a,k;
-
-  b[1]=0.0;
-  xmin=c[n]-fabs(b[n]);
-  xmax=c[n]+fabs(b[n]);
-  for(i=1;i<n;i++){
-    h=fabs(b[i])+fabs(b[i+1]);
-    if(c[i]+h>xmax) xmax= c[i]+h;
-    if(c[i]-h<xmin) xmin= c[i]-h;
-  }
-  xmax *=2.;
-
-  eps2=relfeh*((xmin+xmax)>0.0 ? xmax : -xmin);
-  if(eps1<=0.0) eps1=eps2;
-  eps2=0.5*eps1+7.0*(eps2);
-  x0=xmax;
-  for(i=m1;i<=m2;i++){
-    x[i]=xmax;
-    wu[i]=xmin;
-  }
-
-  for(k=m2;k>=m1;k--){
-    xu=xmin;
-    i=k;
-    do{
-      if(xu<wu[i]){
-	xu=wu[i];
-	i=m1-1;
-      }
-      i--;
-    }while(i>=m1);
-    if(x0>x[k]) x0=x[k];
-    while((x0-xu)>2*relfeh*(fabs(xu)+fabs(x0))+eps1){
-      x1=(xu+x0)/2;
-
-      a=0;
-      q=1.0;
-      for(i=1;i<=n;i++){
-	q=c[i]-x1-((q!=0.0)? b[i]*b[i]/q:fabs(b[i])/relfeh);
-	if(q<0) a++;
-      }
-      //			printf("x1=%e a=%d\n",x1,a);
-      if(a<k){
-	if(a<m1){
-	  xu=x1;
-	  wu[m1]=x1;
-	}else {
-	  xu=x1;
-	  wu[a+1]=x1;
-	  if(x[a]>x1) x[a]=x1;
-	}
-      }else x0=x1;
-    }
-    x[k]=(x0+xu)/2;
-  }
-}
-}

lib/algorithms/iterative/get_eig.c

@@ -1 +0,0 @@
-

lib/cartesian/Cartesian_base.h

@@ -6,8 +6,9 @@
 
     Copyright (C) 2015
 
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+    Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -49,7 +50,6 @@ public:
 
     GridBase(const std::vector<int> & processor_grid) : CartesianCommunicator(processor_grid) {};
 
-
     // Physics Grid information.
     std::vector<int> _simd_layout;// Which dimensions get relayed out over simd lanes.
     std::vector<int> _fdimensions;// (full) Global dimensions of array prior to cb removal
@@ -62,13 +62,12 @@ public:
     int _isites;
     int _fsites;                  // _isites*_osites = product(dimensions).
     int _gsites;
-    std::vector<int> _slice_block;   // subslice information
+    std::vector<int> _slice_block;// subslice information
     std::vector<int> _slice_stride;
     std::vector<int> _slice_nblock;
 
-    // Might need these at some point
-    //    std::vector<int> _lstart;     // local start of array in gcoors. _processor_coor[d]*_ldimensions[d]
-    //    std::vector<int> _lend;       // local end of array in gcoors    _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1
+    std::vector<int> _lstart;     // local start of array in gcoors _processor_coor[d]*_ldimensions[d]
+    std::vector<int> _lend  ;     // local end of array in gcoors   _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1
 
 public:
 
@@ -99,7 +98,7 @@ public:
     virtual int oIndex(std::vector<int> &coor)
     {
         int idx=0;
-	// Works with either global or local coordinates
+        // Works with either global or local coordinates
         for(int d=0;d<_ndimension;d++) idx+=_ostride[d]*(coor[d]%_rdimensions[d]);
         return idx;
     }
@@ -121,6 +120,12 @@ public:
       Lexicographic::CoorFromIndex(coor,Oindex,_rdimensions);
     }
 
+    inline void InOutCoorToLocalCoor (std::vector<int> &ocoor, std::vector<int> &icoor, std::vector<int> &lcoor) {
+      lcoor.resize(_ndimension);
+      for (int d = 0; d < _ndimension; d++)
+        lcoor[d] = ocoor[d] + _rdimensions[d] * icoor[d];
+    }
+
     //////////////////////////////////////////////////////////
     // SIMD lane addressing
     //////////////////////////////////////////////////////////
@@ -128,6 +133,7 @@ public:
     {
       Lexicographic::CoorFromIndex(coor,lane,_simd_layout);
     }
+
     inline int PermuteDim(int dimension){
       return _simd_layout[dimension]>1;
     }
@@ -145,15 +151,15 @@ public:
       // Distance should be either 0,1,2..
       //
       if ( _simd_layout[dimension] > 2 ) { 
-	for(int d=0;d<_ndimension;d++){
-	  if ( d != dimension ) assert ( (_simd_layout[d]==1)  );
-	}
-	permute_type = RotateBit; // How to specify distance; this is not just direction.
-	return permute_type;
+        for(int d=0;d<_ndimension;d++){
+          if ( d != dimension ) assert ( (_simd_layout[d]==1)  );
+        }
+        permute_type = RotateBit; // How to specify distance; this is not just direction.
+        return permute_type;
       }
 
       for(int d=_ndimension-1;d>dimension;d--){
-	if (_simd_layout[d]>1 ) permute_type++;
+        if (_simd_layout[d]>1 ) permute_type++;
       }
       return permute_type;
     }
@@ -168,11 +174,30 @@ public:
     inline int gSites(void) const { return _isites*_osites*_Nprocessors; }; 
     inline int Nd    (void) const { return _ndimension;};
 
+    inline const std::vector<int> LocalStarts(void)             { return _lstart;    };
     inline const std::vector<int> &FullDimensions(void)         { return _fdimensions;};
     inline const std::vector<int> &GlobalDimensions(void)       { return _gdimensions;};
     inline const std::vector<int> &LocalDimensions(void)        { return _ldimensions;};
     inline const std::vector<int> &VirtualLocalDimensions(void) { return _ldimensions;};
 
+    ////////////////////////////////////////////////////////////////
+    // Utility to print the full decomposition details 
+    ////////////////////////////////////////////////////////////////
+
+    void show_decomposition(){
+      std::cout << GridLogMessage << "Full Dimensions    : " << _fdimensions << std::endl;
+      std::cout << GridLogMessage << "Global Dimensions  : " << _gdimensions << std::endl;
+      std::cout << GridLogMessage << "Local Dimensions   : " << _ldimensions << std::endl;
+      std::cout << GridLogMessage << "Reduced Dimensions : " << _rdimensions << std::endl;
+      std::cout << GridLogMessage << "Outer strides      : " << _ostride << std::endl;
+      std::cout << GridLogMessage << "Inner strides      : " << _istride << std::endl;
+      std::cout << GridLogMessage << "iSites             : " << _isites << std::endl;
+      std::cout << GridLogMessage << "oSites             : " << _osites << std::endl;
+      std::cout << GridLogMessage << "lSites             : " << lSites() << std::endl;        
+      std::cout << GridLogMessage << "gSites             : " << gSites() << std::endl;
+      std::cout << GridLogMessage << "Nd                 : " << _ndimension << std::endl;             
+    } 
+
     ////////////////////////////////////////////////////////////////
     // Global addressing
     ////////////////////////////////////////////////////////////////
@@ -184,12 +209,15 @@ public:
       assert(lidx<lSites());
       Lexicographic::CoorFromIndex(lcoor,lidx,_ldimensions);
     }
+
+
+
     void GlobalCoorToGlobalIndex(const std::vector<int> & gcoor,int & gidx){
       gidx=0;
       int mult=1;
       for(int mu=0;mu<_ndimension;mu++) {
-	gidx+=mult*gcoor[mu];
-	mult*=_gdimensions[mu];
+        gidx+=mult*gcoor[mu];
+        mult*=_gdimensions[mu];
       }
     }
     void GlobalCoorToProcessorCoorLocalCoor(std::vector<int> &pcoor,std::vector<int> &lcoor,const std::vector<int> &gcoor)
@@ -197,9 +225,9 @@ public:
       pcoor.resize(_ndimension);
       lcoor.resize(_ndimension);
       for(int mu=0;mu<_ndimension;mu++){
-	int _fld  = _fdimensions[mu]/_processors[mu];
-	pcoor[mu] = gcoor[mu]/_fld;
-	lcoor[mu] = gcoor[mu]%_fld;
+        int _fld  = _fdimensions[mu]/_processors[mu];
+        pcoor[mu] = gcoor[mu]/_fld;
+        lcoor[mu] = gcoor[mu]%_fld;
       }
     }
     void GlobalCoorToRankIndex(int &rank, int &o_idx, int &i_idx ,const std::vector<int> &gcoor)
@@ -211,9 +239,9 @@ public:
       /*
       std::vector<int> cblcoor(lcoor);
       for(int d=0;d<cblcoor.size();d++){
-	if( this->CheckerBoarded(d) ) {
-	  cblcoor[d] = lcoor[d]/2;
-	}
+        if( this->CheckerBoarded(d) ) {
+          cblcoor[d] = lcoor[d]/2;
+        }
       }
       */
       i_idx= iIndex(lcoor);
@@ -239,7 +267,7 @@ public:
     {
       RankIndexToGlobalCoor(rank,o_idx,i_idx ,fcoor);
       if(CheckerBoarded(0)){
-	fcoor[0] = fcoor[0]*2+cb;
+        fcoor[0] = fcoor[0]*2+cb;
       }
     }
     void ProcessorCoorLocalCoorToGlobalCoor(std::vector<int> &Pcoor,std::vector<int> &Lcoor,std::vector<int> &gcoor)

lib/cartesian/Cartesian_full.h

@@ -76,6 +76,8 @@ public:
         _ldimensions.resize(_ndimension);
         _rdimensions.resize(_ndimension);
         _simd_layout.resize(_ndimension);
+	_lstart.resize(_ndimension);
+	_lend.resize(_ndimension);
             
         _ostride.resize(_ndimension);
         _istride.resize(_ndimension);
@@ -94,8 +96,10 @@ public:
 	  // Use a reduced simd grid
 	  _ldimensions[d]= _gdimensions[d]/_processors[d];  //local dimensions
 	  _rdimensions[d]= _ldimensions[d]/_simd_layout[d]; //overdecomposition
-	  _osites *= _rdimensions[d];
-	  _isites *= _simd_layout[d];
+	  _lstart[d]     = _processor_coor[d]*_ldimensions[d];
+	  _lend[d]       = _processor_coor[d]*_ldimensions[d]+_ldimensions[d]-1;
+	  _osites  *= _rdimensions[d];
+	  _isites  *= _simd_layout[d];
                 
 	  // Addressing support
 	  if ( d==0 ) {

lib/cartesian/Cartesian_red_black.h

@@ -151,6 +151,8 @@ public:
       _ldimensions.resize(_ndimension);
       _rdimensions.resize(_ndimension);
       _simd_layout.resize(_ndimension);
+      _lstart.resize(_ndimension);
+      _lend.resize(_ndimension);
       
       _ostride.resize(_ndimension);
       _istride.resize(_ndimension);
@@ -169,6 +171,8 @@ public:
 	  _gdimensions[d] = _gdimensions[d]/2; // Remove a checkerboard
 	}
 	_ldimensions[d] = _gdimensions[d]/_processors[d];
+	_lstart[d]     = _processor_coor[d]*_ldimensions[d];
+	_lend[d]       = _processor_coor[d]*_ldimensions[d]+_ldimensions[d]-1;
 
 	// Use a reduced simd grid
 	_simd_layout[d] = simd_layout[d];

lib/communicator/Communicator_base.cc

@@ -60,6 +60,7 @@ void CartesianCommunicator::ShmBufferFreeAll(void) {
 /////////////////////////////////
 // Grid information queries
 /////////////////////////////////
+int                      CartesianCommunicator::Dimensions(void)         { return _ndimension; };
 int                      CartesianCommunicator::IsBoss(void)            { return _processor==0; };
 int                      CartesianCommunicator::BossRank(void)          { return 0; };
 int                      CartesianCommunicator::ThisRank(void)          { return _processor; };
@@ -91,6 +92,7 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
 #if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPI3L)
 
 int                      CartesianCommunicator::NodeCount(void)    { return ProcessorCount();};
+int                      CartesianCommunicator::RankCount(void)    { return ProcessorCount();};
 
 double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 						       void *xmit,

lib/communicator/Communicator_base.h

@@ -148,6 +148,7 @@ class CartesianCommunicator {
   int  RankFromProcessorCoor(std::vector<int> &coor);
   void ProcessorCoorFromRank(int rank,std::vector<int> &coor);
   
+  int                      Dimensions(void)        ;
   int                      IsBoss(void)            ;
   int                      BossRank(void)          ;
   int                      ThisRank(void)          ;
@@ -155,6 +156,7 @@ class CartesianCommunicator {
   const std::vector<int> & ProcessorGrid(void)     ;
   int                      ProcessorCount(void)    ;
   int                      NodeCount(void)    ;
+  int                      RankCount(void)    ;
 
   ////////////////////////////////////////////////////////////////////////////////
   // very VERY rarely (Log, serial RNG) we need world without a grid
@@ -175,6 +177,8 @@ class CartesianCommunicator {
   void GlobalSumVector(ComplexF *c,int N);
   void GlobalSum(ComplexD &c);
   void GlobalSumVector(ComplexD *c,int N);
+  void GlobalXOR(uint32_t &);
+  void GlobalXOR(uint64_t &);
   
   template<class obj> void GlobalSum(obj &o){
     typedef typename obj::scalar_type scalar_type;

lib/communicator/Communicator_mpi.cc

@@ -83,6 +83,14 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
   assert(ierr==0);
 }
+void CartesianCommunicator::GlobalXOR(uint32_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalXOR(uint64_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
+  assert(ierr==0);
+}
 void CartesianCommunicator::GlobalSum(float &f){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
   assert(ierr==0);

lib/communicator/Communicator_mpi3.cc

@@ -65,6 +65,7 @@ std::vector<int> CartesianCommunicator::MyGroup;
 std::vector<void *> CartesianCommunicator::ShmCommBufs;
 
 int CartesianCommunicator::NodeCount(void)    { return GroupSize;};
+int CartesianCommunicator::RankCount(void)    { return WorldSize;};
 
 
 #undef FORCE_COMMS
@@ -206,7 +207,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",GroupRank,r);
 
       shm_unlink(shm_name);
-      int fd=shm_open(shm_name,O_RDWR|O_CREAT,0660);
+      int fd=shm_open(shm_name,O_RDWR|O_CREAT,0666);
       if ( fd < 0 ) {	perror("failed shm_open");	assert(0);      }
       ftruncate(fd, size);
 
@@ -226,7 +227,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     
       sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",GroupRank,r);
 
-      int fd=shm_open(shm_name,O_RDWR,0660);
+      int fd=shm_open(shm_name,O_RDWR,0666);
       if ( fd<0 ) {	perror("failed shm_open");	assert(0);      }
 
       void * ptr =  mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
@@ -509,6 +510,14 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
   assert(ierr==0);
 }
+void CartesianCommunicator::GlobalXOR(uint32_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalXOR(uint64_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
+  assert(ierr==0);
+}
 void CartesianCommunicator::GlobalSum(float &f){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
   assert(ierr==0);

lib/communicator/Communicator_none.cc

@@ -59,6 +59,8 @@ void CartesianCommunicator::GlobalSum(double &){}
 void CartesianCommunicator::GlobalSum(uint32_t &){}
 void CartesianCommunicator::GlobalSum(uint64_t &){}
 void CartesianCommunicator::GlobalSumVector(double *,int N){}
+void CartesianCommunicator::GlobalXOR(uint32_t &){}
+void CartesianCommunicator::GlobalXOR(uint64_t &){}
 
 void CartesianCommunicator::SendRecvPacket(void *xmit,
 					   void *recv,

lib/cshift/Cshift_common.h

@@ -30,21 +30,11 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
-template<class vobj>
-class SimpleCompressor {
-public:
-  void Point(int) {};
-
-  vobj operator() (const vobj &arg) {
-    return arg;
-  }
-};
-
 ///////////////////////////////////////////////////////////////////
-// Gather for when there is no need to SIMD split with compression
+// Gather for when there is no need to SIMD split 
 ///////////////////////////////////////////////////////////////////
-template<class vobj,class cobj,class compressor> void 
-Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0)
+template<class vobj> void 
+Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 
@@ -62,7 +52,7 @@ Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimen
       for(int b=0;b<e2;b++){
 	int o  = n*stride;
 	int bo = n*e2;
-	buffer[off+bo+b]=compress(rhs._odata[so+o+b]);
+	buffer[off+bo+b]=rhs._odata[so+o+b];
       }
     }
   } else { 
@@ -78,17 +68,16 @@ Gather_plane_simple (const Lattice<vobj> &rhs,commVector<cobj> &buffer,int dimen
        }
      }
      parallel_for(int i=0;i<table.size();i++){
-       buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
+       buffer[off+table[i].first]=rhs._odata[so+table[i].second];
      }
   }
 }
 
-
 ///////////////////////////////////////////////////////////////////
-// Gather for when there *is* need to SIMD split with compression
+// Gather for when there *is* need to SIMD split 
 ///////////////////////////////////////////////////////////////////
-template<class cobj,class vobj,class compressor> void 
-Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename cobj::scalar_object *> pointers,int dimension,int plane,int cbmask,compressor &compress)
+template<class vobj> void 
+Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename vobj::scalar_object *> pointers,int dimension,int plane,int cbmask)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 
@@ -109,8 +98,8 @@ Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename cobj::scalar_
 	int o      =   n*n1;
 	int offset = b+n*e2;
 	
-	cobj temp =compress(rhs._odata[so+o+b]);
-	extract<cobj>(temp,pointers,offset);
+	vobj temp =rhs._odata[so+o+b];
+	extract<vobj>(temp,pointers,offset);
 
       }
     }
@@ -127,32 +116,14 @@ Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename cobj::scalar_
 	int offset = b+n*e2;
 
 	if ( ocb & cbmask ) {
-	  cobj temp =compress(rhs._odata[so+o+b]);
-	  extract<cobj>(temp,pointers,offset);
+	  vobj temp =rhs._odata[so+o+b];
+	  extract<vobj>(temp,pointers,offset);
 	}
       }
     }
   }
 }
 
-//////////////////////////////////////////////////////
-// Gather for when there is no need to SIMD split
-//////////////////////////////////////////////////////
-template<class vobj> void Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
-{
-  SimpleCompressor<vobj> dontcompress;
-  Gather_plane_simple (rhs,buffer,dimension,plane,cbmask,dontcompress);
-}
-
-//////////////////////////////////////////////////////
-// Gather for when there *is* need to SIMD split
-//////////////////////////////////////////////////////
-template<class vobj> void Gather_plane_extract(const Lattice<vobj> &rhs,std::vector<typename vobj::scalar_object *> pointers,int dimension,int plane,int cbmask)
-{
-  SimpleCompressor<vobj> dontcompress;
-  Gather_plane_extract<vobj,vobj,decltype(dontcompress)>(rhs,pointers,dimension,plane,cbmask,dontcompress);
-}
-
 //////////////////////////////////////////////////////
 // Scatter for when there is no need to SIMD split
 //////////////////////////////////////////////////////
@@ -200,7 +171,7 @@ template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,commVector<vo
 //////////////////////////////////////////////////////
 // Scatter for when there *is* need to SIMD split
 //////////////////////////////////////////////////////
- template<class vobj,class cobj> void Scatter_plane_merge(Lattice<vobj> &rhs,std::vector<cobj *> pointers,int dimension,int plane,int cbmask)
+template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,std::vector<typename vobj::scalar_object *> pointers,int dimension,int plane,int cbmask)
 {
   int rd = rhs._grid->_rdimensions[dimension];
 

lib/cshift/Cshift_mpi.h

@@ -154,13 +154,7 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
 			   recv_from_rank,
 			   bytes);
       grid->Barrier();
-      /*
-      for(int i=0;i<send_buf.size();i++){
-	assert(recv_buf.size()==buffer_size);
-	assert(send_buf.size()==buffer_size);
-	std::cout << "SendRecv_Cshift_comms ["<<i<<" "<< dimension<<"] snd "<<send_buf[i]<<" rcv " << recv_buf[i] << "  0x" << cbmask<<std::endl;
-      }
-      */
+
       Scatter_plane_simple (ret,recv_buf,dimension,x,cbmask);
     }
   }
@@ -246,13 +240,6 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
 			     (void *)&recv_buf_extract[i][0],
 			     recv_from_rank,
 			     bytes);
-	/*
-	for(int w=0;w<recv_buf_extract[i].size();w++){
-	  assert(recv_buf_extract[i].size()==buffer_size);
-	  assert(send_buf_extract[i].size()==buffer_size);
-	  std::cout << "SendRecv_Cshift_comms ["<<w<<" "<< dimension<<"] recv "<<recv_buf_extract[i][w]<<" send " << send_buf_extract[nbr_lane][w]  << cbmask<<std::endl;
-	}
-	*/	
 	grid->Barrier();
 	rpointers[i] = &recv_buf_extract[i][0];
       } else { 

lib/lattice/Lattice_base.h

@@ -235,64 +235,74 @@ public:
     }
   };
 
-    //////////////////////////////////////////////////////////////////
-    // Constructor requires "grid" passed.
-    // what about a default grid?
-    //////////////////////////////////////////////////////////////////
-    Lattice(GridBase *grid) : _odata(grid->oSites()) {
-        _grid = grid;
+  //////////////////////////////////////////////////////////////////
+  // Constructor requires "grid" passed.
+  // what about a default grid?
+  //////////////////////////////////////////////////////////////////
+  Lattice(GridBase *grid) : _odata(grid->oSites()) {
+    _grid = grid;
     //        _odata.reserve(_grid->oSites());
     //        _odata.resize(_grid->oSites());
     //      std::cout << "Constructing lattice object with Grid pointer "<<_grid<<std::endl;
-        assert((((uint64_t)&_odata[0])&0xF) ==0);
-        checkerboard=0;
-    }
-
-    Lattice(const Lattice& r){ // copy constructor
-    	_grid = r._grid;
-    	checkerboard = r.checkerboard;
-    	_odata.resize(_grid->oSites());// essential
-	parallel_for(int ss=0;ss<_grid->oSites();ss++){
-            _odata[ss]=r._odata[ss];
-        }  	
-    }
-
-
-
-    virtual ~Lattice(void) = default;
+    assert((((uint64_t)&_odata[0])&0xF) ==0);
+    checkerboard=0;
+  }
+  
+  Lattice(const Lattice& r){ // copy constructor
+    _grid = r._grid;
+    checkerboard = r.checkerboard;
+    _odata.resize(_grid->oSites());// essential
+    parallel_for(int ss=0;ss<_grid->oSites();ss++){
+      _odata[ss]=r._odata[ss];
+    }  	
+  }
+  
+  
+  
+  virtual ~Lattice(void) = default;
     
-    template<class sobj> strong_inline Lattice<vobj> & operator = (const sobj & r){
-      parallel_for(int ss=0;ss<_grid->oSites();ss++){
-            this->_odata[ss]=r;
-        }
-        return *this;
-    }
-    template<class robj> strong_inline Lattice<vobj> & operator = (const Lattice<robj> & r){
-      this->checkerboard = r.checkerboard;
-      conformable(*this,r);
-      
-      parallel_for(int ss=0;ss<_grid->oSites();ss++){
-            this->_odata[ss]=r._odata[ss];
-        }
-        return *this;
+  void reset(GridBase* grid) {
+    if (_grid != grid) {
+      _grid = grid;
+      _odata.resize(grid->oSites());
+      checkerboard = 0;
     }
+  }
+  
 
-    // *=,+=,-= operators inherit behvour from correspond */+/- operation
-    template<class T> strong_inline Lattice<vobj> &operator *=(const T &r) {
-        *this = (*this)*r;
-        return *this;
+  template<class sobj> strong_inline Lattice<vobj> & operator = (const sobj & r){
+    parallel_for(int ss=0;ss<_grid->oSites();ss++){
+      this->_odata[ss]=r;
     }
-
-    template<class T> strong_inline Lattice<vobj> &operator -=(const T &r) {
-        *this = (*this)-r;
-        return *this;
+    return *this;
+  }
+  
+  template<class robj> strong_inline Lattice<vobj> & operator = (const Lattice<robj> & r){
+    this->checkerboard = r.checkerboard;
+    conformable(*this,r);
+    
+    parallel_for(int ss=0;ss<_grid->oSites();ss++){
+      this->_odata[ss]=r._odata[ss];
     }
-    template<class T> strong_inline Lattice<vobj> &operator +=(const T &r) {
-        *this = (*this)+r;
-        return *this;
-    }
- }; // class Lattice
-
+    return *this;
+  }
+  
+  // *=,+=,-= operators inherit behvour from correspond */+/- operation
+  template<class T> strong_inline Lattice<vobj> &operator *=(const T &r) {
+    *this = (*this)*r;
+    return *this;
+  }
+  
+  template<class T> strong_inline Lattice<vobj> &operator -=(const T &r) {
+    *this = (*this)-r;
+    return *this;
+  }
+  template<class T> strong_inline Lattice<vobj> &operator +=(const T &r) {
+    *this = (*this)+r;
+    return *this;
+  }
+}; // class Lattice
+  
   template<class vobj> std::ostream& operator<< (std::ostream& stream, const Lattice<vobj> &o){
     std::vector<int> gcoor;
     typedef typename vobj::scalar_object sobj;
@@ -310,7 +320,7 @@ public:
     }
     return stream;
   }
-
+  
 }
 
 

lib/lattice/Lattice_reduction.h

@@ -1,157 +1,154 @@
-    /*************************************************************************************
-
+ /*************************************************************************************
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./lib/lattice/Lattice_reduction.h
-
     Copyright (C) 2015
-
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
-
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
 #ifndef GRID_LATTICE_REDUCTION_H
 #define GRID_LATTICE_REDUCTION_H
 
+#include <Grid/Grid_Eigen_Dense.h>
+
 namespace Grid {
 #ifdef GRID_WARN_SUBOPTIMAL
 #warning "Optimisation alert all these reduction loops are NOT threaded "
 #endif     
 
-    ////////////////////////////////////////////////////////////////////////////////////////////////////
-    // Deterministic Reduction operations
-    ////////////////////////////////////////////////////////////////////////////////////////////////////
-  template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
-    ComplexD nrm = innerProduct(arg,arg);
-    return std::real(nrm); 
+  ////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Deterministic Reduction operations
+  ////////////////////////////////////////////////////////////////////////////////////////////////////
+template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
+  ComplexD nrm = innerProduct(arg,arg);
+  return std::real(nrm); 
+}
+
+// Double inner product
+template<class vobj>
+inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right) 
+{
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_typeD vector_type;
+  scalar_type  nrm;
+  
+  GridBase *grid = left._grid;
+  
+  std::vector<vector_type,alignedAllocator<vector_type> > sumarray(grid->SumArraySize());
+  
+  parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
+    int nwork, mywork, myoff;
+    GridThread::GetWork(left._grid->oSites(),thr,mywork,myoff);
+    
+    decltype(innerProductD(left._odata[0],right._odata[0])) vnrm=zero; // private to thread; sub summation
+    for(int ss=myoff;ss<mywork+myoff; ss++){
+      vnrm = vnrm + innerProductD(left._odata[ss],right._odata[ss]);
+    }
+    sumarray[thr]=TensorRemove(vnrm) ;
   }
+  
+  vector_type vvnrm; vvnrm=zero;  // sum across threads
+  for(int i=0;i<grid->SumArraySize();i++){
+    vvnrm = vvnrm+sumarray[i];
+  } 
+  nrm = Reduce(vvnrm);// sum across simd
+  right._grid->GlobalSum(nrm);
+  return nrm;
+}
+ 
+template<class Op,class T1>
+inline auto sum(const LatticeUnaryExpression<Op,T1> & expr)
+  ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second))))::scalar_object
+{
+  return sum(closure(expr));
+}
 
-    template<class vobj>
-    inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right) 
-    {
-      typedef typename vobj::scalar_type scalar_type;
-      typedef typename vobj::vector_type vector_type;
-      scalar_type  nrm;
-
-      GridBase *grid = left._grid;
-
-      std::vector<vector_type,alignedAllocator<vector_type> > sumarray(grid->SumArraySize());
-      for(int i=0;i<grid->SumArraySize();i++){
-	sumarray[i]=zero;
-      }
-
-      parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
-	int nwork, mywork, myoff;
-	GridThread::GetWork(left._grid->oSites(),thr,mywork,myoff);
-	
-	decltype(innerProduct(left._odata[0],right._odata[0])) vnrm=zero; // private to thread; sub summation
-        for(int ss=myoff;ss<mywork+myoff; ss++){
-	  vnrm = vnrm + innerProduct(left._odata[ss],right._odata[ss]);
-	}
-	sumarray[thr]=TensorRemove(vnrm) ;
-      }
-      
-      vector_type vvnrm; vvnrm=zero;  // sum across threads
-      for(int i=0;i<grid->SumArraySize();i++){
-	vvnrm = vvnrm+sumarray[i];
-      } 
-      nrm = Reduce(vvnrm);// sum across simd
-      right._grid->GlobalSum(nrm);
-      return nrm;
-    }
-
-    template<class Op,class T1>
-      inline auto sum(const LatticeUnaryExpression<Op,T1> & expr)
-      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second))))::scalar_object
-    {
-      return sum(closure(expr));
-    }
-
-    template<class Op,class T1,class T2>
-      inline auto sum(const LatticeBinaryExpression<Op,T1,T2> & expr)
+template<class Op,class T1,class T2>
+inline auto sum(const LatticeBinaryExpression<Op,T1,T2> & expr)
       ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),eval(0,std::get<1>(expr.second))))::scalar_object
-    {
-      return sum(closure(expr));
-    }
-
-
-    template<class Op,class T1,class T2,class T3>
-      inline auto sum(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr)
-      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),
-				 eval(0,std::get<1>(expr.second)),
-				 eval(0,std::get<2>(expr.second))
-				 ))::scalar_object
-    {
-      return sum(closure(expr));
-    }
-
-    template<class vobj>
-    inline typename vobj::scalar_object sum(const Lattice<vobj> &arg){
-
-      GridBase *grid=arg._grid;
-      int Nsimd = grid->Nsimd();
-
-      std::vector<vobj,alignedAllocator<vobj> > sumarray(grid->SumArraySize());
-      for(int i=0;i<grid->SumArraySize();i++){
-	sumarray[i]=zero;
-      }
-
-      parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
-	int nwork, mywork, myoff;
-	GridThread::GetWork(grid->oSites(),thr,mywork,myoff);
-	
-	vobj vvsum=zero;
-        for(int ss=myoff;ss<mywork+myoff; ss++){
-	  vvsum = vvsum + arg._odata[ss];
-	}
-	sumarray[thr]=vvsum;
-      }
-      
-      vobj vsum=zero;  // sum across threads
-      for(int i=0;i<grid->SumArraySize();i++){
-	vsum = vsum+sumarray[i];
-      } 
-
-      typedef typename vobj::scalar_object sobj;
-      sobj ssum=zero;
-
-      std::vector<sobj>               buf(Nsimd);
-      extract(vsum,buf);
-
-      for(int i=0;i<Nsimd;i++) ssum = ssum + buf[i];
-      arg._grid->GlobalSum(ssum);
-
-      return ssum;
+{
+  return sum(closure(expr));
+}
+
+
+template<class Op,class T1,class T2,class T3>
+inline auto sum(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr)
+  ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),
+				      eval(0,std::get<1>(expr.second)),
+				      eval(0,std::get<2>(expr.second))
+				      ))::scalar_object
+{
+  return sum(closure(expr));
+}
+
+template<class vobj>
+inline typename vobj::scalar_object sum(const Lattice<vobj> &arg)
+{
+  GridBase *grid=arg._grid;
+  int Nsimd = grid->Nsimd();
+  
+  std::vector<vobj,alignedAllocator<vobj> > sumarray(grid->SumArraySize());
+  for(int i=0;i<grid->SumArraySize();i++){
+    sumarray[i]=zero;
+  }
+  
+  parallel_for(int thr=0;thr<grid->SumArraySize();thr++){
+    int nwork, mywork, myoff;
+    GridThread::GetWork(grid->oSites(),thr,mywork,myoff);
+    
+    vobj vvsum=zero;
+    for(int ss=myoff;ss<mywork+myoff; ss++){
+      vvsum = vvsum + arg._odata[ss];
     }
+    sumarray[thr]=vvsum;
+  }
+  
+  vobj vsum=zero;  // sum across threads
+  for(int i=0;i<grid->SumArraySize();i++){
+    vsum = vsum+sumarray[i];
+  } 
+  
+  typedef typename vobj::scalar_object sobj;
+  sobj ssum=zero;
+  
+  std::vector<sobj>               buf(Nsimd);
+  extract(vsum,buf);
+  
+  for(int i=0;i<Nsimd;i++) ssum = ssum + buf[i];
+  arg._grid->GlobalSum(ssum);
+  
+  return ssum;
+}
 
 
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// sliceSum, sliceInnerProduct, sliceAxpy, sliceNorm etc...
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<typename vobj::scalar_object> &result,int orthogdim)
 {
+  ///////////////////////////////////////////////////////
+  // FIXME precision promoted summation
+  // may be important for correlation functions
+  // But easily avoided by using double precision fields
+  ///////////////////////////////////////////////////////
   typedef typename vobj::scalar_object sobj;
   GridBase  *grid = Data._grid;
   assert(grid!=NULL);
 
-  // FIXME
-  // std::cout<<GridLogMessage<<"WARNING ! SliceSum is unthreaded "<<grid->SumArraySize()<<" threads "<<std::endl;
-
   const int    Nd = grid->_ndimension;
   const int Nsimd = grid->Nsimd();
 
@@ -163,23 +160,31 @@ template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<
   int rd=grid->_rdimensions[orthogdim];
 
   std::vector<vobj,alignedAllocator<vobj> > lvSum(rd); // will locally sum vectors first
-  std::vector<sobj> lsSum(ld,zero); // sum across these down to scalars
-  std::vector<sobj> extracted(Nsimd);     // splitting the SIMD
+  std::vector<sobj> lsSum(ld,zero);                    // sum across these down to scalars
+  std::vector<sobj> extracted(Nsimd);                  // splitting the SIMD
 
-  result.resize(fd); // And then global sum to return the same vector to every node for IO to file
+  result.resize(fd); // And then global sum to return the same vector to every node 
   for(int r=0;r<rd;r++){
     lvSum[r]=zero;
   }
 
-  std::vector<int>  coor(Nd);  
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
 
   // sum over reduced dimension planes, breaking out orthog dir
+  // Parallel over orthog direction
+  parallel_for(int r=0;r<rd;r++){
 
-  for(int ss=0;ss<grid->oSites();ss++){
-    Lexicographic::CoorFromIndex(coor,ss,grid->_rdimensions);
-    int r = coor[orthogdim];
-    lvSum[r]=lvSum[r]+Data._odata[ss];
-  }  
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	lvSum[r]=lvSum[r]+Data._odata[ss];
+      }
+    }
+  }
 
   // Sum across simd lanes in the plane, breaking out orthog dir.
   std::vector<int> icoor(Nd);
@@ -214,10 +219,305 @@ template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<
 
     result[t]=gsum;
   }
+}
 
+template<class vobj>
+static void sliceInnerProductVector( std::vector<ComplexD> & result, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int orthogdim) 
+{
+  typedef typename vobj::vector_type   vector_type;
+  typedef typename vobj::scalar_type   scalar_type;
+  GridBase  *grid = lhs._grid;
+  assert(grid!=NULL);
+  conformable(grid,rhs._grid);
+
+  const int    Nd = grid->_ndimension;
+  const int Nsimd = grid->Nsimd();
+
+  assert(orthogdim >= 0);
+  assert(orthogdim < Nd);
+
+  int fd=grid->_fdimensions[orthogdim];
+  int ld=grid->_ldimensions[orthogdim];
+  int rd=grid->_rdimensions[orthogdim];
+
+  std::vector<vector_type,alignedAllocator<vector_type> > lvSum(rd); // will locally sum vectors first
+  std::vector<scalar_type > lsSum(ld,scalar_type(0.0));                    // sum across these down to scalars
+  std::vector<iScalar<scalar_type> > extracted(Nsimd);                  // splitting the SIMD
+
+  result.resize(fd); // And then global sum to return the same vector to every node for IO to file
+  for(int r=0;r<rd;r++){
+    lvSum[r]=zero;
+  }
+
+  int e1=    grid->_slice_nblock[orthogdim];
+  int e2=    grid->_slice_block [orthogdim];
+  int stride=grid->_slice_stride[orthogdim];
+
+  parallel_for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    for(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	vector_type vv = TensorRemove(innerProduct(lhs._odata[ss],rhs._odata[ss]));
+	lvSum[r]=lvSum[r]+vv;
+      }
+    }
+  }
+
+  // Sum across simd lanes in the plane, breaking out orthog dir.
+  std::vector<int> icoor(Nd);
+  for(int rt=0;rt<rd;rt++){
+
+    iScalar<vector_type> temp; 
+    temp._internal = lvSum[rt];
+    extract(temp,extracted);
+
+    for(int idx=0;idx<Nsimd;idx++){
+
+      grid->iCoorFromIindex(icoor,idx);
+
+      int ldx =rt+icoor[orthogdim]*rd;
+
+      lsSum[ldx]=lsSum[ldx]+extracted[idx]._internal;
+
+    }
+  }
+  
+  // sum over nodes.
+  scalar_type gsum;
+  for(int t=0;t<fd;t++){
+    int pt = t/ld; // processor plane
+    int lt = t%ld;
+    if ( pt == grid->_processor_coor[orthogdim] ) {
+      gsum=lsSum[lt];
+    } else {
+      gsum=scalar_type(0.0);
+    }
+
+    grid->GlobalSum(gsum);
+
+    result[t]=gsum;
+  }
+}
+template<class vobj>
+static void sliceNorm (std::vector<RealD> &sn,const Lattice<vobj> &rhs,int Orthog) 
+{
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  int Nblock = rhs._grid->GlobalDimensions()[Orthog];
+  std::vector<ComplexD> ip(Nblock);
+  sn.resize(Nblock);
+  
+  sliceInnerProductVector(ip,rhs,rhs,Orthog);
+  for(int ss=0;ss<Nblock;ss++){
+    sn[ss] = real(ip[ss]);
+  }
+};
+
+
+template<class vobj>
+static void sliceMaddVector(Lattice<vobj> &R,std::vector<RealD> &a,const Lattice<vobj> &X,const Lattice<vobj> &Y,
+			    int orthogdim,RealD scale=1.0) 
+{    
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  typedef typename vobj::tensor_reduced tensor_reduced;
+  
+  scalar_type zscale(scale);
+
+  GridBase *grid  = X._grid;
+
+  int Nsimd  =grid->Nsimd();
+  int Nblock =grid->GlobalDimensions()[orthogdim];
+
+  int fd     =grid->_fdimensions[orthogdim];
+  int ld     =grid->_ldimensions[orthogdim];
+  int rd     =grid->_rdimensions[orthogdim];
+
+  int e1     =grid->_slice_nblock[orthogdim];
+  int e2     =grid->_slice_block [orthogdim];
+  int stride =grid->_slice_stride[orthogdim];
+
+  std::vector<int> icoor;
+
+  for(int r=0;r<rd;r++){
+
+    int so=r*grid->_ostride[orthogdim]; // base offset for start of plane 
+
+    vector_type    av;
+
+    for(int l=0;l<Nsimd;l++){
+      grid->iCoorFromIindex(icoor,l);
+      int ldx =r+icoor[orthogdim]*rd;
+      scalar_type *as =(scalar_type *)&av;
+      as[l] = scalar_type(a[ldx])*zscale;
+    }
+
+    tensor_reduced at; at=av;
+
+    parallel_for_nest2(int n=0;n<e1;n++){
+      for(int b=0;b<e2;b++){
+	int ss= so+n*stride+b;
+	R._odata[ss] = at*X._odata[ss]+Y._odata[ss];
+      }
+    }
+  }
+};
+
+
+/*
+template<class vobj>
+static void sliceMaddVectorSlow (Lattice<vobj> &R,std::vector<RealD> &a,const Lattice<vobj> &X,const Lattice<vobj> &Y,
+			     int Orthog,RealD scale=1.0) 
+{    
+  // FIXME: Implementation is slow
+  // Best base the linear combination by constructing a 
+  // set of vectors of size grid->_rdimensions[Orthog].
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  int Nblock = X._grid->GlobalDimensions()[Orthog];
+  
+  GridBase *FullGrid  = X._grid;
+  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  
+  Lattice<vobj> Xslice(SliceGrid);
+  Lattice<vobj> Rslice(SliceGrid);
+  // If we based this on Cshift it would work for spread out
+  // but it would be even slower
+  for(int i=0;i<Nblock;i++){
+    ExtractSlice(Rslice,Y,i,Orthog);
+    ExtractSlice(Xslice,X,i,Orthog);
+    Rslice = Rslice + Xslice*(scale*a[i]);
+    InsertSlice(Rslice,R,i,Orthog);
+  }
+};
+template<class vobj>
+static void sliceInnerProductVectorSlow( std::vector<ComplexD> & vec, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog) 
+  {
+    // FIXME: Implementation is slow
+    // Look at localInnerProduct implementation,
+    // and do inside a site loop with block strided iterators
+    typedef typename vobj::scalar_object sobj;
+    typedef typename vobj::scalar_type scalar_type;
+    typedef typename vobj::vector_type vector_type;
+    typedef typename vobj::tensor_reduced scalar;
+    typedef typename scalar::scalar_object  scomplex;
+  
+    int Nblock = lhs._grid->GlobalDimensions()[Orthog];
+    vec.resize(Nblock);
+    std::vector<scomplex> sip(Nblock);
+    Lattice<scalar> IP(lhs._grid); 
+    IP=localInnerProduct(lhs,rhs);
+    sliceSum(IP,sip,Orthog);
+  
+    for(int ss=0;ss<Nblock;ss++){
+      vec[ss] = TensorRemove(sip[ss]);
+    }
+  }
+*/
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// FIXME: Implementation is slow
+// If we based this on Cshift it would work for spread out
+// but it would be even slower
+//
+// Repeated extract slice is inefficient
+//
+// Best base the linear combination by constructing a 
+// set of vectors of size grid->_rdimensions[Orthog].
+//////////////////////////////////////////////////////////////////////////////////////////
+
+inline GridBase         *makeSubSliceGrid(const GridBase *BlockSolverGrid,int Orthog)
+{
+  int NN    = BlockSolverGrid->_ndimension;
+  int nsimd = BlockSolverGrid->Nsimd();
+  
+  std::vector<int> latt_phys(0);
+  std::vector<int> simd_phys(0);
+  std::vector<int>  mpi_phys(0);
+  
+  for(int d=0;d<NN;d++){
+    if( d!=Orthog ) { 
+      latt_phys.push_back(BlockSolverGrid->_fdimensions[d]);
+      simd_phys.push_back(BlockSolverGrid->_simd_layout[d]);
+      mpi_phys.push_back(BlockSolverGrid->_processors[d]);
+    }
+  }
+  return (GridBase *)new GridCartesian(latt_phys,simd_phys,mpi_phys); 
 }
 
 
+template<class vobj>
+static void sliceMaddMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,const Lattice<vobj> &Y,int Orthog,RealD scale=1.0) 
+{    
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+
+  int Nblock = X._grid->GlobalDimensions()[Orthog];
+  
+  GridBase *FullGrid  = X._grid;
+  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  
+  Lattice<vobj> Xslice(SliceGrid);
+  Lattice<vobj> Rslice(SliceGrid);
+  
+  for(int i=0;i<Nblock;i++){
+    ExtractSlice(Rslice,Y,i,Orthog);
+    for(int j=0;j<Nblock;j++){
+      ExtractSlice(Xslice,X,j,Orthog);
+      Rslice = Rslice + Xslice*(scale*aa(j,i));
+    }
+    InsertSlice(Rslice,R,i,Orthog);
+  }
+};
+
+template<class vobj>
+static void sliceInnerProductMatrix(  Eigen::MatrixXcd &mat, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog) 
+{
+  // FIXME: Implementation is slow
+  // Not sure of best solution.. think about it
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+  
+  GridBase *FullGrid  = lhs._grid;
+  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  
+  int Nblock = FullGrid->GlobalDimensions()[Orthog];
+  
+  Lattice<vobj> Lslice(SliceGrid);
+  Lattice<vobj> Rslice(SliceGrid);
+  
+  mat = Eigen::MatrixXcd::Zero(Nblock,Nblock);
+  
+  for(int i=0;i<Nblock;i++){
+    ExtractSlice(Lslice,lhs,i,Orthog);
+    for(int j=0;j<Nblock;j++){
+      ExtractSlice(Rslice,rhs,j,Orthog);
+      mat(i,j) = innerProduct(Lslice,Rslice);
+    }
+  }
+#undef FORCE_DIAG
+#ifdef FORCE_DIAG
+  for(int i=0;i<Nblock;i++){
+    for(int j=0;j<Nblock;j++){
+      if ( i != j ) mat(i,j)=0.0;
+    }
+  }
+#endif
+  return;
 }
+
+} /*END NAMESPACE GRID*/
 #endif
 
+
+

lib/lattice/Lattice_rng.h

@@ -6,8 +6,8 @@
 
     Copyright (C) 2015
 
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -75,6 +75,55 @@ namespace Grid {
     return multiplicity;
   }
 
+  
+// merge of April 11 2017
+//<<<<<<< HEAD
+
+
+  // this function is necessary for the LS vectorised field
+  inline int RNGfillable_general(GridBase *coarse,GridBase *fine)
+  {
+    int rngdims = coarse->_ndimension;
+    
+    // trivially extended in higher dims, with locality guaranteeing RNG state is local to node
+    int lowerdims   = fine->_ndimension - coarse->_ndimension;  assert(lowerdims >= 0);
+    // assumes that the higher dimensions are not using more processors
+    // all further divisions are local
+    for(int d=0;d<lowerdims;d++) assert(fine->_processors[d]==1);
+    for(int d=0;d<rngdims;d++) assert(coarse->_processors[d] == fine->_processors[d+lowerdims]);
+    
+
+    // then divide the number of local sites
+    // check that the total number of sims agree, meanse the iSites are the same
+    assert(fine->Nsimd() == coarse->Nsimd());
+
+    // check that the two grids divide cleanly
+    assert( (fine->lSites() / coarse->lSites() ) * coarse->lSites() == fine->lSites() );
+
+    return fine->lSites() / coarse->lSites();
+  }
+
+  /*
+  // Wrap seed_seq to give common interface with random_device
+  class fixedSeed {
+  public:
+    typedef std::seed_seq::result_type result_type;
+    std::seed_seq src;
+    
+    fixedSeed(const std::vector<int> &seeds) : src(seeds.begin(),seeds.end()) {};
+
+    result_type operator () (void){
+      std::vector<result_type> list(1);
+      src.generate(list.begin(),list.end());
+      return list[0];
+    }
+
+  };
+
+=======
+>>>>>>> develop
+  */
+  
   // real scalars are one component
   template<class scalar,class distribution,class generator> 
   void fillScalar(scalar &s,distribution &dist,generator & gen)
@@ -109,7 +158,7 @@ namespace Grid {
 #ifdef RNG_SITMO
     typedef sitmo::prng_engine 	RngEngine;
     typedef uint64_t    	RngStateType;
-    static const int    	RngStateCount = 4;
+    static const int    	RngStateCount = 13;
 #endif
 
     std::vector<RngEngine>                             _generators;
@@ -164,7 +213,7 @@ namespace Grid {
       ss<<eng;
       ss.seekg(0,ss.beg);
       for(int i=0;i<RngStateCount;i++){
-	ss>>saved[i];
+        ss>>saved[i];
       }
     }
     void GetState(std::vector<RngStateType> & saved,int gen) {
@@ -174,7 +223,7 @@ namespace Grid {
       assert(saved.size()==RngStateCount);
       std::stringstream ss;
       for(int i=0;i<RngStateCount;i++){
-	ss<< saved[i]<<" ";
+        ss<< saved[i]<<" ";
       }
       ss.seekg(0,ss.beg);
       ss>>eng;
@@ -215,7 +264,7 @@ namespace Grid {
 
       dist[0].reset();
       for(int idx=0;idx<words;idx++){
-	fillScalar(buf[idx],dist[0],_generators[0]);
+  fillScalar(buf[idx],dist[0],_generators[0]);
       }
 
       CartesianCommunicator::BroadcastWorld(0,(void *)&l,sizeof(l));
@@ -247,7 +296,7 @@ namespace Grid {
       RealF *pointer=(RealF *)&l;
       dist[0].reset();
       for(int i=0;i<2*vComplexF::Nsimd();i++){
-	fillScalar(pointer[i],dist[0],_generators[0]);
+  fillScalar(pointer[i],dist[0],_generators[0]);
       }
       CartesianCommunicator::BroadcastWorld(0,(void *)&l,sizeof(l));
     }
@@ -255,7 +304,7 @@ namespace Grid {
       RealD *pointer=(RealD *)&l;
       dist[0].reset();
       for(int i=0;i<2*vComplexD::Nsimd();i++){
-	fillScalar(pointer[i],dist[0],_generators[0]);
+  fillScalar(pointer[i],dist[0],_generators[0]);
       }
       CartesianCommunicator::BroadcastWorld(0,(void *)&l,sizeof(l));
     }
@@ -263,7 +312,7 @@ namespace Grid {
       RealF *pointer=(RealF *)&l;
       dist[0].reset();
       for(int i=0;i<vRealF::Nsimd();i++){
-	fillScalar(pointer[i],dist[0],_generators[0]);
+  fillScalar(pointer[i],dist[0],_generators[0]);
       }
       CartesianCommunicator::BroadcastWorld(0,(void *)&l,sizeof(l));
     }
@@ -275,7 +324,7 @@ namespace Grid {
       }
       CartesianCommunicator::BroadcastWorld(0,(void *)&l,sizeof(l));
     }
-
+    
     void SeedFixedIntegers(const std::vector<int> &seeds){
       CartesianCommunicator::BroadcastWorld(0,(void *)&seeds[0],sizeof(int)*seeds.size());
       std::seed_seq src(seeds.begin(),seeds.end());
@@ -284,18 +333,20 @@ namespace Grid {
   };
 
   class GridParallelRNG : public GridRNGbase {
+
+    double _time_counter;
+
   public:
     GridBase *_grid;
-    int _vol;
-  public:
+    unsigned int _vol;
 
-    int generator_idx(int os,int is){
+    int generator_idx(int os,int is) {
       return is*_grid->oSites()+os;
     }
 
     GridParallelRNG(GridBase *grid) : GridRNGbase() {
-      _grid=grid;
-      _vol =_grid->iSites()*_grid->oSites();
+      _grid = grid;
+      _vol  =_grid->iSites()*_grid->oSites();
 
       _generators.resize(_vol);
       _uniform.resize(_vol,std::uniform_real_distribution<RealD>{0,1});
@@ -309,33 +360,34 @@ namespace Grid {
       typedef typename vobj::scalar_object scalar_object;
       typedef typename vobj::scalar_type scalar_type;
       typedef typename vobj::vector_type vector_type;
-      
-      int multiplicity = RNGfillable(_grid,l._grid);
 
-      int     Nsimd =_grid->Nsimd();
-      int     osites=_grid->oSites();
-      int words=sizeof(scalar_object)/sizeof(scalar_type);
+      double inner_time_counter = usecond();
+
+      int multiplicity = RNGfillable_general(_grid, l._grid); // l has finer or same grid
+      int Nsimd  = _grid->Nsimd();  // guaranteed to be the same for l._grid too
+      int osites = _grid->oSites();  // guaranteed to be <= l._grid->oSites() by a factor multiplicity
+      int words  = sizeof(scalar_object) / sizeof(scalar_type);
 
       parallel_for(int ss=0;ss<osites;ss++){
+        std::vector<scalar_object> buf(Nsimd);
+        for (int m = 0; m < multiplicity; m++) {  // Draw from same generator multiplicity times
 
-	std::vector<scalar_object> buf(Nsimd);
-	for(int m=0;m<multiplicity;m++) {// Draw from same generator multiplicity times
+          int sm = multiplicity * ss + m;  // Maps the generator site to the fine site
 
-	  int sm=multiplicity*ss+m;      // Maps the generator site to the fine site
-
-	  for(int si=0;si<Nsimd;si++){
-	    int gdx = generator_idx(ss,si); // index of generator state
-	    scalar_type *pointer = (scalar_type *)&buf[si];
-	    dist[gdx].reset();
-	    for(int idx=0;idx<words;idx++){
-	      fillScalar(pointer[idx],dist[gdx],_generators[gdx]);
-	    }
-	  }
-
-	  // merge into SIMD lanes
-	  merge(l._odata[sm],buf);
-	}
+          for (int si = 0; si < Nsimd; si++) {
+            
+            int gdx = generator_idx(ss, si);  // index of generator state
+            scalar_type *pointer = (scalar_type *)&buf[si];
+            dist[gdx].reset();
+            for (int idx = 0; idx < words; idx++) 
+              fillScalar(pointer[idx], dist[gdx], _generators[gdx]);
+          }
+          // merge into SIMD lanes, FIXME suboptimal implementation
+          merge(l._odata[sm], buf);
+        }
       }
+
+      _time_counter += usecond()- inner_time_counter;
     };
 
     void SeedFixedIntegers(const std::vector<int> &seeds){
@@ -412,6 +464,12 @@ namespace Grid {
       }
 #endif
     }
+
+    void Report(){
+      std::cout << GridLogMessage << "Time spent in the fill() routine by GridParallelRNG: "<< _time_counter/1e3 << " ms" << std::endl;
+    }
+
+
     ////////////////////////////////////////////////////////////////////////
     // Support for rigorous test of RNG's
     // Return uniform random uint32_t from requested site generator
@@ -419,7 +477,6 @@ namespace Grid {
     uint32_t GlobalU01(int gsite){
 
       uint32_t the_number;
-
       // who
       std::vector<int> gcoor;
       int rank,o_idx,i_idx;

lib/lattice/Lattice_transfer.h

@@ -1,4 +1,4 @@
-    /*************************************************************************************
+/*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
 
@@ -359,7 +359,7 @@ void localConvert(const Lattice<vobj> &in,Lattice<vvobj> &out)
 
 
 template<class vobj>
-void InsertSlice(Lattice<vobj> &lowDim,Lattice<vobj> & higherDim,int slice, int orthog)
+void InsertSlice(const Lattice<vobj> &lowDim,Lattice<vobj> & higherDim,int slice, int orthog)
 {
   typedef typename vobj::scalar_object sobj;
 
@@ -401,7 +401,7 @@ void InsertSlice(Lattice<vobj> &lowDim,Lattice<vobj> & higherDim,int slice, int
 }
 
 template<class vobj>
-void ExtractSlice(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice, int orthog)
+void ExtractSlice(Lattice<vobj> &lowDim,const Lattice<vobj> & higherDim,int slice, int orthog)
 {
   typedef typename vobj::scalar_object sobj;
 
@@ -444,7 +444,7 @@ void ExtractSlice(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice, in
 
 
 template<class vobj>
-void InsertSliceLocal(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice_lo,int slice_hi, int orthog)
+void InsertSliceLocal(const Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slice_lo,int slice_hi, int orthog)
 {
   typedef typename vobj::scalar_object sobj;
 
@@ -551,7 +551,10 @@ void Replicate(Lattice<vobj> &coarse,Lattice<vobj> & fine)
 
 //Copy SIMD-vectorized lattice to array of scalar objects in lexicographic order
 template<typename vobj, typename sobj>
-typename std::enable_if<isSIMDvectorized<vobj>::value && !isSIMDvectorized<sobj>::value, void>::type unvectorizeToLexOrdArray(std::vector<sobj> &out, const Lattice<vobj> &in){
+typename std::enable_if<isSIMDvectorized<vobj>::value && !isSIMDvectorized<sobj>::value, void>::type 
+unvectorizeToLexOrdArray(std::vector<sobj> &out, const Lattice<vobj> &in)
+{
+
   typedef typename vobj::vector_type vtype;
   
   GridBase* in_grid = in._grid;
@@ -590,6 +593,54 @@ typename std::enable_if<isSIMDvectorized<vobj>::value && !isSIMDvectorized<sobj>
     extract1(in_vobj, out_ptrs, 0);
   }
 }
+//Copy SIMD-vectorized lattice to array of scalar objects in lexicographic order
+template<typename vobj, typename sobj>
+typename std::enable_if<isSIMDvectorized<vobj>::value 
+                    && !isSIMDvectorized<sobj>::value, void>::type 
+vectorizeFromLexOrdArray( std::vector<sobj> &in, Lattice<vobj> &out)
+{
+
+  typedef typename vobj::vector_type vtype;
+  
+  GridBase* grid = out._grid;
+  assert(in.size()==grid->lSites());
+  
+  int ndim     = grid->Nd();
+  int nsimd    = vtype::Nsimd();
+
+  std::vector<std::vector<int> > icoor(nsimd);
+      
+  for(int lane=0; lane < nsimd; lane++){
+    icoor[lane].resize(ndim);
+    grid->iCoorFromIindex(icoor[lane],lane);
+  }
+  
+  parallel_for(uint64_t oidx = 0; oidx < grid->oSites(); oidx++){ //loop over outer index
+    //Assemble vector of pointers to output elements
+    std::vector<sobj*> ptrs(nsimd);
+
+    std::vector<int> ocoor(ndim);
+    grid->oCoorFromOindex(ocoor, oidx);
+
+    std::vector<int> lcoor(grid->Nd());
+      
+    for(int lane=0; lane < nsimd; lane++){
+
+      for(int mu=0;mu<ndim;mu++){
+	lcoor[mu] = ocoor[mu] + grid->_rdimensions[mu]*icoor[lane][mu];
+      }
+
+      int lex;
+      Lexicographic::IndexFromCoor(lcoor, lex, grid->_ldimensions);
+      ptrs[lane] = &in[lex];
+    }
+    
+    //pack from those ptrs
+    vobj vecobj;
+    merge1(vecobj, ptrs, 0);
+    out._odata[oidx] = vecobj; 
+  }
+}
 
 //Convert a Lattice from one precision to another
 template<class VobjOut, class VobjIn>
@@ -615,7 +666,7 @@ void precisionChange(Lattice<VobjOut> &out, const Lattice<VobjIn> &in){
   std::vector<SobjOut> in_slex_conv(in_grid->lSites());
   unvectorizeToLexOrdArray(in_slex_conv, in);
     
-  parallel_for(int out_oidx=0;out_oidx<out_grid->oSites();out_oidx++){
+  parallel_for(uint64_t out_oidx=0;out_oidx<out_grid->oSites();out_oidx++){
     std::vector<int> out_ocoor(ndim);
     out_grid->oCoorFromOindex(out_ocoor, out_oidx);
 

lib/lattice/Lattice_unary.h

@@ -66,10 +66,21 @@ namespace Grid {
     Lattice<obj> ret(rhs._grid);
     ret.checkerboard = rhs.checkerboard;
     conformable(ret,rhs);
+    obj unit(1.0);
     parallel_for(int ss=0;ss<rhs._grid->oSites();ss++){
-      ret._odata[ss]=Exponentiate(rhs._odata[ss],alpha, Nexp);
+      //ret._odata[ss]=Exponentiate(rhs._odata[ss],alpha, Nexp);
+      ret._odata[ss] = unit;
+      for(int i=Nexp; i>=1;--i)
+	ret._odata[ss] = unit + ret._odata[ss]*rhs._odata[ss]*(alpha/RealD(i));
+
     }
+
     return ret;
+
+    
+    
+
+    
   }
 
 

lib/log/Log.cc

@@ -30,6 +30,7 @@ directory
 *************************************************************************************/
 /*  END LEGAL */
 #include <Grid/GridCore.h>
+#include <Grid/util/CompilerCompatible.h>
 
 #include <cxxabi.h>
 #include <memory>

lib/log/Log.h

@@ -110,8 +110,8 @@ public:
   friend std::ostream& operator<< (std::ostream& stream, Logger& log){
 
     if ( log.active ) {
-      stream << log.background()<< std::setw(10) << std::left << log.topName << log.background()<< " : ";
-      stream << log.colour() << std::setw(14) << std::left << log.name << log.background() << " : ";
+      stream << log.background()<< std::setw(8) << std::left << log.topName << log.background()<< " : ";
+      stream << log.colour() << std::setw(10) << std::left << log.name << log.background() << " : ";
       if ( log.timestamp ) {
 	StopWatch.Stop();
 	GridTime now = StopWatch.Elapsed();

lib/parallelIO/IldgIO.h

@@ -0,0 +1,711 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/parallelIO/IldgIO.h
+
+Copyright (C) 2015
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_ILDG_IO_H
+#define GRID_ILDG_IO_H
+
+#ifdef HAVE_LIME
+#include <algorithm>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <map>
+
+#include <pwd.h>
+#include <sys/utsname.h>
+#include <unistd.h>
+
+//C-Lime is a must have for this functionality
+extern "C" {  
+#include "lime.h"
+}
+
+namespace Grid {
+namespace QCD {
+
+  /////////////////////////////////
+  // Encode word types as strings
+  /////////////////////////////////
+ template<class word> inline std::string ScidacWordMnemonic(void){ return std::string("unknown"); }
+ template<> inline std::string ScidacWordMnemonic<double>  (void){ return std::string("D"); }
+ template<> inline std::string ScidacWordMnemonic<float>   (void){ return std::string("F"); }
+ template<> inline std::string ScidacWordMnemonic< int32_t>(void){ return std::string("I32_t"); }
+ template<> inline std::string ScidacWordMnemonic<uint32_t>(void){ return std::string("U32_t"); }
+ template<> inline std::string ScidacWordMnemonic< int64_t>(void){ return std::string("I64_t"); }
+ template<> inline std::string ScidacWordMnemonic<uint64_t>(void){ return std::string("U64_t"); }
+
+  /////////////////////////////////////////
+  // Encode a generic tensor as a string
+  /////////////////////////////////////////
+ template<class vobj> std::string ScidacRecordTypeString(int &colors, int &spins, int & typesize,int &datacount) { 
+
+   typedef typename getPrecision<vobj>::real_scalar_type stype;
+
+   int _ColourN       = indexRank<ColourIndex,vobj>();
+   int _ColourScalar  =  isScalar<ColourIndex,vobj>();
+   int _ColourVector  =  isVector<ColourIndex,vobj>();
+   int _ColourMatrix  =  isMatrix<ColourIndex,vobj>();
+
+   int _SpinN       = indexRank<SpinIndex,vobj>();
+   int _SpinScalar  =  isScalar<SpinIndex,vobj>();
+   int _SpinVector  =  isVector<SpinIndex,vobj>();
+   int _SpinMatrix  =  isMatrix<SpinIndex,vobj>();
+
+   int _LorentzN       = indexRank<LorentzIndex,vobj>();
+   int _LorentzScalar  =  isScalar<LorentzIndex,vobj>();
+   int _LorentzVector  =  isVector<LorentzIndex,vobj>();
+   int _LorentzMatrix  =  isMatrix<LorentzIndex,vobj>();
+
+   std::stringstream stream;
+
+   stream << "GRID_";
+   stream << ScidacWordMnemonic<stype>();
+
+   //   std::cout << " Lorentz N/S/V/M : " << _LorentzN<<" "<<_LorentzScalar<<"/"<<_LorentzVector<<"/"<<_LorentzMatrix<<std::endl;
+   //   std::cout << " Spin    N/S/V/M : " << _SpinN   <<" "<<_SpinScalar   <<"/"<<_SpinVector   <<"/"<<_SpinMatrix<<std::endl;
+   //   std::cout << " Colour  N/S/V/M : " << _ColourN <<" "<<_ColourScalar <<"/"<<_ColourVector <<"/"<<_ColourMatrix<<std::endl;
+
+   if ( _LorentzVector )   stream << "_LorentzVector"<<_LorentzN;
+   if ( _LorentzMatrix )   stream << "_LorentzMatrix"<<_LorentzN;
+
+   if ( _SpinVector )   stream << "_SpinVector"<<_SpinN;
+   if ( _SpinMatrix )   stream << "_SpinMatrix"<<_SpinN;
+
+   if ( _ColourVector )   stream << "_ColourVector"<<_ColourN;
+   if ( _ColourMatrix )   stream << "_ColourMatrix"<<_ColourN;
+
+   if ( _ColourScalar && _LorentzScalar && _SpinScalar )   stream << "_Complex";
+
+
+   typesize = sizeof(typename vobj::scalar_type);
+
+   if ( _ColourMatrix ) typesize*= _ColourN*_ColourN;
+   else                 typesize*= _ColourN;
+
+   if ( _SpinMatrix )   typesize*= _SpinN*_SpinN;
+   else                 typesize*= _SpinN;
+
+   colors    = _ColourN;
+   spins     = _SpinN;
+   datacount = _LorentzN;
+
+   return stream.str();
+ }
+ 
+ template<class vobj> std::string ScidacRecordTypeString(Lattice<vobj> & lat,int &colors, int &spins, int & typesize,int &datacount) { 
+   return ScidacRecordTypeString<vobj>(colors,spins,typesize,datacount);
+ };
+
+
+ ////////////////////////////////////////////////////////////
+ // Helper to fill out metadata
+ ////////////////////////////////////////////////////////////
+ template<class vobj> void ScidacMetaData(Lattice<vobj> & field,
+					  FieldMetaData &header,
+					  scidacRecord & _scidacRecord,
+					  scidacFile   & _scidacFile) 
+ {
+   typedef typename getPrecision<vobj>::real_scalar_type stype;
+
+   /////////////////////////////////////
+   // Pull Grid's metadata
+   /////////////////////////////////////
+   PrepareMetaData(field,header);
+
+   /////////////////////////////////////
+   // Scidac Private File structure
+   /////////////////////////////////////
+   _scidacFile              = scidacFile(field._grid);
+
+   /////////////////////////////////////
+   // Scidac Private Record structure
+   /////////////////////////////////////
+   scidacRecord sr;
+   sr.datatype   = ScidacRecordTypeString(field,sr.colors,sr.spins,sr.typesize,sr.datacount);
+   sr.date       = header.creation_date;
+   sr.precision  = ScidacWordMnemonic<stype>();
+   sr.recordtype = GRID_IO_FIELD;
+
+   _scidacRecord = sr;
+
+   std::cout << GridLogMessage << "Build SciDAC datatype " <<sr.datatype<<std::endl;
+ }
+ 
+ ///////////////////////////////////////////////////////
+ // Scidac checksum
+ ///////////////////////////////////////////////////////
+ static int scidacChecksumVerify(scidacChecksum &scidacChecksum_,uint32_t scidac_csuma,uint32_t scidac_csumb)
+ {
+   uint32_t scidac_checksuma = stoull(scidacChecksum_.suma,0,16);
+   uint32_t scidac_checksumb = stoull(scidacChecksum_.sumb,0,16);
+   if ( scidac_csuma !=scidac_checksuma) return 0;
+   if ( scidac_csumb !=scidac_checksumb) return 0;
+    return 1;
+ }
+
+////////////////////////////////////////////////////////////////////////////////////
+// Lime, ILDG and Scidac I/O classes
+////////////////////////////////////////////////////////////////////////////////////
+class GridLimeReader : public BinaryIO {
+ public:
+   ///////////////////////////////////////////////////
+   // FIXME: format for RNG? Now just binary out instead
+   ///////////////////////////////////////////////////
+
+   FILE       *File;
+   LimeReader *LimeR;
+   std::string filename;
+
+   /////////////////////////////////////////////
+   // Open the file
+   /////////////////////////////////////////////
+   void open(std::string &_filename) 
+   {
+     filename= _filename;
+     File = fopen(filename.c_str(), "r");
+     LimeR = limeCreateReader(File);
+   }
+   /////////////////////////////////////////////
+   // Close the file
+   /////////////////////////////////////////////
+   void close(void){
+     fclose(File);
+     //     limeDestroyReader(LimeR);
+   }
+
+  ////////////////////////////////////////////
+  // Read a generic lattice field and verify checksum
+  ////////////////////////////////////////////
+  template<class vobj>
+  void readLimeLatticeBinaryObject(Lattice<vobj> &field,std::string record_name)
+  {
+    typedef typename vobj::scalar_object sobj;
+    scidacChecksum scidacChecksum_;
+    uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+
+    std::string format = getFormatString<vobj>();
+
+    while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
+
+      std::cout << GridLogMessage << limeReaderType(LimeR) <<std::endl;
+	
+      if ( strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
+
+
+	off_t offset= ftell(File);
+	BinarySimpleMunger<sobj,sobj> munge;
+	BinaryIO::readLatticeObject< sobj, sobj >(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb);
+
+	/////////////////////////////////////////////
+	// Insist checksum is next record
+	/////////////////////////////////////////////
+	readLimeObject(scidacChecksum_,std::string("scidacChecksum"),record_name);
+
+	/////////////////////////////////////////////
+	// Verify checksums
+	/////////////////////////////////////////////
+	scidacChecksumVerify(scidacChecksum_,scidac_csuma,scidac_csumb);
+	return;
+      }
+    }
+  }
+  ////////////////////////////////////////////
+  // Read a generic serialisable object
+  ////////////////////////////////////////////
+  template<class serialisable_object>
+  void readLimeObject(serialisable_object &object,std::string object_name,std::string record_name)
+  {
+    std::string xmlstring;
+    // should this be a do while; can we miss a first record??
+    while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
+
+      uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration)
+
+      if ( strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
+	std::vector<char> xmlc(nbytes+1,'\0');
+	limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR);    
+	XmlReader RD(&xmlc[0],"");
+	read(RD,object_name,object);
+	return;
+      }
+
+    }  
+    assert(0);
+  }
+};
+
+class GridLimeWriter : public BinaryIO {
+ public:
+   ///////////////////////////////////////////////////
+   // FIXME: format for RNG? Now just binary out instead
+   ///////////////////////////////////////////////////
+
+   FILE       *File;
+   LimeWriter *LimeW;
+   std::string filename;
+
+   void open(std::string &_filename) { 
+     filename= _filename;
+     File = fopen(filename.c_str(), "w");
+     LimeW = limeCreateWriter(File); assert(LimeW != NULL );
+   }
+   /////////////////////////////////////////////
+   // Close the file
+   /////////////////////////////////////////////
+   void close(void) {
+     fclose(File);
+     //  limeDestroyWriter(LimeW);
+   }
+  ///////////////////////////////////////////////////////
+  // Lime utility functions
+  ///////////////////////////////////////////////////////
+  int createLimeRecordHeader(std::string message, int MB, int ME, size_t PayloadSize)
+  {
+    LimeRecordHeader *h;
+    h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
+    assert(limeWriteRecordHeader(h, LimeW) >= 0);
+    limeDestroyHeader(h);
+    return LIME_SUCCESS;
+  }
+  ////////////////////////////////////////////
+  // Write a generic serialisable object
+  ////////////////////////////////////////////
+  template<class serialisable_object>
+  void writeLimeObject(int MB,int ME,serialisable_object &object,std::string object_name,std::string record_name)
+  {
+    std::string xmlstring;
+    {
+      XmlWriter WR("","");
+      write(WR,object_name,object);
+      xmlstring = WR.XmlString();
+    }
+    uint64_t nbytes = xmlstring.size();
+    int err;
+    LimeRecordHeader *h = limeCreateHeader(MB, ME,(char *)record_name.c_str(), nbytes); assert(h!= NULL);
+
+    err=limeWriteRecordHeader(h, LimeW);                    assert(err>=0);
+    err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
+    err=limeWriterCloseRecord(LimeW);                       assert(err>=0);
+    limeDestroyHeader(h);
+  }
+  ////////////////////////////////////////////
+  // Write a generic lattice field and csum
+  ////////////////////////////////////////////
+  template<class vobj>
+  void writeLimeLatticeBinaryObject(Lattice<vobj> &field,std::string record_name)
+  {
+    ////////////////////////////////////////////
+    // Create record header
+    ////////////////////////////////////////////
+    typedef typename vobj::scalar_object sobj;
+    int err;
+    uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+    uint64_t PayloadSize = sizeof(sobj) * field._grid->_gsites;
+    createLimeRecordHeader(record_name, 0, 0, PayloadSize);
+
+    ////////////////////////////////////////////////////////////////////
+    // NB: FILE and iostream are jointly writing disjoint sequences in the
+    // the same file through different file handles (integer units).
+    // 
+    // These are both buffered, so why I think this code is right is as follows.
+    //
+    // i)  write record header to FILE *File, telegraphing the size. 
+    // ii) ftell reads the offset from FILE *File .
+    // iii) iostream / MPI Open independently seek this offset. Write sequence direct to disk.
+    //      Closes iostream and flushes.
+    // iv) fseek on FILE * to end of this disjoint section.
+    //  v) Continue writing scidac record.
+    ////////////////////////////////////////////////////////////////////
+    off_t offset = ftell(File);
+    std::string format = getFormatString<vobj>();
+    BinarySimpleMunger<sobj,sobj> munge;
+    BinaryIO::writeLatticeObject<vobj,sobj>(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb);
+    err=limeWriterCloseRecord(LimeW);  assert(err>=0);
+    ////////////////////////////////////////
+    // Write checksum element, propagaing forward from the BinaryIO
+    // Always pair a checksum with a binary object, and close message
+    ////////////////////////////////////////
+    scidacChecksum checksum;
+    std::stringstream streama; streama << std::hex << scidac_csuma;
+    std::stringstream streamb; streamb << std::hex << scidac_csumb;
+    checksum.suma= streama.str();
+    checksum.sumb= streamb.str();
+    std::cout << GridLogMessage<<" writing scidac checksums "<<std::hex<<scidac_csuma<<"/"<<scidac_csumb<<std::dec<<std::endl;
+    writeLimeObject(0,1,checksum,std::string("scidacChecksum"    ),std::string(SCIDAC_CHECKSUM));
+  }
+};
+
+class ScidacWriter : public GridLimeWriter {
+ public:
+
+   template<class SerialisableUserFile>
+   void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
+   {
+     scidacFile    _scidacFile(grid);
+     writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
+     writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
+   }
+  ////////////////////////////////////////////////
+  // Write generic lattice field in scidac format
+  ////////////////////////////////////////////////
+   template <class vobj, class userRecord>
+  void writeScidacFieldRecord(Lattice<vobj> &field,userRecord _userRecord) 
+  {
+    typedef typename vobj::scalar_object sobj;
+    uint64_t nbytes;
+    GridBase * grid = field._grid;
+
+    ////////////////////////////////////////
+    // fill the Grid header
+    ////////////////////////////////////////
+    FieldMetaData header;
+    scidacRecord  _scidacRecord;
+    scidacFile    _scidacFile;
+
+    ScidacMetaData(field,header,_scidacRecord,_scidacFile);
+
+    //////////////////////////////////////////////
+    // Fill the Lime file record by record
+    //////////////////////////////////////////////
+    writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message 
+    writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
+    writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
+    writeLimeLatticeBinaryObject(field,std::string(ILDG_BINARY_DATA));      // Closes message with checksum
+  }
+};
+
+class IldgWriter : public ScidacWriter {
+ public:
+
+  ///////////////////////////////////
+  // A little helper
+  ///////////////////////////////////
+  void writeLimeIldgLFN(std::string &LFN)
+  {
+    uint64_t PayloadSize = LFN.size();
+    int err;
+    createLimeRecordHeader(ILDG_DATA_LFN, 0 , 0, PayloadSize);
+    err=limeWriteRecordData(const_cast<char*>(LFN.c_str()), &PayloadSize,LimeW); assert(err>=0);
+    err=limeWriterCloseRecord(LimeW); assert(err>=0);
+  }
+
+  ////////////////////////////////////////////////////////////////
+  // Special ILDG operations ; gauge configs only.
+  // Don't require scidac records EXCEPT checksum
+  // Use Grid MetaData object if present.
+  ////////////////////////////////////////////////////////////////
+  template <class vsimd>
+  void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,int sequence,std::string LFN,std::string description) 
+  {
+    GridBase * grid = Umu._grid;
+    typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
+    typedef iLorentzColourMatrix<vsimd> vobj;
+    typedef typename vobj::scalar_object sobj;
+
+    uint64_t nbytes;
+
+    ////////////////////////////////////////
+    // fill the Grid header
+    ////////////////////////////////////////
+    FieldMetaData header;
+    scidacRecord  _scidacRecord;
+    scidacFile    _scidacFile;
+
+    ScidacMetaData(Umu,header,_scidacRecord,_scidacFile);
+
+    std::string format = header.floating_point;
+    header.ensemble_id    = description;
+    header.ensemble_label = description;
+    header.sequence_number = sequence;
+    header.ildg_lfn = LFN;
+
+    assert ( (format == std::string("IEEE32BIG"))  
+           ||(format == std::string("IEEE64BIG")) );
+
+    //////////////////////////////////////////////////////
+    // Fill ILDG header data struct
+    //////////////////////////////////////////////////////
+    ildgFormat ildgfmt ;
+    ildgfmt.field     = std::string("su3gauge");
+
+    if ( format == std::string("IEEE32BIG") ) { 
+      ildgfmt.precision = 32;
+    } else { 
+      ildgfmt.precision = 64;
+    }
+    ildgfmt.version = 1.0;
+    ildgfmt.lx = header.dimension[0];
+    ildgfmt.ly = header.dimension[1];
+    ildgfmt.lz = header.dimension[2];
+    ildgfmt.lt = header.dimension[3];
+    assert(header.nd==4);
+    assert(header.nd==header.dimension.size());
+
+    //////////////////////////////////////////////////////////////////////////////
+    // Fill the USQCD info field
+    //////////////////////////////////////////////////////////////////////////////
+    usqcdInfo info;
+    info.version=1.0;
+    info.plaq   = header.plaquette;
+    info.linktr = header.link_trace;
+
+    std::cout << GridLogMessage << " Writing config; IldgIO "<<std::endl;
+    //////////////////////////////////////////////
+    // Fill the Lime file record by record
+    //////////////////////////////////////////////
+    writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message 
+    writeLimeObject(0,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
+    writeLimeObject(0,1,info,info.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
+    writeLimeObject(1,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
+    writeLimeObject(0,0,info,info.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
+    writeLimeObject(0,0,ildgfmt,std::string("ildgFormat")   ,std::string(ILDG_FORMAT)); // rec
+    writeLimeIldgLFN(header.ildg_lfn);                                                 // rec
+    writeLimeLatticeBinaryObject(Umu,std::string(ILDG_BINARY_DATA));      // Closes message with checksum
+    //    limeDestroyWriter(LimeW);
+    fclose(File);
+  }
+};
+
+class IldgReader : public GridLimeReader {
+ public:
+
+  ////////////////////////////////////////////////////////////////
+  // Read either Grid/SciDAC/ILDG configuration
+  // Don't require scidac records EXCEPT checksum
+  // Use Grid MetaData object if present.
+  // Else use ILDG MetaData object if present.
+  // Else use SciDAC MetaData object if present.
+  ////////////////////////////////////////////////////////////////
+  template <class vsimd>
+  void readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu, FieldMetaData &FieldMetaData_) {
+
+    typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
+    typedef typename GaugeField::vector_object  vobj;
+    typedef typename vobj::scalar_object sobj;
+
+    typedef LorentzColourMatrixF fobj;
+    typedef LorentzColourMatrixD dobj;
+
+    GridBase *grid = Umu._grid;
+
+    std::vector<int> dims = Umu._grid->FullDimensions();
+
+    assert(dims.size()==4);
+
+    // Metadata holders
+    ildgFormat     ildgFormat_    ;
+    std::string    ildgLFN_       ;
+    scidacChecksum scidacChecksum_; 
+    usqcdInfo      usqcdInfo_     ;
+
+    // track what we read from file
+    int found_ildgFormat    =0;
+    int found_ildgLFN       =0;
+    int found_scidacChecksum=0;
+    int found_usqcdInfo     =0;
+    int found_ildgBinary =0;
+    int found_FieldMetaData =0;
+
+    uint32_t nersc_csum;
+    uint32_t scidac_csuma;
+    uint32_t scidac_csumb;
+
+    // Binary format
+    std::string format;
+
+    //////////////////////////////////////////////////////////////////////////
+    // Loop over all records
+    // -- Order is poorly guaranteed except ILDG header preceeds binary section.
+    // -- Run like an event loop.
+    // -- Impose trust hierarchy. Grid takes precedence & look for ILDG, and failing
+    //    that Scidac. 
+    // -- Insist on Scidac checksum record.
+    //////////////////////////////////////////////////////////////////////////
+
+    while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
+
+      uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration)
+      
+      //////////////////////////////////////////////////////////////////
+      // If not BINARY_DATA read a string and parse
+      //////////////////////////////////////////////////////////////////
+      if ( strncmp(limeReaderType(LimeR), ILDG_BINARY_DATA,strlen(ILDG_BINARY_DATA) )  ) {
+	
+	// Copy out the string
+	std::vector<char> xmlc(nbytes+1,'\0');
+	limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR);    
+	std::cout << GridLogMessage<< "Non binary record :" <<limeReaderType(LimeR) <<std::endl; //<<"\n"<<(&xmlc[0])<<std::endl;
+
+	//////////////////////////////////
+	// ILDG format record
+	if ( !strncmp(limeReaderType(LimeR), ILDG_FORMAT,strlen(ILDG_FORMAT)) ) { 
+
+	  XmlReader RD(&xmlc[0],"");
+	  read(RD,"ildgFormat",ildgFormat_);
+
+	  if ( ildgFormat_.precision == 64 ) format = std::string("IEEE64BIG");
+	  if ( ildgFormat_.precision == 32 ) format = std::string("IEEE32BIG");
+
+	  assert( ildgFormat_.lx == dims[0]);
+	  assert( ildgFormat_.ly == dims[1]);
+	  assert( ildgFormat_.lz == dims[2]);
+	  assert( ildgFormat_.lt == dims[3]);
+
+	  found_ildgFormat = 1;
+	}
+
+	if ( !strncmp(limeReaderType(LimeR), ILDG_DATA_LFN,strlen(ILDG_DATA_LFN)) ) {
+	  FieldMetaData_.ildg_lfn = std::string(&xmlc[0]);
+	  found_ildgLFN = 1;
+	}
+
+	if ( !strncmp(limeReaderType(LimeR), GRID_FORMAT,strlen(ILDG_FORMAT)) ) { 
+
+	  XmlReader RD(&xmlc[0],"");
+	  read(RD,"FieldMetaData",FieldMetaData_);
+
+	  format = FieldMetaData_.floating_point;
+
+	  assert(FieldMetaData_.dimension[0] == dims[0]);
+	  assert(FieldMetaData_.dimension[1] == dims[1]);
+	  assert(FieldMetaData_.dimension[2] == dims[2]);
+	  assert(FieldMetaData_.dimension[3] == dims[3]);
+
+	  found_FieldMetaData = 1;
+	}
+
+	if ( !strncmp(limeReaderType(LimeR), SCIDAC_RECORD_XML,strlen(SCIDAC_RECORD_XML)) ) { 
+	  XmlReader RD(&xmlc[0],"");
+	  read(RD,"usqcdInfo",usqcdInfo_);
+	  found_usqcdInfo = 1;
+	}
+
+	if ( !strncmp(limeReaderType(LimeR), SCIDAC_CHECKSUM,strlen(SCIDAC_CHECKSUM)) ) { 
+	  XmlReader RD(&xmlc[0],"");
+	  read(RD,"scidacChecksum",scidacChecksum_);
+	  found_scidacChecksum = 1;
+	}
+
+      } else {  
+	/////////////////////////////////
+	// Binary data
+	/////////////////////////////////
+	std::cout << GridLogMessage << "ILDG Binary record found : "  ILDG_BINARY_DATA << std::endl;
+	off_t offset= ftell(File);
+
+	if ( format == std::string("IEEE64BIG") ) {
+	  GaugeSimpleMunger<dobj, sobj> munge;
+	  BinaryIO::readLatticeObject< vobj, dobj >(Umu, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb);
+	} else { 
+	  GaugeSimpleMunger<fobj, sobj> munge;
+	  BinaryIO::readLatticeObject< vobj, fobj >(Umu, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb);
+	}
+
+	found_ildgBinary = 1;
+      }
+
+    }
+
+    //////////////////////////////////////////////////////
+    // Minimally must find binary segment and checksum
+    // Since this is an ILDG reader require ILDG format
+    //////////////////////////////////////////////////////
+    assert(found_ildgBinary);
+    assert(found_ildgFormat);
+    assert(found_scidacChecksum);
+
+    // Must find something with the lattice dimensions
+    assert(found_FieldMetaData||found_ildgFormat);
+
+    if ( found_FieldMetaData ) {
+
+      std::cout << GridLogMessage<<"Grid MetaData was record found: configuration was probably written by Grid ! Yay ! "<<std::endl;
+
+    } else { 
+
+      assert(found_ildgFormat);
+      assert ( ildgFormat_.field == std::string("su3gauge") );
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // Populate our Grid metadata as best we can
+      ///////////////////////////////////////////////////////////////////////////////////////
+
+      std::ostringstream vers; vers << ildgFormat_.version;
+      FieldMetaData_.hdr_version = vers.str();
+      FieldMetaData_.data_type = std::string("4D_SU3_GAUGE_3X3");
+
+      FieldMetaData_.nd=4;
+      FieldMetaData_.dimension.resize(4);
+
+      FieldMetaData_.dimension[0] = ildgFormat_.lx ;
+      FieldMetaData_.dimension[1] = ildgFormat_.ly ;
+      FieldMetaData_.dimension[2] = ildgFormat_.lz ;
+      FieldMetaData_.dimension[3] = ildgFormat_.lt ;
+
+      if ( found_usqcdInfo ) { 
+	FieldMetaData_.plaquette = usqcdInfo_.plaq;
+	FieldMetaData_.link_trace= usqcdInfo_.linktr;
+	std::cout << GridLogMessage <<"This configuration was probably written by USQCD "<<std::endl;
+	std::cout << GridLogMessage <<"USQCD xml record Plaquette : "<<FieldMetaData_.plaquette<<std::endl;
+	std::cout << GridLogMessage <<"USQCD xml record LinkTrace : "<<FieldMetaData_.link_trace<<std::endl;
+      } else { 
+	FieldMetaData_.plaquette = 0.0;
+	FieldMetaData_.link_trace= 0.0;
+	std::cout << GridLogWarning << "This configuration is unsafe with no plaquette records that can verify it !!! "<<std::endl;
+      }
+    }
+
+    ////////////////////////////////////////////////////////////
+    // Really really want to mandate a scidac checksum
+    ////////////////////////////////////////////////////////////
+    if ( found_scidacChecksum ) {
+      FieldMetaData_.scidac_checksuma = stoull(scidacChecksum_.suma,0,16);
+      FieldMetaData_.scidac_checksumb = stoull(scidacChecksum_.sumb,0,16);
+      scidacChecksumVerify(scidacChecksum_,scidac_csuma,scidac_csumb);
+      assert( scidac_csuma ==FieldMetaData_.scidac_checksuma);
+      assert( scidac_csumb ==FieldMetaData_.scidac_checksumb);
+      std::cout << GridLogMessage<<"SciDAC checksums match " << std::endl;
+    } else { 
+      std::cout << GridLogWarning<<"SciDAC checksums not found. This is unsafe. " << std::endl;
+      assert(0); // Can I insist always checksum ?
+    }
+
+    if ( found_FieldMetaData || found_usqcdInfo ) {
+      FieldMetaData checker;
+      GaugeStatistics(Umu,checker);
+      assert(fabs(checker.plaquette  - FieldMetaData_.plaquette )<1.0e-5);
+      assert(fabs(checker.link_trace - FieldMetaData_.link_trace)<1.0e-5);
+      std::cout << GridLogMessage<<"Plaquette and link trace match " << std::endl;
+    }
+  }
+ };
+
+}}
+
+//HAVE_LIME
+#endif
+
+#endif

lib/parallelIO/IldgIOtypes.h

@@ -0,0 +1,231 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/parallelIO/IldgIO.h
+
+Copyright (C) 2015
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_ILDGTYPES_IO_H
+#define GRID_ILDGTYPES_IO_H
+
+#ifdef HAVE_LIME
+extern "C" { // for linkage
+#include "lime.h"
+}
+
+namespace Grid {
+
+/////////////////////////////////////////////////////////////////////////////////
+// Data representation of records that enter ILDG and SciDac formats
+/////////////////////////////////////////////////////////////////////////////////
+
+#define GRID_FORMAT      "grid-format"
+#define ILDG_FORMAT      "ildg-format"
+#define ILDG_BINARY_DATA "ildg-binary-data"
+#define ILDG_DATA_LFN    "ildg-data-lfn"
+#define SCIDAC_CHECKSUM           "scidac-checksum"
+#define SCIDAC_PRIVATE_FILE_XML   "scidac-private-file-xml"
+#define SCIDAC_FILE_XML           "scidac-file-xml"
+#define SCIDAC_PRIVATE_RECORD_XML "scidac-private-record-xml"
+#define SCIDAC_RECORD_XML         "scidac-record-xml"
+#define SCIDAC_BINARY_DATA        "scidac-binary-data"
+// Unused SCIDAC records names; could move to support this functionality
+#define SCIDAC_SITELIST           "scidac-sitelist"
+
+  ////////////////////////////////////////////////////////////
+  const int GRID_IO_SINGLEFILE = 0; // hardcode lift from QIO compat
+  const int GRID_IO_MULTIFILE  = 1; // hardcode lift from QIO compat
+  const int GRID_IO_FIELD      = 0; // hardcode lift from QIO compat
+  const int GRID_IO_GLOBAL     = 1; // hardcode lift from QIO compat
+  ////////////////////////////////////////////////////////////
+
+/////////////////////////////////////////////////////////////////////////////////
+// QIO uses mandatory "private" records fixed format
+// Private is in principle "opaque" however it can't be changed now because that would break existing 
+// file compatability, so should be correct to assume the undocumented but defacto file structure.
+/////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////
+// Scidac private file xml
+// <?xml version="1.0" encoding="UTF-8"?><scidacFile><version>1.1</version><spacetime>4</spacetime><dims>16 16 16 32 </dims><volfmt>0</volfmt></scidacFile>
+////////////////////////
+struct scidacFile : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(scidacFile,
+                                  double, version,
+                                  int, spacetime,
+				  std::string, dims, // must convert to int
+                                  int, volfmt);
+
+  std::vector<int> getDimensions(void) { 
+    std::stringstream stream(dims);
+    std::vector<int> dimensions;
+    int n;
+    while(stream >> n){
+      dimensions.push_back(n);
+    }
+    return dimensions;
+  }
+
+  void setDimensions(std::vector<int> dimensions) { 
+    char delimiter = ' ';
+    std::stringstream stream;
+    for(int i=0;i<dimensions.size();i++){ 
+      stream << dimensions[i];
+      if ( i != dimensions.size()-1) { 
+	stream << delimiter <<std::endl;
+      }
+    }
+    dims = stream.str();
+  }
+
+  // Constructor provides Grid
+  scidacFile() =default; // default constructor
+  scidacFile(GridBase * grid){
+    version      = 1.0;
+    spacetime    = grid->_ndimension;
+    setDimensions(grid->FullDimensions()); 
+    volfmt       = GRID_IO_SINGLEFILE;
+  }
+
+};
+
+///////////////////////////////////////////////////////////////////////
+// scidac-private-record-xml : example
+// <scidacRecord>
+// <version>1.1</version><date>Tue Jul 26 21:14:44 2011 UTC</date><recordtype>0</recordtype>
+// <datatype>QDP_D3_ColorMatrix</datatype><precision>D</precision><colors>3</colors><spins>4</spins>
+// <typesize>144</typesize><datacount>4</datacount>
+// </scidacRecord>
+///////////////////////////////////////////////////////////////////////
+
+struct scidacRecord : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(scidacRecord,
+                                  double, version,
+                                  std::string, date,
+				  int, recordtype,
+				  std::string, datatype,
+				  std::string, precision,
+				  int, colors,
+				  int, spins,
+				  int, typesize,
+				  int, datacount);
+
+  scidacRecord() { version =1.0; }
+
+};
+
+////////////////////////
+// ILDG format
+////////////////////////
+struct ildgFormat : Serializable {
+public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(ildgFormat,
+				  double, version,
+				  std::string, field,
+				  int, precision,
+				  int, lx,
+				  int, ly,
+				  int, lz,
+				  int, lt);
+  ildgFormat() { version=1.0; };
+};
+////////////////////////
+// USQCD info
+////////////////////////
+struct usqcdInfo : Serializable { 
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdInfo,
+				  double, version,
+				  double, plaq,
+				  double, linktr,
+				  std::string, info);
+  usqcdInfo() { 
+    version=1.0; 
+  };
+};
+////////////////////////
+// Scidac Checksum
+////////////////////////
+struct scidacChecksum : Serializable { 
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(scidacChecksum,
+				  double, version,
+				  std::string, suma,
+				  std::string, sumb);
+  scidacChecksum() { 
+    version=1.0; 
+  };
+};
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Type:           scidac-file-xml         <title>MILC ILDG archival gauge configuration</title>
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Type:           
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////
+// Scidac private file xml 
+// <?xml version="1.0" encoding="UTF-8"?><scidacFile><version>1.1</version><spacetime>4</spacetime><dims>16 16 16 32 </dims><volfmt>0</volfmt></scidacFile> 
+////////////////////////                                                                                                                                                                              
+
+#if 0
+////////////////////////////////////////////////////////////////////////////////////////
+// From http://www.physics.utah.edu/~detar/scidac/qio_2p3.pdf
+////////////////////////////////////////////////////////////////////////////////////////
+struct usqcdPropFile : Serializable { 
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdPropFile,
+				  double, version,
+				  std::string, type,
+				  std::string, info);
+  usqcdPropFile() { 
+    version=1.0; 
+  };
+};
+struct usqcdSourceInfo : Serializable { 
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdSourceInfo,
+				  double, version,
+				  std::string, info);
+  usqcdSourceInfo() { 
+    version=1.0; 
+  };
+};
+struct usqcdPropInfo : Serializable { 
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdPropInfo,
+				  double, version,
+				  int, spin,
+				  int, color,
+				  std::string, info);
+  usqcdPropInfo() { 
+    version=1.0; 
+  };
+};
+#endif
+
+}
+#endif
+#endif

lib/parallelIO/MetaData.h

@@ -0,0 +1,325 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/parallelIO/NerscIO.h
+
+    Copyright (C) 2015
+
+
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <algorithm>
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <map>
+#include <unistd.h>
+#include <sys/utsname.h>
+#include <pwd.h>
+
+namespace Grid {
+
+  ///////////////////////////////////////////////////////
+  // Precision mapping
+  ///////////////////////////////////////////////////////
+  template<class vobj> static std::string getFormatString (void)
+  {
+    std::string format;
+    typedef typename getPrecision<vobj>::real_scalar_type stype;
+    if ( sizeof(stype) == sizeof(float) ) {
+      format = std::string("IEEE32BIG");
+    }
+    if ( sizeof(stype) == sizeof(double) ) {
+      format = std::string("IEEE64BIG");
+    }
+    return format;
+  }
+  ////////////////////////////////////////////////////////////////////////////////
+  // header specification/interpretation
+  ////////////////////////////////////////////////////////////////////////////////
+    class FieldMetaData : Serializable {
+    public:
+
+      GRID_SERIALIZABLE_CLASS_MEMBERS(FieldMetaData,
+				      int, nd,
+				      std::vector<int>, dimension,
+				      std::vector<std::string>, boundary,
+				      int, data_start,
+				      std::string, hdr_version,
+				      std::string, storage_format,
+				      double, link_trace,
+				      double, plaquette,
+				      uint32_t, checksum,
+				      uint32_t, scidac_checksuma,
+				      uint32_t, scidac_checksumb,
+				      unsigned int, sequence_number,
+				      std::string, data_type,
+				      std::string, ensemble_id,
+				      std::string, ensemble_label,
+				      std::string, ildg_lfn,
+				      std::string, creator,
+				      std::string, creator_hardware,
+				      std::string, creation_date,
+				      std::string, archive_date,
+				      std::string, floating_point);
+      FieldMetaData(void) { 
+	nd=4;
+	dimension.resize(4);
+	boundary.resize(4);
+      }
+    };
+
+
+
+  namespace QCD {
+
+    using namespace Grid;
+
+
+    //////////////////////////////////////////////////////////////////////
+    // Bit and Physical Checksumming and QA of data
+    //////////////////////////////////////////////////////////////////////
+    inline void GridMetaData(GridBase *grid,FieldMetaData &header)
+    {
+      int nd = grid->_ndimension;
+      header.nd = nd;
+      header.dimension.resize(nd);
+      header.boundary.resize(nd);
+      for(int d=0;d<nd;d++) {
+	header.dimension[d] = grid->_fdimensions[d];
+      }
+      for(int d=0;d<nd;d++) {
+	header.boundary[d] = std::string("PERIODIC");
+      }
+    }
+
+    inline void MachineCharacteristics(FieldMetaData &header)
+    {
+      // Who
+      struct passwd *pw = getpwuid (getuid());
+      if (pw) header.creator = std::string(pw->pw_name); 
+
+      // When
+      std::time_t t = std::time(nullptr);
+      std::tm tm_ = *std::localtime(&t);
+      std::ostringstream oss; 
+      //      oss << std::put_time(&tm_, "%c %Z");
+      header.creation_date = oss.str();
+      header.archive_date  = header.creation_date;
+
+      // What
+      struct utsname name;  uname(&name);
+      header.creator_hardware = std::string(name.nodename)+"-";
+      header.creator_hardware+= std::string(name.machine)+"-";
+      header.creator_hardware+= std::string(name.sysname)+"-";
+      header.creator_hardware+= std::string(name.release);
+    }
+
+#define dump_meta_data(field, s)					\
+      s << "BEGIN_HEADER"      << std::endl;				\
+      s << "HDR_VERSION = "    << field.hdr_version    << std::endl;	\
+      s << "DATATYPE = "       << field.data_type      << std::endl;	\
+      s << "STORAGE_FORMAT = " << field.storage_format << std::endl;	\
+      for(int i=0;i<4;i++){						\
+	s << "DIMENSION_" << i+1 << " = " << field.dimension[i] << std::endl ; \
+      }									\
+      s << "LINK_TRACE = " << std::setprecision(10) << field.link_trace << std::endl; \
+      s << "PLAQUETTE  = " << std::setprecision(10) << field.plaquette  << std::endl; \
+      for(int i=0;i<4;i++){						\
+	s << "BOUNDARY_"<<i+1<<" = " << field.boundary[i] << std::endl;	\
+      }									\
+									\
+      s << "CHECKSUM = "<< std::hex << std::setw(10) << field.checksum << std::dec<<std::endl; \
+      s << "SCIDAC_CHECKSUMA = "<< std::hex << std::setw(10) << field.scidac_checksuma << std::dec<<std::endl; \
+      s << "SCIDAC_CHECKSUMB = "<< std::hex << std::setw(10) << field.scidac_checksumb << std::dec<<std::endl; \
+      s << "ENSEMBLE_ID = "     << field.ensemble_id      << std::endl;	\
+      s << "ENSEMBLE_LABEL = "  << field.ensemble_label   << std::endl;	\
+      s << "SEQUENCE_NUMBER = " << field.sequence_number  << std::endl;	\
+      s << "CREATOR = "         << field.creator          << std::endl;	\
+      s << "CREATOR_HARDWARE = "<< field.creator_hardware << std::endl;	\
+      s << "CREATION_DATE = "   << field.creation_date    << std::endl;	\
+      s << "ARCHIVE_DATE = "    << field.archive_date     << std::endl;	\
+      s << "FLOATING_POINT = "  << field.floating_point   << std::endl;	\
+      s << "END_HEADER"         << std::endl;
+
+template<class vobj> inline void PrepareMetaData(Lattice<vobj> & field, FieldMetaData &header)
+{
+  GridBase *grid = field._grid;
+  std::string format = getFormatString<vobj>();
+   header.floating_point = format;
+   header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac
+   GridMetaData(grid,header); 
+   MachineCharacteristics(header);
+ }
+ inline void GaugeStatistics(Lattice<vLorentzColourMatrixF> & data,FieldMetaData &header)
+ {
+   // How to convert data precision etc...
+   header.link_trace=Grid::QCD::WilsonLoops<PeriodicGimplF>::linkTrace(data);
+   header.plaquette =Grid::QCD::WilsonLoops<PeriodicGimplF>::avgPlaquette(data);
+ }
+ inline void GaugeStatistics(Lattice<vLorentzColourMatrixD> & data,FieldMetaData &header)
+ {
+   // How to convert data precision etc...
+   header.link_trace=Grid::QCD::WilsonLoops<PeriodicGimplD>::linkTrace(data);
+   header.plaquette =Grid::QCD::WilsonLoops<PeriodicGimplD>::avgPlaquette(data);
+ }
+ template<> inline void PrepareMetaData<vLorentzColourMatrixF>(Lattice<vLorentzColourMatrixF> & field, FieldMetaData &header)
+ {
+   
+   GridBase *grid = field._grid;
+   std::string format = getFormatString<vLorentzColourMatrixF>();
+   header.floating_point = format;
+   header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac
+   GridMetaData(grid,header); 
+   GaugeStatistics(field,header);
+   MachineCharacteristics(header);
+ }
+ template<> inline void PrepareMetaData<vLorentzColourMatrixD>(Lattice<vLorentzColourMatrixD> & field, FieldMetaData &header)
+ {
+   GridBase *grid = field._grid;
+   std::string format = getFormatString<vLorentzColourMatrixD>();
+   header.floating_point = format;
+   header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac
+   GridMetaData(grid,header); 
+   GaugeStatistics(field,header);
+   MachineCharacteristics(header);
+ }
+
+    //////////////////////////////////////////////////////////////////////
+    // Utilities ; these are QCD aware
+    //////////////////////////////////////////////////////////////////////
+    inline void reconstruct3(LorentzColourMatrix & cm)
+    {
+      const int x=0;
+      const int y=1;
+      const int z=2;
+      for(int mu=0;mu<Nd;mu++){
+	cm(mu)()(2,x) = adj(cm(mu)()(0,y)*cm(mu)()(1,z)-cm(mu)()(0,z)*cm(mu)()(1,y)); //x= yz-zy
+	cm(mu)()(2,y) = adj(cm(mu)()(0,z)*cm(mu)()(1,x)-cm(mu)()(0,x)*cm(mu)()(1,z)); //y= zx-xz
+	cm(mu)()(2,z) = adj(cm(mu)()(0,x)*cm(mu)()(1,y)-cm(mu)()(0,y)*cm(mu)()(1,x)); //z= xy-yx
+      }
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+    // Some data types for intermediate storage
+    ////////////////////////////////////////////////////////////////////////////////
+    template<typename vtype> using iLorentzColour2x3 = iVector<iVector<iVector<vtype, Nc>, 2>, Nd >;
+
+    typedef iLorentzColour2x3<Complex>  LorentzColour2x3;
+    typedef iLorentzColour2x3<ComplexF> LorentzColour2x3F;
+    typedef iLorentzColour2x3<ComplexD> LorentzColour2x3D;
+
+/////////////////////////////////////////////////////////////////////////////////
+// Simple classes for precision conversion
+/////////////////////////////////////////////////////////////////////////////////
+template <class fobj, class sobj>
+struct BinarySimpleUnmunger {
+  typedef typename getPrecision<fobj>::real_scalar_type fobj_stype;
+  typedef typename getPrecision<sobj>::real_scalar_type sobj_stype;
+  
+  void operator()(sobj &in, fobj &out) {
+    // take word by word and transform accoding to the status
+    fobj_stype *out_buffer = (fobj_stype *)&out;
+    sobj_stype *in_buffer = (sobj_stype *)&in;
+    size_t fobj_words = sizeof(out) / sizeof(fobj_stype);
+    size_t sobj_words = sizeof(in) / sizeof(sobj_stype);
+    assert(fobj_words == sobj_words);
+    
+    for (unsigned int word = 0; word < sobj_words; word++)
+      out_buffer[word] = in_buffer[word];  // type conversion on the fly
+    
+  }
+};
+
+template <class fobj, class sobj>
+struct BinarySimpleMunger {
+  typedef typename getPrecision<fobj>::real_scalar_type fobj_stype;
+  typedef typename getPrecision<sobj>::real_scalar_type sobj_stype;
+
+  void operator()(fobj &in, sobj &out) {
+    // take word by word and transform accoding to the status
+    fobj_stype *in_buffer = (fobj_stype *)&in;
+    sobj_stype *out_buffer = (sobj_stype *)&out;
+    size_t fobj_words = sizeof(in) / sizeof(fobj_stype);
+    size_t sobj_words = sizeof(out) / sizeof(sobj_stype);
+    assert(fobj_words == sobj_words);
+    
+    for (unsigned int word = 0; word < sobj_words; word++)
+      out_buffer[word] = in_buffer[word];  // type conversion on the fly
+    
+  }
+};
+
+
+    template<class fobj,class sobj>
+    struct GaugeSimpleMunger{
+      void operator()(fobj &in, sobj &out) {
+        for (int mu = 0; mu < Nd; mu++) {
+          for (int i = 0; i < Nc; i++) {
+          for (int j = 0; j < Nc; j++) {
+	    out(mu)()(i, j) = in(mu)()(i, j);
+	  }}
+        }
+      };
+    };
+
+    template <class fobj, class sobj>
+    struct GaugeSimpleUnmunger {
+
+      void operator()(sobj &in, fobj &out) {
+        for (int mu = 0; mu < Nd; mu++) {
+          for (int i = 0; i < Nc; i++) {
+          for (int j = 0; j < Nc; j++) {
+	    out(mu)()(i, j) = in(mu)()(i, j);
+	  }}
+        }
+      };
+    };
+
+    template<class fobj,class sobj>
+    struct Gauge3x2munger{
+      void operator() (fobj &in,sobj &out){
+	for(int mu=0;mu<Nd;mu++){
+	  for(int i=0;i<2;i++){
+	  for(int j=0;j<3;j++){
+	    out(mu)()(i,j) = in(mu)(i)(j);
+	  }}
+	}
+	reconstruct3(out);
+      }
+    };
+
+    template<class fobj,class sobj>
+    struct Gauge3x2unmunger{
+      void operator() (sobj &in,fobj &out){
+	for(int mu=0;mu<Nd;mu++){
+	  for(int i=0;i<2;i++){
+	  for(int j=0;j<3;j++){
+	    out(mu)(i)(j) = in(mu)()(i,j);
+	  }}
+	}
+      }
+    };
+  }
+
+
+}

lib/parallelIO/NerscIO.h

@@ -1,4 +1,4 @@
-    /*************************************************************************************
+/*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
 
@@ -6,9 +6,9 @@
 
     Copyright (C) 2015
 
-Author: Matt Spraggs <matthew.spraggs@gmail.com>
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Matt Spraggs <matthew.spraggs@gmail.com>
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+    Author: paboyle <paboyle@ph.ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -25,253 +25,59 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
     See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
+*************************************************************************************/
+/*  END LEGAL */
 #ifndef GRID_NERSC_IO_H
 #define GRID_NERSC_IO_H
 
-#include <algorithm>
-#include <iostream>
-#include <iomanip>
-#include <fstream>
-#include <map>
-
-#include <unistd.h>
-#include <sys/utsname.h>
-#include <pwd.h>
-
 namespace Grid {
-namespace QCD {
+  namespace QCD {
 
-using namespace Grid;
+    using namespace Grid;
 
-////////////////////////////////////////////////////////////////////////////////
-// Some data types for intermediate storage
-////////////////////////////////////////////////////////////////////////////////
-  template<typename vtype> using iLorentzColour2x3 = iVector<iVector<iVector<vtype, Nc>, 2>, 4 >;
+    ////////////////////////////////////////////////////////////////////////////////
+    // Write and read from fstream; comput header offset for payload
+    ////////////////////////////////////////////////////////////////////////////////
+    class NerscIO : public BinaryIO { 
+    public:
 
-  typedef iLorentzColour2x3<Complex>  LorentzColour2x3;
-  typedef iLorentzColour2x3<ComplexF> LorentzColour2x3F;
-  typedef iLorentzColour2x3<ComplexD> LorentzColour2x3D;
-
-////////////////////////////////////////////////////////////////////////////////
-// header specification/interpretation
-////////////////////////////////////////////////////////////////////////////////
-class NerscField {
- public:
-    // header strings (not in order)
-    int dimension[4];
-    std::string boundary[4]; 
-    int data_start;
-    std::string hdr_version;
-    std::string storage_format;
-    // Checks on data
-    double link_trace;
-    double plaquette;
-    uint32_t checksum;
-    unsigned int sequence_number;
-    std::string data_type;
-    std::string ensemble_id ;
-    std::string ensemble_label ;
-    std::string creator ;
-    std::string creator_hardware ;
-    std::string creation_date ;
-    std::string archive_date ;
-    std::string floating_point;
-};
-
-//////////////////////////////////////////////////////////////////////
-// Bit and Physical Checksumming and QA of data
-//////////////////////////////////////////////////////////////////////
-
-inline void NerscGrid(GridBase *grid,NerscField &header)
-{
-  assert(grid->_ndimension==4);
-  for(int d=0;d<4;d++) {
-    header.dimension[d] = grid->_fdimensions[d];
-  }
-  for(int d=0;d<4;d++) {
-    header.boundary[d] = std::string("PERIODIC");
-  }
-}
-template<class GaugeField>
-inline void NerscStatistics(GaugeField & data,NerscField &header)
-{
-  // How to convert data precision etc...
-  header.link_trace=Grid::QCD::WilsonLoops<PeriodicGimplR>::linkTrace(data);
-  header.plaquette =Grid::QCD::WilsonLoops<PeriodicGimplR>::avgPlaquette(data);
-}
-
-inline void NerscMachineCharacteristics(NerscField &header)
-{
-  // Who
-  struct passwd *pw = getpwuid (getuid());
-  if (pw) header.creator = std::string(pw->pw_name); 
-
-  // When
-  std::time_t t = std::time(nullptr);
-  std::tm tm = *std::localtime(&t);
-  std::ostringstream oss; 
-  //  oss << std::put_time(&tm, "%c %Z");
-  header.creation_date = oss.str();
-  header.archive_date  = header.creation_date;
-
-  // What
-  struct utsname name;  uname(&name);
-  header.creator_hardware = std::string(name.nodename)+"-";
-  header.creator_hardware+= std::string(name.machine)+"-";
-  header.creator_hardware+= std::string(name.sysname)+"-";
-  header.creator_hardware+= std::string(name.release);
-
-}
-//////////////////////////////////////////////////////////////////////
-// Utilities ; these are QCD aware
-//////////////////////////////////////////////////////////////////////
-    inline void NerscChecksum(uint32_t *buf,uint32_t buf_size_bytes,uint32_t &csum)
-    {
-      BinaryIO::Uint32Checksum(buf,buf_size_bytes,csum);
-    }
-    inline void reconstruct3(LorentzColourMatrix & cm)
-    {
-      const int x=0;
-      const int y=1;
-      const int z=2;
-      for(int mu=0;mu<4;mu++){
-	cm(mu)()(2,x) = adj(cm(mu)()(0,y)*cm(mu)()(1,z)-cm(mu)()(0,z)*cm(mu)()(1,y)); //x= yz-zy
-	cm(mu)()(2,y) = adj(cm(mu)()(0,z)*cm(mu)()(1,x)-cm(mu)()(0,x)*cm(mu)()(1,z)); //y= zx-xz
-	cm(mu)()(2,z) = adj(cm(mu)()(0,x)*cm(mu)()(1,y)-cm(mu)()(0,y)*cm(mu)()(1,x)); //z= xy-yx
+      static inline void truncate(std::string file){
+	std::ofstream fout(file,std::ios::out);
       }
+  
+      static inline unsigned int writeHeader(FieldMetaData &field,std::string file)
+      {
+      std::ofstream fout(file,std::ios::out|std::ios::in);
+      fout.seekp(0,std::ios::beg);
+      dump_meta_data(field, fout);
+      field.data_start = fout.tellp();
+      return field.data_start;
     }
 
-    template<class fobj,class sobj>
-    struct NerscSimpleMunger{
+      // for the header-reader
+      static inline int readHeader(std::string file,GridBase *grid,  FieldMetaData &field)
+      {
+      int offset=0;
+      std::map<std::string,std::string> header;
+      std::string line;
 
-      void operator() (fobj &in,sobj &out,uint32_t &csum){
+      //////////////////////////////////////////////////
+      // read the header
+      //////////////////////////////////////////////////
+      std::ifstream fin(file);
 
-      for(int mu=0;mu<4;mu++){
-      for(int i=0;i<3;i++){
-      for(int j=0;j<3;j++){
-	out(mu)()(i,j) = in(mu)()(i,j);
-      }}}
-      NerscChecksum((uint32_t *)&in,sizeof(in),csum); 
-      };
-    };
+      getline(fin,line); // read one line and insist is 
 
-    template<class fobj,class sobj>
-    struct NerscSimpleUnmunger{
-      void operator() (sobj &in,fobj &out,uint32_t &csum){
-	for(int mu=0;mu<Nd;mu++){
-	for(int i=0;i<Nc;i++){
-	for(int j=0;j<Nc;j++){
-	  out(mu)()(i,j) = in(mu)()(i,j);
-	}}}
-	NerscChecksum((uint32_t *)&out,sizeof(out),csum); 
-      };
-    };
- 
-    template<class fobj,class sobj>
-    struct Nersc3x2munger{
-      void operator() (fobj &in,sobj &out,uint32_t &csum){
-     
-	NerscChecksum((uint32_t *)&in,sizeof(in),csum); 
+      removeWhitespace(line);
+      std::cout << GridLogMessage << "* " << line << std::endl;
 
-	for(int mu=0;mu<4;mu++){
-	  for(int i=0;i<2;i++){
-	    for(int j=0;j<3;j++){
-	      out(mu)()(i,j) = in(mu)(i)(j);
-	    }}
-	}
-	reconstruct3(out);
-      }
-    };
+      assert(line==std::string("BEGIN_HEADER"));
 
-    template<class fobj,class sobj>
-    struct Nersc3x2unmunger{
-
-      void operator() (sobj &in,fobj &out,uint32_t &csum){
-
-
-	for(int mu=0;mu<4;mu++){
-	  for(int i=0;i<2;i++){
-	    for(int j=0;j<3;j++){
-	      out(mu)(i)(j) = in(mu)()(i,j);
-	    }}
-	}
-
-	NerscChecksum((uint32_t *)&out,sizeof(out),csum); 
-
-      }
-    };
-
-
-////////////////////////////////////////////////////////////////////////////////
-// Write and read from fstream; comput header offset for payload
-////////////////////////////////////////////////////////////////////////////////
-class NerscIO : public BinaryIO { 
- public:
-
-  static inline void truncate(std::string file){
-    std::ofstream fout(file,std::ios::out);
-  }
-  
-  #define dump_nersc_header(field, s)\
-  s << "BEGIN_HEADER"      << std::endl;\
-  s << "HDR_VERSION = "    << field.hdr_version    << std::endl;\
-  s << "DATATYPE = "       << field.data_type      << std::endl;\
-  s << "STORAGE_FORMAT = " << field.storage_format << std::endl;\
-  for(int i=0;i<4;i++){\
-    s << "DIMENSION_" << i+1 << " = " << field.dimension[i] << std::endl ;\
-  }\
-  s << "LINK_TRACE = " << std::setprecision(10) << field.link_trace << std::endl;\
-  s << "PLAQUETTE  = " << std::setprecision(10) << field.plaquette  << std::endl;\
-  for(int i=0;i<4;i++){\
-    s << "BOUNDARY_"<<i+1<<" = " << field.boundary[i] << std::endl;\
-  }\
-  \
-  s << "CHECKSUM = "<< std::hex << std::setw(10) << field.checksum << std::dec<<std::endl;\
-  s << "ENSEMBLE_ID = "     << field.ensemble_id      << std::endl;\
-  s << "ENSEMBLE_LABEL = "  << field.ensemble_label   << std::endl;\
-  s << "SEQUENCE_NUMBER = " << field.sequence_number  << std::endl;\
-  s << "CREATOR = "         << field.creator          << std::endl;\
-  s << "CREATOR_HARDWARE = "<< field.creator_hardware << std::endl;\
-  s << "CREATION_DATE = "   << field.creation_date    << std::endl;\
-  s << "ARCHIVE_DATE = "    << field.archive_date     << std::endl;\
-  s << "FLOATING_POINT = "  << field.floating_point   << std::endl;\
-  s << "END_HEADER"         << std::endl;
-  
-  static inline unsigned int writeHeader(NerscField &field,std::string file)
-  {
-    std::ofstream fout(file,std::ios::out|std::ios::in);
-    fout.seekp(0,std::ios::beg);
-    dump_nersc_header(field, fout);
-    field.data_start = fout.tellp();
-    return field.data_start;
-}
-
-// for the header-reader
-static inline int readHeader(std::string file,GridBase *grid,  NerscField &field)
-{
-  int offset=0;
-  std::map<std::string,std::string> header;
-  std::string line;
-
-  //////////////////////////////////////////////////
-  // read the header
-  //////////////////////////////////////////////////
-  std::ifstream fin(file);
-
-  getline(fin,line); // read one line and insist is 
-
-  removeWhitespace(line);
-  std::cout << GridLogMessage << "* " << line << std::endl;
-
-  assert(line==std::string("BEGIN_HEADER"));
-
-  do {
-    getline(fin,line); // read one line
-    std::cout << GridLogMessage << "* "<<line<< std::endl;
-    int eq = line.find("=");
-    if(eq >0) {
+      do {
+      getline(fin,line); // read one line
+      std::cout << GridLogMessage << "* "<<line<< std::endl;
+      int eq = line.find("=");
+      if(eq >0) {
       std::string key=line.substr(0,eq);
       std::string val=line.substr(eq+1);
       removeWhitespace(key);
@@ -279,275 +85,269 @@ static inline int readHeader(std::string file,GridBase *grid,  NerscField &field
       
       header[key] = val;
     }
-  } while( line.find("END_HEADER") == std::string::npos );
+    } while( line.find("END_HEADER") == std::string::npos );
 
-  field.data_start = fin.tellg();
+      field.data_start = fin.tellg();
 
-  //////////////////////////////////////////////////
-  // chomp the values
-  //////////////////////////////////////////////////
-  field.hdr_version    = header["HDR_VERSION"];
-  field.data_type      = header["DATATYPE"];
-  field.storage_format = header["STORAGE_FORMAT"];
+      //////////////////////////////////////////////////
+      // chomp the values
+      //////////////////////////////////////////////////
+      field.hdr_version    = header["HDR_VERSION"];
+      field.data_type      = header["DATATYPE"];
+      field.storage_format = header["STORAGE_FORMAT"];
   
-  field.dimension[0] = std::stol(header["DIMENSION_1"]);
-  field.dimension[1] = std::stol(header["DIMENSION_2"]);
-  field.dimension[2] = std::stol(header["DIMENSION_3"]);
-  field.dimension[3] = std::stol(header["DIMENSION_4"]);
+      field.dimension[0] = std::stol(header["DIMENSION_1"]);
+      field.dimension[1] = std::stol(header["DIMENSION_2"]);
+      field.dimension[2] = std::stol(header["DIMENSION_3"]);
+      field.dimension[3] = std::stol(header["DIMENSION_4"]);
 
-  assert(grid->_ndimension == 4);
-  for(int d=0;d<4;d++){
-    assert(grid->_fdimensions[d]==field.dimension[d]);
-  }
-
-  field.link_trace = std::stod(header["LINK_TRACE"]);
-  field.plaquette  = std::stod(header["PLAQUETTE"]);
-
-  field.boundary[0] = header["BOUNDARY_1"];
-  field.boundary[1] = header["BOUNDARY_2"];
-  field.boundary[2] = header["BOUNDARY_3"];
-  field.boundary[3] = header["BOUNDARY_4"];
-
-  field.checksum = std::stoul(header["CHECKSUM"],0,16);
-  field.ensemble_id      = header["ENSEMBLE_ID"];
-  field.ensemble_label   = header["ENSEMBLE_LABEL"];
-  field.sequence_number  = std::stol(header["SEQUENCE_NUMBER"]);
-  field.creator          = header["CREATOR"];
-  field.creator_hardware = header["CREATOR_HARDWARE"];
-  field.creation_date    = header["CREATION_DATE"];
-  field.archive_date     = header["ARCHIVE_DATE"];
-  field.floating_point   = header["FLOATING_POINT"];
-
-  return field.data_start;
-}
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-// Now the meat: the object readers
-/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-#define PARALLEL_READ
-#define PARALLEL_WRITE
-
-template<class vsimd>
-static inline void readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,NerscField& header,std::string file)
-{
-  typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
-
-  GridBase *grid = Umu._grid;
-  int offset = readHeader(file,Umu._grid,header);
-
-  NerscField clone(header);
-
-  std::string format(header.floating_point);
-
-  int ieee32big = (format == std::string("IEEE32BIG"));
-  int ieee32    = (format == std::string("IEEE32"));
-  int ieee64big = (format == std::string("IEEE64BIG"));
-  int ieee64    = (format == std::string("IEEE64"));
-
-  uint32_t csum;
-  // depending on datatype, set up munger;
-  // munger is a function of <floating point, Real, data_type>
-  if ( header.data_type == std::string("4D_SU3_GAUGE") ) {
-    if ( ieee32 || ieee32big ) {
-#ifdef PARALLEL_READ
-      csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>, LorentzColour2x3F> 
-	(Umu,file,Nersc3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format);
-#else
-      csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>, LorentzColour2x3F> 
-	(Umu,file,Nersc3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format);
-#endif
+      assert(grid->_ndimension == 4);
+      for(int d=0;d<4;d++){
+      assert(grid->_fdimensions[d]==field.dimension[d]);
     }
-    if ( ieee64 || ieee64big ) {
-#ifdef PARALLEL_READ
-      csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>, LorentzColour2x3D> 
-      	(Umu,file,Nersc3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format);
-#else 
-      csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>, LorentzColour2x3D> 
-      	(Umu,file,Nersc3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format);
-#endif
+
+      field.link_trace = std::stod(header["LINK_TRACE"]);
+      field.plaquette  = std::stod(header["PLAQUETTE"]);
+
+      field.boundary[0] = header["BOUNDARY_1"];
+      field.boundary[1] = header["BOUNDARY_2"];
+      field.boundary[2] = header["BOUNDARY_3"];
+      field.boundary[3] = header["BOUNDARY_4"];
+
+      field.checksum = std::stoul(header["CHECKSUM"],0,16);
+      field.ensemble_id      = header["ENSEMBLE_ID"];
+      field.ensemble_label   = header["ENSEMBLE_LABEL"];
+      field.sequence_number  = std::stol(header["SEQUENCE_NUMBER"]);
+      field.creator          = header["CREATOR"];
+      field.creator_hardware = header["CREATOR_HARDWARE"];
+      field.creation_date    = header["CREATION_DATE"];
+      field.archive_date     = header["ARCHIVE_DATE"];
+      field.floating_point   = header["FLOATING_POINT"];
+
+      return field.data_start;
     }
-  } else if ( header.data_type == std::string("4D_SU3_GAUGE_3x3") ) {
-    if ( ieee32 || ieee32big ) {
-#ifdef PARALLEL_READ
-      csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>,LorentzColourMatrixF>
-	(Umu,file,NerscSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format);
-#else
-      csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>,LorentzColourMatrixF>
-	(Umu,file,NerscSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format);
-#endif
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    // Now the meat: the object readers
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+    template<class vsimd>
+    static inline void readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,
+					 FieldMetaData& header,
+					 std::string file)
+    {
+      typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
+
+      GridBase *grid = Umu._grid;
+      int offset = readHeader(file,Umu._grid,header);
+
+      FieldMetaData clone(header);
+
+      std::string format(header.floating_point);
+
+      int ieee32big = (format == std::string("IEEE32BIG"));
+      int ieee32    = (format == std::string("IEEE32"));
+      int ieee64big = (format == std::string("IEEE64BIG"));
+      int ieee64    = (format == std::string("IEEE64"));
+
+      uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+      // depending on datatype, set up munger;
+      // munger is a function of <floating point, Real, data_type>
+      if ( header.data_type == std::string("4D_SU3_GAUGE") ) {
+	if ( ieee32 || ieee32big ) {
+	  BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>, LorentzColour2x3F> 
+	    (Umu,file,Gauge3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format,
+	     nersc_csum,scidac_csuma,scidac_csumb);
+	}
+	if ( ieee64 || ieee64big ) {
+	  BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>, LorentzColour2x3D> 
+	    (Umu,file,Gauge3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format,
+	     nersc_csum,scidac_csuma,scidac_csumb);
+	}
+      } else if ( header.data_type == std::string("4D_SU3_GAUGE_3x3") ) {
+	if ( ieee32 || ieee32big ) {
+	  BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>,LorentzColourMatrixF>
+	    (Umu,file,GaugeSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format,
+	     nersc_csum,scidac_csuma,scidac_csumb);
+	}
+	if ( ieee64 || ieee64big ) {
+	  BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>,LorentzColourMatrixD>
+	    (Umu,file,GaugeSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format,
+	     nersc_csum,scidac_csuma,scidac_csumb);
+	}
+      } else {
+	assert(0);
+      }
+
+      GaugeStatistics(Umu,clone);
+
+      std::cout<<GridLogMessage <<"NERSC Configuration "<<file<<" checksum "<<std::hex<<nersc_csum<< std::dec
+	       <<" header   "<<std::hex<<header.checksum<<std::dec <<std::endl;
+      std::cout<<GridLogMessage <<"NERSC Configuration "<<file<<" plaquette "<<clone.plaquette
+	       <<" header    "<<header.plaquette<<std::endl;
+      std::cout<<GridLogMessage <<"NERSC Configuration "<<file<<" link_trace "<<clone.link_trace
+	       <<" header    "<<header.link_trace<<std::endl;
+
+      if ( fabs(clone.plaquette -header.plaquette ) >=  1.0e-5 ) { 
+	std::cout << " Plaquette mismatch "<<std::endl;
+	std::cout << Umu[0]<<std::endl;
+	std::cout << Umu[1]<<std::endl;
+      }
+      if ( nersc_csum != header.checksum ) { 
+	std::cerr << " checksum mismatch " << std::endl;
+	std::cerr << " plaqs " << clone.plaquette << " " << header.plaquette << std::endl;
+	std::cerr << " trace " << clone.link_trace<< " " << header.link_trace<< std::endl;
+	std::cerr << " nersc_csum  " <<std::hex<< nersc_csum << " " << header.checksum<< std::dec<< std::endl;
+	exit(0);
+      }
+      assert(fabs(clone.plaquette -header.plaquette ) < 1.0e-5 );
+      assert(fabs(clone.link_trace-header.link_trace) < 1.0e-6 );
+      assert(nersc_csum == header.checksum );
+      
+      std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
     }
-    if ( ieee64 || ieee64big ) {
-#ifdef PARALLEL_READ
-      csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>,LorentzColourMatrixD>
-	(Umu,file,NerscSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format);
-#else
-      csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>,LorentzColourMatrixD>
-	(Umu,file,NerscSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format);
-#endif
-    }
-  } else {
-    assert(0);
-  }
 
-  NerscStatistics<GaugeField>(Umu,clone);
+      template<class vsimd>
+      static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,
+					    std::string file, 
+					    int two_row,
+					    int bits32)
+      {
+	typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
 
-  std::cout<<GridLogMessage <<"NERSC Configuration "<<file<<" checksum "<<std::hex<<            csum<< std::dec
-	                                                  <<" header   "<<std::hex<<header.checksum<<std::dec <<std::endl;
-  std::cout<<GridLogMessage <<"NERSC Configuration "<<file<<" plaquette "<<clone.plaquette
-	                                                  <<" header    "<<header.plaquette<<std::endl;
-  std::cout<<GridLogMessage <<"NERSC Configuration "<<file<<" link_trace "<<clone.link_trace
-	                                                  <<" header    "<<header.link_trace<<std::endl;
-  assert(fabs(clone.plaquette -header.plaquette ) < 1.0e-5 );
-  assert(fabs(clone.link_trace-header.link_trace) < 1.0e-6 );
-  assert(csum == header.checksum );
+	typedef iLorentzColourMatrix<vsimd> vobj;
+	typedef typename vobj::scalar_object sobj;
 
-  std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
-}
+	FieldMetaData header;
+	///////////////////////////////////////////
+	// Following should become arguments
+	///////////////////////////////////////////
+	header.sequence_number = 1;
+	header.ensemble_id     = "UKQCD";
+	header.ensemble_label  = "DWF";
 
-template<class vsimd>
-static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,std::string file, int two_row,int bits32)
-{
-  typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
-
-  typedef iLorentzColourMatrix<vsimd> vobj;
-  typedef typename vobj::scalar_object sobj;
-
-  // Following should become arguments
-  NerscField header;
-  header.sequence_number = 1;
-  header.ensemble_id     = "UKQCD";
-  header.ensemble_label  = "DWF";
-
-  typedef LorentzColourMatrixD fobj3D;
-  typedef LorentzColour2x3D    fobj2D;
-  typedef LorentzColourMatrixF fobj3f;
-  typedef LorentzColour2x3F    fobj2f;
-
-  GridBase *grid = Umu._grid;
-
-  NerscGrid(grid,header);
-  NerscStatistics<GaugeField>(Umu,header);
-  NerscMachineCharacteristics(header);
-
-  uint32_t csum;
-  int offset;
+	typedef LorentzColourMatrixD fobj3D;
+	typedef LorentzColour2x3D    fobj2D;
   
-  truncate(file);
+	GridBase *grid = Umu._grid;
 
-  if ( two_row ) { 
+	GridMetaData(grid,header);
+	assert(header.nd==4);
+	GaugeStatistics(Umu,header);
+	MachineCharacteristics(header);
 
-    header.floating_point = std::string("IEEE64BIG");
-    header.data_type      = std::string("4D_SU3_GAUGE");
-    Nersc3x2unmunger<fobj2D,sobj> munge;
-    BinaryIO::Uint32Checksum<vobj,fobj2D>(Umu, munge,header.checksum);
-    offset = writeHeader(header,file);
-#ifdef PARALLEL_WRITE
-    csum=BinaryIO::writeObjectParallel<vobj,fobj2D>(Umu,file,munge,offset,header.floating_point);
-#else
-    csum=BinaryIO::writeObjectSerial<vobj,fobj2D>(Umu,file,munge,offset,header.floating_point);
-#endif
+	int offset;
+  
+	truncate(file);
 
-  } else { 
-    header.floating_point = std::string("IEEE64BIG");
-    header.data_type      = std::string("4D_SU3_GAUGE_3x3");
-    NerscSimpleUnmunger<fobj3D,sobj> munge;
-    BinaryIO::Uint32Checksum<vobj,fobj3D>(Umu, munge,header.checksum);
-    offset = writeHeader(header,file);
-#ifdef PARALLEL_WRITE
-    csum=BinaryIO::writeObjectParallel<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point);
-#else
-    csum=BinaryIO::writeObjectSerial<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point);
-#endif
-  }
+	// Sod it -- always write 3x3 double
+	header.floating_point = std::string("IEEE64BIG");
+	header.data_type      = std::string("4D_SU3_GAUGE_3x3");
+	GaugeSimpleUnmunger<fobj3D,sobj> munge;
+	offset = writeHeader(header,file);
 
-  std::cout<<GridLogMessage <<"Written NERSC Configuration "<<file<< " checksum "<<std::hex<<csum<< std::dec<<" plaq "<< header.plaquette <<std::endl;
+	uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+	BinaryIO::writeLatticeObject<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point,
+								  nersc_csum,scidac_csuma,scidac_csumb);
+	header.checksum = nersc_csum;
+	writeHeader(header,file);
 
- }
+	std::cout<<GridLogMessage <<"Written NERSC Configuration on "<< file << " checksum "
+		 <<std::hex<<header.checksum
+		 <<std::dec<<" plaq "<< header.plaquette <<std::endl;
 
+      }
+      ///////////////////////////////
+      // RNG state
+      ///////////////////////////////
+      static inline void writeRNGState(GridSerialRNG &serial,GridParallelRNG &parallel,std::string file)
+      {
+	typedef typename GridParallelRNG::RngStateType RngStateType;
 
-    ///////////////////////////////
-    // RNG state
-    ///////////////////////////////
-static inline void writeRNGState(GridSerialRNG &serial,GridParallelRNG &parallel,std::string file)
-{
-  typedef typename GridParallelRNG::RngStateType RngStateType;
+	// Following should become arguments
+	FieldMetaData header;
+	header.sequence_number = 1;
+	header.ensemble_id     = "UKQCD";
+	header.ensemble_label  = "DWF";
 
-  // Following should become arguments
-  NerscField header;
-  header.sequence_number = 1;
-  header.ensemble_id     = "UKQCD";
-  header.ensemble_label  = "DWF";
+	GridBase *grid = parallel._grid;
 
-  GridBase *grid = parallel._grid;
+	GridMetaData(grid,header);
+	assert(header.nd==4);
+	header.link_trace=0.0;
+	header.plaquette=0.0;
+	MachineCharacteristics(header);
 
-  NerscGrid(grid,header);
-  header.link_trace=0.0;
-  header.plaquette=0.0;
-  NerscMachineCharacteristics(header);
-
-  uint32_t csum;
-  int offset;
+	int offset;
   
 #ifdef RNG_RANLUX
-    header.floating_point = std::string("UINT64");
-    header.data_type      = std::string("RANLUX48");
+	header.floating_point = std::string("UINT64");
+	header.data_type      = std::string("RANLUX48");
 #endif
 #ifdef RNG_MT19937
-    header.floating_point = std::string("UINT32");
-    header.data_type      = std::string("MT19937");
+	header.floating_point = std::string("UINT32");
+	header.data_type      = std::string("MT19937");
 #endif
 #ifdef RNG_SITMO
-    header.floating_point = std::string("UINT64");
-    header.data_type      = std::string("SITMO");
+	header.floating_point = std::string("UINT64");
+	header.data_type      = std::string("SITMO");
 #endif
 
-  truncate(file);
-  offset = writeHeader(header,file);
-  csum=BinaryIO::writeRNGSerial(serial,parallel,file,offset);
-  header.checksum = csum;
-  offset = writeHeader(header,file);
+	truncate(file);
+	offset = writeHeader(header,file);
+	uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+	BinaryIO::writeRNG(serial,parallel,file,offset,nersc_csum,scidac_csuma,scidac_csumb);
+	header.checksum = nersc_csum;
+	offset = writeHeader(header,file);
 
-  std::cout<<GridLogMessage <<"Written NERSC RNG STATE "<<file<< " checksum "<<std::hex<<csum<<std::dec<<std::endl;
+	std::cout<<GridLogMessage 
+		 <<"Written NERSC RNG STATE "<<file<< " checksum "
+		 <<std::hex<<header.checksum
+		 <<std::dec<<std::endl;
 
- }
+      }
     
-static inline void readRNGState(GridSerialRNG &serial,GridParallelRNG & parallel,NerscField& header,std::string file)
-{
-  typedef typename GridParallelRNG::RngStateType RngStateType;
+      static inline void readRNGState(GridSerialRNG &serial,GridParallelRNG & parallel,FieldMetaData& header,std::string file)
+      {
+	typedef typename GridParallelRNG::RngStateType RngStateType;
 
-  GridBase *grid = parallel._grid;
+	GridBase *grid = parallel._grid;
 
-  int offset = readHeader(file,grid,header);
+	int offset = readHeader(file,grid,header);
 
-  NerscField clone(header);
+	FieldMetaData clone(header);
 
-  std::string format(header.floating_point);
-  std::string data_type(header.data_type);
+	std::string format(header.floating_point);
+	std::string data_type(header.data_type);
 
 #ifdef RNG_RANLUX
-  assert(format == std::string("UINT64"));
-  assert(data_type == std::string("RANLUX48"));
+	assert(format == std::string("UINT64"));
+	assert(data_type == std::string("RANLUX48"));
 #endif
 #ifdef RNG_MT19937
-  assert(format == std::string("UINT32"));
-  assert(data_type == std::string("MT19937"));
+	assert(format == std::string("UINT32"));
+	assert(data_type == std::string("MT19937"));
 #endif
 #ifdef RNG_SITMO
-  assert(format == std::string("UINT64"));
-  assert(data_type == std::string("SITMO"));
+	assert(format == std::string("UINT64"));
+	assert(data_type == std::string("SITMO"));
 #endif
 
-  // depending on datatype, set up munger;
-  // munger is a function of <floating point, Real, data_type>
-  uint32_t csum=BinaryIO::readRNGSerial(serial,parallel,file,offset);
+	// depending on datatype, set up munger;
+	// munger is a function of <floating point, Real, data_type>
+	uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+	BinaryIO::readRNG(serial,parallel,file,offset,nersc_csum,scidac_csuma,scidac_csumb);
 
-  assert(csum == header.checksum );
+	if ( nersc_csum != header.checksum ) { 
+	  std::cerr << "checksum mismatch "<<std::hex<< nersc_csum <<" "<<header.checksum<<std::dec<<std::endl;
+	  exit(0);
+	}
+	assert(nersc_csum == header.checksum );
 
-  std::cout<<GridLogMessage <<"Read NERSC RNG file "<<file<< " format "<< data_type <<std::endl;
-}
+	std::cout<<GridLogMessage <<"Read NERSC RNG file "<<file<< " format "<< data_type <<std::endl;
+      }
 
-};
+    };
 
-
-}}
+  }}
 #endif

lib/perfmon/PerfCount.h

@@ -205,13 +205,14 @@ public:
   void Stop(void) {
     count=0;
     cycles=0;
-    size_t ign;
 #ifdef __linux__
+    ssize_t ign;
     if ( fd!= -1) {
       ::ioctl(fd, PERF_EVENT_IOC_DISABLE, 0);
       ::ioctl(cyclefd, PERF_EVENT_IOC_DISABLE, 0);
       ign=::read(fd, &count, sizeof(long long));
-      ign=::read(cyclefd, &cycles, sizeof(long long));
+      ign+=::read(cyclefd, &cycles, sizeof(long long));
+      assert(ign=2*sizeof(long long));
     }
     elapsed = cyclecount() - begin;
 #else

lib/qcd/LatticeTheories.h

@@ -0,0 +1,124 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/QCD.h
+
+Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: neo <cossu@post.kek.jp>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_LT_H
+#define GRID_LT_H
+namespace Grid{
+
+// First steps in the complete generalization of the Physics part
+// Design not final
+namespace LatticeTheories {
+
+template <int Dimensions>
+struct LatticeTheory {
+  static const int Nd = Dimensions;
+  static const int Nds = Dimensions * 2;  // double stored field
+  template <typename vtype>
+  using iSinglet = iScalar<iScalar<iScalar<vtype> > >;
+};
+
+template <int Dimensions, int Colours>
+struct LatticeGaugeTheory : public LatticeTheory<Dimensions> {
+  static const int Nds = Dimensions * 2;
+  static const int Nd = Dimensions;
+  static const int Nc = Colours;
+
+  template <typename vtype> 
+  using iColourMatrix = iScalar<iScalar<iMatrix<vtype, Nc> > >;
+  template <typename vtype>
+  using iLorentzColourMatrix = iVector<iScalar<iMatrix<vtype, Nc> >, Nd>;
+  template <typename vtype>
+  using iDoubleStoredColourMatrix = iVector<iScalar<iMatrix<vtype, Nc> >, Nds>;
+  template <typename vtype>
+  using iColourVector = iScalar<iScalar<iVector<vtype, Nc> > >;
+};
+
+template <int Dimensions, int Colours, int Spin>
+struct FermionicLatticeGaugeTheory
+    : public LatticeGaugeTheory<Dimensions, Colours> {
+  static const int Nd = Dimensions;
+  static const int Nds = Dimensions * 2;
+  static const int Nc = Colours;
+  static const int Ns = Spin;
+
+  template <typename vtype>
+  using iSpinMatrix = iScalar<iMatrix<iScalar<vtype>, Ns> >;
+  template <typename vtype>
+  using iSpinColourMatrix = iScalar<iMatrix<iMatrix<vtype, Nc>, Ns> >;
+  template <typename vtype>
+  using iSpinVector = iScalar<iVector<iScalar<vtype>, Ns> >;
+  template <typename vtype>
+  using iSpinColourVector = iScalar<iVector<iVector<vtype, Nc>, Ns> >;
+  // These 2 only if Spin is a multiple of 2
+  static const int Nhs = Spin / 2;
+  template <typename vtype>
+  using iHalfSpinVector = iScalar<iVector<iScalar<vtype>, Nhs> >;
+  template <typename vtype>
+  using iHalfSpinColourVector = iScalar<iVector<iVector<vtype, Nc>, Nhs> >;
+
+  //tests
+  typedef iColourMatrix<Complex> ColourMatrix;
+  typedef iColourMatrix<ComplexF> ColourMatrixF;
+  typedef iColourMatrix<ComplexD> ColourMatrixD;
+
+
+};
+
+// Examples, not complete now.
+struct QCD : public FermionicLatticeGaugeTheory<4, 3, 4> {
+    static const int Xp = 0;
+    static const int Yp = 1;
+    static const int Zp = 2;
+    static const int Tp = 3;
+    static const int Xm = 4;
+    static const int Ym = 5;
+    static const int Zm = 6;
+    static const int Tm = 7;
+
+    typedef FermionicLatticeGaugeTheory FLGT;
+
+    typedef FLGT::iSpinMatrix<Complex  >          SpinMatrix;
+    typedef FLGT::iSpinMatrix<ComplexF >          SpinMatrixF;
+    typedef FLGT::iSpinMatrix<ComplexD >          SpinMatrixD;
+
+};
+struct QED : public FermionicLatticeGaugeTheory<4, 1, 4> {//fill
+};
+
+template <int Dimensions>
+struct Scalar : public LatticeTheory<Dimensions> {};
+
+};  // LatticeTheories
+
+} // Grid
+
+#endif

lib/qcd/QCD.h

@@ -32,9 +32,12 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_QCD_BASE_H
 #define GRID_QCD_BASE_H
 namespace Grid{
-
 namespace QCD {
 
+    static const int Xdir = 0;
+    static const int Ydir = 1;
+    static const int Zdir = 2;
+    static const int Tdir = 3;
 
     static const int Xp = 0;
     static const int Yp = 1;
@@ -354,36 +357,36 @@ namespace QCD {
     //////////////////////////////////////////////
     template<class vobj> 
       void pokeColour(Lattice<vobj> &lhs,
-		      const Lattice<decltype(peekIndex<ColourIndex>(lhs._odata[0],0))> & rhs,
-		      int i)
+              const Lattice<decltype(peekIndex<ColourIndex>(lhs._odata[0],0))> & rhs,
+              int i)
     {
       PokeIndex<ColourIndex>(lhs,rhs,i);
     }
     template<class vobj> 
       void pokeColour(Lattice<vobj> &lhs,
-		      const Lattice<decltype(peekIndex<ColourIndex>(lhs._odata[0],0,0))> & rhs,
-		      int i,int j)
+              const Lattice<decltype(peekIndex<ColourIndex>(lhs._odata[0],0,0))> & rhs,
+              int i,int j)
     {
       PokeIndex<ColourIndex>(lhs,rhs,i,j);
     }
     template<class vobj> 
       void pokeSpin(Lattice<vobj> &lhs,
-		      const Lattice<decltype(peekIndex<SpinIndex>(lhs._odata[0],0))> & rhs,
-		      int i)
+              const Lattice<decltype(peekIndex<SpinIndex>(lhs._odata[0],0))> & rhs,
+              int i)
     {
       PokeIndex<SpinIndex>(lhs,rhs,i);
     }
     template<class vobj> 
       void pokeSpin(Lattice<vobj> &lhs,
-		      const Lattice<decltype(peekIndex<SpinIndex>(lhs._odata[0],0,0))> & rhs,
-		      int i,int j)
+              const Lattice<decltype(peekIndex<SpinIndex>(lhs._odata[0],0,0))> & rhs,
+              int i,int j)
     {
       PokeIndex<SpinIndex>(lhs,rhs,i,j);
     }
     template<class vobj> 
       void pokeLorentz(Lattice<vobj> &lhs,
-		      const Lattice<decltype(peekIndex<LorentzIndex>(lhs._odata[0],0))> & rhs,
-		      int i)
+              const Lattice<decltype(peekIndex<LorentzIndex>(lhs._odata[0],0))> & rhs,
+              int i)
     {
       PokeIndex<LorentzIndex>(lhs,rhs,i);
     }
@@ -492,6 +495,38 @@ namespace QCD {
 }   //namespace QCD
 } // Grid
 
+/*
+<<<<<<< HEAD
+#include <Grid/qcd/utils/SpaceTimeGrid.h>
+#include <Grid/qcd/spin/Dirac.h>
+#include <Grid/qcd/spin/TwoSpinor.h>
+#include <Grid/qcd/utils/LinalgUtils.h>
+#include <Grid/qcd/utils/CovariantCshift.h>
+
+// Include representations  
+#include <Grid/qcd/utils/SUn.h>
+#include <Grid/qcd/utils/SUnAdjoint.h>
+#include <Grid/qcd/utils/SUnTwoIndex.h>
+#include <Grid/qcd/representations/hmc_types.h>
+
+// Scalar field
+#include <Grid/qcd/utils/ScalarObjs.h>
+
+#include <Grid/qcd/action/Actions.h>
+
+#include <Grid/qcd/smearing/Smearing.h>
+
+#include <Grid/qcd/hmc/integrators/Integrator.h>
+#include <Grid/qcd/hmc/integrators/Integrator_algorithm.h>
+#include <Grid/qcd/observables/hmc_observable.h>
+#include <Grid/qcd/hmc/HMC.h>
+
+
+//#include <Grid/qcd/modules/mods.h>
+=======
+
+>>>>>>> develop
+*/
 
 
 #endif

lib/qcd/action/ActionBase.h

@@ -4,10 +4,11 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: ./lib/qcd/action/ActionBase.h
 
-Copyright (C) 2015
+Copyright (C) 2015-2016
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: neo <cossu@post.kek.jp>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -27,128 +28,29 @@ See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
 /*  END LEGAL */
-#ifndef QCD_ACTION_BASE
-#define QCD_ACTION_BASE
+
+#ifndef ACTION_BASE_H
+#define ACTION_BASE_H
+
 namespace Grid {
 namespace QCD {
 
-template <class GaugeField>
-class Action {
+template <class GaugeField >
+class Action 
+{
+
  public:
   bool is_smeared = false;
-  // Boundary conditions? // Heatbath?
-  virtual void refresh(const GaugeField& U,
-                       GridParallelRNG& pRNG) = 0;  // refresh pseudofermions
-  virtual RealD S(const GaugeField& U) = 0;         // evaluate the action
-  virtual void deriv(const GaugeField& U,
-                     GaugeField& dSdU) = 0;  // evaluate the action derivative
-  virtual ~Action(){};
+  // Heatbath?
+  virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
+  virtual RealD S(const GaugeField& U) = 0;                             // evaluate the action
+  virtual void deriv(const GaugeField& U, GaugeField& dSdU) = 0;        // evaluate the action derivative
+  virtual std::string action_name()    = 0;                             // return the action name
+  virtual std::string LogParameters()  = 0;                             // prints action parameters
+  virtual ~Action(){}
 };
 
-// Indexing of tuple types
-template <class T, class Tuple>
-struct Index;
-
-template <class T, class... Types>
-struct Index<T, std::tuple<T, Types...>> {
-  static const std::size_t value = 0;
-};
-
-template <class T, class U, class... Types>
-struct Index<T, std::tuple<U, Types...>> {
-  static const std::size_t value = 1 + Index<T, std::tuple<Types...>>::value;
-};
-
-/*
-template <class GaugeField>
-struct ActionLevel {
- public:
-  typedef Action<GaugeField>*
-      ActPtr;  // now force the same colours as the rest of the code
-
-  //Add supported representations here
-
-
-  unsigned int multiplier;
-
-  std::vector<ActPtr> actions;
-
-  ActionLevel(unsigned int mul = 1) : actions(0), multiplier(mul) {
-    assert(mul >= 1);
-  };
-
-  void push_back(ActPtr ptr) { actions.push_back(ptr); }
-};
-*/
-
-template <class GaugeField, class Repr = NoHirep >
-struct ActionLevel {
- public:
-  unsigned int multiplier; 
-
-  // Fundamental repr actions separated because of the smearing
-  typedef Action<GaugeField>* ActPtr;
-
-  // construct a tuple of vectors of the actions for the corresponding higher
-  // representation fields
-  typedef typename AccessTypes<Action, Repr>::VectorCollection action_collection;
-  action_collection actions_hirep;
-  typedef typename  AccessTypes<Action, Repr>::FieldTypeCollection action_hirep_types;
-
-  std::vector<ActPtr>& actions;
-
-  // Temporary conversion between ActionLevel and ActionLevelHirep
-  //ActionLevelHirep(ActionLevel<GaugeField>& AL ):actions(AL.actions), multiplier(AL.multiplier){}
-
-  ActionLevel(unsigned int mul = 1) : actions(std::get<0>(actions_hirep)), multiplier(mul) {
-    // initialize the hirep vectors to zero.
-    //apply(this->resize, actions_hirep, 0); //need a working resize
-    assert(mul >= 1);
-  };
-
-  //void push_back(ActPtr ptr) { actions.push_back(ptr); }
-
-
-
-  template < class Field >
-  void push_back(Action<Field>* ptr) {
-    // insert only in the correct vector
-    std::get< Index < Field, action_hirep_types>::value >(actions_hirep).push_back(ptr);
-  };
-
- 
-
-  template < class ActPtr>
-  static void resize(ActPtr ap, unsigned int n){
-    ap->resize(n);
-
-  }
-
-  //template <std::size_t I>
-  //auto getRepresentation(Repr& R)->decltype(std::get<I>(R).U)  {return std::get<I>(R).U;}
-
-  // Loop on tuple for a callable function
-  template <std::size_t I = 1, typename Callable, typename ...Args>
-  inline typename std::enable_if<I == std::tuple_size<action_collection>::value, void>::type apply(
-      Callable, Repr& R,Args&...) const {}
-
-  template <std::size_t I = 1, typename Callable, typename ...Args>
-  inline typename std::enable_if<I < std::tuple_size<action_collection>::value, void>::type apply(
-      Callable fn, Repr& R, Args&... arguments) const {
-    fn(std::get<I>(actions_hirep), std::get<I>(R.rep), arguments...);
-    apply<I + 1>(fn, R, arguments...);
-  }  
-
-};
-
-
-//template <class GaugeField>
-//using ActionSet = std::vector<ActionLevel<GaugeField> >;
-
-template <class GaugeField, class R>
-using ActionSet = std::vector<ActionLevel<GaugeField, R> >;
-
 }
 }
 
-#endif
+#endif // ACTION_BASE_H

lib/qcd/action/ActionCore.h

@@ -31,15 +31,31 @@ directory
 #define QCD_ACTION_CORE
 
 #include <Grid/qcd/action/ActionBase.h>
+#include <Grid/qcd/action/ActionSet.h>
 #include <Grid/qcd/action/ActionParams.h>
 
 ////////////////////////////////////////////
 // Gauge Actions
 ////////////////////////////////////////////
 #include <Grid/qcd/action/gauge/Gauge.h>
+
 ////////////////////////////////////////////
 // Fermion prereqs
 ////////////////////////////////////////////
 #include <Grid/qcd/action/fermion/FermionCore.h>
 
+////////////////////////////////////////////
+// Scalar Actions
+////////////////////////////////////////////
+#include <Grid/qcd/action/scalar/Scalar.h>
+
+////////////////////////////////////////////
+// Utility functions
+////////////////////////////////////////////
+#include <Grid/qcd/utils/Metric.h>
+#include <Grid/qcd/utils/CovariantLaplacian.h>
+
+
+
+
 #endif

lib/qcd/action/ActionParams.h

@@ -1,67 +1,92 @@
-    /*************************************************************************************
+/*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+Grid physics library, www.github.com/paboyle/Grid
 
-    Source file: ./lib/qcd/action/ActionParams.h
+Source file: ./lib/qcd/action/ActionParams.h
 
-    Copyright (C) 2015
+Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
 
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
 
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
 
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
 #ifndef GRID_QCD_ACTION_PARAMS_H
 #define GRID_QCD_ACTION_PARAMS_H
 
 namespace Grid {
 namespace QCD {
 
-    // These can move into a params header and be given MacroMagic serialisation
-    struct GparityWilsonImplParams {
-      bool overlapCommsCompute;
-      std::vector<int> twists; 
-      GparityWilsonImplParams () : twists(Nd,0), overlapCommsCompute(false) {};
-
+  // These can move into a params header and be given MacroMagic serialisation
+  struct GparityWilsonImplParams {
+    bool overlapCommsCompute;
+    std::vector<int> twists;
+    GparityWilsonImplParams() : twists(Nd, 0), overlapCommsCompute(false){};
+  };
+  
+  struct WilsonImplParams {
+    bool overlapCommsCompute;
+    std::vector<Complex> boundary_phases;
+    WilsonImplParams() : overlapCommsCompute(false) {
+      boundary_phases.resize(Nd, 1.0);
     };
+    WilsonImplParams(const std::vector<Complex> phi)
+      : boundary_phases(phi), overlapCommsCompute(false) {}
+  };
 
-    struct WilsonImplParams {
-      bool overlapCommsCompute;
-      WilsonImplParams() : overlapCommsCompute(false) {};
-    };
+  struct StaggeredImplParams {
+    StaggeredImplParams()  {};
+  };
+  
+  struct OneFlavourRationalParams : Serializable {
+    GRID_SERIALIZABLE_CLASS_MEMBERS(OneFlavourRationalParams, 
+				    RealD, lo, 
+				    RealD, hi, 
+				    int,   MaxIter, 
+				    RealD, tolerance, 
+				    int,   degree, 
+				    int,   precision);
+    
+    // MaxIter and tolerance, vectors??
+    
+    // constructor 
+    OneFlavourRationalParams(	RealD _lo      = 0.0, 
+				RealD _hi      = 1.0, 
+				int _maxit     = 1000,
+				RealD tol      = 1.0e-8, 
+                           	int _degree    = 10,
+				int _precision = 64)
+      : lo(_lo),
+	hi(_hi),
+	MaxIter(_maxit),
+	tolerance(tol),
+	degree(_degree),
+	precision(_precision){};
+  };
+  
+  
+}
+}
 
-    struct StaggeredImplParams {
-      StaggeredImplParams()  {};
-    };
 
-    struct OneFlavourRationalParams { 
-      RealD  lo;
-      RealD  hi;
-      int MaxIter;   // Vector?
-      RealD tolerance; // Vector? 
-      int    degree=10;
-      int precision=64;
 
-      OneFlavourRationalParams (RealD _lo,RealD _hi,int _maxit,RealD tol=1.0e-8,int _degree = 10,int _precision=64) :
-        lo(_lo), hi(_hi), MaxIter(_maxit), tolerance(tol), degree(_degree), precision(_precision)
-      {};
-    };
-
-}}
 
 #endif

lib/qcd/action/ActionSet.h

@@ -0,0 +1,116 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/ActionSet.h
+
+Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef ACTION_SET_H
+#define ACTION_SET_H
+
+namespace Grid {
+
+// Should drop this namespace here
+namespace QCD {
+
+//////////////////////////////////
+// Indexing of tuple types
+//////////////////////////////////
+
+template <class T, class Tuple>
+struct Index;
+
+template <class T, class... Types>
+struct Index<T, std::tuple<T, Types...>> {
+  static const std::size_t value = 0;
+};
+
+template <class T, class U, class... Types>
+struct Index<T, std::tuple<U, Types...>> {
+  static const std::size_t value = 1 + Index<T, std::tuple<Types...>>::value;
+};
+
+
+////////////////////////////////////////////
+// Action Level
+// Action collection 
+// in a integration level
+// (for multilevel integration schemes)
+////////////////////////////////////////////
+
+template <class Field, class Repr = NoHirep >
+struct ActionLevel {
+ public:
+  unsigned int multiplier;
+
+  // Fundamental repr actions separated because of the smearing
+  typedef Action<Field>* ActPtr;
+
+  // construct a tuple of vectors of the actions for the corresponding higher
+  // representation fields
+  typedef typename AccessTypes<Action, Repr>::VectorCollection action_collection;
+  typedef typename  AccessTypes<Action, Repr>::FieldTypeCollection action_hirep_types;
+
+  action_collection actions_hirep;
+  std::vector<ActPtr>& actions;
+
+  explicit ActionLevel(unsigned int mul = 1) : 
+  actions(std::get<0>(actions_hirep)), multiplier(mul) {
+    // initialize the hirep vectors to zero.
+    // apply(this->resize, actions_hirep, 0); //need a working resize
+    assert(mul >= 1);
+  }
+
+  template < class GenField >
+  void push_back(Action<GenField>* ptr) {
+    // insert only in the correct vector
+    std::get< Index < GenField, action_hirep_types>::value >(actions_hirep).push_back(ptr);
+  };
+
+  template <class ActPtr>
+  static void resize(ActPtr ap, unsigned int n) {
+    ap->resize(n);
+  }
+
+  // Loop on tuple for a callable function
+  template <std::size_t I = 1, typename Callable, typename ...Args>
+  inline typename std::enable_if<I == std::tuple_size<action_collection>::value, void>::type apply(Callable, Repr& R,Args&...) const {}
+
+  template <std::size_t I = 1, typename Callable, typename ...Args>
+  inline typename std::enable_if<I < std::tuple_size<action_collection>::value, void>::type apply(Callable fn, Repr& R, Args&... arguments) const {
+    fn(std::get<I>(actions_hirep), std::get<I>(R.rep), arguments...);
+    apply<I + 1>(fn, R, arguments...);
+  }  
+
+};
+
+// Define the ActionSet
+template <class GaugeField, class R>
+using ActionSet = std::vector<ActionLevel<GaugeField, R> >;
+
+} // QCD
+} // Grid
+
+#endif  // ACTION_SET_H

lib/qcd/action/fermion/CayleyFermion5D.cc

@@ -29,7 +29,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Eigen/Dense>
+#include <Grid/Grid_Eigen_Dense.h>
 #include <Grid/qcd/action/fermion/FermionCore.h>
 #include <Grid/qcd/action/fermion/CayleyFermion5D.h>
 
@@ -320,7 +320,7 @@ void CayleyFermion5D<Impl>::MDeriv  (GaugeField &mat,const FermionField &U,const
     this->DhopDeriv(mat,U,Din,dag);
   } else {
     //      U d/du [D_w D5]^dag V = U D5^dag d/du DW^dag Y // implicit adj on U in call
-    MeooeDag5D(U,Din);
+    Meooe5D(U,Din);
     this->DhopDeriv(mat,Din,V,dag);
   }
 };
@@ -335,8 +335,8 @@ void CayleyFermion5D<Impl>::MoeDeriv(GaugeField &mat,const FermionField &U,const
     this->DhopDerivOE(mat,U,Din,dag);
   } else {
     //      U d/du [D_w D5]^dag V = U D5^dag d/du DW^dag Y // implicit adj on U in call
-      MeooeDag5D(U,Din);
-      this->DhopDerivOE(mat,Din,V,dag);
+    Meooe5D(U,Din);
+    this->DhopDerivOE(mat,Din,V,dag);
   }
 };
 template<class Impl>
@@ -350,7 +350,7 @@ void CayleyFermion5D<Impl>::MeoDeriv(GaugeField &mat,const FermionField &U,const
     this->DhopDerivEO(mat,U,Din,dag);
   } else {
     //      U d/du [D_w D5]^dag V = U D5^dag d/du DW^dag Y // implicit adj on U in call
-    MeooeDag5D(U,Din);
+    Meooe5D(U,Din);
     this->DhopDerivEO(mat,Din,V,dag);
   }
 };
@@ -380,6 +380,8 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   ///////////////////////////////////////////////////////////
   // The Cayley coeffs (unprec)
   ///////////////////////////////////////////////////////////
+  assert(gamma.size()==Ls);
+
   omega.resize(Ls);
   bs.resize(Ls);
   cs.resize(Ls);
@@ -412,10 +414,11 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   for(int i=0; i < Ls; i++){
     as[i] = 1.0;
     omega[i] = gamma[i]*zolo_hi; //NB reciprocal relative to Chroma NEF code
+    //    assert(fabs(omega[i])>0.0);
     bs[i] = 0.5*(bpc/omega[i] + bmc);
     cs[i] = 0.5*(bpc/omega[i] - bmc);
   }
-  
+
   ////////////////////////////////////////////////////////
   // Constants for the preconditioned matrix Cayley form
   ////////////////////////////////////////////////////////
@@ -425,12 +428,12 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   ceo.resize(Ls);
   
   for(int i=0;i<Ls;i++){
-    bee[i]=as[i]*(bs[i]*(4.0-this->M5) +1.0);
+    bee[i]=as[i]*(bs[i]*(4.0-this->M5) +1.0);     
+    //    assert(fabs(bee[i])>0.0);
     cee[i]=as[i]*(1.0-cs[i]*(4.0-this->M5));
     beo[i]=as[i]*bs[i];
     ceo[i]=-as[i]*cs[i];
   }
-  
   aee.resize(Ls);
   aeo.resize(Ls);
   for(int i=0;i<Ls;i++){
@@ -474,14 +477,16 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
 	
   { 
     Coeff_t delta_d=mass*cee[Ls-1];
-    for(int j=0;j<Ls-1;j++) delta_d *= cee[j]/bee[j];
+    for(int j=0;j<Ls-1;j++) {
+      //      assert(fabs(bee[j])>0.0);
+      delta_d *= cee[j]/bee[j];
+    }
     dee[Ls-1] += delta_d;
   }  
 
   int inv=1;
   this->MooeeInternalCompute(0,inv,MatpInv,MatmInv);
   this->MooeeInternalCompute(1,inv,MatpInvDag,MatmInvDag);
-
 }
 
 
@@ -495,7 +500,9 @@ void CayleyFermion5D<Impl>::MooeeInternalCompute(int dag, int inv,
   GridBase *grid = this->FermionRedBlackGrid();
   int LLs = grid->_rdimensions[0];
 
-  if ( LLs == Ls ) return; // Not vectorised in 5th direction
+  if ( LLs == Ls ) {
+    return; // Not vectorised in 5th direction
+  }
 
   Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
   Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);

lib/qcd/action/fermion/CayleyFermion5Dcache.cc

@@ -237,6 +237,13 @@ void CayleyFermion5D<Impl>::MooeeInvDag (const FermionField &psi, FermionField &
   INSTANTIATE_DPERP(GparityWilsonImplD);
   INSTANTIATE_DPERP(ZWilsonImplF);
   INSTANTIATE_DPERP(ZWilsonImplD);
+
+  INSTANTIATE_DPERP(WilsonImplFH);
+  INSTANTIATE_DPERP(WilsonImplDF);
+  INSTANTIATE_DPERP(GparityWilsonImplFH);
+  INSTANTIATE_DPERP(GparityWilsonImplDF);
+  INSTANTIATE_DPERP(ZWilsonImplFH);
+  INSTANTIATE_DPERP(ZWilsonImplDF);
 #endif
 
 }}

lib/qcd/action/fermion/CayleyFermion5Ddense.cc

@@ -29,7 +29,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Eigen/Dense>
+#include <Grid/Grid_Eigen_Dense.h>
 #include <Grid/qcd/action/fermion/FermionCore.h>
 #include <Grid/qcd/action/fermion/CayleyFermion5D.h>
 
@@ -137,6 +137,20 @@ template void CayleyFermion5D<WilsonImplF>::MooeeInternal(const FermionField &ps
 template void CayleyFermion5D<WilsonImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZWilsonImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZWilsonImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+
+INSTANTIATE_DPERP(GparityWilsonImplFH);
+INSTANTIATE_DPERP(GparityWilsonImplDF);
+INSTANTIATE_DPERP(WilsonImplFH);
+INSTANTIATE_DPERP(WilsonImplDF);
+INSTANTIATE_DPERP(ZWilsonImplFH);
+INSTANTIATE_DPERP(ZWilsonImplDF);
+
+template void CayleyFermion5D<GparityWilsonImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<GparityWilsonImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<WilsonImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<WilsonImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZWilsonImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZWilsonImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 #endif
 
 }}

lib/qcd/action/fermion/CayleyFermion5Dssp.cc

@@ -37,7 +37,6 @@ namespace Grid {
 namespace QCD {
 
   // FIXME -- make a version of these routines with site loop outermost for cache reuse.
-
   // Pminus fowards
   // Pplus  backwards
 template<class Impl>  
@@ -152,6 +151,13 @@ void CayleyFermion5D<Impl>::MooeeInvDag (const FermionField &psi, FermionField &
   INSTANTIATE_DPERP(GparityWilsonImplD);
   INSTANTIATE_DPERP(ZWilsonImplF);
   INSTANTIATE_DPERP(ZWilsonImplD);
+
+  INSTANTIATE_DPERP(WilsonImplFH);
+  INSTANTIATE_DPERP(WilsonImplDF);
+  INSTANTIATE_DPERP(GparityWilsonImplFH);
+  INSTANTIATE_DPERP(GparityWilsonImplDF);
+  INSTANTIATE_DPERP(ZWilsonImplFH);
+  INSTANTIATE_DPERP(ZWilsonImplDF);
 #endif
 
 }

lib/qcd/action/fermion/CayleyFermion5Dvec.cc

@@ -808,10 +808,21 @@ INSTANTIATE_DPERP(DomainWallVec5dImplF);
 INSTANTIATE_DPERP(ZDomainWallVec5dImplD);
 INSTANTIATE_DPERP(ZDomainWallVec5dImplF);
 
+INSTANTIATE_DPERP(DomainWallVec5dImplDF);
+INSTANTIATE_DPERP(DomainWallVec5dImplFH);
+INSTANTIATE_DPERP(ZDomainWallVec5dImplDF);
+INSTANTIATE_DPERP(ZDomainWallVec5dImplFH);
+
 template void CayleyFermion5D<DomainWallVec5dImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<DomainWallVec5dImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZDomainWallVec5dImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZDomainWallVec5dImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 
+template void CayleyFermion5D<DomainWallVec5dImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<DomainWallVec5dImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZDomainWallVec5dImplFH>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+template void CayleyFermion5D<ZDomainWallVec5dImplDF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
+
+
 
 }}

lib/qcd/action/fermion/DomainWallFermion.h

@@ -68,7 +68,7 @@ namespace Grid {
 	Approx::zolotarev_data *zdata = Approx::higham(eps,this->Ls);// eps is ignored for higham
 	assert(zdata->n==this->Ls);
 	
-	//	std::cout<<GridLogMessage << "DomainWallFermion with Ls="<<this->Ls<<std::endl;
+	std::cout<<GridLogMessage << "DomainWallFermion with Ls="<<this->Ls<<std::endl;
 	// Call base setter
 	this->SetCoefficientsTanh(zdata,1.0,0.0);
 

lib/qcd/action/fermion/Fermion.h

@@ -1,4 +1,4 @@
-    /*************************************************************************************
+/*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
 
@@ -58,6 +58,7 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
 #include <Grid/qcd/action/fermion/DomainWallFermion.h>
 #include <Grid/qcd/action/fermion/MobiusFermion.h>
 #include <Grid/qcd/action/fermion/ZMobiusFermion.h>
+#include <Grid/qcd/action/fermion/SchurDiagTwoKappa.h>
 #include <Grid/qcd/action/fermion/ScaledShamirFermion.h>
 #include <Grid/qcd/action/fermion/MobiusZolotarevFermion.h>
 #include <Grid/qcd/action/fermion/ShamirZolotarevFermion.h>
@@ -88,6 +89,10 @@ typedef WilsonFermion<WilsonImplR> WilsonFermionR;
 typedef WilsonFermion<WilsonImplF> WilsonFermionF;
 typedef WilsonFermion<WilsonImplD> WilsonFermionD;
 
+typedef WilsonFermion<WilsonImplRL> WilsonFermionRL;
+typedef WilsonFermion<WilsonImplFH> WilsonFermionFH;
+typedef WilsonFermion<WilsonImplDF> WilsonFermionDF;
+
 typedef WilsonFermion<WilsonAdjImplR> WilsonAdjFermionR;
 typedef WilsonFermion<WilsonAdjImplF> WilsonAdjFermionF;
 typedef WilsonFermion<WilsonAdjImplD> WilsonAdjFermionD;
@@ -104,27 +109,50 @@ typedef DomainWallFermion<WilsonImplR> DomainWallFermionR;
 typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;
 typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
 
+typedef DomainWallFermion<WilsonImplRL> DomainWallFermionRL;
+typedef DomainWallFermion<WilsonImplFH> DomainWallFermionFH;
+typedef DomainWallFermion<WilsonImplDF> DomainWallFermionDF;
+
 typedef MobiusFermion<WilsonImplR> MobiusFermionR;
 typedef MobiusFermion<WilsonImplF> MobiusFermionF;
 typedef MobiusFermion<WilsonImplD> MobiusFermionD;
 
+typedef MobiusFermion<WilsonImplRL> MobiusFermionRL;
+typedef MobiusFermion<WilsonImplFH> MobiusFermionFH;
+typedef MobiusFermion<WilsonImplDF> MobiusFermionDF;
+
 typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
 typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
 typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
 
+typedef ZMobiusFermion<ZWilsonImplRL> ZMobiusFermionRL;
+typedef ZMobiusFermion<ZWilsonImplFH> ZMobiusFermionFH;
+typedef ZMobiusFermion<ZWilsonImplDF> ZMobiusFermionDF;
+
 // Ls vectorised 
 typedef DomainWallFermion<DomainWallVec5dImplR> DomainWallFermionVec5dR;
 typedef DomainWallFermion<DomainWallVec5dImplF> DomainWallFermionVec5dF;
 typedef DomainWallFermion<DomainWallVec5dImplD> DomainWallFermionVec5dD;
 
+typedef DomainWallFermion<DomainWallVec5dImplRL> DomainWallFermionVec5dRL;
+typedef DomainWallFermion<DomainWallVec5dImplFH> DomainWallFermionVec5dFH;
+typedef DomainWallFermion<DomainWallVec5dImplDF> DomainWallFermionVec5dDF;
+
 typedef MobiusFermion<DomainWallVec5dImplR> MobiusFermionVec5dR;
 typedef MobiusFermion<DomainWallVec5dImplF> MobiusFermionVec5dF;
 typedef MobiusFermion<DomainWallVec5dImplD> MobiusFermionVec5dD;
 
+typedef MobiusFermion<DomainWallVec5dImplRL> MobiusFermionVec5dRL;
+typedef MobiusFermion<DomainWallVec5dImplFH> MobiusFermionVec5dFH;
+typedef MobiusFermion<DomainWallVec5dImplDF> MobiusFermionVec5dDF;
+
 typedef ZMobiusFermion<ZDomainWallVec5dImplR> ZMobiusFermionVec5dR;
 typedef ZMobiusFermion<ZDomainWallVec5dImplF> ZMobiusFermionVec5dF;
 typedef ZMobiusFermion<ZDomainWallVec5dImplD> ZMobiusFermionVec5dD;
 
+typedef ZMobiusFermion<ZDomainWallVec5dImplRL> ZMobiusFermionVec5dRL;
+typedef ZMobiusFermion<ZDomainWallVec5dImplFH> ZMobiusFermionVec5dFH;
+typedef ZMobiusFermion<ZDomainWallVec5dImplDF> ZMobiusFermionVec5dDF;
 
 typedef ScaledShamirFermion<WilsonImplR> ScaledShamirFermionR;
 typedef ScaledShamirFermion<WilsonImplF> ScaledShamirFermionF;
@@ -165,17 +193,35 @@ typedef OverlapWilsonPartialFractionZolotarevFermion<WilsonImplD> OverlapWilsonP
 typedef WilsonFermion<GparityWilsonImplR>     GparityWilsonFermionR;
 typedef WilsonFermion<GparityWilsonImplF>     GparityWilsonFermionF;
 typedef WilsonFermion<GparityWilsonImplD>     GparityWilsonFermionD;
+
+typedef WilsonFermion<GparityWilsonImplRL>     GparityWilsonFermionRL;
+typedef WilsonFermion<GparityWilsonImplFH>     GparityWilsonFermionFH;
+typedef WilsonFermion<GparityWilsonImplDF>     GparityWilsonFermionDF;
+
 typedef DomainWallFermion<GparityWilsonImplR> GparityDomainWallFermionR;
 typedef DomainWallFermion<GparityWilsonImplF> GparityDomainWallFermionF;
 typedef DomainWallFermion<GparityWilsonImplD> GparityDomainWallFermionD;
 
+typedef DomainWallFermion<GparityWilsonImplRL> GparityDomainWallFermionRL;
+typedef DomainWallFermion<GparityWilsonImplFH> GparityDomainWallFermionFH;
+typedef DomainWallFermion<GparityWilsonImplDF> GparityDomainWallFermionDF;
+
 typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionR;
 typedef WilsonTMFermion<GparityWilsonImplF> GparityWilsonTMFermionF;
 typedef WilsonTMFermion<GparityWilsonImplD> GparityWilsonTMFermionD;
+
+typedef WilsonTMFermion<GparityWilsonImplRL> GparityWilsonTMFermionRL;
+typedef WilsonTMFermion<GparityWilsonImplFH> GparityWilsonTMFermionFH;
+typedef WilsonTMFermion<GparityWilsonImplDF> GparityWilsonTMFermionDF;
+
 typedef MobiusFermion<GparityWilsonImplR> GparityMobiusFermionR;
 typedef MobiusFermion<GparityWilsonImplF> GparityMobiusFermionF;
 typedef MobiusFermion<GparityWilsonImplD> GparityMobiusFermionD;
 
+typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
+typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
+typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
+
 typedef ImprovedStaggeredFermion<StaggeredImplR> ImprovedStaggeredFermionR;
 typedef ImprovedStaggeredFermion<StaggeredImplF> ImprovedStaggeredFermionF;
 typedef ImprovedStaggeredFermion<StaggeredImplD> ImprovedStaggeredFermionD;

lib/qcd/action/fermion/FermionCore.h

@@ -55,7 +55,14 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
   template class A<ZWilsonImplF>;		\
   template class A<ZWilsonImplD>;		\
   template class A<GparityWilsonImplF>;		\
-  template class A<GparityWilsonImplD>;		
+  template class A<GparityWilsonImplD>;		\
+  template class A<WilsonImplFH>;		\
+  template class A<WilsonImplDF>;		\
+  template class A<ZWilsonImplFH>;		\
+  template class A<ZWilsonImplDF>;		\
+  template class A<GparityWilsonImplFH>;		\
+  template class A<GparityWilsonImplDF>;		
+
 
 #define AdjointFermOpTemplateInstantiate(A) \
   template class A<WilsonAdjImplF>; \
@@ -69,7 +76,11 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
   template class A<DomainWallVec5dImplF>;	\
   template class A<DomainWallVec5dImplD>;	\
   template class A<ZDomainWallVec5dImplF>;	\
-  template class A<ZDomainWallVec5dImplD>;	
+  template class A<ZDomainWallVec5dImplD>;	\
+  template class A<DomainWallVec5dImplFH>;	\
+  template class A<DomainWallVec5dImplDF>;	\
+  template class A<ZDomainWallVec5dImplFH>;	\
+  template class A<ZDomainWallVec5dImplDF>;	
 
 #define FermOpTemplateInstantiate(A) \
  FermOp4dVecTemplateInstantiate(A) \

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -35,7 +35,6 @@ directory
 namespace Grid {
 namespace QCD {
 
-
   //////////////////////////////////////////////
   // Template parameter class constructs to package
   // externally control Fermion implementations
@@ -44,7 +43,7 @@ namespace QCD {
   // Ultimately need Impl to always define types where XXX is opaque
   //
   //    typedef typename XXX               Simd;
-  //    typedef typename XXX     GaugeLinkField;	
+  //    typedef typename XXX     GaugeLinkField;        
   //    typedef typename XXX         GaugeField;
   //    typedef typename XXX      GaugeActField;
   //    typedef typename XXX       FermionField;
@@ -89,7 +88,53 @@ namespace QCD {
   //    
   //  }
   //////////////////////////////////////////////
-  
+
+  template <class T> struct SamePrecisionMapper {
+    typedef T HigherPrecVector ;
+    typedef T LowerPrecVector ;
+  };
+  template <class T> struct LowerPrecisionMapper {  };
+  template <> struct LowerPrecisionMapper<vRealF> {
+    typedef vRealF HigherPrecVector ;
+    typedef vRealH LowerPrecVector ;
+  };
+  template <> struct LowerPrecisionMapper<vRealD> {
+    typedef vRealD HigherPrecVector ;
+    typedef vRealF LowerPrecVector ;
+  };
+  template <> struct LowerPrecisionMapper<vComplexF> {
+    typedef vComplexF HigherPrecVector ;
+    typedef vComplexH LowerPrecVector ;
+  };
+  template <> struct LowerPrecisionMapper<vComplexD> {
+    typedef vComplexD HigherPrecVector ;
+    typedef vComplexF LowerPrecVector ;
+  };
+
+  struct CoeffReal {
+  public:
+    typedef RealD _Coeff_t;
+    static const int Nhcs = 2;
+    template<class Simd> using PrecisionMapper = SamePrecisionMapper<Simd>;
+  };
+  struct CoeffRealHalfComms {
+  public:
+    typedef RealD _Coeff_t;
+    static const int Nhcs = 1;
+    template<class Simd> using PrecisionMapper = LowerPrecisionMapper<Simd>;
+  };
+  struct CoeffComplex {
+  public:
+    typedef ComplexD _Coeff_t;
+    static const int Nhcs = 2;
+    template<class Simd> using PrecisionMapper = SamePrecisionMapper<Simd>;
+  };
+  struct CoeffComplexHalfComms {
+  public:
+    typedef ComplexD _Coeff_t;
+    static const int Nhcs = 1;
+    template<class Simd> using PrecisionMapper = LowerPrecisionMapper<Simd>;
+  };
 
   ////////////////////////////////////////////////////////////////////////
   // Implementation dependent fermion types
@@ -108,58 +153,63 @@ namespace QCD {
   typedef typename Impl::Coeff_t                     Coeff_t;           \
   
 #define INHERIT_IMPL_TYPES(Base) \
-  INHERIT_GIMPL_TYPES(Base)	 \
+  INHERIT_GIMPL_TYPES(Base)      \
   INHERIT_FIMPL_TYPES(Base)
   
   /////////////////////////////////////////////////////////////////////////////
   // Single flavour four spinors with colour index
   /////////////////////////////////////////////////////////////////////////////
-  template <class S, class Representation = FundamentalRepresentation,class _Coeff_t = RealD >
+  template <class S, class Representation = FundamentalRepresentation,class Options = CoeffReal >
   class WilsonImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
-
     public:
 
     static const int Dimension = Representation::Dimension;
+    static const bool LsVectorised=false;
+    static const int Nhcs = Options::Nhcs;
+
     typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
+    INHERIT_GIMPL_TYPES(Gimpl);
       
     //Necessary?
     constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
     
-    const bool LsVectorised=false;
-    typedef _Coeff_t Coeff_t;
-
-    INHERIT_GIMPL_TYPES(Gimpl);
+    typedef typename Options::_Coeff_t Coeff_t;
+    typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
       
     template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
     template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Dimension>, Ns> >;
     template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
+    template <typename vtype> using iImplHalfCommSpinor    = iScalar<iVector<iVector<vtype, Dimension>, Nhcs> >;
     template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
     
     typedef iImplSpinor<Simd>            SiteSpinor;
     typedef iImplPropagator<Simd>        SitePropagator;
     typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
+    typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
     typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
     
     typedef Lattice<SiteSpinor>            FermionField;
     typedef Lattice<SitePropagator>        PropagatorField;
     typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
     
-    typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+    typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
     typedef WilsonImplParams ImplParams;
     typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
     
     ImplParams Params;
     
-    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
+    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){
+      assert(Params.boundary_phases.size() == Nd);
+    };
       
     bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
       
     inline void multLink(SiteHalfSpinor &phi,
-			 const SiteDoubledGaugeField &U,
-			 const SiteHalfSpinor &chi,
-			 int mu,
-			 StencilEntry *SE,
-			 StencilImpl &St) {
+                         const SiteDoubledGaugeField &U,
+                         const SiteHalfSpinor &chi,
+                         int mu,
+                         StencilEntry *SE,
+                         StencilImpl &St) {
       mult(&phi(), &U(mu), &chi());
     }
       
@@ -169,16 +219,34 @@ namespace QCD {
     }
       
     inline void DoubleStore(GridBase *GaugeGrid,
-			    DoubledGaugeField &Uds,
-			    const GaugeField &Umu) {
+                            DoubledGaugeField &Uds,
+                            const GaugeField &Umu) 
+    {
+      typedef typename Simd::scalar_type scalar_type;
+
       conformable(Uds._grid, GaugeGrid);
       conformable(Umu._grid, GaugeGrid);
+
       GaugeLinkField U(GaugeGrid);
+      GaugeLinkField tmp(GaugeGrid);
+
+      Lattice<iScalar<vInteger> > coor(GaugeGrid);
       for (int mu = 0; mu < Nd; mu++) {
-	U = PeekIndex<LorentzIndex>(Umu, mu);
-	PokeIndex<LorentzIndex>(Uds, U, mu);
-	U = adj(Cshift(U, mu, -1));
-	PokeIndex<LorentzIndex>(Uds, U, mu + 4);
+
+	      auto pha = Params.boundary_phases[mu];
+	      scalar_type phase( real(pha),imag(pha) );
+
+        int Lmu = GaugeGrid->GlobalDimensions()[mu] - 1;
+
+        LatticeCoordinate(coor, mu);
+
+        U = PeekIndex<LorentzIndex>(Umu, mu);
+        tmp = where(coor == Lmu, phase * U, U);
+        PokeIndex<LorentzIndex>(Uds, tmp, mu);
+
+        U = adj(Cshift(U, mu, -1));
+        U = where(coor == 0, conjugate(phase) * U, U); 
+        PokeIndex<LorentzIndex>(Uds, U, mu + 4);
       }
     }
 
@@ -195,11 +263,11 @@ namespace QCD {
       tmp = zero;
       
       parallel_for(int sss=0;sss<tmp._grid->oSites();sss++){
-	int sU=sss;
-	for(int s=0;s<Ls;s++){
-	  int sF = s+Ls*sU;
-	  tmp[sU] = tmp[sU]+ traceIndex<SpinIndex>(outerProduct(Btilde[sF],Atilde[sF])); // ordering here
-	}
+	    int sU=sss;
+	    for(int s=0;s<Ls;s++){
+	        int sF = s+Ls*sU;
+	        tmp[sU] = tmp[sU]+ traceIndex<SpinIndex>(outerProduct(Btilde[sF],Atilde[sF])); // ordering here
+	    }
       }
       PokeIndex<LorentzIndex>(mat,tmp,mu);
       
@@ -209,31 +277,34 @@ namespace QCD {
   ////////////////////////////////////////////////////////////////////////////////////
   // Single flavour four spinors with colour index, 5d redblack
   ////////////////////////////////////////////////////////////////////////////////////
-
-template<class S,int Nrepresentation=Nc,class _Coeff_t = RealD>
+template<class S,int Nrepresentation=Nc, class Options=CoeffReal>
 class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { 
   public:
-      
-  static const int Dimension = Nrepresentation;
-  const bool LsVectorised=true;
-  typedef _Coeff_t Coeff_t;      
+
   typedef PeriodicGaugeImpl<GaugeImplTypes<S, Nrepresentation> > Gimpl;
-  
   INHERIT_GIMPL_TYPES(Gimpl);
+
+  static const int Dimension = Nrepresentation;
+  static const bool LsVectorised=true;
+  static const int Nhcs = Options::Nhcs;
+      
+  typedef typename Options::_Coeff_t Coeff_t;      
+  typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
   
   template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Nrepresentation>, Ns> >;
   template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Nrepresentation>, Ns> >;
   template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhs> >;
+  template <typename vtype> using iImplHalfCommSpinor    = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhcs> >;
   template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>;
   template <typename vtype> using iImplGaugeField        = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nd>;
   template <typename vtype> using iImplGaugeLink         = iScalar<iScalar<iMatrix<vtype, Nrepresentation> > >;
   
-  typedef iImplSpinor<Simd> SiteSpinor;
-  typedef iImplPropagator<Simd> SitePropagator;
-  typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
-  typedef Lattice<SiteSpinor> FermionField;
-  typedef Lattice<SitePropagator> PropagatorField;
-  
+  typedef iImplSpinor<Simd>            SiteSpinor;
+  typedef iImplPropagator<Simd>        SitePropagator;
+  typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
+  typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
+  typedef Lattice<SiteSpinor>          FermionField;
+  typedef Lattice<SitePropagator>      PropagatorField;
 
   /////////////////////////////////////////////////
   // Make the doubled gauge field a *scalar*
@@ -241,9 +312,9 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
   typedef iImplDoubledGaugeField<typename Simd::scalar_type>  SiteDoubledGaugeField;  // This is a scalar
   typedef iImplGaugeField<typename Simd::scalar_type>         SiteScalarGaugeField;  // scalar
   typedef iImplGaugeLink<typename Simd::scalar_type>          SiteScalarGaugeLink;  // scalar
-  typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
+  typedef Lattice<SiteDoubledGaugeField>                      DoubledGaugeField;
       
-  typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+  typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
   typedef WilsonImplParams ImplParams;
   typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
   
@@ -259,12 +330,12 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
   }
 
   inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
-		       const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
-		       StencilImpl &St) {
+                       const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
+                       StencilImpl &St) {
     SiteGaugeLink UU;
     for (int i = 0; i < Nrepresentation; i++) {
       for (int j = 0; j < Nrepresentation; j++) {
-	vsplat(UU()()(i, j), U(mu)()(i, j));
+        vsplat(UU()()(i, j), U(mu)()(i, j));
       }
     }
     mult(&phi(), &UU(), &chi());
@@ -301,45 +372,90 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
   {
     assert(0);
   }
-      
-  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,FermionField &Atilde, int mu) 
-  {
-	assert(0);
+
+  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde, int mu) {
+
+    assert(0);
+    // Following lines to be revised after Peter's addition of half prec
+    // missing put lane...
+    /*
+    typedef decltype(traceIndex<SpinIndex>(outerProduct(Btilde[0], Atilde[0]))) result_type;
+    unsigned int LLs = Btilde._grid->_rdimensions[0];
+    conformable(Atilde._grid,Btilde._grid);
+    GridBase* grid = mat._grid;
+    GridBase* Bgrid = Btilde._grid;
+    unsigned int dimU = grid->Nd();
+    unsigned int dimF = Bgrid->Nd();
+    GaugeLinkField tmp(grid); 
+    tmp = zero;
+    
+    // FIXME 
+    // Current implementation works, thread safe, probably suboptimal
+    // Passing through the local coordinate for grid transformation
+    // the force grid is in general very different from the Ls vectorized grid
+
+    PARALLEL_FOR_LOOP
+    for (int so = 0; so < grid->oSites(); so++) {
+      std::vector<typename result_type::scalar_object> vres(Bgrid->Nsimd());
+      std::vector<int> ocoor;  grid->oCoorFromOindex(ocoor,so); 
+      for (int si = 0; si < tmp._grid->iSites(); si++){
+        typename result_type::scalar_object scalar_object; scalar_object = zero;
+        std::vector<int> local_coor;      
+        std::vector<int> icoor; grid->iCoorFromIindex(icoor,si);
+        grid->InOutCoorToLocalCoor(ocoor, icoor, local_coor);
+        for (int s = 0; s < LLs; s++) {
+          std::vector<int> slocal_coor(dimF);
+          slocal_coor[0] = s;
+          for (int s4d = 1; s4d< dimF; s4d++) slocal_coor[s4d] = local_coor[s4d-1];
+          int sF = Bgrid->oIndexReduced(slocal_coor);  
+          assert(sF < Bgrid->oSites());
+
+          extract(traceIndex<SpinIndex>(outerProduct(Btilde[sF], Atilde[sF])), vres); 
+          // sum across the 5d dimension
+          for (auto v : vres) scalar_object += v;  
+        }
+        tmp._odata[so].putlane(scalar_object, si);
+      }
+    }
+    PokeIndex<LorentzIndex>(mat, tmp, mu);
+    */
   }
 };
     
     ////////////////////////////////////////////////////////////////////////////////////////
     // Flavour doubled spinors; is Gparity the only? what about C*?
     ////////////////////////////////////////////////////////////////////////////////////////
-    
-template <class S, int Nrepresentation,class _Coeff_t = RealD>
+template <class S, int Nrepresentation, class Options=CoeffReal>
 class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
  public:
 
  static const int Dimension = Nrepresentation;
+ static const int Nhcs = Options::Nhcs;
+ static const bool LsVectorised=false;
 
- const bool LsVectorised=false;
-
- typedef _Coeff_t Coeff_t;
  typedef ConjugateGaugeImpl< GaugeImplTypes<S,Nrepresentation> > Gimpl;
- 
  INHERIT_GIMPL_TYPES(Gimpl);
+
+ typedef typename Options::_Coeff_t Coeff_t;
+ typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
       
- template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
- template <typename vtype> using iImplPropagator        = iVector<iMatrix<iMatrix<vtype, Nrepresentation>, Ns>, Ngp >;
- template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
+ template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>,   Ngp>;
+ template <typename vtype> using iImplPropagator        = iVector<iMatrix<iMatrix<vtype, Nrepresentation>, Ns>,   Ngp>;
+ template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>,  Ngp>;
+ template <typename vtype> using iImplHalfCommSpinor    = iVector<iVector<iVector<vtype, Nrepresentation>, Nhcs>, Ngp>;
  template <typename vtype> using iImplDoubledGaugeField = iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
-      
- typedef iImplSpinor<Simd> SiteSpinor;
- typedef iImplPropagator<Simd> SitePropagator;
- typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
+
+ typedef iImplSpinor<Simd>            SiteSpinor;
+ typedef iImplPropagator<Simd>        SitePropagator;
+ typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
+ typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
  typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
 
  typedef Lattice<SiteSpinor> FermionField;
  typedef Lattice<SitePropagator> PropagatorField;
  typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
  
- typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
+ typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
  typedef WilsonStencil<SiteSpinor, SiteHalfSpinor> StencilImpl;
  
  typedef GparityWilsonImplParams ImplParams;
@@ -353,19 +469,19 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
  // provide the multiply by link that is differentiated between Gparity (with
  // flavour index) and non-Gparity
  inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U,
-		      const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
-		      StencilImpl &St) {
+                      const SiteHalfSpinor &chi, int mu, StencilEntry *SE,
+                      StencilImpl &St) {
 
-  typedef SiteHalfSpinor vobj;
-  typedef typename SiteHalfSpinor::scalar_object sobj;
+   typedef SiteHalfSpinor vobj;
+   typedef typename SiteHalfSpinor::scalar_object sobj;
 	
    vobj vtmp;
    sobj stmp;
-	
+        
    GridBase *grid = St._grid;
-	
+        
    const int Nsimd = grid->Nsimd();
-	
+        
    int direction = St._directions[mu];
    int distance = St._distances[mu];
    int ptype = St._permute_type[mu];
@@ -373,13 +489,13 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
    
    // Fixme X.Y.Z.T hardcode in stencil
    int mmu = mu % Nd;
-	
+        
    // assert our assumptions
    assert((distance == 1) || (distance == -1));  // nearest neighbour stencil hard code
    assert((sl == 1) || (sl == 2));
    
    std::vector<int> icoor;
-	
+        
    if ( SE->_around_the_world && Params.twists[mmu] ) {
 
      if ( sl == 2 ) {
@@ -389,25 +505,25 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
        extract(chi,vals);
        for(int s=0;s<Nsimd;s++){
 
-	 grid->iCoorFromIindex(icoor,s);
-	      
-	 assert((icoor[direction]==0)||(icoor[direction]==1));
-	      
-	 int permute_lane;
-	 if ( distance == 1) {
-	   permute_lane = icoor[direction]?1:0;
-	 } else {
-	   permute_lane = icoor[direction]?0:1;
-	 }
-	      
-	 if ( permute_lane ) { 
-	   stmp(0) = vals[s](1);
-	   stmp(1) = vals[s](0);
-	   vals[s] = stmp;
-	      }
+         grid->iCoorFromIindex(icoor,s);
+              
+         assert((icoor[direction]==0)||(icoor[direction]==1));
+              
+         int permute_lane;
+         if ( distance == 1) {
+           permute_lane = icoor[direction]?1:0;
+         } else {
+           permute_lane = icoor[direction]?0:1;
+         }
+              
+         if ( permute_lane ) { 
+           stmp(0) = vals[s](1);
+           stmp(1) = vals[s](0);
+           vals[s] = stmp;
+              }
        }
        merge(vtmp,vals);
-	    
+            
      } else { 
        vtmp(0) = chi(1);
        vtmp(1) = chi(0);
@@ -432,11 +548,11 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
    GaugeLinkField Uconj(GaugeGrid);
    
    Lattice<iScalar<vInteger> > coor(GaugeGrid);
-	
+        
    for(int mu=0;mu<Nd;mu++){
-	  
+          
      LatticeCoordinate(coor,mu);
-	  
+          
      U     = PeekIndex<LorentzIndex>(Umu,mu);
      Uconj = conjugate(U);
      
@@ -450,7 +566,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
        Uds[ss](0)(mu) = U[ss]();
        Uds[ss](1)(mu) = Uconj[ss]();
      }
-	  
+          
      U     = adj(Cshift(U    ,mu,-1));      // correct except for spanning the boundary
      Uconj = adj(Cshift(Uconj,mu,-1));
  
@@ -458,11 +574,12 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
      if ( Params.twists[mu] ) { 
        Utmp = where(coor==0,Uconj,Utmp);
      }
+
 	  
      parallel_for(auto ss=U.begin();ss<U.end();ss++){
        Uds[ss](0)(mu+4) = Utmp[ss]();
      }
-	  
+          
      Utmp = Uconj;
      if ( Params.twists[mu] ) { 
        Utmp = where(coor==0,U,Utmp);
@@ -471,11 +588,10 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
      parallel_for(auto ss=U.begin();ss<U.end();ss++){
        Uds[ss](1)(mu+4) = Utmp[ss]();
      }
-	  
+          
    }
  }
       
-      
  inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A, int mu) {
 
    // DhopDir provides U or Uconj depending on coor/flavour.
@@ -483,7 +599,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
    // use lorentz for flavour as hack.
    auto tmp = TraceIndex<SpinIndex>(outerProduct(Btilde, A));
    parallel_for(auto ss = tmp.begin(); ss < tmp.end(); ss++) {
-     link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1));
+     link[ss]() = tmp[ss](0, 0) + conjugate(tmp[ss](1, 1));
    }
    PokeIndex<LorentzIndex>(mat, link, mu);
    return;
@@ -492,7 +608,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde, int mu) {
 
    int Ls = Btilde._grid->_fdimensions[0];
-	
+        
    GaugeLinkField tmp(mat._grid);
    tmp = zero;
    parallel_for(int ss = 0; ss < tmp._grid->oSites(); ss++) {
@@ -508,40 +624,36 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 
 };
 
-
-  /////////////////////////////////////////////////////////////////////////////
-  // Single flavour one component spinors with colour index
-  /////////////////////////////////////////////////////////////////////////////
-  template <class S, class Representation = FundamentalRepresentation >
-  class StaggeredImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
+/////////////////////////////////////////////////////////////////////////////
+// Single flavour one component spinors with colour index
+/////////////////////////////////////////////////////////////////////////////
+template <class S, class Representation = FundamentalRepresentation >
+class StaggeredImpl : public PeriodicGaugeImpl<GaugeImplTypes<S, Representation::Dimension > > {
 
     public:
 
     typedef RealD  _Coeff_t ;
     static const int Dimension = Representation::Dimension;
+    static const bool LsVectorised=false;
     typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
       
     //Necessary?
     constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
     
-    const bool LsVectorised=false;
     typedef _Coeff_t Coeff_t;
 
     INHERIT_GIMPL_TYPES(Gimpl);
       
-    template <typename vtype> using iImplScalar            = iScalar<iScalar<iScalar<vtype> > >;
     template <typename vtype> using iImplSpinor            = iScalar<iScalar<iVector<vtype, Dimension> > >;
     template <typename vtype> using iImplHalfSpinor        = iScalar<iScalar<iVector<vtype, Dimension> > >;
     template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
     template <typename vtype> using iImplPropagator        = iScalar<iScalar<iMatrix<vtype, Dimension> > >;
     
-    typedef iImplScalar<Simd>            SiteComplex;
     typedef iImplSpinor<Simd>            SiteSpinor;
     typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
     typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
     typedef iImplPropagator<Simd>        SitePropagator;
     
-    typedef Lattice<SiteComplex>           ComplexField;
     typedef Lattice<SiteSpinor>            FermionField;
     typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
     typedef Lattice<SitePropagator> PropagatorField;
@@ -641,8 +753,6 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
     }
   };
 
-
-
   /////////////////////////////////////////////////////////////////////////////
   // Single flavour one component spinors with colour index. 5d vec
   /////////////////////////////////////////////////////////////////////////////
@@ -651,20 +761,17 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 
     public:
 
-    typedef RealD  _Coeff_t ;
     static const int Dimension = Representation::Dimension;
+    static const bool LsVectorised=true;
+    typedef RealD   Coeff_t ;
     typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
       
     //Necessary?
     constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
-    
-    const bool LsVectorised=true;
 
-    typedef _Coeff_t Coeff_t;
 
     INHERIT_GIMPL_TYPES(Gimpl);
 
-    template <typename vtype> using iImplScalar            = iScalar<iScalar<iScalar<vtype> > >;
     template <typename vtype> using iImplSpinor            = iScalar<iScalar<iVector<vtype, Dimension> > >;
     template <typename vtype> using iImplHalfSpinor        = iScalar<iScalar<iVector<vtype, Dimension> > >;
     template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
@@ -681,12 +788,10 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
     typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
     typedef Lattice<SitePropagator> PropagatorField;
     
-    typedef iImplScalar<Simd>            SiteComplex;
     typedef iImplSpinor<Simd>            SiteSpinor;
     typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
 
     
-    typedef Lattice<SiteComplex>           ComplexField;
     typedef Lattice<SiteSpinor>            FermionField;
     
     typedef SimpleCompressor<SiteSpinor> Compressor;
@@ -823,43 +928,61 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
     }
   };
 
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffReal > WilsonImplR;  // Real.. whichever prec
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffReal > WilsonImplF;  // Float
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffReal > WilsonImplD;  // Double
 
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffRealHalfComms > WilsonImplRL;  // Real.. whichever prec
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplFH;  // Float
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffRealHalfComms > WilsonImplDF;  // Double
 
- typedef WilsonImpl<vComplex,  FundamentalRepresentation > WilsonImplR;   // Real.. whichever prec
- typedef WilsonImpl<vComplexF, FundamentalRepresentation > WilsonImplF;  // Float
- typedef WilsonImpl<vComplexD, FundamentalRepresentation > WilsonImplD;  // Double
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffComplex > ZWilsonImplR; // Real.. whichever prec
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplex > ZWilsonImplF; // Float
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplex > ZWilsonImplD; // Double
 
- typedef WilsonImpl<vComplex,  FundamentalRepresentation, ComplexD > ZWilsonImplR; // Real.. whichever prec
- typedef WilsonImpl<vComplexF, FundamentalRepresentation, ComplexD > ZWilsonImplF; // Float
- typedef WilsonImpl<vComplexD, FundamentalRepresentation, ComplexD > ZWilsonImplD; // Double
+typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplRL; // Real.. whichever prec
+typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplFH; // Float
+typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplexHalfComms > ZWilsonImplDF; // Double
  
- typedef WilsonImpl<vComplex,  AdjointRepresentation > WilsonAdjImplR;   // Real.. whichever prec
- typedef WilsonImpl<vComplexF, AdjointRepresentation > WilsonAdjImplF;  // Float
- typedef WilsonImpl<vComplexD, AdjointRepresentation > WilsonAdjImplD;  // Double
+typedef WilsonImpl<vComplex,  AdjointRepresentation, CoeffReal > WilsonAdjImplR;   // Real.. whichever prec
+typedef WilsonImpl<vComplexF, AdjointRepresentation, CoeffReal > WilsonAdjImplF;  // Float
+typedef WilsonImpl<vComplexD, AdjointRepresentation, CoeffReal > WilsonAdjImplD;  // Double
  
- typedef WilsonImpl<vComplex,  TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplR;   // Real.. whichever prec
- typedef WilsonImpl<vComplexF, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplF;  // Float
- typedef WilsonImpl<vComplexD, TwoIndexSymmetricRepresentation > WilsonTwoIndexSymmetricImplD;  // Double
+typedef WilsonImpl<vComplex,  TwoIndexSymmetricRepresentation, CoeffReal > WilsonTwoIndexSymmetricImplR;   // Real.. whichever prec
+typedef WilsonImpl<vComplexF, TwoIndexSymmetricRepresentation, CoeffReal > WilsonTwoIndexSymmetricImplF;  // Float
+typedef WilsonImpl<vComplexD, TwoIndexSymmetricRepresentation, CoeffReal > WilsonTwoIndexSymmetricImplD;  // Double
  
- typedef DomainWallVec5dImpl<vComplex ,Nc> DomainWallVec5dImplR; // Real.. whichever prec
- typedef DomainWallVec5dImpl<vComplexF,Nc> DomainWallVec5dImplF; // Float
- typedef DomainWallVec5dImpl<vComplexD,Nc> DomainWallVec5dImplD; // Double
+typedef DomainWallVec5dImpl<vComplex ,Nc, CoeffReal> DomainWallVec5dImplR; // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplexF,Nc, CoeffReal> DomainWallVec5dImplF; // Float
+typedef DomainWallVec5dImpl<vComplexD,Nc, CoeffReal> DomainWallVec5dImplD; // Double
  
- typedef DomainWallVec5dImpl<vComplex ,Nc,ComplexD> ZDomainWallVec5dImplR; // Real.. whichever prec
- typedef DomainWallVec5dImpl<vComplexF,Nc,ComplexD> ZDomainWallVec5dImplF; // Float
- typedef DomainWallVec5dImpl<vComplexD,Nc,ComplexD> ZDomainWallVec5dImplD; // Double
+typedef DomainWallVec5dImpl<vComplex ,Nc, CoeffRealHalfComms> DomainWallVec5dImplRL; // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplexF,Nc, CoeffRealHalfComms> DomainWallVec5dImplFH; // Float
+typedef DomainWallVec5dImpl<vComplexD,Nc, CoeffRealHalfComms> DomainWallVec5dImplDF; // Double
  
- typedef GparityWilsonImpl<vComplex , Nc> GparityWilsonImplR;  // Real.. whichever prec
- typedef GparityWilsonImpl<vComplexF, Nc> GparityWilsonImplF;  // Float
- typedef GparityWilsonImpl<vComplexD, Nc> GparityWilsonImplD;  // Double
+typedef DomainWallVec5dImpl<vComplex ,Nc,CoeffComplex> ZDomainWallVec5dImplR; // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplexF,Nc,CoeffComplex> ZDomainWallVec5dImplF; // Float
+typedef DomainWallVec5dImpl<vComplexD,Nc,CoeffComplex> ZDomainWallVec5dImplD; // Double
+ 
+typedef DomainWallVec5dImpl<vComplex ,Nc,CoeffComplexHalfComms> ZDomainWallVec5dImplRL; // Real.. whichever prec
+typedef DomainWallVec5dImpl<vComplexF,Nc,CoeffComplexHalfComms> ZDomainWallVec5dImplFH; // Float
+typedef DomainWallVec5dImpl<vComplexD,Nc,CoeffComplexHalfComms> ZDomainWallVec5dImplDF; // Double
+ 
+typedef GparityWilsonImpl<vComplex , Nc,CoeffReal> GparityWilsonImplR;  // Real.. whichever prec
+typedef GparityWilsonImpl<vComplexF, Nc,CoeffReal> GparityWilsonImplF;  // Float
+typedef GparityWilsonImpl<vComplexD, Nc,CoeffReal> GparityWilsonImplD;  // Double
+ 
+typedef GparityWilsonImpl<vComplex , Nc,CoeffRealHalfComms> GparityWilsonImplRL;  // Real.. whichever prec
+typedef GparityWilsonImpl<vComplexF, Nc,CoeffRealHalfComms> GparityWilsonImplFH;  // Float
+typedef GparityWilsonImpl<vComplexD, Nc,CoeffRealHalfComms> GparityWilsonImplDF;  // Double
 
- typedef StaggeredImpl<vComplex,  FundamentalRepresentation > StaggeredImplR;   // Real.. whichever prec
- typedef StaggeredImpl<vComplexF, FundamentalRepresentation > StaggeredImplF;  // Float
- typedef StaggeredImpl<vComplexD, FundamentalRepresentation > StaggeredImplD;  // Double
+typedef StaggeredImpl<vComplex,  FundamentalRepresentation > StaggeredImplR;   // Real.. whichever prec
+typedef StaggeredImpl<vComplexF, FundamentalRepresentation > StaggeredImplF;  // Float
+typedef StaggeredImpl<vComplexD, FundamentalRepresentation > StaggeredImplD;  // Double
 
- typedef StaggeredVec5dImpl<vComplex,  FundamentalRepresentation > StaggeredVec5dImplR;   // Real.. whichever prec
- typedef StaggeredVec5dImpl<vComplexF, FundamentalRepresentation > StaggeredVec5dImplF;  // Float
- typedef StaggeredVec5dImpl<vComplexD, FundamentalRepresentation > StaggeredVec5dImplD;  // Double
+typedef StaggeredVec5dImpl<vComplex,  FundamentalRepresentation > StaggeredVec5dImplR;   // Real.. whichever prec
+typedef StaggeredVec5dImpl<vComplexF, FundamentalRepresentation > StaggeredVec5dImplF;  // Float
+typedef StaggeredVec5dImpl<vComplexD, FundamentalRepresentation > StaggeredVec5dImplD;  // Double
 
 }}
 

lib/qcd/action/fermion/ImprovedStaggeredFermion.cc

@@ -160,8 +160,6 @@ void ImprovedStaggeredFermion<Impl>::ImportGauge(const GaugeField &_Uthin,const
     PokeIndex<LorentzIndex>(UUUmu, U*(-0.5*c2/u0/u0/u0), mu+4);
   }
 
-  std::cout << " Umu " << Umu._odata[0]<<std::endl;
-  std::cout << " UUUmu " << UUUmu._odata[0]<<std::endl;
   pickCheckerboard(Even, UmuEven, Umu);
   pickCheckerboard(Odd,  UmuOdd , Umu);
   pickCheckerboard(Even, UUUmuEven, UUUmu);

lib/qcd/action/fermion/SchurDiagTwoKappa.h

@@ -0,0 +1,102 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: SchurDiagTwoKappa.h
+
+    Copyright (C) 2017
+
+Author: Christoph Lehner
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef  _SCHUR_DIAG_TWO_KAPPA_H
+#define  _SCHUR_DIAG_TWO_KAPPA_H
+
+namespace Grid {
+
+  // This is specific to (Z)mobius fermions
+  template<class Matrix, class Field>
+    class KappaSimilarityTransform {
+  public:
+    INHERIT_IMPL_TYPES(Matrix);
+    std::vector<Coeff_t> kappa, kappaDag, kappaInv, kappaInvDag;
+
+    KappaSimilarityTransform (Matrix &zmob) {
+      for (int i=0;i<(int)zmob.bs.size();i++) {
+	Coeff_t k = 1.0 / ( 2.0 * (zmob.bs[i] *(4 - zmob.M5) + 1.0) );
+	kappa.push_back( k );
+	kappaDag.push_back( conj(k) );
+	kappaInv.push_back( 1.0 / k );
+	kappaInvDag.push_back( 1.0 / conj(k) );
+      }
+    }
+
+  template<typename vobj>
+    void sscale(const Lattice<vobj>& in, Lattice<vobj>& out, Coeff_t* s) {
+    GridBase *grid=out._grid;
+    out.checkerboard = in.checkerboard;
+    assert(grid->_simd_layout[0] == 1); // should be fine for ZMobius for now
+    int Ls = grid->_rdimensions[0];
+    parallel_for(int ss=0;ss<grid->oSites();ss++){
+      vobj tmp = s[ss % Ls]*in._odata[ss];
+      vstream(out._odata[ss],tmp);
+    }
+  }
+
+  RealD sscale_norm(const Field& in, Field& out, Coeff_t* s) {
+    sscale(in,out,s);
+    return norm2(out);
+  }
+
+  virtual RealD M       (const Field& in, Field& out) { return sscale_norm(in,out,&kappa[0]);   }
+  virtual RealD MDag    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaDag[0]);}
+  virtual RealD MInv    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInv[0]);}
+  virtual RealD MInvDag (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInvDag[0]);}
+
+  };
+
+  template<class Matrix,class Field>
+    class SchurDiagTwoKappaOperator :  public SchurOperatorBase<Field> {
+  public:
+    KappaSimilarityTransform<Matrix, Field> _S;
+    SchurDiagTwoOperator<Matrix, Field> _Mat;
+
+    SchurDiagTwoKappaOperator (Matrix &Mat): _S(Mat), _Mat(Mat) {};
+
+    virtual  RealD Mpc      (const Field &in, Field &out) {
+      Field tmp(in._grid);
+
+      _S.MInv(in,out);
+      _Mat.Mpc(out,tmp);
+      return _S.M(tmp,out);
+
+    }
+    virtual  RealD MpcDag   (const Field &in, Field &out){
+      Field tmp(in._grid);
+
+      _S.MDag(in,out);
+      _Mat.MpcDag(out,tmp);
+      return _S.MInvDag(tmp,out);
+    }
+  };
+
+}
+
+#endif

lib/qcd/action/fermion/WilsonCompressor.h

@@ -33,228 +33,321 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 namespace Grid {
 namespace QCD {
 
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonCompressor {
-  public:
-    int mu;
-    int dag;
+/////////////////////////////////////////////////////////////////////////////////////////////
+// optimised versions supporting half precision too
+/////////////////////////////////////////////////////////////////////////////////////////////
 
-    WilsonCompressor(int _dag){
-      mu=0;
-      dag=_dag;
-      assert((dag==0)||(dag==1));
-    }
-    void Point(int p) { 
-      mu=p;
-    };
+template<class _HCspinor,class _Hspinor,class _Spinor, class projector,typename SFINAE = void >
+class WilsonCompressorTemplate;
 
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      int mudag=mu;
-      if (!dag) {
-	mudag=(mu+Nd)%(2*Nd);
-      }
-      switch(mudag) {
-      case Xp:
-	spProjXp(ret,in);
-	break;
-      case Yp:
-	spProjYp(ret,in);
-	break;
-      case Zp:
-	spProjZp(ret,in);
-	break;
-      case Tp:
-	spProjTp(ret,in);
-	break;
-      case Xm:
-	spProjXm(ret,in);
-	break;
-      case Ym:
-	spProjYm(ret,in);
-	break;
-      case Zm:
-	spProjZm(ret,in);
-	break;
-      case Tm:
-	spProjTm(ret,in);
-	break;
-      default: 
-	assert(0);
-	break;
-      }
-      return ret;
-    }
-  };
 
-  /////////////////////////
-  // optimised versions
-  /////////////////////////
+template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
+class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector,
+  typename std::enable_if<std::is_same<_HCspinor,_Hspinor>::value>::type >
+{
+ public:
+  
+  int mu,dag;  
 
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonXpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjXp(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonYpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjYp(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonZpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjZp(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonTpCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjTp(ret,in);
-      return ret;
-    }
-  };
+  void Point(int p) { mu=p; };
 
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonXmCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjXm(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonYmCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjYm(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonZmCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjZm(ret,in);
-      return ret;
-    }
-  };
-  template<class SiteHalfSpinor,class SiteSpinor>
-  class WilsonTmCompressor {
-  public:
-    inline SiteHalfSpinor operator () (const SiteSpinor &in) {
-      SiteHalfSpinor ret;
-      spProjTm(ret,in);
-      return ret;
-    }
-  };
+  WilsonCompressorTemplate(int _dag=0){
+    dag = _dag;
+  }
 
-    // Fast comms buffer manipulation which should inline right through (avoid direction
-    // dependent logic that prevents inlining
-  template<class vobj,class cobj>
-  class WilsonStencil : public CartesianStencil<vobj,cobj> {
-  public:
+  typedef _Spinor         SiteSpinor;
+  typedef _Hspinor     SiteHalfSpinor;
+  typedef _HCspinor SiteHalfCommSpinor;
+  typedef typename SiteHalfCommSpinor::vector_type vComplexLow;
+  typedef typename SiteHalfSpinor::vector_type     vComplexHigh;
+  constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
 
-    typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
+  inline int CommDatumSize(void) {
+    return sizeof(SiteHalfCommSpinor);
+  }
 
-    WilsonStencil(GridBase *grid,
+  /*****************************************************/
+  /* Compress includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Compress(SiteHalfSpinor *buf,Integer o,const SiteSpinor &in) {
+    projector::Proj(buf[o],in,mu,dag);
+  }
+
+  /*****************************************************/
+  /* Exchange includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Exchange(SiteHalfSpinor *mp,
+                       SiteHalfSpinor *vp0,
+                       SiteHalfSpinor *vp1,
+		       Integer type,Integer o){
+    exchange(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type);
+  }
+
+  /*****************************************************/
+  /* Have a decompression step if mpi data is not same */
+  /*****************************************************/
+  inline void Decompress(SiteHalfSpinor *out,
+			 SiteHalfSpinor *in, Integer o) {    
+    assert(0);
+  }
+
+  /*****************************************************/
+  /* Compress Exchange                                 */
+  /*****************************************************/
+  inline void CompressExchange(SiteHalfSpinor *out0,
+			       SiteHalfSpinor *out1,
+			       const SiteSpinor *in,
+			       Integer j,Integer k, Integer m,Integer type){
+    SiteHalfSpinor temp1, temp2,temp3,temp4;
+    projector::Proj(temp1,in[k],mu,dag);
+    projector::Proj(temp2,in[m],mu,dag);
+    exchange(out0[j],out1[j],temp1,temp2,type);
+  }
+
+  /*****************************************************/
+  /* Pass the info to the stencil */
+  /*****************************************************/
+  inline bool DecompressionStep(void) { return false; }
+
+};
+
+template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
+class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector,
+  typename std::enable_if<!std::is_same<_HCspinor,_Hspinor>::value>::type >
+{
+ public:
+  
+  int mu,dag;  
+
+  void Point(int p) { mu=p; };
+
+  WilsonCompressorTemplate(int _dag=0){
+    dag = _dag;
+  }
+
+  typedef _Spinor         SiteSpinor;
+  typedef _Hspinor     SiteHalfSpinor;
+  typedef _HCspinor SiteHalfCommSpinor;
+  typedef typename SiteHalfCommSpinor::vector_type vComplexLow;
+  typedef typename SiteHalfSpinor::vector_type     vComplexHigh;
+  constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
+
+  inline int CommDatumSize(void) {
+    return sizeof(SiteHalfCommSpinor);
+  }
+
+  /*****************************************************/
+  /* Compress includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Compress(SiteHalfSpinor *buf,Integer o,const SiteSpinor &in) {
+    SiteHalfSpinor hsp;
+    SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf;
+    projector::Proj(hsp,in,mu,dag);
+    precisionChange((vComplexLow *)&hbuf[o],(vComplexHigh *)&hsp,Nw);
+  }
+
+  /*****************************************************/
+  /* Exchange includes precision change if mpi data is not same */
+  /*****************************************************/
+  inline void Exchange(SiteHalfSpinor *mp,
+                       SiteHalfSpinor *vp0,
+                       SiteHalfSpinor *vp1,
+		       Integer type,Integer o){
+    SiteHalfSpinor vt0,vt1;
+    SiteHalfCommSpinor *vpp0 = (SiteHalfCommSpinor *)vp0;
+    SiteHalfCommSpinor *vpp1 = (SiteHalfCommSpinor *)vp1;
+    precisionChange((vComplexHigh *)&vt0,(vComplexLow *)&vpp0[o],Nw);
+    precisionChange((vComplexHigh *)&vt1,(vComplexLow *)&vpp1[o],Nw);
+    exchange(mp[2*o],mp[2*o+1],vt0,vt1,type);
+  }
+
+  /*****************************************************/
+  /* Have a decompression step if mpi data is not same */
+  /*****************************************************/
+  inline void Decompress(SiteHalfSpinor *out,
+			 SiteHalfSpinor *in, Integer o){
+    SiteHalfCommSpinor *hin=(SiteHalfCommSpinor *)in;
+    precisionChange((vComplexHigh *)&out[o],(vComplexLow *)&hin[o],Nw);
+  }
+
+  /*****************************************************/
+  /* Compress Exchange                                 */
+  /*****************************************************/
+  inline void CompressExchange(SiteHalfSpinor *out0,
+			       SiteHalfSpinor *out1,
+			       const SiteSpinor *in,
+			       Integer j,Integer k, Integer m,Integer type){
+    SiteHalfSpinor temp1, temp2,temp3,temp4;
+    SiteHalfCommSpinor *hout0 = (SiteHalfCommSpinor *)out0;
+    SiteHalfCommSpinor *hout1 = (SiteHalfCommSpinor *)out1;
+    projector::Proj(temp1,in[k],mu,dag);
+    projector::Proj(temp2,in[m],mu,dag);
+    exchange(temp3,temp4,temp1,temp2,type);
+    precisionChange((vComplexLow *)&hout0[j],(vComplexHigh *)&temp3,Nw);
+    precisionChange((vComplexLow *)&hout1[j],(vComplexHigh *)&temp4,Nw);
+  }
+
+  /*****************************************************/
+  /* Pass the info to the stencil */
+  /*****************************************************/
+  inline bool DecompressionStep(void) { return true; }
+
+};
+
+#define DECLARE_PROJ(Projector,Compressor,spProj)			\
+  class Projector {							\
+  public:								\
+    template<class hsp,class fsp>					\
+    static void Proj(hsp &result,const fsp &in,int mu,int dag){			\
+      spProj(result,in);						\
+    }									\
+  };									\
+template<typename HCS,typename HS,typename S> using Compressor = WilsonCompressorTemplate<HCS,HS,S,Projector>;
+
+DECLARE_PROJ(WilsonXpProjector,WilsonXpCompressor,spProjXp);
+DECLARE_PROJ(WilsonYpProjector,WilsonYpCompressor,spProjYp);
+DECLARE_PROJ(WilsonZpProjector,WilsonZpCompressor,spProjZp);
+DECLARE_PROJ(WilsonTpProjector,WilsonTpCompressor,spProjTp);
+DECLARE_PROJ(WilsonXmProjector,WilsonXmCompressor,spProjXm);
+DECLARE_PROJ(WilsonYmProjector,WilsonYmCompressor,spProjYm);
+DECLARE_PROJ(WilsonZmProjector,WilsonZmCompressor,spProjZm);
+DECLARE_PROJ(WilsonTmProjector,WilsonTmCompressor,spProjTm);
+
+class WilsonProjector {
+ public:
+  template<class hsp,class fsp>
+  static void Proj(hsp &result,const fsp &in,int mu,int dag){
+    int mudag=dag? mu : (mu+Nd)%(2*Nd);
+    switch(mudag) {
+    case Xp:	spProjXp(result,in);	break;
+    case Yp:	spProjYp(result,in);	break;
+    case Zp:	spProjZp(result,in);	break;
+    case Tp:	spProjTp(result,in);	break;
+    case Xm:	spProjXm(result,in);	break;
+    case Ym:	spProjYm(result,in);	break;
+    case Zm:	spProjZm(result,in);	break;
+    case Tm:	spProjTm(result,in);	break;
+    default: 	assert(0);	        break;
+    }
+  }
+};
+template<typename HCS,typename HS,typename S> using WilsonCompressor = WilsonCompressorTemplate<HCS,HS,S,WilsonProjector>;
+
+// Fast comms buffer manipulation which should inline right through (avoid direction
+// dependent logic that prevents inlining
+template<class vobj,class cobj>
+class WilsonStencil : public CartesianStencil<vobj,cobj> {
+public:
+
+  typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
+
+  std::vector<int> same_node;
+  std::vector<int> surface_list;
+
+  WilsonStencil(GridBase *grid,
 		int npoints,
 		int checkerboard,
 		const std::vector<int> &directions,
-		const std::vector<int> &distances)  : CartesianStencil<vobj,cobj> (grid,npoints,checkerboard,directions,distances) 
-      {    };
-
-    template < class compressor>
-    void HaloExchangeOpt(const Lattice<vobj> &source,compressor &compress) 
-    {
-      std::vector<std::vector<CommsRequest_t> > reqs;
-      HaloExchangeOptGather(source,compress);
-      this->CommunicateBegin(reqs);
-      this->calls++;
-      this->CommunicateComplete(reqs);
-      this->CommsMerge();
-    }
-
-    template < class compressor>
-    void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress) 
-    {
-      this->calls++;
-      this->Mergers.resize(0); 
-      this->Packets.resize(0);
-      this->HaloGatherOpt(source,compress);
-    }
-
-
-    template < class compressor>
-    void HaloGatherOpt(const Lattice<vobj> &source,compressor &compress)
-    {
-      this->_grid->StencilBarrier();
-      // conformable(source._grid,_grid);
-      assert(source._grid==this->_grid);
-      this->halogtime-=usecond();
-      
-      this->u_comm_offset=0;
-      
-      int dag = compress.dag;
-      
-      WilsonXpCompressor<cobj,vobj> XpCompress; 
-      WilsonYpCompressor<cobj,vobj> YpCompress; 
-      WilsonZpCompressor<cobj,vobj> ZpCompress; 
-      WilsonTpCompressor<cobj,vobj> TpCompress;
-      WilsonXmCompressor<cobj,vobj> XmCompress;
-      WilsonYmCompressor<cobj,vobj> YmCompress;
-      WilsonZmCompressor<cobj,vobj> ZmCompress;
-      WilsonTmCompressor<cobj,vobj> TmCompress;
-
-      // Gather all comms buffers
-      //    for(int point = 0 ; point < _npoints; point++) {
-      //      compress.Point(point);
-      //      HaloGatherDir(source,compress,point,face_idx);
-      //    }
-      int face_idx=0;
-      if ( dag ) { 
-	//	std::cout << " Optimised Dagger compress " <<std::endl;
-	this->HaloGatherDir(source,XpCompress,Xp,face_idx);
-	this->HaloGatherDir(source,YpCompress,Yp,face_idx);
-	this->HaloGatherDir(source,ZpCompress,Zp,face_idx);
-	this->HaloGatherDir(source,TpCompress,Tp,face_idx);
-	this->HaloGatherDir(source,XmCompress,Xm,face_idx);
-	this->HaloGatherDir(source,YmCompress,Ym,face_idx);
-	this->HaloGatherDir(source,ZmCompress,Zm,face_idx);
-	this->HaloGatherDir(source,TmCompress,Tm,face_idx);
-      } else {
-	this->HaloGatherDir(source,XmCompress,Xp,face_idx);
-	this->HaloGatherDir(source,YmCompress,Yp,face_idx);
-	this->HaloGatherDir(source,ZmCompress,Zp,face_idx);
-	this->HaloGatherDir(source,TmCompress,Tp,face_idx);
-	this->HaloGatherDir(source,XpCompress,Xm,face_idx);
-	this->HaloGatherDir(source,YpCompress,Ym,face_idx);
-	this->HaloGatherDir(source,ZpCompress,Zm,face_idx);
-	this->HaloGatherDir(source,TpCompress,Tm,face_idx);
-      }
-      this->face_table_computed=1;
-      assert(this->u_comm_offset==this->_unified_buffer_size);
-      this->halogtime+=usecond();
-    }
-
+		const std::vector<int> &distances)  
+    : CartesianStencil<vobj,cobj> (grid,npoints,checkerboard,directions,distances) ,
+    same_node(npoints)
+  { 
+    surface_list.resize(0);
   };
 
+  void BuildSurfaceList(int Ls,int vol4){
+
+    // find same node for SHM
+    // Here we know the distance is 1 for WilsonStencil
+    for(int point=0;point<this->_npoints;point++){
+      same_node[point] = this->SameNode(point);
+      //      std::cout << " dir " <<point<<" same_node " <<same_node[point]<<std::endl;
+    }
+    
+    for(int site = 0 ;site< vol4;site++){
+      int local = 1;
+      for(int point=0;point<this->_npoints;point++){
+	if( (!this->GetNodeLocal(site*Ls,point)) && (!same_node[point]) ){ 
+	  local = 0;
+	}
+      }
+      if(local == 0) { 
+	surface_list.push_back(site);
+      }
+    }
+  }
+
+  template < class compressor>
+  void HaloExchangeOpt(const Lattice<vobj> &source,compressor &compress) 
+  {
+    std::vector<std::vector<CommsRequest_t> > reqs;
+    this->HaloExchangeOptGather(source,compress);
+    this->CommunicateBegin(reqs);
+    this->CommunicateComplete(reqs);
+    this->CommsMerge(compress);
+    this->CommsMergeSHM(compress);
+  }
+  
+  template <class compressor>
+  void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress) 
+  {
+    this->Prepare();
+    this->HaloGatherOpt(source,compress);
+  }
+
+  template <class compressor>
+  void HaloGatherOpt(const Lattice<vobj> &source,compressor &compress)
+  {
+    // Strategy. Inherit types from Compressor.
+    // Use types to select the write direction by directon compressor
+    typedef typename compressor::SiteSpinor         SiteSpinor;
+    typedef typename compressor::SiteHalfSpinor     SiteHalfSpinor;
+    typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor;
+
+    this->_grid->StencilBarrier();
+
+    assert(source._grid==this->_grid);
+    this->halogtime-=usecond();
+    
+    this->u_comm_offset=0;
+      
+    WilsonXpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> XpCompress; 
+    WilsonYpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> YpCompress; 
+    WilsonZpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> ZpCompress; 
+    WilsonTpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> TpCompress;
+    WilsonXmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> XmCompress; 
+    WilsonYmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> YmCompress; 
+    WilsonZmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> ZmCompress; 
+    WilsonTmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> TmCompress;
+
+    int dag = compress.dag;
+    int face_idx=0;
+    if ( dag ) { 
+      //	std::cout << " Optimised Dagger compress " <<std::endl;
+      assert(same_node[Xp]==this->HaloGatherDir(source,XpCompress,Xp,face_idx));
+      assert(same_node[Yp]==this->HaloGatherDir(source,YpCompress,Yp,face_idx));
+      assert(same_node[Zp]==this->HaloGatherDir(source,ZpCompress,Zp,face_idx));
+      assert(same_node[Tp]==this->HaloGatherDir(source,TpCompress,Tp,face_idx));
+      assert(same_node[Xm]==this->HaloGatherDir(source,XmCompress,Xm,face_idx));
+      assert(same_node[Ym]==this->HaloGatherDir(source,YmCompress,Ym,face_idx));
+      assert(same_node[Zm]==this->HaloGatherDir(source,ZmCompress,Zm,face_idx));
+      assert(same_node[Tm]==this->HaloGatherDir(source,TmCompress,Tm,face_idx));
+    } else {
+      assert(same_node[Xp]==this->HaloGatherDir(source,XmCompress,Xp,face_idx));
+      assert(same_node[Yp]==this->HaloGatherDir(source,YmCompress,Yp,face_idx));
+      assert(same_node[Zp]==this->HaloGatherDir(source,ZmCompress,Zp,face_idx));
+      assert(same_node[Tp]==this->HaloGatherDir(source,TmCompress,Tp,face_idx));
+      assert(same_node[Xm]==this->HaloGatherDir(source,XpCompress,Xm,face_idx));
+      assert(same_node[Ym]==this->HaloGatherDir(source,YpCompress,Ym,face_idx));
+      assert(same_node[Zm]==this->HaloGatherDir(source,ZpCompress,Zm,face_idx));
+      assert(same_node[Tm]==this->HaloGatherDir(source,TpCompress,Tm,face_idx));
+    }
+    this->face_table_computed=1;
+    assert(this->u_comm_offset==this->_unified_buffer_size);
+    this->halogtime+=usecond();
+  }
+
+ };
 
 }} // namespace close
 #endif