From ee11678b1fe595f5442ff800a655929c4e03a31b Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 3 Nov 2020 12:41:35 +0000
Subject: [PATCH 01/82] added Xi-to-Sigma rare decays

---
 Grid/qcd/utils/BaryonUtils.h | 169 +++++++++++++++++++++++++++++++++++
 1 file changed, 169 insertions(+)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 15516b56..e6b52a43 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -1292,4 +1292,173 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
   }  );//end loop over lattice sites
 }
 
+
+/***********************************************************************
+ * The following code is for Xi -> Sigma rare hypeon decays            *
+ **********************************************************************/
+
+/* Dq_loop is a quark line from t_H to t_H
+ * Dd_spec is a quark line from t_i to t_f
+ * Ds_spec is a quark line from t_i to t_f
+ * Dd_tf is a quark line from t_f to t_H
+ * Ds_ti is a quark line from t_i to t_H */
+template <class FImpl>
+template <class mobj, class mobj2, class robj>
+void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
+						 const mobj2 &Dd_spec,
+						 const mobj2 &Ds_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_xi,
+		                 		 const Gamma GammaB_sigma,
+						 robj &result)
+{
+
+  Gamma g5(Gamma::Algebra::Gamma5); 
+
+  auto DdG = Dd_spec * GammaB_sigma;
+  auto GDs = GammaB_xi * Ds_spec;
+  // Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
+  auto DsGDd = Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
+  // DsGDd * GammaB
+  auto DsGDdG = DsGDd * GammaB_sigma;
+  // GammaB * DsGDd
+  auto GDsGDd = GammaB_xi * DsGDd;
+  // GammaB * DsGDd * GammaB
+  auto GDsGDdG = GDsGDd * GammaB_sigma;
+  // \gamma_\mu^L * Dq_loop 
+  auto trGDq = trace(GammaH * Dq_loop);
+
+  for (int ie_s=0; ie_s < 6 ; ie_s++){
+    int a_s = epsilon[ie_s][0]; //a
+    int b_s = epsilon[ie_s][1]; //b
+    int c_s = epsilon[ie_s][2]; //c
+    for (int ie_x=0; ie_x < 6 ; ie_x++){
+      int a_x = epsilon[ie_x][0]; //a'
+      int b_x = epsilon[ie_x][1]; //b'
+      int c_x = epsilon[ie_x][2]; //c'
+      auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq;
+      for (int alpha_x=0; alpha_x<Ns; alpha_x++){
+      for (int beta_s=0; beta_s<Ns; beta_s++){
+        auto GDsGDdG_ab_ba = GDsGDd()(alpha_x,beta_s)(b_x,a_s);
+        auto Ds_ab_ab = Ds_spec()(alpha_x,beta_s)(a_x,b_s);
+        auto DsGDdG_ab_aa = DsGDd()(alpha_x,beta_s)(a_x,a_s);
+        auto GDs_ab_bb = GDs()(alpha_x,beta_s)(b_x,b_s);
+        for (int gamma_x=0; gamma_x<Ns; gamma_x++){
+          auto DdG_cb_ca = DdG()(gamma_x,beta_s)(c_x,a_s);
+          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+            result()(gamma_x,gamma_s)() -= ee_GD * Dd_spec()(gamma_x,gamma_s)(c_x,c_s) * GDsGDdG_ab_ba * Ds_ab_ab;
+            result()(gamma_x,gamma_s)() += ee_GD * DdG_cb_ca * GDsGDd()(alpha_x,gamma_s)(b_x,c_s) * Ds_ab_ab;
+            result()(gamma_x,gamma_s)() += ee_GD * Dd_spec()(gamma_x,gamma_s)(c_x,c_s) * DsGDdG_ab_aa * GDs_ab_bb;
+            result()(gamma_x,gamma_s)() -= ee_GD * DdG_cb_ca * DsGDd()(alpha_x,gamma_s)(a_x,c_s) * GDs_ab_bb;
+          }
+	}
+      }}
+    }
+  }
+}
+
+
+//Equivalent to "One-trace"
+/* Dq_loop is a quark line from t_H to t_H
+ * Dd_spec is a quark line from t_i to t_f
+ * Ds_spec is a quark line from t_i to t_f
+ * Dd_tf is a quark line from t_f to t_H
+ * Ds_ti is a quark line from t_i to t_H */
+template <class FImpl>
+template <class mobj, class mobj2, class robj>
+void BaryonUtils<FImpl>::XiToSigmaQ2EyeSite(const mobj &Dq_loop,
+						 const mobj2 &Dd_spec,
+						 const mobj2 &Ds_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_xi,
+		                 		 const Gamma GammaB_sigma,
+						 robj &result)
+{
+
+  Gamma g5(Gamma::Algebra::Gamma5); 
+
+  auto DdG = Dd_spec * GammaB_sigma;
+  auto GDs = GammaB_xi * Ds_spec;
+  // Ds * \gamma_\mu^L * Dq_loop * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
+  auto DsGDqGDd = Ds_ti * Gamma_H * Dq_loop * Gamma_H * g5 * adj(Dd_tf) * g5;
+  // DsGDd * GammaB
+  auto DsGDqGDdG = DsGDqGDd * GammaB_sigma;
+  // GammaB * DsGDd
+  auto GDsGDqGDd = GammaB_xi * DsGDqGDd;
+  // GammaB * DsGDd * GammaB
+  auto GDsGDqGDdG = GDsGDqGDd * GammaB_sigma;
+
+  for (int ie_s=0; ie_s < 6 ; ie_s++){
+    int a_s = epsilon[ie_s][0]; //a
+    int b_s = epsilon[ie_s][1]; //b
+    int c_s = epsilon[ie_s][2]; //c
+    for (int ie_x=0; ie_x < 6 ; ie_x++){
+      int a_x = epsilon[ie_x][0]; //a'
+      int b_x = epsilon[ie_x][1]; //b'
+      int c_x = epsilon[ie_x][2]; //c'
+      auto ee = epsilon_sgn[ie_s] * epsilon_sgn[ie_x];
+      for (int alpha_x=0; alpha_x<Ns; alpha_x++){
+      for (int beta_s=0; beta_s<Ns; beta_s++){
+        auto GDsGDqGDdG_ab_ba = GDsGDqGDdG()(alpha_x,beta_s)(b_x,a_s);
+        auto Ds_ab_ab = Ds_spec()(alpha_x,beta_s)(a_x,b_s);
+        auto DsGDqGDdG_ab_aa = GDsGDqGDdG()(alpha_x,beta_s)(a_x,a_s);
+        auto GDs_ab_bb = GDs()(alpha_x,beta_s)(b_x,b_s);
+          for (int gamma_x=0; gamma_x<Ns; gamma_x++){
+            auto DdG_cb_ca = DdG()(gamma_x, beta_s)(c_x,a_s);
+          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+            result()(gamma_x,gamma_s)() -= ee * Dd_spec()(gamma_x,gamma_s)(c_x,c_s) * GDsGDqGDdG_ab_ba * Ds_ab_ab;
+            result()(gamma_x,gamma_s)() += ee * DdG_cb_ca * GDsGDqGDd()(alpha_x,gamma_s)(b_x,c_s) * Ds_ab_ab;
+            result()(gamma_x,gamma_s)() += ee * Dd_spec()(gamma_x,gamma_s)(c_x,c_s) * DsGDqGDdG_ab_aa * GDs_ab_bb;
+            result()(gamma_x,gamma_s)() -= ee * DdG_cb_ca * DsGDqGDd()(alpha_x,gamma_s)(a_x,c_s) * GDs_ab_bb;
+          }}
+      }}
+    }
+  }
+}
+
+template<class FImpl>
+template <class mobj>
+void BaryonUtils<FImpl>::XiToSigmaEye(const PropagatorField &qq_loop,
+						 const mobj &Dd_spec,
+						 const mobj &Ds_spec,
+						 const PropagatorField &qd_tf,
+						 const PropagatorField &qs_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_xi,
+		                 		 const Gamma GammaB_sigma,
+						 const std::string op,
+						 SpinMatrixField &stn_corr)
+{
+
+  assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
+  assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
+
+  GridBase *grid = qs_ti.Grid();
+
+  autoView( vcorr, stn_corr, CpuWrite);
+  autoView( vq_loop , qq_loop, CpuRead);
+  autoView( vd_tf , qd_tf, CpuRead);
+  autoView( vs_ti , qs_ti, CpuRead);
+
+  accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+    auto Dq_loop = vq_loop[ss];
+    auto Dd_tf = vd_tf[ss];
+    auto Ds_ti = vs_ti[ss];
+    sobj result=Zero();
+    if(op == "Q1"){
+      XiToSigmaQ1EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
+    } else if(op == "Q2"){
+      XiToSigmaQ2EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
+    } else {
+      assert(0 && "Weak Operator not correctly specified");
+    }
+      vcorr[ss] = result; 
+  }  );//end loop over lattice sites
+}
+
+
 NAMESPACE_END(Grid);

From a3de7026c8d091b4d42eba5e5cb8e57419e0963e Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 3 Nov 2020 12:51:50 +0000
Subject: [PATCH 02/82] bugfix

---
 Grid/qcd/utils/BaryonUtils.h | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index e6b52a43..8e6bf722 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -207,6 +207,26 @@ public:
 				                 const Gamma GammaB_sigma,
 		                 		 const Gamma GammaB_nucl,
 						 robj &result);
+  template <class mobj, class mobj2, class robj>
+  static void XiToSigmaQ1EyeSite(const mobj &Dq_loop,
+						 const mobj2 &Dd_spec,
+						 const mobj2 &Ds_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_sigma,
+		                 		 const Gamma GammaB_nucl,
+						 robj &result);
+  template <class mobj, class mobj2, class robj>
+  static void XiToSigmaQ2EyeSite(const mobj &Dq_loop,
+						 const mobj2 &Dd_spec,
+						 const mobj2 &Ds_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_sigma,
+		                 		 const Gamma GammaB_nucl,
+						 robj &result);
   public:
   template <class mobj>
   static void SigmaToNucleonEye(const PropagatorField &qq_loop,
@@ -229,6 +249,17 @@ public:
 				 const Gamma GammaB_nucl,
 		                 const std::string op,
 				 SpinMatrixField &stn_corr);
+  template <class mobj>
+  static void XiToSigmaEye(const PropagatorField &qq_loop,
+				 const mobj &Dd_spec,
+				 const mobj &Ds_spec,
+				 const PropagatorField &qd_tf,
+				 const PropagatorField &qs_ti,
+				 const Gamma Gamma_H,
+				 const Gamma GammaB_sigma,
+				 const Gamma GammaB_nucl,
+		                 const std::string op,
+				 SpinMatrixField &stn_corr);
 };
 
 template <class FImpl> 

From 67023c334b16938c36af9f41647ba42891008e03 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 3 Nov 2020 13:07:37 +0000
Subject: [PATCH 03/82] bugfix

---
 Grid/qcd/utils/BaryonUtils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 8e6bf722..a281a669 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -1359,7 +1359,7 @@ void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
   // GammaB * DsGDd * GammaB
   auto GDsGDdG = GDsGDd * GammaB_sigma;
   // \gamma_\mu^L * Dq_loop 
-  auto trGDq = trace(GammaH * Dq_loop);
+  auto trGDq = trace(Gamma_H * Dq_loop);
 
   for (int ie_s=0; ie_s < 6 ; ie_s++){
     int a_s = epsilon[ie_s][0]; //a

From 4014dfd5b9a79223023ab2bee3789a7efde45945 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 3 Nov 2020 16:13:08 +0000
Subject: [PATCH 04/82] first tested version

---
 Grid/qcd/utils/BaryonUtils.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index a281a669..f5bc1480 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -1359,7 +1359,7 @@ void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
   // GammaB * DsGDd * GammaB
   auto GDsGDdG = GDsGDd * GammaB_sigma;
   // \gamma_\mu^L * Dq_loop 
-  auto trGDq = trace(Gamma_H * Dq_loop);
+  auto trGDq = TensorRemove(trace(Gamma_H * Dq_loop)); 
 
   for (int ie_s=0; ie_s < 6 ; ie_s++){
     int a_s = epsilon[ie_s][0]; //a
@@ -1369,7 +1369,7 @@ void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
       int a_x = epsilon[ie_x][0]; //a'
       int b_x = epsilon[ie_x][1]; //b'
       int c_x = epsilon[ie_x][2]; //c'
-      auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq;
+      auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq; 
       for (int alpha_x=0; alpha_x<Ns; alpha_x++){
       for (int beta_s=0; beta_s<Ns; beta_s++){
         auto GDsGDdG_ab_ba = GDsGDd()(alpha_x,beta_s)(b_x,a_s);

From 9bae6b889a94024b60c39bc2fb993eed7456e1b9 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 3 Nov 2020 20:03:09 +0000
Subject: [PATCH 05/82] speedup in Sigma-to-nucleon

---
 Grid/qcd/utils/BaryonUtils.h | 193 ++++++++++++++++++++++++++++++++++-
 1 file changed, 189 insertions(+), 4 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index f5bc1480..c0999a4a 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -1027,7 +1027,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-template <class FImpl>
+/*template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
 						 const mobj2 &Du_spec,
@@ -1071,6 +1071,50 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
       }}
     }
   }
+}*/
+template <class FImpl>
+template <class mobj, class mobj2, class robj>
+void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
+						 const mobj2 &Du_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_sigma,
+		                 		 const Gamma GammaB_nucl,
+						 robj &result)
+{
+
+  Gamma g5(Gamma::Algebra::Gamma5); 
+
+  auto DuG = Du_spec * GammaB_nucl;
+  // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
+  auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
+  // Dq_loop * \gamma_\mu^L
+  auto trDqG = TensorRemove(trace(Dq_loop * Gamma_H));
+
+  for (int ie_n=0; ie_n < 6 ; ie_n++){
+    int a_n = epsilon[ie_n][0]; //a
+    int b_n = epsilon[ie_n][1]; //b
+    int c_n = epsilon[ie_n][2]; //c
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = epsilon[ie_s][0]; //a'
+      int b_s = epsilon[ie_s][1]; //b'
+      int c_s = epsilon[ie_s][2]; //c'
+      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
+      for (int beta_n=0; beta_n<Ns; beta_n++){
+        auto GDsGDd_ab_bb = GDsGDd()(alpha_s,beta_n)(b_s,b_n);
+        auto ee_GDGDDG = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd_ab_bb * trDqG;
+	auto DuG_ab_aa = DuG()(alpha_s, beta_n)(a_s,a_n);
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+	  auto DuG_cb_ca = DuG()(gamma_s, beta_n)(c_s,a_n);
+          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+            result()(gamma_s,gamma_n)() += ee_GDGDDG * DuG_ab_aa * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
+            result()(gamma_s,gamma_n)() -= ee_GDGDDG * DuG_cb_ca * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
+	  }
+	}
+      }}
+    }
+  }
 }
 
 /* Du_ti is a quark line from t_i to t_H
@@ -1078,7 +1122,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-template <class FImpl>
+/*template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
 						 const mobj &Du_tf,
@@ -1130,6 +1174,55 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
       }}
     }
   }
+}*/
+template <class FImpl>
+template <class mobj, class mobj2, class robj>
+void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
+						 const mobj &Du_tf,
+						 const mobj2 &Du_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_sigma,
+		                 		 const Gamma GammaB_nucl,
+						 robj &result)
+{
+
+  Gamma g5(Gamma::Algebra::Gamma5); 
+
+  auto DuG = Du_spec * GammaB_nucl;
+  // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
+  auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
+  // Du_ti * \gamma_\mu^L * adj(Du_tf)
+  auto DuGHDu = Du_ti * Gamma_H * g5 * adj(Du_tf) * g5;
+  auto DuGHDuG = DuGHDu * GammaB_nucl;
+
+  for (int ie_n=0; ie_n < 6 ; ie_n++){
+    int a_n = epsilon[ie_n][0]; //a
+    int b_n = epsilon[ie_n][1]; //b
+    int c_n = epsilon[ie_n][2]; //c
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = epsilon[ie_s][0]; //a'
+      int b_s = epsilon[ie_s][1]; //b'
+      int c_s = epsilon[ie_s][2]; //c'
+      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
+      for (int beta_n=0; beta_n<Ns; beta_n++){
+        auto ee_GDGDDG = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd()(alpha_s,beta_n)(b_s,b_n);
+	auto DuGHDuG_ab_aa = DuGHDuG()(alpha_s, beta_n)(a_s,a_n);
+	auto DuG_ab_aa = DuG()(alpha_s,beta_n)(a_s,a_n);
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+	  auto DuGHDuG_cb_ca = DuGHDuG()(gamma_s, beta_n)(c_s,a_n);
+          auto DuG_cb_ca = DuG()(gamma_s,beta_n)(c_s,a_n);
+          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+            result()(gamma_s,gamma_n)() += ee_GDGDDG * DuG_cb_ca * DuGHDu()(alpha_s,gamma_n)(a_s,c_n);
+            result()(gamma_s,gamma_n)() += ee_GDGDDG * DuGHDuG_cb_ca * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
+            result()(gamma_s,gamma_n)() -= ee_GDGDDG * DuGHDuG_ab_aa * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
+            result()(gamma_s,gamma_n)() -= ee_GDGDDG * DuG_ab_aa * DuGHDu()(gamma_s,gamma_n)(c_s,c_n);
+          }
+        }
+      }}
+    }
+  }
 }
 
 //Equivalent to "One-trace"
@@ -1137,7 +1230,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-template <class FImpl>
+/*template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
 						 const mobj2 &Du_spec,
@@ -1181,6 +1274,47 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
       }}}
     }
   }
+}*/
+template <class FImpl>
+template <class mobj, class mobj2, class robj>
+void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
+						 const mobj2 &Du_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_sigma,
+		                 		 const Gamma GammaB_nucl,
+						 robj &result)
+{
+
+  Gamma g5(Gamma::Algebra::Gamma5); 
+
+  auto DuG = Du_spec * GammaB_nucl;
+  // Gamma^B * Ds * \gamma_\mu^L * Dq_loop * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
+  auto GDsGDqGDd = GammaB_sigma * Ds_ti * Gamma_H * Dq_loop * Gamma_H * g5 * adj(Dd_tf) * g5;
+
+  for (int ie_n=0; ie_n < 6 ; ie_n++){
+    int a_n = epsilon[ie_n][0]; //a
+    int b_n = epsilon[ie_n][1]; //b
+    int c_n = epsilon[ie_n][2]; //c
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = epsilon[ie_s][0]; //a'
+      int b_s = epsilon[ie_s][1]; //b'
+      int c_s = epsilon[ie_s][2]; //c'
+      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
+      for (int beta_n=0; beta_n<Ns; beta_n++){
+        auto ee_GDGDGD = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDqGDd()(alpha_s,beta_n)(b_s,b_n);
+	auto DuG_ab_aa = DuG()(alpha_s, beta_n)(a_s,a_n);
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+          auto DuG_cb_ca = DuG()(gamma_s, beta_n)(c_s,a_n);
+          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+            result()(gamma_s,gamma_n)() -= ee_GDGDGD * DuG_ab_aa * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
+            result()(gamma_s,gamma_n)() += ee_GDGDGD * DuG_cb_ca * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
+          }
+	}
+      }}
+    }
+  }
 }
 
 /* Du_ti is a quark line from t_i to t_H
@@ -1188,7 +1322,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-template <class FImpl>
+/*template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
 						 const mobj &Du_tf,
@@ -1240,6 +1374,57 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
       }}}
     }
   }
+}*/
+template <class FImpl>
+template <class mobj, class mobj2, class robj>
+void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
+						 const mobj &Du_tf,
+						 const mobj2 &Du_spec,
+						 const mobj &Dd_tf,
+						 const mobj &Ds_ti,
+				                 const Gamma Gamma_H,
+				                 const Gamma GammaB_sigma,
+		                 		 const Gamma GammaB_nucl,
+						 robj &result)
+{
+
+  Gamma g5(Gamma::Algebra::Gamma5); 
+
+  auto DuG = Du_spec * GammaB_nucl;
+  // Gamma^B * Ds * \gamma_\mu^L * adj(Du)
+  auto GDsGDu = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Du_tf) * g5;
+  // GDsGDu * GammaB
+  auto GDsGDuG = GDsGDu * GammaB_nucl;
+  // Du * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
+  auto DuGDd = Du_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
+
+  for (int ie_n=0; ie_n < 6 ; ie_n++){
+    int a_n = epsilon[ie_n][0]; //a
+    int b_n = epsilon[ie_n][1]; //b
+    int c_n = epsilon[ie_n][2]; //c
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = epsilon[ie_s][0]; //a'
+      int b_s = epsilon[ie_s][1]; //b'
+      int c_s = epsilon[ie_s][2]; //c'
+      auto ee = epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
+      for (int beta_n=0; beta_n<Ns; beta_n++){
+        auto DuGDd_ab_ab = DuGDd()(alpha_s,beta_n)(a_s,b_n);
+	auto GDsGDuG_ab_ba = GDsGDuG()(alpha_s, beta_n)(b_s,a_n);
+        auto DuG_ab_aa = DuG()(alpha_s,beta_n)(a_s,a_n);
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+          auto DuGDd_cb_cb = DuGDd()(gamma_s,beta_n)(c_s,b_n);
+	  auto DuG_cb_ca = DuG()(gamma_s,beta_n)(c_s,a_n);
+          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+            result()(gamma_s,gamma_n)() -= ee * DuGDd_ab_ab * DuG_cb_ca * GDsGDu()(alpha_s,gamma_n)(b_s,c_n);
+            result()(gamma_s,gamma_n)() -= ee * DuGDd_cb_cb * GDsGDuG_ab_ba * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
+            result()(gamma_s,gamma_n)() += ee * DuGDd_ab_ab * GDsGDuG_ab_ba * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
+            result()(gamma_s,gamma_n)() += ee * DuGDd_cb_cb * DuG_ab_aa * GDsGDu()(alpha_s,gamma_n)(b_s,c_n);
+          }
+        }
+      }}
+    }
+  }
 }
 
 template<class FImpl>

From 3594ce877beb22161dcce972f14bd89598972fd6 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 3 Nov 2020 20:04:30 +0000
Subject: [PATCH 06/82] speedup in Sigma-to-nucleon

---
 Grid/qcd/utils/BaryonUtils.h | 196 -----------------------------------
 1 file changed, 196 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index c0999a4a..8a4ff6ac 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -1027,51 +1027,6 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-/*template <class FImpl>
-template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
-{
-
-  Gamma g5(Gamma::Algebra::Gamma5); 
-
-  auto DuG = Du_spec * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
-  // Dq_loop * \gamma_\mu^L
-  auto DqG = Dq_loop * Gamma_H;
-
-  for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int beta_n=0; beta_n<Ns; beta_n++){
-        auto GDsGDd_ab_bb = GDsGDd()(alpha_s,beta_n)(b_s,b_n);
-        for (int tau2=0; tau2<Ns; tau2++){
-        for (int j=0; j<Nc; j++){
-          auto DqG_tt_jj = DqG()(tau2,tau2)(j,j);
-          auto ee_GDGDDG = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd_ab_bb * DqG_tt_jj;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-            result()(gamma_s,gamma_n)() += ee_GDGDDG * DuG()(alpha_s, beta_n)(a_s,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-            result()(gamma_s,gamma_n)() -= ee_GDGDDG * DuG()(gamma_s, beta_n)(c_s,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-          }}
-	}}
-      }}
-    }
-  }
-}*/
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
@@ -1122,59 +1077,6 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-/*template <class FImpl>
-template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
-						 const mobj &Du_tf,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
-{
-
-  Gamma g5(Gamma::Algebra::Gamma5); 
-
-  auto DuG = Du_spec * GammaB_nucl;
-  auto adjDu = g5 * adj(Du_tf) * g5;
-  auto adjDuG = adjDu * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
-  // Dq_loop * \gamma_\mu^L
-  auto DuGH = Du_ti * Gamma_H;
-
-  for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int beta_n=0; beta_n<Ns; beta_n++){
-        auto GDsGDd_ab_bb = GDsGDd()(alpha_s,beta_n)(b_s,b_n);
-        for (int tau2=0; tau2<Ns; tau2++){
-        for (int j=0; j<Nc; j++){
-          auto DuGH_at_aj = DuGH()(alpha_s,tau2)(a_s,j);
-          auto ee_GDGDDG_a = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd_ab_bb * DuGH_at_aj;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-            auto DuGH_gt_cj = DuGH()(gamma_s,tau2)(c_s,j);
-            auto ee_GDGDDG_c = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd_ab_bb * DuGH_gt_cj;
-            for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-              result()(gamma_s,gamma_n)() += ee_GDGDDG_a * DuG()(gamma_s, beta_n)(c_s,a_n) * adjDu()(tau2,gamma_n)(j,c_n);
-              result()(gamma_s,gamma_n)() += ee_GDGDDG_c * adjDuG()(tau2, beta_n)(j,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-              result()(gamma_s,gamma_n)() -= ee_GDGDDG_a * adjDuG()(tau2, beta_n)(j,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-              result()(gamma_s,gamma_n)() -= ee_GDGDDG_c * DuG()(alpha_s, beta_n)(a_s,a_n) * adjDu()(tau2,gamma_n)(j,c_n);
-            }
-	  }
-	}}
-      }}
-    }
-  }
-}*/
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
@@ -1230,51 +1132,6 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-/*template <class FImpl>
-template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
-{
-
-  Gamma g5(Gamma::Algebra::Gamma5); 
-
-  auto DuG = Du_spec * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L
-  auto GDsG = GammaB_sigma * Ds_ti * Gamma_H;
-  // Dq_loop * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto DqGDd = Dq_loop * Gamma_H * g5 * adj(Dd_tf) * g5;
-
-  for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int tau=0; tau<Ns; tau++){
-      for (int i=0; i<Nc; i++){
-	auto GDsG_at_bi = GDsG()(alpha_s,tau)(b_s,i);
-        for (int beta_n=0; beta_n<Ns; beta_n++){
-          auto DqGDd_tb_ib = DqGDd()(tau,beta_n)(i,b_n);
-	  auto ee_GDGDGD = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsG_at_bi * DqGDd_tb_ib;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-            result()(gamma_s,gamma_n)() -= ee_GDGDGD * DuG()(alpha_s, beta_n)(a_s,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-            result()(gamma_s,gamma_n)() += ee_GDGDGD * DuG()(gamma_s, beta_n)(c_s,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-          }}
-	}
-      }}}
-    }
-  }
-}*/
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
@@ -1322,59 +1179,6 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
  * Du_spec is a quark line from t_i to t_f
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
-/*template <class FImpl>
-template <class mobj, class mobj2, class robj>
-void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
-						 const mobj &Du_tf,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
-{
-
-  Gamma g5(Gamma::Algebra::Gamma5); 
-
-  auto DuG = Du_spec * GammaB_nucl;
-  auto adjDu = g5 * adj(Du_tf) * g5;
-  auto adjDuG = adjDu * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L
-  auto GDsG = GammaB_sigma * Ds_ti * Gamma_H;
-  // Du * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto DuGDd = Du_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
-
-  for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int tau=0; tau<Ns; tau++){
-      for (int i=0; i<Nc; i++){
-	auto GDsG_at_bi = GDsG()(alpha_s,tau)(b_s,i);
-        for (int beta_n=0; beta_n<Ns; beta_n++){
-          auto DuGDd_ab_ab = DuGDd()(alpha_s,beta_n)(a_s,b_n);
-	  auto ee_GDGDGD_a = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsG_at_bi * DuGDd_ab_ab;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-            auto DuGDd_gb_cb = DuGDd()(gamma_s,beta_n)(c_s,b_n);
-	    auto ee_GDGDGD_c = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsG_at_bi * DuGDd_gb_cb;
-            for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-              result()(gamma_s,gamma_n)() -= ee_GDGDGD_a * DuG()(gamma_s, beta_n)(c_s,a_n) * adjDu()(tau,gamma_n)(i,c_n);
-              result()(gamma_s,gamma_n)() -= ee_GDGDGD_c * adjDuG()(tau, beta_n)(i,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-              result()(gamma_s,gamma_n)() += ee_GDGDGD_a * adjDuG()(tau, beta_n)(i,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-              result()(gamma_s,gamma_n)() += ee_GDGDGD_c * DuG()(alpha_s, beta_n)(a_s,a_n) * adjDu()(tau,gamma_n)(i,c_n);
-            }
-	  }
-	}
-      }}}
-    }
-  }
-}*/
 template <class FImpl>
 template <class mobj, class mobj2, class robj>
 void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,

From c438118fd719b5fd908dc72dcd6ff3d6db83923c Mon Sep 17 00:00:00 2001
From: Daniel Richtmann <daniel.richtmann@gmail.com>
Date: Tue, 8 Dec 2020 14:42:11 +0100
Subject: [PATCH 07/82] Change access specifier of clover fields in order to
 allow deriving classes to access these

---
 Grid/qcd/action/fermion/WilsonCloverFermion.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/qcd/action/fermion/WilsonCloverFermion.h b/Grid/qcd/action/fermion/WilsonCloverFermion.h
index 91ad6d6d..92af7111 100644
--- a/Grid/qcd/action/fermion/WilsonCloverFermion.h
+++ b/Grid/qcd/action/fermion/WilsonCloverFermion.h
@@ -245,7 +245,7 @@ public:
     return out;
   }
 
-private:
+protected:
   // here fixing the 4 dimensions, make it more general?
 
   RealD csw_r;                                               // Clover coefficient - spatial

From 808f1e0e8c199204c7369fe2a033bc6041cbaa91 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 15 Dec 2020 16:33:29 +0000
Subject: [PATCH 08/82] merge develop

---
 tests/solver/Test_zMADWF_prec.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/solver/Test_zMADWF_prec.cc b/tests/solver/Test_zMADWF_prec.cc
index d1168764..f18e1d86 100644
--- a/tests/solver/Test_zMADWF_prec.cc
+++ b/tests/solver/Test_zMADWF_prec.cc
@@ -52,7 +52,7 @@ struct TestParams{
   bool zmobius_inner;
   double lambda_max; //upper bound of H_T eigenvalue range required to generate zMobius approximation
   
-  TestParams(): load_config(true), config_file("ckpoint_lat.1000"), mass(0.01),
+  TestParams(): load_config(false), config_file("ckpoint_lat.1000"), mass(0.01),
 		Ls_outer(24), b_plus_c_outer(2.0), resid_outer(1e-8),
 		Ls_inner(12), b_plus_c_inner(1.0), resid_inner(1e-8), zmobius_inner(true), lambda_max(1.42), outer_precon("Standard"), inner_precon("Standard")
   {}
@@ -246,7 +246,7 @@ void run(const TestParams &params){
   typename RunParamsInner::SchurSolverType SchurSolver_inner(CG_inner);
 
   ZeroGuesser<LatticeFermion> Guess;
-  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 100, &update);
+  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 10000, &update);
   
   LatticeFermionD result_MADWF(FGrid_outer);
   result_MADWF = Zero();

From f36d6f3923b7632e169b6740c94cc39ecc6bc8a9 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Thu, 17 Dec 2020 17:04:08 +0000
Subject: [PATCH 09/82] compiles on GPU. 3pt still wrong!!!!

---
 Grid/qcd/utils/A2Autils.h    |   2 +-
 Grid/qcd/utils/BaryonUtils.h | 991 +++++++++++++++++++----------------
 2 files changed, 544 insertions(+), 449 deletions(-)

diff --git a/Grid/qcd/utils/A2Autils.h b/Grid/qcd/utils/A2Autils.h
index b63d8571..497927dd 100644
--- a/Grid/qcd/utils/A2Autils.h
+++ b/Grid/qcd/utils/A2Autils.h
@@ -1047,7 +1047,7 @@ A2Autils<FImpl>::ContractWWVV(std::vector<PropagatorField> &WWVV,
 {
   GridBase *grid = vs[0].Grid();
 
-  int nd    = grid->_ndimension;
+  //int nd    = grid->_ndimension;
   int Nsimd = grid->Nsimd();
   int N_t = WW_sd.dimensions()[0];
   int N_s = WW_sd.dimensions()[1];
diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 15516b56..25c71e3a 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -44,24 +44,24 @@ public:
   typedef typename ComplexField::vector_object vobj;
 
   typedef Lattice<iSpinMatrix<typename FImpl::Simd>> SpinMatrixField;
-  typedef typename SpinMatrixField::vector_object sobj;
+  //typedef typename SpinMatrixField::vector_object sobj;
 
-  static const int epsilon[6][3] ;
-  static const Real epsilon_sgn[6];
+  //static const int epsilon[6][3] ;
+  //static const Real epsilon_sgn[6];
 
   private: 
-  template <class mobj, class robj>
+  template <class mobj, class robj> accelerator_inline
   static void BaryonSite(const mobj &D1,
-				 const mobj &D2,
-				 const mobj &D3,
-				 const Gamma GammaA_left,
-				 const Gamma GammaB_left,
-				 const Gamma GammaA_right,
-				 const Gamma GammaB_right,
-				 const int parity,
-				 const bool * wick_contractions,
-  				 robj &result);
-  template <class mobj, class robj>
+         const mobj &D2,
+         const mobj &D3,
+         const Gamma GammaA_left,
+         const Gamma GammaB_left,
+         const Gamma GammaA_right,
+         const Gamma GammaB_right,
+         const int parity,
+         const bool * wick_contractions,
+           robj &result);
+  template <class mobj, class robj> accelerator_inline
   static void BaryonSiteMatrix(const mobj &D1,
          const mobj &D2,
          const mobj &D3,
@@ -76,15 +76,15 @@ public:
                  std::string qf, 
                  bool* wick_contractions);
   static void ContractBaryons(const PropagatorField &q1_left,
-				 const PropagatorField &q2_left,
-				 const PropagatorField &q3_left,
-				 const Gamma GammaA_left,
-				 const Gamma GammaB_left,
-				 const Gamma GammaA_right,
-				 const Gamma GammaB_right,
-				 const bool* wick_contractions,
-				 const int parity,
-				 ComplexField &baryon_corr);
+         const PropagatorField &q2_left,
+         const PropagatorField &q3_left,
+         const Gamma GammaA_left,
+         const Gamma GammaB_left,
+         const Gamma GammaA_right,
+         const Gamma GammaB_right,
+         const bool* wick_contractions,
+         const int parity,
+         ComplexField &baryon_corr);
   static void ContractBaryonsMatrix(const PropagatorField &q1_left,
          const PropagatorField &q2_left,
          const PropagatorField &q3_left,
@@ -96,16 +96,16 @@ public:
          SpinMatrixField &baryon_corr);
   template <class mobj, class robj>
   static void ContractBaryonsSliced(const mobj &D1,
-				 const mobj &D2,
-				 const mobj &D3,
-				 const Gamma GammaA_left,
-				 const Gamma GammaB_left,
-				 const Gamma GammaA_right,
-				 const Gamma GammaB_right,
-				 const bool* wick_contractions,
-				 const int parity,
-				 const int nt,
-				 robj &result);
+         const mobj &D2,
+         const mobj &D3,
+         const Gamma GammaA_left,
+         const Gamma GammaB_left,
+         const Gamma GammaA_right,
+         const Gamma GammaB_right,
+         const bool* wick_contractions,
+         const int parity,
+         const int nt,
+         robj &result);
   template <class mobj, class robj>
   static void ContractBaryonsSlicedMatrix(const mobj &D1,
          const mobj &D2,
@@ -118,11 +118,11 @@ public:
          const int nt,
          robj &result);
   private:
-  template <class mobj, class mobj2, class robj>
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void BaryonGamma3ptGroup1Site(
            const mobj &Dq1_ti,
            const mobj2 &Dq2_spec,
-           const mobj2 &Dq3_spec,
+          // const mobj2 &Dq3_spec,
            const mobj &Dq4_tf,
                    const Gamma GammaJ,
                    const Gamma GammaBi,
@@ -130,11 +130,11 @@ public:
            int wick_contraction,
            robj &result);
 
-  template <class mobj, class mobj2, class robj>
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void BaryonGamma3ptGroup2Site(
            const mobj2 &Dq1_spec,
            const mobj &Dq2_ti,
-           const mobj2 &Dq3_spec,
+           //const mobj2 &Dq3_spec,
            const mobj &Dq4_tf,
                    const Gamma GammaJ,
                    const Gamma GammaBi,
@@ -142,10 +142,10 @@ public:
            int wick_contraction,
            robj &result);
 
-  template <class mobj, class mobj2, class robj>
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void BaryonGamma3ptGroup3Site(
            const mobj2 &Dq1_spec,
-           const mobj2 &Dq2_spec,
+           //const mobj2 &Dq2_spec,
            const mobj &Dq3_ti,
            const mobj &Dq4_tf,
                    const Gamma GammaJ,
@@ -167,86 +167,78 @@ public:
                    const Gamma GammaBf,
            SpinMatrixField &stn_corr);
   private: 
-  template <class mobj, class mobj2, class robj>
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result);
-  template <class mobj, class mobj2, class robj>
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result);
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
-						 const mobj &Du_tf,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result);
+             const mobj &Du_tf,
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result);
 
 
-  template <class mobj, class mobj2, class robj>
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result);
-  template <class mobj, class mobj2, class robj>
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result);
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
-						 const mobj &Du_tf,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result);
+             const mobj &Du_tf,
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result);
   public:
   template <class mobj>
   static void SigmaToNucleonEye(const PropagatorField &qq_loop,
-				 const mobj &Du_spec,
-				 const PropagatorField &qd_tf,
-				 const PropagatorField &qs_ti,
-				 const Gamma Gamma_H,
-				 const Gamma GammaB_sigma,
-				 const Gamma GammaB_nucl,
-		                 const std::string op,
-				 SpinMatrixField &stn_corr);
+         const mobj &Du_spec,
+         const PropagatorField &qd_tf,
+         const PropagatorField &qs_ti,
+         const Gamma Gamma_H,
+         const Gamma GammaB_sigma,
+         const Gamma GammaB_nucl,
+                     const std::string op,
+         SpinMatrixField &stn_corr);
   template <class mobj>
   static void SigmaToNucleonNonEye(const PropagatorField &qq_ti,
-				 const PropagatorField &qq_tf,
-				 const mobj &Du_spec,
-				 const PropagatorField &qd_tf,
-				 const PropagatorField &qs_ti,
-				 const Gamma Gamma_H,
-				 const Gamma GammaB_sigma,
-				 const Gamma GammaB_nucl,
-		                 const std::string op,
-				 SpinMatrixField &stn_corr);
+         const PropagatorField &qq_tf,
+         const mobj &Du_spec,
+         const PropagatorField &qd_tf,
+         const PropagatorField &qs_ti,
+         const Gamma Gamma_H,
+         const Gamma GammaB_sigma,
+         const Gamma GammaB_nucl,
+                     const std::string op,
+         SpinMatrixField &stn_corr);
 };
-
+/*
 template <class FImpl> 
 const int BaryonUtils<FImpl>::epsilon[6][3] = {{0,1,2},{1,2,0},{2,0,1},{0,2,1},{2,1,0},{1,0,2}};
-/*template <class FImpl> 
-const Complex BaryonUtils<FImpl>::epsilon_sgn[6] = {Complex(1),
-						    Complex(1),
-						    Complex(1),
-						    Complex(-1),
-						    Complex(-1),
-						    Complex(-1)};
-*/
 template <class FImpl> 
 const Real BaryonUtils<FImpl>::epsilon_sgn[6] = {1.,1.,1.,-1.,-1.,-1.};
-
+*/
 //This is the old version
 template <class FImpl>
-template <class mobj, class robj>
+template <class mobj, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
                 const mobj &D2,
                 const mobj &D3,
@@ -274,16 +266,20 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
     auto GBf_D3 = GammaB_f * D3;
     auto GAf_D3 = GammaA_f * D3;
 
-    for (int ie_f=0; ie_f < 6 ; ie_f++){
-        int a_f = epsilon[ie_f][0]; //a
-        int b_f = epsilon[ie_f][1]; //b
-        int c_f = epsilon[ie_f][2]; //c
-    for (int ie_i=0; ie_i < 6 ; ie_i++){
-        int a_i = epsilon[ie_i][0]; //a'
-        int b_i = epsilon[ie_i][1]; //b'
-        int c_i = epsilon[ie_i][2]; //c'
+  Real ee;
 
-        Real ee = epsilon_sgn[ie_f] * epsilon_sgn[ie_i];
+  for (int ie_f=0; ie_f < 6 ; ie_f++){
+    int a_f    = (ie_f < 3 ? ie_f       : (6-ie_f)%3 ); //epsilon[ie_n][0]; //a
+    int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
+    int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_f = (ie_f < 3 ? 1 : -1);
+  for (int ie_i=0; ie_i < 6 ; ie_i++){
+    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_i = (ie_i < 3 ? 1 : -1);
+
+    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
         //This is the \delta_{456}^{123} part
         if (wick_contraction[0]){
             for (int rho=0; rho<Ns; rho++){
@@ -361,7 +357,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
 
 //New version without parity projection or trace
 template <class FImpl>
-template <class mobj, class robj>
+template <class mobj, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
                 const mobj &D2,
                 const mobj &D3,
@@ -384,16 +380,21 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
     auto GBf_D3 = GammaB_f * D3;
     auto GAf_D3 = GammaA_f * D3;
 
-    for (int ie_f=0; ie_f < 6 ; ie_f++){
-        int a_f = epsilon[ie_f][0]; //a
-        int b_f = epsilon[ie_f][1]; //b
-        int c_f = epsilon[ie_f][2]; //c
-    for (int ie_i=0; ie_i < 6 ; ie_i++){
-        int a_i = epsilon[ie_i][0]; //a'
-        int b_i = epsilon[ie_i][1]; //b'
-        int c_i = epsilon[ie_i][2]; //c'
 
-        Real ee = epsilon_sgn[ie_f] * epsilon_sgn[ie_i];
+  Real ee;
+
+  for (int ie_f=0; ie_f < 6 ; ie_f++){
+    int a_f    = (ie_f < 3 ? ie_f       : (6-ie_f)%3 ); //epsilon[ie_n][0]; //a
+    int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
+    int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_f = (ie_f < 3 ? 1 : -1);
+  for (int ie_i=0; ie_i < 6 ; ie_i++){
+    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_i = (ie_i < 3 ? 1 : -1);
+
+    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
         //This is the \delta_{456}^{123} part
         if (wick_contraction[0]){
             for (int rho_i=0; rho_i<Ns; rho_i++){
@@ -501,15 +502,15 @@ void BaryonUtils<FImpl>::WickContractions(std::string qi, std::string qf, bool*
  * Wick_Contractions function above                               */
 template<class FImpl>
 void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
-						 const PropagatorField &q2_left,
-						 const PropagatorField &q3_left,
-				                 const Gamma GammaA_left,
-				                 const Gamma GammaB_left,
-				                 const Gamma GammaA_right,
-		                 		 const Gamma GammaB_right,
-						 const bool* wick_contractions,
-						 const int parity,
-						 ComplexField &baryon_corr)
+             const PropagatorField &q2_left,
+             const PropagatorField &q3_left,
+                         const Gamma GammaA_left,
+                         const Gamma GammaB_left,
+                         const Gamma GammaA_right,
+                         const Gamma GammaB_right,
+             const bool* wick_contractions,
+             const int parity,
+             ComplexField &baryon_corr)
 {
 
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
@@ -519,10 +520,10 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
 
   GridBase *grid = q1_left.Grid();
   
-  autoView(vbaryon_corr, baryon_corr,CpuWrite);
-  autoView( v1 , q1_left, CpuRead);
-  autoView( v2 , q2_left, CpuRead);
-  autoView( v3 , q3_left, CpuRead);
+  autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( v1          , q1_left     , AcceleratorRead);
+  autoView( v2          , q2_left     , AcceleratorRead);
+  autoView( v3          , q3_left     , AcceleratorRead);
 
   Real bytes =0.;
   bytes += grid->oSites() * (432.*sizeof(vComplex) + 126.*sizeof(int) + 36.*sizeof(Real));
@@ -538,12 +539,13 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
   t =-usecond();
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto D1 = v1[ss];
-    auto D2 = v2[ss];
-    auto D3 = v3[ss];
-    vobj result=Zero();
+    auto D1 = v1(ss);
+    auto D2 = v2(ss);
+    auto D3 = v3(ss);
+    typedef decltype(coalescedRead(vbaryon_corr[0])) cVec;
+    cVec result=Zero();
     BaryonSite(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,parity,wick_contractions,result);
-    vbaryon_corr[ss] = result; 
+    coalescedWrite(vbaryon_corr[ss],result);
   }  );//end loop over lattice sites
 
   t += usecond();
@@ -567,38 +569,21 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
   assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
  
   GridBase *grid = q1_left.Grid();
-  
-  autoView(vbaryon_corr, baryon_corr,CpuWrite);
-  autoView( v1 , q1_left, CpuRead);
-  autoView( v2 , q2_left, CpuRead);
-  autoView( v3 , q3_left, CpuRead);
 
-  // Real bytes =0.;
-  // bytes += grid->oSites() * (432.*sizeof(vComplex) + 126.*sizeof(int) + 36.*sizeof(Real));
-  // for (int ie=0; ie < 6 ; ie++){
-  //   if(ie==0 or ie==3){
-  //      bytes += grid->oSites() * (4.*sizeof(int) + 4752.*sizeof(vComplex)) * wick_contractions[ie];
-  //   }
-  //   else{
-  //      bytes += grid->oSites() * (64.*sizeof(int) + 5184.*sizeof(vComplex)) * wick_contractions[ie];
-  //   }
-  // }
-  // Real t=0.;
-  // t =-usecond();
+  autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( v1          , q1_left     , AcceleratorRead);
+  autoView( v2          , q2_left     , AcceleratorRead);
+  autoView( v3          , q3_left     , AcceleratorRead);
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto D1 = v1[ss];
-    auto D2 = v2[ss];
-    auto D3 = v3[ss];
-    sobj result=Zero();
+    auto D1 = v1(ss);
+    auto D2 = v2(ss);
+    auto D3 = v3(ss);
+    typedef decltype(coalescedRead(vbaryon_corr[0])) spinor;
+    spinor result=Zero();
     BaryonSiteMatrix(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,wick_contractions,result);
-    vbaryon_corr[ss] = result; 
+    coalescedWrite(vbaryon_corr[ss],result);
   }  );//end loop over lattice sites
-
-  // t += usecond();
-
-  // std::cout << GridLogDebug << std::setw(10) << bytes/t*1.0e6/1024/1024/1024 << " GB/s " << std::endl;
-
 }
 
 /* The array wick_contractions must be of length 6. The order     * 
@@ -609,16 +594,16 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
 template <class FImpl>
 template <class mobj, class robj>
 void BaryonUtils<FImpl>::ContractBaryonsSliced(const mobj &D1,
-						 const mobj &D2,
-						 const mobj &D3,
-				                 const Gamma GammaA_left,
-				                 const Gamma GammaB_left,
-				                 const Gamma GammaA_right,
-		                 		 const Gamma GammaB_right,
-						 const bool* wick_contractions,
-						 const int parity,
-						 const int nt,
-						 robj &result)
+             const mobj &D2,
+             const mobj &D3,
+                         const Gamma GammaA_left,
+                         const Gamma GammaB_left,
+                         const Gamma GammaA_right,
+                         const Gamma GammaB_right,
+             const bool* wick_contractions,
+             const int parity,
+             const int nt,
+             robj &result)
 {
 
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
@@ -664,11 +649,11 @@ void BaryonUtils<FImpl>::ContractBaryonsSlicedMatrix(const mobj &D1,
  * Dq3_spec is a quark line from t_i to t_f
  * Dq4_tf is a quark line from t_f to t_J */
 template<class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
                         const mobj &Dq1_ti,
                         const mobj2 &Dq2_spec,
-                        const mobj2 &Dq3_spec,
+           //             const mobj2 &Dq3_spec,
                         const mobj &Dq4_tf,
                                 const Gamma GammaJ,
                                 const Gamma GammaBi,
@@ -678,41 +663,47 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
 {
     Gamma g5(Gamma::Algebra::Gamma5); 
 
-    auto adjD4_g_D1     = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq1_ti;
+//    auto adjD4_g_D1     = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq1_ti;
+    auto adjD4          = g5 * adj(Dq4_tf) * g5 ;
+    auto adjD4_g_D1     = adjD4 * GammaJ * Dq1_ti;
     auto Gf_adjD4_g_D1  = GammaBf * adjD4_g_D1;
     auto D2_Gi          = Dq2_spec * GammaBi;
     auto Gf_D2_Gi       = GammaBf * D2_Gi;
-    auto Gf_D3          = GammaBf * Dq3_spec;
 
-    int a_f, b_f, c_f;
-    int a_i, b_i, c_i;
+//    auto Gf_D3          = GammaBf * Dq3_spec;  // including a second mobj2 parameter leads to compilation error
+    auto Gf_D3          = GammaBf * Dq2_spec;  //WRONG!!!!!
 
-    Real ee;
 
-    for (int ie_f=0; ie_f < 6 ; ie_f++){
-        a_f = epsilon[ie_f][0]; //a
-        b_f = epsilon[ie_f][1]; //b
-        c_f = epsilon[ie_f][2]; //c
-    for (int ie_i=0; ie_i < 6 ; ie_i++){
-        a_i = epsilon[ie_i][0]; //a'
-        b_i = epsilon[ie_i][1]; //b'
-        c_i = epsilon[ie_i][2]; //c'
+  Real ee;
 
-        ee = epsilon_sgn[ie_f] * epsilon_sgn[ie_i];
+  for (int ie_f=0; ie_f < 6 ; ie_f++){
+    int a_f    = (ie_f < 3 ? ie_f       : (6-ie_f)%3 ); //epsilon[ie_n][0]; //a
+    int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
+    int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_f = (ie_f < 3 ? 1 : -1);
+  for (int ie_i=0; ie_i < 6 ; ie_i++){
+    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_i = (ie_i < 3 ? 1 : -1);
+
+    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
         for (int beta_i=0; beta_i<Ns; beta_i++){
             auto D2_Gi_ab_aa        = D2_Gi     ()(alpha_f,beta_i)(a_f,a_i);
             auto Gf_D3_ab_bb        = Gf_D3     ()(alpha_f,beta_i)(b_f,b_i);
             auto Gf_D2_Gi_ab_ba     = Gf_D2_Gi  ()(alpha_f,beta_i)(b_f,a_i);
-            auto Dq3_spec_ab_ab     = Dq3_spec  ()(alpha_f,beta_i)(a_f,b_i);
+            //auto Dq3_spec_ab_ab     = Dq3_spec  ()(alpha_f,beta_i)(a_f,b_i);
+            auto Dq3_spec_ab_ab     = Dq2_spec  ()(alpha_f,beta_i)(a_f,b_i); // WRONG!!!!
 
             for (int gamma_i=0; gamma_i<Ns; gamma_i++){
                 auto ee_adjD4_g_D1_ag_ac        = ee * adjD4_g_D1   ()(alpha_f,gamma_i)(a_f,c_i);
                 auto ee_Gf_adjD4_g_D1_ag_bc     = ee * Gf_adjD4_g_D1()(alpha_f,gamma_i)(b_f,c_i);
             for (int gamma_f=0; gamma_f<Ns; gamma_f++){
                 auto ee_adjD4_g_D1_gg_cc        = ee * adjD4_g_D1   ()(gamma_f,gamma_i)(c_f,c_i);
-                auto Dq3_spec_gb_cb             = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
+                //auto Dq3_spec_gb_cb             = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
+                auto Dq3_spec_gb_cb             = Dq2_spec          ()(gamma_f,beta_i)(c_f,b_i); //WRONG!!!!
                 auto D2_Gi_gb_ca                = D2_Gi             ()(gamma_f,beta_i)(c_f,a_i);
 
 
@@ -756,11 +747,11 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
  * Dq3_spec is a quark line from t_i to t_f
  * Dq4_tf is a quark line from t_f to t_J */
 template<class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
                         const mobj2 &Dq1_spec,
                         const mobj &Dq2_ti,
-                        const mobj2 &Dq3_spec,
+                    //    const mobj2 &Dq3_spec,
                         const mobj &Dq4_tf,
                                 const Gamma GammaJ,
                                 const Gamma GammaBi,
@@ -773,37 +764,40 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
     auto adjD4_g_D2_Gi      = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq2_ti * GammaBi;
     auto Gf_adjD4_g_D2_Gi   = GammaBf * adjD4_g_D2_Gi;
     auto Gf_D1              = GammaBf * Dq1_spec;
-    auto Gf_D3              = GammaBf * Dq3_spec;
+    //auto Gf_D3              = GammaBf * Dq3_spec;
+    auto Gf_D3              = GammaBf * Dq1_spec; // WRONG!!!!!
 
-    int a_f, b_f, c_f;
-    int a_i, b_i, c_i;
 
-    Real ee;
+  Real ee;
 
-    for (int ie_f=0; ie_f < 6 ; ie_f++){
-        a_f = epsilon[ie_f][0]; //a
-        b_f = epsilon[ie_f][1]; //b
-        c_f = epsilon[ie_f][2]; //c
-    for (int ie_i=0; ie_i < 6 ; ie_i++){
-        a_i = epsilon[ie_i][0]; //a'
-        b_i = epsilon[ie_i][1]; //b'
-        c_i = epsilon[ie_i][2]; //c'
+  for (int ie_f=0; ie_f < 6 ; ie_f++){
+    int a_f    = (ie_f < 3 ? ie_f       : (6-ie_f)%3 ); //epsilon[ie_n][0]; //a
+    int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
+    int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_f = (ie_f < 3 ? 1 : -1);
+  for (int ie_i=0; ie_i < 6 ; ie_i++){
+    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_i = (ie_i < 3 ? 1 : -1);
 
-        ee = epsilon_sgn[ie_f] * epsilon_sgn[ie_i];
+    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
         for (int beta_i=0; beta_i<Ns; beta_i++){
             auto adjD4_g_D2_Gi_ab_aa        = adjD4_g_D2_Gi     ()(alpha_f,beta_i)(a_f,a_i);
             auto Gf_D3_ab_bb                = Gf_D3             ()(alpha_f,beta_i)(b_f,b_i);
             auto Gf_adjD4_g_D2_Gi_ab_ba     = Gf_adjD4_g_D2_Gi  ()(alpha_f,beta_i)(b_f,a_i);
-            auto Dq3_spec_ab_ab             = Dq3_spec          ()(alpha_f,beta_i)(a_f,b_i);
+            //auto Dq3_spec_ab_ab             = Dq3_spec          ()(alpha_f,beta_i)(a_f,b_i);
+            auto Dq3_spec_ab_ab             = Dq1_spec          ()(alpha_f,beta_i)(a_f,b_i); //WRONG!!!
 
             for (int gamma_i=0; gamma_i<Ns; gamma_i++){ 
                 auto ee_Dq1_spec_ag_ac      = ee * Dq1_spec     ()(alpha_f,gamma_i)(a_f,c_i);
                 auto ee_Gf_D1_ag_bc         = ee * Gf_D1        ()(alpha_f,gamma_i)(b_f,c_i);
             for (int gamma_f=0; gamma_f<Ns; gamma_f++){
                 auto ee_Dq1_spec_gg_cc      = ee * Dq1_spec     ()(gamma_f,gamma_i)(c_f,c_i);
-                auto Dq3_spec_gb_cb         = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
+                //auto Dq3_spec_gb_cb         = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
+                auto Dq3_spec_gb_cb         = Dq1_spec          ()(gamma_f,beta_i)(c_f,b_i); //WRONG!!!
                 auto adjD4_g_D2_Gi_gb_ca    = adjD4_g_D2_Gi     ()(gamma_f,beta_i)(c_f,a_i);
 
                 if(wick_contraction == 1) { // Do contraction II1
@@ -846,10 +840,10 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
  * Dq3_ti is a quark line from t_i to t_J
  * Dq4_tf is a quark line from t_f to t_J */
 template<class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
                         const mobj2 &Dq1_spec,
-                        const mobj2 &Dq2_spec,
+                       // const mobj2 &Dq2_spec,
                         const mobj &Dq3_ti,
                         const mobj &Dq4_tf,
                                 const Gamma GammaJ,
@@ -863,24 +857,25 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
     auto adjD4_g_D3     = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq3_ti;
     auto Gf_adjD4_g_D3  = GammaBf * adjD4_g_D3;
     auto Gf_D1          = GammaBf * Dq1_spec;
-    auto D2_Gi          = Dq2_spec * GammaBi;
+    //auto D2_Gi          = Dq2_spec * GammaBi;
+    auto D2_Gi          = Dq1_spec * GammaBi; //WRONG!!!!!!!!!!!!!!!!!
     auto Gf_D2_Gi       = GammaBf * D2_Gi;
 
-    int a_f, b_f, c_f;
-    int a_i, b_i, c_i;
 
-    Real ee;
+  Real ee;
 
-    for (int ie_f=0; ie_f < 6 ; ie_f++){
-        a_f = epsilon[ie_f][0]; //a
-        b_f = epsilon[ie_f][1]; //b
-        c_f = epsilon[ie_f][2]; //c
-    for (int ie_i=0; ie_i < 6 ; ie_i++){
-        a_i = epsilon[ie_i][0]; //a'
-        b_i = epsilon[ie_i][1]; //b'
-        c_i = epsilon[ie_i][2]; //c'
+  for (int ie_f=0; ie_f < 6 ; ie_f++){
+    int a_f    = (ie_f < 3 ? ie_f       : (6-ie_f)%3 ); //epsilon[ie_n][0]; //a
+    int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
+    int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_f = (ie_f < 3 ? 1 : -1);
+  for (int ie_i=0; ie_i < 6 ; ie_i++){
+    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_i = (ie_i < 3 ? 1 : -1);
 
-        ee = epsilon_sgn[ie_f] * epsilon_sgn[ie_i];
+    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
         for (int beta_i=0; beta_i<Ns; beta_i++){
@@ -951,44 +946,91 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
                                 const Gamma GammaBf,
                         SpinMatrixField &stn_corr)
 {
+  assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
+  assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
+
     GridBase *grid = q_tf.Grid();
 
-    autoView( vcorr, stn_corr, CpuWrite);
-    autoView( vq_ti , q_ti, CpuRead);
-    autoView( vq_tf , q_tf, CpuRead);
+    // autoView( vcorr, stn_corr, CpuWrite);
+    // autoView( vq_ti , q_ti, CpuRead);
+    // autoView( vq_tf , q_tf, CpuRead);
+
+    // if (group == 1) {
+    //     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+    //         auto Dq_ti = vq_ti[ss];
+    //         auto Dq_tf = vq_tf[ss];
+    //         sobj result=Zero();
+    //         BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+    //         vcorr[ss] += result; 
+    //     });//end loop over lattice sites
+    // } else if (group == 2) {
+    //     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+    //         auto Dq_ti = vq_ti[ss];
+    //         auto Dq_tf = vq_tf[ss];
+    //         sobj result=Zero();
+    //         BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+    //         vcorr[ss] += result; 
+    //     });//end loop over lattice sites
+    // } else if (group == 3) {
+    //     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+    //         auto Dq_ti = vq_ti[ss];
+    //         auto Dq_tf = vq_tf[ss];
+    //         sobj result=Zero();
+    //         BaryonGamma3ptGroup3Site(Dq_spec1,Dq_spec2,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+
+    //         vcorr[ss] += result; 
+    //     });//end loop over lattice sites
+    // }
+
+    autoView( vcorr , stn_corr , AcceleratorWrite);
+    autoView( vq_ti , q_ti     , AcceleratorRead);
+    autoView( vq_tf , q_tf     , AcceleratorRead);
 
     if (group == 1) {
         accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-            auto Dq_ti = vq_ti[ss];
-            auto Dq_tf = vq_tf[ss];
-            sobj result=Zero();
-            BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-            vcorr[ss] += result; 
+            auto Dq_ti = vq_ti(ss);
+            auto Dq_tf = vq_tf(ss);
+            //sobj result=Zero();
+            typedef decltype(coalescedRead(vcorr[0])) spinor;
+            spinor result=Zero();
+            //BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+            BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); //WRONG
+            // vcorr[ss] += result; 
+            coalescedWrite(vcorr[ss],result);
         });//end loop over lattice sites
+
     } else if (group == 2) {
         accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-            auto Dq_ti = vq_ti[ss];
-            auto Dq_tf = vq_tf[ss];
-            sobj result=Zero();
-            BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-            vcorr[ss] += result; 
+            auto Dq_ti = vq_ti(ss);
+            auto Dq_tf = vq_tf(ss);
+            //sobj result=Zero();
+            typedef decltype(coalescedRead(vcorr[0])) spinor;
+            spinor result=Zero();
+           // BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+            BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); //WRONG
+            // vcorr[ss] += result; 
+            coalescedWrite(vcorr[ss],result);
         });//end loop over lattice sites
     } else if (group == 3) {
         accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-            auto Dq_ti = vq_ti[ss];
-            auto Dq_tf = vq_tf[ss];
-            sobj result=Zero();
-            BaryonGamma3ptGroup3Site(Dq_spec1,Dq_spec2,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-
-            vcorr[ss] += result; 
+            auto Dq_ti = vq_ti(ss);
+            auto Dq_tf = vq_tf(ss);
+            //sobj result=Zero();
+            typedef decltype(coalescedRead(vcorr[0])) spinor;
+            spinor result=Zero();
+            //BaryonGamma3ptGroup3Site(Dq_spec1,Dq_spec2,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+            BaryonGamma3ptGroup3Site(Dq_spec1,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); //WRONG
+            // vcorr[ss] += result; 
+            coalescedWrite(vcorr[ss],result);
         });//end loop over lattice sites
     }
+
 }
 
 
 /***********************************************************************
  * End of BaryonGamma3pt-function code.                                *
- *																	   *
+ *                                     *
  * The following code is for Sigma -> N rare hypeon decays             *
  **********************************************************************/
 
@@ -997,49 +1039,60 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result)
 {
 
   Gamma g5(Gamma::Algebra::Gamma5); 
 
-  auto DuG = Du_spec * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
-  // Dq_loop * \gamma_\mu^L
-  auto DqG = Dq_loop * Gamma_H;
+  //auto Gn_adjDd_GH_Ds   = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
+  auto adjDd_GH_Ds      = g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
+  auto Gn_adjDd_GH_Ds   = GammaB_nucl * adjDd_GH_Ds;
+  auto Du_Gs            = Du_spec * GammaB_sigma;
+  auto Dq_GH            = Dq_loop * Gamma_H;
+  auto Tr_Dq_GH         = trace(Dq_GH)()()();
+
+  Real ee;
 
   for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int beta_n=0; beta_n<Ns; beta_n++){
-        auto GDsGDd_ab_bb = GDsGDd()(alpha_s,beta_n)(b_s,b_n);
-        for (int tau2=0; tau2<Ns; tau2++){
-        for (int j=0; j<Nc; j++){
-          auto DqG_tt_jj = DqG()(tau2,tau2)(j,j);
-          auto ee_GDGDDG = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd_ab_bb * DqG_tt_jj;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-            result()(gamma_s,gamma_n)() += ee_GDGDDG * DuG()(alpha_s, beta_n)(a_s,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-            result()(gamma_s,gamma_n)() -= ee_GDGDDG * DuG()(gamma_s, beta_n)(c_s,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-          }}
-	}}
+    int a_n    = (ie_n < 3 ? ie_n       : (6-ie_n)%3 ); //epsilon[ie_n][0]; //a
+    int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
+    int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_n = (ie_n < 3 ? 1 : -1);
+  for (int ie_s=0; ie_s < 6 ; ie_s++){
+    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_s = (ie_s < 3 ? 1 : -1);
+
+    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+
+    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+    for (int beta_s=0;  beta_s<Ns;  beta_s++){
+
+      auto Gn_adjDd_GH_Ds_ab_bb = Gn_adjDd_GH_Ds ()(alpha_n, beta_s)(b_n,b_s);
+
+      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+        result()(gamma_n,gamma_s)() += ee   * Gn_adjDd_GH_Ds_ab_bb
+                                            * Du_spec         ()(gamma_n,gamma_s)(c_n,c_s)
+                                            * Du_Gs           ()(alpha_n, beta_s)(a_n,a_s) 
+                                            * Tr_Dq_GH;
+
+        result()(gamma_n,gamma_s)() -= ee   * Gn_adjDd_GH_Ds_ab_bb
+                                            * Du_spec         ()(alpha_n,gamma_s)(a_n,c_s)
+                                            * Du_Gs           ()(gamma_n, beta_s)(c_n,a_s) 
+                                            * Tr_Dq_GH;
       }}
-    }
-  }
+    }}
+  }}
 }
 
 /* Du_ti is a quark line from t_i to t_H
@@ -1048,57 +1101,68 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
-						 const mobj &Du_tf,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
+             const mobj &Du_tf,
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result)
 {
 
   Gamma g5(Gamma::Algebra::Gamma5); 
 
-  auto DuG = Du_spec * GammaB_nucl;
-  auto adjDu = g5 * adj(Du_tf) * g5;
-  auto adjDuG = adjDu * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
-  // Dq_loop * \gamma_\mu^L
-  auto DuGH = Du_ti * Gamma_H;
+  auto Du_Gs          = Du_spec * GammaB_sigma;
+  //auto Gn_adjDd_GH_Ds = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
+  auto adjDd_GH_Ds = g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
+  auto Gn_adjDd_GH_Ds = GammaB_nucl * adjDd_GH_Ds;
+  auto adjDu_GH_Du    = g5 * adj(Du_tf) * g5 * Gamma_H * Du_ti;
+  auto adjDu_GH_Du_Gs = adjDu_GH_Du * GammaB_sigma;
+
+  Real ee;
 
   for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int beta_n=0; beta_n<Ns; beta_n++){
-        auto GDsGDd_ab_bb = GDsGDd()(alpha_s,beta_n)(b_s,b_n);
-        for (int tau2=0; tau2<Ns; tau2++){
-        for (int j=0; j<Nc; j++){
-          auto DuGH_at_aj = DuGH()(alpha_s,tau2)(a_s,j);
-          auto ee_GDGDDG_a = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd_ab_bb * DuGH_at_aj;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-            auto DuGH_gt_cj = DuGH()(gamma_s,tau2)(c_s,j);
-            auto ee_GDGDDG_c = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsGDd_ab_bb * DuGH_gt_cj;
-            for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-              result()(gamma_s,gamma_n)() += ee_GDGDDG_a * DuG()(gamma_s, beta_n)(c_s,a_n) * adjDu()(tau2,gamma_n)(j,c_n);
-              result()(gamma_s,gamma_n)() += ee_GDGDDG_c * adjDuG()(tau2, beta_n)(j,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-              result()(gamma_s,gamma_n)() -= ee_GDGDDG_a * adjDuG()(tau2, beta_n)(j,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-              result()(gamma_s,gamma_n)() -= ee_GDGDDG_c * DuG()(alpha_s, beta_n)(a_s,a_n) * adjDu()(tau2,gamma_n)(j,c_n);
-            }
-	  }
-	}}
+    int a_n    = (ie_n < 3 ? ie_n       : (6-ie_n)%3 ); //epsilon[ie_n][0]; //a
+    int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
+    int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_n = (ie_n < 3 ? 1 : -1);
+  for (int ie_s=0; ie_s < 6 ; ie_s++){
+    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_s = (ie_s < 3 ? 1 : -1);
+
+    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+
+    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+    for (int beta_s=0;  beta_s<Ns;  beta_s++){
+
+      auto Gn_adjDd_GH_Ds_ab_bb = Gn_adjDd_GH_Ds  ()(alpha_n, beta_s)(b_n,b_s);
+
+      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+
+        result()(gamma_n,gamma_s)() += ee * Gn_adjDd_GH_Ds_ab_bb
+                                          * adjDu_GH_Du     ()(alpha_n,gamma_s)(a_n,c_s)
+                                          * Du_Gs           ()(gamma_n, beta_s)(c_n,a_s);
+
+        result()(gamma_n,gamma_s)() += ee * Gn_adjDd_GH_Ds_ab_bb
+                                          * adjDu_GH_Du_Gs  ()(gamma_n, beta_s)(c_n,a_s)
+                                          * Du_spec         ()(alpha_n,gamma_s)(a_n,c_s);
+
+        result()(gamma_n,gamma_s)() -= ee * Gn_adjDd_GH_Ds_ab_bb
+                                          * adjDu_GH_Du_Gs  ()(alpha_n, beta_s)(a_n,a_s)
+                                          * Du_spec         ()(gamma_n,gamma_s)(c_n,c_s);
+
+        result()(gamma_n,gamma_s)() -= ee * Gn_adjDd_GH_Ds_ab_bb
+                                          * adjDu_GH_Du     ()(gamma_n,gamma_s)(c_n,c_s)
+                                          * Du_Gs           ()(alpha_n, beta_s)(a_n,a_s);
       }}
-    }
-  }
+    }}
+  }}
 }
 
 //Equivalent to "One-trace"
@@ -1107,49 +1171,57 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result)
 {
 
   Gamma g5(Gamma::Algebra::Gamma5); 
 
-  auto DuG = Du_spec * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L
-  auto GDsG = GammaB_sigma * Ds_ti * Gamma_H;
-  // Dq_loop * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto DqGDd = Dq_loop * Gamma_H * g5 * adj(Dd_tf) * g5;
+  //auto Gn_adjDd_GH_Duloop_GH_Ds = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Dq_loop * Gamma_H * Ds_ti;
+  auto adjDd_GH_Duloop_GH_Ds = g5 * adj(Dd_tf) * g5 * Gamma_H * Dq_loop * Gamma_H * Ds_ti;
+  auto Gn_adjDd_GH_Duloop_GH_Ds = GammaB_nucl * adjDd_GH_Duloop_GH_Ds;
+  auto Du_Gs = Du_spec * GammaB_sigma;
+
+  Real ee;
 
   for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int tau=0; tau<Ns; tau++){
-      for (int i=0; i<Nc; i++){
-	auto GDsG_at_bi = GDsG()(alpha_s,tau)(b_s,i);
-        for (int beta_n=0; beta_n<Ns; beta_n++){
-          auto DqGDd_tb_ib = DqGDd()(tau,beta_n)(i,b_n);
-	  auto ee_GDGDGD = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsG_at_bi * DqGDd_tb_ib;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-            result()(gamma_s,gamma_n)() -= ee_GDGDGD * DuG()(alpha_s, beta_n)(a_s,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-            result()(gamma_s,gamma_n)() += ee_GDGDGD * DuG()(gamma_s, beta_n)(c_s,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-          }}
-	}
-      }}}
-    }
-  }
+    int a_n    = (ie_n < 3 ? ie_n       : (6-ie_n)%3 ); //epsilon[ie_n][0]; //a
+    int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
+    int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_n = (ie_n < 3 ? 1 : -1);
+  for (int ie_s=0; ie_s < 6 ; ie_s++){
+    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_s = (ie_s < 3 ? 1 : -1);
+
+    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+
+    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+    for (int beta_s=0; beta_s<Ns; beta_s++){
+
+      auto Gn_adjDd_GH_Duloop_GH_Ds_ab_bb = Gn_adjDd_GH_Duloop_GH_Ds ()(alpha_n,beta_s)(b_n,b_s);
+
+      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+
+        result()(gamma_n,gamma_s)() -= ee   * Du_spec         ()(gamma_n,gamma_s)(c_n,c_s)
+                                            * Du_Gs           ()(alpha_n,beta_s)(a_n,a_s) 
+                                            * Gn_adjDd_GH_Duloop_GH_Ds_ab_bb;
+
+        result()(gamma_n,gamma_s)() += ee   * Du_Gs         ()(alpha_n,gamma_s)(a_n,c_s)
+                                            * Du_spec       ()(gamma_n,beta_s)(c_n,a_s) 
+                                            * Gn_adjDd_GH_Duloop_GH_Ds_ab_bb;
+      }}
+    }}
+  }}
 }
 
 /* Du_ti is a quark line from t_i to t_H
@@ -1158,70 +1230,85 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
-						 const mobj &Du_tf,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result)
+             const mobj &Du_tf,
+             const mobj2 &Du_spec,
+             const mobj &Dd_tf,
+             const mobj &Ds_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             robj &result)
 {
 
   Gamma g5(Gamma::Algebra::Gamma5); 
 
-  auto DuG = Du_spec * GammaB_nucl;
-  auto adjDu = g5 * adj(Du_tf) * g5;
-  auto adjDuG = adjDu * GammaB_nucl;
-  // Gamma^B * Ds * \gamma_\mu^L
-  auto GDsG = GammaB_sigma * Ds_ti * Gamma_H;
-  // Du * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5)
-  auto DuGDd = Du_ti * Gamma_H * g5 * adj(Dd_tf) * g5;
+  auto Du_Gs              = Du_spec * GammaB_sigma;
+  auto adjDu_GH_Ds        = g5 * adj(Du_tf) * g5 * Gamma_H * Ds_ti;
+  //auto Gn_adjDd_GH_Du     = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Du_ti;
+  auto adjDd_GH_Du        = g5 * adj(Dd_tf) * g5 * Gamma_H * Du_ti;
+  auto Gn_adjDd_GH_Du     = GammaB_nucl * adjDd_GH_Du; // for some reason I needed to split this into two lines to avoid the compilation error 'error: identifier "Grid::Gamma::mul" is undefined in device code'
+
+  auto Gn_adjDd_GH_Du_Gs  = Gn_adjDd_GH_Du * GammaB_sigma;
+
+  Real ee;
 
   for (int ie_n=0; ie_n < 6 ; ie_n++){
-    int a_n = epsilon[ie_n][0]; //a
-    int b_n = epsilon[ie_n][1]; //b
-    int c_n = epsilon[ie_n][2]; //c
-    for (int ie_s=0; ie_s < 6 ; ie_s++){
-      int a_s = epsilon[ie_s][0]; //a'
-      int b_s = epsilon[ie_s][1]; //b'
-      int c_s = epsilon[ie_s][2]; //c'
-      for (int alpha_s=0; alpha_s<Ns; alpha_s++){
-      for (int tau=0; tau<Ns; tau++){
-      for (int i=0; i<Nc; i++){
-	auto GDsG_at_bi = GDsG()(alpha_s,tau)(b_s,i);
-        for (int beta_n=0; beta_n<Ns; beta_n++){
-          auto DuGDd_ab_ab = DuGDd()(alpha_s,beta_n)(a_s,b_n);
-	  auto ee_GDGDGD_a = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsG_at_bi * DuGDd_ab_ab;
-          for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-            auto DuGDd_gb_cb = DuGDd()(gamma_s,beta_n)(c_s,b_n);
-	    auto ee_GDGDGD_c = epsilon_sgn[ie_n] * epsilon_sgn[ie_s] * GDsG_at_bi * DuGDd_gb_cb;
-            for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-              result()(gamma_s,gamma_n)() -= ee_GDGDGD_a * DuG()(gamma_s, beta_n)(c_s,a_n) * adjDu()(tau,gamma_n)(i,c_n);
-              result()(gamma_s,gamma_n)() -= ee_GDGDGD_c * adjDuG()(tau, beta_n)(i,a_n) * Du_spec()(alpha_s,gamma_n)(a_s,c_n);
-              result()(gamma_s,gamma_n)() += ee_GDGDGD_a * adjDuG()(tau, beta_n)(i,a_n) * Du_spec()(gamma_s,gamma_n)(c_s,c_n);
-              result()(gamma_s,gamma_n)() += ee_GDGDGD_c * DuG()(alpha_s, beta_n)(a_s,a_n) * adjDu()(tau,gamma_n)(i,c_n);
-            }
-	  }
-	}
-      }}}
-    }
-  }
+    int a_n    = (ie_n < 3 ? ie_n       : (6-ie_n)%3 ); //epsilon[ie_n][0]; //a
+    int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
+    int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
+    int eSgn_n = (ie_n < 3 ? 1 : -1);
+  for (int ie_s=0; ie_s < 6 ; ie_s++){
+    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+    int eSgn_s = (ie_s < 3 ? 1 : -1);
+
+    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+
+    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+    for (int beta_s=0;  beta_s<Ns;   beta_s++){
+
+      auto adjDu_GH_Ds_ab_ab = adjDu_GH_Ds()(alpha_n, beta_s)(a_n,b_s);
+      auto Gn_adjDd_GH_Du_Gs_ab_ba = Gn_adjDd_GH_Du_Gs()(alpha_n, beta_s)(b_n,a_s);
+
+      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+        auto Gn_adjDd_GH_Du_ag_bc = Gn_adjDd_GH_Du()(alpha_n,gamma_s)(b_n,c_s);
+      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+        auto adjDu_GH_Ds_gb_cb = adjDu_GH_Ds()(gamma_n, beta_s)(c_n,b_s);
+
+        result()(gamma_n,gamma_s)() += ee * adjDu_GH_Ds_ab_ab
+                                          * Gn_adjDd_GH_Du_Gs_ab_ba
+                                          * Du_spec()(gamma_n,gamma_s)(c_n,c_s);
+
+        result()(gamma_n,gamma_s)() -= ee * adjDu_GH_Ds_gb_cb
+                                          * Gn_adjDd_GH_Du_Gs_ab_ba
+                                          * Du_spec()(alpha_n,gamma_s)(a_n,c_s);
+
+        result()(gamma_n,gamma_s)() += ee * adjDu_GH_Ds_gb_cb
+                                          * Gn_adjDd_GH_Du_ag_bc
+                                          * Du_Gs()(alpha_n, beta_s)(a_n,a_s);
+
+        result()(gamma_n,gamma_s)() -= ee * adjDu_GH_Ds_ab_ab
+                                          * Gn_adjDd_GH_Du_ag_bc
+                                          * Du_Gs()(gamma_n, beta_s)(c_n,a_s);
+      }}
+    }}
+  }}
 }
 
 template<class FImpl>
 template <class mobj>
 void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
-						 const mobj &Du_spec,
-						 const PropagatorField &qd_tf,
-						 const PropagatorField &qs_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 const std::string op,
-						 SpinMatrixField &stn_corr)
+             const mobj &Du_spec,
+             const PropagatorField &qd_tf,
+             const PropagatorField &qs_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             const std::string op,
+             SpinMatrixField &stn_corr)
 {
 
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
@@ -1229,39 +1316,43 @@ void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
 
   GridBase *grid = qs_ti.Grid();
 
-  autoView( vcorr, stn_corr, CpuWrite);
-  autoView( vq_loop , qq_loop, CpuRead);
-  autoView( vd_tf , qd_tf, CpuRead);
-  autoView( vs_ti , qs_ti, CpuRead);
+  autoView( vcorr   , stn_corr , AcceleratorWrite);
+  autoView( vq_loop , qq_loop  , AcceleratorRead);
+  autoView( vd_tf   , qd_tf    , AcceleratorRead);
+  autoView( vs_ti   , qs_ti    , AcceleratorRead);
+
+  bool doQ1 = (op == "Q1");
+  bool doQ2 = (op == "Q2");
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto Dq_loop = vq_loop[ss];
-    auto Dd_tf = vd_tf[ss];
-    auto Ds_ti = vs_ti[ss];
-    sobj result=Zero();
-    if(op == "Q1"){
+    auto Dq_loop = vq_loop(ss);
+    auto Dd_tf = vd_tf(ss);
+    auto Ds_ti = vs_ti(ss);
+    typedef decltype(coalescedRead(vcorr[0])) spinor;
+    spinor result=Zero();
+    if(doQ1){
       SigmaToNucleonQ1EyeSite(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
-    } else if(op == "Q2"){
+    } else if(doQ2){
       SigmaToNucleonQ2EyeSite(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else {
       assert(0 && "Weak Operator not correctly specified");
     }
-      vcorr[ss] = result; 
-  }  );//end loop over lattice sites
+    coalescedWrite(vcorr[ss],result);
+  });//end loop over lattice sites
 }
 
 template<class FImpl>
 template <class mobj>
 void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
-						 const PropagatorField &qq_tf,
-						 const mobj &Du_spec,
-						 const PropagatorField &qd_tf,
-						 const PropagatorField &qs_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 const std::string op,
-						 SpinMatrixField &stn_corr)
+             const PropagatorField &qq_tf,
+             const mobj &Du_spec,
+             const PropagatorField &qd_tf,
+             const PropagatorField &qs_ti,
+                         const Gamma Gamma_H,
+                         const Gamma GammaB_sigma,
+                         const Gamma GammaB_nucl,
+             const std::string op,
+             SpinMatrixField &stn_corr)
 {
 
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
@@ -1269,27 +1360,31 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
 
   GridBase *grid = qs_ti.Grid();
 
-  autoView( vcorr , stn_corr, CpuWrite);
-  autoView( vq_ti , qq_ti, CpuRead);
-  autoView( vq_tf , qq_tf, CpuRead);
-  autoView( vd_tf , qd_tf, CpuRead);
-  autoView( vs_ti , qs_ti, CpuRead);
- // accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-  thread_for(ss,grid->oSites(),{
-    auto Dq_ti = vq_ti[ss];
-    auto Dq_tf = vq_tf[ss];
-    auto Dd_tf = vd_tf[ss];
-    auto Ds_ti = vs_ti[ss];
-    sobj result=Zero();
-    if(op == "Q1"){
+  autoView( vcorr , stn_corr , AcceleratorWrite );
+  autoView( vq_ti , qq_ti    , AcceleratorRead  );
+  autoView( vq_tf , qq_tf    , AcceleratorRead  );
+  autoView( vd_tf , qd_tf    , AcceleratorRead  );
+  autoView( vs_ti , qs_ti    , AcceleratorRead  );
+
+  bool doQ1 = (op == "Q1");
+  bool doQ2 = (op == "Q2");
+
+  accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+    auto Dq_ti = vq_ti(ss);
+    auto Dq_tf = vq_tf(ss);
+    auto Dd_tf = vd_tf(ss);
+    auto Ds_ti = vs_ti(ss);
+    typedef decltype(coalescedRead(vcorr[0])) spinor;
+    spinor result=Zero();
+    if(doQ1){
       SigmaToNucleonQ1NonEyeSite(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
-    } else if(op == "Q2"){
+    } else if(doQ2){
       SigmaToNucleonQ2NonEyeSite(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else {
       assert(0 && "Weak Operator not correctly specified");
     }
-      vcorr[ss] = result; 
-  }  );//end loop over lattice sites
+    coalescedWrite(vcorr[ss],result);
+  });//end loop over lattice sites
 }
 
 NAMESPACE_END(Grid);

From e759367d42c14c2de4eac0349b9800b780b63988 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Fri, 8 Jan 2021 18:04:50 +0000
Subject: [PATCH 10/82] tested and working

---
 Grid/qcd/utils/BaryonUtils.h | 906 +++++++++++++++++------------------
 1 file changed, 427 insertions(+), 479 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 25c71e3a..35358d05 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -40,14 +40,7 @@ public:
   typedef typename FImpl::FermionField FermionField;
   typedef typename FImpl::PropagatorField PropagatorField;
 
-  typedef typename FImpl::SitePropagator pobj;
-  typedef typename ComplexField::vector_object vobj;
-
   typedef Lattice<iSpinMatrix<typename FImpl::Simd>> SpinMatrixField;
-  //typedef typename SpinMatrixField::vector_object sobj;
-
-  //static const int epsilon[6][3] ;
-  //static const Real epsilon_sgn[6];
 
   private: 
   template <class mobj, class robj> accelerator_inline
@@ -122,7 +115,7 @@ public:
   static void BaryonGamma3ptGroup1Site(
            const mobj &Dq1_ti,
            const mobj2 &Dq2_spec,
-          // const mobj2 &Dq3_spec,
+           const mobj2 &Dq3_spec,
            const mobj &Dq4_tf,
                    const Gamma GammaJ,
                    const Gamma GammaBi,
@@ -134,7 +127,7 @@ public:
   static void BaryonGamma3ptGroup2Site(
            const mobj2 &Dq1_spec,
            const mobj &Dq2_ti,
-           //const mobj2 &Dq3_spec,
+           const mobj2 &Dq3_spec,
            const mobj &Dq4_tf,
                    const Gamma GammaJ,
                    const Gamma GammaBi,
@@ -145,7 +138,7 @@ public:
   template <class mobj, class mobj2, class robj> accelerator_inline
   static void BaryonGamma3ptGroup3Site(
            const mobj2 &Dq1_spec,
-           //const mobj2 &Dq2_spec,
+           const mobj2 &Dq2_spec,
            const mobj &Dq3_ti,
            const mobj &Dq4_tf,
                    const Gamma GammaJ,
@@ -230,13 +223,7 @@ public:
                      const std::string op,
          SpinMatrixField &stn_corr);
 };
-/*
-template <class FImpl> 
-const int BaryonUtils<FImpl>::epsilon[6][3] = {{0,1,2},{1,2,0},{2,0,1},{0,2,1},{2,1,0},{1,0,2}};
-template <class FImpl> 
-const Real BaryonUtils<FImpl>::epsilon_sgn[6] = {1.,1.,1.,-1.,-1.,-1.};
-*/
-//This is the old version
+//This computes a baryon contraction on a lattice site, including the spin-trace of the correlation matrix
 template <class FImpl>
 template <class mobj, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
@@ -251,20 +238,20 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
                 robj &result)
 {
 
-    Gamma g4(Gamma::Algebra::GammaT); //needed for parity P_\pm = 0.5*(1 \pm \gamma_4)
+  Gamma g4(Gamma::Algebra::GammaT); //needed for parity P_\pm = 0.5*(1 \pm \gamma_4)
     
-    auto D1_GAi =  D1 * GammaA_i;
-    auto D1_GAi_g4 = D1_GAi * g4;
-    auto D1_GAi_P = 0.5*(D1_GAi + (Real)parity * D1_GAi_g4);
-    auto GAf_D1_GAi_P = GammaA_f * D1_GAi_P;
-    auto GBf_D1_GAi_P = GammaB_f * D1_GAi_P;
+  auto D1_GAi =  D1 * GammaA_i;
+  auto D1_GAi_g4 = D1_GAi * g4;
+  auto D1_GAi_P = 0.5*(D1_GAi + (Real)parity * D1_GAi_g4);
+  auto GAf_D1_GAi_P = GammaA_f * D1_GAi_P;
+  auto GBf_D1_GAi_P = GammaB_f * D1_GAi_P;
 
-    auto D2_GBi = D2 * GammaB_i;
-    auto GBf_D2_GBi = GammaB_f * D2_GBi;
-    auto GAf_D2_GBi = GammaA_f * D2_GBi;
+  auto D2_GBi = D2 * GammaB_i;
+  auto GBf_D2_GBi = GammaB_f * D2_GBi;
+  auto GAf_D2_GBi = GammaA_f * D2_GBi;
 
-    auto GBf_D3 = GammaB_f * D3;
-    auto GAf_D3 = GammaA_f * D3;
+  auto GBf_D3 = GammaB_f * D3;
+  auto GAf_D3 = GammaA_f * D3;
 
   Real ee;
 
@@ -273,86 +260,87 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
     int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
     int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_f = (ie_f < 3 ? 1 : -1);
-  for (int ie_i=0; ie_i < 6 ; ie_i++){
-    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
-    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
-    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_i = (ie_i < 3 ? 1 : -1);
+    for (int ie_i=0; ie_i < 6 ; ie_i++){
+      int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+      int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+      int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_i = (ie_i < 3 ? 1 : -1);
 
-    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
-        //This is the \delta_{456}^{123} part
-        if (wick_contraction[0]){
-            for (int rho=0; rho<Ns; rho++){
-                auto GAf_D1_GAi_P_rr_cc = GAf_D1_GAi_P()(rho,rho)(c_f,c_i);
-                for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()()() += ee  * GAf_D1_GAi_P_rr_cc
+      ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      //This is the \delta_{456}^{123} part
+      if (wick_contraction[0]){
+        for (int rho=0; rho<Ns; rho++){
+          auto GAf_D1_GAi_P_rr_cc = GAf_D1_GAi_P()(rho,rho)(c_f,c_i);
+          for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()()() += ee  * GAf_D1_GAi_P_rr_cc
                                         * D2_GBi    ()(alpha_f,beta_i)(a_f,a_i)
                                         * GBf_D3    ()(alpha_f,beta_i)(b_f,b_i);
-                }}
-            }
-        }   
-        //This is the \delta_{456}^{231} part
-        if (wick_contraction[1]){
-            for (int rho=0; rho<Ns; rho++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto D1_GAi_P_ar_ac = D1_GAi_P()(alpha_f,rho)(a_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()()() += ee  * D1_GAi_P_ar_ac
+          }}
+        }
+      }   
+      //This is the \delta_{456}^{231} part
+      if (wick_contraction[1]){
+        for (int rho=0; rho<Ns; rho++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto D1_GAi_P_ar_ac = D1_GAi_P()(alpha_f,rho)(a_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()()() += ee  * D1_GAi_P_ar_ac
                                         * GBf_D2_GBi    ()(alpha_f,beta_i)(b_f,a_i)
                                         * GAf_D3        ()(rho,beta_i)(c_f,b_i);
-                }
-            }}
-        }   
-        //This is the \delta_{456}^{312} part
-        if (wick_contraction[2]){
-            for (int rho=0; rho<Ns; rho++){
-                for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto GBf_D1_GAi_P_ar_bc = GBf_D1_GAi_P()(alpha_f,rho)(b_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()()() += ee  * GBf_D1_GAi_P_ar_bc
+          }
+        }}
+      }   
+      //This is the \delta_{456}^{312} part
+      if (wick_contraction[2]){
+        for (int rho=0; rho<Ns; rho++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto GBf_D1_GAi_P_ar_bc = GBf_D1_GAi_P()(alpha_f,rho)(b_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()()() += ee  * GBf_D1_GAi_P_ar_bc
                                         * GAf_D2_GBi    ()(rho,beta_i)(c_f,a_i)
                                         * D3            ()(alpha_f,beta_i)(a_f,b_i);
-                }
-            }}
-        }   
-        //This is the \delta_{456}^{132} part
-        if (wick_contraction[3]){
-            for (int rho=0; rho<Ns; rho++){
-                auto GAf_D1_GAi_P_rr_cc = GAf_D1_GAi_P()(rho,rho)(c_f,c_i);
-                for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()()() -= ee  * GAf_D1_GAi_P_rr_cc
+          }
+        }}
+      }   
+      //This is the \delta_{456}^{132} part
+      if (wick_contraction[3]){
+        for (int rho=0; rho<Ns; rho++){
+          auto GAf_D1_GAi_P_rr_cc = GAf_D1_GAi_P()(rho,rho)(c_f,c_i);
+          for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()()() -= ee  * GAf_D1_GAi_P_rr_cc
                                         * GBf_D2_GBi    ()(alpha_f,beta_i)(b_f,a_i)
                                         * D3            ()(alpha_f,beta_i)(a_f,b_i);
-              }
-        }}
-        }   
-        //This is the \delta_{456}^{321} part
-        if (wick_contraction[4]){
-            for (int rho=0; rho<Ns; rho++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto GBf_D1_GAi_P_ar_bc = GBf_D1_GAi_P()(alpha_f,rho)(b_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()()() -= ee  * GBf_D1_GAi_P_ar_bc
+          }}
+        }
+      }   
+      //This is the \delta_{456}^{321} part
+      if (wick_contraction[4]){
+        for (int rho=0; rho<Ns; rho++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto GBf_D1_GAi_P_ar_bc = GBf_D1_GAi_P()(alpha_f,rho)(b_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()()() -= ee  * GBf_D1_GAi_P_ar_bc
                                         * D2_GBi    ()(alpha_f,beta_i)(a_f,a_i)
                                         * GAf_D3    ()(rho,beta_i)(c_f,b_i);
-                }
-            }}
-        }   
-        //This is the \delta_{456}^{213} part
-        if (wick_contraction[5]){
-            for (int rho=0; rho<Ns; rho++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto D1_GAi_P_ar_ac = D1_GAi_P()(alpha_f,rho)(a_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()()() -= ee  * D1_GAi_P_ar_ac
+          }
+        }}
+      }   
+      //This is the \delta_{456}^{213} part
+      if (wick_contraction[5]){
+        for (int rho=0; rho<Ns; rho++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto D1_GAi_P_ar_ac = D1_GAi_P()(alpha_f,rho)(a_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()()() -= ee  * D1_GAi_P_ar_ac
                                         * GAf_D2_GBi    ()(rho,beta_i)(c_f,a_i)
                                         * GBf_D3        ()(alpha_f,beta_i)(b_f,b_i);
-                }
-            }}
-        }
-    }}
+          }
+        }}
+      }
+    }
+  }
 }
 
 //New version without parity projection or trace
@@ -369,17 +357,16 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
                 robj &result)
 {
 
-    auto D1_GAi =  D1 * GammaA_i;
-    auto GAf_D1_GAi = GammaA_f * D1_GAi;
-    auto GBf_D1_GAi = GammaB_f * D1_GAi;
+  auto D1_GAi =  D1 * GammaA_i;
+  auto GAf_D1_GAi = GammaA_f * D1_GAi;
+  auto GBf_D1_GAi = GammaB_f * D1_GAi;
 
-    auto D2_GBi = D2 * GammaB_i;
-    auto GBf_D2_GBi = GammaB_f * D2_GBi;
-    auto GAf_D2_GBi = GammaA_f * D2_GBi;
-
-    auto GBf_D3 = GammaB_f * D3;
-    auto GAf_D3 = GammaA_f * D3;
+  auto D2_GBi = D2 * GammaB_i;
+  auto GBf_D2_GBi = GammaB_f * D2_GBi;
+  auto GAf_D2_GBi = GammaA_f * D2_GBi;
 
+  auto GBf_D3 = GammaB_f * D3;
+  auto GAf_D3 = GammaA_f * D3;
 
   Real ee;
 
@@ -388,96 +375,101 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
     int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
     int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_f = (ie_f < 3 ? 1 : -1);
-  for (int ie_i=0; ie_i < 6 ; ie_i++){
-    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
-    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
-    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_i = (ie_i < 3 ? 1 : -1);
+    for (int ie_i=0; ie_i < 6 ; ie_i++){
+      int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+      int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+      int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_i = (ie_i < 3 ? 1 : -1);
 
-    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
-        //This is the \delta_{456}^{123} part
-        if (wick_contraction[0]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int rho_f=0; rho_f<Ns; rho_f++){
-                auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
-                for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()(rho_f,rho_i)() += ee  * GAf_D1_GAi_rr_cc
+      ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      //This is the \delta_{456}^{123} part
+      if (wick_contraction[0]){
+        for (int rho_i=0; rho_i<Ns; rho_i++){
+        for (int rho_f=0; rho_f<Ns; rho_f++){
+          auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
+          for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()(rho_f,rho_i)() += ee  * GAf_D1_GAi_rr_cc
                                         * D2_GBi    ()(alpha_f,beta_i)(a_f,a_i)
                                         * GBf_D3    ()(alpha_f,beta_i)(b_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{231} part
-        if (wick_contraction[1]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto GBf_D2_GBi_ab_ba = GBf_D2_GBi ()(alpha_f,beta_i)(b_f,a_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() += ee  * D1_GAi_ar_ac
+          }}
+        }}
+      }   
+      //This is the \delta_{456}^{231} part
+      if (wick_contraction[1]){
+        for (int rho_i=0; rho_i<Ns; rho_i++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            auto GBf_D2_GBi_ab_ba = GBf_D2_GBi ()(alpha_f,beta_i)(b_f,a_i);
+            for (int rho_f=0; rho_f<Ns; rho_f++){
+              result()(rho_f,rho_i)() += ee  * D1_GAi_ar_ac
                                         * GBf_D2_GBi_ab_ba
                                         * GAf_D3        ()(rho_f,beta_i)(c_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{312} part
-        if (wick_contraction[2]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto D3_ab_ab = D3 ()(alpha_f,beta_i)(a_f,b_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() += ee  * GBf_D1_GAi_ar_bc
+            }
+          }
+        }}
+      }   
+      //This is the \delta_{456}^{312} part
+      if (wick_contraction[2]){
+        for (int rho_i=0; rho_i<Ns; rho_i++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            auto D3_ab_ab = D3 ()(alpha_f,beta_i)(a_f,b_i);
+            for (int rho_f=0; rho_f<Ns; rho_f++){
+              result()(rho_f,rho_i)() += ee  * GBf_D1_GAi_ar_bc
                                         * GAf_D2_GBi    ()(rho_f,beta_i)(c_f,a_i)
                                         * D3_ab_ab;
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{132} part
-        if (wick_contraction[3]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int rho_f=0; rho_f<Ns; rho_f++){
-                auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
-                for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                    result()(rho_f,rho_i)() -= ee  * GAf_D1_GAi_rr_cc
+            }
+          }
+        }}
+      }   
+      //This is the \delta_{456}^{132} part
+      if (wick_contraction[3]){
+        for (int rho_i=0; rho_i<Ns; rho_i++){
+        for (int rho_f=0; rho_f<Ns; rho_f++){
+          auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
+          for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            result()(rho_f,rho_i)() -= ee  * GAf_D1_GAi_rr_cc
                                         * GBf_D2_GBi    ()(alpha_f,beta_i)(b_f,a_i)
                                         * D3            ()(alpha_f,beta_i)(a_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{321} part
-        if (wick_contraction[4]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto D2_GBi_ab_aa = D2_GBi()(alpha_f,beta_i)(a_f,a_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() -= ee  * GBf_D1_GAi_ar_bc
+          }}
+        }}
+      }   
+      //This is the \delta_{456}^{321} part
+      if (wick_contraction[4]){
+        for (int rho_i=0; rho_i<Ns; rho_i++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            auto D2_GBi_ab_aa = D2_GBi()(alpha_f,beta_i)(a_f,a_i);
+            for (int rho_f=0; rho_f<Ns; rho_f++){
+              result()(rho_f,rho_i)() -= ee  * GBf_D1_GAi_ar_bc
                                         * D2_GBi_ab_aa
                                         * GAf_D3    ()(rho_f,beta_i)(c_f,b_i);
-                }}
-            }}
-        }   
-        //This is the \delta_{456}^{213} part
-        if (wick_contraction[5]){
-            for (int rho_i=0; rho_i<Ns; rho_i++){
-            for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-                auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
-                for (int beta_i=0; beta_i<Ns; beta_i++){
-                  auto GBf_D3_ab_bb = GBf_D3()(alpha_f,beta_i)(b_f,b_i);
-                for (int rho_f=0; rho_f<Ns; rho_f++){
-                    result()(rho_f,rho_i)() -= ee  * D1_GAi_ar_ac
+            }
+          }
+        }}
+      }   
+      //This is the \delta_{456}^{213} part
+      if (wick_contraction[5]){
+        for (int rho_i=0; rho_i<Ns; rho_i++){
+        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+          auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
+          for (int beta_i=0; beta_i<Ns; beta_i++){
+            auto GBf_D3_ab_bb = GBf_D3()(alpha_f,beta_i)(b_f,b_i);
+            for (int rho_f=0; rho_f<Ns; rho_f++){
+              result()(rho_f,rho_i)() -= ee  * D1_GAi_ar_ac
                                         * GAf_D2_GBi    ()(rho_f,beta_i)(c_f,a_i)
                                         * GBf_D3_ab_bb;
-                }}
-            }}
-        }
-    }}
+            }
+          }
+        }}
+      }
+    }
+  }
 }
 
 /* Computes which wick contractions should be performed for a    *
@@ -530,8 +522,7 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
   for (int ie=0; ie < 6 ; ie++){
     if(ie==0 or ie==3){
        bytes += grid->oSites() * (4.*sizeof(int) + 4752.*sizeof(vComplex)) * wick_contractions[ie];
-    }
-    else{
+    } else{
        bytes += grid->oSites() * (64.*sizeof(int) + 5184.*sizeof(vComplex)) * wick_contractions[ie];
     }
   }
@@ -653,7 +644,7 @@ template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
                         const mobj &Dq1_ti,
                         const mobj2 &Dq2_spec,
-           //             const mobj2 &Dq3_spec,
+                        const mobj2 &Dq3_spec,
                         const mobj &Dq4_tf,
                                 const Gamma GammaJ,
                                 const Gamma GammaBi,
@@ -661,18 +652,14 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
                         int wick_contraction,
                         robj &result)
 {
-    Gamma g5(Gamma::Algebra::Gamma5); 
-
-//    auto adjD4_g_D1     = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq1_ti;
-    auto adjD4          = g5 * adj(Dq4_tf) * g5 ;
-    auto adjD4_g_D1     = adjD4 * GammaJ * Dq1_ti;
-    auto Gf_adjD4_g_D1  = GammaBf * adjD4_g_D1;
-    auto D2_Gi          = Dq2_spec * GammaBi;
-    auto Gf_D2_Gi       = GammaBf * D2_Gi;
-
-//    auto Gf_D3          = GammaBf * Dq3_spec;  // including a second mobj2 parameter leads to compilation error
-    auto Gf_D3          = GammaBf * Dq2_spec;  //WRONG!!!!!
+  Gamma g5(Gamma::Algebra::Gamma5); 
 
+  auto adjD4          = g5 * adj(Dq4_tf) * g5 ;
+  auto adjD4_g_D1     = adjD4 * GammaJ * Dq1_ti;
+  auto Gf_adjD4_g_D1  = GammaBf * adjD4_g_D1;
+  auto D2_Gi          = Dq2_spec * GammaBi;
+  auto Gf_D2_Gi       = GammaBf * D2_Gi;
+  auto Gf_D3          = GammaBf * Dq3_spec;  
 
   Real ee;
 
@@ -681,65 +668,65 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
     int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
     int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_f = (ie_f < 3 ? 1 : -1);
-  for (int ie_i=0; ie_i < 6 ; ie_i++){
-    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
-    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
-    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_i = (ie_i < 3 ? 1 : -1);
+    for (int ie_i=0; ie_i < 6 ; ie_i++){
+      int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+      int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+      int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_i = (ie_i < 3 ? 1 : -1);
 
-    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      ee = Real(eSgn_f * eSgn_i);
 
-        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-        for (int beta_i=0; beta_i<Ns; beta_i++){
-            auto D2_Gi_ab_aa        = D2_Gi     ()(alpha_f,beta_i)(a_f,a_i);
-            auto Gf_D3_ab_bb        = Gf_D3     ()(alpha_f,beta_i)(b_f,b_i);
-            auto Gf_D2_Gi_ab_ba     = Gf_D2_Gi  ()(alpha_f,beta_i)(b_f,a_i);
-            //auto Dq3_spec_ab_ab     = Dq3_spec  ()(alpha_f,beta_i)(a_f,b_i);
-            auto Dq3_spec_ab_ab     = Dq2_spec  ()(alpha_f,beta_i)(a_f,b_i); // WRONG!!!!
+      for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+      for (int beta_i=0; beta_i<Ns; beta_i++){
+        auto D2_Gi_ab_aa        = D2_Gi     ()(alpha_f,beta_i)(a_f,a_i);
+        auto Gf_D3_ab_bb        = Gf_D3     ()(alpha_f,beta_i)(b_f,b_i);
+        auto Gf_D2_Gi_ab_ba     = Gf_D2_Gi  ()(alpha_f,beta_i)(b_f,a_i);
+        auto Dq3_spec_ab_ab     = Dq3_spec  ()(alpha_f,beta_i)(a_f,b_i);
 
-            for (int gamma_i=0; gamma_i<Ns; gamma_i++){
-                auto ee_adjD4_g_D1_ag_ac        = ee * adjD4_g_D1   ()(alpha_f,gamma_i)(a_f,c_i);
-                auto ee_Gf_adjD4_g_D1_ag_bc     = ee * Gf_adjD4_g_D1()(alpha_f,gamma_i)(b_f,c_i);
-            for (int gamma_f=0; gamma_f<Ns; gamma_f++){
-                auto ee_adjD4_g_D1_gg_cc        = ee * adjD4_g_D1   ()(gamma_f,gamma_i)(c_f,c_i);
-                //auto Dq3_spec_gb_cb             = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
-                auto Dq3_spec_gb_cb             = Dq2_spec          ()(gamma_f,beta_i)(c_f,b_i); //WRONG!!!!
-                auto D2_Gi_gb_ca                = D2_Gi             ()(gamma_f,beta_i)(c_f,a_i);
+        for (int gamma_i=0; gamma_i<Ns; gamma_i++){
+          auto ee_adjD4_g_D1_ag_ac        = ee * adjD4_g_D1   ()(alpha_f,gamma_i)(a_f,c_i);
+          auto ee_Gf_adjD4_g_D1_ag_bc     = ee * Gf_adjD4_g_D1()(alpha_f,gamma_i)(b_f,c_i);
+          for (int gamma_f=0; gamma_f<Ns; gamma_f++){
+            auto ee_adjD4_g_D1_gg_cc        = ee * adjD4_g_D1   ()(gamma_f,gamma_i)(c_f,c_i);
+            auto Dq3_spec_gb_cb             = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
+            auto D2_Gi_gb_ca                = D2_Gi             ()(gamma_f,beta_i)(c_f,a_i);
 
 
-                if(wick_contraction == 1) { // Do contraction I1
-                    result()(gamma_f,gamma_i)() -= ee_adjD4_g_D1_gg_cc
+            if(wick_contraction == 1) { // Do contraction I1
+              result()(gamma_f,gamma_i)() -= ee_adjD4_g_D1_gg_cc
                                                         * D2_Gi_ab_aa
                                                         * Gf_D3_ab_bb;
-                }
-                if(wick_contraction == 2) { // Do contraction I2
-                    result()(gamma_f,gamma_i)() -= ee_adjD4_g_D1_ag_ac
+            }
+            if(wick_contraction == 2) { // Do contraction I2
+              result()(gamma_f,gamma_i)() -= ee_adjD4_g_D1_ag_ac
                                                         * Gf_D2_Gi_ab_ba
                                                         * Dq3_spec_gb_cb;
-                }
-                if(wick_contraction == 3) { // Do contraction I3
-                    result()(gamma_f,gamma_i)() -= ee_Gf_adjD4_g_D1_ag_bc
+            }
+            if(wick_contraction == 3) { // Do contraction I3
+              result()(gamma_f,gamma_i)() -= ee_Gf_adjD4_g_D1_ag_bc
                                                         * D2_Gi_gb_ca
                                                         * Dq3_spec_ab_ab;
-                }
-                if(wick_contraction == 4) { // Do contraction I4
-                    result()(gamma_f,gamma_i)() += ee_adjD4_g_D1_gg_cc
+            }
+            if(wick_contraction == 4) { // Do contraction I4
+              result()(gamma_f,gamma_i)() += ee_adjD4_g_D1_gg_cc
                                                         * Gf_D2_Gi_ab_ba
                                                         * Dq3_spec_ab_ab;
-                }
-                if(wick_contraction == 5) { // Do contraction I5
-                    result()(gamma_f,gamma_i)() += ee_Gf_adjD4_g_D1_ag_bc
+            }
+            if(wick_contraction == 5) { // Do contraction I5
+              result()(gamma_f,gamma_i)() += ee_Gf_adjD4_g_D1_ag_bc
                                                         * D2_Gi_ab_aa
                                                         * Dq3_spec_gb_cb;
-                }
-                if(wick_contraction == 6) { // Do contraction I6
-                    result()(gamma_f,gamma_i)() += ee_adjD4_g_D1_ag_ac
+            }
+            if(wick_contraction == 6) { // Do contraction I6
+              result()(gamma_f,gamma_i)() += ee_adjD4_g_D1_ag_ac
                                                         * D2_Gi_gb_ca
                                                         * Gf_D3_ab_bb;
-                }
-            }}
-        }}
-    }}
+            }
+          }
+        }
+      }}
+    }
+  }
 }
 
 /* Dq1_spec is a quark line from t_i to t_f
@@ -751,7 +738,7 @@ template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
                         const mobj2 &Dq1_spec,
                         const mobj &Dq2_ti,
-                    //    const mobj2 &Dq3_spec,
+                        const mobj2 &Dq3_spec,
                         const mobj &Dq4_tf,
                                 const Gamma GammaJ,
                                 const Gamma GammaBi,
@@ -759,14 +746,12 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
                         int wick_contraction,
                         robj &result)
 {
-    Gamma g5(Gamma::Algebra::Gamma5); 
-
-    auto adjD4_g_D2_Gi      = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq2_ti * GammaBi;
-    auto Gf_adjD4_g_D2_Gi   = GammaBf * adjD4_g_D2_Gi;
-    auto Gf_D1              = GammaBf * Dq1_spec;
-    //auto Gf_D3              = GammaBf * Dq3_spec;
-    auto Gf_D3              = GammaBf * Dq1_spec; // WRONG!!!!!
+  Gamma g5(Gamma::Algebra::Gamma5); 
 
+  auto adjD4_g_D2_Gi      = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq2_ti * GammaBi;
+  auto Gf_adjD4_g_D2_Gi   = GammaBf * adjD4_g_D2_Gi;
+  auto Gf_D1              = GammaBf * Dq1_spec;
+  auto Gf_D3              = GammaBf * Dq3_spec;
 
   Real ee;
 
@@ -775,64 +760,64 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
     int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
     int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_f = (ie_f < 3 ? 1 : -1);
-  for (int ie_i=0; ie_i < 6 ; ie_i++){
-    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
-    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
-    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_i = (ie_i < 3 ? 1 : -1);
+    for (int ie_i=0; ie_i < 6 ; ie_i++){
+      int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+      int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+      int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_i = (ie_i < 3 ? 1 : -1);
 
-    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
-        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-        for (int beta_i=0; beta_i<Ns; beta_i++){
-            auto adjD4_g_D2_Gi_ab_aa        = adjD4_g_D2_Gi     ()(alpha_f,beta_i)(a_f,a_i);
-            auto Gf_D3_ab_bb                = Gf_D3             ()(alpha_f,beta_i)(b_f,b_i);
-            auto Gf_adjD4_g_D2_Gi_ab_ba     = Gf_adjD4_g_D2_Gi  ()(alpha_f,beta_i)(b_f,a_i);
-            //auto Dq3_spec_ab_ab             = Dq3_spec          ()(alpha_f,beta_i)(a_f,b_i);
-            auto Dq3_spec_ab_ab             = Dq1_spec          ()(alpha_f,beta_i)(a_f,b_i); //WRONG!!!
+      for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+      for (int beta_i=0; beta_i<Ns; beta_i++){
+        auto adjD4_g_D2_Gi_ab_aa        = adjD4_g_D2_Gi     ()(alpha_f,beta_i)(a_f,a_i);
+        auto Gf_D3_ab_bb                = Gf_D3             ()(alpha_f,beta_i)(b_f,b_i);
+        auto Gf_adjD4_g_D2_Gi_ab_ba     = Gf_adjD4_g_D2_Gi  ()(alpha_f,beta_i)(b_f,a_i);
+        auto Dq3_spec_ab_ab             = Dq3_spec          ()(alpha_f,beta_i)(a_f,b_i);
 
-            for (int gamma_i=0; gamma_i<Ns; gamma_i++){ 
-                auto ee_Dq1_spec_ag_ac      = ee * Dq1_spec     ()(alpha_f,gamma_i)(a_f,c_i);
-                auto ee_Gf_D1_ag_bc         = ee * Gf_D1        ()(alpha_f,gamma_i)(b_f,c_i);
-            for (int gamma_f=0; gamma_f<Ns; gamma_f++){
-                auto ee_Dq1_spec_gg_cc      = ee * Dq1_spec     ()(gamma_f,gamma_i)(c_f,c_i);
-                //auto Dq3_spec_gb_cb         = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
-                auto Dq3_spec_gb_cb         = Dq1_spec          ()(gamma_f,beta_i)(c_f,b_i); //WRONG!!!
-                auto adjD4_g_D2_Gi_gb_ca    = adjD4_g_D2_Gi     ()(gamma_f,beta_i)(c_f,a_i);
+        for (int gamma_i=0; gamma_i<Ns; gamma_i++){ 
+          auto ee_Dq1_spec_ag_ac      = ee * Dq1_spec     ()(alpha_f,gamma_i)(a_f,c_i);
+          auto ee_Gf_D1_ag_bc         = ee * Gf_D1        ()(alpha_f,gamma_i)(b_f,c_i);
+          for (int gamma_f=0; gamma_f<Ns; gamma_f++){
+            auto ee_Dq1_spec_gg_cc      = ee * Dq1_spec     ()(gamma_f,gamma_i)(c_f,c_i);
+            auto Dq3_spec_gb_cb         = Dq3_spec          ()(gamma_f,beta_i)(c_f,b_i);
+            auto adjD4_g_D2_Gi_gb_ca    = adjD4_g_D2_Gi     ()(gamma_f,beta_i)(c_f,a_i);
 
-                if(wick_contraction == 1) { // Do contraction II1
-                    result()(gamma_f,gamma_i)() -= ee_Dq1_spec_gg_cc
+            if(wick_contraction == 1) { // Do contraction II1
+              result()(gamma_f,gamma_i)() -= ee_Dq1_spec_gg_cc
                                                         * adjD4_g_D2_Gi_ab_aa
                                                         * Gf_D3_ab_bb;
-                }
-                if(wick_contraction == 2) { // Do contraction II2
-                    result()(gamma_f,gamma_i)() -= ee_Dq1_spec_ag_ac
+            }
+            if(wick_contraction == 2) { // Do contraction II2
+              result()(gamma_f,gamma_i)() -= ee_Dq1_spec_ag_ac
                                                         * Gf_adjD4_g_D2_Gi_ab_ba
                                                         * Dq3_spec_gb_cb;
-                }
-                if(wick_contraction == 3) { // Do contraction II3
-                    result()(gamma_f,gamma_i)() -= ee_Gf_D1_ag_bc
+            }
+            if(wick_contraction == 3) { // Do contraction II3
+              result()(gamma_f,gamma_i)() -= ee_Gf_D1_ag_bc
                                                         * adjD4_g_D2_Gi_gb_ca
                                                         * Dq3_spec_ab_ab;
-                }
-                if(wick_contraction == 4) { // Do contraction II4
-                    result()(gamma_f,gamma_i)() += ee_Dq1_spec_gg_cc
+            }
+            if(wick_contraction == 4) { // Do contraction II4
+              result()(gamma_f,gamma_i)() += ee_Dq1_spec_gg_cc
                                                         * Gf_adjD4_g_D2_Gi_ab_ba
                                                         * Dq3_spec_ab_ab;
-                }
-                if(wick_contraction == 5) { // Do contraction II5
-                    result()(gamma_f,gamma_i)() += ee_Gf_D1_ag_bc
+            }
+            if(wick_contraction == 5) { // Do contraction II5
+              result()(gamma_f,gamma_i)() += ee_Gf_D1_ag_bc
                                                         * adjD4_g_D2_Gi_ab_aa
                                                         * Dq3_spec_gb_cb;
-                }
-                if(wick_contraction == 6) { // Do contraction II6
-                    result()(gamma_f,gamma_i)() += ee_Dq1_spec_ag_ac
+            }
+            if(wick_contraction == 6) { // Do contraction II6
+              result()(gamma_f,gamma_i)() += ee_Dq1_spec_ag_ac
                                                         * adjD4_g_D2_Gi_gb_ca
                                                         * Gf_D3_ab_bb;
-                }
-            }}
-        }}
-    }}
+            }
+          }
+        }
+      }}
+    }
+  }
 }
 
 /* Dq1_spec is a quark line from t_i to t_f
@@ -843,7 +828,7 @@ template<class FImpl>
 template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
                         const mobj2 &Dq1_spec,
-                       // const mobj2 &Dq2_spec,
+                        const mobj2 &Dq2_spec,
                         const mobj &Dq3_ti,
                         const mobj &Dq4_tf,
                                 const Gamma GammaJ,
@@ -852,15 +837,13 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
                         int wick_contraction,
                         robj &result)
 {
-    Gamma g5(Gamma::Algebra::Gamma5);
-
-    auto adjD4_g_D3     = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq3_ti;
-    auto Gf_adjD4_g_D3  = GammaBf * adjD4_g_D3;
-    auto Gf_D1          = GammaBf * Dq1_spec;
-    //auto D2_Gi          = Dq2_spec * GammaBi;
-    auto D2_Gi          = Dq1_spec * GammaBi; //WRONG!!!!!!!!!!!!!!!!!
-    auto Gf_D2_Gi       = GammaBf * D2_Gi;
+  Gamma g5(Gamma::Algebra::Gamma5);
 
+  auto adjD4_g_D3     = g5 * adj(Dq4_tf) * g5 * GammaJ * Dq3_ti;
+  auto Gf_adjD4_g_D3  = GammaBf * adjD4_g_D3;
+  auto Gf_D1          = GammaBf * Dq1_spec;
+  auto D2_Gi          = Dq2_spec * GammaBi;
+  auto Gf_D2_Gi       = GammaBf * D2_Gi;
 
   Real ee;
 
@@ -869,62 +852,64 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
     int b_f    = (ie_f < 3 ? (ie_f+1)%3 : (8-ie_f)%3 ); //epsilon[ie_n][1]; //b
     int c_f    = (ie_f < 3 ? (ie_f+2)%3 : (7-ie_f)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_f = (ie_f < 3 ? 1 : -1);
-  for (int ie_i=0; ie_i < 6 ; ie_i++){
-    int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
-    int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
-    int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_i = (ie_i < 3 ? 1 : -1);
+    for (int ie_i=0; ie_i < 6 ; ie_i++){
+      int a_i = (ie_i < 3 ? ie_i       : (6-ie_i)%3 ); //epsilon[ie_s][0]; //a'
+      int b_i = (ie_i < 3 ? (ie_i+1)%3 : (8-ie_i)%3 ); //epsilon[ie_s][1]; //b'
+      int c_i = (ie_i < 3 ? (ie_i+2)%3 : (7-ie_i)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_i = (ie_i < 3 ? 1 : -1);
 
-    ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
-        for (int alpha_f=0; alpha_f<Ns; alpha_f++){
-        for (int beta_i=0; beta_i<Ns; beta_i++){
-            auto D2_Gi_ab_aa            = D2_Gi         ()(alpha_f,beta_i)(a_f,a_i);
-            auto Gf_adjD4_g_D3_ab_bb    = Gf_adjD4_g_D3 ()(alpha_f,beta_i)(b_f,b_i);
-            auto Gf_D2_Gi_ab_ba         = Gf_D2_Gi      ()(alpha_f,beta_i)(b_f,a_i);
-            auto adjD4_g_D3_ab_ab       = adjD4_g_D3    ()(alpha_f,beta_i)(a_f,b_i);
+      for (int alpha_f=0; alpha_f<Ns; alpha_f++){
+      for (int beta_i=0; beta_i<Ns; beta_i++){
+        auto D2_Gi_ab_aa            = D2_Gi         ()(alpha_f,beta_i)(a_f,a_i);
+        auto Gf_adjD4_g_D3_ab_bb    = Gf_adjD4_g_D3 ()(alpha_f,beta_i)(b_f,b_i);
+        auto Gf_D2_Gi_ab_ba         = Gf_D2_Gi      ()(alpha_f,beta_i)(b_f,a_i);
+        auto adjD4_g_D3_ab_ab       = adjD4_g_D3    ()(alpha_f,beta_i)(a_f,b_i);
 
-            for (int gamma_i=0; gamma_i<Ns; gamma_i++) {
-                auto ee_Dq1_spec_ag_ac  = ee * Dq1_spec ()(alpha_f,gamma_i)(a_f,c_i);
-                auto ee_Gf_D1_ag_bc     = ee * Gf_D1    ()(alpha_f,gamma_i)(b_f,c_i);
-            for (int gamma_f=0; gamma_f<Ns; gamma_f++) {
-                auto ee_Dq1_spec_gg_cc  = ee * Dq1_spec ()(gamma_f,gamma_i)(c_f,c_i);
-                auto adjD4_g_D3_gb_cb   = adjD4_g_D3    ()(gamma_f,beta_i)(c_f,b_i);
-                auto D2_Gi_gb_ca        = D2_Gi         ()(gamma_f,beta_i)(c_f,a_i);
+        for (int gamma_i=0; gamma_i<Ns; gamma_i++) {
+          auto ee_Dq1_spec_ag_ac  = ee * Dq1_spec ()(alpha_f,gamma_i)(a_f,c_i);
+          auto ee_Gf_D1_ag_bc     = ee * Gf_D1    ()(alpha_f,gamma_i)(b_f,c_i);
+          for (int gamma_f=0; gamma_f<Ns; gamma_f++) {
+            auto ee_Dq1_spec_gg_cc  = ee * Dq1_spec ()(gamma_f,gamma_i)(c_f,c_i);
+            auto adjD4_g_D3_gb_cb   = adjD4_g_D3    ()(gamma_f,beta_i)(c_f,b_i);
+            auto D2_Gi_gb_ca        = D2_Gi         ()(gamma_f,beta_i)(c_f,a_i);
 
-                if(wick_contraction == 1) { // Do contraction III1
-                    result()(gamma_f,gamma_i)() -= ee_Dq1_spec_gg_cc
+            if(wick_contraction == 1) { // Do contraction III1
+              result()(gamma_f,gamma_i)() -= ee_Dq1_spec_gg_cc
                                                         * D2_Gi_ab_aa
                                                         * Gf_adjD4_g_D3_ab_bb;
-                }
-                if(wick_contraction == 2) { // Do contraction III2
-                    result()(gamma_f,gamma_i)() -= ee_Dq1_spec_ag_ac
+            }
+            if(wick_contraction == 2) { // Do contraction III2
+              result()(gamma_f,gamma_i)() -= ee_Dq1_spec_ag_ac
                                                         * Gf_D2_Gi_ab_ba
                                                         * adjD4_g_D3_gb_cb;
-                }
-                if(wick_contraction == 3) { // Do contraction III3
-                    result()(gamma_f,gamma_i)() -= ee_Gf_D1_ag_bc
+            }
+            if(wick_contraction == 3) { // Do contraction III3
+              result()(gamma_f,gamma_i)() -= ee_Gf_D1_ag_bc
                                                         * D2_Gi_gb_ca
                                                         * adjD4_g_D3_ab_ab;
-                }
-                if(wick_contraction == 4) { // Do contraction III4
-                    result()(gamma_f,gamma_i)() += ee_Dq1_spec_gg_cc
+            }
+            if(wick_contraction == 4) { // Do contraction III4
+              result()(gamma_f,gamma_i)() += ee_Dq1_spec_gg_cc
                                                         * Gf_D2_Gi_ab_ba
                                                         * adjD4_g_D3_ab_ab;
-                }
-                if(wick_contraction == 5) { // Do contraction III5
-                    result()(gamma_f,gamma_i)() += ee_Gf_D1_ag_bc
+            }
+            if(wick_contraction == 5) { // Do contraction III5
+              result()(gamma_f,gamma_i)() += ee_Gf_D1_ag_bc
                                                         * D2_Gi_ab_aa
                                                         * adjD4_g_D3_gb_cb;
-                }
-                if(wick_contraction == 6) { // Do contraction III6
-                    result()(gamma_f,gamma_i)() += ee_Dq1_spec_ag_ac
+            }
+            if(wick_contraction == 6) { // Do contraction III6
+              result()(gamma_f,gamma_i)() += ee_Dq1_spec_ag_ac
                                                         * D2_Gi_gb_ca
                                                         * Gf_adjD4_g_D3_ab_bb;
-                }
-            }}
-        }}
-    }}
+            }
+          }
+        }
+      }}
+    }
+  }
 }
 
 /* The group indicates which inital state quarks the current is  * 
@@ -949,81 +934,44 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
   assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
 
-    GridBase *grid = q_tf.Grid();
+  GridBase *grid = q_tf.Grid();
 
-    // autoView( vcorr, stn_corr, CpuWrite);
-    // autoView( vq_ti , q_ti, CpuRead);
-    // autoView( vq_tf , q_tf, CpuRead);
+  autoView( vcorr , stn_corr , AcceleratorWrite);
+  autoView( vq_ti , q_ti     , AcceleratorRead);
+  autoView( vq_tf , q_tf     , AcceleratorRead);
 
-    // if (group == 1) {
-    //     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    //         auto Dq_ti = vq_ti[ss];
-    //         auto Dq_tf = vq_tf[ss];
-    //         sobj result=Zero();
-    //         BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-    //         vcorr[ss] += result; 
-    //     });//end loop over lattice sites
-    // } else if (group == 2) {
-    //     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    //         auto Dq_ti = vq_ti[ss];
-    //         auto Dq_tf = vq_tf[ss];
-    //         sobj result=Zero();
-    //         BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-    //         vcorr[ss] += result; 
-    //     });//end loop over lattice sites
-    // } else if (group == 3) {
-    //     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    //         auto Dq_ti = vq_ti[ss];
-    //         auto Dq_tf = vq_tf[ss];
-    //         sobj result=Zero();
-    //         BaryonGamma3ptGroup3Site(Dq_spec1,Dq_spec2,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+  Vector<mobj> my_Dq_spec{Dq_spec1,Dq_spec2};
+  mobj * Dq_spec_p = &my_Dq_spec[0];
 
-    //         vcorr[ss] += result; 
-    //     });//end loop over lattice sites
-    // }
+  if (group == 1) {
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_ti = vq_ti(ss);
+      auto Dq_tf = vq_tf(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
+      BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec_p[0],Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
+      coalescedWrite(vcorr[ss],result);
+    });//end loop over lattice sites
 
-    autoView( vcorr , stn_corr , AcceleratorWrite);
-    autoView( vq_ti , q_ti     , AcceleratorRead);
-    autoView( vq_tf , q_tf     , AcceleratorRead);
-
-    if (group == 1) {
-        accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-            auto Dq_ti = vq_ti(ss);
-            auto Dq_tf = vq_tf(ss);
-            //sobj result=Zero();
-            typedef decltype(coalescedRead(vcorr[0])) spinor;
-            spinor result=Zero();
-            //BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-            BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec1,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); //WRONG
-            // vcorr[ss] += result; 
-            coalescedWrite(vcorr[ss],result);
-        });//end loop over lattice sites
-
-    } else if (group == 2) {
-        accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-            auto Dq_ti = vq_ti(ss);
-            auto Dq_tf = vq_tf(ss);
-            //sobj result=Zero();
-            typedef decltype(coalescedRead(vcorr[0])) spinor;
-            spinor result=Zero();
-           // BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_spec2,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-            BaryonGamma3ptGroup2Site(Dq_spec1,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); //WRONG
-            // vcorr[ss] += result; 
-            coalescedWrite(vcorr[ss],result);
-        });//end loop over lattice sites
-    } else if (group == 3) {
-        accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-            auto Dq_ti = vq_ti(ss);
-            auto Dq_tf = vq_tf(ss);
-            //sobj result=Zero();
-            typedef decltype(coalescedRead(vcorr[0])) spinor;
-            spinor result=Zero();
-            //BaryonGamma3ptGroup3Site(Dq_spec1,Dq_spec2,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-            BaryonGamma3ptGroup3Site(Dq_spec1,Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); //WRONG
-            // vcorr[ss] += result; 
-            coalescedWrite(vcorr[ss],result);
-        });//end loop over lattice sites
-    }
+  } else if (group == 2) {
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_ti = vq_ti(ss);
+      auto Dq_tf = vq_tf(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
+      BaryonGamma3ptGroup2Site(Dq_spec_p[0],Dq_ti,Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
+      coalescedWrite(vcorr[ss],result);
+    });//end loop over lattice sites
+  } else if (group == 3) {
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_ti = vq_ti(ss);
+      auto Dq_tf = vq_tf(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
+      BaryonGamma3ptGroup3Site(Dq_spec_p[0],Dq_spec_p[1],Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
+      coalescedWrite(vcorr[ss],result);
+    });//end loop over lattice sites
+  }
 
 }
 
@@ -1052,7 +1000,6 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
 
   Gamma g5(Gamma::Algebra::Gamma5); 
 
-  //auto Gn_adjDd_GH_Ds   = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
   auto adjDd_GH_Ds      = g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
   auto Gn_adjDd_GH_Ds   = GammaB_nucl * adjDd_GH_Ds;
   auto Du_Gs            = Du_spec * GammaB_sigma;
@@ -1066,33 +1013,33 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
     int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
     int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_n = (ie_n < 3 ? 1 : -1);
-  for (int ie_s=0; ie_s < 6 ; ie_s++){
-    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
-    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
-    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_s = (ie_s < 3 ? 1 : -1);
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+      int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+      int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_s = (ie_s < 3 ? 1 : -1);
 
-    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      ee = Real(eSgn_n * eSgn_s); 
+      for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+      for (int beta_s=0;  beta_s<Ns;  beta_s++){
 
-    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
-    for (int beta_s=0;  beta_s<Ns;  beta_s++){
+        auto Gn_adjDd_GH_Ds_ab_bb = Gn_adjDd_GH_Ds ()(alpha_n, beta_s)(b_n,b_s);
 
-      auto Gn_adjDd_GH_Ds_ab_bb = Gn_adjDd_GH_Ds ()(alpha_n, beta_s)(b_n,b_s);
-
-      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-        result()(gamma_n,gamma_s)() += ee   * Gn_adjDd_GH_Ds_ab_bb
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+        for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+          result()(gamma_n,gamma_s)() += ee   * Gn_adjDd_GH_Ds_ab_bb
                                             * Du_spec         ()(gamma_n,gamma_s)(c_n,c_s)
                                             * Du_Gs           ()(alpha_n, beta_s)(a_n,a_s) 
                                             * Tr_Dq_GH;
 
-        result()(gamma_n,gamma_s)() -= ee   * Gn_adjDd_GH_Ds_ab_bb
+          result()(gamma_n,gamma_s)() -= ee   * Gn_adjDd_GH_Ds_ab_bb
                                             * Du_spec         ()(alpha_n,gamma_s)(a_n,c_s)
                                             * Du_Gs           ()(gamma_n, beta_s)(c_n,a_s) 
                                             * Tr_Dq_GH;
+        }}
       }}
-    }}
-  }}
+    }
+  }
 }
 
 /* Du_ti is a quark line from t_i to t_H
@@ -1116,7 +1063,6 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
   Gamma g5(Gamma::Algebra::Gamma5); 
 
   auto Du_Gs          = Du_spec * GammaB_sigma;
-  //auto Gn_adjDd_GH_Ds = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
   auto adjDd_GH_Ds = g5 * adj(Dd_tf) * g5 * Gamma_H * Ds_ti;
   auto Gn_adjDd_GH_Ds = GammaB_nucl * adjDd_GH_Ds;
   auto adjDu_GH_Du    = g5 * adj(Du_tf) * g5 * Gamma_H * Du_ti;
@@ -1129,40 +1075,41 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
     int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
     int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_n = (ie_n < 3 ? 1 : -1);
-  for (int ie_s=0; ie_s < 6 ; ie_s++){
-    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
-    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
-    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_s = (ie_s < 3 ? 1 : -1);
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+      int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+      int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_s = (ie_s < 3 ? 1 : -1);
 
-    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
-    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
-    for (int beta_s=0;  beta_s<Ns;  beta_s++){
+      for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+      for (int beta_s=0;  beta_s<Ns;  beta_s++){
 
-      auto Gn_adjDd_GH_Ds_ab_bb = Gn_adjDd_GH_Ds  ()(alpha_n, beta_s)(b_n,b_s);
+        auto Gn_adjDd_GH_Ds_ab_bb = Gn_adjDd_GH_Ds  ()(alpha_n, beta_s)(b_n,b_s);
 
-      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+        for (int gamma_n=0; gamma_n<Ns; gamma_n++){
 
-        result()(gamma_n,gamma_s)() += ee * Gn_adjDd_GH_Ds_ab_bb
+          result()(gamma_n,gamma_s)() += ee * Gn_adjDd_GH_Ds_ab_bb
                                           * adjDu_GH_Du     ()(alpha_n,gamma_s)(a_n,c_s)
                                           * Du_Gs           ()(gamma_n, beta_s)(c_n,a_s);
 
-        result()(gamma_n,gamma_s)() += ee * Gn_adjDd_GH_Ds_ab_bb
+          result()(gamma_n,gamma_s)() += ee * Gn_adjDd_GH_Ds_ab_bb
                                           * adjDu_GH_Du_Gs  ()(gamma_n, beta_s)(c_n,a_s)
                                           * Du_spec         ()(alpha_n,gamma_s)(a_n,c_s);
 
-        result()(gamma_n,gamma_s)() -= ee * Gn_adjDd_GH_Ds_ab_bb
+          result()(gamma_n,gamma_s)() -= ee * Gn_adjDd_GH_Ds_ab_bb
                                           * adjDu_GH_Du_Gs  ()(alpha_n, beta_s)(a_n,a_s)
                                           * Du_spec         ()(gamma_n,gamma_s)(c_n,c_s);
 
-        result()(gamma_n,gamma_s)() -= ee * Gn_adjDd_GH_Ds_ab_bb
+          result()(gamma_n,gamma_s)() -= ee * Gn_adjDd_GH_Ds_ab_bb
                                           * adjDu_GH_Du     ()(gamma_n,gamma_s)(c_n,c_s)
                                           * Du_Gs           ()(alpha_n, beta_s)(a_n,a_s);
+        }}
       }}
-    }}
-  }}
+    }
+  }
 }
 
 //Equivalent to "One-trace"
@@ -1184,7 +1131,6 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
 
   Gamma g5(Gamma::Algebra::Gamma5); 
 
-  //auto Gn_adjDd_GH_Duloop_GH_Ds = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Dq_loop * Gamma_H * Ds_ti;
   auto adjDd_GH_Duloop_GH_Ds = g5 * adj(Dd_tf) * g5 * Gamma_H * Dq_loop * Gamma_H * Ds_ti;
   auto Gn_adjDd_GH_Duloop_GH_Ds = GammaB_nucl * adjDd_GH_Duloop_GH_Ds;
   auto Du_Gs = Du_spec * GammaB_sigma;
@@ -1196,32 +1142,33 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
     int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
     int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_n = (ie_n < 3 ? 1 : -1);
-  for (int ie_s=0; ie_s < 6 ; ie_s++){
-    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
-    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
-    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_s = (ie_s < 3 ? 1 : -1);
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+      int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+      int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_s = (ie_s < 3 ? 1 : -1);
 
-    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
-    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
-    for (int beta_s=0; beta_s<Ns; beta_s++){
+      for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+      for (int beta_s=0; beta_s<Ns; beta_s++){
 
-      auto Gn_adjDd_GH_Duloop_GH_Ds_ab_bb = Gn_adjDd_GH_Duloop_GH_Ds ()(alpha_n,beta_s)(b_n,b_s);
+        auto Gn_adjDd_GH_Duloop_GH_Ds_ab_bb = Gn_adjDd_GH_Duloop_GH_Ds ()(alpha_n,beta_s)(b_n,b_s);
 
-      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+        for (int gamma_n=0; gamma_n<Ns; gamma_n++){
 
-        result()(gamma_n,gamma_s)() -= ee   * Du_spec         ()(gamma_n,gamma_s)(c_n,c_s)
+          result()(gamma_n,gamma_s)() -= ee   * Du_spec         ()(gamma_n,gamma_s)(c_n,c_s)
                                             * Du_Gs           ()(alpha_n,beta_s)(a_n,a_s) 
                                             * Gn_adjDd_GH_Duloop_GH_Ds_ab_bb;
 
-        result()(gamma_n,gamma_s)() += ee   * Du_Gs         ()(alpha_n,gamma_s)(a_n,c_s)
+          result()(gamma_n,gamma_s)() += ee   * Du_Gs         ()(alpha_n,gamma_s)(a_n,c_s)
                                             * Du_spec       ()(gamma_n,beta_s)(c_n,a_s) 
                                             * Gn_adjDd_GH_Duloop_GH_Ds_ab_bb;
+        }}
       }}
-    }}
-  }}
+    }
+  }
 }
 
 /* Du_ti is a quark line from t_i to t_H
@@ -1246,7 +1193,6 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
 
   auto Du_Gs              = Du_spec * GammaB_sigma;
   auto adjDu_GH_Ds        = g5 * adj(Du_tf) * g5 * Gamma_H * Ds_ti;
-  //auto Gn_adjDd_GH_Du     = GammaB_nucl * g5 * adj(Dd_tf) * g5 * Gamma_H * Du_ti;
   auto adjDd_GH_Du        = g5 * adj(Dd_tf) * g5 * Gamma_H * Du_ti;
   auto Gn_adjDd_GH_Du     = GammaB_nucl * adjDd_GH_Du; // for some reason I needed to split this into two lines to avoid the compilation error 'error: identifier "Grid::Gamma::mul" is undefined in device code'
 
@@ -1259,43 +1205,45 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
     int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
     int c_n    = (ie_n < 3 ? (ie_n+2)%3 : (7-ie_n)%3 ); //epsilon[ie_n][2]; //c
     int eSgn_n = (ie_n < 3 ? 1 : -1);
-  for (int ie_s=0; ie_s < 6 ; ie_s++){
-    int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
-    int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
-    int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
-    int eSgn_s = (ie_s < 3 ? 1 : -1);
+    for (int ie_s=0; ie_s < 6 ; ie_s++){
+      int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+      int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+      int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_s = (ie_s < 3 ? 1 : -1);
 
-    ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
+      ee = Real(eSgn_n * eSgn_s); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
 
-    for (int alpha_n=0; alpha_n<Ns; alpha_n++){
-    for (int beta_s=0;  beta_s<Ns;   beta_s++){
+      for (int alpha_n=0; alpha_n<Ns; alpha_n++){
+      for (int beta_s=0;  beta_s<Ns;   beta_s++){
 
-      auto adjDu_GH_Ds_ab_ab = adjDu_GH_Ds()(alpha_n, beta_s)(a_n,b_s);
-      auto Gn_adjDd_GH_Du_Gs_ab_ba = Gn_adjDd_GH_Du_Gs()(alpha_n, beta_s)(b_n,a_s);
+        auto adjDu_GH_Ds_ab_ab = adjDu_GH_Ds()(alpha_n, beta_s)(a_n,b_s);
+        auto Gn_adjDd_GH_Du_Gs_ab_ba = Gn_adjDd_GH_Du_Gs()(alpha_n, beta_s)(b_n,a_s);
 
-      for (int gamma_s=0; gamma_s<Ns; gamma_s++){
-        auto Gn_adjDd_GH_Du_ag_bc = Gn_adjDd_GH_Du()(alpha_n,gamma_s)(b_n,c_s);
-      for (int gamma_n=0; gamma_n<Ns; gamma_n++){
-        auto adjDu_GH_Ds_gb_cb = adjDu_GH_Ds()(gamma_n, beta_s)(c_n,b_s);
+        for (int gamma_s=0; gamma_s<Ns; gamma_s++){
+          auto Gn_adjDd_GH_Du_ag_bc = Gn_adjDd_GH_Du()(alpha_n,gamma_s)(b_n,c_s);
+          for (int gamma_n=0; gamma_n<Ns; gamma_n++){
+            auto adjDu_GH_Ds_gb_cb = adjDu_GH_Ds()(gamma_n, beta_s)(c_n,b_s);
 
-        result()(gamma_n,gamma_s)() += ee * adjDu_GH_Ds_ab_ab
+            result()(gamma_n,gamma_s)() += ee * adjDu_GH_Ds_ab_ab
                                           * Gn_adjDd_GH_Du_Gs_ab_ba
                                           * Du_spec()(gamma_n,gamma_s)(c_n,c_s);
 
-        result()(gamma_n,gamma_s)() -= ee * adjDu_GH_Ds_gb_cb
+            result()(gamma_n,gamma_s)() -= ee * adjDu_GH_Ds_gb_cb
                                           * Gn_adjDd_GH_Du_Gs_ab_ba
                                           * Du_spec()(alpha_n,gamma_s)(a_n,c_s);
 
-        result()(gamma_n,gamma_s)() += ee * adjDu_GH_Ds_gb_cb
+            result()(gamma_n,gamma_s)() += ee * adjDu_GH_Ds_gb_cb
                                           * Gn_adjDd_GH_Du_ag_bc
                                           * Du_Gs()(alpha_n, beta_s)(a_n,a_s);
 
-        result()(gamma_n,gamma_s)() -= ee * adjDu_GH_Ds_ab_ab
+            result()(gamma_n,gamma_s)() -= ee * adjDu_GH_Ds_ab_ab
                                           * Gn_adjDd_GH_Du_ag_bc
                                           * Du_Gs()(gamma_n, beta_s)(c_n,a_s);
+          }
+        }
       }}
-    }}
-  }}
+    }
+  }
 }
 
 template<class FImpl>

From 74de2d9742de7a85850cf40cbbe0215edc3405da Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Fri, 8 Jan 2021 18:28:36 +0000
Subject: [PATCH 11/82] whitespace changes

---
 Grid/qcd/utils/BaryonUtils.h | 156 +++++++++++++++++------------------
 1 file changed, 78 insertions(+), 78 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 35358d05..80a80a76 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -53,7 +53,7 @@ public:
          const Gamma GammaB_right,
          const int parity,
          const bool * wick_contractions,
-           robj &result);
+         robj &result);
   template <class mobj, class robj> accelerator_inline
   static void BaryonSiteMatrix(const mobj &D1,
          const mobj &D2,
@@ -63,7 +63,7 @@ public:
          const Gamma GammaA_right,
          const Gamma GammaB_right,
          const bool * wick_contractions,
-           robj &result);
+         robj &result);
   public:
   static void WickContractions(std::string qi, 
                  std::string qf, 
@@ -117,9 +117,9 @@ public:
            const mobj2 &Dq2_spec,
            const mobj2 &Dq3_spec,
            const mobj &Dq4_tf,
-                   const Gamma GammaJ,
-                   const Gamma GammaBi,
-                   const Gamma GammaBf,
+           const Gamma GammaJ,
+           const Gamma GammaBi,
+           const Gamma GammaBf,
            int wick_contraction,
            robj &result);
 
@@ -129,9 +129,9 @@ public:
            const mobj &Dq2_ti,
            const mobj2 &Dq3_spec,
            const mobj &Dq4_tf,
-                   const Gamma GammaJ,
-                   const Gamma GammaBi,
-                   const Gamma GammaBf,
+           const Gamma GammaJ,
+           const Gamma GammaBi,
+           const Gamma GammaBf,
            int wick_contraction,
            robj &result);
 
@@ -141,9 +141,9 @@ public:
            const mobj2 &Dq2_spec,
            const mobj &Dq3_ti,
            const mobj &Dq4_tf,
-                   const Gamma GammaJ,
-                   const Gamma GammaBi,
-                   const Gamma GammaBf,
+           const Gamma GammaJ,
+           const Gamma GammaBi,
+           const Gamma GammaBf,
            int wick_contraction,
            robj &result);
   public:
@@ -155,9 +155,9 @@ public:
            const PropagatorField &q_tf,
            int group,
            int wick_contraction,
-                   const Gamma GammaJ,
-                   const Gamma GammaBi,
-                   const Gamma GammaBf,
+           const Gamma GammaJ,
+           const Gamma GammaBi,
+           const Gamma GammaBf,
            SpinMatrixField &stn_corr);
   private: 
   template <class mobj, class mobj2, class robj> accelerator_inline
@@ -165,9 +165,9 @@ public:
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result);
   template <class mobj, class mobj2, class robj> accelerator_inline
   static void SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
@@ -175,9 +175,9 @@ public:
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result);
 
 
@@ -186,9 +186,9 @@ public:
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result);
   template <class mobj, class mobj2, class robj> accelerator_inline
   static void SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
@@ -196,9 +196,9 @@ public:
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result);
   public:
   template <class mobj>
@@ -209,7 +209,7 @@ public:
          const Gamma Gamma_H,
          const Gamma GammaB_sigma,
          const Gamma GammaB_nucl,
-                     const std::string op,
+         const std::string op,
          SpinMatrixField &stn_corr);
   template <class mobj>
   static void SigmaToNucleonNonEye(const PropagatorField &qq_ti,
@@ -220,7 +220,7 @@ public:
          const Gamma Gamma_H,
          const Gamma GammaB_sigma,
          const Gamma GammaB_nucl,
-                     const std::string op,
+         const std::string op,
          SpinMatrixField &stn_corr);
 };
 //This computes a baryon contraction on a lattice site, including the spin-trace of the correlation matrix
@@ -229,10 +229,10 @@ template <class mobj, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
                 const mobj &D2,
                 const mobj &D3,
-                         const Gamma GammaA_i,
-                         const Gamma GammaB_i,
-                         const Gamma GammaA_f,
-                         const Gamma GammaB_f,
+                const Gamma GammaA_i,
+                const Gamma GammaB_i,
+                const Gamma GammaA_f,
+                const Gamma GammaB_f,
                 const int parity,
                 const bool * wick_contraction,
                 robj &result)
@@ -349,10 +349,10 @@ template <class mobj, class robj> accelerator_inline
 void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
                 const mobj &D2,
                 const mobj &D3,
-                         const Gamma GammaA_i,
-                         const Gamma GammaB_i,
-                         const Gamma GammaA_f,
-                         const Gamma GammaB_f,
+                const Gamma GammaA_i,
+                const Gamma GammaB_i,
+                const Gamma GammaA_f,
+                const Gamma GammaB_f,
                 const bool * wick_contraction,
                 robj &result)
 {
@@ -496,10 +496,10 @@ template<class FImpl>
 void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
              const PropagatorField &q2_left,
              const PropagatorField &q3_left,
-                         const Gamma GammaA_left,
-                         const Gamma GammaB_left,
-                         const Gamma GammaA_right,
-                         const Gamma GammaB_right,
+             const Gamma GammaA_left,
+             const Gamma GammaB_left,
+             const Gamma GammaA_right,
+             const Gamma GammaB_right,
              const bool* wick_contractions,
              const int parity,
              ComplexField &baryon_corr)
@@ -548,10 +548,10 @@ template<class FImpl>
 void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
              const PropagatorField &q2_left,
              const PropagatorField &q3_left,
-                         const Gamma GammaA_left,
-                         const Gamma GammaB_left,
-                         const Gamma GammaA_right,
-                         const Gamma GammaB_right,
+             const Gamma GammaA_left,
+             const Gamma GammaB_left,
+             const Gamma GammaA_right,
+             const Gamma GammaB_right,
              const bool* wick_contractions,
              SpinMatrixField &baryon_corr)
 {
@@ -612,10 +612,10 @@ template <class mobj, class robj>
 void BaryonUtils<FImpl>::ContractBaryonsSlicedMatrix(const mobj &D1,
              const mobj &D2,
              const mobj &D3,
-                         const Gamma GammaA_left,
-                         const Gamma GammaB_left,
-                         const Gamma GammaA_right,
-                         const Gamma GammaB_right,
+             const Gamma GammaA_left,
+             const Gamma GammaB_left,
+             const Gamma GammaA_right,
+             const Gamma GammaB_right,
              const bool* wick_contractions,
              const int nt,
              robj &result)
@@ -646,9 +646,9 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup1Site(
                         const mobj2 &Dq2_spec,
                         const mobj2 &Dq3_spec,
                         const mobj &Dq4_tf,
-                                const Gamma GammaJ,
-                                const Gamma GammaBi,
-                                const Gamma GammaBf,
+                        const Gamma GammaJ,
+                        const Gamma GammaBi,
+                        const Gamma GammaBf,
                         int wick_contraction,
                         robj &result)
 {
@@ -740,9 +740,9 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup2Site(
                         const mobj &Dq2_ti,
                         const mobj2 &Dq3_spec,
                         const mobj &Dq4_tf,
-                                const Gamma GammaJ,
-                                const Gamma GammaBi,
-                                const Gamma GammaBf,
+                        const Gamma GammaJ,
+                        const Gamma GammaBi,
+                        const Gamma GammaBf,
                         int wick_contraction,
                         robj &result)
 {
@@ -831,9 +831,9 @@ void BaryonUtils<FImpl>::BaryonGamma3ptGroup3Site(
                         const mobj2 &Dq2_spec,
                         const mobj &Dq3_ti,
                         const mobj &Dq4_tf,
-                                const Gamma GammaJ,
-                                const Gamma GammaBi,
-                                const Gamma GammaBf,
+                        const Gamma GammaJ,
+                        const Gamma GammaBi,
+                        const Gamma GammaBf,
                         int wick_contraction,
                         robj &result)
 {
@@ -926,9 +926,9 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
                         const PropagatorField &q_tf,
                         int group,
                         int wick_contraction,
-                                const Gamma GammaJ,
-                                const Gamma GammaBi,
-                                const Gamma GammaBf,
+                        const Gamma GammaJ,
+                        const Gamma GammaBi,
+                        const Gamma GammaBf,
                         SpinMatrixField &stn_corr)
 {
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
@@ -992,9 +992,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop,
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result)
 {
 
@@ -1054,9 +1054,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti,
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result)
 {
 
@@ -1123,9 +1123,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop,
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result)
 {
 
@@ -1183,9 +1183,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
              const mobj2 &Du_spec,
              const mobj &Dd_tf,
              const mobj &Ds_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              robj &result)
 {
 
@@ -1252,9 +1252,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
              const mobj &Du_spec,
              const PropagatorField &qd_tf,
              const PropagatorField &qs_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              const std::string op,
              SpinMatrixField &stn_corr)
 {
@@ -1296,9 +1296,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
              const mobj &Du_spec,
              const PropagatorField &qd_tf,
              const PropagatorField &qs_ti,
-                         const Gamma Gamma_H,
-                         const Gamma GammaB_sigma,
-                         const Gamma GammaB_nucl,
+             const Gamma Gamma_H,
+             const Gamma GammaB_sigma,
+             const Gamma GammaB_nucl,
              const std::string op,
              SpinMatrixField &stn_corr)
 {

From 45fc7ded3a18b8fc905127fb4d987a5bb16654e1 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 12 Jan 2021 09:10:37 +0000
Subject: [PATCH 12/82] test for sum

---
 Grid/qcd/utils/BaryonUtils.h | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 80a80a76..828b4085 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -950,7 +950,8 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
       typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec_p[0],Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-      coalescedWrite(vcorr[ss],result);
+      //coalescedWrite(vcorr[ss],vcorr[ss]+result); //diff by factor 10???
+      coalescedWrite(vcorr[ss],vcorr[ss]+result);
     });//end loop over lattice sites
 
   } else if (group == 2) {
@@ -1271,6 +1272,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
 
   bool doQ1 = (op == "Q1");
   bool doQ2 = (op == "Q2");
+  
+  Vector<mobj> my_Dq_spec{Du_spec};
+  mobj * Dq_spec_p = &my_Dq_spec[0];
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
     auto Dq_loop = vq_loop(ss);
@@ -1279,9 +1283,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
     typedef decltype(coalescedRead(vcorr[0])) spinor;
     spinor result=Zero();
     if(doQ1){
-      SigmaToNucleonQ1EyeSite(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      SigmaToNucleonQ1EyeSite(Dq_loop,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else if(doQ2){
-      SigmaToNucleonQ2EyeSite(Dq_loop,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      SigmaToNucleonQ2EyeSite(Dq_loop,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else {
       assert(0 && "Weak Operator not correctly specified");
     }
@@ -1316,6 +1320,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
 
   bool doQ1 = (op == "Q1");
   bool doQ2 = (op == "Q2");
+  
+  Vector<mobj> my_Dq_spec{Du_spec};
+  mobj * Dq_spec_p = &my_Dq_spec[0];
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
     auto Dq_ti = vq_ti(ss);
@@ -1325,9 +1332,9 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
     typedef decltype(coalescedRead(vcorr[0])) spinor;
     spinor result=Zero();
     if(doQ1){
-      SigmaToNucleonQ1NonEyeSite(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      SigmaToNucleonQ1NonEyeSite(Dq_ti,Dq_tf,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else if(doQ2){
-      SigmaToNucleonQ2NonEyeSite(Dq_ti,Dq_tf,Du_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
+      SigmaToNucleonQ2NonEyeSite(Dq_ti,Dq_tf,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
     } else {
       assert(0 && "Weak Operator not correctly specified");
     }

From fa12b9a3295f1fdf392b0367de4136e576b5401e Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Wed, 13 Jan 2021 10:04:17 +0000
Subject: [PATCH 13/82] bugfix

---
 Grid/qcd/utils/BaryonUtils.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 828b4085..69bf8959 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -961,7 +961,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
       typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup2Site(Dq_spec_p[0],Dq_ti,Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],result);
+      coalescedWrite(vcorr[ss],vcorr[ss]+result);
     });//end loop over lattice sites
   } else if (group == 3) {
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
@@ -970,7 +970,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
       typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup3Site(Dq_spec_p[0],Dq_spec_p[1],Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],result);
+      coalescedWrite(vcorr[ss],vcorr[ss]+result);
     });//end loop over lattice sites
   }
 

From 9b73a937e7dbfca0ff0c49b5003c0416e572bb82 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Mon, 18 Jan 2021 18:57:05 +0000
Subject: [PATCH 14/82] bugfix

---
 Grid/qcd/utils/BaryonUtils.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 69bf8959..edc5c8d5 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -951,7 +951,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
       spinor result=Zero();
       BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec_p[0],Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
       //coalescedWrite(vcorr[ss],vcorr[ss]+result); //diff by factor 10???
-      coalescedWrite(vcorr[ss],vcorr[ss]+result);
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result);
     });//end loop over lattice sites
 
   } else if (group == 2) {
@@ -961,7 +961,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
       typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup2Site(Dq_spec_p[0],Dq_ti,Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],vcorr[ss]+result);
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result);
     });//end loop over lattice sites
   } else if (group == 3) {
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
@@ -970,7 +970,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
       typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup3Site(Dq_spec_p[0],Dq_spec_p[1],Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],vcorr[ss]+result);
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result);
     });//end loop over lattice sites
   }
 

From 8bfa0e74f837c914efaf95929149f9ce5b0a5487 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 19 Jan 2021 12:27:57 +0000
Subject: [PATCH 15/82] final version, tested on CPU and GPU

---
 Grid/qcd/utils/BaryonUtils.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index edc5c8d5..ca8b66ef 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -950,7 +950,6 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
       typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec_p[0],Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-      //coalescedWrite(vcorr[ss],vcorr[ss]+result); //diff by factor 10???
       coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result);
     });//end loop over lattice sites
 

From fc6d07897fe4a9f7f1a80bd9a7849fe373648b42 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 19 Jan 2021 12:32:48 +0000
Subject: [PATCH 16/82] revert changes

---
 tests/solver/Test_zMADWF_prec.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/solver/Test_zMADWF_prec.cc b/tests/solver/Test_zMADWF_prec.cc
index f18e1d86..d1168764 100644
--- a/tests/solver/Test_zMADWF_prec.cc
+++ b/tests/solver/Test_zMADWF_prec.cc
@@ -52,7 +52,7 @@ struct TestParams{
   bool zmobius_inner;
   double lambda_max; //upper bound of H_T eigenvalue range required to generate zMobius approximation
   
-  TestParams(): load_config(false), config_file("ckpoint_lat.1000"), mass(0.01),
+  TestParams(): load_config(true), config_file("ckpoint_lat.1000"), mass(0.01),
 		Ls_outer(24), b_plus_c_outer(2.0), resid_outer(1e-8),
 		Ls_inner(12), b_plus_c_inner(1.0), resid_inner(1e-8), zmobius_inner(true), lambda_max(1.42), outer_precon("Standard"), inner_precon("Standard")
   {}
@@ -246,7 +246,7 @@ void run(const TestParams &params){
   typename RunParamsInner::SchurSolverType SchurSolver_inner(CG_inner);
 
   ZeroGuesser<LatticeFermion> Guess;
-  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 10000, &update);
+  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 100, &update);
   
   LatticeFermionD result_MADWF(FGrid_outer);
   result_MADWF = Zero();

From df16202865e5ff8277b805336b12a054b316ea08 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 19 Jan 2021 19:25:27 +0000
Subject: [PATCH 17/82] weird bug in 2pt function...

---
 Grid/qcd/utils/BaryonUtils.h | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index ca8b66ef..7393c232 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -513,6 +513,7 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
   GridBase *grid = q1_left.Grid();
   
   autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( vcorr_read , baryon_corr , AcceleratorRead);
   autoView( v1          , q1_left     , AcceleratorRead);
   autoView( v2          , q2_left     , AcceleratorRead);
   autoView( v3          , q3_left     , AcceleratorRead);
@@ -533,7 +534,8 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
     auto D1 = v1(ss);
     auto D2 = v2(ss);
     auto D3 = v3(ss);
-    typedef decltype(coalescedRead(vbaryon_corr[0])) cVec;
+    //typedef decltype(coalescedRead(vbaryon_corr[0])) cVec;
+    typedef decltype(coalescedRead(vcorr_read[0])) cVec;
     cVec result=Zero();
     BaryonSite(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,parity,wick_contractions,result);
     coalescedWrite(vbaryon_corr[ss],result);
@@ -562,6 +564,7 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
   GridBase *grid = q1_left.Grid();
 
   autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( vcorr_read , baryon_corr , AcceleratorRead);
   autoView( v1          , q1_left     , AcceleratorRead);
   autoView( v2          , q2_left     , AcceleratorRead);
   autoView( v3          , q3_left     , AcceleratorRead);
@@ -570,7 +573,8 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
     auto D1 = v1(ss);
     auto D2 = v2(ss);
     auto D3 = v3(ss);
-    typedef decltype(coalescedRead(vbaryon_corr[0])) spinor;
+    //typedef decltype(coalescedRead(vbaryon_corr[0])) spinor;
+    typedef decltype(coalescedRead(vcorr_read[0])) spinor;
     spinor result=Zero();
     BaryonSiteMatrix(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,wick_contractions,result);
     coalescedWrite(vbaryon_corr[ss],result);
@@ -937,6 +941,7 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
   GridBase *grid = q_tf.Grid();
 
   autoView( vcorr , stn_corr , AcceleratorWrite);
+  autoView( vcorr_read , stn_corr , AcceleratorRead);
   autoView( vq_ti , q_ti     , AcceleratorRead);
   autoView( vq_tf , q_tf     , AcceleratorRead);
 
@@ -947,29 +952,29 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
       auto Dq_ti = vq_ti(ss);
       auto Dq_tf = vq_tf(ss);
-      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      typedef decltype(coalescedRead(vcorr_read[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec_p[0],Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result);
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr_read[ss])+result); 
     });//end loop over lattice sites
 
   } else if (group == 2) {
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
       auto Dq_ti = vq_ti(ss);
       auto Dq_tf = vq_tf(ss);
-      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      typedef decltype(coalescedRead(vcorr_read[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup2Site(Dq_spec_p[0],Dq_ti,Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result);
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr_read[ss])+result); 
     });//end loop over lattice sites
   } else if (group == 3) {
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
       auto Dq_ti = vq_ti(ss);
       auto Dq_tf = vq_tf(ss);
-      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      typedef decltype(coalescedRead(vcorr_read[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup3Site(Dq_spec_p[0],Dq_spec_p[1],Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result);
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr_read[ss])+result); 
     });//end loop over lattice sites
   }
 

From ff1fa988085ae868ae603fe9562968b3dcd57da2 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 21 Jan 2021 21:38:23 -0500
Subject: [PATCH 18/82] Fix for GPU conserveed current

---
 Grid/qcd/action/fermion/WilsonImpl.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Grid/qcd/action/fermion/WilsonImpl.h b/Grid/qcd/action/fermion/WilsonImpl.h
index 52e1ee00..d7941d1f 100644
--- a/Grid/qcd/action/fermion/WilsonImpl.h
+++ b/Grid/qcd/action/fermion/WilsonImpl.h
@@ -106,11 +106,15 @@ public:
 			    const _SpinorField & phi,
 			    int mu)
   {
+    const int Nsimd = SiteHalfSpinor::Nsimd();
     autoView( out_v, out, AcceleratorWrite);
     autoView( phi_v, phi, AcceleratorRead);
     autoView( Umu_v, Umu, AcceleratorRead);
-    accelerator_for(sss,out.Grid()->oSites(),1,{
-	multLink(out_v[sss],Umu_v[sss],phi_v[sss],mu);
+    typedef decltype(coalescedRead(out_v[0]))   calcSpinor;
+    accelerator_for(sss,out.Grid()->oSites(),Nsimd,{
+	calcSpinor tmp;
+	multLink(tmp,Umu_v[sss],phi_v(sss),mu);
+	coalescedWrite(out_v[sss],tmp);
     });
   }
 					   

From 11a5fd09d65427aaaa68d9dc28318bfe92b08097 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 21 Jan 2021 21:39:41 -0500
Subject: [PATCH 19/82] Hot config

---
 tests/debug/Test_cayley_mres.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/debug/Test_cayley_mres.cc b/tests/debug/Test_cayley_mres.cc
index 5282c756..ea88885e 100644
--- a/tests/debug/Test_cayley_mres.cc
+++ b/tests/debug/Test_cayley_mres.cc
@@ -117,8 +117,8 @@ int main (int argc, char ** argv)
   else
   {
     std::cout<<GridLogMessage <<"Using cold configuration"<<std::endl;
-    SU<Nc>::ColdConfiguration(Umu);
-    //  SU<Nc>::HotConfiguration(RNG4,Umu);
+    //SU<Nc>::ColdConfiguration(Umu);
+    SU<Nc>::HotConfiguration(RNG4,Umu);
   }
 
   RealD mass=0.3;

From 2983b6fdf6485569c7d621bb22e4dcec23633e22 Mon Sep 17 00:00:00 2001
From: Michael Marshall <43034299+mmphys@users.noreply.github.com>
Date: Sat, 23 Jan 2021 12:41:48 +0000
Subject: [PATCH 20/82] Optional (superficial) changes to make comparison with
 Hadrons WardIdentity module easier: use Schur solver; example of Hadrons
 random gauge init; logging updates; only solve reverse propagator if provided

---
 tests/debug/Test_cayley_mres.cc | 86 +++++++++++++++------------------
 1 file changed, 40 insertions(+), 46 deletions(-)

diff --git a/tests/debug/Test_cayley_mres.cc b/tests/debug/Test_cayley_mres.cc
index ea88885e..bfbc3cf7 100644
--- a/tests/debug/Test_cayley_mres.cc
+++ b/tests/debug/Test_cayley_mres.cc
@@ -33,13 +33,14 @@ using namespace Grid;
 
 
 template<class What> 
-void  TestConserved(What & Ddwf, What & Ddwfrev, 
+void  TestConserved(What & Ddwf,
 		    LatticeGaugeField &Umu,
 		    GridCartesian         * FGrid,	       GridRedBlackCartesian * FrbGrid,
 		    GridCartesian         * UGrid,	       GridRedBlackCartesian * UrbGrid,
 		    RealD mass, RealD M5,
 		    GridParallelRNG *RNG4,
-		    GridParallelRNG *RNG5);
+		    GridParallelRNG *RNG5,
+                    What *Ddwfrev=nullptr);
 
   Gamma::Algebra Gmu [] = {
     Gamma::Algebra::GammaX,
@@ -102,10 +103,11 @@ int main (int argc, char ** argv)
   GridRedBlackCartesian * FrbGridF = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGridF);
 
 
-  std::vector<int> seeds4({1,2,3,4});
   std::vector<int> seeds5({5,6,7,8});
   GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
-  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  GridParallelRNG          RNG4(UGrid);
+  std::vector<int> seeds4({1,2,3,4}); RNG4.SeedFixedIntegers(seeds4);
+  //const std::string seeds4{ "test-gauge-3000" }; RNG4.SeedUniqueString( seeds4 );
 
   LatticeGaugeField Umu(UGrid);
   if( argc > 1 && argv[1][0] != '-' )
@@ -116,8 +118,8 @@ int main (int argc, char ** argv)
   }
   else
   {
-    std::cout<<GridLogMessage <<"Using cold configuration"<<std::endl;
-    //SU<Nc>::ColdConfiguration(Umu);
+    std::cout<<GridLogMessage <<"Using hot configuration"<<std::endl;
+    // SU<Nc>::ColdConfiguration(Umu);
     SU<Nc>::HotConfiguration(RNG4,Umu);
   }
 
@@ -127,7 +129,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage <<"DomainWallFermion test"<<std::endl;
   std::cout<<GridLogMessage <<"======================"<<std::endl;
   DomainWallFermionR Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-  TestConserved<DomainWallFermionR>(Ddwf,Ddwf,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5);
+  TestConserved<DomainWallFermionR>(Ddwf,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5);
 
   RealD b=1.5;// Scale factor b+c=2, b-c=1
   RealD c=0.5;
@@ -137,13 +139,13 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage <<"MobiusFermion test"<<std::endl;
   std::cout<<GridLogMessage <<"======================"<<std::endl;
   MobiusFermionR Dmob(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
-  TestConserved<MobiusFermionR>(Dmob,Dmob,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5);
+  TestConserved<MobiusFermionR>(Dmob,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5);
 
   std::cout<<GridLogMessage <<"======================"<<std::endl;
   std::cout<<GridLogMessage <<"ScaledShamirFermion test"<<std::endl;
   std::cout<<GridLogMessage <<"======================"<<std::endl;
   ScaledShamirFermionR Dsham(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,2.0);
-  TestConserved<ScaledShamirFermionR>(Dsham,Dsham,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5);
+  TestConserved<ScaledShamirFermionR>(Dsham,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5);
 
   std::cout<<GridLogMessage <<"======================"<<std::endl;
   std::cout<<GridLogMessage <<"ZMobiusFermion test"<<std::endl;
@@ -152,8 +154,7 @@ int main (int argc, char ** argv)
   //  for(int s=0;s<Ls;s++) omegasrev[s]=omegas[Ls-1-s];
   ZMobiusFermionR ZDmob(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,omegas,b,c);
   ZMobiusFermionR ZDmobrev(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,omegasrev,b,c);
-
-  TestConserved<ZMobiusFermionR>(ZDmob,ZDmobrev,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5);
+  TestConserved<ZMobiusFermionR>(ZDmob,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5,&ZDmobrev);
 
   Grid_finalize();
 }
@@ -161,22 +162,17 @@ int main (int argc, char ** argv)
 
 
 template<class Action> 
-void  TestConserved(Action & Ddwf, 
-		    Action & Ddwfrev, 
+void  TestConserved(Action & Ddwf,
 		    LatticeGaugeField &Umu,
 		    GridCartesian         * FGrid,	       GridRedBlackCartesian * FrbGrid,
 		    GridCartesian         * UGrid,	       GridRedBlackCartesian * UrbGrid,
 		    RealD mass, RealD M5,
 		    GridParallelRNG *RNG4,
-		    GridParallelRNG *RNG5)
+		    GridParallelRNG *RNG5,
+                    Action * Ddwfrev)
 {
-  int Ls=Ddwf.Ls;
-
-  LatticePropagator phys_src(UGrid); 
-
-  std::vector<LatticeColourMatrix> U(4,UGrid);
-  
-  LatticePropagator seqsrc(FGrid); 
+  LatticePropagator phys_src(UGrid);
+  LatticePropagator seqsrc(FGrid);
   LatticePropagator prop5(FGrid); 
   LatticePropagator prop5rev(FGrid); 
   LatticePropagator prop4(UGrid); 
@@ -194,9 +190,9 @@ void  TestConserved(Action & Ddwf,
   phys_src=Zero();
   pokeSite(kronecker,phys_src,coor);
   
-  MdagMLinearOperator<Action,LatticeFermion> HermOp(Ddwf);
-  MdagMLinearOperator<Action,LatticeFermion> HermOprev(Ddwfrev);
   ConjugateGradient<LatticeFermion> CG(1.0e-16,100000);
+  SchurRedBlackDiagTwoSolve<LatticeFermion> schur(CG);
+  ZeroGuesser<LatticeFermion> zpg;
   for(int s=0;s<Nd;s++){
     for(int c=0;c<Nc;c++){
       LatticeFermion src4  (UGrid); 
@@ -206,20 +202,20 @@ void  TestConserved(Action & Ddwf,
       Ddwf.ImportPhysicalFermionSource(src4,src5);
 
       LatticeFermion result5(FGrid); result5=Zero();
-
-      // CGNE
-      LatticeFermion Mdagsrc5  (FGrid); 
-      Ddwf.Mdag(src5,Mdagsrc5);
-      CG(HermOp,Mdagsrc5,result5);
+      schur(Ddwf,src5,result5,zpg);
+      std::cout<<GridLogMessage<<"spin "<<s<<" color "<<c<<" norm2(sourc5d) "<<norm2(src5)
+               <<" norm2(result5d) "<<norm2(result5)<<std::endl;
       FermToProp<Action>(prop5,result5,s,c);
 
       LatticeFermion result4(UGrid);
       Ddwf.ExportPhysicalFermionSolution(result5,result4);
       FermToProp<Action>(prop4,result4,s,c);
 
-      Ddwfrev.ImportPhysicalFermionSource(src4,src5);
-      Ddwfrev.Mdag(src5,Mdagsrc5);
-      CG(HermOprev,Mdagsrc5,result5);
+      if( Ddwfrev ) {
+        Ddwfrev->ImportPhysicalFermionSource(src4,src5);
+        result5 = Zero();
+        schur(*Ddwfrev,src5,result5,zpg);
+      }
       FermToProp<Action>(prop5rev,result5,s,c);
     }
   }
@@ -251,11 +247,7 @@ void  TestConserved(Action & Ddwf,
       PropToFerm<Action>(src5,seqsrc,s,c);
 
       LatticeFermion result5(FGrid); result5=Zero();
-
-      // CGNE
-      LatticeFermion Mdagsrc5  (FGrid); 
-      Ddwf.Mdag(src5,Mdagsrc5);
-      CG(HermOp,Mdagsrc5,result5);
+      schur(Ddwf,src5,result5,zpg);
 
       LatticeFermion result4(UGrid);
       Ddwf.ExportPhysicalFermionSolution(result5,result4);
@@ -276,10 +268,10 @@ void  TestConserved(Action & Ddwf,
   Ddwf.ContractConservedCurrent(prop5rev,prop5,Vector_mu,phys_src,Current::Vector,Tdir);
   Ddwf.ContractJ5q(prop5,PJ5q);
   
-  PA       = trace(g5*Axial_mu);
-  SV       = trace(Vector_mu);
-  VV       = trace(gT*Vector_mu);
-  PP       = trace(adj(prop4)*prop4);
+  PA       = trace(g5*Axial_mu);      // Pseudoscalar-Axial conserved current
+  SV       = trace(Vector_mu);        // Scalar-Vector conserved current
+  VV       = trace(gT*Vector_mu);     // (local) Vector-Vector conserved current
+  PP       = trace(adj(prop4)*prop4); // Pseudoscalar density
   
   // Spatial sum
   sliceSum(PA,sumPA,Tdir);
@@ -288,15 +280,17 @@ void  TestConserved(Action & Ddwf,
   sliceSum(PP,sumPP,Tdir);
   sliceSum(PJ5q,sumPJ5q,Tdir);
 
-  int Nt=sumPA.size();
+  const int Nt{static_cast<int>(sumPA.size())};
+  std::cout<<GridLogMessage<<"Vector Ward identity by timeslice (~ 0)"<<std::endl;
   for(int t=0;t<Nt;t++){
-    std::cout <<" SV "<<real(TensorRemove(sumSV[t]));
-    std::cout <<" VV "<<real(TensorRemove(sumVV[t]))<<std::endl;
+    std::cout<<GridLogMessage <<" t "<<t<<" SV "<<real(TensorRemove(sumSV[t]))<<" VV "<<real(TensorRemove(sumVV[t]))<<std::endl;
   }
+  std::cout<<GridLogMessage<<"Axial Ward identity by timeslice (defect ~ 0)"<<std::endl;
   for(int t=0;t<Nt;t++){
-    std::cout <<" PAc "<<real(TensorRemove(sumPA[t]));
-    std::cout <<" PJ5q "<<real(TensorRemove(sumPJ5q[t]));
-    std::cout <<" Ward Identity defect " <<real(TensorRemove(sumPA[t]-sumPA[(t-1+Nt)%Nt] - 2.0*(Ddwf.mass*sumPP[t] + sumPJ5q[t]) ))<<"\n";
+    const RealD DmuPAmu{real(TensorRemove(sumPA[t]-sumPA[(t-1+Nt)%Nt]))};
+    std::cout<<GridLogMessage<<" t "<<t<<" DmuPAmu "<<DmuPAmu
+             <<" PP "<<real(TensorRemove(sumPP[t]))<<" PJ5q "<<real(TensorRemove(sumPJ5q[t]))
+             <<" Ward Identity defect " <<(DmuPAmu - 2.*real(TensorRemove(Ddwf.mass*sumPP[t] + sumPJ5q[t])))<<std::endl;
   }
   
   ///////////////////////////////

From 77063418daf3d696df80af012a375daa47aca246 Mon Sep 17 00:00:00 2001
From: Michael Marshall <43034299+mmphys@users.noreply.github.com>
Date: Mon, 25 Jan 2021 15:09:36 +0000
Subject: [PATCH 21/82] Fix issue for GPU by ensuring accelerator_inline
 version of convertType is available for Grid::complex<T>. This removes many
 warnings in Hadrons Simplify the SFINAE syntax and correct convertType for
 iScalar

---
 Grid/lattice/Lattice_transfer.h | 39 +++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/Grid/lattice/Lattice_transfer.h b/Grid/lattice/Lattice_transfer.h
index 91de721f..c91fa4d1 100644
--- a/Grid/lattice/Lattice_transfer.h
+++ b/Grid/lattice/Lattice_transfer.h
@@ -97,6 +97,21 @@ accelerator_inline void convertType(ComplexF & out, const std::complex<float> &
   out = in;
 }
 
+template<typename T>
+accelerator_inline typename std::enable_if<isGridFundamental<T>::value>::type
+convertType(T & out, const T & in) {
+  out = in;
+}
+
+// This would allow for conversions between GridFundamental types, but is not strictly needed as yet
+/*template<typename T1, typename T2>
+accelerator_inline typename std::enable_if<isGridFundamental<T1>::value && isGridFundamental<T2>::value>::type
+// Or to make this very broad, conversions between anything that's not a GridTensor could be allowed
+//accelerator_inline typename std::enable_if<!isGridTensor<T1>::value && !isGridTensor<T2>::value>::type
+convertType(T1 & out, const T2 & in) {
+  out = in;
+}*/
+
 #ifdef GRID_SIMT
 accelerator_inline void convertType(vComplexF & out, const ComplexF & in) {
   ((ComplexF*)&out)[acceleratorSIMTlane(vComplexF::Nsimd())] = in;
@@ -117,23 +132,20 @@ accelerator_inline void convertType(vComplexD2 & out, const vComplexF & in) {
   Optimization::PrecisionChange::StoD(in.v,out._internal[0].v,out._internal[1].v);
 }
 
-template<typename T1,typename T2,int N>
-  accelerator_inline void convertType(iMatrix<T1,N> & out, const iMatrix<T2,N> & in);
-template<typename T1,typename T2,int N>
-  accelerator_inline void convertType(iVector<T1,N> & out, const iVector<T2,N> & in);
-
-template<typename T1,typename T2, typename std::enable_if<!isGridScalar<T1>::value, T1>::type* = nullptr>
-accelerator_inline void convertType(T1 & out, const iScalar<T2> & in) {
-  convertType(out,in._internal);
+template<typename T1,typename T2>
+accelerator_inline void convertType(iScalar<T1> & out, const iScalar<T2> & in) {
+  convertType(out._internal,in._internal);
 }
 
-template<typename T1, typename std::enable_if<!isGridScalar<T1>::value, T1>::type* = nullptr>
-accelerator_inline void convertType(T1 & out, const iScalar<T1> & in) {
+template<typename T1,typename T2>
+accelerator_inline typename std::enable_if<!isGridScalar<T1>::value>::type
+convertType(T1 & out, const iScalar<T2> & in) {
   convertType(out,in._internal);
 }
 
 template<typename T1,typename T2>
-accelerator_inline void convertType(iScalar<T1> & out, const T2 & in) {
+accelerator_inline typename std::enable_if<!isGridScalar<T2>::value>::type
+convertType(iScalar<T1> & out, const T2 & in) {
   convertType(out._internal,in);
 }
 
@@ -150,11 +162,6 @@ accelerator_inline void convertType(iVector<T1,N> & out, const iVector<T2,N> & i
     convertType(out._internal[i],in._internal[i]);
 }
 
-template<typename T, typename std::enable_if<isGridFundamental<T>::value, T>::type* = nullptr>
-accelerator_inline void convertType(T & out, const T & in) {
-  out = in;
-}
-
 template<typename T1,typename T2>
 accelerator_inline void convertType(Lattice<T1> & out, const Lattice<T2> & in) {
   autoView( out_v , out,AcceleratorWrite);

From 81d88d9f4df1c9944c8b33db6dd123c36560757d Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Wed, 27 Jan 2021 21:09:51 +0000
Subject: [PATCH 22/82] fixes

---
 Grid/qcd/utils/BaryonUtils.h | 63 ++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 31 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 7393c232..d6b48ba0 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -52,7 +52,7 @@ public:
          const Gamma GammaA_right,
          const Gamma GammaB_right,
          const int parity,
-         const bool * wick_contractions,
+         const int wick_contractions,
          robj &result);
   template <class mobj, class robj> accelerator_inline
   static void BaryonSiteMatrix(const mobj &D1,
@@ -62,12 +62,12 @@ public:
          const Gamma GammaB_left,
          const Gamma GammaA_right,
          const Gamma GammaB_right,
-         const bool * wick_contractions,
+         const int wick_contractions,
          robj &result);
   public:
   static void WickContractions(std::string qi, 
                  std::string qf, 
-                 bool* wick_contractions);
+                 int &wick_contractions);
   static void ContractBaryons(const PropagatorField &q1_left,
          const PropagatorField &q2_left,
          const PropagatorField &q3_left,
@@ -75,7 +75,7 @@ public:
          const Gamma GammaB_left,
          const Gamma GammaA_right,
          const Gamma GammaB_right,
-         const bool* wick_contractions,
+         const int wick_contractions,
          const int parity,
          ComplexField &baryon_corr);
   static void ContractBaryonsMatrix(const PropagatorField &q1_left,
@@ -85,7 +85,7 @@ public:
          const Gamma GammaB_left,
          const Gamma GammaA_right,
          const Gamma GammaB_right,
-         const bool* wick_contractions,
+         const int wick_contractions,
          SpinMatrixField &baryon_corr);
   template <class mobj, class robj>
   static void ContractBaryonsSliced(const mobj &D1,
@@ -95,7 +95,7 @@ public:
          const Gamma GammaB_left,
          const Gamma GammaA_right,
          const Gamma GammaB_right,
-         const bool* wick_contractions,
+         const int wick_contractions,
          const int parity,
          const int nt,
          robj &result);
@@ -107,7 +107,7 @@ public:
          const Gamma GammaB_left,
          const Gamma GammaA_right,
          const Gamma GammaB_right,
-         const bool* wick_contractions,
+         const int wick_contractions,
          const int nt,
          robj &result);
   private:
@@ -234,7 +234,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
                 const Gamma GammaA_f,
                 const Gamma GammaB_f,
                 const int parity,
-                const bool * wick_contraction,
+                const int wick_contraction,
                 robj &result)
 {
 
@@ -268,7 +268,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
 
       ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
       //This is the \delta_{456}^{123} part
-      if (wick_contraction[0]){
+      if (wick_contraction & 1){
         for (int rho=0; rho<Ns; rho++){
           auto GAf_D1_GAi_P_rr_cc = GAf_D1_GAi_P()(rho,rho)(c_f,c_i);
           for (int alpha_f=0; alpha_f<Ns; alpha_f++){
@@ -280,7 +280,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
         }
       }   
       //This is the \delta_{456}^{231} part
-      if (wick_contraction[1]){
+      if (wick_contraction & 2){
         for (int rho=0; rho<Ns; rho++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto D1_GAi_P_ar_ac = D1_GAi_P()(alpha_f,rho)(a_f,c_i);
@@ -292,7 +292,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{312} part
-      if (wick_contraction[2]){
+      if (wick_contraction & 4){
         for (int rho=0; rho<Ns; rho++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto GBf_D1_GAi_P_ar_bc = GBf_D1_GAi_P()(alpha_f,rho)(b_f,c_i);
@@ -304,7 +304,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{132} part
-      if (wick_contraction[3]){
+      if (wick_contraction & 8){
         for (int rho=0; rho<Ns; rho++){
           auto GAf_D1_GAi_P_rr_cc = GAf_D1_GAi_P()(rho,rho)(c_f,c_i);
           for (int alpha_f=0; alpha_f<Ns; alpha_f++){
@@ -316,7 +316,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
         }
       }   
       //This is the \delta_{456}^{321} part
-      if (wick_contraction[4]){
+      if (wick_contraction & 16){
         for (int rho=0; rho<Ns; rho++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto GBf_D1_GAi_P_ar_bc = GBf_D1_GAi_P()(alpha_f,rho)(b_f,c_i);
@@ -328,7 +328,7 @@ void BaryonUtils<FImpl>::BaryonSite(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{213} part
-      if (wick_contraction[5]){
+      if (wick_contraction & 32){
         for (int rho=0; rho<Ns; rho++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto D1_GAi_P_ar_ac = D1_GAi_P()(alpha_f,rho)(a_f,c_i);
@@ -353,7 +353,7 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
                 const Gamma GammaB_i,
                 const Gamma GammaA_f,
                 const Gamma GammaB_f,
-                const bool * wick_contraction,
+                const int wick_contraction,
                 robj &result)
 {
 
@@ -383,7 +383,7 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
 
       ee = Real(eSgn_f * eSgn_i); //epsilon_sgn[ie_n] * epsilon_sgn[ie_s];
       //This is the \delta_{456}^{123} part
-      if (wick_contraction[0]){
+      if (wick_contraction & 1){
         for (int rho_i=0; rho_i<Ns; rho_i++){
         for (int rho_f=0; rho_f<Ns; rho_f++){
           auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
@@ -396,7 +396,7 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{231} part
-      if (wick_contraction[1]){
+      if (wick_contraction & 2){
         for (int rho_i=0; rho_i<Ns; rho_i++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
@@ -411,7 +411,7 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{312} part
-      if (wick_contraction[2]){
+      if (wick_contraction & 4){
         for (int rho_i=0; rho_i<Ns; rho_i++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
@@ -426,7 +426,7 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{132} part
-      if (wick_contraction[3]){
+      if (wick_contraction & 8){
         for (int rho_i=0; rho_i<Ns; rho_i++){
         for (int rho_f=0; rho_f<Ns; rho_f++){
           auto GAf_D1_GAi_rr_cc = GAf_D1_GAi()(rho_f,rho_i)(c_f,c_i);
@@ -439,7 +439,7 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{321} part
-      if (wick_contraction[4]){
+      if (wick_contraction & 16){
         for (int rho_i=0; rho_i<Ns; rho_i++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto GBf_D1_GAi_ar_bc = GBf_D1_GAi()(alpha_f,rho_i)(b_f,c_i);
@@ -454,7 +454,7 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
         }}
       }   
       //This is the \delta_{456}^{213} part
-      if (wick_contraction[5]){
+      if (wick_contraction & 32){
         for (int rho_i=0; rho_i<Ns; rho_i++){
         for (int alpha_f=0; alpha_f<Ns; alpha_f++){
           auto D1_GAi_ar_ac = D1_GAi()(alpha_f,rho_i)(a_f,c_i);
@@ -477,13 +477,14 @@ void BaryonUtils<FImpl>::BaryonSiteMatrix(const mobj &D1,
  * flavours.                                                     *
  * The array wick_contractions must be of length 6               */
 template<class FImpl>
-void BaryonUtils<FImpl>::WickContractions(std::string qi, std::string qf, bool* wick_contractions) {
+void BaryonUtils<FImpl>::WickContractions(std::string qi, std::string qf, int &wick_contractions) {
+    assert(qi.size() == 3 && qf.size() == 3 && "Only sets of 3 quarks accepted.");
     const int epsilon[6][3] = {{0,1,2},{1,2,0},{2,0,1},{0,2,1},{2,1,0},{1,0,2}};
+    wick_contractions=0;
     for (int ie=0; ie < 6 ; ie++) {
-        wick_contractions[ie] = (qi.size() == 3 && qf.size() == 3
-                                    && qi[0] == qf[epsilon[ie][0]] 
+        wick_contractions += ( ( qi[0] == qf[epsilon[ie][0]] 
                                     && qi[1] == qf[epsilon[ie][1]] 
-                                    && qi[2] == qf[epsilon[ie][2]]);
+                                    && qi[2] == qf[epsilon[ie][2]]) ? 1 : 0) << ie;
     }
 }
 
@@ -500,7 +501,7 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
              const Gamma GammaB_left,
              const Gamma GammaA_right,
              const Gamma GammaB_right,
-             const bool* wick_contractions,
+             const int wick_contractions,
              const int parity,
              ComplexField &baryon_corr)
 {
@@ -522,9 +523,9 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
   bytes += grid->oSites() * (432.*sizeof(vComplex) + 126.*sizeof(int) + 36.*sizeof(Real));
   for (int ie=0; ie < 6 ; ie++){
     if(ie==0 or ie==3){
-       bytes += grid->oSites() * (4.*sizeof(int) + 4752.*sizeof(vComplex)) * wick_contractions[ie];
+       //bytes += grid->oSites() * (4.*sizeof(int) + 4752.*sizeof(vComplex)) * wick_contractions[ie];
     } else{
-       bytes += grid->oSites() * (64.*sizeof(int) + 5184.*sizeof(vComplex)) * wick_contractions[ie];
+       //bytes += grid->oSites() * (64.*sizeof(int) + 5184.*sizeof(vComplex)) * wick_contractions[ie];
     }
   }
   Real t=0.;
@@ -554,7 +555,7 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
              const Gamma GammaB_left,
              const Gamma GammaA_right,
              const Gamma GammaB_right,
-             const bool* wick_contractions,
+             const int wick_contractions,
              SpinMatrixField &baryon_corr)
 {
 
@@ -595,7 +596,7 @@ void BaryonUtils<FImpl>::ContractBaryonsSliced(const mobj &D1,
                          const Gamma GammaB_left,
                          const Gamma GammaA_right,
                          const Gamma GammaB_right,
-             const bool* wick_contractions,
+             const int wick_contractions,
              const int parity,
              const int nt,
              robj &result)
@@ -620,7 +621,7 @@ void BaryonUtils<FImpl>::ContractBaryonsSlicedMatrix(const mobj &D1,
              const Gamma GammaB_left,
              const Gamma GammaA_right,
              const Gamma GammaB_right,
-             const bool* wick_contractions,
+             const int wick_contractions,
              const int nt,
              robj &result)
 {

From 712bb406502922fb0ceb733e7a74f0ce2b902e2c Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 15 Dec 2020 16:33:29 +0000
Subject: [PATCH 23/82] merge develop

---
 tests/solver/Test_zMADWF_prec.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/solver/Test_zMADWF_prec.cc b/tests/solver/Test_zMADWF_prec.cc
index d1168764..f18e1d86 100644
--- a/tests/solver/Test_zMADWF_prec.cc
+++ b/tests/solver/Test_zMADWF_prec.cc
@@ -52,7 +52,7 @@ struct TestParams{
   bool zmobius_inner;
   double lambda_max; //upper bound of H_T eigenvalue range required to generate zMobius approximation
   
-  TestParams(): load_config(true), config_file("ckpoint_lat.1000"), mass(0.01),
+  TestParams(): load_config(false), config_file("ckpoint_lat.1000"), mass(0.01),
 		Ls_outer(24), b_plus_c_outer(2.0), resid_outer(1e-8),
 		Ls_inner(12), b_plus_c_inner(1.0), resid_inner(1e-8), zmobius_inner(true), lambda_max(1.42), outer_precon("Standard"), inner_precon("Standard")
   {}
@@ -246,7 +246,7 @@ void run(const TestParams &params){
   typename RunParamsInner::SchurSolverType SchurSolver_inner(CG_inner);
 
   ZeroGuesser<LatticeFermion> Guess;
-  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 100, &update);
+  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 10000, &update);
   
   LatticeFermionD result_MADWF(FGrid_outer);
   result_MADWF = Zero();

From 7905afa9f5b19b147b59eb809b87d1a27e4fc950 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 19 Jan 2021 12:32:48 +0000
Subject: [PATCH 24/82] revert changes

---
 tests/solver/Test_zMADWF_prec.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/solver/Test_zMADWF_prec.cc b/tests/solver/Test_zMADWF_prec.cc
index f18e1d86..d1168764 100644
--- a/tests/solver/Test_zMADWF_prec.cc
+++ b/tests/solver/Test_zMADWF_prec.cc
@@ -52,7 +52,7 @@ struct TestParams{
   bool zmobius_inner;
   double lambda_max; //upper bound of H_T eigenvalue range required to generate zMobius approximation
   
-  TestParams(): load_config(false), config_file("ckpoint_lat.1000"), mass(0.01),
+  TestParams(): load_config(true), config_file("ckpoint_lat.1000"), mass(0.01),
 		Ls_outer(24), b_plus_c_outer(2.0), resid_outer(1e-8),
 		Ls_inner(12), b_plus_c_inner(1.0), resid_inner(1e-8), zmobius_inner(true), lambda_max(1.42), outer_precon("Standard"), inner_precon("Standard")
   {}
@@ -246,7 +246,7 @@ void run(const TestParams &params){
   typename RunParamsInner::SchurSolverType SchurSolver_inner(CG_inner);
 
   ZeroGuesser<LatticeFermion> Guess;
-  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 10000, &update);
+  MADWF<MobiusFermionD, ZMobiusFermionD, PVtype, typename RunParamsInner::SchurSolverType, ZeroGuesser<LatticeFermion> > madwf(D_outer, D_inner, PV_outer, SchurSolver_inner, Guess, params.resid_outer, 100, &update);
   
   LatticeFermionD result_MADWF(FGrid_outer);
   result_MADWF = Zero();

From 96dd7a8fbd66d438618962ca930e7bd1eef34d08 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Mon, 16 Nov 2020 17:15:34 +0100
Subject: [PATCH 25/82] Flop cout matches DiRAC-ITT-2020

---
 benchmarks/Benchmark_ITT.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 032535b3..5d602ce9 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -445,7 +445,7 @@ public:
 	// 1344= 3*(2*8+6)*2*8 + 8*3*2*2 + 3*4*2*8
 	// 1344 = Nc* (6+(Nc-1)*8)*2*Nd + Nd*Nc*2*2  + Nd*Nc*Ns*2
 	//	double flops=(1344.0*volume)/2;
-#if 1
+#if 0
 	double fps = Nc* (6+(Nc-1)*8)*Ns*Nd + Nd*Nc*Ns  + Nd*Nc*Ns*2;
 #else
 	double fps = Nc* (6+(Nc-1)*8)*Ns*Nd + 2*Nd*Nc*Ns  + 2*Nd*Nc*Ns*2;

From a673b6a54da17b319d8f6707893a9a3f4005be32 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Thu, 28 Jan 2021 14:15:09 +0000
Subject: [PATCH 26/82] prettify

---
 Grid/qcd/utils/BaryonUtils.h | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index d6b48ba0..56c5781d 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -513,11 +513,11 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
 
   GridBase *grid = q1_left.Grid();
   
-  autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
-  autoView( vcorr_read , baryon_corr , AcceleratorRead);
-  autoView( v1          , q1_left     , AcceleratorRead);
-  autoView( v2          , q2_left     , AcceleratorRead);
-  autoView( v3          , q3_left     , AcceleratorRead);
+  autoView( vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( vcorr_read   , baryon_corr , AcceleratorRead);
+  autoView( v1           , q1_left     , AcceleratorRead);
+  autoView( v2           , q2_left     , AcceleratorRead);
+  autoView( v3           , q3_left     , AcceleratorRead);
 
   Real bytes =0.;
   bytes += grid->oSites() * (432.*sizeof(vComplex) + 126.*sizeof(int) + 36.*sizeof(Real));
@@ -564,11 +564,11 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
  
   GridBase *grid = q1_left.Grid();
 
-  autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
-  autoView( vcorr_read , baryon_corr , AcceleratorRead);
-  autoView( v1          , q1_left     , AcceleratorRead);
-  autoView( v2          , q2_left     , AcceleratorRead);
-  autoView( v3          , q3_left     , AcceleratorRead);
+  autoView( vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( vcorr_read   , baryon_corr , AcceleratorRead);
+  autoView( v1           , q1_left     , AcceleratorRead);
+  autoView( v2           , q2_left     , AcceleratorRead);
+  autoView( v3           , q3_left     , AcceleratorRead);
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
     auto D1 = v1(ss);
@@ -941,10 +941,10 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
 
   GridBase *grid = q_tf.Grid();
 
-  autoView( vcorr , stn_corr , AcceleratorWrite);
+  autoView( vcorr      , stn_corr , AcceleratorWrite);
   autoView( vcorr_read , stn_corr , AcceleratorRead);
-  autoView( vq_ti , q_ti     , AcceleratorRead);
-  autoView( vq_tf , q_tf     , AcceleratorRead);
+  autoView( vq_ti      , q_ti     , AcceleratorRead);
+  autoView( vq_tf      , q_tf     , AcceleratorRead);
 
   Vector<mobj> my_Dq_spec{Dq_spec1,Dq_spec2};
   mobj * Dq_spec_p = &my_Dq_spec[0];

From bc496dd8440449a2e9d1bc5dce074192640cd094 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Thu, 28 Jan 2021 14:29:56 +0000
Subject: [PATCH 27/82] change back benchmark_ITT

---
 benchmarks/Benchmark_ITT.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 5d602ce9..032535b3 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -445,7 +445,7 @@ public:
 	// 1344= 3*(2*8+6)*2*8 + 8*3*2*2 + 3*4*2*8
 	// 1344 = Nc* (6+(Nc-1)*8)*2*Nd + Nd*Nc*2*2  + Nd*Nc*Ns*2
 	//	double flops=(1344.0*volume)/2;
-#if 0
+#if 1
 	double fps = Nc* (6+(Nc-1)*8)*Ns*Nd + Nd*Nc*Ns  + Nd*Nc*Ns*2;
 #else
 	double fps = Nc* (6+(Nc-1)*8)*Ns*Nd + 2*Nd*Nc*Ns  + 2*Nd*Nc*Ns*2;

From 019ffe17d4f1ba9d167cb45f62ea7a0df0c19adc Mon Sep 17 00:00:00 2001
From: Christoph Lehner <christoph@lhnr.de>
Date: Tue, 2 Feb 2021 11:32:23 +0100
Subject: [PATCH 28/82] Allow for GPU vector width beyond 64

---
 Grid/util/Coordinate.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/util/Coordinate.h b/Grid/util/Coordinate.h
index 004fbc72..89f73264 100644
--- a/Grid/util/Coordinate.h
+++ b/Grid/util/Coordinate.h
@@ -88,7 +88,7 @@ public:
 // Coordinate class, maxdims = 8 for now.
 ////////////////////////////////////////////////////////////////
 #define GRID_MAX_LATTICE_DIMENSION (8)
-#define GRID_MAX_SIMD              (16)
+#define GRID_MAX_SIMD              (sizeof(vInteger)/sizeof(Integer))
 
 static constexpr int MaxDims = GRID_MAX_LATTICE_DIMENSION;
 

From 9b9a53f87066b5c67b607f56cb73c56efa5d4243 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 2 Feb 2021 13:06:43 +0000
Subject: [PATCH 29/82] ...

---
 Grid/qcd/utils/BaryonUtils.h | 53 ++++++++++++++++++++++--------------
 1 file changed, 32 insertions(+), 21 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 8a4ff6ac..94cc07b1 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -1350,15 +1350,19 @@ void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
   // \gamma_\mu^L * Dq_loop 
   auto trGDq = TensorRemove(trace(Gamma_H * Dq_loop)); 
 
+  Real ee;
+
   for (int ie_s=0; ie_s < 6 ; ie_s++){
-    int a_s = epsilon[ie_s][0]; //a
-    int b_s = epsilon[ie_s][1]; //b
-    int c_s = epsilon[ie_s][2]; //c
+      int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+      int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+      int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_s = (ie_s < 3 ? 1 : -1);
     for (int ie_x=0; ie_x < 6 ; ie_x++){
-      int a_x = epsilon[ie_x][0]; //a'
-      int b_x = epsilon[ie_x][1]; //b'
-      int c_x = epsilon[ie_x][2]; //c'
-      auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq; 
+      int a_x = (ie_x < 3 ? ie_x       : (6-ie_x)%3 ); //epsilon[ie_x][0]; //a'
+      int b_x = (ie_x < 3 ? (ie_x+1)%3 : (8-ie_x)%3 ); //epsilon[ie_x][1]; //b'
+      int c_x = (ie_x < 3 ? (ie_x+2)%3 : (7-ie_x)%3 ); //epsilon[ie_x][2]; //c'
+      int eSgn_x = (ie_x < 3 ? 1 : -1);
+      ee = Real(eSgn_s * eSgn_x);
       for (int alpha_x=0; alpha_x<Ns; alpha_x++){
       for (int beta_s=0; beta_s<Ns; beta_s++){
         auto GDsGDdG_ab_ba = GDsGDd()(alpha_x,beta_s)(b_x,a_s);
@@ -1451,7 +1455,7 @@ void BaryonUtils<FImpl>::XiToSigmaEye(const PropagatorField &qq_loop,
 				                 const Gamma GammaB_xi,
 		                 		 const Gamma GammaB_sigma,
 						 const std::string op,
-						 SpinMatrixField &stn_corr)
+						 SpinMatrixField &xts_corr)
 {
 
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
@@ -1459,24 +1463,31 @@ void BaryonUtils<FImpl>::XiToSigmaEye(const PropagatorField &qq_loop,
 
   GridBase *grid = qs_ti.Grid();
 
-  autoView( vcorr, stn_corr, CpuWrite);
-  autoView( vq_loop , qq_loop, CpuRead);
-  autoView( vd_tf , qd_tf, CpuRead);
-  autoView( vs_ti , qs_ti, CpuRead);
+  autoView( vcorr   , xts_corr , AcceleratorWrite);
+  autoView( vq_loop , qq_loop  , AcceleratorRead);
+  autoView( vd_tf   , qd_tf    , AcceleratorRead);
+  autoView( vs_ti   , qs_ti    , AcceleratorRead);
+
+  bool doQ1 = (op == "Q1");
+  bool doQ2 = (op == "Q2");
+
+  Vector<mobj> my_Dq_spec{Dd_spec,Ds_spec};
+  mobj * Dq_spec_p = &my_Dq_spec[0];
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto Dq_loop = vq_loop[ss];
-    auto Dd_tf = vd_tf[ss];
-    auto Ds_ti = vs_ti[ss];
-    sobj result=Zero();
-    if(op == "Q1"){
-      XiToSigmaQ1EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
-    } else if(op == "Q2"){
-      XiToSigmaQ2EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
+    auto Dq_loop = vq_loop(ss);
+    auto Dd_tf   = vd_tf(ss);
+    auto Ds_ti   = vs_ti(ss);
+    typedef decltype(coalescedRead(vcorr[0])) spinor;
+    spinor result=Zero();
+    if(doQ1){
+      XiToSigmaQ1EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[1],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
+    } else if(doQ2){
+      XiToSigmaQ2EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
     } else {
       assert(0 && "Weak Operator not correctly specified");
     }
-      vcorr[ss] = result; 
+    coalescedWrite(vcorr[ss],result);
   }  );//end loop over lattice sites
 }
 

From 3215d88a91214bc6bbed75cf21cf1a4a28b0bbfb Mon Sep 17 00:00:00 2001
From: Michael Marshall <43034299+mmphys@users.noreply.github.com>
Date: Wed, 3 Feb 2021 15:17:03 +0000
Subject: [PATCH 30/82] Simplify syntax with Grid::EnableIf post code review.
 Updated EnableIf so that ReturnType defaults to void in same way as
 std::enable_if see https://en.cppreference.com/w/cpp/types/enable_if

---
 Grid/lattice/Lattice_transfer.h | 9 +++------
 Grid/simd/Grid_vector_types.h   | 4 ++--
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/Grid/lattice/Lattice_transfer.h b/Grid/lattice/Lattice_transfer.h
index c91fa4d1..5a26cce9 100644
--- a/Grid/lattice/Lattice_transfer.h
+++ b/Grid/lattice/Lattice_transfer.h
@@ -98,8 +98,7 @@ accelerator_inline void convertType(ComplexF & out, const std::complex<float> &
 }
 
 template<typename T>
-accelerator_inline typename std::enable_if<isGridFundamental<T>::value>::type
-convertType(T & out, const T & in) {
+accelerator_inline EnableIf<isGridFundamental<T>> convertType(T & out, const T & in) {
   out = in;
 }
 
@@ -138,14 +137,12 @@ accelerator_inline void convertType(iScalar<T1> & out, const iScalar<T2> & in) {
 }
 
 template<typename T1,typename T2>
-accelerator_inline typename std::enable_if<!isGridScalar<T1>::value>::type
-convertType(T1 & out, const iScalar<T2> & in) {
+accelerator_inline NotEnableIf<isGridScalar<T1>> convertType(T1 & out, const iScalar<T2> & in) {
   convertType(out,in._internal);
 }
 
 template<typename T1,typename T2>
-accelerator_inline typename std::enable_if<!isGridScalar<T2>::value>::type
-convertType(iScalar<T1> & out, const T2 & in) {
+accelerator_inline NotEnableIf<isGridScalar<T2>> convertType(iScalar<T1> & out, const T2 & in) {
   convertType(out._internal,in);
 }
 
diff --git a/Grid/simd/Grid_vector_types.h b/Grid/simd/Grid_vector_types.h
index c07077a3..4f952bb2 100644
--- a/Grid/simd/Grid_vector_types.h
+++ b/Grid/simd/Grid_vector_types.h
@@ -208,8 +208,8 @@ struct RealPart<complex<T> > {
 //////////////////////////////////////
 // type alias used to simplify the syntax of std::enable_if
 template <typename T> using Invoke = typename T::type;
-template <typename Condition, typename ReturnType> using EnableIf    = Invoke<std::enable_if<Condition::value, ReturnType> >;
-template <typename Condition, typename ReturnType> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
+template <typename Condition, typename ReturnType = void> using EnableIf    = Invoke<std::enable_if<Condition::value, ReturnType> >;
+template <typename Condition, typename ReturnType = void> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
 
 ////////////////////////////////////////////////////////
 // Check for complexity with type traits

From 4705aa541d62e16b070452e4a3f329d9f9565afa Mon Sep 17 00:00:00 2001
From: Christoph Lehner <christoph@lhnr.de>
Date: Thu, 4 Feb 2021 14:25:55 +0100
Subject: [PATCH 31/82] Allow user to configure ShmDims via environment
 variables

---
 Grid/communicator/SharedMemoryMPI.cc | 18 ++++++++++++++++++
 Grid/util/Init.cc                    |  2 +-
 Grid/util/Init.h                     |  2 +-
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc
index a12418e6..466f6a1e 100644
--- a/Grid/communicator/SharedMemoryMPI.cc
+++ b/Grid/communicator/SharedMemoryMPI.cc
@@ -7,6 +7,7 @@
     Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Christoph Lehner <christoph@lhnr.de>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -169,6 +170,23 @@ static inline int divides(int a,int b)
 }
 void GlobalSharedMemory::GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims)
 {
+  ////////////////////////////////////////////////////////////////
+  // Allow user to configure through environment variable
+  ////////////////////////////////////////////////////////////////
+  char* str = getenv(("GRID_SHM_DIMS_" + std::to_string(ShmDims.size())).c_str());
+  if ( str ) {
+    std::vector<int> IntShmDims;
+    GridCmdOptionIntVector(std::string(str),IntShmDims);
+    assert(IntShmDims.size() == WorldDims.size());
+    long ShmSize = 1;
+    for (int dim=0;dim<WorldDims.size();dim++) {
+      ShmSize *= (ShmDims[dim] = IntShmDims[dim]);
+      assert(divides(ShmDims[dim],WorldDims[dim]));
+    }
+    assert(ShmSize == WorldShmSize);
+    return;
+  }
+  
   ////////////////////////////////////////////////////////////////
   // Powers of 2,3,5 only in prime decomposition for now
   ////////////////////////////////////////////////////////////////
diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc
index 9be39e94..55d8c5bf 100644
--- a/Grid/util/Init.cc
+++ b/Grid/util/Init.cc
@@ -140,7 +140,7 @@ void GridCmdOptionCSL(std::string str,std::vector<std::string> & vec)
 }
 
 template<class VectorInt>
-void GridCmdOptionIntVector(std::string &str,VectorInt & vec)
+void GridCmdOptionIntVector(const std::string &str,VectorInt & vec)
 {
   vec.resize(0);
   std::stringstream ss(str);
diff --git a/Grid/util/Init.h b/Grid/util/Init.h
index dad963a0..4eb8f06c 100644
--- a/Grid/util/Init.h
+++ b/Grid/util/Init.h
@@ -55,7 +55,7 @@ template<class VectorInt>
 std::string GridCmdVectorIntToString(const VectorInt & vec);
 void GridCmdOptionCSL(std::string str,std::vector<std::string> & vec);
 template<class VectorInt>
-void GridCmdOptionIntVector(std::string &str,VectorInt & vec);
+void GridCmdOptionIntVector(const std::string &str,VectorInt & vec);
 void GridCmdOptionInt(std::string &str,int & val);
 
 

From cd99edcc5f0e3b12106c07652f073eeb5be985c7 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 4 Feb 2021 18:25:49 -0500
Subject: [PATCH 32/82] maxLocalNorm2()

---
 Grid/communicator/Communicator_base.h  |  3 +-
 Grid/communicator/Communicator_mpi3.cc | 10 +++++
 Grid/communicator/Communicator_none.cc |  2 +
 Grid/lattice/Lattice_reduction.h       | 53 +++++++++++++++++++++++++-
 tests/core/Test_main.cc                | 16 +++++++-
 5 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/Grid/communicator/Communicator_base.h b/Grid/communicator/Communicator_base.h
index bb06d43f..a15f9789 100644
--- a/Grid/communicator/Communicator_base.h
+++ b/Grid/communicator/Communicator_base.h
@@ -1,4 +1,3 @@
-
 /*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
@@ -108,6 +107,8 @@ public:
   ////////////////////////////////////////////////////////////
   // Reduction
   ////////////////////////////////////////////////////////////
+  void GlobalMax(RealD &);
+  void GlobalMax(RealF &);
   void GlobalSum(RealF &);
   void GlobalSumVector(RealF *,int N);
   void GlobalSum(RealD &);
diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc
index c6543851..5713fe35 100644
--- a/Grid/communicator/Communicator_mpi3.cc
+++ b/Grid/communicator/Communicator_mpi3.cc
@@ -275,6 +275,16 @@ void CartesianCommunicator::GlobalXOR(uint64_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
   assert(ierr==0);
 }
+void CartesianCommunicator::GlobalMax(float &f)
+{
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_MAX,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalMax(double &d)
+{
+  int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_MAX,communicator);
+  assert(ierr==0);
+}
 void CartesianCommunicator::GlobalSum(float &f){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
   assert(ierr==0);
diff --git a/Grid/communicator/Communicator_none.cc b/Grid/communicator/Communicator_none.cc
index 6cb431a2..beb2cc97 100644
--- a/Grid/communicator/Communicator_none.cc
+++ b/Grid/communicator/Communicator_none.cc
@@ -67,6 +67,8 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
 
 CartesianCommunicator::~CartesianCommunicator(){}
 
+void CartesianCommunicator::GlobalMax(float &){}
+void CartesianCommunicator::GlobalMax(double &){}
 void CartesianCommunicator::GlobalSum(float &){}
 void CartesianCommunicator::GlobalSumVector(float *,int N){}
 void CartesianCommunicator::GlobalSum(double &){}
diff --git a/Grid/lattice/Lattice_reduction.h b/Grid/lattice/Lattice_reduction.h
index c2955485..7338fd41 100644
--- a/Grid/lattice/Lattice_reduction.h
+++ b/Grid/lattice/Lattice_reduction.h
@@ -96,8 +96,34 @@ inline typename vobj::scalar_objectD sumD_cpu(const vobj *arg, Integer osites)
   ssobj ret = ssum;
   return ret;
 }
+/*
+Threaded max, don't use for now
+template<class Double>
+inline Double max(const Double *arg, Integer osites)
+{
+  //  const int Nsimd = vobj::Nsimd();
+  const int nthread = GridThread::GetThreads();
 
-
+  std::vector<Double> maxarray(nthread);
+  
+  thread_for(thr,nthread, {
+    int nwork, mywork, myoff;
+    nwork = osites;
+    GridThread::GetWork(nwork,thr,mywork,myoff);
+    Double max=arg[0];
+    for(int ss=myoff;ss<mywork+myoff; ss++){
+      if( arg[ss] > max ) max = arg[ss];
+    }
+    maxarray[thr]=max;
+  });
+  
+  Double tmax=maxarray[0];
+  for(int i=0;i<nthread;i++){
+    if (maxarray[i]>tmax) tmax = maxarray[i];
+  } 
+  return tmax;
+}
+*/
 template<class vobj>
 inline typename vobj::scalar_object sum(const vobj *arg, Integer osites)
 {
@@ -140,6 +166,31 @@ template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
   ComplexD nrm = innerProduct(arg,arg);
   return real(nrm); 
 }
+template<class vobj> inline RealD maxLocalNorm2(const Lattice<vobj> &arg)
+{
+  typedef typename vobj::tensor_reduced vscalar;
+  typedef typename vobj::scalar_object  scalar;
+  typedef typename getPrecision<vobj>::real_scalar_type rscalar;
+
+  Lattice<vscalar> inner = localNorm2(arg);
+
+  auto grid = arg.Grid();
+
+  RealD max;
+  for(int l=0;l<grid->lSites();l++){
+    Coordinate coor;
+    scalar val;
+    RealD r;
+    grid->LocalIndexToLocalCoor(l,coor);
+    peekLocalSite(val,inner,coor);
+    r=real(TensorRemove(val));
+    if( (l==0) || (r>max)){
+      max=r;
+    }
+  }
+  grid->GlobalMax(max);
+  return max;
+}
 
 // Double inner product
 template<class vobj>
diff --git a/tests/core/Test_main.cc b/tests/core/Test_main.cc
index d7ed04ba..d3e6bfbd 100644
--- a/tests/core/Test_main.cc
+++ b/tests/core/Test_main.cc
@@ -231,6 +231,19 @@ int main(int argc, char **argv) {
       scalar = localInnerProduct(cVec, cVec);
       scalar = localNorm2(cVec);
 
+      std::cout << "Testing maxLocalNorm2" <<std::endl;
+      for(Integer gsite=0;gsite<Fine.gSites();gsite++){
+
+	TComplex big(10.0);
+	Coordinate coor;
+
+	random(FineRNG, scalar);
+	Fine.GlobalIndexToGlobalCoor(gsite,coor);
+        pokeSite(big,scalar,coor);
+	
+	RealD Linfty = maxLocalNorm2(scalar);
+	assert(Linfty == 100.0);
+      }
       //     -=,+=,*=,()
       //     add,+,sub,-,mult,mac,*
       //     adj,conjugate
@@ -549,7 +562,8 @@ int main(int argc, char **argv) {
 
                   std::vector<int> shiftcoor = coor;
                   shiftcoor[dir] = (shiftcoor[dir] + shift + latt_size[dir]) %
-                                   (latt_size[dir] / mpi_layout[dir]);
+                                   (latt_size[dir]);
+		  //                                   (latt_size[dir] / mpi_layout[dir]);
 
                   std::vector<int> rl(4);
                   for (int dd = 0; dd < 4; dd++) {

From eda9ab487babbf2409fef80fd69f3b60ed532480 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Mon, 8 Feb 2021 10:47:22 -0500
Subject: [PATCH 33/82] MADWF 5d source option for hadrons - look at Grid of
 source Abort on GPU error

---
 Grid/qcd/action/fermion/MADWF.h | 14 +++++++++++---
 Grid/threads/Accelerator.cc     |  1 +
 Grid/threads/Accelerator.h      |  3 +++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/Grid/qcd/action/fermion/MADWF.h b/Grid/qcd/action/fermion/MADWF.h
index 6b3c6e71..5d17e865 100644
--- a/Grid/qcd/action/fermion/MADWF.h
+++ b/Grid/qcd/action/fermion/MADWF.h
@@ -85,7 +85,7 @@ class MADWF
       maxiter     =_maxiter;
     };
    
-  void operator() (const FermionFieldo &src4,FermionFieldo &sol5)
+  void operator() (const FermionFieldo &src,FermionFieldo &sol5)
   {
     std::cout << GridLogMessage<< " ************************************************" << std::endl;
     std::cout << GridLogMessage<< "  MADWF-like algorithm                           " << std::endl;
@@ -114,8 +114,16 @@ class MADWF
     ///////////////////////////////////////
     //Import source, include Dminus factors
     ///////////////////////////////////////
-    Mato.ImportPhysicalFermionSource(src4,b); 
-    std::cout << GridLogMessage << " src4 " <<norm2(src4)<<std::endl;
+    GridBase *src_grid = src.Grid();
+
+    assert( (src_grid == Mato.GaugeGrid()) || (src_grid == Mato.FermionGrid()));
+
+    if ( src_grid == Mato.GaugeGrid() ) {
+      Mato.ImportPhysicalFermionSource(src,b);
+    } else {
+      b=src;
+    }
+    std::cout << GridLogMessage << " src " <<norm2(src)<<std::endl;
     std::cout << GridLogMessage << " b    " <<norm2(b)<<std::endl;
 
     defect = b;
diff --git a/Grid/threads/Accelerator.cc b/Grid/threads/Accelerator.cc
index 6fc1afe2..4bf7f395 100644
--- a/Grid/threads/Accelerator.cc
+++ b/Grid/threads/Accelerator.cc
@@ -1,6 +1,7 @@
 #include <Grid/GridCore.h>
 
 NAMESPACE_BEGIN(Grid);
+int      acceleratorAbortOnGpuError=1;
 uint32_t accelerator_threads=2;
 uint32_t acceleratorThreads(void)       {return accelerator_threads;};
 void     acceleratorThreads(uint32_t t) {accelerator_threads = t;};
diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h
index 6232aea8..59645546 100644
--- a/Grid/threads/Accelerator.h
+++ b/Grid/threads/Accelerator.h
@@ -100,6 +100,8 @@ void     acceleratorInit(void);
 #define accelerator        __host__ __device__
 #define accelerator_inline __host__ __device__ inline
 
+extern int acceleratorAbortOnGpuError;
+
 accelerator_inline int acceleratorSIMTlane(int Nsimd) {
 #ifdef GRID_SIMT
   return threadIdx.z; 
@@ -140,6 +142,7 @@ void LambdaApply(uint64_t num1, uint64_t num2, uint64_t num3, lambda Lambda)
       printf("Cuda error %s \n", cudaGetErrorString( err ));		\
       puts(__FILE__);							\
       printf("Line %d\n",__LINE__);					\
+      if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);		\
     }									\
   }
 

From 55de69a56953f12380e3a276262ef0f547b5e28c Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@bnl.gov>
Date: Mon, 8 Feb 2021 12:03:16 -0500
Subject: [PATCH 34/82] Fixed compile issues with maxLocalNorm2 for non-scalar
 lattices maxLocalNorm2 test now reuses the random field

---
 Grid/lattice/Lattice_reduction.h | 7 ++++---
 tests/core/Test_main.cc          | 9 +++++----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/Grid/lattice/Lattice_reduction.h b/Grid/lattice/Lattice_reduction.h
index 7338fd41..0a5fbcb6 100644
--- a/Grid/lattice/Lattice_reduction.h
+++ b/Grid/lattice/Lattice_reduction.h
@@ -166,11 +166,12 @@ template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
   ComplexD nrm = innerProduct(arg,arg);
   return real(nrm); 
 }
+
+//The global maximum of the site norm2
 template<class vobj> inline RealD maxLocalNorm2(const Lattice<vobj> &arg)
 {
-  typedef typename vobj::tensor_reduced vscalar;
-  typedef typename vobj::scalar_object  scalar;
-  typedef typename getPrecision<vobj>::real_scalar_type rscalar;
+  typedef typename vobj::tensor_reduced vscalar;  //iScalar<iScalar<.... <vPODtype> > >
+  typedef typename vscalar::scalar_object  scalar;   //iScalar<iScalar<.... <PODtype> > >
 
   Lattice<vscalar> inner = localNorm2(arg);
 
diff --git a/tests/core/Test_main.cc b/tests/core/Test_main.cc
index d3e6bfbd..6e316aa6 100644
--- a/tests/core/Test_main.cc
+++ b/tests/core/Test_main.cc
@@ -232,12 +232,13 @@ int main(int argc, char **argv) {
       scalar = localNorm2(cVec);
 
       std::cout << "Testing maxLocalNorm2" <<std::endl;
-      for(Integer gsite=0;gsite<Fine.gSites();gsite++){
-
+      
+      LatticeComplex rand_scalar(&Fine);
+      random(FineRNG, rand_scalar);  //uniform [0,1]
+      for(Integer gsite=0;gsite<Fine.gSites();gsite++){ //check on every site independently
+	scalar = rand_scalar;
 	TComplex big(10.0);
 	Coordinate coor;
-
-	random(FineRNG, scalar);
 	Fine.GlobalIndexToGlobalCoor(gsite,coor);
         pokeSite(big,scalar,coor);
 	

From a5033329247f6410928965ce564b2989c7bac908 Mon Sep 17 00:00:00 2001
From: Michael Marshall <43034299+mmphys@users.noreply.github.com>
Date: Sun, 14 Feb 2021 21:27:54 +0000
Subject: [PATCH 35/82] Seems the intention with AutoConf produced
 Grid/Config.h was to use sed to translate standard PACKAGE_ #defines into
 GRID_ however due to missing '' after -i this hasn't been working. Perhaps it
 is too late to fix this, since we don't know who/what is relying on this
 downstream? ... but if they are, and AutoConf is being used, then likely
 these #defines have just been redefined anyway. Seems reasonable to redefine
 PACKAGE and VERSION as well, as none of these macros are used throughout Grid
 or Hadrons.

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index f077ca93..7d95e4e2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -7,7 +7,7 @@ AM_INIT_AUTOMAKE([subdir-objects 1.13])
 AM_EXTRA_RECURSIVE_TARGETS([tests bench])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([Grid/Grid.h])
-AC_CONFIG_HEADERS([Grid/Config.h],[sed -i 's|PACKAGE_|GRID_|' Grid/Config.h])
+AC_CONFIG_HEADERS([Grid/Config.h],[[sed -i '' -e 's|PACKAGE_|GRID_|' -e 's|[[:space:]]PACKAGE[[:space:]]| GRID_PACKAGE |' -e 's|[[:space:]]VERSION[[:space:]]| GRID_PACKAGE_VERSION |' Grid/Config.h]])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 ################ Get git info

From 35114c9e629c53546c8e95edd6def7b3e1692c7a Mon Sep 17 00:00:00 2001
From: Michael Marshall <43034299+mmphys@users.noreply.github.com>
Date: Wed, 17 Feb 2021 13:24:15 +0000
Subject: [PATCH 36/82] Mac OS (Darwin) sed -i flag for in-place editing
 differs from posix / gnu

---
 configure.ac | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 7d95e4e2..702ce826 100644
--- a/configure.ac
+++ b/configure.ac
@@ -7,7 +7,12 @@ AM_INIT_AUTOMAKE([subdir-objects 1.13])
 AM_EXTRA_RECURSIVE_TARGETS([tests bench])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([Grid/Grid.h])
-AC_CONFIG_HEADERS([Grid/Config.h],[[sed -i '' -e 's|PACKAGE_|GRID_|' -e 's|[[:space:]]PACKAGE[[:space:]]| GRID_PACKAGE |' -e 's|[[:space:]]VERSION[[:space:]]| GRID_PACKAGE_VERSION |' Grid/Config.h]])
+AC_CONFIG_HEADERS([Grid/Config.h],[[$SED_INPLACE -e 's|PACKAGE_|GRID_|' -e 's|[[:space:]]PACKAGE[[:space:]]| GRID_PACKAGE |' -e 's|[[:space:]]VERSION[[:space:]]| GRID_PACKAGE_VERSION |' Grid/Config.h]],
+    [if test x"$host_os" == x"${host_os#darwin}" ; then]
+        [SED_INPLACE="sed -i"]
+    [else]
+        [SED_INPLACE="sed -i .bak"]
+    [fi])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 ################ Get git info

From 86b58d5aff2adac0b41b4fc7be22e77dfbd583e2 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Thu, 18 Feb 2021 12:04:32 +0000
Subject: [PATCH 37/82] changed if and accelerator_for - no runtime errors any
 more

---
 Grid/qcd/utils/BaryonUtils.h | 129 ++++++++++++++++++-----------------
 1 file changed, 68 insertions(+), 61 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 56c5781d..4ac5f685 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -513,19 +513,18 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
 
   GridBase *grid = q1_left.Grid();
   
-  autoView( vbaryon_corr , baryon_corr , AcceleratorWrite);
-  autoView( vcorr_read   , baryon_corr , AcceleratorRead);
-  autoView( v1           , q1_left     , AcceleratorRead);
-  autoView( v2           , q2_left     , AcceleratorRead);
-  autoView( v3           , q3_left     , AcceleratorRead);
+  autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( v1          , q1_left     , AcceleratorRead);
+  autoView( v2          , q2_left     , AcceleratorRead);
+  autoView( v3          , q3_left     , AcceleratorRead);
 
   Real bytes =0.;
   bytes += grid->oSites() * (432.*sizeof(vComplex) + 126.*sizeof(int) + 36.*sizeof(Real));
   for (int ie=0; ie < 6 ; ie++){
     if(ie==0 or ie==3){
-       //bytes += grid->oSites() * (4.*sizeof(int) + 4752.*sizeof(vComplex)) * wick_contractions[ie];
+       bytes += ( wick_contractions & (1 << ie) ) ? grid->oSites() * (4.*sizeof(int) + 4752.*sizeof(vComplex)) : 0.;
     } else{
-       //bytes += grid->oSites() * (64.*sizeof(int) + 5184.*sizeof(vComplex)) * wick_contractions[ie];
+       bytes += ( wick_contractions & (1 << ie) ) ? grid->oSites() * (64.*sizeof(int) + 5184.*sizeof(vComplex)) : 0.;
     }
   }
   Real t=0.;
@@ -535,8 +534,7 @@ void BaryonUtils<FImpl>::ContractBaryons(const PropagatorField &q1_left,
     auto D1 = v1(ss);
     auto D2 = v2(ss);
     auto D3 = v3(ss);
-    //typedef decltype(coalescedRead(vbaryon_corr[0])) cVec;
-    typedef decltype(coalescedRead(vcorr_read[0])) cVec;
+    typedef decltype(coalescedRead(vbaryon_corr[0])) cVec;
     cVec result=Zero();
     BaryonSite(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,parity,wick_contractions,result);
     coalescedWrite(vbaryon_corr[ss],result);
@@ -561,21 +559,19 @@ void BaryonUtils<FImpl>::ContractBaryonsMatrix(const PropagatorField &q1_left,
 
   assert(Ns==4 && "Baryon code only implemented for N_spin = 4");
   assert(Nc==3 && "Baryon code only implemented for N_colour = 3");
- 
+
   GridBase *grid = q1_left.Grid();
 
-  autoView( vbaryon_corr , baryon_corr , AcceleratorWrite);
-  autoView( vcorr_read   , baryon_corr , AcceleratorRead);
-  autoView( v1           , q1_left     , AcceleratorRead);
-  autoView( v2           , q2_left     , AcceleratorRead);
-  autoView( v3           , q3_left     , AcceleratorRead);
+  autoView(vbaryon_corr , baryon_corr , AcceleratorWrite);
+  autoView( v1          , q1_left     , AcceleratorRead);
+  autoView( v2          , q2_left     , AcceleratorRead);
+  autoView( v3          , q3_left     , AcceleratorRead);
 
   accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
     auto D1 = v1(ss);
     auto D2 = v2(ss);
     auto D3 = v3(ss);
-    //typedef decltype(coalescedRead(vbaryon_corr[0])) spinor;
-    typedef decltype(coalescedRead(vcorr_read[0])) spinor;
+    typedef decltype(coalescedRead(vbaryon_corr[0])) spinor;
     spinor result=Zero();
     BaryonSiteMatrix(D1,D2,D3,GammaA_left,GammaB_left,GammaA_right,GammaB_right,wick_contractions,result);
     coalescedWrite(vbaryon_corr[ss],result);
@@ -941,10 +937,9 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
 
   GridBase *grid = q_tf.Grid();
 
-  autoView( vcorr      , stn_corr , AcceleratorWrite);
-  autoView( vcorr_read , stn_corr , AcceleratorRead);
-  autoView( vq_ti      , q_ti     , AcceleratorRead);
-  autoView( vq_tf      , q_tf     , AcceleratorRead);
+  autoView( vcorr , stn_corr , AcceleratorWrite);
+  autoView( vq_ti , q_ti     , AcceleratorRead);
+  autoView( vq_tf , q_tf     , AcceleratorRead);
 
   Vector<mobj> my_Dq_spec{Dq_spec1,Dq_spec2};
   mobj * Dq_spec_p = &my_Dq_spec[0];
@@ -953,29 +948,28 @@ void BaryonUtils<FImpl>::BaryonGamma3pt(
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
       auto Dq_ti = vq_ti(ss);
       auto Dq_tf = vq_tf(ss);
-      typedef decltype(coalescedRead(vcorr_read[0])) spinor;
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup1Site(Dq_ti,Dq_spec_p[0],Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result);
-      coalescedWrite(vcorr[ss],coalescedRead(vcorr_read[ss])+result); 
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result); 
     });//end loop over lattice sites
-
   } else if (group == 2) {
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
       auto Dq_ti = vq_ti(ss);
       auto Dq_tf = vq_tf(ss);
-      typedef decltype(coalescedRead(vcorr_read[0])) spinor;
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup2Site(Dq_spec_p[0],Dq_ti,Dq_spec_p[1],Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],coalescedRead(vcorr_read[ss])+result); 
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result); 
     });//end loop over lattice sites
   } else if (group == 3) {
     accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
       auto Dq_ti = vq_ti(ss);
       auto Dq_tf = vq_tf(ss);
-      typedef decltype(coalescedRead(vcorr_read[0])) spinor;
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
       spinor result=Zero();
       BaryonGamma3ptGroup3Site(Dq_spec_p[0],Dq_spec_p[1],Dq_ti,Dq_tf,GammaJ,GammaBi,GammaBf,wick_contraction,result); 
-      coalescedWrite(vcorr[ss],coalescedRead(vcorr_read[ss])+result); 
+      coalescedWrite(vcorr[ss],coalescedRead(vcorr[ss])+result); 
     });//end loop over lattice sites
   }
 
@@ -1206,6 +1200,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
 
   Real ee;
 
+
   for (int ie_n=0; ie_n < 6 ; ie_n++){
     int a_n    = (ie_n < 3 ? ie_n       : (6-ie_n)%3 ); //epsilon[ie_n][0]; //a
     int b_n    = (ie_n < 3 ? (ie_n+1)%3 : (8-ie_n)%3 ); //epsilon[ie_n][1]; //b
@@ -1250,6 +1245,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
       }}
     }
   }
+
 }
 
 template<class FImpl>
@@ -1275,27 +1271,32 @@ void BaryonUtils<FImpl>::SigmaToNucleonEye(const PropagatorField &qq_loop,
   autoView( vd_tf   , qd_tf    , AcceleratorRead);
   autoView( vs_ti   , qs_ti    , AcceleratorRead);
 
-  bool doQ1 = (op == "Q1");
-  bool doQ2 = (op == "Q2");
-  
   Vector<mobj> my_Dq_spec{Du_spec};
   mobj * Dq_spec_p = &my_Dq_spec[0];
 
-  accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto Dq_loop = vq_loop(ss);
-    auto Dd_tf = vd_tf(ss);
-    auto Ds_ti = vs_ti(ss);
-    typedef decltype(coalescedRead(vcorr[0])) spinor;
-    spinor result=Zero();
-    if(doQ1){
+  if(op == "Q1"){
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_loop = vq_loop(ss);
+      auto Dd_tf = vd_tf(ss);
+      auto Ds_ti = vs_ti(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
       SigmaToNucleonQ1EyeSite(Dq_loop,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
-    } else if(doQ2){
+      coalescedWrite(vcorr[ss],result);
+    });//end loop over lattice sites
+  } else if(op == "Q2"){
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_loop = vq_loop(ss);
+      auto Dd_tf = vd_tf(ss);
+      auto Ds_ti = vs_ti(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
       SigmaToNucleonQ2EyeSite(Dq_loop,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
-    } else {
-      assert(0 && "Weak Operator not correctly specified");
-    }
-    coalescedWrite(vcorr[ss],result);
-  });//end loop over lattice sites
+      coalescedWrite(vcorr[ss],result);
+    });//end loop over lattice sites
+  } else {
+    assert(0 && "Weak Operator not correctly specified");
+  }
 }
 
 template<class FImpl>
@@ -1322,29 +1323,35 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
   autoView( vq_tf , qq_tf    , AcceleratorRead  );
   autoView( vd_tf , qd_tf    , AcceleratorRead  );
   autoView( vs_ti , qs_ti    , AcceleratorRead  );
-
-  bool doQ1 = (op == "Q1");
-  bool doQ2 = (op == "Q2");
   
   Vector<mobj> my_Dq_spec{Du_spec};
   mobj * Dq_spec_p = &my_Dq_spec[0];
 
-  accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto Dq_ti = vq_ti(ss);
-    auto Dq_tf = vq_tf(ss);
-    auto Dd_tf = vd_tf(ss);
-    auto Ds_ti = vs_ti(ss);
-    typedef decltype(coalescedRead(vcorr[0])) spinor;
-    spinor result=Zero();
-    if(doQ1){
+  if(op == "Q1"){
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_ti = vq_ti(ss);
+      auto Dq_tf = vq_tf(ss);
+      auto Dd_tf = vd_tf(ss);
+      auto Ds_ti = vs_ti(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
       SigmaToNucleonQ1NonEyeSite(Dq_ti,Dq_tf,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
-    } else if(doQ2){
+      coalescedWrite(vcorr[ss],result);
+    });//end loop over lattice sites
+  } else if(op == "Q2"){
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_ti = vq_ti(ss);
+      auto Dq_tf = vq_tf(ss);
+      auto Dd_tf = vd_tf(ss);
+      auto Ds_ti = vs_ti(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
       SigmaToNucleonQ2NonEyeSite(Dq_ti,Dq_tf,Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_sigma,GammaB_nucl,result);
-    } else {
-      assert(0 && "Weak Operator not correctly specified");
-    }
-    coalescedWrite(vcorr[ss],result);
-  });//end loop over lattice sites
+      coalescedWrite(vcorr[ss],result);
+    });//end loop over lattice sites
+  } else {
+    assert(0 && "Weak Operator not correctly specified");
+  }
 }
 
 NAMESPACE_END(Grid);

From 7ae030f5851e8a9d74bedb79a28ef598b4006c26 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Thu, 18 Feb 2021 13:24:50 +0000
Subject: [PATCH 38/82] changed back A2AUtils warning

---
 Grid/qcd/utils/A2Autils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/qcd/utils/A2Autils.h b/Grid/qcd/utils/A2Autils.h
index 497927dd..b63d8571 100644
--- a/Grid/qcd/utils/A2Autils.h
+++ b/Grid/qcd/utils/A2Autils.h
@@ -1047,7 +1047,7 @@ A2Autils<FImpl>::ContractWWVV(std::vector<PropagatorField> &WWVV,
 {
   GridBase *grid = vs[0].Grid();
 
-  //int nd    = grid->_ndimension;
+  int nd    = grid->_ndimension;
   int Nsimd = grid->Nsimd();
   int N_t = WW_sd.dimensions()[0];
   int N_s = WW_sd.dimensions()[1];

From e3d019bc2f46c8c338bc3306a1954180a8b94dd7 Mon Sep 17 00:00:00 2001
From: Daniel Richtmann <daniel.richtmann@gmail.com>
Date: Mon, 22 Feb 2021 14:56:52 +0100
Subject: [PATCH 39/82] Enable performance counting in WilsonFermion like in
 others

---
 .../fermion/implementation/WilsonFermionImplementation.h       | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h
index 4977ea68..84ac25c1 100644
--- a/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h
@@ -397,6 +397,7 @@ void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U, co
 template <class Impl>
 void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int dag)
 {
+  DhopCalls+=2;
   conformable(in.Grid(), _grid);  // verifies full grid
   conformable(in.Grid(), out.Grid());
 
@@ -408,6 +409,7 @@ void WilsonFermion<Impl>::Dhop(const FermionField &in, FermionField &out, int da
 template <class Impl>
 void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int dag)
 {
+  DhopCalls++;
   conformable(in.Grid(), _cbgrid);    // verifies half grid
   conformable(in.Grid(), out.Grid());  // drops the cb check
 
@@ -420,6 +422,7 @@ void WilsonFermion<Impl>::DhopOE(const FermionField &in, FermionField &out, int
 template <class Impl>
 void WilsonFermion<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls++;
   conformable(in.Grid(), _cbgrid);    // verifies half grid
   conformable(in.Grid(), out.Grid());  // drops the cb check
 

From c073e62e0be2f2e421588ad76f90dce33e7154c0 Mon Sep 17 00:00:00 2001
From: Daniel Richtmann <daniel.richtmann@gmail.com>
Date: Mon, 22 Feb 2021 15:17:07 +0100
Subject: [PATCH 40/82] Correct misleading ac help string

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 702ce826..fb0c78fc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -130,7 +130,7 @@ esac
 
 ############### fermions
 AC_ARG_ENABLE([fermion-reps],
-     [AC_HELP_STRING([--fermion-reps=yes|no], [enable extra fermion representation support])],
+     [AC_HELP_STRING([--enable-fermion-reps=yes|no], [enable extra fermion representation support])],
      [ac_FERMION_REPS=${enable_fermion_reps}], [ac_FERMION_REPS=yes])
 
 AM_CONDITIONAL(BUILD_FERMION_REPS, [ test "${ac_FERMION_REPS}X" == "yesX" ])

From d5ab571a894682e36bc24488958f01a18983c7f7 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@bnl.gov>
Date: Tue, 23 Feb 2021 11:49:56 -0500
Subject: [PATCH 41/82] Added the ability to apply a custom "filter" to the
 conjugate momentum in the Integrator classes, applied both after refresh and
 after applying the forces Added a conjugate momentum "filter" that applies a
 phase to each site. With sites set to 1.0 or 0.0 this acts as a mask and
 enables, for example, the freezing of inactive gauge links in DDHMC Added
 tests/forces/Test_momentum_filter demonstrating the use of the filter to
 freeze boundary links

---
 Grid/qcd/hmc/integrators/Integrator.h     |  28 ++++
 Grid/qcd/hmc/integrators/MomentumFilter.h |  94 +++++++++++++
 tests/forces/Test_momentum_filter.cc      | 154 ++++++++++++++++++++++
 3 files changed, 276 insertions(+)
 create mode 100644 Grid/qcd/hmc/integrators/MomentumFilter.h
 create mode 100644 tests/forces/Test_momentum_filter.cc

diff --git a/Grid/qcd/hmc/integrators/Integrator.h b/Grid/qcd/hmc/integrators/Integrator.h
index 70055754..77b7de52 100644
--- a/Grid/qcd/hmc/integrators/Integrator.h
+++ b/Grid/qcd/hmc/integrators/Integrator.h
@@ -33,6 +33,7 @@ directory
 #define INTEGRATOR_INCLUDED
 
 #include <memory>
+#include "MomentumFilter.h"
 
 NAMESPACE_BEGIN(Grid);
 
@@ -78,8 +79,19 @@ protected:
   RepresentationPolicy Representations;
   IntegratorParameters Params;
 
+  //Filters allow the user to manipulate the conjugate momentum, for example to freeze links in DDHMC
+  //It is applied whenever the momentum is updated / refreshed
+  //The default filter does nothing
+  MomentumFilterBase<MomentaField> const* MomFilter;
+
   const ActionSet<Field, RepresentationPolicy> as;
 
+  //Get a pointer to a shared static instance of the "do-nothing" momentum filter to serve as a default
+  static MomentumFilterBase<MomentaField> const* getDefaultMomFilter(){ 
+    static MomentumFilterNone<MomentaField> filter;
+    return &filter;
+  }
+
   void update_P(Field& U, int level, double ep) 
   {
     t_P[level] += ep;
@@ -135,6 +147,8 @@ protected:
 
     // Force from the other representations
     as[level].apply(update_P_hireps, Representations, Mom, U, ep);
+
+    MomFilter->applyFilter(Mom);
   }
 
   void update_U(Field& U, double ep) 
@@ -174,11 +188,23 @@ public:
     t_P.resize(levels, 0.0);
     t_U = 0.0;
     // initialization of smearer delegated outside of Integrator
+
+    //Default the momentum filter to "do-nothing"
+    MomFilter = getDefaultMomFilter();
   };
 
   virtual ~Integrator() {}
 
   virtual std::string integrator_name() = 0;
+  
+  //Set the momentum filter allowing for manipulation of the conjugate momentum
+  void setMomentumFilter(const MomentumFilterBase<MomentaField> &filter){
+    MomFilter = &filter;
+  }
+
+  //Access the conjugate momentum
+  const MomentaField & getMomentum() const{ return P; }
+  
 
   void print_parameters()
   {
@@ -249,6 +275,8 @@ public:
       // Refresh the higher representation actions
       as[level].apply(refresh_hireps, Representations, pRNG);
     }
+
+    MomFilter->applyFilter(P);
   }
 
   // to be used by the actionlevel class to iterate
diff --git a/Grid/qcd/hmc/integrators/MomentumFilter.h b/Grid/qcd/hmc/integrators/MomentumFilter.h
new file mode 100644
index 00000000..2a15d80c
--- /dev/null
+++ b/Grid/qcd/hmc/integrators/MomentumFilter.h
@@ -0,0 +1,94 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/integrators/MomentumFilter.h
+
+Copyright (C) 2015
+
+Author: Christopher Kelly <ckelly@bnl.gov>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+//--------------------------------------------------------------------
+#ifndef MOMENTUM_FILTER
+#define MOMENTUM_FILTER
+
+NAMESPACE_BEGIN(Grid);
+
+//These filter objects allow the user to manipulate the conjugate momentum as part of the update / refresh
+
+template<typename MomentaField>
+struct MomentumFilterBase{
+  virtual void applyFilter(MomentaField &P) const;
+};
+
+//Do nothing
+template<typename MomentaField>
+struct MomentumFilterNone: public MomentumFilterBase<MomentaField>{
+  void applyFilter(MomentaField &P) const override{}
+};
+
+//Multiply each site/direction by a Lorentz vector complex number field
+//Can be used to implement a mask, zeroing out sites
+template<typename MomentaField>
+struct MomentumFilterApplyPhase: public MomentumFilterBase<MomentaField>{
+  typedef typename MomentaField::vector_type vector_type; //SIMD-vectorized complex type
+  typedef typename MomentaField::scalar_type scalar_type; //scalar complex type
+  typedef iVector<iScalar<iScalar<vector_type> >, Nd > LorentzScalarType; //complex phase for each site/direction
+  typedef Lattice<LorentzScalarType> LatticeLorentzScalarType;
+  
+  LatticeLorentzScalarType phase;
+ 
+  MomentumFilterApplyPhase(const LatticeLorentzScalarType _phase): phase(_phase){}
+
+  //Default to uniform field of (1,0)
+  MomentumFilterApplyPhase(GridBase* _grid): phase(_grid){
+    LorentzScalarType one;
+    for(int mu=0;mu<Nd;mu++)
+      one(mu)()() = scalar_type(1.);
+    
+    phase = one;
+  }
+
+  void applyFilter(MomentaField &P) const override{
+    conformable(P,phase);
+    autoView( P_v , P, AcceleratorWrite);
+    autoView( phase_v , phase, AcceleratorRead);
+
+    accelerator_for(ss,P_v.size(),MomentaField::vector_type::Nsimd(),{
+    	auto site_mom = P_v(ss);
+    	auto site_phase = phase_v(ss);
+	for(int mu=0;mu<Nd;mu++)
+	  site_mom(mu) = site_mom(mu) * site_phase(mu);
+    	coalescedWrite(P_v[ss], site_mom);
+      });
+    
+  }
+
+
+};
+
+
+
+
+NAMESPACE_END(Grid);
+
+#endif
diff --git a/tests/forces/Test_momentum_filter.cc b/tests/forces/Test_momentum_filter.cc
new file mode 100644
index 00000000..856ea0f2
--- /dev/null
+++ b/tests/forces/Test_momentum_filter.cc
@@ -0,0 +1,154 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_wilson_force.cc
+
+    Copyright (C) 2015
+
+Author: Christopher Kelly <ckelly@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+
+//Get the mu-directected links on the upper boundary and the bulk remainder
+template<typename Field>
+void getLinksBoundaryBulk(Field &bound, Field &bulk,  Field &from, const Coordinate &latt_size){
+  bound = Zero(); bulk = Zero();
+  for(int mu=0;mu<Nd;mu++){
+    LatticeInteger mucoor(bound.Grid());
+    LatticeCoordinate(mucoor, mu);
+
+    bound = where( mucoor == (Integer)(latt_size[mu] - 1), from, bound );
+    bulk = where( mucoor != (Integer)(latt_size[mu] - 1), from, bulk );
+  }
+}
+
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  Coordinate latt_size   = GridDefaultLatt();
+  Coordinate simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  Coordinate mpi_layout  = GridDefaultMpi();
+
+  GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  std::vector<int> seeds({1,2,3,4});
+
+  GridParallelRNG          pRNG(&Grid);
+  pRNG.SeedFixedIntegers(seeds);
+
+  typedef PeriodicGimplR Gimpl;
+  typedef WilsonGaugeAction<Gimpl> GaugeAction;
+  typedef NoHirep Representation; //fundamental
+  typedef NoSmearing<Gimpl> Smearing;
+  typedef MinimumNorm2<Gimpl, Smearing> Omelyan;
+  typedef Gimpl::Field Field;
+  typedef MomentumFilterApplyPhase<Field> Filter;
+  Filter filter(&Grid);
+  
+  //Setup a filter that disables link update on links passing through the global lattice boundary
+  typedef Filter::LatticeLorentzScalarType MaskType;
+  typedef Filter::LorentzScalarType MaskSiteType;
+
+  MaskSiteType zero, one;
+  for(int mu=0;mu<Nd;mu++){
+    zero(mu)()() = 0.;
+    one(mu)()() = 1.;
+  }
+  MaskType zeroField(&Grid), oneField(&Grid);
+  zeroField = zero;
+  oneField = one;
+
+  
+  filter.phase = oneField; //make every site 1.0
+
+  //Zero mu-directed links at upper boundary
+  for(int mu=0;mu<Nd;mu++){
+    LatticeInteger mucoor(&Grid);
+    LatticeCoordinate(mucoor, mu);
+    
+    filter.phase = where( mucoor == (Integer)(latt_size[mu] - 1) , zeroField, filter.phase );
+  }
+
+  //Start with a random gauge field
+  Field U(&Grid);
+  SU<Nc>::HotConfiguration(pRNG,U);
+
+  //Get the original links on the bulk and boundary for later use
+  Field Ubnd_orig(&Grid), Ubulk_orig(&Grid);
+  getLinksBoundaryBulk(Ubnd_orig, Ubulk_orig, U, latt_size);
+
+  ActionSet<Field,Representation> actions(1);  
+  double beta=6;
+  GaugeAction gauge_action(beta);
+  actions[0].push_back(&gauge_action);
+
+  Smearing smear;
+  IntegratorParameters params(1,1.); //1 MD step
+  Omelyan integrator(&Grid, params, actions, smear);
+  
+  integrator.setMomentumFilter(filter);
+
+  integrator.refresh(U, pRNG); //doesn't actually change the gauge field
+
+  //Check the momentum is zero on the boundary
+  const auto &P = integrator.getMomentum();
+  Field Pbnd(&Grid), Pbulk(&Grid);
+  getLinksBoundaryBulk(Pbnd, Pbulk, const_cast<Field&>(P), latt_size);
+
+  RealD Pbnd_nrm = norm2(Pbnd); //expect zero
+  std::cout << GridLogMessage << "After refresh, norm2 of mu-directed conjugate momentum on boundary is: " << Pbnd_nrm << " (expect 0)" << std::endl;
+  RealD Pbulk_nrm = norm2(Pbulk); //expect non-zero
+  std::cout << GridLogMessage << "After refresh, norm2 of bulk conjugate momentum is: " << Pbulk_nrm << " (expect non-zero)" << std::endl;
+
+  //Evolve the gauge field
+  integrator.integrate(U);
+
+  //Check momentum is still zero on boundary
+  getLinksBoundaryBulk(Pbnd, Pbulk, const_cast<Field&>(P), latt_size);
+  
+  Pbnd_nrm = norm2(Pbnd); //expect zero
+  std::cout << GridLogMessage << "After integrate, norm2 of mu-directed conjugate momentum on boundary is: " << Pbnd_nrm << " (expect 0)" << std::endl;
+  Pbulk_nrm = norm2(Pbulk); //expect non-zero
+  std::cout << GridLogMessage << "After integrate, norm2 of bulk conjugate momentum is: " << Pbulk_nrm << " (expect non-zero)" << std::endl;
+
+  //Get the new bulk and bound links
+  Field Ubnd_new(&Grid), Ubulk_new(&Grid);
+  getLinksBoundaryBulk(Ubnd_new, Ubulk_new, U, latt_size);
+
+  Field Ubnd_diff = Ubnd_new - Ubnd_orig;
+  Field Ubulk_diff = Ubulk_new - Ubulk_orig;
+
+  RealD Ubnd_change = norm2( Ubnd_diff );
+  RealD Ubulk_change = norm2( Ubulk_diff );
+  std::cout << GridLogMessage << "After integrate, norm2 of change in mu-directed boundary links is : " << Ubnd_change << " (expect 0)" << std::endl;
+  std::cout << GridLogMessage << "After integrate, norm2 of change in bulk links is : " << Ubulk_change << " (expect non-zero)" << std::endl;
+
+  Grid_finalize();
+}

From f9b1f240f6af103ff437b459bce2937026c0f4d3 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 26 Feb 2021 17:51:41 +0100
Subject: [PATCH 42/82] Better SIMD usage/coalescence

---
 Grid/lattice/Lattice_view.h |  7 ++++-
 Grid/simd/Grid_gpu_vec.h    | 23 ++++++++++++--
 Grid/simd/Simd.h            | 12 ++++---
 Grid/tensors/Tensor_SIMT.h  | 62 +++++++++++++++++++++++++++++++++++++
 Grid/threads/Accelerator.h  | 53 ++++++++++++++++++++++++++-----
 5 files changed, 143 insertions(+), 14 deletions(-)

diff --git a/Grid/lattice/Lattice_view.h b/Grid/lattice/Lattice_view.h
index 3b76b921..cb568abd 100644
--- a/Grid/lattice/Lattice_view.h
+++ b/Grid/lattice/Lattice_view.h
@@ -67,9 +67,14 @@ public:
   accelerator_inline const vobj & operator()(size_t i) const { return this->_odata[i]; }
 #endif
 
+#if 1
+  //  accelerator_inline const vobj & operator[](size_t i) const { return this->_odata[i]; };
+  accelerator_inline vobj       & operator[](size_t i) const { return this->_odata[i]; };
+#else
   accelerator_inline const vobj & operator[](size_t i) const { return this->_odata[i]; };
   accelerator_inline vobj       & operator[](size_t i)       { return this->_odata[i]; };
-
+#endif
+  
   accelerator_inline uint64_t begin(void) const { return 0;};
   accelerator_inline uint64_t end(void)   const { return this->_odata_size; };
   accelerator_inline uint64_t size(void)  const { return this->_odata_size; };
diff --git a/Grid/simd/Grid_gpu_vec.h b/Grid/simd/Grid_gpu_vec.h
index 8e55ce2f..2c1a38e7 100644
--- a/Grid/simd/Grid_gpu_vec.h
+++ b/Grid/simd/Grid_gpu_vec.h
@@ -60,11 +60,25 @@ template<class pair>
 class GpuComplex {
 public:
   pair z;
-  typedef decltype(z.x) real;
+  typedef decltype(z.x) Real;
 public: 
   accelerator_inline GpuComplex() = default;
-  accelerator_inline GpuComplex(real re,real im) { z.x=re; z.y=im; };
+  accelerator_inline GpuComplex(Real re,Real im) { z.x=re; z.y=im; };
   accelerator_inline GpuComplex(const GpuComplex &zz) { z = zz.z;};
+  accelerator_inline Real real(void) const { return z.x; };
+  accelerator_inline Real imag(void) const { return z.y; };
+  accelerator_inline GpuComplex &operator*=(const GpuComplex &r) {
+    *this = (*this) * r;
+    return *this;
+  }
+  accelerator_inline GpuComplex &operator+=(const GpuComplex &r) {
+    *this = (*this) + r;
+    return *this;
+  }
+  accelerator_inline GpuComplex &operator-=(const GpuComplex &r) {
+    *this = (*this) - r;
+    return *this;
+  }
   friend accelerator_inline  GpuComplex operator+(const GpuComplex &lhs,const GpuComplex &rhs) { 
     GpuComplex r ; 
     r.z.x = lhs.z.x + rhs.z.x; 
@@ -157,6 +171,11 @@ typedef GpuVector<NSIMD_RealD,    double      > GpuVectorRD;
 typedef GpuVector<NSIMD_ComplexD, GpuComplexD > GpuVectorCD;
 typedef GpuVector<NSIMD_Integer,  Integer     > GpuVectorI;
 
+accelerator_inline GpuComplexF timesI(const GpuComplexF &r)     { return(GpuComplexF(-r.imag(),r.real()));}
+accelerator_inline GpuComplexD timesI(const GpuComplexD &r)     { return(GpuComplexD(-r.imag(),r.real()));}
+accelerator_inline GpuComplexF timesMinusI(const GpuComplexF &r){ return(GpuComplexF(r.imag(),-r.real()));}
+accelerator_inline GpuComplexD timesMinusI(const GpuComplexD &r){ return(GpuComplexD(r.imag(),-r.real()));}
+
 accelerator_inline float half2float(half h)
 {
   float f;
diff --git a/Grid/simd/Simd.h b/Grid/simd/Simd.h
index 1dc86c1b..76ca3bef 100644
--- a/Grid/simd/Simd.h
+++ b/Grid/simd/Simd.h
@@ -148,10 +148,14 @@ accelerator_inline void sub (ComplexF * __restrict__ y,const ComplexF * __restri
 accelerator_inline void add (ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) + (*r); }
   
 //conjugate already supported for complex
-accelerator_inline ComplexF timesI(const ComplexF &r)     { return(r*ComplexF(0.0,1.0));}
-accelerator_inline ComplexD timesI(const ComplexD &r)     { return(r*ComplexD(0.0,1.0));}
-accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(r*ComplexF(0.0,-1.0));}
-accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(r*ComplexD(0.0,-1.0));}
+accelerator_inline ComplexF timesI(const ComplexF &r)     { return(ComplexF(-r.imag(),r.real()));}
+accelerator_inline ComplexD timesI(const ComplexD &r)     { return(ComplexD(-r.imag(),r.real()));}
+accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(ComplexF(r.imag(),-r.real()));}
+accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(ComplexD(r.imag(),-r.real()));}
+//accelerator_inline ComplexF timesI(const ComplexF &r)     { return(r*ComplexF(0.0,1.0));}
+//accelerator_inline ComplexD timesI(const ComplexD &r)     { return(r*ComplexD(0.0,1.0));}
+//accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(r*ComplexF(0.0,-1.0));}
+//accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(r*ComplexD(0.0,-1.0));}
 
 // define projections to real and imaginay parts
 accelerator_inline ComplexF projReal(const ComplexF &r){return( ComplexF(r.real(), 0.0));}
diff --git a/Grid/tensors/Tensor_SIMT.h b/Grid/tensors/Tensor_SIMT.h
index ec57a679..ede24fbe 100644
--- a/Grid/tensors/Tensor_SIMT.h
+++ b/Grid/tensors/Tensor_SIMT.h
@@ -64,6 +64,68 @@ void coalescedWriteNonTemporal(vobj & __restrict__ vec,const vobj & __restrict__
 }
 #else
 
+
+#ifndef GRID_SYCL
+// Use the scalar as our own complex on GPU
+template<class vsimd,IfSimd<vsimd> = 0> accelerator_inline
+typename vsimd::scalar_type
+coalescedRead(const vsimd & __restrict__ vec,int lane=acceleratorSIMTlane(vsimd::Nsimd()))
+{
+  typedef typename vsimd::scalar_type S;
+  S * __restrict__ p=(S *)&vec;
+  return p[lane];
+}
+template<int ptype,class vsimd,IfSimd<vsimd> = 0> accelerator_inline
+typename vsimd::scalar_type
+coalescedReadPermute(const vsimd & __restrict__ vec,int doperm,int lane=acceleratorSIMTlane(vsimd::Nsimd()))
+{
+  typedef typename vsimd::scalar_type S;
+
+  S * __restrict__ p=(S *)&vec;
+  int mask = vsimd::Nsimd() >> (ptype + 1);
+  int plane= doperm ? lane ^ mask : lane;
+  return p[plane];
+}
+template<class vsimd,IfSimd<vsimd> = 0> accelerator_inline
+void coalescedWrite(vsimd & __restrict__ vec,
+		    const typename vsimd::scalar_type & __restrict__ extracted,
+		    int lane=acceleratorSIMTlane(vsimd::Nsimd()))
+{
+  typedef typename vsimd::scalar_type S;
+  S * __restrict__ p=(S *)&vec;
+  p[lane]=extracted;
+}
+#else
+template<class vsimd,IfSimd<vsimd> = 0> accelerator_inline
+typename vsimd::vector_type::datum
+coalescedRead(const vsimd & __restrict__ vec,int lane=acceleratorSIMTlane(vsimd::Nsimd()))
+{
+  typedef typename vsimd::vector_type::datum S;
+  S * __restrict__ p=(S *)&vec;
+  return p[lane];
+}
+template<int ptype,class vsimd,IfSimd<vsimd> = 0> accelerator_inline
+typename vsimd::vector_type::datum
+coalescedReadPermute(const vsimd & __restrict__ vec,int doperm,int lane=acceleratorSIMTlane(vsimd::Nsimd()))
+{
+  typedef typename vsimd::vector_type::datum S;
+
+  S * __restrict__ p=(S *)&vec;
+  int mask = vsimd::Nsimd() >> (ptype + 1);
+  int plane= doperm ? lane ^ mask : lane;
+  return p[plane];
+}
+template<class vsimd,IfSimd<vsimd> = 0> accelerator_inline
+void coalescedWrite(vsimd & __restrict__ vec,
+		    const typename vsimd::vector_type::datum & __restrict__ extracted,
+		    int lane=acceleratorSIMTlane(vsimd::Nsimd()))
+{
+  typedef typename vsimd::vector_type::datum S;
+  S * __restrict__ p=(S *)&vec;
+  p[lane]=extracted;
+}
+#endif
+
 //////////////////////////////////////////
 // Extract and insert slices on the GPU
 //////////////////////////////////////////
diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h
index 59645546..2b7bf53a 100644
--- a/Grid/threads/Accelerator.h
+++ b/Grid/threads/Accelerator.h
@@ -104,7 +104,7 @@ extern int acceleratorAbortOnGpuError;
 
 accelerator_inline int acceleratorSIMTlane(int Nsimd) {
 #ifdef GRID_SIMT
-  return threadIdx.z; 
+  return threadIdx.x; 
 #else
   return 0;
 #endif
@@ -112,28 +112,67 @@ accelerator_inline int acceleratorSIMTlane(int Nsimd) {
 
 #define accelerator_for2dNB( iter1, num1, iter2, num2, nsimd, ... )	\
   {									\
+    int nt=acceleratorThreads();					\
     typedef uint64_t Iterator;						\
     auto lambda = [=] accelerator					\
       (Iterator iter1,Iterator iter2,Iterator lane) mutable {		\
       __VA_ARGS__;							\
     };									\
-    int nt=acceleratorThreads();					\
-    dim3 cu_threads(acceleratorThreads(),1,nsimd);			\
+    dim3 cu_threads(nsimd,acceleratorThreads(),1);			\
     dim3 cu_blocks ((num1+nt-1)/nt,num2,1);				\
     LambdaApply<<<cu_blocks,cu_threads>>>(num1,num2,nsimd,lambda);	\
   }
 
+#define accelerator_for6dNB(iter1, num1,				\
+                            iter2, num2,				\
+                            iter3, num3,				\
+                            iter4, num4,				\
+                            iter5, num5,				\
+			    iter6, num6, ... )				\
+  {									\
+    typedef uint64_t Iterator;						\
+    auto lambda = [=] accelerator					\
+      (Iterator iter1,Iterator iter2,					\
+       Iterator iter3,Iterator iter4,					\
+       Iterator iter5,Iterator iter6) mutable {				\
+      __VA_ARGS__;							\
+    };									\
+    dim3 cu_blocks (num1,num2,num3);					\
+    dim3 cu_threads(num4,num5,num6);					\
+    Lambda6Apply<<<cu_blocks,cu_threads>>>(num1,num2,num3,num4,num5,num6,lambda); \
+  }
+
 template<typename lambda>  __global__
 void LambdaApply(uint64_t num1, uint64_t num2, uint64_t num3, lambda Lambda)
 {
-  uint64_t x = threadIdx.x + blockDim.x*blockIdx.x;
-  uint64_t y = threadIdx.y + blockDim.y*blockIdx.y;
-  uint64_t z = threadIdx.z;
+  // Weird permute is to make lane coalesce for large blocks
+  uint64_t x = threadIdx.y + blockDim.y*blockIdx.x;
+  uint64_t y = threadIdx.z + blockDim.z*blockIdx.y;
+  uint64_t z = threadIdx.x;
   if ( (x < num1) && (y<num2) && (z<num3) ) {
     Lambda(x,y,z);
   }
 }
 
+template<typename lambda>  __global__
+void Lambda6Apply(uint64_t num1, uint64_t num2, uint64_t num3,
+		  uint64_t num4, uint64_t num5, uint64_t num6,
+		  lambda Lambda)
+{
+  uint64_t iter1 = blockIdx.x;
+  uint64_t iter2 = blockIdx.y;
+  uint64_t iter3 = blockIdx.z;
+  uint64_t iter4 = threadIdx.x;
+  uint64_t iter5 = threadIdx.y;
+  uint64_t iter6 = threadIdx.z;
+
+  if ( (iter1 < num1) && (iter2<num2) && (iter3<num3)
+    && (iter4 < num4) && (iter5<num5) && (iter6<num6) )
+  {
+    Lambda(iter1,iter2,iter3,iter4,iter5,iter6);
+  }
+}
+
 #define accelerator_barrier(dummy)					\
   {									\
     cudaDeviceSynchronize();						\
@@ -221,7 +260,7 @@ accelerator_inline int acceleratorSIMTlane(int Nsimd) {
       cl::sycl::range<3> global{unum1,unum2,nsimd};			\
       cgh.parallel_for<class dslash>(					\
       cl::sycl::nd_range<3>(global,local), \
-      [=] (cl::sycl::nd_item<3> item) mutable {       \
+      [=] (cl::sycl::nd_item<3> item) /*mutable*/ {   \
       auto iter1    = item.get_global_id(0);	      \
       auto iter2    = item.get_global_id(1);	      \
       auto lane     = item.get_global_id(2);	      \

From 442336bd96b9c0062a9c42704d2013331b3b10e1 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Tue, 2 Mar 2021 14:50:51 +0100
Subject: [PATCH 43/82] Hand unrolled to use optimised code paths on GPU for
 coalesced reads in Wilson case. Other cases to do. This now includes comms
 code path.

---
 .../WilsonKernelsHandImplementation.h         | 217 ++++++++++--------
 .../WilsonKernelsImplementation.h             |  12 +-
 2 files changed, 123 insertions(+), 106 deletions(-)

diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
index 89ae5668..b867369f 100644
--- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
@@ -76,7 +76,24 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #define REGISTER
 
-#define LOAD_CHIMU \
+#ifdef GRID_SIMT
+#define LOAD_CHIMU(ptype)		\
+  {const SiteSpinor & ref (in[offset]);	\
+    Chimu_00=coalescedReadPermute<ptype>(ref()(0)(0),perm);	\
+    Chimu_01=coalescedReadPermute<ptype>(ref()(0)(1),perm);	\
+    Chimu_02=coalescedReadPermute<ptype>(ref()(0)(2),perm);	\
+    Chimu_10=coalescedReadPermute<ptype>(ref()(1)(0),perm);	\
+    Chimu_11=coalescedReadPermute<ptype>(ref()(1)(1),perm);	\
+    Chimu_12=coalescedReadPermute<ptype>(ref()(1)(2),perm);	\
+    Chimu_20=coalescedReadPermute<ptype>(ref()(2)(0),perm);	\
+    Chimu_21=coalescedReadPermute<ptype>(ref()(2)(1),perm);	\
+    Chimu_22=coalescedReadPermute<ptype>(ref()(2)(2),perm);	\
+    Chimu_30=coalescedReadPermute<ptype>(ref()(3)(0),perm);	\
+    Chimu_31=coalescedReadPermute<ptype>(ref()(3)(1),perm);	\
+    Chimu_32=coalescedReadPermute<ptype>(ref()(3)(2),perm);	}
+#define PERMUTE_DIR(dir) ;
+#else
+#define LOAD_CHIMU			\
   {const SiteSpinor & ref (in[offset]);	\
     Chimu_00=ref()(0)(0);\
     Chimu_01=ref()(0)(1);\
@@ -91,55 +108,55 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     Chimu_31=ref()(3)(1);\
     Chimu_32=ref()(3)(2);}
 
-#define LOAD_CHI\
-  {const SiteHalfSpinor &ref(buf[offset]);	\
-    Chi_00 = ref()(0)(0);\
-    Chi_01 = ref()(0)(1);\
-    Chi_02 = ref()(0)(2);\
-    Chi_10 = ref()(1)(0);\
-    Chi_11 = ref()(1)(1);\
-    Chi_12 = ref()(1)(2);}
-
-// To splat or not to splat depends on the implementation
-#define MULT_2SPIN(A)\
-  {auto & ref(U[sU](A));			\
-   Impl::loadLinkElement(U_00,ref()(0,0));	\
-   Impl::loadLinkElement(U_10,ref()(1,0));	\
-   Impl::loadLinkElement(U_20,ref()(2,0));	\
-   Impl::loadLinkElement(U_01,ref()(0,1));	\
-   Impl::loadLinkElement(U_11,ref()(1,1));	\
-   Impl::loadLinkElement(U_21,ref()(2,1));	\
-    UChi_00 = U_00*Chi_00;\
-    UChi_10 = U_00*Chi_10;\
-    UChi_01 = U_10*Chi_00;\
-    UChi_11 = U_10*Chi_10;\
-    UChi_02 = U_20*Chi_00;\
-    UChi_12 = U_20*Chi_10;\
-    UChi_00+= U_01*Chi_01;\
-    UChi_10+= U_01*Chi_11;\
-    UChi_01+= U_11*Chi_01;\
-    UChi_11+= U_11*Chi_11;\
-    UChi_02+= U_21*Chi_01;\
-    UChi_12+= U_21*Chi_11;\
-    Impl::loadLinkElement(U_00,ref()(0,2));	\
-    Impl::loadLinkElement(U_10,ref()(1,2));	\
-    Impl::loadLinkElement(U_20,ref()(2,2));	\
-    UChi_00+= U_00*Chi_02;\
-    UChi_10+= U_00*Chi_12;\
-    UChi_01+= U_10*Chi_02;\
-    UChi_11+= U_10*Chi_12;\
-    UChi_02+= U_20*Chi_02;\
-    UChi_12+= U_20*Chi_12;}
-
-
 #define PERMUTE_DIR(dir)			\
-      permute##dir(Chi_00,Chi_00);\
+  permute##dir(Chi_00,Chi_00);	\
       permute##dir(Chi_01,Chi_01);\
       permute##dir(Chi_02,Chi_02);\
-      permute##dir(Chi_10,Chi_10);\
+      permute##dir(Chi_10,Chi_10);	\
       permute##dir(Chi_11,Chi_11);\
       permute##dir(Chi_12,Chi_12);
 
+#endif
+
+#define LOAD_CHI				\
+  {const SiteHalfSpinor &ref(buf[offset]);	\
+    Chi_00 = coalescedRead(ref()(0)(0));	\
+    Chi_01 = coalescedRead(ref()(0)(1));	\
+    Chi_02 = coalescedRead(ref()(0)(2));	\
+    Chi_10 = coalescedRead(ref()(1)(0));	\
+    Chi_11 = coalescedRead(ref()(1)(1));	\
+    Chi_12 = coalescedRead(ref()(1)(2));}
+
+#define MULT_2SPIN(A)\
+  {auto & ref(U[sU](A));					\
+  U_00=coalescedRead(ref()(0,0));				\
+  U_10=coalescedRead(ref()(1,0));					\
+  U_20=coalescedRead(ref()(2,0));					\
+  U_01=coalescedRead(ref()(0,1));					\
+  U_11=coalescedRead(ref()(1,1));					\
+  U_21=coalescedRead(ref()(2,1));					\
+    UChi_00 = U_00*Chi_00;					\
+    UChi_10 = U_00*Chi_10;					\
+    UChi_01 = U_10*Chi_00;					\
+    UChi_11 = U_10*Chi_10;					\
+    UChi_02 = U_20*Chi_00;					\
+    UChi_12 = U_20*Chi_10;					\
+    UChi_00+= U_01*Chi_01;					\
+    UChi_10+= U_01*Chi_11;					\
+    UChi_01+= U_11*Chi_01;					\
+    UChi_11+= U_11*Chi_11;					\
+    UChi_02+= U_21*Chi_01;					\
+    UChi_12+= U_21*Chi_11;					\
+    U_00=coalescedRead(ref()(0,2));				\
+    U_10=coalescedRead(ref()(1,2));				\
+    U_20=coalescedRead(ref()(2,2));				\
+    UChi_00+= U_00*Chi_02;					\
+    UChi_10+= U_00*Chi_12;					\
+    UChi_01+= U_10*Chi_02;					\
+    UChi_11+= U_10*Chi_12;					\
+    UChi_02+= U_20*Chi_02;					\
+    UChi_12+= U_20*Chi_12;}
+
 //      hspin(0)=fspin(0)+timesI(fspin(3));
 //      hspin(1)=fspin(1)+timesI(fspin(2));
 #define XP_PROJ \
@@ -359,7 +376,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   local  = SE->_is_local;			\
   perm   = SE->_permute;			\
   if ( local ) {				\
-    LOAD_CHIMU;					\
+    LOAD_CHIMU(PERM);				\
     PROJ;					\
     if ( perm) {				\
       PERMUTE_DIR(PERM);			\
@@ -376,7 +393,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   local  = SE->_is_local;			\
   perm   = SE->_permute;			\
   if ( local ) {				\
-    LOAD_CHIMU;					\
+    LOAD_CHIMU(PERM);				\
     PROJ;					\
     if ( perm) {				\
       PERMUTE_DIR(PERM);			\
@@ -401,40 +418,39 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #define HAND_RESULT(ss)				\
   {						\
-    SiteSpinor & ref (out[ss]);		\
-    vstream(ref()(0)(0),result_00);		\
-    vstream(ref()(0)(1),result_01);		\
-    vstream(ref()(0)(2),result_02);		\
-    vstream(ref()(1)(0),result_10);		\
-    vstream(ref()(1)(1),result_11);		\
-    vstream(ref()(1)(2),result_12);		\
-    vstream(ref()(2)(0),result_20);		\
-    vstream(ref()(2)(1),result_21);		\
-    vstream(ref()(2)(2),result_22);		\
-    vstream(ref()(3)(0),result_30);		\
-    vstream(ref()(3)(1),result_31);		\
-    vstream(ref()(3)(2),result_32);		\
+    SiteSpinor & ref (out[ss]);			\
+    coalescedWrite(ref()(0)(0),result_00);		\
+    coalescedWrite(ref()(0)(1),result_01);		\
+    coalescedWrite(ref()(0)(2),result_02);		\
+    coalescedWrite(ref()(1)(0),result_10);		\
+    coalescedWrite(ref()(1)(1),result_11);		\
+    coalescedWrite(ref()(1)(2),result_12);		\
+    coalescedWrite(ref()(2)(0),result_20);		\
+    coalescedWrite(ref()(2)(1),result_21);		\
+    coalescedWrite(ref()(2)(2),result_22);		\
+    coalescedWrite(ref()(3)(0),result_30);		\
+    coalescedWrite(ref()(3)(1),result_31);		\
+    coalescedWrite(ref()(3)(2),result_32);		\
   }
 
-#define HAND_RESULT_EXT(ss)			\
-  if (nmu){					\
-    SiteSpinor & ref (out[ss]);		\
-    ref()(0)(0)+=result_00;		\
-    ref()(0)(1)+=result_01;		\
-    ref()(0)(2)+=result_02;		\
-    ref()(1)(0)+=result_10;		\
-    ref()(1)(1)+=result_11;		\
-    ref()(1)(2)+=result_12;		\
-    ref()(2)(0)+=result_20;		\
-    ref()(2)(1)+=result_21;		\
-    ref()(2)(2)+=result_22;		\
-    ref()(3)(0)+=result_30;		\
-    ref()(3)(1)+=result_31;		\
-    ref()(3)(2)+=result_32;		\
+#define HAND_RESULT_EXT(ss)				\
+  {							\
+    SiteSpinor & ref (out[ss]);				\
+    coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00);	\
+    coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01);	\
+    coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02);	\
+    coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10);	\
+    coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11);	\
+    coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12);	\
+    coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20);	\
+    coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21);	\
+    coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22);	\
+    coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30);	\
+    coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31);	\
+    coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32);	\
   }
 
-
-#define HAND_DECLARATIONS(a)			\
+#define HAND_DECLARATIONS(Simd)			\
   Simd result_00;				\
   Simd result_01;				\
   Simd result_02;				\
@@ -467,18 +483,18 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   Simd U_21;
 
 #define ZERO_RESULT				\
-  result_00=Zero();				\
-  result_01=Zero();				\
-  result_02=Zero();				\
-  result_10=Zero();				\
-  result_11=Zero();				\
-  result_12=Zero();				\
-  result_20=Zero();				\
-  result_21=Zero();				\
-  result_22=Zero();				\
-  result_30=Zero();				\
-  result_31=Zero();				\
-  result_32=Zero();			
+  result_00=S(0.0,0.0);			\
+  result_01=S(0.0,0.0);			\
+  result_02=S(0.0,0.0);			\
+  result_10=S(0.0,0.0);			\
+  result_11=S(0.0,0.0);			\
+  result_12=S(0.0,0.0);			\
+  result_20=S(0.0,0.0);			\
+  result_21=S(0.0,0.0);			\
+  result_22=S(0.0,0.0);			\
+  result_30=S(0.0,0.0);			\
+  result_31=S(0.0,0.0);			\
+  result_32=S(0.0,0.0);			
 
 #define Chimu_00 Chi_00
 #define Chimu_01 Chi_01
@@ -502,8 +518,8 @@ WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,Site
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
+  typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
   int offset,local,perm, ptype;
   StencilEntry *SE;
@@ -525,8 +541,8 @@ void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView
 {
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
+  typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
   int offset,local,perm, ptype;
@@ -549,8 +565,8 @@ WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,Si
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
+  typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
   int offset,local,perm, ptype;
   StencilEntry *SE;
@@ -572,8 +588,8 @@ void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldVi
 {
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
+  typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
   int offset,local,perm, ptype;
@@ -596,8 +612,8 @@ WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,Si
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
+  typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
   int offset, ptype;
   StencilEntry *SE;
@@ -620,8 +636,8 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldVi
 {
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
+  typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
   int offset, ptype;
@@ -682,3 +698,4 @@ NAMESPACE_END(Grid);
 #undef HAND_RESULT
 #undef HAND_RESULT_INT
 #undef HAND_RESULT_EXT
+#undef HAND_DECLARATIONS
diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h
index c5f50bbb..937d13af 100644
--- a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h
@@ -445,20 +445,20 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st,  DoubledGaugeField
 
    if( interior && exterior ) {
      if (Opt == WilsonKernelsStatic::OptGeneric    ) { KERNEL_CALL(GenericDhopSite); return;}
-#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSite);    return;}
+#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptInlineAsm  ) {  ASM_CALL(AsmDhopSite);    return;}
 #endif
    } else if( interior ) {
      if (Opt == WilsonKernelsStatic::OptGeneric    ) { KERNEL_CALLNB(GenericDhopSiteInt); return;}
-#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALLNB(HandDhopSiteInt);    return;}
+#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptInlineAsm  ) {  ASM_CALL(AsmDhopSiteInt);    return;}
 #endif
    } else if( exterior ) {
      if (Opt == WilsonKernelsStatic::OptGeneric    ) { KERNEL_CALL(GenericDhopSiteExt); return;}
-#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteExt);    return;}
+#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptInlineAsm  ) {  ASM_CALL(AsmDhopSiteExt);    return;}
 #endif
    }
@@ -476,20 +476,20 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st,  DoubledGaugeField
 
    if( interior && exterior ) {
      if (Opt == WilsonKernelsStatic::OptGeneric    ) { KERNEL_CALL(GenericDhopSiteDag); return;}
-#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDag);    return;}
+#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptInlineAsm  ) {  ASM_CALL(AsmDhopSiteDag);     return;}
 #endif
    } else if( interior ) {
      if (Opt == WilsonKernelsStatic::OptGeneric    ) { KERNEL_CALL(GenericDhopSiteDagInt); return;}
-#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagInt);    return;}
+#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptInlineAsm  ) {  ASM_CALL(AsmDhopSiteDagInt);     return;}
 #endif
    } else if( exterior ) {
      if (Opt == WilsonKernelsStatic::OptGeneric    ) { KERNEL_CALL(GenericDhopSiteDagExt); return;}
-#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagExt);    return;}
+#ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptInlineAsm  ) {  ASM_CALL(AsmDhopSiteDagExt);     return;}
 #endif
    }

From 679d1d22f7e920163acf7b88515c30ceb297387b Mon Sep 17 00:00:00 2001
From: u61464 <u61464@login-2.aidevcloud>
Date: Wed, 3 Mar 2021 11:21:43 -0800
Subject: [PATCH 44/82] Sycl happier

---
 Grid/Makefile.am                              |  4 ++-
 Grid/cartesian/Cartesian_red_black.h          |  2 +-
 Grid/qcd/action/fermion/FermionOperatorImpl.h |  4 +--
 Grid/qcd/action/fermion/WilsonCompressor.h    | 24 ++++++++--------
 Grid/qcd/action/fermion/WilsonImpl.h          |  2 +-
 .../WilsonKernelsHandImplementation.h         | 28 +++++++++----------
 Grid/simd/Grid_gpu_vec.h                      |  1 +
 Grid/stencil/SimpleCompressor.h               | 14 +++++-----
 Grid/stencil/Stencil.h                        | 14 +++++-----
 Grid/tensors/Tensor_SIMT.h                    |  4 ++-
 Grid/threads/Accelerator.h                    |  2 +-
 benchmarks/Benchmark_dwf_fp32.cc              |  2 +-
 configure.ac                                  | 11 ++++++++
 13 files changed, 64 insertions(+), 48 deletions(-)

diff --git a/Grid/Makefile.am b/Grid/Makefile.am
index ded6d146..7c3c151b 100644
--- a/Grid/Makefile.am
+++ b/Grid/Makefile.am
@@ -54,9 +54,11 @@ Version.h: version-cache
 include Make.inc
 include Eigen.inc
 
-extra_sources+=$(ZWILS_FERMION_FILES)
 extra_sources+=$(WILS_FERMION_FILES)
 extra_sources+=$(STAG_FERMION_FILES)
+if BUILD_ZMOBIUS
+  extra_sources+=$(ZWILS_FERMION_FILES)
+endif
 if BUILD_GPARITY
   extra_sources+=$(GP_FERMION_FILES)
 endif
diff --git a/Grid/cartesian/Cartesian_red_black.h b/Grid/cartesian/Cartesian_red_black.h
index b71981f5..092d4910 100644
--- a/Grid/cartesian/Cartesian_red_black.h
+++ b/Grid/cartesian/Cartesian_red_black.h
@@ -36,7 +36,7 @@ static const int CbBlack=1;
 static const int Even   =CbRed;
 static const int Odd    =CbBlack;
 
-accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex, Coordinate &rdim, Coordinate &chk_dim_msk)
+accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex,const Coordinate &rdim,const Coordinate &chk_dim_msk)
 {
   int nd=rdim.size();
   Coordinate coor(nd);
diff --git a/Grid/qcd/action/fermion/FermionOperatorImpl.h b/Grid/qcd/action/fermion/FermionOperatorImpl.h
index b444f6dc..9345c0e6 100644
--- a/Grid/qcd/action/fermion/FermionOperatorImpl.h
+++ b/Grid/qcd/action/fermion/FermionOperatorImpl.h
@@ -153,8 +153,8 @@ public:
   typedef typename Impl::StencilImpl             StencilImpl;		\
   typedef typename Impl::ImplParams               ImplParams;	        \
   typedef typename Impl::StencilImpl::View_type  StencilView;		\
-  typedef typename ViewMap<FermionField>::Type      FermionFieldView;	\
-  typedef typename ViewMap<DoubledGaugeField>::Type DoubledGaugeFieldView;
+  typedef const typename ViewMap<FermionField>::Type      FermionFieldView;	\
+  typedef const typename ViewMap<DoubledGaugeField>::Type DoubledGaugeFieldView;
 
 #define INHERIT_IMPL_TYPES(Base)		\
   INHERIT_GIMPL_TYPES(Base)			\
diff --git a/Grid/qcd/action/fermion/WilsonCompressor.h b/Grid/qcd/action/fermion/WilsonCompressor.h
index 10e98f33..0760bcba 100644
--- a/Grid/qcd/action/fermion/WilsonCompressor.h
+++ b/Grid/qcd/action/fermion/WilsonCompressor.h
@@ -61,7 +61,7 @@ public:
   typedef typename SiteHalfSpinor::vector_type     vComplexHigh;
   constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
 
-  accelerator_inline int CommDatumSize(void) {
+  accelerator_inline int CommDatumSize(void) const {
     return sizeof(SiteHalfCommSpinor);
   }
 
@@ -69,7 +69,7 @@ public:
   /* Compress includes precision change if mpi data is not same */
   /*****************************************************/
   template<class _SiteHalfSpinor, class _SiteSpinor>
-  accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) {
+  accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) const {
     _SiteHalfSpinor tmp;
     projector::Proj(tmp,in,mu,dag);
     vstream(buf[o],tmp);
@@ -81,7 +81,7 @@ public:
   accelerator_inline void Exchange(SiteHalfSpinor *mp,
 				   const SiteHalfSpinor * __restrict__ vp0,
 				   const SiteHalfSpinor * __restrict__ vp1,
-				   Integer type,Integer o){
+				   Integer type,Integer o) const {
     SiteHalfSpinor tmp1;
     SiteHalfSpinor tmp2;
     exchange(tmp1,tmp2,vp0[o],vp1[o],type);
@@ -93,7 +93,7 @@ public:
   /* Have a decompression step if mpi data is not same */
   /*****************************************************/
   accelerator_inline void Decompress(SiteHalfSpinor * __restrict__ out,
-				     SiteHalfSpinor * __restrict__ in, Integer o) {    
+				     SiteHalfSpinor * __restrict__ in, Integer o) const {    
     assert(0);
   }
 
@@ -103,7 +103,7 @@ public:
   accelerator_inline void CompressExchange(SiteHalfSpinor * __restrict__ out0,
 					   SiteHalfSpinor * __restrict__ out1,
 					   const SiteSpinor * __restrict__ in,
-					   Integer j,Integer k, Integer m,Integer type)
+					   Integer j,Integer k, Integer m,Integer type) const
   {
     SiteHalfSpinor temp1, temp2;
     SiteHalfSpinor temp3, temp4;
@@ -117,7 +117,7 @@ public:
   /*****************************************************/
   /* Pass the info to the stencil */
   /*****************************************************/
-  accelerator_inline bool DecompressionStep(void) { return false; }
+  accelerator_inline bool DecompressionStep(void) const { return false; }
 
 };
 
@@ -142,7 +142,7 @@ public:
   typedef typename SiteHalfSpinor::vector_type     vComplexHigh;
   constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
 
-  accelerator_inline int CommDatumSize(void) {
+  accelerator_inline int CommDatumSize(void) const {
     return sizeof(SiteHalfCommSpinor);
   }
 
@@ -150,7 +150,7 @@ public:
   /* Compress includes precision change if mpi data is not same */
   /*****************************************************/
   template<class _SiteHalfSpinor, class _SiteSpinor>
-  accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) {
+  accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) const {
     _SiteHalfSpinor hsp;
     SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf;
     projector::Proj(hsp,in,mu,dag);
@@ -163,7 +163,7 @@ public:
   accelerator_inline void Exchange(SiteHalfSpinor *mp,
                        SiteHalfSpinor *vp0,
                        SiteHalfSpinor *vp1,
-		       Integer type,Integer o){
+		       Integer type,Integer o) const {
     SiteHalfSpinor vt0,vt1;
     SiteHalfCommSpinor *vpp0 = (SiteHalfCommSpinor *)vp0;
     SiteHalfCommSpinor *vpp1 = (SiteHalfCommSpinor *)vp1;
@@ -175,7 +175,7 @@ public:
   /*****************************************************/
   /* Have a decompression step if mpi data is not same */
   /*****************************************************/
-  accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o){
+  accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o) const {
     SiteHalfCommSpinor *hin=(SiteHalfCommSpinor *)in;
     precisionChange((vComplexHigh *)&out[o],(vComplexLow *)&hin[o],Nw);
   }
@@ -186,7 +186,7 @@ public:
   accelerator_inline void CompressExchange(SiteHalfSpinor *out0,
 			       SiteHalfSpinor *out1,
 			       const SiteSpinor *in,
-			       Integer j,Integer k, Integer m,Integer type){
+			       Integer j,Integer k, Integer m,Integer type) const {
     SiteHalfSpinor temp1, temp2,temp3,temp4;
     SiteHalfCommSpinor *hout0 = (SiteHalfCommSpinor *)out0;
     SiteHalfCommSpinor *hout1 = (SiteHalfCommSpinor *)out1;
@@ -200,7 +200,7 @@ public:
   /*****************************************************/
   /* Pass the info to the stencil */
   /*****************************************************/
-  accelerator_inline bool DecompressionStep(void) { return true; }
+  accelerator_inline bool DecompressionStep(void) const { return true; }
 
 };
 
diff --git a/Grid/qcd/action/fermion/WilsonImpl.h b/Grid/qcd/action/fermion/WilsonImpl.h
index d7941d1f..94676b6b 100644
--- a/Grid/qcd/action/fermion/WilsonImpl.h
+++ b/Grid/qcd/action/fermion/WilsonImpl.h
@@ -72,7 +72,7 @@ public:
   typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
   typedef WilsonImplParams ImplParams;
   typedef WilsonStencil<SiteSpinor, SiteHalfSpinor,ImplParams> StencilImpl;
-  typedef typename StencilImpl::View_type StencilView;
+  typedef const typename StencilImpl::View_type StencilView;
     
   ImplParams Params;
 
diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
index b867369f..688cb75a 100644
--- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
@@ -93,7 +93,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     Chimu_32=coalescedReadPermute<ptype>(ref()(3)(2),perm);	}
 #define PERMUTE_DIR(dir) ;
 #else
-#define LOAD_CHIMU			\
+#define LOAD_CHIMU(ptype)		\
   {const SiteSpinor & ref (in[offset]);	\
     Chimu_00=ref()(0)(0);\
     Chimu_01=ref()(0)(1);\
@@ -482,19 +482,19 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   Simd U_11;					\
   Simd U_21;
 
-#define ZERO_RESULT				\
-  result_00=S(0.0,0.0);			\
-  result_01=S(0.0,0.0);			\
-  result_02=S(0.0,0.0);			\
-  result_10=S(0.0,0.0);			\
-  result_11=S(0.0,0.0);			\
-  result_12=S(0.0,0.0);			\
-  result_20=S(0.0,0.0);			\
-  result_21=S(0.0,0.0);			\
-  result_22=S(0.0,0.0);			\
-  result_30=S(0.0,0.0);			\
-  result_31=S(0.0,0.0);			\
-  result_32=S(0.0,0.0);			
+#define ZERO_RESULT							\
+  zeroit(result_00);							\
+  zeroit(result_01);							\
+  zeroit(result_02);							\
+  zeroit(result_10);							\
+  zeroit(result_11);							\
+  zeroit(result_12);							\
+  zeroit(result_20);							\
+  zeroit(result_21);							\
+  zeroit(result_22);							\
+  zeroit(result_30);							\
+  zeroit(result_31);							\
+  zeroit(result_32);			
 
 #define Chimu_00 Chi_00
 #define Chimu_01 Chi_01
diff --git a/Grid/simd/Grid_gpu_vec.h b/Grid/simd/Grid_gpu_vec.h
index 2c1a38e7..b2c7588f 100644
--- a/Grid/simd/Grid_gpu_vec.h
+++ b/Grid/simd/Grid_gpu_vec.h
@@ -67,6 +67,7 @@ public:
   accelerator_inline GpuComplex(const GpuComplex &zz) { z = zz.z;};
   accelerator_inline Real real(void) const { return z.x; };
   accelerator_inline Real imag(void) const { return z.y; };
+  accelerator_inline GpuComplex &operator=(const Zero &zz) { z.x = 0; z.y=0; return *this; };
   accelerator_inline GpuComplex &operator*=(const GpuComplex &r) {
     *this = (*this) * r;
     return *this;
diff --git a/Grid/stencil/SimpleCompressor.h b/Grid/stencil/SimpleCompressor.h
index be7c89c0..2ce48369 100644
--- a/Grid/stencil/SimpleCompressor.h
+++ b/Grid/stencil/SimpleCompressor.h
@@ -7,20 +7,20 @@ template<class vobj>
 class SimpleCompressor {
 public:
   void Point(int) {};
-  accelerator_inline int  CommDatumSize(void) { return sizeof(vobj); }
-  accelerator_inline bool DecompressionStep(void) { return false; }
-  template<class cobj> accelerator_inline void Compress(cobj *buf,int o,const cobj &in) { buf[o]=in; }
-  accelerator_inline void Exchange(vobj *mp,vobj *vp0,vobj *vp1,Integer type,Integer o){
+  accelerator_inline int  CommDatumSize(void) const { return sizeof(vobj); }
+  accelerator_inline bool DecompressionStep(void) const { return false; }
+  template<class cobj> accelerator_inline void Compress(cobj *buf,int o,const cobj &in) const { buf[o]=in; }
+  accelerator_inline void Exchange(vobj *mp,vobj *vp0,vobj *vp1,Integer type,Integer o) const {
     exchange(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type);
   }
-  accelerator_inline void Decompress(vobj *out,vobj *in, int o){ assert(0); }
+  accelerator_inline void Decompress(vobj *out,vobj *in, int o) const { assert(0); }
   accelerator_inline void CompressExchange(vobj *out0,vobj *out1,const vobj *in,
-			       int j,int k, int m,int type){
+			       int j,int k, int m,int type) const {
     exchange(out0[j],out1[j],in[k],in[m],type);
   }
   // For cshift. Cshift should drop compressor coupling altogether 
   // because I had to decouple the code from the Stencil anyway
-  accelerator_inline vobj operator() (const vobj &arg) {
+  accelerator_inline vobj operator() (const vobj &arg) const {
     return arg;
   }
 };
diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h
index 23fc8203..58cebed3 100644
--- a/Grid/stencil/Stencil.h
+++ b/Grid/stencil/Stencil.h
@@ -147,16 +147,16 @@ class CartesianStencilAccelerator {
   cobj* u_recv_buf_p;
   cobj* u_send_buf_p;
 
-  accelerator_inline cobj *CommBuf(void) { return u_recv_buf_p; }
+  accelerator_inline cobj *CommBuf(void) const { return u_recv_buf_p; }
 
-  accelerator_inline int GetNodeLocal(int osite,int point) {
+  accelerator_inline int GetNodeLocal(int osite,int point) const {
     return this->_entries_p[point+this->_npoints*osite]._is_local;
   }
-  accelerator_inline StencilEntry * GetEntry(int &ptype,int point,int osite) {
+  accelerator_inline StencilEntry * GetEntry(int &ptype,int point,int osite) const {
     ptype = this->_permute_type[point]; return & this->_entries_p[point+this->_npoints*osite];
   }
 
-  accelerator_inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) {
+  accelerator_inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) const {
     uint64_t cbase = (uint64_t)&u_recv_buf_p[0];
     local = this->_entries_p[ent]._is_local;
     perm  = this->_entries_p[ent]._permute;
@@ -168,14 +168,14 @@ class CartesianStencilAccelerator {
     }
   }
 
-  accelerator_inline uint64_t GetPFInfo(int ent,uint64_t base) {
+  accelerator_inline uint64_t GetPFInfo(int ent,uint64_t base) const {
     uint64_t cbase = (uint64_t)&u_recv_buf_p[0];
     int local = this->_entries_p[ent]._is_local;
     if (local) return  base + this->_entries_p[ent]._byte_offset;
     else       return cbase + this->_entries_p[ent]._byte_offset;
   }
 
-  accelerator_inline void iCoorFromIindex(Coordinate &coor,int lane)
+  accelerator_inline void iCoorFromIindex(Coordinate &coor,int lane) const
   {
     Lexicographic::CoorFromIndex(coor,lane,this->_simd_layout);
   }
@@ -221,7 +221,7 @@ public:
   typedef typename cobj::vector_type vector_type;
   typedef typename cobj::scalar_type scalar_type;
   typedef typename cobj::scalar_object scalar_object;
-  typedef CartesianStencilView<vobj,cobj,Parameters> View_type;
+  typedef const CartesianStencilView<vobj,cobj,Parameters> View_type;
   typedef typename View_type::StencilVector StencilVector;
   ///////////////////////////////////////////
   // Helper structs
diff --git a/Grid/tensors/Tensor_SIMT.h b/Grid/tensors/Tensor_SIMT.h
index ede24fbe..672f385f 100644
--- a/Grid/tensors/Tensor_SIMT.h
+++ b/Grid/tensors/Tensor_SIMT.h
@@ -66,7 +66,7 @@ void coalescedWriteNonTemporal(vobj & __restrict__ vec,const vobj & __restrict__
 
 
 #ifndef GRID_SYCL
-// Use the scalar as our own complex on GPU
+// Use the scalar as our own complex on GPU ... thrust::complex or std::complex
 template<class vsimd,IfSimd<vsimd> = 0> accelerator_inline
 typename vsimd::scalar_type
 coalescedRead(const vsimd & __restrict__ vec,int lane=acceleratorSIMTlane(vsimd::Nsimd()))
@@ -96,6 +96,8 @@ void coalescedWrite(vsimd & __restrict__ vec,
   p[lane]=extracted;
 }
 #else
+// For SyCL have option to use GpuComplex from inside the vector type in SIMT loops
+// Faster for some reason
 template<class vsimd,IfSimd<vsimd> = 0> accelerator_inline
 typename vsimd::vector_type::datum
 coalescedRead(const vsimd & __restrict__ vec,int lane=acceleratorSIMTlane(vsimd::Nsimd()))
diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h
index 2b7bf53a..f1a694fb 100644
--- a/Grid/threads/Accelerator.h
+++ b/Grid/threads/Accelerator.h
@@ -456,7 +456,7 @@ accelerator_inline void acceleratorSynchronise(void)
   __syncwarp();
 #endif
 #ifdef GRID_SYCL
-  // No barrier call on SYCL??  // Option get __spir:: stuff to do warp barrier
+  cl::sycl::detail::workGroupBarrier();
 #endif
 #ifdef GRID_HIP
   __syncthreads();
diff --git a/benchmarks/Benchmark_dwf_fp32.cc b/benchmarks/Benchmark_dwf_fp32.cc
index cb86177e..03f3ee61 100644
--- a/benchmarks/Benchmark_dwf_fp32.cc
+++ b/benchmarks/Benchmark_dwf_fp32.cc
@@ -53,7 +53,7 @@ int main (int argc, char ** argv)
   int threads = GridThread::GetThreads();
 
   Coordinate latt4 = GridDefaultLatt();
-  int Ls=8;
+  int Ls=16;
   for(int i=0;i<argc;i++)
     if(std::string(argv[i]) == "-Ls"){
       std::stringstream ss(argv[i+1]); ss >> Ls;
diff --git a/configure.ac b/configure.ac
index fb0c78fc..5f165412 100644
--- a/configure.ac
+++ b/configure.ac
@@ -140,12 +140,23 @@ AC_ARG_ENABLE([gparity],
      [ac_GPARITY=${enable_gparity}], [ac_GPARITY=yes])
 
 AM_CONDITIONAL(BUILD_GPARITY, [ test "${ac_GPARITY}X" == "yesX" ])
+
+AC_ARG_ENABLE([zmobius],
+     [AC_HELP_STRING([--enable-zmobius=yes|no], [enable Zmobius support])],
+     [ac_ZMOBIUS=${enable_zmobius}], [ac_ZMOBIUS=yes])
+
+AM_CONDITIONAL(BUILD_ZMOBIUS, [ test "${ac_ZMOBIUS}X" == "yesX" ])
+
+
 case ${ac_FERMION_REPS} in
    yes) AC_DEFINE([ENABLE_FERMION_REPS],[1],[non QCD fermion reps]);;
 esac
 case ${ac_GPARITY} in
    yes) AC_DEFINE([ENABLE_GPARITY],[1],[fermion actions with GPARITY BCs]);;
 esac
+case ${ac_ZMOBIUS} in
+   yes) AC_DEFINE([ENABLE_ZMOBIUS],[1],[Zmobius fermion actions]);;
+esac
 ############### Nc
 AC_ARG_ENABLE([Nc],
     [AC_HELP_STRING([--enable-Nc=2|3|4], [enable number of colours])],

From 1eea9d73b984252c69869f65991f2f33d6e93193 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Wed, 3 Mar 2021 23:50:01 +0100
Subject: [PATCH 45/82] Pass serial RNG around

---
 Grid/qcd/action/ActionBase.h                        |  2 +-
 Grid/qcd/action/gauge/GaugeImplTypes.h              |  2 +-
 Grid/qcd/action/gauge/PlaqPlusRectangleAction.h     |  2 +-
 Grid/qcd/action/gauge/WilsonGaugeAction.h           |  3 +--
 .../qcd/action/pseudofermion/ExactOneFlavourRatio.h |  2 +-
 .../pseudofermion/OneFlavourEvenOddRational.h       | 11 ++++++-----
 .../pseudofermion/OneFlavourEvenOddRationalRatio.h  |  7 +++++--
 Grid/qcd/action/pseudofermion/OneFlavourRational.h  |  7 +++++--
 .../action/pseudofermion/OneFlavourRationalRatio.h  |  7 +++++--
 Grid/qcd/action/pseudofermion/TwoFlavour.h          |  2 +-
 Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h   |  2 +-
 .../action/pseudofermion/TwoFlavourEvenOddRatio.h   |  2 +-
 Grid/qcd/action/pseudofermion/TwoFlavourRatio.h     |  2 +-
 Grid/qcd/action/scalar/ScalarAction.h               |  2 +-
 Grid/qcd/action/scalar/ScalarImpl.h                 |  4 ++--
 Grid/qcd/action/scalar/ScalarInteractionAction.h    |  2 +-
 Grid/qcd/hmc/HMC.h                                  |  2 +-
 Grid/qcd/hmc/integrators/Integrator.h               | 13 ++++++-------
 benchmarks/Benchmark_dwf_fp32.cc                    |  2 +-
 19 files changed, 42 insertions(+), 34 deletions(-)

diff --git a/Grid/qcd/action/ActionBase.h b/Grid/qcd/action/ActionBase.h
index bff21d1d..17980ee0 100644
--- a/Grid/qcd/action/ActionBase.h
+++ b/Grid/qcd/action/ActionBase.h
@@ -41,7 +41,7 @@ class Action
 public:
   bool is_smeared = false;
   // Heatbath?
-  virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
+  virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
   virtual RealD S(const GaugeField& U) = 0;                             // evaluate the action
   virtual void deriv(const GaugeField& U, GaugeField& dSdU) = 0;        // evaluate the action derivative
   virtual std::string action_name()    = 0;                             // return the action name
diff --git a/Grid/qcd/action/gauge/GaugeImplTypes.h b/Grid/qcd/action/gauge/GaugeImplTypes.h
index 55a20eca..2499e0e9 100644
--- a/Grid/qcd/action/gauge/GaugeImplTypes.h
+++ b/Grid/qcd/action/gauge/GaugeImplTypes.h
@@ -96,7 +96,7 @@ public:
   ///////////////////////////////////////////////////////////
   // Move these to another class
   // HMC auxiliary functions
-  static inline void generate_momenta(Field &P, GridParallelRNG &pRNG) 
+  static inline void generate_momenta(Field &P, GridSerialRNG & sRNG, GridParallelRNG &pRNG) 
   {
     // Zbigniew Srocinsky thesis:
     //
diff --git a/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h b/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h
index 639aca19..7690092d 100644
--- a/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h
+++ b/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h
@@ -49,7 +49,7 @@ public:
 
   virtual std::string action_name(){return "PlaqPlusRectangleAction";}
       
-  virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {}; // noop as no pseudoferms
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {}; // noop as no pseudoferms
       
   virtual std::string LogParameters(){
     std::stringstream sstream;
diff --git a/Grid/qcd/action/gauge/WilsonGaugeAction.h b/Grid/qcd/action/gauge/WilsonGaugeAction.h
index 40d600d2..f535b54f 100644
--- a/Grid/qcd/action/gauge/WilsonGaugeAction.h
+++ b/Grid/qcd/action/gauge/WilsonGaugeAction.h
@@ -54,8 +54,7 @@ public:
     return sstream.str();
   }
 
-  virtual void refresh(const GaugeField &U,
-                       GridParallelRNG &pRNG){};  // noop as no pseudoferms
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG){};  // noop as no pseudoferms
 
   virtual RealD S(const GaugeField &U) {
     RealD plaq = WilsonLoops<Gimpl>::avgPlaquette(U);
diff --git a/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h b/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h
index 9fc0a3b0..576a8cf6 100644
--- a/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h
+++ b/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h
@@ -124,7 +124,7 @@ NAMESPACE_BEGIN(Grid);
       //
       // As a check of rational require \Phi^dag M_{EOFA} \Phi == eta^dag M^-1/2^dag M M^-1/2 eta = eta^dag eta
       //
-      virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG)
+      virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG)
       {
         Lop.ImportGauge(U);
         Rop.ImportGauge(U);
diff --git a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h
index 56dff94d..656e9b2f 100644
--- a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h
+++ b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h
@@ -1,4 +1,3 @@
-
 /*************************************************************************************
 
 Grid physics library, www.github.com/paboyle/Grid
@@ -43,8 +42,7 @@ NAMESPACE_BEGIN(Grid);
 //
 
 template <class Impl>
-class OneFlavourEvenOddRationalPseudoFermionAction
-  : public Action<typename Impl::GaugeField> {
+class OneFlavourEvenOddRationalPseudoFermionAction : public Action<typename Impl::GaugeField> {
 public:
   INHERIT_IMPL_TYPES(Impl);
 
@@ -103,7 +101,7 @@ public:
     return sstream.str();
   }
   
-  virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) {
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {
     // P(phi) = e^{- phi^dag (MpcdagMpc)^-1/2 phi}
     //        = e^{- phi^dag (MpcdagMpc)^-1/4 (MpcdagMpc)^-1/4 phi}
     // Phi = MpcdagMpc^{1/4} eta
@@ -156,7 +154,10 @@ public:
 
     msCG(Mpc, PhiOdd, Y);
 
-    if ( (rand()%param.BoundsCheckFreq)==0 ) { 
+    auto grid = FermOp.FermionGrid();
+    auto r=rand();
+    grid->Broadcast(0,r);
+    if ( (r%param.BoundsCheckFreq)==0 ) { 
       FermionField gauss(FermOp.FermionRedBlackGrid());
       gauss = PhiOdd;
       HighBoundCheck(Mpc,gauss,param.hi);
diff --git a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h
index e5f0b602..e968b8e4 100644
--- a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h
+++ b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h
@@ -101,7 +101,7 @@ NAMESPACE_BEGIN(Grid);
       }
       
       
-      virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
+      virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
 
 	// S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi       
 	//
@@ -170,7 +170,10 @@ NAMESPACE_BEGIN(Grid);
 	msCG_M(MdagM,X,Y);
 
 	// Randomly apply rational bounds checks.
-	if ( (rand()%param.BoundsCheckFreq)==0 ) { 
+	auto grid = NumOp.FermionGrid();
+        auto r=rand();
+        grid->Broadcast(0,r);
+        if ( (r%param.BoundsCheckFreq)==0 ) { 
 	  FermionField gauss(NumOp.FermionRedBlackGrid());
 	  gauss = PhiOdd;
 	  HighBoundCheck(MdagM,gauss,param.hi);
diff --git a/Grid/qcd/action/pseudofermion/OneFlavourRational.h b/Grid/qcd/action/pseudofermion/OneFlavourRational.h
index f6c823c9..aa647445 100644
--- a/Grid/qcd/action/pseudofermion/OneFlavourRational.h
+++ b/Grid/qcd/action/pseudofermion/OneFlavourRational.h
@@ -98,7 +98,7 @@ NAMESPACE_BEGIN(Grid);
 
 
       
-      virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
+      virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
 
 	
 	// P(phi) = e^{- phi^dag (MdagM)^-1/2 phi}
@@ -142,7 +142,10 @@ NAMESPACE_BEGIN(Grid);
 
 	msCG(MdagMOp,Phi,Y);
 
-	if ( (rand()%param.BoundsCheckFreq)==0 ) { 
+	auto grid = FermOp.FermionGrid();
+        auto r=rand();
+        grid->Broadcast(0,r);
+        if ( (r%param.BoundsCheckFreq)==0 ) { 
 	  FermionField gauss(FermOp.FermionGrid());
 	  gauss = Phi;
 	  HighBoundCheck(MdagMOp,gauss,param.hi);
diff --git a/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h b/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h
index 5fae2fe9..128c869a 100644
--- a/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h
+++ b/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h
@@ -95,7 +95,7 @@ NAMESPACE_BEGIN(Grid);
       }
       
 
-      virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
+      virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
 
 	// S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi       
 	//
@@ -156,7 +156,10 @@ NAMESPACE_BEGIN(Grid);
 	msCG_M(MdagM,X,Y);
 
 	// Randomly apply rational bounds checks.
-	if ( (rand()%param.BoundsCheckFreq)==0 ) { 
+        auto grid = NumOp.FermionGrid();
+        auto r=rand();
+        grid->Broadcast(0,r);
+        if ( (r%param.BoundsCheckFreq)==0 ) { 	
 	  FermionField gauss(NumOp.FermionGrid());
 	  gauss = Phi;
 	  HighBoundCheck(MdagM,gauss,param.hi);
diff --git a/Grid/qcd/action/pseudofermion/TwoFlavour.h b/Grid/qcd/action/pseudofermion/TwoFlavour.h
index f905a675..2ac97ddd 100644
--- a/Grid/qcd/action/pseudofermion/TwoFlavour.h
+++ b/Grid/qcd/action/pseudofermion/TwoFlavour.h
@@ -73,7 +73,7 @@ public:
   //////////////////////////////////////////////////////////////////////////////////////
   // Push the gauge field in to the dops. Assume any BC's and smearing already applied
   //////////////////////////////////////////////////////////////////////////////////////
-  virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) {
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {
     // P(phi) = e^{- phi^dag (MdagM)^-1 phi}
     // Phi = Mdag eta
     // P(eta) = e^{- eta^dag eta}
diff --git a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h
index a3cf8f08..2e5208a8 100644
--- a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h
+++ b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h
@@ -77,7 +77,7 @@ public:
   //////////////////////////////////////////////////////////////////////////////////////
   // Push the gauge field in to the dops. Assume any BC's and smearing already applied
   //////////////////////////////////////////////////////////////////////////////////////
-  virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
     
     // P(phi) = e^{- phi^dag (MpcdagMpc)^-1 phi}
     // Phi = McpDag eta 
diff --git a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h
index d1d6f336..da628c75 100644
--- a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h
+++ b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h
@@ -84,7 +84,7 @@ NAMESPACE_BEGIN(Grid);
       } 
 
       
-      virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
+      virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
 
         // P(phi) = e^{- phi^dag Vpc (MpcdagMpc)^-1 Vpcdag phi}
         //
diff --git a/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h b/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h
index 4d72faba..f584706d 100644
--- a/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h
+++ b/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h
@@ -64,7 +64,7 @@ public:
     return sstream.str();
   }  
       
-  virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
 
     // P(phi) = e^{- phi^dag V (MdagM)^-1 Vdag phi}
     //
diff --git a/Grid/qcd/action/scalar/ScalarAction.h b/Grid/qcd/action/scalar/ScalarAction.h
index 34fc4fac..8b4f4f79 100644
--- a/Grid/qcd/action/scalar/ScalarAction.h
+++ b/Grid/qcd/action/scalar/ScalarAction.h
@@ -55,7 +55,7 @@ public:
   }
   virtual std::string action_name() {return "ScalarAction";}
 
-  virtual void refresh(const Field &U, GridParallelRNG &pRNG) {}  // noop as no pseudoferms
+  virtual void refresh(const Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {}  // noop as no pseudoferms
 
   virtual RealD S(const Field &p) {
     return (mass_square * 0.5 + Nd) * ScalarObs<Impl>::sumphisquared(p) +
diff --git a/Grid/qcd/action/scalar/ScalarImpl.h b/Grid/qcd/action/scalar/ScalarImpl.h
index 403ea573..13bd6c90 100644
--- a/Grid/qcd/action/scalar/ScalarImpl.h
+++ b/Grid/qcd/action/scalar/ScalarImpl.h
@@ -27,7 +27,7 @@ public:
   typedef Field              FermionField;
   typedef Field              PropagatorField;
     
-  static inline void generate_momenta(Field& P, GridParallelRNG& pRNG){
+  static inline void generate_momenta(Field& P, GridSerialRNG &sRNG, GridParallelRNG& pRNG){
     RealD scale = ::sqrt(HMC_MOMENTUM_DENOMINATOR); // CPS/UKQCD momentum rescaling
     gaussian(pRNG, P);
     P *= scale; 
@@ -151,7 +151,7 @@ public:
       out = one / out;
     }
 
-    static inline void generate_momenta(Field &P, GridParallelRNG &pRNG)
+    static inline void generate_momenta(Field &P, GridSerialRNG & sRNG, GridParallelRNG &pRNG)
     {
       RealD scale = ::sqrt(HMC_MOMENTUM_DENOMINATOR); // CPS/UKQCD momentum rescaling
 #ifndef USE_FFT_ACCELERATION
diff --git a/Grid/qcd/action/scalar/ScalarInteractionAction.h b/Grid/qcd/action/scalar/ScalarInteractionAction.h
index 5a5f9251..e04dd486 100644
--- a/Grid/qcd/action/scalar/ScalarInteractionAction.h
+++ b/Grid/qcd/action/scalar/ScalarInteractionAction.h
@@ -77,7 +77,7 @@ public:
 
   virtual std::string action_name() { return "ScalarAction"; }
 
-  virtual void refresh(const Field &U, GridParallelRNG &pRNG) {}
+  virtual void refresh(const Field &U, GridSerialRNG & sRNG, GridParallelRNG &pRNG) {}
 
   virtual RealD S(const Field &p)
   {
diff --git a/Grid/qcd/hmc/HMC.h b/Grid/qcd/hmc/HMC.h
index f168b69a..44674ea5 100644
--- a/Grid/qcd/hmc/HMC.h
+++ b/Grid/qcd/hmc/HMC.h
@@ -139,7 +139,7 @@ private:
   // Evolution
   /////////////////////////////////////////////////////////
   RealD evolve_hmc_step(Field &U) {
-    TheIntegrator.refresh(U, pRNG);  // set U and initialize P and phi's
+    TheIntegrator.refresh(U, sRNG, pRNG);  // set U and initialize P and phi's
 
     RealD H0 = TheIntegrator.S(U);  // initial state action
 
diff --git a/Grid/qcd/hmc/integrators/Integrator.h b/Grid/qcd/hmc/integrators/Integrator.h
index 77b7de52..aa28c6c8 100644
--- a/Grid/qcd/hmc/integrators/Integrator.h
+++ b/Grid/qcd/hmc/integrators/Integrator.h
@@ -236,10 +236,9 @@ public:
   // over the representations
   struct _refresh {
     template <class FieldType, class Repr>
-    void operator()(std::vector<Action<FieldType>*> repr_set, Repr& Rep,
-                    GridParallelRNG& pRNG) {
+    void operator()(std::vector<Action<FieldType>*> repr_set, Repr& Rep, GridSerialRNG & sRNG, GridParallelRNG& pRNG) {
       for (int a = 0; a < repr_set.size(); ++a){
-        repr_set.at(a)->refresh(Rep.U, pRNG);
+        repr_set.at(a)->refresh(Rep.U, sRNG, pRNG);
       
 	std::cout << GridLogDebug << "Hirep refreshing pseudofermions" << std::endl;
       }
@@ -247,12 +246,12 @@ public:
   } refresh_hireps{};
 
   // Initialization of momenta and actions
-  void refresh(Field& U, GridParallelRNG& pRNG) 
+  void refresh(Field& U,  GridSerialRNG & sRNG, GridParallelRNG& pRNG) 
   {
     assert(P.Grid() == U.Grid());
     std::cout << GridLogIntegrator << "Integrator refresh\n";
 
-    FieldImplementation::generate_momenta(P, pRNG);
+    FieldImplementation::generate_momenta(P, sRNG, pRNG);
 
     // Update the smeared fields, can be implemented as observer
     // necessary to keep the fields updated even after a reject
@@ -269,11 +268,11 @@ public:
         // get gauge field from the SmearingPolicy and
         // based on the boolean is_smeared in actionID
         Field& Us = Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
-        as[level].actions.at(actionID)->refresh(Us, pRNG);
+        as[level].actions.at(actionID)->refresh(Us, sRNG, pRNG);
       }
 
       // Refresh the higher representation actions
-      as[level].apply(refresh_hireps, Representations, pRNG);
+      as[level].apply(refresh_hireps, Representations, sRNG, pRNG);
     }
 
     MomFilter->applyFilter(P);
diff --git a/benchmarks/Benchmark_dwf_fp32.cc b/benchmarks/Benchmark_dwf_fp32.cc
index cb86177e..03f3ee61 100644
--- a/benchmarks/Benchmark_dwf_fp32.cc
+++ b/benchmarks/Benchmark_dwf_fp32.cc
@@ -53,7 +53,7 @@ int main (int argc, char ** argv)
   int threads = GridThread::GetThreads();
 
   Coordinate latt4 = GridDefaultLatt();
-  int Ls=8;
+  int Ls=16;
   for(int i=0;i<argc;i++)
     if(std::string(argv[i]) == "-Ls"){
       std::stringstream ss(argv[i+1]); ss >> Ls;

From d4b4de8f428c625757c31c6d265c288671a0ef5b Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Thu, 4 Mar 2021 20:01:24 +0000
Subject: [PATCH 46/82] changes

---
 Grid/qcd/utils/BaryonUtils.h | 60 +++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 25 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index b74a5b20..1a7a4d38 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -252,7 +252,7 @@ public:
 				 const Gamma GammaB_sigma,
 				 const Gamma GammaB_nucl,
 		                 const std::string op,
-				 SpinMatrixField &stn_corr);
+				 SpinMatrixField &xts_corr);
 };
 //This computes a baryon contraction on a lattice site, including the spin-trace of the correlation matrix
 template <class FImpl>
@@ -1435,6 +1435,7 @@ void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
       int c_x = (ie_x < 3 ? (ie_x+2)%3 : (7-ie_x)%3 ); //epsilon[ie_x][2]; //c'
       int eSgn_x = (ie_x < 3 ? 1 : -1);
       ee = Real(eSgn_s * eSgn_x);
+      auto ee_GD = ee * trGDq;
       for (int alpha_x=0; alpha_x<Ns; alpha_x++){
       for (int beta_s=0; beta_s<Ns; beta_s++){
         auto GDsGDdG_ab_ba = GDsGDd()(alpha_x,beta_s)(b_x,a_s);
@@ -1488,15 +1489,19 @@ void BaryonUtils<FImpl>::XiToSigmaQ2EyeSite(const mobj &Dq_loop,
   // GammaB * DsGDd * GammaB
   auto GDsGDqGDdG = GDsGDqGDd * GammaB_sigma;
 
+  Real ee;
+
   for (int ie_s=0; ie_s < 6 ; ie_s++){
-    int a_s = epsilon[ie_s][0]; //a
-    int b_s = epsilon[ie_s][1]; //b
-    int c_s = epsilon[ie_s][2]; //c
+      int a_s = (ie_s < 3 ? ie_s       : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a'
+      int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b'
+      int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c'
+      int eSgn_s = (ie_s < 3 ? 1 : -1);
     for (int ie_x=0; ie_x < 6 ; ie_x++){
-      int a_x = epsilon[ie_x][0]; //a'
-      int b_x = epsilon[ie_x][1]; //b'
-      int c_x = epsilon[ie_x][2]; //c'
-      auto ee = epsilon_sgn[ie_s] * epsilon_sgn[ie_x];
+      int a_x = (ie_x < 3 ? ie_x       : (6-ie_x)%3 ); //epsilon[ie_x][0]; //a'
+      int b_x = (ie_x < 3 ? (ie_x+1)%3 : (8-ie_x)%3 ); //epsilon[ie_x][1]; //b'
+      int c_x = (ie_x < 3 ? (ie_x+2)%3 : (7-ie_x)%3 ); //epsilon[ie_x][2]; //c'
+      int eSgn_x = (ie_x < 3 ? 1 : -1);
+      ee = Real(eSgn_s * eSgn_x);
       for (int alpha_x=0; alpha_x<Ns; alpha_x++){
       for (int beta_s=0; beta_s<Ns; beta_s++){
         auto GDsGDqGDdG_ab_ba = GDsGDqGDdG()(alpha_x,beta_s)(b_x,a_s);
@@ -1540,27 +1545,32 @@ void BaryonUtils<FImpl>::XiToSigmaEye(const PropagatorField &qq_loop,
   autoView( vd_tf   , qd_tf    , AcceleratorRead);
   autoView( vs_ti   , qs_ti    , AcceleratorRead);
 
-  bool doQ1 = (op == "Q1");
-  bool doQ2 = (op == "Q2");
-
   Vector<mobj> my_Dq_spec{Dd_spec,Ds_spec};
   mobj * Dq_spec_p = &my_Dq_spec[0];
 
-  accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
-    auto Dq_loop = vq_loop(ss);
-    auto Dd_tf   = vd_tf(ss);
-    auto Ds_ti   = vs_ti(ss);
-    typedef decltype(coalescedRead(vcorr[0])) spinor;
-    spinor result=Zero();
-    if(doQ1){
+  if(op == "Q1"){
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_loop = vq_loop(ss);
+      auto Dd_tf   = vd_tf(ss);
+      auto Ds_ti   = vs_ti(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
       XiToSigmaQ1EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[1],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
-    } else if(doQ2){
-      XiToSigmaQ2EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
-    } else {
-      assert(0 && "Weak Operator not correctly specified");
-    }
-    coalescedWrite(vcorr[ss],result);
-  }  );//end loop over lattice sites
+      coalescedWrite(vcorr[ss],result);
+    }  );//end loop over lattice sites
+  } else if(op == "Q2"){
+    accelerator_for(ss, grid->oSites(), grid->Nsimd(), {
+      auto Dq_loop = vq_loop(ss);
+      auto Dd_tf   = vd_tf(ss);
+      auto Ds_ti   = vs_ti(ss);
+      typedef decltype(coalescedRead(vcorr[0])) spinor;
+      spinor result=Zero();
+      XiToSigmaQ2EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[1],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result);
+      coalescedWrite(vcorr[ss],result);
+    }  );//end loop over lattice sites
+  } else {
+    assert(0 && "Weak Operator not correctly specified");
+  }
 }
 
 

From 9b15704290048b936e71c3ddd652a6eec9bf5fff Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Fri, 5 Mar 2021 10:42:32 +0000
Subject: [PATCH 47/82] tested and consitent

---
 Grid/qcd/utils/BaryonUtils.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index 1a7a4d38..b69865e8 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -27,7 +27,6 @@
  *************************************************************************************/
 /*  END LEGAL */
 #pragma once
-//#include <Grid/Hadrons/Global.hpp>
 #include <Grid/Eigen/unsupported/CXX11/Tensor>
 
 NAMESPACE_BEGIN(Grid);
@@ -200,7 +199,7 @@ public:
 				                 const Gamma GammaB_sigma,
 		                 		 const Gamma GammaB_nucl,
 						 robj &result);
-  template <class mobj, class mobj2, class robj>
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void XiToSigmaQ1EyeSite(const mobj &Dq_loop,
 						 const mobj2 &Dd_spec,
 						 const mobj2 &Ds_spec,
@@ -210,7 +209,7 @@ public:
 				                 const Gamma GammaB_sigma,
 		                 		 const Gamma GammaB_nucl,
 						 robj &result);
-  template <class mobj, class mobj2, class robj>
+  template <class mobj, class mobj2, class robj> accelerator_inline
   static void XiToSigmaQ2EyeSite(const mobj &Dq_loop,
 						 const mobj2 &Dd_spec,
 						 const mobj2 &Ds_spec,
@@ -1395,7 +1394,7 @@ void BaryonUtils<FImpl>::SigmaToNucleonNonEye(const PropagatorField &qq_ti,
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
 						 const mobj2 &Dd_spec,
 						 const mobj2 &Ds_spec,
@@ -1464,7 +1463,7 @@ void BaryonUtils<FImpl>::XiToSigmaQ1EyeSite(const mobj &Dq_loop,
  * Dd_tf is a quark line from t_f to t_H
  * Ds_ti is a quark line from t_i to t_H */
 template <class FImpl>
-template <class mobj, class mobj2, class robj>
+template <class mobj, class mobj2, class robj> accelerator_inline
 void BaryonUtils<FImpl>::XiToSigmaQ2EyeSite(const mobj &Dq_loop,
 						 const mobj2 &Dd_spec,
 						 const mobj2 &Ds_spec,

From 7a19432e0b77fff8943ad6701f448ab366dab27f Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Fri, 5 Mar 2021 10:57:09 +0000
Subject: [PATCH 48/82] whitespace

---
 Grid/qcd/utils/BaryonUtils.h | 84 ++++++++++++++++++------------------
 1 file changed, 42 insertions(+), 42 deletions(-)

diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h
index b69865e8..9d9cb508 100644
--- a/Grid/qcd/utils/BaryonUtils.h
+++ b/Grid/qcd/utils/BaryonUtils.h
@@ -191,34 +191,34 @@ public:
              robj &result);
   template <class mobj, class mobj2, class robj> accelerator_inline
   static void SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti,
-						 const mobj &Du_tf,
-						 const mobj2 &Du_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result);
+	     const mobj &Du_tf,
+	     const mobj2 &Du_spec,
+	     const mobj &Dd_tf,
+	     const mobj &Ds_ti,
+	     const Gamma Gamma_H,
+	     const Gamma GammaB_sigma,
+	     const Gamma GammaB_nucl,
+	     robj &result);
   template <class mobj, class mobj2, class robj> accelerator_inline
   static void XiToSigmaQ1EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Dd_spec,
-						 const mobj2 &Ds_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result);
+	     const mobj2 &Dd_spec,
+	     const mobj2 &Ds_spec,
+	     const mobj &Dd_tf,
+	     const mobj &Ds_ti,
+	     const Gamma Gamma_H,
+	     const Gamma GammaB_sigma,
+	     const Gamma GammaB_nucl,
+	     robj &result);
   template <class mobj, class mobj2, class robj> accelerator_inline
   static void XiToSigmaQ2EyeSite(const mobj &Dq_loop,
-						 const mobj2 &Dd_spec,
-						 const mobj2 &Ds_spec,
-						 const mobj &Dd_tf,
-						 const mobj &Ds_ti,
-				                 const Gamma Gamma_H,
-				                 const Gamma GammaB_sigma,
-		                 		 const Gamma GammaB_nucl,
-						 robj &result);
+	     const mobj2 &Dd_spec,
+	     const mobj2 &Ds_spec,
+	     const mobj &Dd_tf,
+	     const mobj &Ds_ti,
+	     const Gamma Gamma_H,
+	     const Gamma GammaB_sigma,
+	     const Gamma GammaB_nucl,
+	     robj &result);
   public:
   template <class mobj>
   static void SigmaToNucleonEye(const PropagatorField &qq_loop,
@@ -232,26 +232,26 @@ public:
          SpinMatrixField &stn_corr);
   template <class mobj>
   static void SigmaToNucleonNonEye(const PropagatorField &qq_ti,
-				 const PropagatorField &qq_tf,
-				 const mobj &Du_spec,
-				 const PropagatorField &qd_tf,
-				 const PropagatorField &qs_ti,
-				 const Gamma Gamma_H,
-				 const Gamma GammaB_sigma,
-				 const Gamma GammaB_nucl,
-		                 const std::string op,
-				 SpinMatrixField &stn_corr);
+	 const PropagatorField &qq_tf,
+	 const mobj &Du_spec,
+	 const PropagatorField &qd_tf,
+	 const PropagatorField &qs_ti,
+	 const Gamma Gamma_H,
+	 const Gamma GammaB_sigma,
+	 const Gamma GammaB_nucl,
+         const std::string op,
+	 SpinMatrixField &stn_corr);
   template <class mobj>
   static void XiToSigmaEye(const PropagatorField &qq_loop,
-				 const mobj &Dd_spec,
-				 const mobj &Ds_spec,
-				 const PropagatorField &qd_tf,
-				 const PropagatorField &qs_ti,
-				 const Gamma Gamma_H,
-				 const Gamma GammaB_sigma,
-				 const Gamma GammaB_nucl,
-		                 const std::string op,
-				 SpinMatrixField &xts_corr);
+	 const mobj &Dd_spec,
+	 const mobj &Ds_spec,
+	 const PropagatorField &qd_tf,
+	 const PropagatorField &qs_ti,
+	 const Gamma Gamma_H,
+	 const Gamma GammaB_sigma,
+	 const Gamma GammaB_nucl,
+         const std::string op,
+	 SpinMatrixField &xts_corr);
 };
 //This computes a baryon contraction on a lattice site, including the spin-trace of the correlation matrix
 template <class FImpl>

From aa173e29989b93445cc28655dd7d0feadb746961 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@users.noreply.github.com>
Date: Fri, 5 Mar 2021 10:25:33 -0500
Subject: [PATCH 49/82] Update README.md

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index 4cbae720..fff68dc6 100644
--- a/README.md
+++ b/README.md
@@ -149,7 +149,6 @@ If you want to build all the tests at once just use `make tests`.
 - `--enable-numa`: enable NUMA first touch optimisation
 - `--enable-simd=<code>`: setup Grid for the SIMD target `<code>` (default: `GEN`). A list of possible SIMD targets is detailed in a section below.
 - `--enable-gen-simd-width=<size>`: select the size (in bytes) of the generic SIMD vector type (default: 32 bytes).
-- `--enable-precision={single|double}`: set the default precision (default: `double`). **Deprecated option**
 - `--enable-comms=<comm>`: Use `<comm>` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below.
 - `--enable-rng={sitmo|ranlux48|mt19937}`: choose the RNG (default: `sitmo `).
 - `--disable-timers`: disable system dependent high-resolution timers.

From b24181aa4f21a59e601237947c5ea465d0e1ecaf Mon Sep 17 00:00:00 2001
From: Christoph Lehner <christoph@lhnr.de>
Date: Fri, 5 Mar 2021 16:56:58 +0100
Subject: [PATCH 50/82] Update Coordinate.h

Revert GRID_MAX_SIMD change
---
 Grid/util/Coordinate.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/util/Coordinate.h b/Grid/util/Coordinate.h
index 89f73264..004fbc72 100644
--- a/Grid/util/Coordinate.h
+++ b/Grid/util/Coordinate.h
@@ -88,7 +88,7 @@ public:
 // Coordinate class, maxdims = 8 for now.
 ////////////////////////////////////////////////////////////////
 #define GRID_MAX_LATTICE_DIMENSION (8)
-#define GRID_MAX_SIMD              (sizeof(vInteger)/sizeof(Integer))
+#define GRID_MAX_SIMD              (16)
 
 static constexpr int MaxDims = GRID_MAX_LATTICE_DIMENSION;
 

From a76cb005e007bf887e0573175f9e9806fecc61b1 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@users.noreply.github.com>
Date: Mon, 8 Mar 2021 13:37:57 -0500
Subject: [PATCH 51/82] Update Tensor_exp.h

---
 Grid/tensors/Tensor_exp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/tensors/Tensor_exp.h b/Grid/tensors/Tensor_exp.h
index 0a1d6389..1f637d5f 100644
--- a/Grid/tensors/Tensor_exp.h
+++ b/Grid/tensors/Tensor_exp.h
@@ -28,7 +28,7 @@ Author: neo <cossu@post.kek.jp>
 #ifndef GRID_MATH_EXP_H
 #define GRID_MATH_EXP_H
 
-#define DEFAULT_MAT_EXP 12
+#define DEFAULT_MAT_EXP 20
 
 NAMESPACE_BEGIN(Grid);
 

From 4d1ea15c79d30c961272d5020404395db44365dc Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Tue, 9 Mar 2021 04:29:37 +0100
Subject: [PATCH 52/82] More verbosity. The 16bit limit on Grid.y, Grid.z is
 annoying

---
 Grid/threads/Accelerator.cc | 3 +--
 Grid/threads/Accelerator.h  | 7 ++++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Grid/threads/Accelerator.cc b/Grid/threads/Accelerator.cc
index 4bf7f395..9d9d851c 100644
--- a/Grid/threads/Accelerator.cc
+++ b/Grid/threads/Accelerator.cc
@@ -53,7 +53,6 @@ void acceleratorInit(void)
     prop = gpu_props[i];
     totalDeviceMem = prop.totalGlobalMem;
     if ( world_rank == 0) {
-#ifndef GRID_DEFAULT_GPU
       if ( i==rank ) {
 	printf("AcceleratorCudaInit[%d]: ========================\n",rank);
 	printf("AcceleratorCudaInit[%d]: Device Number    : %d\n", rank,i);
@@ -67,8 +66,8 @@ void acceleratorInit(void)
 	GPU_PROP(warpSize);
 	GPU_PROP(pciBusID);
 	GPU_PROP(pciDeviceID);
+ 	printf("AcceleratorCudaInit[%d]: maxGridSize (%d,%d,%d)\n",rank,prop.maxGridSize[0],prop.maxGridSize[1],prop.maxGridSize[2]);
       }
-#endif
       //      GPU_PROP(unifiedAddressing);
       //      GPU_PROP(l2CacheSize);
       //      GPU_PROP(singleToDoublePrecisionPerfRatio);
diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h
index f1a694fb..56b85c72 100644
--- a/Grid/threads/Accelerator.h
+++ b/Grid/threads/Accelerator.h
@@ -178,9 +178,10 @@ void Lambda6Apply(uint64_t num1, uint64_t num2, uint64_t num3,
     cudaDeviceSynchronize();						\
     cudaError err = cudaGetLastError();					\
     if ( cudaSuccess != err ) {						\
-      printf("Cuda error %s \n", cudaGetErrorString( err ));		\
-      puts(__FILE__);							\
-      printf("Line %d\n",__LINE__);					\
+      printf("accelerator_barrier(): Cuda error %s \n",			\
+	     cudaGetErrorString( err ));				\
+      printf("File %s Line %d\n",__FILE__,__LINE__);			\
+      fflush(stdout);							\
       if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);		\
     }									\
   }

From 6a429ee6d3eee71689f650782decb8948bf6fa77 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Tue, 9 Mar 2021 04:31:10 +0100
Subject: [PATCH 53/82] 2d loop hits Nvidia 16bit limit on large local vols

---
 Grid/cshift/Cshift_common.h | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/Grid/cshift/Cshift_common.h b/Grid/cshift/Cshift_common.h
index f2f39815..cf902b58 100644
--- a/Grid/cshift/Cshift_common.h
+++ b/Grid/cshift/Cshift_common.h
@@ -110,9 +110,11 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
   int n1=rhs.Grid()->_slice_stride[dimension];
 
   if ( cbmask ==0x3){
-#ifdef ACCELERATOR_CSHIFT    
+#ifdef ACCELERATOR_CSHIFT
     autoView(rhs_v , rhs, AcceleratorRead);
-    accelerator_for2d(n,e1,b,e2,1,{
+    accelerator_for(nn,e1*e2,1,{
+	int n = nn%e1;
+	int b = nn/e1;
 	int o      =   n*n1;
 	int offset = b+n*e2;
 	
@@ -135,7 +137,9 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
     std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
 #ifdef ACCELERATOR_CSHIFT    
     autoView(rhs_v , rhs, AcceleratorRead);
-    accelerator_for2d(n,e1,b,e2,1,{
+    accelerator_for(nn,e1*e2,1,{
+	int n = nn%e1;
+	int b = nn/e1;
 
 	Coordinate coor;
 
@@ -257,7 +261,9 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
     int _slice_block = rhs.Grid()->_slice_block[dimension];
 #ifdef ACCELERATOR_CSHIFT    
     autoView( rhs_v , rhs, AcceleratorWrite);
-    accelerator_for2d(n,e1,b,e2,1,{
+    accelerator_for(nn,e1*e2,1,{
+	int n = nn%e1;
+	int b = nn/e1;
 	int o      = n*_slice_stride;
 	int offset = b+n*_slice_block;
 	merge(rhs_v[so+o+b],pointers,offset);
@@ -274,7 +280,7 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
 
     // Case of SIMD split AND checker dim cannot currently be hit, except in 
     // Test_cshift_red_black code.
-    //    std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
+    std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
     std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
     assert(0); // This will fail if hit on GPU
     autoView( rhs_v, rhs, CpuWrite);

From 58bf9b9e6d14ba2249ece02c74336c402e92b694 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Wed, 10 Mar 2021 02:45:22 +0100
Subject: [PATCH 54/82] Clean up test

---
 tests/core/Test_where.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/core/Test_where.cc b/tests/core/Test_where.cc
index 050b711b..deb29865 100644
--- a/tests/core/Test_where.cc
+++ b/tests/core/Test_where.cc
@@ -40,9 +40,9 @@ int main (int argc, char ** argv)
 
   int N=16;
   
-  std::vector<int> latt_size  ({N,4,4});
-  std::vector<int> simd_layout({vComplexD::Nsimd(),1,1});
-  std::vector<int> mpi_layout ({1,1,1});
+  std::vector<int> latt_size  ({N,N,N,N});
+  std::vector<int> simd_layout({vComplexD::Nsimd(),1,1,1});
+  std::vector<int> mpi_layout ({1,1,1,1});
 
   int vol = 1;
   int nd  = latt_size.size();
@@ -69,7 +69,7 @@ int main (int argc, char ** argv)
     for(int t=0;t<latt_size[mu];t++){
       LatticeCoordinate(coor,mu);
       sl=where(coor==Integer(t),rn,zz);
-      std::cout <<GridLogMessage<< " sl " << sl<<std::endl;
+      //      std::cout <<GridLogMessage<< " sl " << sl<<std::endl;
       std::cout <<GridLogMessage<<" slice "<<t<<" " << norm2(sl)<<std::endl;
       ns=ns+norm2(sl);
     }

From 0e21adb3f62cc28b0ee51e9304065da8dd7a676d Mon Sep 17 00:00:00 2001
From: u61464 <u61464@login-2.aidevcloud>
Date: Wed, 10 Mar 2021 05:40:51 -0800
Subject: [PATCH 55/82] Gives 200GF/s on SyCL/DG1 8^4, doesn't uglify develop
 for other platforms too badly. Easy to revert to clean more C++ stylistic
 code. Theres a SYCL_HACK macro I will clean up later once dpcpp evolves a
 central nervous systems.

---
 Grid/qcd/action/fermion/WilsonKernels.h       | 10 +-
 .../WilsonKernelsHandImplementation.h         | 99 ++++++++++++++++---
 .../WilsonKernelsImplementation.h             | 20 +++-
 3 files changed, 115 insertions(+), 14 deletions(-)

diff --git a/Grid/qcd/action/fermion/WilsonKernels.h b/Grid/qcd/action/fermion/WilsonKernels.h
index 1bac9211..68422f28 100644
--- a/Grid/qcd/action/fermion/WilsonKernels.h
+++ b/Grid/qcd/action/fermion/WilsonKernels.h
@@ -49,9 +49,17 @@ public:
 
   INHERIT_IMPL_TYPES(Impl);
   typedef FermionOperator<Impl> Base;
-   
+  typedef AcceleratorVector<int,STENCIL_MAX> StencilVector;   
 public:
 
+#ifdef GRID_SYCL
+#define SYCL_HACK
+#endif  
+#ifdef SYCL_HACK
+  static void HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor  *buf,
+			       int ss,int sU,const SiteSpinor *in, SiteSpinor *out);
+#endif
+  
   static void DhopKernel(int Opt,StencilImpl &st,  DoubledGaugeField &U, SiteHalfSpinor * buf,
 			 int Ls, int Nsite, const FermionField &in, FermionField &out,
 			 int interior=1,int exterior=1) ;
diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
index 688cb75a..fb42fe88 100644
--- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
@@ -118,15 +118,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #endif
 
-#define LOAD_CHI				\
-  {const SiteHalfSpinor &ref(buf[offset]);	\
-    Chi_00 = coalescedRead(ref()(0)(0));	\
-    Chi_01 = coalescedRead(ref()(0)(1));	\
-    Chi_02 = coalescedRead(ref()(0)(2));	\
-    Chi_10 = coalescedRead(ref()(1)(0));	\
-    Chi_11 = coalescedRead(ref()(1)(1));	\
-    Chi_12 = coalescedRead(ref()(1)(2));}
-
 #define MULT_2SPIN(A)\
   {auto & ref(U[sU](A));					\
   U_00=coalescedRead(ref()(0,0));				\
@@ -157,6 +148,15 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     UChi_02+= U_20*Chi_02;					\
     UChi_12+= U_20*Chi_12;}
 
+#define LOAD_CHI				\
+  {const SiteHalfSpinor &ref(buf[offset]);	\
+    Chi_00 = coalescedRead(ref()(0)(0));	\
+    Chi_01 = coalescedRead(ref()(0)(1));	\
+    Chi_02 = coalescedRead(ref()(0)(2));	\
+    Chi_10 = coalescedRead(ref()(1)(0));	\
+    Chi_11 = coalescedRead(ref()(1)(1));	\
+    Chi_12 = coalescedRead(ref()(1)(2));}
+
 //      hspin(0)=fspin(0)+timesI(fspin(3));
 //      hspin(1)=fspin(1)+timesI(fspin(2));
 #define XP_PROJ \
@@ -370,7 +370,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   result_31-= UChi_11;	\
   result_32-= UChi_12;
 
-#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON)	\
+#define HAND_STENCIL_LEGB(PROJ,PERM,DIR,RECON)	\
   SE=st.GetEntry(ptype,DIR,ss);			\
   offset = SE->_offset;				\
   local  = SE->_is_local;			\
@@ -384,6 +384,37 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   } else {					\
     LOAD_CHI;					\
   }						\
+  acceleratorSynchronise();			\
+  MULT_2SPIN(DIR);				\
+  RECON;					
+
+#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON)	\
+  SE=&st_p[DIR+8*ss];				\
+  ptype=st_perm[DIR];				\
+  offset = SE->_offset;				\
+  local  = SE->_is_local;			\
+  perm   = SE->_permute;			\
+  if ( local ) {				\
+    LOAD_CHIMU(PERM);				\
+    PROJ;					\
+    if ( perm) {				\
+      PERMUTE_DIR(PERM);			\
+    }						\
+  } else {					\
+    LOAD_CHI;					\
+  }						\
+  acceleratorSynchronise();			\
+  MULT_2SPIN(DIR);				\
+  RECON;					
+
+#define HAND_STENCIL_LEGA(PROJ,PERM,DIR,RECON)				\
+  SE=&st_p[DIR+8*ss];							\
+  ptype=st_perm[DIR];							\
+ /*SE=st.GetEntry(ptype,DIR,ss);*/					\
+  offset = SE->_offset;				\
+  perm   = SE->_permute;			\
+  LOAD_CHIMU(PERM);				\
+  PROJ;						\
   MULT_2SPIN(DIR);				\
   RECON;					
 
@@ -401,10 +432,12 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   } else if ( st.same_node[DIR] ) {		\
     LOAD_CHI;					\
   }						\
+  acceleratorSynchronise();			\
   if (local || st.same_node[DIR] ) {		\
     MULT_2SPIN(DIR);				\
     RECON;					\
-  }
+  }						\
+  acceleratorSynchronise();			
 
 #define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON)	\
   SE=st.GetEntry(ptype,DIR,ss);			\
@@ -414,7 +447,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     MULT_2SPIN(DIR);				\
     RECON;					\
     nmu++;					\
-  }
+  }						\
+  acceleratorSynchronise();			
 
 #define HAND_RESULT(ss)				\
   {						\
@@ -511,10 +545,41 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 NAMESPACE_BEGIN(Grid);
 
+
+#ifdef SYCL_HACK
+template<class Impl> accelerator_inline void 
+WilsonKernels<Impl>::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor  *buf,
+				      int ss,int sU,const SiteSpinor *in, SiteSpinor *out)
+{
+// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+  typedef iSinglet<Simd> vCplx;
+  //  typedef decltype( coalescedRead( vCplx()()() )) Simt;
+  typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+
+  HAND_DECLARATIONS(Simt);
+
+  int offset,local,perm, ptype;
+  StencilEntry *SE;
+  HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
+  HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
+  HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
+  HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
+  HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
+  HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
+  HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
+  HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
+  HAND_RESULT(ss);
+}
+#endif
+
 template<class Impl> accelerator_inline void 
 WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 				  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
@@ -539,6 +604,8 @@ template<class Impl>  accelerator_inline
 void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
@@ -562,6 +629,8 @@ template<class Impl>  accelerator_inline void
 WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
@@ -586,6 +655,8 @@ template<class Impl> accelerator_inline
 void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
@@ -609,6 +680,8 @@ template<class Impl>  accelerator_inline void
 WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
@@ -634,6 +707,8 @@ template<class Impl>  accelerator_inline
 void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h
index 937d13af..9228b84c 100644
--- a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h
@@ -416,7 +416,21 @@ void WilsonKernels<Impl>::DhopDirKernel( StencilImpl &st, DoubledGaugeField &U,S
 #undef LoopBody
 }
 
-#define KERNEL_CALLNB(A) \
+#define KERNEL_CALL_TMP(A) \
+  const uint64_t    NN = Nsite*Ls;					\
+  auto U_p = & U_v[0];							\
+  auto in_p = & in_v[0];						\
+  auto out_p = & out_v[0];						\
+  auto st_p = st_v._entries_p;						\
+  auto st_perm = st_v._permute_type;					\
+  accelerator_forNB( ss, NN, Simd::Nsimd(), {				\
+      int sF = ss;							\
+      int sU = ss/Ls;							\
+      WilsonKernels<Impl>::A(st_perm,st_p,U_p,buf,sF,sU,in_p,out_p);	\
+    });									\
+  accelerator_barrier();
+
+#define KERNEL_CALLNB(A)						\
   const uint64_t    NN = Nsite*Ls;					\
   accelerator_forNB( ss, NN, Simd::Nsimd(), {				\
       int sF = ss;							\
@@ -445,7 +459,11 @@ void WilsonKernels<Impl>::DhopKernel(int Opt,StencilImpl &st,  DoubledGaugeField
 
    if( interior && exterior ) {
      if (Opt == WilsonKernelsStatic::OptGeneric    ) { KERNEL_CALL(GenericDhopSite); return;}
+#ifdef SYCL_HACK     
+     if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL_TMP(HandDhopSiteSycl);    return; }
+#else
      if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSite);    return;}
+#endif     
 #ifndef GRID_CUDA
      if (Opt == WilsonKernelsStatic::OptInlineAsm  ) {  ASM_CALL(AsmDhopSite);    return;}
 #endif

From f786ff8d69534e7885a55a7090a26b4d28619209 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Wed, 10 Mar 2021 14:32:06 -0500
Subject: [PATCH 56/82] Extend test from Fionn, fails on A100 apparently

---
 tests/core/Test_where_extended.cc | 137 ++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)
 create mode 100644 tests/core/Test_where_extended.cc

diff --git a/tests/core/Test_where_extended.cc b/tests/core/Test_where_extended.cc
new file mode 100644
index 00000000..706fd7ee
--- /dev/null
+++ b/tests/core/Test_where_extended.cc
@@ -0,0 +1,137 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_poisson_fft.cc
+
+    Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace Grid;
+ ;
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+
+  int N=16;
+  
+  std::vector<int> latt_size  ({N,4,4});
+  std::vector<int> simd_layout({vComplexD::Nsimd(),1,1});
+  std::vector<int> mpi_layout ({1,1,1});
+
+  int vol = 1;
+  int nd  = latt_size.size();
+  for(int d=0;d<nd;d++){
+    vol = vol * latt_size[d];
+  }
+
+  GridCartesian         GRID(latt_size,simd_layout,mpi_layout);
+
+  LatticeComplexD      zz(&GRID);
+  LatticeInteger     coor(&GRID);
+  LatticeComplexD  rn(&GRID);
+  LatticeComplexD  sl(&GRID);
+
+  zz  = ComplexD(0.0,0.0);
+
+  GridParallelRNG RNG(&GRID);
+  RNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));  
+  gaussian(RNG,rn);
+
+  RealD nn=norm2(rn);
+  for(int mu=0;mu<nd;mu++){
+    RealD ns=0.0;
+    for(int t=0;t<latt_size[mu];t++){
+      LatticeCoordinate(coor,mu);
+      sl=where(coor==Integer(t),rn,zz);
+      std::cout <<GridLogMessage<< " sl " << sl<<std::endl;
+      std::cout <<GridLogMessage<<" slice "<<t<<" " << norm2(sl)<<std::endl;
+      ns=ns+norm2(sl);
+    }
+    std::cout <<GridLogMessage <<" sliceNorm" <<mu<<" "<< nn <<" "<<ns<<" " << nn-ns<<std::endl;
+  }
+
+  unsigned int tmin = 3;
+
+  int Ls = 2;
+  GridCartesian * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  std::vector<int> seeds4({1,2,3,4});
+  GridParallelRNG  RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
+
+
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"== LatticeFermion =="<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+
+  LatticeFermion  q_outF(FGrid); q_outF=0.0;
+  LatticeFermion  tmpF(UGrid); random(RNG4,tmpF);
+  LatticeFermion  tmp2F(UGrid);
+  LatticeFermion  ZZF (UGrid);   ZZF=0.0;
+
+  RealD nA=0.0;
+  RealD nB=0.0;
+  for(int s=0;s<Ls;s++){
+    nB = nB + norm2(tmpF); 
+    tmp2F   = where((lcoor>=tmin),tmpF,ZZF);
+    nA = nA + norm2(tmp2F); 
+    InsertSlice(tmp2F, q_outF, s , 0);
+  }
+
+  RealD nQO=norm2(q_outF);
+  std::cout <<GridLogMessage << "norm_before_where: " << nB << std::endl;
+  std::cout <<GridLogMessage << "norm_after_where: " << nA << std::endl;
+  std::cout <<GridLogMessage << "norm_q_out: " << nQO << std::endl;
+
+
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"== LatticePropagator =="<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+
+  LatticePropagator  q_outP(FGrid); q_outP=0.0;
+  LatticePropagator  tmpP(UGrid); random(RNG4,tmpP);
+  LatticePropagator  tmp2P(UGrid);
+  LatticePropagator  ZZP (UGrid);   ZZP=0.0;
+
+  nA=0.0;
+  nB=0.0;
+  for(int s=0;s<Ls;s++){
+    nB = nB + norm2(tmpP); 
+    tmp2P   = where((lcoor>=tmin),tmpP,ZZP);
+    nA = nA + norm2(tmp2P); 
+    InsertSlice(tmp2P, q_outP, s , 0);
+  }
+
+  nQO=norm2(q_outP);
+  std::cout <<GridLogMessage << "norm_before_where: " << nB << std::endl;
+  std::cout <<GridLogMessage << "norm_after_where: " << nA << std::endl;
+  std::cout <<GridLogMessage << "norm_q_out: " << nQO << std::endl;
+
+  Grid_finalize();
+}

From d9c4afe5b7a183fec9f7cf6aa99e3534cd258feb Mon Sep 17 00:00:00 2001
From: Peter Georg <peter.georg@physik.uni-regensburg.de>
Date: Thu, 11 Mar 2021 12:58:49 +0100
Subject: [PATCH 57/82] Fix inconsistent configure option AVX512

Before this change AVX512 enabled different instruction sets depending
on the compiler:

For Intel C++ Compiler Classic (ICC):
    AVX512F, AVX512CD, AVX512DQ, AVX512BW, AVX512VL
    i.e. Intel Xeon Skylake and newer

For Intel ICX, gcc, clang:
    AVX512F, AVX512CD, AVX512ER, AVX512PF
    i.e. Intel Xeon Phi x200/x205 (KNL/KNM)

With this commit AVX512 now only enables the common instruction sets
supported by all CPUs supporting any AVX-512 instructions set:
AVX512F and AVX512CD (called COMMON-AVX512 by icc)
---
 configure.ac | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 5f165412..afd5cbad 100644
--- a/configure.ac
+++ b/configure.ac
@@ -444,7 +444,7 @@ case ${ax_cv_cxx_compiler_vendor} in
         SIMD_FLAGS='-mavx2 -mfma -mf16c';;
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
-        SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
+        SIMD_FLAGS='-mavx512f -mavx512cd';;
       SKL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics for SkyLake Xeon])
         SIMD_FLAGS='-march=skylake-avx512';;
@@ -498,7 +498,7 @@ case ${ax_cv_cxx_compiler_vendor} in
         SIMD_FLAGS='-march=core-avx2 -xcore-avx2';;
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
-        SIMD_FLAGS='-xcore-avx512';;
+        SIMD_FLAGS='-xcommon-avx512';;
       KNC)
         AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner])
         SIMD_FLAGS='';;

From 82402c6a7cc3057dd0e4c2f6389ba2b347972cfc Mon Sep 17 00:00:00 2001
From: Peter Georg <peter.georg@physik.uni-regensburg.de>
Date: Thu, 11 Mar 2021 13:08:40 +0100
Subject: [PATCH 58/82] Add simd option SKL for ICC

---
 configure.ac | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/configure.ac b/configure.ac
index afd5cbad..4e5e33c8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -499,6 +499,9 @@ case ${ax_cv_cxx_compiler_vendor} in
       AVX512)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-xcommon-avx512';;
+      SKL)
+        AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
+        SIMD_FLAGS='-xcore-avx512';;
       KNC)
         AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner])
         SIMD_FLAGS='';;

From ce1fc1f48aac5a48512b6ff70c1d85d0810d7623 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 11 Mar 2021 22:20:53 +0100
Subject: [PATCH 59/82] Possible fallback plan for Fionn's compiler bbug in
 nvcc

---
 Grid/lattice/Lattice_where.h | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/Grid/lattice/Lattice_where.h b/Grid/lattice/Lattice_where.h
index 6686d1b3..e9a3f823 100644
--- a/Grid/lattice/Lattice_where.h
+++ b/Grid/lattice/Lattice_where.h
@@ -43,7 +43,7 @@ inline void whereWolf(Lattice<vobj> &ret,const Lattice<iobj> &predicate,Lattice<
   conformable(iftrue,predicate);
   conformable(iftrue,ret);
 
-  GridBase *grid=iftrue._grid;
+  GridBase *grid=iftrue.Grid();
 
   typedef typename vobj::scalar_object scalar_object;
   typedef typename vobj::scalar_type scalar_type;
@@ -52,22 +52,24 @@ inline void whereWolf(Lattice<vobj> &ret,const Lattice<iobj> &predicate,Lattice<
 
   const int Nsimd = grid->Nsimd();
 
-  std::vector<Integer> mask(Nsimd);
-  std::vector<scalar_object> truevals (Nsimd);
-  std::vector<scalar_object> falsevals(Nsimd);
+  Integer mask;
+  scalar_object trueval;
+  scalar_object falseval;
 
-  parallel_for(int ss=0;ss<iftrue._grid->oSites(); ss++){
-
-    extract(iftrue._odata[ss]   ,truevals);
-    extract(iffalse._odata[ss]  ,falsevals);
-    extract<vInteger,Integer>(TensorRemove(predicate._odata[ss]),mask);
-
-    for(int s=0;s<Nsimd;s++){
-      if (mask[s]) falsevals[s]=truevals[s];
+  autoView(iftrue_v,iftrue,CpuRead);
+  autoView(iffalse_v,iffalse,CpuRead);
+  autoView(predicate_v,predicate,CpuRead);
+  autoView(ret_v,ret,CpuWrite);
+  Integer NN= grid->oSites();
+  thread_for(ss,NN,{
+    for(int l=0;l<Nsimd;l++){
+      trueval =extractLane(l,iftrue_v[ss]);
+      falseval=extractLane(l,iffalse_v[ss]);
+      mask    =extractLane(l,predicate_v[ss]);
+      if (mask) falseval=trueval;
+      insertLane(l,ret_v[ss],falseval);
     }
-
-    merge(ret._odata[ss],falsevals);
-  }
+  });
 }
 
 template<class vobj,class iobj>
@@ -76,9 +78,9 @@ inline Lattice<vobj> whereWolf(const Lattice<iobj> &predicate,Lattice<vobj> &ift
   conformable(iftrue,iffalse);
   conformable(iftrue,predicate);
 
-  Lattice<vobj> ret(iftrue._grid);
+  Lattice<vobj> ret(iftrue.Grid());
 
-  where(ret,predicate,iftrue,iffalse);
+  whereWolf(ret,predicate,iftrue,iffalse);
 
   return ret;
 }

From cd5891eecd45b53db3a02498d97d0bd5e29c67eb Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 11 Mar 2021 22:34:28 +0100
Subject: [PATCH 60/82] Test that fails on Cuda 11.0

---
 tests/core/Test_where_extended.cc | 124 ++++++++++++++++--------------
 1 file changed, 65 insertions(+), 59 deletions(-)

diff --git a/tests/core/Test_where_extended.cc b/tests/core/Test_where_extended.cc
index 706fd7ee..9862b3ed 100644
--- a/tests/core/Test_where_extended.cc
+++ b/tests/core/Test_where_extended.cc
@@ -51,87 +51,93 @@ int main (int argc, char ** argv)
   }
 
   GridCartesian         GRID(latt_size,simd_layout,mpi_layout);
-
-  LatticeComplexD      zz(&GRID);
-  LatticeInteger     coor(&GRID);
-  LatticeComplexD  rn(&GRID);
-  LatticeComplexD  sl(&GRID);
-
-  zz  = ComplexD(0.0,0.0);
-
   GridParallelRNG RNG(&GRID);
   RNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));  
-  gaussian(RNG,rn);
 
-  RealD nn=norm2(rn);
-  for(int mu=0;mu<nd;mu++){
-    RealD ns=0.0;
-    for(int t=0;t<latt_size[mu];t++){
-      LatticeCoordinate(coor,mu);
-      sl=where(coor==Integer(t),rn,zz);
-      std::cout <<GridLogMessage<< " sl " << sl<<std::endl;
-      std::cout <<GridLogMessage<<" slice "<<t<<" " << norm2(sl)<<std::endl;
-      ns=ns+norm2(sl);
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  std::cout<<GridLogMessage<<"== LatticeComplex =="<<std::endl;
+  std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  {
+    LatticeComplexD      zz(&GRID);
+    LatticeInteger     coor(&GRID);
+    LatticeComplexD  rn(&GRID);
+    LatticeComplexD  sl(&GRID);
+
+    zz  = ComplexD(0.0,0.0);
+
+    gaussian(RNG,rn);
+    
+    RealD nn=norm2(rn);
+    for(int mu=0;mu<nd;mu++){
+      RealD ns=0.0;
+      for(int t=0;t<latt_size[mu];t++){
+	LatticeCoordinate(coor,mu);
+	sl=where(coor==Integer(t),rn,zz);
+	std::cout <<GridLogMessage<<" slice "<<t<<" " << norm2(sl)<<std::endl;
+	ns=ns+norm2(sl);
+      }
+      std::cout <<GridLogMessage <<" sliceNorm" <<mu<<" "<< nn <<" "<<ns<<" err " << nn-ns<<std::endl;
+      assert(abs(nn-ns) < 1.0e-10);
     }
-    std::cout <<GridLogMessage <<" sliceNorm" <<mu<<" "<< nn <<" "<<ns<<" " << nn-ns<<std::endl;
   }
 
-  unsigned int tmin = 3;
-
-  int Ls = 2;
-  GridCartesian * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
-  GridCartesian * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
-  std::vector<int> seeds4({1,2,3,4});
-  GridParallelRNG  RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
-  LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1);
-
 
   std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
   std::cout<<GridLogMessage<<"== LatticeFermion =="<<std::endl;
   std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
+  {
+    LatticeFermionD      zz(&GRID);
+    LatticeInteger     coor(&GRID);
+    LatticeFermionD  rn(&GRID);
+    LatticeFermionD  sl(&GRID);
 
-  LatticeFermion  q_outF(FGrid); q_outF=0.0;
-  LatticeFermion  tmpF(UGrid); random(RNG4,tmpF);
-  LatticeFermion  tmp2F(UGrid);
-  LatticeFermion  ZZF (UGrid);   ZZF=0.0;
+    zz  = ComplexD(0.0,0.0);
 
-  RealD nA=0.0;
-  RealD nB=0.0;
-  for(int s=0;s<Ls;s++){
-    nB = nB + norm2(tmpF); 
-    tmp2F   = where((lcoor>=tmin),tmpF,ZZF);
-    nA = nA + norm2(tmp2F); 
-    InsertSlice(tmp2F, q_outF, s , 0);
+    gaussian(RNG,rn);
+    
+    RealD nn=norm2(rn);
+    for(int mu=0;mu<nd;mu++){
+      RealD ns=0.0;
+      for(int t=0;t<latt_size[mu];t++){
+	LatticeCoordinate(coor,mu);
+	sl=where(coor==Integer(t),rn,zz);
+	std::cout <<GridLogMessage<<" slice "<<t<<" " << norm2(sl)<<std::endl;
+	ns=ns+norm2(sl);
+      }
+      std::cout <<GridLogMessage <<" sliceNorm" <<mu<<" "<< nn <<" "<<ns<<" err " << nn-ns<<std::endl;
+      assert(abs(nn-ns) < 1.0e-10);
+    }
   }
 
-  RealD nQO=norm2(q_outF);
-  std::cout <<GridLogMessage << "norm_before_where: " << nB << std::endl;
-  std::cout <<GridLogMessage << "norm_after_where: " << nA << std::endl;
-  std::cout <<GridLogMessage << "norm_q_out: " << nQO << std::endl;
 
 
   std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
   std::cout<<GridLogMessage<<"== LatticePropagator =="<<std::endl;
   std::cout<<GridLogMessage<<"=============================================================="<<std::endl;
 
-  LatticePropagator  q_outP(FGrid); q_outP=0.0;
-  LatticePropagator  tmpP(UGrid); random(RNG4,tmpP);
-  LatticePropagator  tmp2P(UGrid);
-  LatticePropagator  ZZP (UGrid);   ZZP=0.0;
+  {
+    LatticePropagatorD      zz(&GRID);
+    LatticeInteger     coor(&GRID);
+    LatticePropagatorD  rn(&GRID);
+    LatticePropagatorD  sl(&GRID);
 
-  nA=0.0;
-  nB=0.0;
-  for(int s=0;s<Ls;s++){
-    nB = nB + norm2(tmpP); 
-    tmp2P   = where((lcoor>=tmin),tmpP,ZZP);
-    nA = nA + norm2(tmp2P); 
-    InsertSlice(tmp2P, q_outP, s , 0);
+    zz  = ComplexD(0.0,0.0);
+
+    gaussian(RNG,rn);
+    
+    RealD nn=norm2(rn);
+    for(int mu=0;mu<nd;mu++){
+      RealD ns=0.0;
+      for(int t=0;t<latt_size[mu];t++){
+	LatticeCoordinate(coor,mu);
+	sl=where(coor==Integer(t),rn,zz);
+	std::cout <<GridLogMessage<<" slice "<<t<<" " << norm2(sl)<<std::endl;
+	ns=ns+norm2(sl);
+      }
+      std::cout <<GridLogMessage <<" sliceNorm" <<mu<<" "<< nn <<" "<<ns<<" err " << nn-ns<<std::endl;
+      assert(abs(nn-ns) < 1.0e-10);
+    }
   }
 
-  nQO=norm2(q_outP);
-  std::cout <<GridLogMessage << "norm_before_where: " << nB << std::endl;
-  std::cout <<GridLogMessage << "norm_after_where: " << nA << std::endl;
-  std::cout <<GridLogMessage << "norm_q_out: " << nQO << std::endl;
-
   Grid_finalize();
 }

From da91a884ef9011455dec36c21f6ca5576b9e58ab Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 11 Mar 2021 23:54:53 +0100
Subject: [PATCH 61/82] NVCC versions found buggy added as guard

---
 Grid/util/CompilerCompatible.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Grid/util/CompilerCompatible.h b/Grid/util/CompilerCompatible.h
index 37331668..7c4a056d 100644
--- a/Grid/util/CompilerCompatible.h
+++ b/Grid/util/CompilerCompatible.h
@@ -1,5 +1,16 @@
 #pragma once 
 
+#if defined(__NVCC__)
+
+#if (__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ == 0)
+#error "NVCC version 11.0 breaks on Ampere, see Github issue 346"
+#endif
+#if (__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ == 1)
+#error "NVCC version 11.1 breaks on Ampere, see Github issue 346"
+#endif
+
+#endif
+
 #if defined(__clang__)
 
   #if __clang_major__ < 3

From db3ac67506bcbdbe0799632dcc1561a96d0e320f Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 12 Mar 2021 14:55:07 +0100
Subject: [PATCH 62/82] Update thread issue

---
 Grid/lattice/Lattice_where.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Grid/lattice/Lattice_where.h b/Grid/lattice/Lattice_where.h
index e9a3f823..777f4015 100644
--- a/Grid/lattice/Lattice_where.h
+++ b/Grid/lattice/Lattice_where.h
@@ -52,16 +52,15 @@ inline void whereWolf(Lattice<vobj> &ret,const Lattice<iobj> &predicate,Lattice<
 
   const int Nsimd = grid->Nsimd();
 
-  Integer mask;
-  scalar_object trueval;
-  scalar_object falseval;
-
   autoView(iftrue_v,iftrue,CpuRead);
   autoView(iffalse_v,iffalse,CpuRead);
   autoView(predicate_v,predicate,CpuRead);
   autoView(ret_v,ret,CpuWrite);
   Integer NN= grid->oSites();
   thread_for(ss,NN,{
+    Integer mask;
+    scalar_object trueval;
+    scalar_object falseval;
     for(int l=0;l<Nsimd;l++){
       trueval =extractLane(l,iftrue_v[ss]);
       falseval=extractLane(l,iffalse_v[ss]);

From 001814b4422d9b17a2751d18e23172e5ded79690 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 12 Mar 2021 09:31:17 -0500
Subject: [PATCH 63/82] updated to do list. Start adding DDHMC work items

---
 TODO | 75 ++++++++++++++++++++++++++++--------------------------------
 1 file changed, 35 insertions(+), 40 deletions(-)

diff --git a/TODO b/TODO
index f1175560..e23e040d 100644
--- a/TODO
+++ b/TODO
@@ -1,3 +1,6 @@
+-- comms threads issue??
+-- Part done: Staggered kernel performance on GPU
+
 =========================================================
 General
 =========================================================
@@ -5,28 +8,18 @@ General
 - Make representations code take Gimpl
 - Simplify the HMCand remove modules
 - Lattice_arith - are the mult, mac etc.. still needed after ET engine?
-- Lattice_rng
-- Lattice_transfer.h
-- accelerate A2Autils -- off critical path for HMC
+- Lattice_rng - faster local only loop in init
+- Audit: accelerate A2Autils -- off critical path for HMC
 
 =========================================================
-GPU branch code item work list
+GPU  work list
 =========================================================
 
-* sum_cpu promote to double during summation for increased precisoin.
+* sum_cpu promote to double during summation for increased precision.
 * Introduce sumD & ReduceD 
 * GPU sum is probably better currently.
-
 * Accelerate the cshift & benchmark
 
-* 0) Single GPU
-- 128 bit integer table load in GPU code.
-  - ImprovedStaggered accelerate & measure perf
-  - Gianluca's changes to Cayley into gpu-port
-  - Mobius kernel fusion.                     -- Gianluca?
-  - Lebesque order reintroduction. StencilView should have pointer to it
-  - Lebesgue reorder in all kernels
-
 * 3) Comms/NVlink
 - OpenMP tasks to run comms threads. Experiment with it 
 - Remove explicit openMP in staggered. 
@@ -35,14 +28,6 @@ GPU branch code item work list
 - Stencil gather ??
 - SIMD dirs in stencil
 
-* 4) ET enhancements
-- eval -> scalar ops in ET engine
-- coalescedRead, coalescedWrite in expressions.
-
-* 5) Misc
-- Conserved current clean up.
-- multLinkProp eliminate
-
 8) Merge develop and test HMC
 
 9) Gamma tables on GPU; check this. Appear to work, but no idea why. Are these done on CPU?
@@ -52,7 +37,7 @@ GPU branch code item work list
 -     Audit NAMESPACE CHANGES
 -     Audit changes
 
------
+---------
 Gianluca's changes
 - Performance impact of construct in aligned allocator???
 ---------
@@ -62,6 +47,33 @@ Gianluca's changes
 -----------------------------
 DONE:
 -----------------------------
+=====
+-- Done: Remez X^-1/2 X^-1/2 X = 1 test.
+         Feed in MdagM^2 as a test and take its sqrt.
+         Automated test that MdagM invsqrt(MdagM)invsqrt(MdagM) = 1 in HMC for bounds satisfaction.
+
+-- Done: Sycl Kernels into develop. Compare to existing unroll and just use.
+-- Done: sRNG into refresh functions
+-- Done: Tuned decomposition on CUDA into develop
+-- Done: Sycl friend accessor. Const view attempt via typedef??
+
+
+* Done 5) Misc
+- Conserved current clean up.
+- multLinkProp eliminate
+
+* Done 0) Single GPU
+- 128 bit integer table load in GPU code.
+  - ImprovedStaggered accelerate & measure perf
+  - Gianluca's changes to Cayley into gpu-port
+  - Mobius kernel fusion.                     -- Gianluca?
+  - Lebesque order reintroduction. StencilView should have pointer to it
+  - Lebesgue reorder in all kernels
+
+* 4) ET enhancements
+- Done eval -> scalar ops in ET engine
+- Done coalescedRead, coalescedWrite in expressions.
+
 =============================================================================================
 AUDIT ContractWWVV with respect to develop    -- DONE
 - GPU accelerate EOFA                                                  -- DONE
@@ -125,23 +137,6 @@ AUDIT ContractWWVV with respect to develop    -- DONE
 - -      (4) omp parallel for collapse(n)
 - - Only (1) has a natural mirror in accelerator_loop
 - - Nested loop macros get cumbersome made a generic interface for N deep
-- - Don't like thread_region and thread_loop_in_region
-- - Could replace with 
-
-    thread_nested(1, 
-      for {
-
-      }
-    );
-    thread_nested(2,
-      for (){
-        for (){
-
-	}
-      }
-    );
-
-    and same "in_region".
 
 
 -----------------------------

From 51f506553c59923fc60f3a68333b14757b8853a5 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 12 Mar 2021 15:33:04 +0100
Subject: [PATCH 64/82] Read out the local ID once, and store

---
 .../WilsonKernelsHandImplementation.h         | 165 ++++++++++--------
 1 file changed, 96 insertions(+), 69 deletions(-)

diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
index fb42fe88..0703b613 100644
--- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h
@@ -79,18 +79,18 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_SIMT
 #define LOAD_CHIMU(ptype)		\
   {const SiteSpinor & ref (in[offset]);	\
-    Chimu_00=coalescedReadPermute<ptype>(ref()(0)(0),perm);	\
-    Chimu_01=coalescedReadPermute<ptype>(ref()(0)(1),perm);	\
-    Chimu_02=coalescedReadPermute<ptype>(ref()(0)(2),perm);	\
-    Chimu_10=coalescedReadPermute<ptype>(ref()(1)(0),perm);	\
-    Chimu_11=coalescedReadPermute<ptype>(ref()(1)(1),perm);	\
-    Chimu_12=coalescedReadPermute<ptype>(ref()(1)(2),perm);	\
-    Chimu_20=coalescedReadPermute<ptype>(ref()(2)(0),perm);	\
-    Chimu_21=coalescedReadPermute<ptype>(ref()(2)(1),perm);	\
-    Chimu_22=coalescedReadPermute<ptype>(ref()(2)(2),perm);	\
-    Chimu_30=coalescedReadPermute<ptype>(ref()(3)(0),perm);	\
-    Chimu_31=coalescedReadPermute<ptype>(ref()(3)(1),perm);	\
-    Chimu_32=coalescedReadPermute<ptype>(ref()(3)(2),perm);	}
+    Chimu_00=coalescedReadPermute<ptype>(ref()(0)(0),perm,lane);	\
+    Chimu_01=coalescedReadPermute<ptype>(ref()(0)(1),perm,lane);		\
+    Chimu_02=coalescedReadPermute<ptype>(ref()(0)(2),perm,lane);		\
+    Chimu_10=coalescedReadPermute<ptype>(ref()(1)(0),perm,lane);		\
+    Chimu_11=coalescedReadPermute<ptype>(ref()(1)(1),perm,lane);		\
+    Chimu_12=coalescedReadPermute<ptype>(ref()(1)(2),perm,lane);		\
+    Chimu_20=coalescedReadPermute<ptype>(ref()(2)(0),perm,lane);		\
+    Chimu_21=coalescedReadPermute<ptype>(ref()(2)(1),perm,lane);		\
+    Chimu_22=coalescedReadPermute<ptype>(ref()(2)(2),perm,lane);		\
+    Chimu_30=coalescedReadPermute<ptype>(ref()(3)(0),perm,lane);		\
+    Chimu_31=coalescedReadPermute<ptype>(ref()(3)(1),perm,lane);		\
+    Chimu_32=coalescedReadPermute<ptype>(ref()(3)(2),perm,lane);	}
 #define PERMUTE_DIR(dir) ;
 #else
 #define LOAD_CHIMU(ptype)		\
@@ -119,43 +119,43 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #endif
 
 #define MULT_2SPIN(A)\
-  {auto & ref(U[sU](A));					\
-  U_00=coalescedRead(ref()(0,0));				\
-  U_10=coalescedRead(ref()(1,0));					\
-  U_20=coalescedRead(ref()(2,0));					\
-  U_01=coalescedRead(ref()(0,1));					\
-  U_11=coalescedRead(ref()(1,1));					\
-  U_21=coalescedRead(ref()(2,1));					\
-    UChi_00 = U_00*Chi_00;					\
-    UChi_10 = U_00*Chi_10;					\
-    UChi_01 = U_10*Chi_00;					\
-    UChi_11 = U_10*Chi_10;					\
-    UChi_02 = U_20*Chi_00;					\
-    UChi_12 = U_20*Chi_10;					\
-    UChi_00+= U_01*Chi_01;					\
-    UChi_10+= U_01*Chi_11;					\
-    UChi_01+= U_11*Chi_01;					\
-    UChi_11+= U_11*Chi_11;					\
-    UChi_02+= U_21*Chi_01;					\
-    UChi_12+= U_21*Chi_11;					\
-    U_00=coalescedRead(ref()(0,2));				\
-    U_10=coalescedRead(ref()(1,2));				\
-    U_20=coalescedRead(ref()(2,2));				\
-    UChi_00+= U_00*Chi_02;					\
-    UChi_10+= U_00*Chi_12;					\
-    UChi_01+= U_10*Chi_02;					\
-    UChi_11+= U_10*Chi_12;					\
-    UChi_02+= U_20*Chi_02;					\
+  {auto & ref(U[sU](A));						\
+    U_00=coalescedRead(ref()(0,0),lane);				\
+    U_10=coalescedRead(ref()(1,0),lane);				\
+    U_20=coalescedRead(ref()(2,0),lane);				\
+    U_01=coalescedRead(ref()(0,1),lane);				\
+    U_11=coalescedRead(ref()(1,1),lane);				\
+    U_21=coalescedRead(ref()(2,1),lane);				\
+    UChi_00 = U_00*Chi_00;						\
+    UChi_10 = U_00*Chi_10;						\
+    UChi_01 = U_10*Chi_00;						\
+    UChi_11 = U_10*Chi_10;						\
+    UChi_02 = U_20*Chi_00;						\
+    UChi_12 = U_20*Chi_10;						\
+    UChi_00+= U_01*Chi_01;						\
+    UChi_10+= U_01*Chi_11;						\
+    UChi_01+= U_11*Chi_01;						\
+    UChi_11+= U_11*Chi_11;						\
+    UChi_02+= U_21*Chi_01;						\
+    UChi_12+= U_21*Chi_11;						\
+    U_00=coalescedRead(ref()(0,2),lane);				\
+    U_10=coalescedRead(ref()(1,2),lane);				\
+    U_20=coalescedRead(ref()(2,2),lane);				\
+    UChi_00+= U_00*Chi_02;						\
+    UChi_10+= U_00*Chi_12;						\
+    UChi_01+= U_10*Chi_02;						\
+    UChi_11+= U_10*Chi_12;						\
+    UChi_02+= U_20*Chi_02;						\
     UChi_12+= U_20*Chi_12;}
 
 #define LOAD_CHI				\
   {const SiteHalfSpinor &ref(buf[offset]);	\
-    Chi_00 = coalescedRead(ref()(0)(0));	\
-    Chi_01 = coalescedRead(ref()(0)(1));	\
-    Chi_02 = coalescedRead(ref()(0)(2));	\
-    Chi_10 = coalescedRead(ref()(1)(0));	\
-    Chi_11 = coalescedRead(ref()(1)(1));	\
-    Chi_12 = coalescedRead(ref()(1)(2));}
+    Chi_00 = coalescedRead(ref()(0)(0),lane);	\
+    Chi_01 = coalescedRead(ref()(0)(1),lane);	\
+    Chi_02 = coalescedRead(ref()(0)(2),lane);	\
+    Chi_10 = coalescedRead(ref()(1)(0),lane);	\
+    Chi_11 = coalescedRead(ref()(1)(1),lane);	\
+    Chi_12 = coalescedRead(ref()(1)(2),lane);}
 
 //      hspin(0)=fspin(0)+timesI(fspin(3));
 //      hspin(1)=fspin(1)+timesI(fspin(2));
@@ -453,35 +453,35 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define HAND_RESULT(ss)				\
   {						\
     SiteSpinor & ref (out[ss]);			\
-    coalescedWrite(ref()(0)(0),result_00);		\
-    coalescedWrite(ref()(0)(1),result_01);		\
-    coalescedWrite(ref()(0)(2),result_02);		\
-    coalescedWrite(ref()(1)(0),result_10);		\
-    coalescedWrite(ref()(1)(1),result_11);		\
-    coalescedWrite(ref()(1)(2),result_12);		\
-    coalescedWrite(ref()(2)(0),result_20);		\
-    coalescedWrite(ref()(2)(1),result_21);		\
-    coalescedWrite(ref()(2)(2),result_22);		\
-    coalescedWrite(ref()(3)(0),result_30);		\
-    coalescedWrite(ref()(3)(1),result_31);		\
-    coalescedWrite(ref()(3)(2),result_32);		\
+    coalescedWrite(ref()(0)(0),result_00,lane);		\
+    coalescedWrite(ref()(0)(1),result_01,lane);		\
+    coalescedWrite(ref()(0)(2),result_02,lane);		\
+    coalescedWrite(ref()(1)(0),result_10,lane);		\
+    coalescedWrite(ref()(1)(1),result_11,lane);		\
+    coalescedWrite(ref()(1)(2),result_12,lane);		\
+    coalescedWrite(ref()(2)(0),result_20,lane);		\
+    coalescedWrite(ref()(2)(1),result_21,lane);		\
+    coalescedWrite(ref()(2)(2),result_22,lane);		\
+    coalescedWrite(ref()(3)(0),result_30,lane);		\
+    coalescedWrite(ref()(3)(1),result_31,lane);		\
+    coalescedWrite(ref()(3)(2),result_32,lane);		\
   }
 
 #define HAND_RESULT_EXT(ss)				\
   {							\
     SiteSpinor & ref (out[ss]);				\
-    coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00);	\
-    coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01);	\
-    coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02);	\
-    coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10);	\
-    coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11);	\
-    coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12);	\
-    coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20);	\
-    coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21);	\
-    coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22);	\
-    coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30);	\
-    coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31);	\
-    coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32);	\
+    coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00,lane);	\
+    coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01,lane);	\
+    coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02,lane);	\
+    coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10,lane);	\
+    coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11,lane);	\
+    coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12,lane);	\
+    coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20,lane);	\
+    coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21,lane);	\
+    coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22,lane);	\
+    coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30,lane);	\
+    coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31,lane);	\
+    coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32,lane);	\
   }
 
 #define HAND_DECLARATIONS(Simd)			\
@@ -558,6 +558,9 @@ WilsonKernels<Impl>::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p,
   //  typedef decltype( coalescedRead( vCplx()()() )) Simt;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
 
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+
   HAND_DECLARATIONS(Simt);
 
   int offset,local,perm, ptype;
@@ -584,6 +587,10 @@ WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,Site
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+
   HAND_DECLARATIONS(Simt);
 
   int offset,local,perm, ptype;
@@ -609,6 +616,10 @@ void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+
   HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
@@ -635,6 +646,10 @@ WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,Si
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+
   HAND_DECLARATIONS(Simt);
 
   int offset,local,perm, ptype;
@@ -660,6 +675,10 @@ void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldVi
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+
   HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
@@ -686,6 +705,10 @@ WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,Si
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+
   HAND_DECLARATIONS(Simt);
 
   int offset, ptype;
@@ -712,6 +735,10 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldVi
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
+
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+
   HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;

From 9c2b37218a8849ece7198fc7a25b56bde10f6b13 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 18 Mar 2021 06:24:11 -0400
Subject: [PATCH 65/82] sRNG parameter added

---
 Grid/qcd/utils/Metric.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Grid/qcd/utils/Metric.h b/Grid/qcd/utils/Metric.h
index 10d06de8..d8ae27dc 100644
--- a/Grid/qcd/utils/Metric.h
+++ b/Grid/qcd/utils/Metric.h
@@ -93,13 +93,13 @@ public:
   GeneralisedMomenta(GridBase* grid, Metric<MomentaField>& M): M(M), Mom(grid), AuxMom(grid), AuxField(grid){}
 
   // Correct
-  void MomentaDistribution(GridParallelRNG& pRNG){
+  void MomentaDistribution(GridSerialRNG & sRNG, GridParallelRNG& pRNG){
     // Generate a distribution for
     // P^dag G P
     // where G = M^-1
 
     // Generate gaussian momenta
-    Implementation::generate_momenta(Mom, pRNG);
+    Implementation::generate_momenta(Mom, sRNG, pRNG);
     // Modify the distribution with the metric
     M.MSquareRoot(Mom);
 
@@ -107,8 +107,8 @@ public:
       // Auxiliary momenta
       // do nothing if trivial, so hide in the metric
       MomentaField AuxMomTemp(Mom.Grid());
-      Implementation::generate_momenta(AuxMom, pRNG);
-      Implementation::generate_momenta(AuxField, pRNG);
+      Implementation::generate_momenta(AuxMom, sRNG, pRNG);
+      Implementation::generate_momenta(AuxField, sRNG, pRNG);
       // Modify the distribution with the metric
       // Aux^dag M Aux
       M.MInvSquareRoot(AuxMom);  // AuxMom = M^{-1/2} AuxMomTemp

From 49b0af2c95bd078b6b0b40b8b28b2ea8309daf45 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 18 Mar 2021 09:10:02 -0400
Subject: [PATCH 66/82] Update of tests to compile with the sRNG addition.
 Audited the code conventions (again) with the CPS momentum denominator and
 added anti periodic in time to the Test_mobius_force.cc and tested the
 Test_dwf_gpforce.

Promoted thesee to test full HMC hamiltonian, tr P^2/2 + phidag MdagM phi

with the same pdot and Udot as audited in the Integrator.h etc...

With full comments and sources for factors.
---
 tests/forces/Test_dwf_force_eofa.cc      |   4 +-
 tests/forces/Test_dwf_gpforce.cc         | 123 +++++++++++++++--------
 tests/forces/Test_dwf_gpforce_eofa.cc    |   3 +-
 tests/forces/Test_laplacian_force.cc     |   3 +-
 tests/forces/Test_mobius_force.cc        | 109 +++++++++++++++++---
 tests/forces/Test_mobius_force_eofa.cc   |   3 +-
 tests/forces/Test_mobius_gpforce_eofa.cc |   3 +-
 7 files changed, 188 insertions(+), 60 deletions(-)

diff --git a/tests/forces/Test_dwf_force_eofa.cc b/tests/forces/Test_dwf_force_eofa.cc
index 80d36934..525178d0 100644
--- a/tests/forces/Test_dwf_force_eofa.cc
+++ b/tests/forces/Test_dwf_force_eofa.cc
@@ -86,7 +86,9 @@ int main (int argc, char** argv)
   ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
   ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, true);
 
-  Meofa.refresh(U, RNG5);
+  GridSerialRNG  sRNG; sRNG.SeedFixedIntegers(seeds4);
+  Meofa.refresh(U, sRNG, RNG5 );
+
   RealD S = Meofa.S(U); // pdag M p
 
   // get the deriv of phidag M phi with respect to "U"
diff --git a/tests/forces/Test_dwf_gpforce.cc b/tests/forces/Test_dwf_gpforce.cc
index 28133cc6..1fa1c6e4 100644
--- a/tests/forces/Test_dwf_gpforce.cc
+++ b/tests/forces/Test_dwf_gpforce.cc
@@ -84,6 +84,13 @@ int main (int argc, char ** argv)
   GparityDomainWallFermionR::ImplParams params;
   params.twists = twists;
 
+  /*
+  params.boundary_phases[0] = 1.0;
+  params.boundary_phases[1] = 1.0;
+  params.boundary_phases[2] = 1.0;
+  params.boundary_phases[3] =- 1.0;
+  */
+  
   GparityDomainWallFermionR Dw(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,params);
 
   Dw.M   (phi,Mphi);
@@ -96,6 +103,16 @@ int main (int argc, char ** argv)
 
   Dw.MDeriv(tmp , Mphi,  phi,DaggerNo );  UdSdU=tmp;
   Dw.MDeriv(tmp , phi,  Mphi,DaggerYes ); UdSdU=(UdSdU+tmp);
+
+  // *****************************************************************************************
+  // *** There is a funny negative sign in all derivatives. This is - UdSdU.               ***
+  // ***                                                                                   ***
+  // *** Deriv in both Wilson gauge action and the TwoFlavour.h seems to miss a minus sign ***
+  // *** UdSdU is negated relative to what I think - call what is returned mUdSdU,         ***
+  // *** and insert minus sign                                                             ***
+  // *****************************************************************************************
+
+  UdSdU = - UdSdU ; // Follow sign convention of actions in Grid. Seems crazy.
   
   FermionField Ftmp      (FGrid);
 
@@ -106,7 +123,7 @@ int main (int argc, char ** argv)
   RealD Hmom = 0.0;
   RealD Hmomprime = 0.0;
   LatticeColourMatrix mommu(UGrid); 
-  LatticeColourMatrix forcemu(UGrid); 
+  LatticeColourMatrix mUdSdUmu(UGrid); 
   LatticeGaugeField mom(UGrid); 
   LatticeGaugeField Uprime(UGrid); 
 
@@ -114,10 +131,20 @@ int main (int argc, char ** argv)
 
     SU<Nc>::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
 
-    Hmom -= real(sum(trace(mommu*mommu)));
+  // Momentum Hamiltonian is - trace(p^2)/HMC_MOM_DENOMINATOR
+  //
+  // Integrator.h:   RealD H = - FieldImplementation::FieldSquareNorm(P)/HMC_MOMENTUM_DENOMINATOR; // - trace (P*P)/denom                                                                                       //     GaugeImplTypes.h:        Hloc += trace(Pmu * Pmu);
+  //                          Sign comes from a sneaky multiply by "i" in GaussianFundemantalLie algebra
+  //                          P is i P^a_\mu T^a, not Pa Ta
+  // 
+  // Integrator.h: H =  Hmom + sum S(action)
+    Hmom -= real(sum(trace(mommu*mommu)))/ HMC_MOMENTUM_DENOMINATOR;
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 
+  // -- Drops factor of "i" in the U update: U' = e^{P dt} U   [ _not_ e^{iPdt}U ]. P is anti hermitian already
+  // -- Udot = p U
+
     // fourth order exponential approx
     autoView( mom_v, mom, CpuRead);
     autoView( U_v , U, CpuRead);
@@ -134,8 +161,8 @@ int main (int argc, char ** argv)
 	;
     });
   }
-
   std::cout << GridLogMessage <<"Initial mom hamiltonian is "<< Hmom <<std::endl;
+
   Dw.ImportGauge(Uprime);
   Dw.M          (phi,MphiPrime);
 
@@ -145,53 +172,60 @@ int main (int argc, char ** argv)
   // Use derivative to estimate dS
   //////////////////////////////////////////////
 
-
-  for(int mu=0;mu<Nd;mu++){
-    std::cout << "" <<std::endl;
-    mommu   = PeekIndex<LorentzIndex>(mom,mu);
-    std::cout << GridLogMessage<< " Mommu  " << norm2(mommu)<<std::endl;
-    mommu   = mommu+adj(mommu);
-    std::cout << GridLogMessage<< " Mommu + Mommudag " << norm2(mommu)<<std::endl;
-    mommu   = PeekIndex<LorentzIndex>(UdSdU,mu);
-    std::cout << GridLogMessage<< " dsdumu  " << norm2(mommu)<<std::endl;
-    mommu   = mommu+adj(mommu);
-    std::cout << GridLogMessage<< " dsdumu + dag  " << norm2(mommu)<<std::endl;
-  }
-
+  //
+  // Ta has 1/2([ F - adj(F) ])_traceless and want the UdSdU _and_ UdagdSdUdag terms so 2x. 
+  //
   LatticeComplex dS(UGrid); dS = Zero();
   LatticeComplex dSmom(UGrid); dSmom = Zero();
   LatticeComplex dSmom2(UGrid); dSmom2 = Zero();
   for(int mu=0;mu<Nd;mu++){
     mommu   = PeekIndex<LorentzIndex>(UdSdU,mu);
-    mommu=Ta(mommu)*2.0;
+    mommu=Ta(mommu); // projectForce , GaugeImplTypes.h
     PokeIndex<LorentzIndex>(UdSdU,mommu,mu);
   }
 
   for(int mu=0;mu<Nd;mu++){
-    mommu   = PeekIndex<LorentzIndex>(mom,mu);
-    std::cout << GridLogMessage<< " Mommu  " << norm2(mommu)<<std::endl;
-    mommu   = mommu+adj(mommu);
-    std::cout << GridLogMessage<< " Mommu + Mommudag " << norm2(mommu)<<std::endl;
-    mommu   = PeekIndex<LorentzIndex>(UdSdU,mu);
-    std::cout << GridLogMessage<< " dsdumu  " << norm2(mommu)<<std::endl;
-    mommu   = mommu+adj(mommu);
-    std::cout << GridLogMessage<< " dsdumu + dag  " << norm2(mommu)<<std::endl;
-  }
 
-  for(int mu=0;mu<Nd;mu++){
-    forcemu = PeekIndex<LorentzIndex>(UdSdU,mu);
+    mUdSdUmu= PeekIndex<LorentzIndex>(UdSdU,mu);
     mommu   = PeekIndex<LorentzIndex>(mom,mu);
 
-    // Update PF action density
-    dS = dS+trace(mommu*forcemu)*dt;
+    //
+    // Derive HMC eom:
+    //
+    // Sdot =  - 2 trace( p p^dot ) / D - trace( p [ mUdSdU - h.c. ] ) = 0 
+    //
+    //
+    // Sdot = 0 = - 2 trace( p p^dot ) / D - 2 trace( p Ta( mUdSdU ) = 0
+    //
+    // EOM: 
+    //
+    // pdot = - D Ta( mUdSdU ) -- source of sign is the "funny sign" above
+    //
+    // dSqcd_dt  = - 2.0*trace(mommu* Ta(mUdSdU) )*dt -- i.e. mUdSdU with adjoint term -> force has a 2x implicit
+    //
+    // dH_mom/dt = - 2 trace (p pdot)/Denom  
+    //
+    // dH_tot / dt = 0 <= pdot = - Denom * mUdSdU 
+    //
+    // dH_mom/dt = 2 trace (p mUdSdU ) 
+    //
+    // True Momentum delta H has a dt^2 piece
+    //
+    // dSmom = [ trace mom*mom - trace ( (mom-Denom*f*dt)(mom-Denom*f*dt) ) ] / Denom
+    //       = 2*trace(mom*f) dt  - Denom*dt*dt * trace(f*f).
+    //       = dSmom + dSmom2
+    //
 
-    dSmom  = dSmom  - trace(mommu*forcemu) * dt;
-    dSmom2 = dSmom2 - trace(forcemu*forcemu) *(0.25* dt*dt);
+    dS     = dS - 2.0*trace(mommu*mUdSdUmu)*dt;   // U and Udagger derivs hence 2x.
 
-    // Update mom action density
-    mommu = mommu + forcemu*(dt*0.5);
+    dSmom  = dSmom  + 2.0*trace(mommu*mUdSdUmu) * dt;  // this 2.0 coms from derivative of p^2 
+    
+    dSmom2 = dSmom2 - trace(mUdSdUmu*mUdSdUmu) * dt*dt* HMC_MOMENTUM_DENOMINATOR; // Remnant
 
-    Hmomprime -= real(sum(trace(mommu*mommu)));
+    // Update mom action density . Verbatim update_P in Integrator.h
+    mommu = mommu - mUdSdUmu * dt* HMC_MOMENTUM_DENOMINATOR;; 
+
+    Hmomprime -= real(sum(trace(mommu*mommu))) / HMC_MOMENTUM_DENOMINATOR;
 
   }
 
@@ -199,20 +233,25 @@ int main (int argc, char ** argv)
   ComplexD dSm       = sum(dSmom);
   ComplexD dSm2      = sum(dSmom2);
 
+  std::cout << GridLogMessage <<"dSm "<< dSm<<std::endl;
+  std::cout << GridLogMessage <<"dSm2 "<< dSm2<<std::endl;
 
   std::cout << GridLogMessage <<"Initial mom hamiltonian is "<< Hmom <<std::endl;
   std::cout << GridLogMessage <<"Final   mom hamiltonian is "<< Hmomprime <<std::endl;
-  std::cout << GridLogMessage <<"Delta   mom hamiltonian is "<< Hmomprime-Hmom <<std::endl;
 
-  std::cout << GridLogMessage << " S      "<<S<<std::endl;
-  std::cout << GridLogMessage << " Sprime "<<Sprime<<std::endl;
-  std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
-  std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
-  std::cout << GridLogMessage <<"dSm "<< dSm<<std::endl;
-  std::cout << GridLogMessage <<"dSm2"<< dSm2<<std::endl;
+  std::cout << GridLogMessage <<"Delta   mom hamiltonian is "<< Hmomprime-Hmom <<std::endl;
+  std::cout << GridLogMessage <<"predict Delta mom hamiltonian is "<< dSm+dSm2 <<std::endl;
+  
+  std::cout << GridLogMessage << "Initial S      "<<S<<std::endl;
+  std::cout << GridLogMessage << "Final   S      "<<Sprime<<std::endl;
+  std::cout << GridLogMessage << "Delta   S      "<<Sprime-S<<std::endl;
+  std::cout << GridLogMessage << "predict delta S"<< dSpred <<std::endl;
+  std::cout << GridLogMessage << "defect "<< Sprime-S-dSpred <<std::endl;
 
   std::cout << GridLogMessage << "Total dS    "<< Hmomprime - Hmom + Sprime - S <<std::endl;
 
+  std::cout << GridLogMessage << "dS - dt^2 term "<< Hmomprime - Hmom + Sprime - S - dSm2 <<std::endl;
+  
   assert( fabs(real(Sprime-S-dSpred)) < 5.0 ) ;
 
   std::cout<< GridLogMessage << "Done" <<std::endl;
diff --git a/tests/forces/Test_dwf_gpforce_eofa.cc b/tests/forces/Test_dwf_gpforce_eofa.cc
index 7e480e7a..fd47d33c 100644
--- a/tests/forces/Test_dwf_gpforce_eofa.cc
+++ b/tests/forces/Test_dwf_gpforce_eofa.cc
@@ -90,7 +90,8 @@ int main (int argc, char** argv)
   ConjugateGradient<FermionField> CG(1.0e-12, 5000);
   ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, true);
 
-  Meofa.refresh(U, RNG5);
+  GridSerialRNG  sRNG; sRNG.SeedFixedIntegers(seeds4);
+  Meofa.refresh(U, sRNG, RNG5);
   RealD S = Meofa.S(U); // pdag M p
 
   // get the deriv of phidag M phi with respect to "U"
diff --git a/tests/forces/Test_laplacian_force.cc b/tests/forces/Test_laplacian_force.cc
index 18508860..dbaf1cbd 100644
--- a/tests/forces/Test_laplacian_force.cc
+++ b/tests/forces/Test_laplacian_force.cc
@@ -46,6 +46,7 @@ int main (int argc, char ** argv)
 
   std::vector<int> seeds({1,2,3,4});
 
+  GridSerialRNG sRNG; sRNG.SeedFixedIntegers({4,5,6,7});
   GridParallelRNG          pRNG(&Grid);
   pRNG.SeedFixedIntegers(std::vector<int>({15,91,21,3}));
 
@@ -67,7 +68,7 @@ int main (int argc, char ** argv)
   LaplacianAdjointField<PeriodicGimplR> Laplacian(&Grid, CG, LapPar, Kappa);
   GeneralisedMomenta<PeriodicGimplR> LaplacianMomenta(&Grid, Laplacian);
   LaplacianMomenta.M.ImportGauge(U);
-  LaplacianMomenta.MomentaDistribution(pRNG);// fills the Momenta with the correct distr
+  LaplacianMomenta.MomentaDistribution(sRNG,pRNG);// fills the Momenta with the correct distr
   
 
   std::cout << std::setprecision(15);
diff --git a/tests/forces/Test_mobius_force.cc b/tests/forces/Test_mobius_force.cc
index ba7bc363..d2326a81 100644
--- a/tests/forces/Test_mobius_force.cc
+++ b/tests/forces/Test_mobius_force.cc
@@ -69,7 +69,14 @@ int main (int argc, char ** argv)
   RealD M5=1.8; 
   RealD b=0.5;
   RealD c=0.5;
-  MobiusFermionR Ddwf(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
+
+  WilsonImplParams p;
+  p.boundary_phases[0] = 1.0;
+  p.boundary_phases[1] = 1.0;
+  p.boundary_phases[2] = 1.0;
+  p.boundary_phases[3] =- 1.0;
+  
+  MobiusFermionR Ddwf(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c,p);
   Ddwf.M   (phi,Mphi);
 
   ComplexD S    = innerProduct(Mphi,Mphi); // pdag MdagM p
@@ -82,24 +89,44 @@ int main (int argc, char ** argv)
   Ddwf.MDeriv(tmp , Mphi,  phi,DaggerNo );  UdSdU=tmp;
   Ddwf.MDeriv(tmp , phi,  Mphi,DaggerYes ); UdSdU=(UdSdU+tmp);  
   
+  // *****************************************************************************************
+  // *** There is a funny negative sign in all derivatives. This is - UdSdU.               ***
+  // ***                                                                                   ***
+  // *** Deriv in both Wilson gauge action and the TwoFlavour.h seems to miss a minus sign ***
+  // *** UdSdU is negated relative to what I think - call what is returned mUdSdU,         ***
+  // *** and insert minus sign                                                             ***
+  // *****************************************************************************************
+
+  UdSdU = - UdSdU ; // Follow sign convention of actions in Grid. Seems crazy.
+
   LatticeFermion Ftmp      (FGrid);
 
   ////////////////////////////////////
   // Modify the gauge field a little 
   ////////////////////////////////////
-  RealD dt = 0.0001;
+  RealD dt = 0.001;
+  RealD Hmom = 0.0;
+  RealD Hmomprime = 0.0;
 
   LatticeColourMatrix mommu(UGrid); 
-  LatticeColourMatrix forcemu(UGrid); 
+  LatticeColourMatrix mUdSdUmu(UGrid); 
   LatticeGaugeField mom(UGrid); 
   LatticeGaugeField Uprime(UGrid); 
 
   for(int mu=0;mu<Nd;mu++){
 
     SU<Nc>::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
-
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 
+    // Momentum Hamiltonian is - trace(p^2)/HMC_MOM_DENOMINATOR
+    //
+    // Integrator.h:   RealD H = - FieldImplementation::FieldSquareNorm(P)/HMC_MOMENTUM_DENOMINATOR; // - trace (P*P)/denom                                                                                       //     GaugeImplTypes.h:        Hloc += trace(Pmu * Pmu);
+    //                          Sign comes from a sneaky multiply by "i" in GaussianFundemantalLie algebra
+    //                          P is i P^a_\mu T^a, not Pa Ta
+    // 
+    // Integrator.h: H =  Hmom + sum S(action)
+    Hmom -= real(sum(trace(mommu*mommu)))/ HMC_MOMENTUM_DENOMINATOR;
+    
     // fourth order exponential approx
     autoView( U_v , U, CpuRead);
     autoView( mom_v, mom, CpuRead);
@@ -115,6 +142,7 @@ int main (int argc, char ** argv)
 	;
     });
   }
+  std::cout << GridLogMessage <<"Initial mom hamiltonian is "<< Hmom <<std::endl;
   
   Ddwf.ImportGauge(Uprime);
   Ddwf.M          (phi,MphiPrime);
@@ -125,32 +153,87 @@ int main (int argc, char ** argv)
   // Use derivative to estimate dS
   //////////////////////////////////////////////
 
-
   LatticeComplex dS(UGrid); dS = Zero();
+  LatticeComplex dSmom(UGrid); dSmom = Zero();
+  LatticeComplex dSmom2(UGrid); dSmom2 = Zero();
+
   for(int mu=0;mu<Nd;mu++){
     mommu   = PeekIndex<LorentzIndex>(UdSdU,mu);
-    mommu=Ta(mommu)*2.0;
+    mommu=Ta(mommu);
     PokeIndex<LorentzIndex>(UdSdU,mommu,mu);
   }
 
   for(int mu=0;mu<Nd;mu++){
-    forcemu = PeekIndex<LorentzIndex>(UdSdU,mu);
+
+    mUdSdUmu= PeekIndex<LorentzIndex>(UdSdU,mu);
     mommu   = PeekIndex<LorentzIndex>(mom,mu);
 
-    // Update PF action density
-    dS = dS+trace(mommu*forcemu)*dt;
+  //
+  // Derive HMC eom:
+  //
+  // Sdot =  - 2 trace( p p^dot ) / D - trace( p [ mUdSdU - h.c. ] ) = 0 
+  //
+  //
+  // Sdot = 0 = - 2 trace( p p^dot ) / D - 2 trace( p Ta( mUdSdU ) = 0
+  //
+  // EOM: 
+  //
+  // pdot = - D Ta( mUdSdU ) -- source of sign is the "funny sign" above
+  //
+  // dSqcd_dt  = - 2.0*trace(mommu* Ta(mUdSdU) )*dt -- i.e. mUdSdU with adjoint term -> force has a 2x implicit
+  //
+  // dH_mom/dt = - 2 trace (p pdot)/Denom  
+  //
+  // dH_tot / dt = 0 <= pdot = - Denom * mUdSdU 
+  //
+  // dH_mom/dt = 2 trace (p mUdSdU ) 
+  //
+  // True Momentum delta H has a dt^2 piece
+  //
+  // dSmom = [ trace mom*mom - trace ( (mom-Denom*f*dt)(mom-Denom*f*dt) ) ] / Denom
+  //       = 2*trace(mom*f) dt  - Denom*dt*dt * trace(f*f).
+  //       = dSmom + dSmom2
+  //
+    
+    dS     = dS - 2.0*trace(mommu*mUdSdUmu)*dt;   // U and Udagger derivs hence 2x.
+
+    dSmom  = dSmom  + 2.0*trace(mommu*mUdSdUmu) * dt;  // this 2.0 coms from derivative of p^2 
+    
+    dSmom2 = dSmom2 - trace(mUdSdUmu*mUdSdUmu) * dt*dt* HMC_MOMENTUM_DENOMINATOR; // Remnant
+
+    mommu = mommu - mUdSdUmu * dt* HMC_MOMENTUM_DENOMINATOR;; 
+
+    Hmomprime -= real(sum(trace(mommu*mommu))) / HMC_MOMENTUM_DENOMINATOR;
 
   }
 
   ComplexD dSpred    = sum(dS);
+  ComplexD dSm       = sum(dSmom);
+  ComplexD dSm2      = sum(dSmom2);
 
-  std::cout << GridLogMessage << " -- S         "<<S<<std::endl;
-  std::cout << GridLogMessage << " -- Sprime    "<<Sprime<<std::endl;
-  std::cout << GridLogMessage << "dS            "<<Sprime-S<<std::endl;
-  std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
+  std::cout << GridLogMessage <<"dSm "<< dSm<<std::endl;
+  std::cout << GridLogMessage <<"dSm2 "<< dSm2<<std::endl;
+
+  std::cout << GridLogMessage <<"Initial mom hamiltonian is "<< Hmom <<std::endl;
+  std::cout << GridLogMessage <<"Final   mom hamiltonian is "<< Hmomprime <<std::endl;
+
+  std::cout << GridLogMessage <<"Delta   mom hamiltonian is "<< Hmomprime-Hmom <<std::endl;
+  std::cout << GridLogMessage <<"predict Delta mom hamiltonian is "<< dSm+dSm2 <<std::endl;
+  
+  std::cout << GridLogMessage << "Initial S      "<<S<<std::endl;
+  std::cout << GridLogMessage << "Final   S      "<<Sprime<<std::endl;
+  std::cout << GridLogMessage << "Delta   S      "<<Sprime-S<<std::endl;
+  std::cout << GridLogMessage << "predict delta S"<< dSpred <<std::endl;
+  std::cout << GridLogMessage << "defect "<< Sprime-S-dSpred <<std::endl;
+
+  std::cout << GridLogMessage << "Total dS    "<< Hmomprime - Hmom + Sprime - S <<std::endl;
+
+  std::cout << GridLogMessage << "dS - dt^2 term "<< Hmomprime - Hmom + Sprime - S - dSm2 <<std::endl;
 
   assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
 
+
+  
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }
diff --git a/tests/forces/Test_mobius_force_eofa.cc b/tests/forces/Test_mobius_force_eofa.cc
index 28523e9c..eea3e3f4 100644
--- a/tests/forces/Test_mobius_force_eofa.cc
+++ b/tests/forces/Test_mobius_force_eofa.cc
@@ -88,7 +88,8 @@ int main (int argc, char** argv)
   ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
   ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, false);
 
-  Meofa.refresh(U, RNG5);
+  GridSerialRNG  sRNG; sRNG.SeedFixedIntegers(seeds4);
+  Meofa.refresh(U, sRNG, RNG5 );
   RealD S = Meofa.S(U); // pdag M p
 
   // get the deriv of phidag M phi with respect to "U"
diff --git a/tests/forces/Test_mobius_gpforce_eofa.cc b/tests/forces/Test_mobius_gpforce_eofa.cc
index 9c80b2aa..7f114615 100644
--- a/tests/forces/Test_mobius_gpforce_eofa.cc
+++ b/tests/forces/Test_mobius_gpforce_eofa.cc
@@ -93,7 +93,8 @@ int main (int argc, char** argv)
   ConjugateGradient<FermionField> CG(1.0e-12, 5000);
   ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, false);
 
-  Meofa.refresh(U, RNG5);
+  GridSerialRNG  sRNG; sRNG.SeedFixedIntegers(seeds4);
+  Meofa.refresh(U, sRNG, RNG5 );
   RealD S = Meofa.S(U); // pdag M p
 
   // get the deriv of phidag M phi with respect to "U"

From 15c50a7442527a7962e53cdc4a6bb3369f41501c Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 18 Mar 2021 15:40:42 -0400
Subject: [PATCH 67/82] Explicit instantiate the template function

---
 Grid/util/Init.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc
index 55d8c5bf..bfbc464d 100644
--- a/Grid/util/Init.cc
+++ b/Grid/util/Init.cc
@@ -153,6 +153,9 @@ void GridCmdOptionIntVector(const std::string &str,VectorInt & vec)
   return;
 }
 
+template void GridCmdOptionIntVector(const std::string &str,std::vector<int> & vec);
+template void GridCmdOptionIntVector(const std::string &str,Coordinate & vec);
+
 void GridCmdOptionInt(std::string &str,int & val)
 {
   std::stringstream ss(str);

From 8bdadbadaca9cffd8d7616cc9deed07187b262f1 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 18 Mar 2021 15:41:14 -0400
Subject: [PATCH 68/82] Cold start

---
 HMC/Mobius2p1fRHMC.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/HMC/Mobius2p1fRHMC.cc b/HMC/Mobius2p1fRHMC.cc
index 82ca4d37..b958d548 100644
--- a/HMC/Mobius2p1fRHMC.cc
+++ b/HMC/Mobius2p1fRHMC.cc
@@ -56,12 +56,12 @@ int main(int argc, char **argv) {
   MD.trajL   = 1.0;
 
   HMCparameters HMCparams;
-  HMCparams.StartTrajectory  = 30;
+  HMCparams.StartTrajectory  = 0;
   HMCparams.Trajectories     = 200;
   HMCparams.NoMetropolisUntil=  0;
   // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
-  //  HMCparams.StartingType     =std::string("ColdStart");
-  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.StartingType     =std::string("ColdStart");
+  //  HMCparams.StartingType     =std::string("CheckpointStart");
   HMCparams.MD = MD;
   HMCWrapper TheHMC(HMCparams);
 

From bb89a82a07be478d50060efd5c76cf71622af870 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Mon, 29 Mar 2021 20:01:15 +0200
Subject: [PATCH 69/82] Staggered coalseced read

---
 Grid/qcd/action/fermion/StaggeredImpl.h       | 16 ++--
 Grid/qcd/action/fermion/WilsonImpl.h          | 35 ++++++++-
 .../StaggeredKernelsImplementation.h          | 76 ++++++++++---------
 3 files changed, 82 insertions(+), 45 deletions(-)

diff --git a/Grid/qcd/action/fermion/StaggeredImpl.h b/Grid/qcd/action/fermion/StaggeredImpl.h
index 8adf45a4..f44d12f4 100644
--- a/Grid/qcd/action/fermion/StaggeredImpl.h
+++ b/Grid/qcd/action/fermion/StaggeredImpl.h
@@ -72,19 +72,23 @@ public:
     
   StaggeredImpl(const ImplParams &p = ImplParams()) : Params(p){};
       
-  static accelerator_inline void multLink(SiteSpinor &phi,
+  template<class _Spinor>
+  static accelerator_inline void multLink(_Spinor &phi,
 		       const SiteDoubledGaugeField &U,
-		       const SiteSpinor &chi,
+		       const _Spinor &chi,
 		       int mu)
   {
-    mult(&phi(), &U(mu), &chi());
+    auto UU = coalescedRead(U(mu));
+    mult(&phi(), &UU, &chi());
   }
-  static accelerator_inline void multLinkAdd(SiteSpinor &phi,
+  template<class _Spinor>
+  static accelerator_inline void multLinkAdd(_Spinor &phi,
 			  const SiteDoubledGaugeField &U,
-			  const SiteSpinor &chi,
+			  const _Spinor &chi,
 			  int mu)
   {
-    mac(&phi(), &U(mu), &chi());
+    auto UU = coalescedRead(U(mu));
+    mac(&phi(), &UU, &chi());
   }
       
   template <class ref>
diff --git a/Grid/qcd/action/fermion/WilsonImpl.h b/Grid/qcd/action/fermion/WilsonImpl.h
index 94676b6b..2ff6feba 100644
--- a/Grid/qcd/action/fermion/WilsonImpl.h
+++ b/Grid/qcd/action/fermion/WilsonImpl.h
@@ -184,18 +184,22 @@ public:
       mat = TraceIndex<SpinIndex>(P); 
     }
       
-    inline void extractLinkField(std::vector<GaugeLinkField> &mat, DoubledGaugeField &Uds){
+    inline void extractLinkField(std::vector<GaugeLinkField> &mat, DoubledGaugeField &Uds)
+    {
       for (int mu = 0; mu < Nd; mu++)
       mat[mu] = PeekIndex<LorentzIndex>(Uds, mu);
     }
 
-
-  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde,int mu){
-      
+  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde,int mu)
+  {
+#undef USE_OLD_INSERT_FORCE    
     int Ls=Btilde.Grid()->_fdimensions[0];
+    autoView( mat_v , mat, AcceleratorWrite);
+#ifdef USE_OLD_INSERT_FORCE    
     GaugeLinkField tmp(mat.Grid());
     tmp = Zero();
     {
+      const int Nsimd = SiteSpinor::Nsimd();
       autoView( tmp_v , tmp, AcceleratorWrite);
       autoView( Btilde_v , Btilde, AcceleratorRead);
       autoView( Atilde_v , Atilde, AcceleratorRead);
@@ -208,6 +212,29 @@ public:
 	});
     }
     PokeIndex<LorentzIndex>(mat,tmp,mu);
+#else
+    {
+      const int Nsimd = SiteSpinor::Nsimd();
+      autoView( Btilde_v , Btilde, AcceleratorRead);
+      autoView( Atilde_v , Atilde, AcceleratorRead);
+      accelerator_for(sss,mat.Grid()->oSites(),Nsimd,{
+	  int sU=sss;
+  	  typedef decltype(coalescedRead(mat_v[sU](mu)() )) ColorMatrixType;
+  	  ColorMatrixType sum;
+	  zeroit(sum);  
+	  for(int s=0;s<Ls;s++){
+	    int sF = s+Ls*sU;
+  	    for(int spn=0;spn<Ns;spn++){ //sum over spin
+  	      auto bb = coalescedRead(Btilde_v[sF]()(spn) ); //color vector
+  	      auto aa = coalescedRead(Atilde_v[sF]()(spn) );
+	      auto op = outerProduct(bb,aa);
+  	      sum = sum + op;
+	    }
+	  }
+  	  coalescedWrite(mat_v[sU](mu)(), sum);
+      });
+    }
+#endif    
   }
 };
 
diff --git a/Grid/qcd/action/fermion/implementation/StaggeredKernelsImplementation.h b/Grid/qcd/action/fermion/implementation/StaggeredKernelsImplementation.h
index 0b6f9fb0..dd62e109 100644
--- a/Grid/qcd/action/fermion/implementation/StaggeredKernelsImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/StaggeredKernelsImplementation.h
@@ -35,39 +35,32 @@ NAMESPACE_BEGIN(Grid);
 #define GENERIC_STENCIL_LEG(U,Dir,skew,multLink)		\
   SE = st.GetEntry(ptype, Dir+skew, sF);			\
   if (SE->_is_local ) {						\
-    if (SE->_permute) {						\
-      chi_p = &chi;						\
-      permute(chi,  in[SE->_offset], ptype);			\
-    } else {							\
-      chi_p = &in[SE->_offset];					\
-    }								\
+    int perm= SE->_permute;						\
+    chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\
   } else {							\
-    chi_p = &buf[SE->_offset];					\
+    chi = coalescedRead(buf[SE->_offset],lane);			\
   }								\
-  multLink(Uchi, U[sU], *chi_p, Dir);			
+  acceleratorSynchronise();					\
+  multLink(Uchi, U[sU], chi, Dir);			
 
 #define GENERIC_STENCIL_LEG_INT(U,Dir,skew,multLink)		\
   SE = st.GetEntry(ptype, Dir+skew, sF);			\
   if (SE->_is_local ) {						\
-    if (SE->_permute) {						\
-      chi_p = &chi;						\
-      permute(chi,  in[SE->_offset], ptype);			\
-    } else {							\
-      chi_p = &in[SE->_offset];					\
-    }								\
+    int perm= SE->_permute;						\
+    chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\
   } else if ( st.same_node[Dir] ) {				\
-    chi_p = &buf[SE->_offset];					\
+    chi = coalescedRead(buf[SE->_offset],lane);                 \
   }								\
   if (SE->_is_local || st.same_node[Dir] ) {			\
-    multLink(Uchi, U[sU], *chi_p, Dir);				\
+    multLink(Uchi, U[sU], chi, Dir);				\
   }
 
 #define GENERIC_STENCIL_LEG_EXT(U,Dir,skew,multLink)		\
   SE = st.GetEntry(ptype, Dir+skew, sF);			\
   if ((!SE->_is_local) && (!st.same_node[Dir]) ) {		\
     nmu++;							\
-    chi_p = &buf[SE->_offset];					\
-    multLink(Uchi, U[sU], *chi_p, Dir);				\
+    chi = coalescedRead(buf[SE->_offset],lane);			\
+    multLink(Uchi, U[sU], chi, Dir);				\
   }
 
 template <class Impl>
@@ -84,12 +77,14 @@ void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st,
 					     SiteSpinor *buf, int sF, int sU, 
 					     const FermionFieldView &in, FermionFieldView &out, int dag) 
 {
-  const SiteSpinor *chi_p;
-  SiteSpinor chi;
-  SiteSpinor Uchi;
+  typedef decltype(coalescedRead(in[0])) calcSpinor;
+  calcSpinor chi;
+  calcSpinor Uchi;
   StencilEntry *SE;
   int ptype;
   int skew;
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
 
   //  for(int s=0;s<LLs;s++){
   //
@@ -118,7 +113,7 @@ void StaggeredKernels<Impl>::DhopSiteGeneric(StencilView &st,
     if ( dag ) { 
       Uchi = - Uchi;
     } 
-    vstream(out[sF], Uchi);
+    coalescedWrite(out[sF], Uchi,lane);
   }
 };
 
@@ -130,13 +125,16 @@ template <int Naik> accelerator_inline
 void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st, 
 						DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 						SiteSpinor *buf, int sF, int sU, 
-						const FermionFieldView &in, FermionFieldView &out,int dag) {
-  const SiteSpinor *chi_p;
-  SiteSpinor chi;
-  SiteSpinor Uchi;
+						const FermionFieldView &in, FermionFieldView &out,int dag)
+{
+  typedef decltype(coalescedRead(in[0])) calcSpinor;
+  calcSpinor chi;
+  calcSpinor Uchi;
   StencilEntry *SE;
   int ptype;
   int skew ;
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
 
   //  for(int s=0;s<LLs;s++){
   //    int sF=LLs*sU+s;
@@ -165,7 +163,7 @@ void StaggeredKernels<Impl>::DhopSiteGenericInt(StencilView &st,
     if ( dag ) {
       Uchi = - Uchi;
     }
-    vstream(out[sF], Uchi);
+    coalescedWrite(out[sF], Uchi,lane);
   }
 };
 
@@ -178,14 +176,17 @@ template <int Naik> accelerator_inline
 void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st, 
 						DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU,
 						SiteSpinor *buf, int sF, int sU,
-						const FermionFieldView &in, FermionFieldView &out,int dag) {
-  const SiteSpinor *chi_p;
-  //  SiteSpinor chi;
-  SiteSpinor Uchi;
+						const FermionFieldView &in, FermionFieldView &out,int dag)
+{
+  typedef decltype(coalescedRead(in[0])) calcSpinor;
+  calcSpinor chi;
+  calcSpinor Uchi;
   StencilEntry *SE;
   int ptype;
   int nmu=0;
   int skew ;
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
 
   //  for(int s=0;s<LLs;s++){
   //    int sF=LLs*sU+s;
@@ -211,11 +212,12 @@ void StaggeredKernels<Impl>::DhopSiteGenericExt(StencilView &st,
     GENERIC_STENCIL_LEG_EXT(UUU,Zm,skew,Impl::multLinkAdd);
     GENERIC_STENCIL_LEG_EXT(UUU,Tm,skew,Impl::multLinkAdd);
     }
-    if ( nmu ) { 
-      if ( dag ) { 
-	out[sF] = out[sF] - Uchi;
+    if ( nmu ) {
+      auto _out = coalescedRead(out[sF],lane);
+      if ( dag ) {
+	coalescedWrite(out[sF], _out-Uchi,lane);
       } else { 
-	out[sF] = out[sF] + Uchi;
+	coalescedWrite(out[sF], _out+Uchi,lane);
       }
     }
   }
@@ -261,6 +263,8 @@ void StaggeredKernels<Impl>::DhopImproved(StencilImpl &st, LebesgueOrder &lo,
   GridBase *FGrid=in.Grid();  
   GridBase *UGrid=U.Grid();  
   typedef StaggeredKernels<Impl> ThisKernel;
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
   autoView( UUU_v , UUU, AcceleratorRead);
   autoView( U_v   ,   U, AcceleratorRead);
   autoView( in_v  ,  in, AcceleratorRead);
@@ -301,6 +305,8 @@ void StaggeredKernels<Impl>::DhopNaive(StencilImpl &st, LebesgueOrder &lo,
   GridBase *FGrid=in.Grid();  
   GridBase *UGrid=U.Grid();  
   typedef StaggeredKernels<Impl> ThisKernel;
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
   autoView( UUU_v ,   U, AcceleratorRead);
   autoView( U_v   ,   U, AcceleratorRead);
   autoView( in_v  ,  in, AcceleratorRead);

From e9479929570e3b6e000f60b6368d7981666fac7c Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Mon, 29 Mar 2021 20:04:06 +0200
Subject: [PATCH 70/82] Improved force terms

---
 Grid/tensors/Tensor_outer.h | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/Grid/tensors/Tensor_outer.h b/Grid/tensors/Tensor_outer.h
index 4902c22f..a32a2a91 100644
--- a/Grid/tensors/Tensor_outer.h
+++ b/Grid/tensors/Tensor_outer.h
@@ -34,6 +34,16 @@ NAMESPACE_BEGIN(Grid);
 // outerProduct Scalar x Scalar -> Scalar
 //              Vector x Vector -> Matrix
 ///////////////////////////////////////////////////////////////////////////////////////
+template<class CC,IfComplex<CC> = 0>
+accelerator_inline CC outerProduct(const CC &l, const CC& r)
+{
+  return l*conj(r);
+}
+template<class RR,IfReal<RR> = 0>
+accelerator_inline RR outerProduct(const RR &l, const RR& r)
+{
+  return l*r;
+}
 
 template<class l,class r,int N> accelerator_inline
 auto outerProduct (const iVector<l,N>& lhs,const iVector<r,N>& rhs) -> iMatrix<decltype(outerProduct(lhs._internal[0],rhs._internal[0])),N>
@@ -57,17 +67,6 @@ auto outerProduct (const iScalar<l>& lhs,const iScalar<r>& rhs) -> iScalar<declt
   return ret;
 }
 
-template<class CC,IfComplex<CC> = 0>
-accelerator_inline CC outerProduct(const CC &l, const CC& r)
-{
-  return l*conj(r);
-}
-template<class RR,IfReal<RR> = 0>
-accelerator_inline RR outerProduct(const RR &l, const RR& r)
-{
-  return l*r;
-}
-
 NAMESPACE_END(Grid);
 
 #endif

From a7fb25adf66996f5de6861228774c803ef87bd4a Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Mon, 29 Mar 2021 21:44:14 +0200
Subject: [PATCH 71/82] Make Cshift fields static to avoid repeated reallocaate
 overhead

---
 Grid/cshift/Cshift_mpi.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Grid/cshift/Cshift_mpi.h b/Grid/cshift/Cshift_mpi.h
index 375d004e..7e93e260 100644
--- a/Grid/cshift/Cshift_mpi.h
+++ b/Grid/cshift/Cshift_mpi.h
@@ -122,8 +122,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
   assert(shift<fd);
   
   int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
-  cshiftVector<vobj> send_buf(buffer_size);
-  cshiftVector<vobj> recv_buf(buffer_size);
+  static cshiftVector<vobj> send_buf; send_buf.resize(buffer_size);
+  static cshiftVector<vobj> recv_buf; recv_buf.resize(buffer_size);
     
   int cb= (cbmask==0x2)? Odd : Even;
   int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
@@ -198,8 +198,8 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
   //  int words = sizeof(vobj)/sizeof(vector_type);
 
-  std::vector<cshiftVector<scalar_object> >  send_buf_extract(Nsimd);
-  std::vector<cshiftVector<scalar_object> >  recv_buf_extract(Nsimd);
+  static std::vector<cshiftVector<scalar_object> >  send_buf_extract; send_buf_extract.resize(Nsimd);
+  static std::vector<cshiftVector<scalar_object> >  recv_buf_extract; recv_buf_extract.resize(Nsimd);
   scalar_object *  recv_buf_extract_mpi;
   scalar_object *  send_buf_extract_mpi;
  
@@ -294,8 +294,8 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
   assert(shift<fd);
   
   int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
-  cshiftVector<vobj> send_buf_v(buffer_size);
-  cshiftVector<vobj> recv_buf_v(buffer_size);
+  static cshiftVector<vobj> send_buf_v; send_buf_v.resize(buffer_size);
+  static cshiftVector<vobj> recv_buf_v; recv_buf_v.resize(buffer_size);
   vobj *send_buf;
   vobj *recv_buf;
   {
@@ -381,8 +381,8 @@ template<class vobj> void  Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
   int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
   //  int words = sizeof(vobj)/sizeof(vector_type);
 
-  std::vector<cshiftVector<scalar_object> >  send_buf_extract(Nsimd);
-  std::vector<cshiftVector<scalar_object> >  recv_buf_extract(Nsimd);
+  static std::vector<cshiftVector<scalar_object> >  send_buf_extract; send_buf_extract.resize(Nsimd);
+  static std::vector<cshiftVector<scalar_object> >  recv_buf_extract; recv_buf_extract.resize(Nsimd);
   scalar_object *  recv_buf_extract_mpi;
   scalar_object *  send_buf_extract_mpi;
   {

From addeb621a7015cb690a08d4650e8f2b10568fbf7 Mon Sep 17 00:00:00 2001
From: Andrew Zhen Ning Yong <andrew.yong@ed.ac.uk>
Date: Tue, 6 Apr 2021 13:45:37 +0100
Subject: [PATCH 72/82] Implemented tadpole operator for Shamir action.

---
 .../implementation/CayleyFermion5DImplementation.h | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h b/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h
index f11e9c44..c3e0f821 100644
--- a/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h
+++ b/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h
@@ -880,11 +880,23 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
   }
 
   std::vector<RealD> G_s(Ls,1.0);
+  Integer sign = 1; // sign flip for vector/tadpole
   if ( curr_type == Current::Axial ) {
     for(int s=0;s<Ls/2;s++){
       G_s[s] = -1.0;
     }
   }
+  else if ( curr_type == Current::Tadpole ) {
+    auto b=this->_b;
+    auto c=this->_c;
+    if ( b == 1 && c == 0 ) {
+      sign = -1;    
+    }
+    else {
+      std::cerr << "Error: Tadpole implementation currently unavailable for non-Shamir actions." << std::endl;
+      assert(b==1 && c==0);
+    }
+  }
 
   for(int s=0;s<Ls;s++){
 
@@ -907,7 +919,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
 
     tmp    = Cshift(tmp,mu,1);
     Impl::multLinkField(Utmp,this->Umu,tmp,mu);
-    tmp    = G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop
+    tmp    = sign*G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop
     tmp    = where((lcoor>=tmin),tmp,zz); // Mask the time 
     L_Q    = where((lcoor<=tmax),tmp,zz); // Position of current complicated
 

From 980e721f6e25864cadfea3611a9c1a052d8d05c4 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@users.noreply.github.com>
Date: Tue, 13 Apr 2021 09:33:01 -0400
Subject: [PATCH 73/82] Update MetaData.h

---
 Grid/parallelIO/MetaData.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/parallelIO/MetaData.h b/Grid/parallelIO/MetaData.h
index d30ba523..af8b3f76 100644
--- a/Grid/parallelIO/MetaData.h
+++ b/Grid/parallelIO/MetaData.h
@@ -128,7 +128,7 @@ inline void MachineCharacteristics(FieldMetaData &header)
   std::time_t t = std::time(nullptr);
   std::tm tm_ = *std::localtime(&t);
   std::ostringstream oss; 
-  //      oss << std::put_time(&tm_, "%c %Z");
+  oss << std::put_time(&tm_, "%c %Z");
   header.creation_date = oss.str();
   header.archive_date  = header.creation_date;
 

From 86e11743ca3bb52ebfbedce8584b95b7be6f7fe0 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Tue, 20 Apr 2021 10:19:11 -0400
Subject: [PATCH 74/82]  set twists

---
 tests/forces/Test_gp_rect_force.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/forces/Test_gp_rect_force.cc b/tests/forces/Test_gp_rect_force.cc
index 98ebb2fa..e277ea6b 100644
--- a/tests/forces/Test_gp_rect_force.cc
+++ b/tests/forces/Test_gp_rect_force.cc
@@ -29,7 +29,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 using namespace std;
 using namespace Grid;
- ;
 
  
 
@@ -59,6 +58,10 @@ int main (int argc, char ** argv)
   double beta = 1.0;
   double c1   = 0.331;
 
+  const int nu = 1;
+  std::vector<int> twists(Nd,0);
+  twists[nu] = 1;
+  ConjugateGimplD::setDirections(twists);
   ConjugatePlaqPlusRectangleActionR Action(beta,c1);
   //ConjugateWilsonGaugeActionR Action(beta);
   //WilsonGaugeActionR Action(beta);

From dbe210dd53405303cf57374c9b658902cbf8072a Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sun, 25 Apr 2021 10:25:59 -0400
Subject: [PATCH 75/82] Open the ens_id

---
 Grid/parallelIO/NerscIO.h | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/Grid/parallelIO/NerscIO.h b/Grid/parallelIO/NerscIO.h
index 3ebdf0cc..1ffda074 100644
--- a/Grid/parallelIO/NerscIO.h
+++ b/Grid/parallelIO/NerscIO.h
@@ -205,11 +205,22 @@ public:
     std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
   }
 
+  // Preferred interface
+  template<class GaugeStats=PeriodicGaugeStatistics>
+  static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
+					std::string file, 
+					std::string ens_id    = std::string("UKQCD"),
+					std::string ens_label = std::string("DWF"))
+  {
+    writeConfiguration(Umu,file,0,1,ens_id,ens_label);
+  }
   template<class GaugeStats=PeriodicGaugeStatistics>
   static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
 					std::string file, 
 					int two_row,
-					int bits32)
+					int bits32,
+					std::string ens_id    = std::string("UKQCD"),
+					std::string ens_label = std::string("DWF"))
   {
     typedef vLorentzColourMatrixD vobj;
     typedef typename vobj::scalar_object sobj;
@@ -219,8 +230,8 @@ public:
     // Following should become arguments
     ///////////////////////////////////////////
     header.sequence_number = 1;
-    header.ensemble_id     = "UKQCD";
-    header.ensemble_label  = "DWF";
+    header.ensemble_id     = ens_id;
+    header.ensemble_label  = ens_label;
 
     typedef LorentzColourMatrixD fobj3D;
     typedef LorentzColour2x3D    fobj2D;
@@ -232,7 +243,7 @@ public:
     GaugeStats Stats; Stats(Umu,header);
     MachineCharacteristics(header);
 
-	uint64_t offset;
+    uint64_t offset;
 
     // Sod it -- always write 3x3 double
     header.floating_point = std::string("IEEE64BIG");

From 955a8113ded53fc55c204fc107832933ad120c2c Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sun, 25 Apr 2021 10:36:38 -0400
Subject: [PATCH 76/82] Expose label only to reduce number of parameters

---
 Grid/parallelIO/NerscIO.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Grid/parallelIO/NerscIO.h b/Grid/parallelIO/NerscIO.h
index 1ffda074..04aae5d8 100644
--- a/Grid/parallelIO/NerscIO.h
+++ b/Grid/parallelIO/NerscIO.h
@@ -209,7 +209,6 @@ public:
   template<class GaugeStats=PeriodicGaugeStatistics>
   static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
 					std::string file, 
-					std::string ens_id    = std::string("UKQCD"),
 					std::string ens_label = std::string("DWF"))
   {
     writeConfiguration(Umu,file,0,1,ens_id,ens_label);
@@ -219,7 +218,6 @@ public:
 					std::string file, 
 					int two_row,
 					int bits32,
-					std::string ens_id    = std::string("UKQCD"),
 					std::string ens_label = std::string("DWF"))
   {
     typedef vLorentzColourMatrixD vobj;
@@ -230,7 +228,7 @@ public:
     // Following should become arguments
     ///////////////////////////////////////////
     header.sequence_number = 1;
-    header.ensemble_id     = ens_id;
+    header.ensemble_id     = std::string("UKQCD");
     header.ensemble_label  = ens_label;
 
     typedef LorentzColourMatrixD fobj3D;

From d45c868656b4e28452946f90a6e71cdf12c21cf2 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sun, 25 Apr 2021 10:53:34 -0400
Subject: [PATCH 77/82] Change interface

---
 Grid/parallelIO/NerscIO.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Grid/parallelIO/NerscIO.h b/Grid/parallelIO/NerscIO.h
index 04aae5d8..99011e25 100644
--- a/Grid/parallelIO/NerscIO.h
+++ b/Grid/parallelIO/NerscIO.h
@@ -211,7 +211,7 @@ public:
 					std::string file, 
 					std::string ens_label = std::string("DWF"))
   {
-    writeConfiguration(Umu,file,0,1,ens_id,ens_label);
+    writeConfiguration(Umu,file,0,1,ens_label);
   }
   template<class GaugeStats=PeriodicGaugeStatistics>
   static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,

From 8cd4263974060f9af3b002604d9036e2552cc307 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sun, 25 Apr 2021 22:20:37 -0400
Subject: [PATCH 78/82] Tests compile

---
 tests/debug/Test_heatbath_dwf_eofa.cc            | 6 ++++--
 tests/debug/Test_heatbath_dwf_eofa_gparity.cc    | 7 +++++--
 tests/debug/Test_heatbath_mobius_eofa.cc         | 6 ++++--
 tests/debug/Test_heatbath_mobius_eofa_gparity.cc | 6 ++++--
 tests/forces/Test_momentum_filter.cc             | 4 +++-
 5 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/tests/debug/Test_heatbath_dwf_eofa.cc b/tests/debug/Test_heatbath_dwf_eofa.cc
index 9d453a96..e1c18021 100644
--- a/tests/debug/Test_heatbath_dwf_eofa.cc
+++ b/tests/debug/Test_heatbath_dwf_eofa.cc
@@ -66,7 +66,9 @@ int main(int argc, char** argv)
   // Set up RNGs
   std::vector<int> seeds4({1, 2, 3, 4});
   std::vector<int> seeds5({5, 6, 7, 8});
+  GridSerialRNG sRNG;
   GridParallelRNG RNG5(FGrid);
+  sRNG.SeedFixedIntegers(seeds5);
   RNG5.SeedFixedIntegers(seeds5);
   GridParallelRNG RNG4(UGrid);
   RNG4.SeedFixedIntegers(seeds4);
@@ -84,7 +86,7 @@ int main(int argc, char** argv)
     ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, false);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu,sRNG, RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
@@ -94,7 +96,7 @@ int main(int argc, char** argv)
     ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, true);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu,sRNG, RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
diff --git a/tests/debug/Test_heatbath_dwf_eofa_gparity.cc b/tests/debug/Test_heatbath_dwf_eofa_gparity.cc
index 22cc1e90..7eabfc65 100644
--- a/tests/debug/Test_heatbath_dwf_eofa_gparity.cc
+++ b/tests/debug/Test_heatbath_dwf_eofa_gparity.cc
@@ -74,6 +74,9 @@ int main(int argc, char** argv)
   RNG5.SeedFixedIntegers(seeds5);
   GridParallelRNG RNG4(UGrid);
   RNG4.SeedFixedIntegers(seeds4);
+  GridSerialRNG sRNG;
+  RNG4.SeedFixedIntegers(seeds4);
+  sRNG.SeedFixedIntegers(seeds5);
 
   // Random gauge field
   LatticeGaugeField Umu(UGrid);
@@ -90,7 +93,7 @@ int main(int argc, char** argv)
     ConjugateGradient<FermionField> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, false);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu,sRNG, RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
@@ -100,7 +103,7 @@ int main(int argc, char** argv)
     ConjugateGradient<FermionField> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, true);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu,sRNG, RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
diff --git a/tests/debug/Test_heatbath_mobius_eofa.cc b/tests/debug/Test_heatbath_mobius_eofa.cc
index 4cf4bf53..48806642 100644
--- a/tests/debug/Test_heatbath_mobius_eofa.cc
+++ b/tests/debug/Test_heatbath_mobius_eofa.cc
@@ -68,8 +68,10 @@ int main(int argc, char** argv)
   // Set up RNGs
   std::vector<int> seeds4({1, 2, 3, 4});
   std::vector<int> seeds5({5, 6, 7, 8});
+  GridSerialRNG sRNG;
   GridParallelRNG RNG5(FGrid);
   RNG5.SeedFixedIntegers(seeds5);
+  sRNG.SeedFixedIntegers(seeds5);
   GridParallelRNG RNG4(UGrid);
   RNG4.SeedFixedIntegers(seeds4);
 
@@ -86,7 +88,7 @@ int main(int argc, char** argv)
     ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, false);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu, sRNG,RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
@@ -96,7 +98,7 @@ int main(int argc, char** argv)
     ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, true);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu, sRNG,RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
diff --git a/tests/debug/Test_heatbath_mobius_eofa_gparity.cc b/tests/debug/Test_heatbath_mobius_eofa_gparity.cc
index 2fcb4b9f..52447e5e 100644
--- a/tests/debug/Test_heatbath_mobius_eofa_gparity.cc
+++ b/tests/debug/Test_heatbath_mobius_eofa_gparity.cc
@@ -73,7 +73,9 @@ int main(int argc, char** argv)
   std::vector<int> seeds4({1, 2, 3, 4});
   std::vector<int> seeds5({5, 6, 7, 8});
   GridParallelRNG RNG5(FGrid);
+  GridSerialRNG   sRNG;
   RNG5.SeedFixedIntegers(seeds5);
+  sRNG.SeedFixedIntegers(seeds5);
   GridParallelRNG RNG4(UGrid);
   RNG4.SeedFixedIntegers(seeds4);
 
@@ -91,7 +93,7 @@ int main(int argc, char** argv)
     ConjugateGradient<FermionField> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, false);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu, sRNG, RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
@@ -101,7 +103,7 @@ int main(int argc, char** argv)
     ConjugateGradient<FermionField> CG(1.0e-12, 5000);
     ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, true);
 
-    Meofa.refresh(Umu, RNG5);
+    Meofa.refresh(Umu, sRNG, RNG5);
     printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
   }
 
diff --git a/tests/forces/Test_momentum_filter.cc b/tests/forces/Test_momentum_filter.cc
index 856ea0f2..794b5fa0 100644
--- a/tests/forces/Test_momentum_filter.cc
+++ b/tests/forces/Test_momentum_filter.cc
@@ -61,7 +61,9 @@ int main (int argc, char ** argv)
   std::vector<int> seeds({1,2,3,4});
 
   GridParallelRNG          pRNG(&Grid);
+  GridSerialRNG            sRNG;
   pRNG.SeedFixedIntegers(seeds);
+  sRNG.SeedFixedIntegers(seeds);
 
   typedef PeriodicGimplR Gimpl;
   typedef WilsonGaugeAction<Gimpl> GaugeAction;
@@ -115,7 +117,7 @@ int main (int argc, char ** argv)
   
   integrator.setMomentumFilter(filter);
 
-  integrator.refresh(U, pRNG); //doesn't actually change the gauge field
+  integrator.refresh(U, sRNG, pRNG); //doesn't actually change the gauge field
 
   //Check the momentum is zero on the boundary
   const auto &P = integrator.getMomentum();

From 009ccd581ede8faf0ba748fa49a1757419106e23 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Mon, 26 Apr 2021 10:36:33 +0100
Subject: [PATCH 79/82] bugfix 3D stout smearing

---
 Grid/qcd/smearing/StoutSmearing.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Grid/qcd/smearing/StoutSmearing.h b/Grid/qcd/smearing/StoutSmearing.h
index ed2ccdb6..629f81e2 100644
--- a/Grid/qcd/smearing/StoutSmearing.h
+++ b/Grid/qcd/smearing/StoutSmearing.h
@@ -89,11 +89,12 @@ public:
     SmearBase->smear(C, U);
 
     for (int mu = 0; mu < Nd; mu++) {
-      if( mu == OrthogDim )
+      Umu = peekLorentz(U, mu);
+      if( mu == OrthogDim ){
         tmp = 1.0;  // Don't smear in the orthogonal direction
+      }
       else {
         tmp = peekLorentz(C, mu);
-        Umu = peekLorentz(U, mu);
         iq_mu = Ta(
                    tmp *
                    adj(Umu));  // iq_mu = Ta(Omega_mu) to match the signs with the paper

From cf2923d5ddb9c190ebb78efadab281e5a06ba247 Mon Sep 17 00:00:00 2001
From: Felix Erben <felix.erben@ed.ac.uk>
Date: Tue, 27 Apr 2021 16:53:37 +0100
Subject: [PATCH 80/82] Jamie's fix

---
 Grid/qcd/smearing/StoutSmearing.h | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/Grid/qcd/smearing/StoutSmearing.h b/Grid/qcd/smearing/StoutSmearing.h
index 629f81e2..6ee78e8c 100644
--- a/Grid/qcd/smearing/StoutSmearing.h
+++ b/Grid/qcd/smearing/StoutSmearing.h
@@ -85,22 +85,18 @@ public:
 
     std::cout << GridLogDebug << "Stout smearing started\n";
 
-    // Smear the configurations
+    // C contains the staples multiplied by some rho
+    u_smr = U ; // set the smeared field to the current gauge field
     SmearBase->smear(C, U);
 
     for (int mu = 0; mu < Nd; mu++) {
+      if( mu == OrthogDim ) continue ;
+      // u_smr = exp(iQ_mu)*U_mu apart from Orthogdim
       Umu = peekLorentz(U, mu);
-      if( mu == OrthogDim ){
-        tmp = 1.0;  // Don't smear in the orthogonal direction
-      }
-      else {
-        tmp = peekLorentz(C, mu);
-        iq_mu = Ta(
-                   tmp *
-                   adj(Umu));  // iq_mu = Ta(Omega_mu) to match the signs with the paper
-        exponentiate_iQ(tmp, iq_mu);
-      }
-      pokeLorentz(u_smr, tmp * Umu, mu);  // u_smr = exp(iQ_mu)*U_mu
+      tmp = peekLorentz(C, mu);
+      iq_mu = Ta( tmp * adj(Umu));  
+      exponentiate_iQ(tmp, iq_mu);
+      pokeLorentz(u_smr, tmp * Umu, mu);
     }
     std::cout << GridLogDebug << "Stout smearing completed\n";
   };

From 834f536b5f426aa0b3a334a89b37d8da39fb4238 Mon Sep 17 00:00:00 2001
From: u61464 <u61464@login-2.aidevcloud>
Date: Tue, 4 May 2021 08:40:18 -0700
Subject: [PATCH 81/82] Fastest option on SyCL is now std::complex

---
 Grid/tensors/Tensor_SIMT.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Grid/tensors/Tensor_SIMT.h b/Grid/tensors/Tensor_SIMT.h
index 672f385f..0a7d3382 100644
--- a/Grid/tensors/Tensor_SIMT.h
+++ b/Grid/tensors/Tensor_SIMT.h
@@ -65,7 +65,8 @@ void coalescedWriteNonTemporal(vobj & __restrict__ vec,const vobj & __restrict__
 #else
 
 
-#ifndef GRID_SYCL
+//#ifndef GRID_SYCL
+#if 1
 // Use the scalar as our own complex on GPU ... thrust::complex or std::complex
 template<class vsimd,IfSimd<vsimd> = 0> accelerator_inline
 typename vsimd::scalar_type

From 8cfc7342cde4b93d1de8f41a6f909ddd6d5d351f Mon Sep 17 00:00:00 2001
From: u61464 <u61464@s011-n002.aidevcloud>
Date: Wed, 5 May 2021 14:17:18 -0700
Subject: [PATCH 82/82] staggered hand unroll read coalesce

---
 Grid/qcd/action/fermion/Fermion.h             |   6 -
 Grid/qcd/action/fermion/FermionOperatorImpl.h |   5 +-
 .../implementation/StaggeredKernelsAsm.h      |   5 +-
 .../implementation/StaggeredKernelsHand.h     | 203 +++++++++---------
 4 files changed, 106 insertions(+), 113 deletions(-)

diff --git a/Grid/qcd/action/fermion/Fermion.h b/Grid/qcd/action/fermion/Fermion.h
index 16252340..09777204 100644
--- a/Grid/qcd/action/fermion/Fermion.h
+++ b/Grid/qcd/action/fermion/Fermion.h
@@ -291,12 +291,6 @@ typedef ImprovedStaggeredFermion5D<StaggeredImplR> ImprovedStaggeredFermion5DR;
 typedef ImprovedStaggeredFermion5D<StaggeredImplF> ImprovedStaggeredFermion5DF;
 typedef ImprovedStaggeredFermion5D<StaggeredImplD> ImprovedStaggeredFermion5DD;
 
-#ifndef GRID_CUDA
-typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplR> ImprovedStaggeredFermionVec5dR;
-typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplF> ImprovedStaggeredFermionVec5dF;
-typedef ImprovedStaggeredFermion5D<StaggeredVec5dImplD> ImprovedStaggeredFermionVec5dD;
-#endif
-
 NAMESPACE_END(Grid);
 
 ////////////////////
diff --git a/Grid/qcd/action/fermion/FermionOperatorImpl.h b/Grid/qcd/action/fermion/FermionOperatorImpl.h
index 9345c0e6..56aaca12 100644
--- a/Grid/qcd/action/fermion/FermionOperatorImpl.h
+++ b/Grid/qcd/action/fermion/FermionOperatorImpl.h
@@ -183,7 +183,8 @@ NAMESPACE_CHECK(ImplStaggered);
 /////////////////////////////////////////////////////////////////////////////
 // Single flavour one component spinors with colour index. 5d vec
 /////////////////////////////////////////////////////////////////////////////
-#include <Grid/qcd/action/fermion/StaggeredVec5dImpl.h> 
-NAMESPACE_CHECK(ImplStaggered5dVec);  
+// Deprecate Vec5d
+//#include <Grid/qcd/action/fermion/StaggeredVec5dImpl.h> 
+//NAMESPACE_CHECK(ImplStaggered5dVec);  
 
 
diff --git a/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h b/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h
index 63fd2a2f..e9cacbcf 100644
--- a/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h
+++ b/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h
@@ -680,7 +680,8 @@ void StaggeredKernels<Impl>::DhopSiteAsm(StencilView &st,
   gauge2 =(uint64_t)&UU[sU]( Z );				\
   gauge3 =(uint64_t)&UU[sU]( T ); 
   
-
+#undef STAG_VEC5D
+#ifdef STAG_VEC5D
   // This is the single precision 5th direction vectorised kernel
 #include <Grid/simd/Intel512single.h>
 template <> void StaggeredKernels<StaggeredVec5dImplF>::DhopSiteAsm(StencilView &st,
@@ -790,7 +791,7 @@ template <> void StaggeredKernels<StaggeredVec5dImplD>::DhopSiteAsm(StencilView
 #endif
 }
    
-   
+#endif   
 
 
 #define PERMUTE_DIR3 __asm__ (	\
diff --git a/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h b/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h
index 6bcb22b4..2b6087bc 100644
--- a/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h
+++ b/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h
@@ -32,25 +32,50 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 NAMESPACE_BEGIN(Grid);
 
-#define LOAD_CHI(b)		\
+#ifdef GRID_SIMT
+
+#define LOAD_CHI(ptype,b)			\
+  const SiteSpinor & ref (b[offset]);				\
+  Chi_0=coalescedReadPermute<ptype>(ref()()(0),perm,lane);	\
+  Chi_1=coalescedReadPermute<ptype>(ref()()(1),perm,lane);	\
+  Chi_2=coalescedReadPermute<ptype>(ref()()(2),perm,lane);
+
+#define LOAD_CHI_COMMS(b)		\
   const SiteSpinor & ref (b[offset]);	\
-    Chi_0=ref()()(0);\
-    Chi_1=ref()()(1);\
-    Chi_2=ref()()(2);
+  Chi_0=coalescedRead(ref()()(0),lane);	\
+  Chi_1=coalescedRead(ref()()(1),lane);	\
+  Chi_2=coalescedRead(ref()()(2),lane);
+
+#define PERMUTE_DIR(dir)	;
+#else
+#define LOAD_CHI(ptype,b)      LOAD_CHI_COMMS(b)
+
+#define LOAD_CHI_COMMS(b)		\
+  const SiteSpinor & ref (b[offset]);	\
+  Chi_0=ref()()(0);			\
+  Chi_1=ref()()(1);			\
+  Chi_2=ref()()(2);
+
+#define PERMUTE_DIR(dir)			\
+  permute##dir(Chi_0,Chi_0);			\
+  permute##dir(Chi_1,Chi_1);			\
+  permute##dir(Chi_2,Chi_2);
+
+#endif
 
 
 // To splat or not to splat depends on the implementation
 #define MULT(A,UChi)				\
   auto & ref(U[sU](A));			\
-   Impl::loadLinkElement(U_00,ref()(0,0));      \
-   Impl::loadLinkElement(U_10,ref()(1,0));      \
-   Impl::loadLinkElement(U_20,ref()(2,0));      \
-   Impl::loadLinkElement(U_01,ref()(0,1));      \
-   Impl::loadLinkElement(U_11,ref()(1,1));      \
-   Impl::loadLinkElement(U_21,ref()(2,1));      \
-   Impl::loadLinkElement(U_02,ref()(0,2));     \
-   Impl::loadLinkElement(U_12,ref()(1,2));     \
-   Impl::loadLinkElement(U_22,ref()(2,2));     \
+    U_00=coalescedRead(ref()(0,0),lane);				\
+    U_10=coalescedRead(ref()(1,0),lane);				\
+    U_20=coalescedRead(ref()(2,0),lane);				\
+    U_01=coalescedRead(ref()(0,1),lane);				\
+    U_11=coalescedRead(ref()(1,1),lane);				\
+    U_21=coalescedRead(ref()(2,1),lane);				\
+    U_02=coalescedRead(ref()(0,2),lane);				\
+    U_12=coalescedRead(ref()(1,2),lane);				\
+    U_22=coalescedRead(ref()(2,2),lane);				\
     UChi ## _0  = U_00*Chi_0;	       \
     UChi ## _1  = U_10*Chi_0;\
     UChi ## _2  = U_20*Chi_0;\
@@ -63,15 +88,15 @@ NAMESPACE_BEGIN(Grid);
 
 #define MULT_ADD(U,A,UChi)			\
   auto & ref(U[sU](A));			\
-   Impl::loadLinkElement(U_00,ref()(0,0));      \
-   Impl::loadLinkElement(U_10,ref()(1,0));      \
-   Impl::loadLinkElement(U_20,ref()(2,0));      \
-   Impl::loadLinkElement(U_01,ref()(0,1));      \
-   Impl::loadLinkElement(U_11,ref()(1,1));      \
-   Impl::loadLinkElement(U_21,ref()(2,1));      \
-   Impl::loadLinkElement(U_02,ref()(0,2));     \
-   Impl::loadLinkElement(U_12,ref()(1,2));     \
-   Impl::loadLinkElement(U_22,ref()(2,2));     \
+    U_00=coalescedRead(ref()(0,0),lane);				\
+    U_10=coalescedRead(ref()(1,0),lane);				\
+    U_20=coalescedRead(ref()(2,0),lane);				\
+    U_01=coalescedRead(ref()(0,1),lane);				\
+    U_11=coalescedRead(ref()(1,1),lane);				\
+    U_21=coalescedRead(ref()(2,1),lane);				\
+    U_02=coalescedRead(ref()(0,2),lane);				\
+    U_12=coalescedRead(ref()(1,2),lane);				\
+    U_22=coalescedRead(ref()(2,2),lane);				\
     UChi ## _0 += U_00*Chi_0;	       \
     UChi ## _1 += U_10*Chi_0;\
     UChi ## _2 += U_20*Chi_0;\
@@ -83,24 +108,18 @@ NAMESPACE_BEGIN(Grid);
     UChi ## _2 += U_22*Chi_2;
 
 
-#define PERMUTE_DIR(dir)			\
-  permute##dir(Chi_0,Chi_0);			\
-  permute##dir(Chi_1,Chi_1);			\
-  permute##dir(Chi_2,Chi_2);
-
-
 #define HAND_STENCIL_LEG_BASE(Dir,Perm,skew)	\
   SE=st.GetEntry(ptype,Dir+skew,sF);	\
   offset = SE->_offset;			\
   local  = SE->_is_local;		\
   perm   = SE->_permute;		\
   if ( local ) {						\
-    LOAD_CHI(in);					\
+    LOAD_CHI(Perm,in);						\
     if ( perm) {						\
       PERMUTE_DIR(Perm);					\
     }								\
   } else {							\
-    LOAD_CHI(buf);						\
+    LOAD_CHI_COMMS(buf);					\
   }								
 
 #define HAND_STENCIL_LEG_BEGIN(Dir,Perm,skew,even)		\
@@ -116,19 +135,18 @@ NAMESPACE_BEGIN(Grid);
   }
 
 
-
 #define HAND_STENCIL_LEG_INT(U,Dir,Perm,skew,even)	\
   SE=st.GetEntry(ptype,Dir+skew,sF);			\
   offset = SE->_offset;					\
   local  = SE->_is_local;				\
   perm   = SE->_permute;				\
   if ( local ) {					\
-    LOAD_CHI(in);				\
+    LOAD_CHI(Perm,in);					\
     if ( perm) {					\
       PERMUTE_DIR(Perm);				\
     }							\
   } else if ( st.same_node[Dir] ) {			\
-    LOAD_CHI(buf);					\
+    LOAD_CHI_COMMS(buf);				\
   }							\
   if (local || st.same_node[Dir] ) {		\
     MULT_ADD(U,Dir,even);				\
@@ -140,10 +158,32 @@ NAMESPACE_BEGIN(Grid);
   local  = SE->_is_local;				\
   if ((!local) && (!st.same_node[Dir]) ) {		\
     nmu++;							\
-    { LOAD_CHI(buf);	  }					\
+    { LOAD_CHI_COMMS(buf);	  }				\
     { MULT_ADD(U,Dir,even); }					\
   }								
 
+#define HAND_DECLARATIONS(Simd) \
+  Simd even_0;			\
+  Simd even_1;			\
+  Simd even_2;			\
+  Simd odd_0;			\
+  Simd odd_1;			\
+  Simd odd_2;		        \
+		      		\
+  Simd Chi_0;			\
+  Simd Chi_1;			\
+  Simd Chi_2;			\
+				\
+  Simd U_00;			\
+  Simd U_10;			\
+  Simd U_20;			\
+  Simd U_01;			\
+  Simd U_11;			\
+  Simd U_21;			\
+  Simd U_02;			\
+  Simd U_12;			\
+  Simd U_22;			
+  
 
 template <class Impl>
 template <int Naik> accelerator_inline
@@ -155,28 +195,14 @@ void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
 
-  Simd even_0; // 12 regs on knc
-  Simd even_1;
-  Simd even_2;
-  Simd odd_0; // 12 regs on knc
-  Simd odd_1;
-  Simd odd_2;
 
-  Simd Chi_0;    // two spinor; 6 regs
-  Simd Chi_1;
-  Simd Chi_2;
-  
-  Simd U_00;  // two rows of U matrix
-  Simd U_10;
-  Simd U_20;  
-  Simd U_01;
-  Simd U_11;
-  Simd U_21;  // 2 reg left.
-  Simd U_02;
-  Simd U_12;
-  Simd U_22; 
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+  typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
-  SiteSpinor result;
+  typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
+  calcSiteSpinor result;
   int offset,local,perm, ptype;
 
   StencilEntry *SE;
@@ -215,7 +241,7 @@ void StaggeredKernels<Impl>::DhopSiteHand(StencilView &st,
       result()()(1) = even_1 + odd_1;
       result()()(2) = even_2 + odd_2;
     }
-    vstream(out[sF],result);
+    coalescedWrite(out[sF],result);
   }
 }
 
@@ -230,28 +256,13 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
 
-  Simd even_0; // 12 regs on knc
-  Simd even_1;
-  Simd even_2;
-  Simd odd_0; // 12 regs on knc
-  Simd odd_1;
-  Simd odd_2;
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+  typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
-  Simd Chi_0;    // two spinor; 6 regs
-  Simd Chi_1;
-  Simd Chi_2;
-  
-  Simd U_00;  // two rows of U matrix
-  Simd U_10;
-  Simd U_20;  
-  Simd U_01;
-  Simd U_11;
-  Simd U_21;  // 2 reg left.
-  Simd U_02;
-  Simd U_12;
-  Simd U_22; 
-
-  SiteSpinor result;
+  typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
+  calcSiteSpinor result;
   int offset, ptype, local, perm;
 
   StencilEntry *SE;
@@ -261,8 +272,8 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
   //    int sF=s+LLs*sU;
   {
 
-    even_0 = Zero();    even_1 = Zero();    even_2 = Zero();
-     odd_0 = Zero();     odd_1 = Zero();     odd_2 = Zero();
+    zeroit(even_0);    zeroit(even_1);    zeroit(even_2);
+    zeroit(odd_0);    zeroit(odd_1);    zeroit(odd_2);
 
     skew = 0;
     HAND_STENCIL_LEG_INT(U,Xp,3,skew,even);  
@@ -294,7 +305,7 @@ void StaggeredKernels<Impl>::DhopSiteHandInt(StencilView &st,
       result()()(1) = even_1 + odd_1;
       result()()(2) = even_2 + odd_2;
     }
-    vstream(out[sF],result);
+    coalescedWrite(out[sF],result);
   }
 }
 
@@ -309,28 +320,13 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
 
-  Simd even_0; // 12 regs on knc
-  Simd even_1;
-  Simd even_2;
-  Simd odd_0; // 12 regs on knc
-  Simd odd_1;
-  Simd odd_2;
+  const int Nsimd = SiteHalfSpinor::Nsimd();
+  const int lane=acceleratorSIMTlane(Nsimd);
+  typedef decltype( coalescedRead( in[0]()()(0) )) Simt;
+  HAND_DECLARATIONS(Simt);
 
-  Simd Chi_0;    // two spinor; 6 regs
-  Simd Chi_1;
-  Simd Chi_2;
-  
-  Simd U_00;  // two rows of U matrix
-  Simd U_10;
-  Simd U_20;  
-  Simd U_01;
-  Simd U_11;
-  Simd U_21;  // 2 reg left.
-  Simd U_02;
-  Simd U_12;
-  Simd U_22; 
-
-  SiteSpinor result;
+  typedef decltype( coalescedRead( in[0] )) calcSiteSpinor;
+  calcSiteSpinor result;
   int offset, ptype, local;
 
   StencilEntry *SE;
@@ -340,8 +336,8 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
   //    int sF=s+LLs*sU;
   {
 
-    even_0 = Zero();    even_1 = Zero();    even_2 = Zero();
-     odd_0 = Zero();     odd_1 = Zero();     odd_2 = Zero();
+    zeroit(even_0);    zeroit(even_1);    zeroit(even_2);
+    zeroit(odd_0);    zeroit(odd_1);    zeroit(odd_2);
     int nmu=0;
     skew = 0;
     HAND_STENCIL_LEG_EXT(U,Xp,3,skew,even);  
@@ -374,7 +370,7 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
 	result()()(1) = even_1 + odd_1;
 	result()()(2) = even_2 + odd_2;
       }
-      out[sF] = out[sF] + result;
+      coalescedWrite(out[sF] , out(sF)+ result);
     }
   }
 }
@@ -397,6 +393,7 @@ void StaggeredKernels<Impl>::DhopSiteHandExt(StencilView &st,
 						     const FermionFieldView &in, FermionFieldView &out, int dag); \
 */
 #undef LOAD_CHI
+#undef HAND_DECLARATIONS
 
 NAMESPACE_END(Grid);