From ee11678b1fe595f5442ff800a655929c4e03a31b Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Tue, 3 Nov 2020 12:41:35 +0000 Subject: [PATCH 01/86] added Xi-to-Sigma rare decays --- Grid/qcd/utils/BaryonUtils.h | 169 +++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index 15516b56..e6b52a43 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -1292,4 +1292,173 @@ void BaryonUtils::SigmaToNucleonNonEye(const PropagatorField &qq_ti, } );//end loop over lattice sites } + +/*********************************************************************** + * The following code is for Xi -> Sigma rare hypeon decays * + **********************************************************************/ + +/* Dq_loop is a quark line from t_H to t_H + * Dd_spec is a quark line from t_i to t_f + * Ds_spec is a quark line from t_i to t_f + * Dd_tf is a quark line from t_f to t_H + * Ds_ti is a quark line from t_i to t_H */ +template +template +void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, + const mobj2 &Dd_spec, + const mobj2 &Ds_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_xi, + const Gamma GammaB_sigma, + robj &result) +{ + + Gamma g5(Gamma::Algebra::Gamma5); + + auto DdG = Dd_spec * GammaB_sigma; + auto GDs = GammaB_xi * Ds_spec; + // Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) + auto DsGDd = Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5; + // DsGDd * GammaB + auto DsGDdG = DsGDd * GammaB_sigma; + // GammaB * DsGDd + auto GDsGDd = GammaB_xi * DsGDd; + // GammaB * DsGDd * GammaB + auto GDsGDdG = GDsGDd * GammaB_sigma; + // \gamma_\mu^L * Dq_loop + auto trGDq = trace(GammaH * Dq_loop); + + for (int ie_s=0; ie_s < 6 ; ie_s++){ + int a_s = epsilon[ie_s][0]; //a + int b_s = epsilon[ie_s][1]; //b + int c_s = epsilon[ie_s][2]; //c + for (int ie_x=0; ie_x < 6 ; ie_x++){ + int a_x = epsilon[ie_x][0]; //a' + int b_x = epsilon[ie_x][1]; //b' + int c_x = epsilon[ie_x][2]; //c' + auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq; + for (int alpha_x=0; alpha_x +template +void BaryonUtils::XiToSigmaQ2EyeSite(const mobj &Dq_loop, + const mobj2 &Dd_spec, + const mobj2 &Ds_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_xi, + const Gamma GammaB_sigma, + robj &result) +{ + + Gamma g5(Gamma::Algebra::Gamma5); + + auto DdG = Dd_spec * GammaB_sigma; + auto GDs = GammaB_xi * Ds_spec; + // Ds * \gamma_\mu^L * Dq_loop * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) + auto DsGDqGDd = Ds_ti * Gamma_H * Dq_loop * Gamma_H * g5 * adj(Dd_tf) * g5; + // DsGDd * GammaB + auto DsGDqGDdG = DsGDqGDd * GammaB_sigma; + // GammaB * DsGDd + auto GDsGDqGDd = GammaB_xi * DsGDqGDd; + // GammaB * DsGDd * GammaB + auto GDsGDqGDdG = GDsGDqGDd * GammaB_sigma; + + for (int ie_s=0; ie_s < 6 ; ie_s++){ + int a_s = epsilon[ie_s][0]; //a + int b_s = epsilon[ie_s][1]; //b + int c_s = epsilon[ie_s][2]; //c + for (int ie_x=0; ie_x < 6 ; ie_x++){ + int a_x = epsilon[ie_x][0]; //a' + int b_x = epsilon[ie_x][1]; //b' + int c_x = epsilon[ie_x][2]; //c' + auto ee = epsilon_sgn[ie_s] * epsilon_sgn[ie_x]; + for (int alpha_x=0; alpha_x +template +void BaryonUtils::XiToSigmaEye(const PropagatorField &qq_loop, + const mobj &Dd_spec, + const mobj &Ds_spec, + const PropagatorField &qd_tf, + const PropagatorField &qs_ti, + const Gamma Gamma_H, + const Gamma GammaB_xi, + const Gamma GammaB_sigma, + const std::string op, + SpinMatrixField &stn_corr) +{ + + assert(Ns==4 && "Baryon code only implemented for N_spin = 4"); + assert(Nc==3 && "Baryon code only implemented for N_colour = 3"); + + GridBase *grid = qs_ti.Grid(); + + autoView( vcorr, stn_corr, CpuWrite); + autoView( vq_loop , qq_loop, CpuRead); + autoView( vd_tf , qd_tf, CpuRead); + autoView( vs_ti , qs_ti, CpuRead); + + accelerator_for(ss, grid->oSites(), grid->Nsimd(), { + auto Dq_loop = vq_loop[ss]; + auto Dd_tf = vd_tf[ss]; + auto Ds_ti = vs_ti[ss]; + sobj result=Zero(); + if(op == "Q1"){ + XiToSigmaQ1EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); + } else if(op == "Q2"){ + XiToSigmaQ2EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); + } else { + assert(0 && "Weak Operator not correctly specified"); + } + vcorr[ss] = result; + } );//end loop over lattice sites +} + + NAMESPACE_END(Grid); From a3de7026c8d091b4d42eba5e5cb8e57419e0963e Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Tue, 3 Nov 2020 12:51:50 +0000 Subject: [PATCH 02/86] bugfix --- Grid/qcd/utils/BaryonUtils.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index e6b52a43..8e6bf722 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -207,6 +207,26 @@ public: const Gamma GammaB_sigma, const Gamma GammaB_nucl, robj &result); + template + static void XiToSigmaQ1EyeSite(const mobj &Dq_loop, + const mobj2 &Dd_spec, + const mobj2 &Ds_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result); + template + static void XiToSigmaQ2EyeSite(const mobj &Dq_loop, + const mobj2 &Dd_spec, + const mobj2 &Ds_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result); public: template static void SigmaToNucleonEye(const PropagatorField &qq_loop, @@ -229,6 +249,17 @@ public: const Gamma GammaB_nucl, const std::string op, SpinMatrixField &stn_corr); + template + static void XiToSigmaEye(const PropagatorField &qq_loop, + const mobj &Dd_spec, + const mobj &Ds_spec, + const PropagatorField &qd_tf, + const PropagatorField &qs_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + const std::string op, + SpinMatrixField &stn_corr); }; template From 67023c334b16938c36af9f41647ba42891008e03 Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Tue, 3 Nov 2020 13:07:37 +0000 Subject: [PATCH 03/86] bugfix --- Grid/qcd/utils/BaryonUtils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index 8e6bf722..a281a669 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -1359,7 +1359,7 @@ void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, // GammaB * DsGDd * GammaB auto GDsGDdG = GDsGDd * GammaB_sigma; // \gamma_\mu^L * Dq_loop - auto trGDq = trace(GammaH * Dq_loop); + auto trGDq = trace(Gamma_H * Dq_loop); for (int ie_s=0; ie_s < 6 ; ie_s++){ int a_s = epsilon[ie_s][0]; //a From 4014dfd5b9a79223023ab2bee3789a7efde45945 Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Tue, 3 Nov 2020 16:13:08 +0000 Subject: [PATCH 04/86] first tested version --- Grid/qcd/utils/BaryonUtils.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index a281a669..f5bc1480 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -1359,7 +1359,7 @@ void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, // GammaB * DsGDd * GammaB auto GDsGDdG = GDsGDd * GammaB_sigma; // \gamma_\mu^L * Dq_loop - auto trGDq = trace(Gamma_H * Dq_loop); + auto trGDq = TensorRemove(trace(Gamma_H * Dq_loop)); for (int ie_s=0; ie_s < 6 ; ie_s++){ int a_s = epsilon[ie_s][0]; //a @@ -1369,7 +1369,7 @@ void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, int a_x = epsilon[ie_x][0]; //a' int b_x = epsilon[ie_x][1]; //b' int c_x = epsilon[ie_x][2]; //c' - auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq; + auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq; for (int alpha_x=0; alpha_x Date: Tue, 3 Nov 2020 20:03:09 +0000 Subject: [PATCH 05/86] speedup in Sigma-to-nucleon --- Grid/qcd/utils/BaryonUtils.h | 193 ++++++++++++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 4 deletions(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index f5bc1480..c0999a4a 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -1027,7 +1027,7 @@ void BaryonUtils::BaryonGamma3pt( * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -template +/*template template void BaryonUtils::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop, const mobj2 &Du_spec, @@ -1071,6 +1071,50 @@ void BaryonUtils::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop, }} } } +}*/ +template +template +void BaryonUtils::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop, + const mobj2 &Du_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result) +{ + + Gamma g5(Gamma::Algebra::Gamma5); + + auto DuG = Du_spec * GammaB_nucl; + // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) + auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5; + // Dq_loop * \gamma_\mu^L + auto trDqG = TensorRemove(trace(Dq_loop * Gamma_H)); + + for (int ie_n=0; ie_n < 6 ; ie_n++){ + int a_n = epsilon[ie_n][0]; //a + int b_n = epsilon[ie_n][1]; //b + int c_n = epsilon[ie_n][2]; //c + for (int ie_s=0; ie_s < 6 ; ie_s++){ + int a_s = epsilon[ie_s][0]; //a' + int b_s = epsilon[ie_s][1]; //b' + int c_s = epsilon[ie_s][2]; //c' + for (int alpha_s=0; alpha_s::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop, * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -template +/*template template void BaryonUtils::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti, const mobj &Du_tf, @@ -1130,6 +1174,55 @@ void BaryonUtils::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti, }} } } +}*/ +template +template +void BaryonUtils::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti, + const mobj &Du_tf, + const mobj2 &Du_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result) +{ + + Gamma g5(Gamma::Algebra::Gamma5); + + auto DuG = Du_spec * GammaB_nucl; + // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) + auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5; + // Du_ti * \gamma_\mu^L * adj(Du_tf) + auto DuGHDu = Du_ti * Gamma_H * g5 * adj(Du_tf) * g5; + auto DuGHDuG = DuGHDu * GammaB_nucl; + + for (int ie_n=0; ie_n < 6 ; ie_n++){ + int a_n = epsilon[ie_n][0]; //a + int b_n = epsilon[ie_n][1]; //b + int c_n = epsilon[ie_n][2]; //c + for (int ie_s=0; ie_s < 6 ; ie_s++){ + int a_s = epsilon[ie_s][0]; //a' + int b_s = epsilon[ie_s][1]; //b' + int c_s = epsilon[ie_s][2]; //c' + for (int alpha_s=0; alpha_s::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti, * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -template +/*template template void BaryonUtils::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop, const mobj2 &Du_spec, @@ -1181,6 +1274,47 @@ void BaryonUtils::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop, }}} } } +}*/ +template +template +void BaryonUtils::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop, + const mobj2 &Du_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result) +{ + + Gamma g5(Gamma::Algebra::Gamma5); + + auto DuG = Du_spec * GammaB_nucl; + // Gamma^B * Ds * \gamma_\mu^L * Dq_loop * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) + auto GDsGDqGDd = GammaB_sigma * Ds_ti * Gamma_H * Dq_loop * Gamma_H * g5 * adj(Dd_tf) * g5; + + for (int ie_n=0; ie_n < 6 ; ie_n++){ + int a_n = epsilon[ie_n][0]; //a + int b_n = epsilon[ie_n][1]; //b + int c_n = epsilon[ie_n][2]; //c + for (int ie_s=0; ie_s < 6 ; ie_s++){ + int a_s = epsilon[ie_s][0]; //a' + int b_s = epsilon[ie_s][1]; //b' + int c_s = epsilon[ie_s][2]; //c' + for (int alpha_s=0; alpha_s::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop, * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -template +/*template template void BaryonUtils::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti, const mobj &Du_tf, @@ -1240,6 +1374,57 @@ void BaryonUtils::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti, }}} } } +}*/ +template +template +void BaryonUtils::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti, + const mobj &Du_tf, + const mobj2 &Du_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result) +{ + + Gamma g5(Gamma::Algebra::Gamma5); + + auto DuG = Du_spec * GammaB_nucl; + // Gamma^B * Ds * \gamma_\mu^L * adj(Du) + auto GDsGDu = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Du_tf) * g5; + // GDsGDu * GammaB + auto GDsGDuG = GDsGDu * GammaB_nucl; + // Du * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) + auto DuGDd = Du_ti * Gamma_H * g5 * adj(Dd_tf) * g5; + + for (int ie_n=0; ie_n < 6 ; ie_n++){ + int a_n = epsilon[ie_n][0]; //a + int b_n = epsilon[ie_n][1]; //b + int c_n = epsilon[ie_n][2]; //c + for (int ie_s=0; ie_s < 6 ; ie_s++){ + int a_s = epsilon[ie_s][0]; //a' + int b_s = epsilon[ie_s][1]; //b' + int c_s = epsilon[ie_s][2]; //c' + auto ee = epsilon_sgn[ie_n] * epsilon_sgn[ie_s]; + for (int alpha_s=0; alpha_s From 3594ce877beb22161dcce972f14bd89598972fd6 Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Tue, 3 Nov 2020 20:04:30 +0000 Subject: [PATCH 06/86] speedup in Sigma-to-nucleon --- Grid/qcd/utils/BaryonUtils.h | 196 ----------------------------------- 1 file changed, 196 deletions(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index c0999a4a..8a4ff6ac 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -1027,51 +1027,6 @@ void BaryonUtils::BaryonGamma3pt( * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -/*template -template -void BaryonUtils::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop, - const mobj2 &Du_spec, - const mobj &Dd_tf, - const mobj &Ds_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - robj &result) -{ - - Gamma g5(Gamma::Algebra::Gamma5); - - auto DuG = Du_spec * GammaB_nucl; - // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) - auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5; - // Dq_loop * \gamma_\mu^L - auto DqG = Dq_loop * Gamma_H; - - for (int ie_n=0; ie_n < 6 ; ie_n++){ - int a_n = epsilon[ie_n][0]; //a - int b_n = epsilon[ie_n][1]; //b - int c_n = epsilon[ie_n][2]; //c - for (int ie_s=0; ie_s < 6 ; ie_s++){ - int a_s = epsilon[ie_s][0]; //a' - int b_s = epsilon[ie_s][1]; //b' - int c_s = epsilon[ie_s][2]; //c' - for (int alpha_s=0; alpha_s template void BaryonUtils::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop, @@ -1122,59 +1077,6 @@ void BaryonUtils::SigmaToNucleonQ1EyeSite(const mobj &Dq_loop, * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -/*template -template -void BaryonUtils::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti, - const mobj &Du_tf, - const mobj2 &Du_spec, - const mobj &Dd_tf, - const mobj &Ds_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - robj &result) -{ - - Gamma g5(Gamma::Algebra::Gamma5); - - auto DuG = Du_spec * GammaB_nucl; - auto adjDu = g5 * adj(Du_tf) * g5; - auto adjDuG = adjDu * GammaB_nucl; - // Gamma^B * Ds * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) - auto GDsGDd = GammaB_sigma * Ds_ti * Gamma_H * g5 * adj(Dd_tf) * g5; - // Dq_loop * \gamma_\mu^L - auto DuGH = Du_ti * Gamma_H; - - for (int ie_n=0; ie_n < 6 ; ie_n++){ - int a_n = epsilon[ie_n][0]; //a - int b_n = epsilon[ie_n][1]; //b - int c_n = epsilon[ie_n][2]; //c - for (int ie_s=0; ie_s < 6 ; ie_s++){ - int a_s = epsilon[ie_s][0]; //a' - int b_s = epsilon[ie_s][1]; //b' - int c_s = epsilon[ie_s][2]; //c' - for (int alpha_s=0; alpha_s template void BaryonUtils::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti, @@ -1230,51 +1132,6 @@ void BaryonUtils::SigmaToNucleonQ1NonEyeSite(const mobj &Du_ti, * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -/*template -template -void BaryonUtils::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop, - const mobj2 &Du_spec, - const mobj &Dd_tf, - const mobj &Ds_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - robj &result) -{ - - Gamma g5(Gamma::Algebra::Gamma5); - - auto DuG = Du_spec * GammaB_nucl; - // Gamma^B * Ds * \gamma_\mu^L - auto GDsG = GammaB_sigma * Ds_ti * Gamma_H; - // Dq_loop * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) - auto DqGDd = Dq_loop * Gamma_H * g5 * adj(Dd_tf) * g5; - - for (int ie_n=0; ie_n < 6 ; ie_n++){ - int a_n = epsilon[ie_n][0]; //a - int b_n = epsilon[ie_n][1]; //b - int c_n = epsilon[ie_n][2]; //c - for (int ie_s=0; ie_s < 6 ; ie_s++){ - int a_s = epsilon[ie_s][0]; //a' - int b_s = epsilon[ie_s][1]; //b' - int c_s = epsilon[ie_s][2]; //c' - for (int alpha_s=0; alpha_s template void BaryonUtils::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop, @@ -1322,59 +1179,6 @@ void BaryonUtils::SigmaToNucleonQ2EyeSite(const mobj &Dq_loop, * Du_spec is a quark line from t_i to t_f * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ -/*template -template -void BaryonUtils::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti, - const mobj &Du_tf, - const mobj2 &Du_spec, - const mobj &Dd_tf, - const mobj &Ds_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - robj &result) -{ - - Gamma g5(Gamma::Algebra::Gamma5); - - auto DuG = Du_spec * GammaB_nucl; - auto adjDu = g5 * adj(Du_tf) * g5; - auto adjDuG = adjDu * GammaB_nucl; - // Gamma^B * Ds * \gamma_\mu^L - auto GDsG = GammaB_sigma * Ds_ti * Gamma_H; - // Du * \gamma_\mu^L * (\gamma_5 * Dd^\dagger * \gamma_5) - auto DuGDd = Du_ti * Gamma_H * g5 * adj(Dd_tf) * g5; - - for (int ie_n=0; ie_n < 6 ; ie_n++){ - int a_n = epsilon[ie_n][0]; //a - int b_n = epsilon[ie_n][1]; //b - int c_n = epsilon[ie_n][2]; //c - for (int ie_s=0; ie_s < 6 ; ie_s++){ - int a_s = epsilon[ie_s][0]; //a' - int b_s = epsilon[ie_s][1]; //b' - int c_s = epsilon[ie_s][2]; //c' - for (int alpha_s=0; alpha_s template void BaryonUtils::SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti, From 2983b6fdf6485569c7d621bb22e4dcec23633e22 Mon Sep 17 00:00:00 2001 From: Michael Marshall <43034299+mmphys@users.noreply.github.com> Date: Sat, 23 Jan 2021 12:41:48 +0000 Subject: [PATCH 07/86] Optional (superficial) changes to make comparison with Hadrons WardIdentity module easier: use Schur solver; example of Hadrons random gauge init; logging updates; only solve reverse propagator if provided --- tests/debug/Test_cayley_mres.cc | 86 +++++++++++++++------------------ 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/tests/debug/Test_cayley_mres.cc b/tests/debug/Test_cayley_mres.cc index ea88885e..bfbc3cf7 100644 --- a/tests/debug/Test_cayley_mres.cc +++ b/tests/debug/Test_cayley_mres.cc @@ -33,13 +33,14 @@ using namespace Grid; template -void TestConserved(What & Ddwf, What & Ddwfrev, +void TestConserved(What & Ddwf, LatticeGaugeField &Umu, GridCartesian * FGrid, GridRedBlackCartesian * FrbGrid, GridCartesian * UGrid, GridRedBlackCartesian * UrbGrid, RealD mass, RealD M5, GridParallelRNG *RNG4, - GridParallelRNG *RNG5); + GridParallelRNG *RNG5, + What *Ddwfrev=nullptr); Gamma::Algebra Gmu [] = { Gamma::Algebra::GammaX, @@ -102,10 +103,11 @@ int main (int argc, char ** argv) GridRedBlackCartesian * FrbGridF = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGridF); - std::vector seeds4({1,2,3,4}); std::vector seeds5({5,6,7,8}); GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5); - GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); + GridParallelRNG RNG4(UGrid); + std::vector seeds4({1,2,3,4}); RNG4.SeedFixedIntegers(seeds4); + //const std::string seeds4{ "test-gauge-3000" }; RNG4.SeedUniqueString( seeds4 ); LatticeGaugeField Umu(UGrid); if( argc > 1 && argv[1][0] != '-' ) @@ -116,8 +118,8 @@ int main (int argc, char ** argv) } else { - std::cout<::ColdConfiguration(Umu); + std::cout<::ColdConfiguration(Umu); SU::HotConfiguration(RNG4,Umu); } @@ -127,7 +129,7 @@ int main (int argc, char ** argv) std::cout<(Ddwf,Ddwf,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5); + TestConserved(Ddwf,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5); RealD b=1.5;// Scale factor b+c=2, b-c=1 RealD c=0.5; @@ -137,13 +139,13 @@ int main (int argc, char ** argv) std::cout<(Dmob,Dmob,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5); + TestConserved(Dmob,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5); std::cout<(Dsham,Dsham,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5); + TestConserved(Dsham,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5); std::cout<(ZDmob,ZDmobrev,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5); + TestConserved(ZDmob,Umu,FGrid,FrbGrid,UGrid,UrbGrid,mass,M5,&RNG4,&RNG5,&ZDmobrev); Grid_finalize(); } @@ -161,22 +162,17 @@ int main (int argc, char ** argv) template -void TestConserved(Action & Ddwf, - Action & Ddwfrev, +void TestConserved(Action & Ddwf, LatticeGaugeField &Umu, GridCartesian * FGrid, GridRedBlackCartesian * FrbGrid, GridCartesian * UGrid, GridRedBlackCartesian * UrbGrid, RealD mass, RealD M5, GridParallelRNG *RNG4, - GridParallelRNG *RNG5) + GridParallelRNG *RNG5, + Action * Ddwfrev) { - int Ls=Ddwf.Ls; - - LatticePropagator phys_src(UGrid); - - std::vector U(4,UGrid); - - LatticePropagator seqsrc(FGrid); + LatticePropagator phys_src(UGrid); + LatticePropagator seqsrc(FGrid); LatticePropagator prop5(FGrid); LatticePropagator prop5rev(FGrid); LatticePropagator prop4(UGrid); @@ -194,9 +190,9 @@ void TestConserved(Action & Ddwf, phys_src=Zero(); pokeSite(kronecker,phys_src,coor); - MdagMLinearOperator HermOp(Ddwf); - MdagMLinearOperator HermOprev(Ddwfrev); ConjugateGradient CG(1.0e-16,100000); + SchurRedBlackDiagTwoSolve schur(CG); + ZeroGuesser zpg; for(int s=0;s(prop5,result5,s,c); LatticeFermion result4(UGrid); Ddwf.ExportPhysicalFermionSolution(result5,result4); FermToProp(prop4,result4,s,c); - Ddwfrev.ImportPhysicalFermionSource(src4,src5); - Ddwfrev.Mdag(src5,Mdagsrc5); - CG(HermOprev,Mdagsrc5,result5); + if( Ddwfrev ) { + Ddwfrev->ImportPhysicalFermionSource(src4,src5); + result5 = Zero(); + schur(*Ddwfrev,src5,result5,zpg); + } FermToProp(prop5rev,result5,s,c); } } @@ -251,11 +247,7 @@ void TestConserved(Action & Ddwf, PropToFerm(src5,seqsrc,s,c); LatticeFermion result5(FGrid); result5=Zero(); - - // CGNE - LatticeFermion Mdagsrc5 (FGrid); - Ddwf.Mdag(src5,Mdagsrc5); - CG(HermOp,Mdagsrc5,result5); + schur(Ddwf,src5,result5,zpg); LatticeFermion result4(UGrid); Ddwf.ExportPhysicalFermionSolution(result5,result4); @@ -276,10 +268,10 @@ void TestConserved(Action & Ddwf, Ddwf.ContractConservedCurrent(prop5rev,prop5,Vector_mu,phys_src,Current::Vector,Tdir); Ddwf.ContractJ5q(prop5,PJ5q); - PA = trace(g5*Axial_mu); - SV = trace(Vector_mu); - VV = trace(gT*Vector_mu); - PP = trace(adj(prop4)*prop4); + PA = trace(g5*Axial_mu); // Pseudoscalar-Axial conserved current + SV = trace(Vector_mu); // Scalar-Vector conserved current + VV = trace(gT*Vector_mu); // (local) Vector-Vector conserved current + PP = trace(adj(prop4)*prop4); // Pseudoscalar density // Spatial sum sliceSum(PA,sumPA,Tdir); @@ -288,15 +280,17 @@ void TestConserved(Action & Ddwf, sliceSum(PP,sumPP,Tdir); sliceSum(PJ5q,sumPJ5q,Tdir); - int Nt=sumPA.size(); + const int Nt{static_cast(sumPA.size())}; + std::cout< Date: Mon, 25 Jan 2021 15:09:36 +0000 Subject: [PATCH 08/86] Fix issue for GPU by ensuring accelerator_inline version of convertType is available for Grid::complex. This removes many warnings in Hadrons Simplify the SFINAE syntax and correct convertType for iScalar --- Grid/lattice/Lattice_transfer.h | 39 +++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/Grid/lattice/Lattice_transfer.h b/Grid/lattice/Lattice_transfer.h index 91de721f..c91fa4d1 100644 --- a/Grid/lattice/Lattice_transfer.h +++ b/Grid/lattice/Lattice_transfer.h @@ -97,6 +97,21 @@ accelerator_inline void convertType(ComplexF & out, const std::complex & out = in; } +template +accelerator_inline typename std::enable_if::value>::type +convertType(T & out, const T & in) { + out = in; +} + +// This would allow for conversions between GridFundamental types, but is not strictly needed as yet +/*template +accelerator_inline typename std::enable_if::value && isGridFundamental::value>::type +// Or to make this very broad, conversions between anything that's not a GridTensor could be allowed +//accelerator_inline typename std::enable_if::value && !isGridTensor::value>::type +convertType(T1 & out, const T2 & in) { + out = in; +}*/ + #ifdef GRID_SIMT accelerator_inline void convertType(vComplexF & out, const ComplexF & in) { ((ComplexF*)&out)[acceleratorSIMTlane(vComplexF::Nsimd())] = in; @@ -117,23 +132,20 @@ accelerator_inline void convertType(vComplexD2 & out, const vComplexF & in) { Optimization::PrecisionChange::StoD(in.v,out._internal[0].v,out._internal[1].v); } -template - accelerator_inline void convertType(iMatrix & out, const iMatrix & in); -template - accelerator_inline void convertType(iVector & out, const iVector & in); - -template::value, T1>::type* = nullptr> -accelerator_inline void convertType(T1 & out, const iScalar & in) { - convertType(out,in._internal); +template +accelerator_inline void convertType(iScalar & out, const iScalar & in) { + convertType(out._internal,in._internal); } -template::value, T1>::type* = nullptr> -accelerator_inline void convertType(T1 & out, const iScalar & in) { +template +accelerator_inline typename std::enable_if::value>::type +convertType(T1 & out, const iScalar & in) { convertType(out,in._internal); } template -accelerator_inline void convertType(iScalar & out, const T2 & in) { +accelerator_inline typename std::enable_if::value>::type +convertType(iScalar & out, const T2 & in) { convertType(out._internal,in); } @@ -150,11 +162,6 @@ accelerator_inline void convertType(iVector & out, const iVector & i convertType(out._internal[i],in._internal[i]); } -template::value, T>::type* = nullptr> -accelerator_inline void convertType(T & out, const T & in) { - out = in; -} - template accelerator_inline void convertType(Lattice & out, const Lattice & in) { autoView( out_v , out,AcceleratorWrite); From 9b9a53f87066b5c67b607f56cb73c56efa5d4243 Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Tue, 2 Feb 2021 13:06:43 +0000 Subject: [PATCH 09/86] ... --- Grid/qcd/utils/BaryonUtils.h | 53 ++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index 8a4ff6ac..94cc07b1 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -1350,15 +1350,19 @@ void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, // \gamma_\mu^L * Dq_loop auto trGDq = TensorRemove(trace(Gamma_H * Dq_loop)); + Real ee; + for (int ie_s=0; ie_s < 6 ; ie_s++){ - int a_s = epsilon[ie_s][0]; //a - int b_s = epsilon[ie_s][1]; //b - int c_s = epsilon[ie_s][2]; //c + int a_s = (ie_s < 3 ? ie_s : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a' + int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b' + int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c' + int eSgn_s = (ie_s < 3 ? 1 : -1); for (int ie_x=0; ie_x < 6 ; ie_x++){ - int a_x = epsilon[ie_x][0]; //a' - int b_x = epsilon[ie_x][1]; //b' - int c_x = epsilon[ie_x][2]; //c' - auto ee_GD = epsilon_sgn[ie_s] * epsilon_sgn[ie_x] * trGDq; + int a_x = (ie_x < 3 ? ie_x : (6-ie_x)%3 ); //epsilon[ie_x][0]; //a' + int b_x = (ie_x < 3 ? (ie_x+1)%3 : (8-ie_x)%3 ); //epsilon[ie_x][1]; //b' + int c_x = (ie_x < 3 ? (ie_x+2)%3 : (7-ie_x)%3 ); //epsilon[ie_x][2]; //c' + int eSgn_x = (ie_x < 3 ? 1 : -1); + ee = Real(eSgn_s * eSgn_x); for (int alpha_x=0; alpha_x::XiToSigmaEye(const PropagatorField &qq_loop, const Gamma GammaB_xi, const Gamma GammaB_sigma, const std::string op, - SpinMatrixField &stn_corr) + SpinMatrixField &xts_corr) { assert(Ns==4 && "Baryon code only implemented for N_spin = 4"); @@ -1459,24 +1463,31 @@ void BaryonUtils::XiToSigmaEye(const PropagatorField &qq_loop, GridBase *grid = qs_ti.Grid(); - autoView( vcorr, stn_corr, CpuWrite); - autoView( vq_loop , qq_loop, CpuRead); - autoView( vd_tf , qd_tf, CpuRead); - autoView( vs_ti , qs_ti, CpuRead); + autoView( vcorr , xts_corr , AcceleratorWrite); + autoView( vq_loop , qq_loop , AcceleratorRead); + autoView( vd_tf , qd_tf , AcceleratorRead); + autoView( vs_ti , qs_ti , AcceleratorRead); + + bool doQ1 = (op == "Q1"); + bool doQ2 = (op == "Q2"); + + Vector my_Dq_spec{Dd_spec,Ds_spec}; + mobj * Dq_spec_p = &my_Dq_spec[0]; accelerator_for(ss, grid->oSites(), grid->Nsimd(), { - auto Dq_loop = vq_loop[ss]; - auto Dd_tf = vd_tf[ss]; - auto Ds_ti = vs_ti[ss]; - sobj result=Zero(); - if(op == "Q1"){ - XiToSigmaQ1EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); - } else if(op == "Q2"){ - XiToSigmaQ2EyeSite(Dq_loop,Dd_spec,Ds_spec,Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); + auto Dq_loop = vq_loop(ss); + auto Dd_tf = vd_tf(ss); + auto Ds_ti = vs_ti(ss); + typedef decltype(coalescedRead(vcorr[0])) spinor; + spinor result=Zero(); + if(doQ1){ + XiToSigmaQ1EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[1],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); + } else if(doQ2){ + XiToSigmaQ2EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); } else { assert(0 && "Weak Operator not correctly specified"); } - vcorr[ss] = result; + coalescedWrite(vcorr[ss],result); } );//end loop over lattice sites } From 3215d88a91214bc6bbed75cf21cf1a4a28b0bbfb Mon Sep 17 00:00:00 2001 From: Michael Marshall <43034299+mmphys@users.noreply.github.com> Date: Wed, 3 Feb 2021 15:17:03 +0000 Subject: [PATCH 10/86] Simplify syntax with Grid::EnableIf post code review. Updated EnableIf so that ReturnType defaults to void in same way as std::enable_if see https://en.cppreference.com/w/cpp/types/enable_if --- Grid/lattice/Lattice_transfer.h | 9 +++------ Grid/simd/Grid_vector_types.h | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Grid/lattice/Lattice_transfer.h b/Grid/lattice/Lattice_transfer.h index c91fa4d1..5a26cce9 100644 --- a/Grid/lattice/Lattice_transfer.h +++ b/Grid/lattice/Lattice_transfer.h @@ -98,8 +98,7 @@ accelerator_inline void convertType(ComplexF & out, const std::complex & } template -accelerator_inline typename std::enable_if::value>::type -convertType(T & out, const T & in) { +accelerator_inline EnableIf> convertType(T & out, const T & in) { out = in; } @@ -138,14 +137,12 @@ accelerator_inline void convertType(iScalar & out, const iScalar & in) { } template -accelerator_inline typename std::enable_if::value>::type -convertType(T1 & out, const iScalar & in) { +accelerator_inline NotEnableIf> convertType(T1 & out, const iScalar & in) { convertType(out,in._internal); } template -accelerator_inline typename std::enable_if::value>::type -convertType(iScalar & out, const T2 & in) { +accelerator_inline NotEnableIf> convertType(iScalar & out, const T2 & in) { convertType(out._internal,in); } diff --git a/Grid/simd/Grid_vector_types.h b/Grid/simd/Grid_vector_types.h index c07077a3..4f952bb2 100644 --- a/Grid/simd/Grid_vector_types.h +++ b/Grid/simd/Grid_vector_types.h @@ -208,8 +208,8 @@ struct RealPart > { ////////////////////////////////////// // type alias used to simplify the syntax of std::enable_if template using Invoke = typename T::type; -template using EnableIf = Invoke >; -template using NotEnableIf = Invoke >; +template using EnableIf = Invoke >; +template using NotEnableIf = Invoke >; //////////////////////////////////////////////////////// // Check for complexity with type traits From 442336bd96b9c0062a9c42704d2013331b3b10e1 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 2 Mar 2021 14:50:51 +0100 Subject: [PATCH 11/86] Hand unrolled to use optimised code paths on GPU for coalesced reads in Wilson case. Other cases to do. This now includes comms code path. --- .../WilsonKernelsHandImplementation.h | 217 ++++++++++-------- .../WilsonKernelsImplementation.h | 12 +- 2 files changed, 123 insertions(+), 106 deletions(-) diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h index 89ae5668..b867369f 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h @@ -76,7 +76,24 @@ Author: paboyle #define REGISTER -#define LOAD_CHIMU \ +#ifdef GRID_SIMT +#define LOAD_CHIMU(ptype) \ + {const SiteSpinor & ref (in[offset]); \ + Chimu_00=coalescedReadPermute(ref()(0)(0),perm); \ + Chimu_01=coalescedReadPermute(ref()(0)(1),perm); \ + Chimu_02=coalescedReadPermute(ref()(0)(2),perm); \ + Chimu_10=coalescedReadPermute(ref()(1)(0),perm); \ + Chimu_11=coalescedReadPermute(ref()(1)(1),perm); \ + Chimu_12=coalescedReadPermute(ref()(1)(2),perm); \ + Chimu_20=coalescedReadPermute(ref()(2)(0),perm); \ + Chimu_21=coalescedReadPermute(ref()(2)(1),perm); \ + Chimu_22=coalescedReadPermute(ref()(2)(2),perm); \ + Chimu_30=coalescedReadPermute(ref()(3)(0),perm); \ + Chimu_31=coalescedReadPermute(ref()(3)(1),perm); \ + Chimu_32=coalescedReadPermute(ref()(3)(2),perm); } +#define PERMUTE_DIR(dir) ; +#else +#define LOAD_CHIMU \ {const SiteSpinor & ref (in[offset]); \ Chimu_00=ref()(0)(0);\ Chimu_01=ref()(0)(1);\ @@ -91,55 +108,55 @@ Author: paboyle Chimu_31=ref()(3)(1);\ Chimu_32=ref()(3)(2);} -#define LOAD_CHI\ - {const SiteHalfSpinor &ref(buf[offset]); \ - Chi_00 = ref()(0)(0);\ - Chi_01 = ref()(0)(1);\ - Chi_02 = ref()(0)(2);\ - Chi_10 = ref()(1)(0);\ - Chi_11 = ref()(1)(1);\ - Chi_12 = ref()(1)(2);} - -// To splat or not to splat depends on the implementation -#define MULT_2SPIN(A)\ - {auto & ref(U[sU](A)); \ - Impl::loadLinkElement(U_00,ref()(0,0)); \ - Impl::loadLinkElement(U_10,ref()(1,0)); \ - Impl::loadLinkElement(U_20,ref()(2,0)); \ - Impl::loadLinkElement(U_01,ref()(0,1)); \ - Impl::loadLinkElement(U_11,ref()(1,1)); \ - Impl::loadLinkElement(U_21,ref()(2,1)); \ - UChi_00 = U_00*Chi_00;\ - UChi_10 = U_00*Chi_10;\ - UChi_01 = U_10*Chi_00;\ - UChi_11 = U_10*Chi_10;\ - UChi_02 = U_20*Chi_00;\ - UChi_12 = U_20*Chi_10;\ - UChi_00+= U_01*Chi_01;\ - UChi_10+= U_01*Chi_11;\ - UChi_01+= U_11*Chi_01;\ - UChi_11+= U_11*Chi_11;\ - UChi_02+= U_21*Chi_01;\ - UChi_12+= U_21*Chi_11;\ - Impl::loadLinkElement(U_00,ref()(0,2)); \ - Impl::loadLinkElement(U_10,ref()(1,2)); \ - Impl::loadLinkElement(U_20,ref()(2,2)); \ - UChi_00+= U_00*Chi_02;\ - UChi_10+= U_00*Chi_12;\ - UChi_01+= U_10*Chi_02;\ - UChi_11+= U_10*Chi_12;\ - UChi_02+= U_20*Chi_02;\ - UChi_12+= U_20*Chi_12;} - - #define PERMUTE_DIR(dir) \ - permute##dir(Chi_00,Chi_00);\ + permute##dir(Chi_00,Chi_00); \ permute##dir(Chi_01,Chi_01);\ permute##dir(Chi_02,Chi_02);\ - permute##dir(Chi_10,Chi_10);\ + permute##dir(Chi_10,Chi_10); \ permute##dir(Chi_11,Chi_11);\ permute##dir(Chi_12,Chi_12); +#endif + +#define LOAD_CHI \ + {const SiteHalfSpinor &ref(buf[offset]); \ + Chi_00 = coalescedRead(ref()(0)(0)); \ + Chi_01 = coalescedRead(ref()(0)(1)); \ + Chi_02 = coalescedRead(ref()(0)(2)); \ + Chi_10 = coalescedRead(ref()(1)(0)); \ + Chi_11 = coalescedRead(ref()(1)(1)); \ + Chi_12 = coalescedRead(ref()(1)(2));} + +#define MULT_2SPIN(A)\ + {auto & ref(U[sU](A)); \ + U_00=coalescedRead(ref()(0,0)); \ + U_10=coalescedRead(ref()(1,0)); \ + U_20=coalescedRead(ref()(2,0)); \ + U_01=coalescedRead(ref()(0,1)); \ + U_11=coalescedRead(ref()(1,1)); \ + U_21=coalescedRead(ref()(2,1)); \ + UChi_00 = U_00*Chi_00; \ + UChi_10 = U_00*Chi_10; \ + UChi_01 = U_10*Chi_00; \ + UChi_11 = U_10*Chi_10; \ + UChi_02 = U_20*Chi_00; \ + UChi_12 = U_20*Chi_10; \ + UChi_00+= U_01*Chi_01; \ + UChi_10+= U_01*Chi_11; \ + UChi_01+= U_11*Chi_01; \ + UChi_11+= U_11*Chi_11; \ + UChi_02+= U_21*Chi_01; \ + UChi_12+= U_21*Chi_11; \ + U_00=coalescedRead(ref()(0,2)); \ + U_10=coalescedRead(ref()(1,2)); \ + U_20=coalescedRead(ref()(2,2)); \ + UChi_00+= U_00*Chi_02; \ + UChi_10+= U_00*Chi_12; \ + UChi_01+= U_10*Chi_02; \ + UChi_11+= U_10*Chi_12; \ + UChi_02+= U_20*Chi_02; \ + UChi_12+= U_20*Chi_12;} + // hspin(0)=fspin(0)+timesI(fspin(3)); // hspin(1)=fspin(1)+timesI(fspin(2)); #define XP_PROJ \ @@ -359,7 +376,7 @@ Author: paboyle local = SE->_is_local; \ perm = SE->_permute; \ if ( local ) { \ - LOAD_CHIMU; \ + LOAD_CHIMU(PERM); \ PROJ; \ if ( perm) { \ PERMUTE_DIR(PERM); \ @@ -376,7 +393,7 @@ Author: paboyle local = SE->_is_local; \ perm = SE->_permute; \ if ( local ) { \ - LOAD_CHIMU; \ + LOAD_CHIMU(PERM); \ PROJ; \ if ( perm) { \ PERMUTE_DIR(PERM); \ @@ -401,40 +418,39 @@ Author: paboyle #define HAND_RESULT(ss) \ { \ - SiteSpinor & ref (out[ss]); \ - vstream(ref()(0)(0),result_00); \ - vstream(ref()(0)(1),result_01); \ - vstream(ref()(0)(2),result_02); \ - vstream(ref()(1)(0),result_10); \ - vstream(ref()(1)(1),result_11); \ - vstream(ref()(1)(2),result_12); \ - vstream(ref()(2)(0),result_20); \ - vstream(ref()(2)(1),result_21); \ - vstream(ref()(2)(2),result_22); \ - vstream(ref()(3)(0),result_30); \ - vstream(ref()(3)(1),result_31); \ - vstream(ref()(3)(2),result_32); \ + SiteSpinor & ref (out[ss]); \ + coalescedWrite(ref()(0)(0),result_00); \ + coalescedWrite(ref()(0)(1),result_01); \ + coalescedWrite(ref()(0)(2),result_02); \ + coalescedWrite(ref()(1)(0),result_10); \ + coalescedWrite(ref()(1)(1),result_11); \ + coalescedWrite(ref()(1)(2),result_12); \ + coalescedWrite(ref()(2)(0),result_20); \ + coalescedWrite(ref()(2)(1),result_21); \ + coalescedWrite(ref()(2)(2),result_22); \ + coalescedWrite(ref()(3)(0),result_30); \ + coalescedWrite(ref()(3)(1),result_31); \ + coalescedWrite(ref()(3)(2),result_32); \ } -#define HAND_RESULT_EXT(ss) \ - if (nmu){ \ - SiteSpinor & ref (out[ss]); \ - ref()(0)(0)+=result_00; \ - ref()(0)(1)+=result_01; \ - ref()(0)(2)+=result_02; \ - ref()(1)(0)+=result_10; \ - ref()(1)(1)+=result_11; \ - ref()(1)(2)+=result_12; \ - ref()(2)(0)+=result_20; \ - ref()(2)(1)+=result_21; \ - ref()(2)(2)+=result_22; \ - ref()(3)(0)+=result_30; \ - ref()(3)(1)+=result_31; \ - ref()(3)(2)+=result_32; \ +#define HAND_RESULT_EXT(ss) \ + { \ + SiteSpinor & ref (out[ss]); \ + coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00); \ + coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01); \ + coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02); \ + coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10); \ + coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11); \ + coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12); \ + coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20); \ + coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21); \ + coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22); \ + coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30); \ + coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31); \ + coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32); \ } - -#define HAND_DECLARATIONS(a) \ +#define HAND_DECLARATIONS(Simd) \ Simd result_00; \ Simd result_01; \ Simd result_02; \ @@ -467,18 +483,18 @@ Author: paboyle Simd U_21; #define ZERO_RESULT \ - result_00=Zero(); \ - result_01=Zero(); \ - result_02=Zero(); \ - result_10=Zero(); \ - result_11=Zero(); \ - result_12=Zero(); \ - result_20=Zero(); \ - result_21=Zero(); \ - result_22=Zero(); \ - result_30=Zero(); \ - result_31=Zero(); \ - result_32=Zero(); + result_00=S(0.0,0.0); \ + result_01=S(0.0,0.0); \ + result_02=S(0.0,0.0); \ + result_10=S(0.0,0.0); \ + result_11=S(0.0,0.0); \ + result_12=S(0.0,0.0); \ + result_20=S(0.0,0.0); \ + result_21=S(0.0,0.0); \ + result_22=S(0.0,0.0); \ + result_30=S(0.0,0.0); \ + result_31=S(0.0,0.0); \ + result_32=S(0.0,0.0); #define Chimu_00 Chi_00 #define Chimu_01 Chi_01 @@ -502,8 +518,8 @@ WilsonKernels::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,Site // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc... typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - - HAND_DECLARATIONS(ignore); + typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + HAND_DECLARATIONS(Simt); int offset,local,perm, ptype; StencilEntry *SE; @@ -525,8 +541,8 @@ void WilsonKernels::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView { typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - - HAND_DECLARATIONS(ignore); + typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + HAND_DECLARATIONS(Simt); StencilEntry *SE; int offset,local,perm, ptype; @@ -549,8 +565,8 @@ WilsonKernels::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,Si // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc... typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - - HAND_DECLARATIONS(ignore); + typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + HAND_DECLARATIONS(Simt); int offset,local,perm, ptype; StencilEntry *SE; @@ -572,8 +588,8 @@ void WilsonKernels::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldVi { typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - - HAND_DECLARATIONS(ignore); + typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + HAND_DECLARATIONS(Simt); StencilEntry *SE; int offset,local,perm, ptype; @@ -596,8 +612,8 @@ WilsonKernels::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,Si // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc... typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - - HAND_DECLARATIONS(ignore); + typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + HAND_DECLARATIONS(Simt); int offset, ptype; StencilEntry *SE; @@ -620,8 +636,8 @@ void WilsonKernels::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldVi { typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - - HAND_DECLARATIONS(ignore); + typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + HAND_DECLARATIONS(Simt); StencilEntry *SE; int offset, ptype; @@ -682,3 +698,4 @@ NAMESPACE_END(Grid); #undef HAND_RESULT #undef HAND_RESULT_INT #undef HAND_RESULT_EXT +#undef HAND_DECLARATIONS diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h index c5f50bbb..937d13af 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h @@ -445,20 +445,20 @@ void WilsonKernels::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField if( interior && exterior ) { if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSite); return;} -#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSite); return;} +#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSite); return;} #endif } else if( interior ) { if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALLNB(GenericDhopSiteInt); return;} -#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALLNB(HandDhopSiteInt); return;} +#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteInt); return;} #endif } else if( exterior ) { if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteExt); return;} -#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteExt); return;} +#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteExt); return;} #endif } @@ -476,20 +476,20 @@ void WilsonKernels::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField if( interior && exterior ) { if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDag); return;} -#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDag); return;} +#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDag); return;} #endif } else if( interior ) { if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagInt); return;} -#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagInt); return;} +#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagInt); return;} #endif } else if( exterior ) { if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSiteDagExt); return;} -#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSiteDagExt); return;} +#ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagExt); return;} #endif } From 679d1d22f7e920163acf7b88515c30ceb297387b Mon Sep 17 00:00:00 2001 From: u61464 Date: Wed, 3 Mar 2021 11:21:43 -0800 Subject: [PATCH 12/86] Sycl happier --- Grid/Makefile.am | 4 ++- Grid/cartesian/Cartesian_red_black.h | 2 +- Grid/qcd/action/fermion/FermionOperatorImpl.h | 4 +-- Grid/qcd/action/fermion/WilsonCompressor.h | 24 ++++++++-------- Grid/qcd/action/fermion/WilsonImpl.h | 2 +- .../WilsonKernelsHandImplementation.h | 28 +++++++++---------- Grid/simd/Grid_gpu_vec.h | 1 + Grid/stencil/SimpleCompressor.h | 14 +++++----- Grid/stencil/Stencil.h | 14 +++++----- Grid/tensors/Tensor_SIMT.h | 4 ++- Grid/threads/Accelerator.h | 2 +- benchmarks/Benchmark_dwf_fp32.cc | 2 +- configure.ac | 11 ++++++++ 13 files changed, 64 insertions(+), 48 deletions(-) diff --git a/Grid/Makefile.am b/Grid/Makefile.am index ded6d146..7c3c151b 100644 --- a/Grid/Makefile.am +++ b/Grid/Makefile.am @@ -54,9 +54,11 @@ Version.h: version-cache include Make.inc include Eigen.inc -extra_sources+=$(ZWILS_FERMION_FILES) extra_sources+=$(WILS_FERMION_FILES) extra_sources+=$(STAG_FERMION_FILES) +if BUILD_ZMOBIUS + extra_sources+=$(ZWILS_FERMION_FILES) +endif if BUILD_GPARITY extra_sources+=$(GP_FERMION_FILES) endif diff --git a/Grid/cartesian/Cartesian_red_black.h b/Grid/cartesian/Cartesian_red_black.h index b71981f5..092d4910 100644 --- a/Grid/cartesian/Cartesian_red_black.h +++ b/Grid/cartesian/Cartesian_red_black.h @@ -36,7 +36,7 @@ static const int CbBlack=1; static const int Even =CbRed; static const int Odd =CbBlack; -accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex, Coordinate &rdim, Coordinate &chk_dim_msk) +accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex,const Coordinate &rdim,const Coordinate &chk_dim_msk) { int nd=rdim.size(); Coordinate coor(nd); diff --git a/Grid/qcd/action/fermion/FermionOperatorImpl.h b/Grid/qcd/action/fermion/FermionOperatorImpl.h index b444f6dc..9345c0e6 100644 --- a/Grid/qcd/action/fermion/FermionOperatorImpl.h +++ b/Grid/qcd/action/fermion/FermionOperatorImpl.h @@ -153,8 +153,8 @@ public: typedef typename Impl::StencilImpl StencilImpl; \ typedef typename Impl::ImplParams ImplParams; \ typedef typename Impl::StencilImpl::View_type StencilView; \ - typedef typename ViewMap::Type FermionFieldView; \ - typedef typename ViewMap::Type DoubledGaugeFieldView; + typedef const typename ViewMap::Type FermionFieldView; \ + typedef const typename ViewMap::Type DoubledGaugeFieldView; #define INHERIT_IMPL_TYPES(Base) \ INHERIT_GIMPL_TYPES(Base) \ diff --git a/Grid/qcd/action/fermion/WilsonCompressor.h b/Grid/qcd/action/fermion/WilsonCompressor.h index 10e98f33..0760bcba 100644 --- a/Grid/qcd/action/fermion/WilsonCompressor.h +++ b/Grid/qcd/action/fermion/WilsonCompressor.h @@ -61,7 +61,7 @@ public: typedef typename SiteHalfSpinor::vector_type vComplexHigh; constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh); - accelerator_inline int CommDatumSize(void) { + accelerator_inline int CommDatumSize(void) const { return sizeof(SiteHalfCommSpinor); } @@ -69,7 +69,7 @@ public: /* Compress includes precision change if mpi data is not same */ /*****************************************************/ template - accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) { + accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) const { _SiteHalfSpinor tmp; projector::Proj(tmp,in,mu,dag); vstream(buf[o],tmp); @@ -81,7 +81,7 @@ public: accelerator_inline void Exchange(SiteHalfSpinor *mp, const SiteHalfSpinor * __restrict__ vp0, const SiteHalfSpinor * __restrict__ vp1, - Integer type,Integer o){ + Integer type,Integer o) const { SiteHalfSpinor tmp1; SiteHalfSpinor tmp2; exchange(tmp1,tmp2,vp0[o],vp1[o],type); @@ -93,7 +93,7 @@ public: /* Have a decompression step if mpi data is not same */ /*****************************************************/ accelerator_inline void Decompress(SiteHalfSpinor * __restrict__ out, - SiteHalfSpinor * __restrict__ in, Integer o) { + SiteHalfSpinor * __restrict__ in, Integer o) const { assert(0); } @@ -103,7 +103,7 @@ public: accelerator_inline void CompressExchange(SiteHalfSpinor * __restrict__ out0, SiteHalfSpinor * __restrict__ out1, const SiteSpinor * __restrict__ in, - Integer j,Integer k, Integer m,Integer type) + Integer j,Integer k, Integer m,Integer type) const { SiteHalfSpinor temp1, temp2; SiteHalfSpinor temp3, temp4; @@ -117,7 +117,7 @@ public: /*****************************************************/ /* Pass the info to the stencil */ /*****************************************************/ - accelerator_inline bool DecompressionStep(void) { return false; } + accelerator_inline bool DecompressionStep(void) const { return false; } }; @@ -142,7 +142,7 @@ public: typedef typename SiteHalfSpinor::vector_type vComplexHigh; constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh); - accelerator_inline int CommDatumSize(void) { + accelerator_inline int CommDatumSize(void) const { return sizeof(SiteHalfCommSpinor); } @@ -150,7 +150,7 @@ public: /* Compress includes precision change if mpi data is not same */ /*****************************************************/ template - accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) { + accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) const { _SiteHalfSpinor hsp; SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf; projector::Proj(hsp,in,mu,dag); @@ -163,7 +163,7 @@ public: accelerator_inline void Exchange(SiteHalfSpinor *mp, SiteHalfSpinor *vp0, SiteHalfSpinor *vp1, - Integer type,Integer o){ + Integer type,Integer o) const { SiteHalfSpinor vt0,vt1; SiteHalfCommSpinor *vpp0 = (SiteHalfCommSpinor *)vp0; SiteHalfCommSpinor *vpp1 = (SiteHalfCommSpinor *)vp1; @@ -175,7 +175,7 @@ public: /*****************************************************/ /* Have a decompression step if mpi data is not same */ /*****************************************************/ - accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o){ + accelerator_inline void Decompress(SiteHalfSpinor *out, SiteHalfSpinor *in, Integer o) const { SiteHalfCommSpinor *hin=(SiteHalfCommSpinor *)in; precisionChange((vComplexHigh *)&out[o],(vComplexLow *)&hin[o],Nw); } @@ -186,7 +186,7 @@ public: accelerator_inline void CompressExchange(SiteHalfSpinor *out0, SiteHalfSpinor *out1, const SiteSpinor *in, - Integer j,Integer k, Integer m,Integer type){ + Integer j,Integer k, Integer m,Integer type) const { SiteHalfSpinor temp1, temp2,temp3,temp4; SiteHalfCommSpinor *hout0 = (SiteHalfCommSpinor *)out0; SiteHalfCommSpinor *hout1 = (SiteHalfCommSpinor *)out1; @@ -200,7 +200,7 @@ public: /*****************************************************/ /* Pass the info to the stencil */ /*****************************************************/ - accelerator_inline bool DecompressionStep(void) { return true; } + accelerator_inline bool DecompressionStep(void) const { return true; } }; diff --git a/Grid/qcd/action/fermion/WilsonImpl.h b/Grid/qcd/action/fermion/WilsonImpl.h index d7941d1f..94676b6b 100644 --- a/Grid/qcd/action/fermion/WilsonImpl.h +++ b/Grid/qcd/action/fermion/WilsonImpl.h @@ -72,7 +72,7 @@ public: typedef WilsonCompressor Compressor; typedef WilsonImplParams ImplParams; typedef WilsonStencil StencilImpl; - typedef typename StencilImpl::View_type StencilView; + typedef const typename StencilImpl::View_type StencilView; ImplParams Params; diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h index b867369f..688cb75a 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h @@ -93,7 +93,7 @@ Author: paboyle Chimu_32=coalescedReadPermute(ref()(3)(2),perm); } #define PERMUTE_DIR(dir) ; #else -#define LOAD_CHIMU \ +#define LOAD_CHIMU(ptype) \ {const SiteSpinor & ref (in[offset]); \ Chimu_00=ref()(0)(0);\ Chimu_01=ref()(0)(1);\ @@ -482,19 +482,19 @@ Author: paboyle Simd U_11; \ Simd U_21; -#define ZERO_RESULT \ - result_00=S(0.0,0.0); \ - result_01=S(0.0,0.0); \ - result_02=S(0.0,0.0); \ - result_10=S(0.0,0.0); \ - result_11=S(0.0,0.0); \ - result_12=S(0.0,0.0); \ - result_20=S(0.0,0.0); \ - result_21=S(0.0,0.0); \ - result_22=S(0.0,0.0); \ - result_30=S(0.0,0.0); \ - result_31=S(0.0,0.0); \ - result_32=S(0.0,0.0); +#define ZERO_RESULT \ + zeroit(result_00); \ + zeroit(result_01); \ + zeroit(result_02); \ + zeroit(result_10); \ + zeroit(result_11); \ + zeroit(result_12); \ + zeroit(result_20); \ + zeroit(result_21); \ + zeroit(result_22); \ + zeroit(result_30); \ + zeroit(result_31); \ + zeroit(result_32); #define Chimu_00 Chi_00 #define Chimu_01 Chi_01 diff --git a/Grid/simd/Grid_gpu_vec.h b/Grid/simd/Grid_gpu_vec.h index 2c1a38e7..b2c7588f 100644 --- a/Grid/simd/Grid_gpu_vec.h +++ b/Grid/simd/Grid_gpu_vec.h @@ -67,6 +67,7 @@ public: accelerator_inline GpuComplex(const GpuComplex &zz) { z = zz.z;}; accelerator_inline Real real(void) const { return z.x; }; accelerator_inline Real imag(void) const { return z.y; }; + accelerator_inline GpuComplex &operator=(const Zero &zz) { z.x = 0; z.y=0; return *this; }; accelerator_inline GpuComplex &operator*=(const GpuComplex &r) { *this = (*this) * r; return *this; diff --git a/Grid/stencil/SimpleCompressor.h b/Grid/stencil/SimpleCompressor.h index be7c89c0..2ce48369 100644 --- a/Grid/stencil/SimpleCompressor.h +++ b/Grid/stencil/SimpleCompressor.h @@ -7,20 +7,20 @@ template class SimpleCompressor { public: void Point(int) {}; - accelerator_inline int CommDatumSize(void) { return sizeof(vobj); } - accelerator_inline bool DecompressionStep(void) { return false; } - template accelerator_inline void Compress(cobj *buf,int o,const cobj &in) { buf[o]=in; } - accelerator_inline void Exchange(vobj *mp,vobj *vp0,vobj *vp1,Integer type,Integer o){ + accelerator_inline int CommDatumSize(void) const { return sizeof(vobj); } + accelerator_inline bool DecompressionStep(void) const { return false; } + template accelerator_inline void Compress(cobj *buf,int o,const cobj &in) const { buf[o]=in; } + accelerator_inline void Exchange(vobj *mp,vobj *vp0,vobj *vp1,Integer type,Integer o) const { exchange(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type); } - accelerator_inline void Decompress(vobj *out,vobj *in, int o){ assert(0); } + accelerator_inline void Decompress(vobj *out,vobj *in, int o) const { assert(0); } accelerator_inline void CompressExchange(vobj *out0,vobj *out1,const vobj *in, - int j,int k, int m,int type){ + int j,int k, int m,int type) const { exchange(out0[j],out1[j],in[k],in[m],type); } // For cshift. Cshift should drop compressor coupling altogether // because I had to decouple the code from the Stencil anyway - accelerator_inline vobj operator() (const vobj &arg) { + accelerator_inline vobj operator() (const vobj &arg) const { return arg; } }; diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h index 23fc8203..58cebed3 100644 --- a/Grid/stencil/Stencil.h +++ b/Grid/stencil/Stencil.h @@ -147,16 +147,16 @@ class CartesianStencilAccelerator { cobj* u_recv_buf_p; cobj* u_send_buf_p; - accelerator_inline cobj *CommBuf(void) { return u_recv_buf_p; } + accelerator_inline cobj *CommBuf(void) const { return u_recv_buf_p; } - accelerator_inline int GetNodeLocal(int osite,int point) { + accelerator_inline int GetNodeLocal(int osite,int point) const { return this->_entries_p[point+this->_npoints*osite]._is_local; } - accelerator_inline StencilEntry * GetEntry(int &ptype,int point,int osite) { + accelerator_inline StencilEntry * GetEntry(int &ptype,int point,int osite) const { ptype = this->_permute_type[point]; return & this->_entries_p[point+this->_npoints*osite]; } - accelerator_inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) { + accelerator_inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) const { uint64_t cbase = (uint64_t)&u_recv_buf_p[0]; local = this->_entries_p[ent]._is_local; perm = this->_entries_p[ent]._permute; @@ -168,14 +168,14 @@ class CartesianStencilAccelerator { } } - accelerator_inline uint64_t GetPFInfo(int ent,uint64_t base) { + accelerator_inline uint64_t GetPFInfo(int ent,uint64_t base) const { uint64_t cbase = (uint64_t)&u_recv_buf_p[0]; int local = this->_entries_p[ent]._is_local; if (local) return base + this->_entries_p[ent]._byte_offset; else return cbase + this->_entries_p[ent]._byte_offset; } - accelerator_inline void iCoorFromIindex(Coordinate &coor,int lane) + accelerator_inline void iCoorFromIindex(Coordinate &coor,int lane) const { Lexicographic::CoorFromIndex(coor,lane,this->_simd_layout); } @@ -221,7 +221,7 @@ public: typedef typename cobj::vector_type vector_type; typedef typename cobj::scalar_type scalar_type; typedef typename cobj::scalar_object scalar_object; - typedef CartesianStencilView View_type; + typedef const CartesianStencilView View_type; typedef typename View_type::StencilVector StencilVector; /////////////////////////////////////////// // Helper structs diff --git a/Grid/tensors/Tensor_SIMT.h b/Grid/tensors/Tensor_SIMT.h index ede24fbe..672f385f 100644 --- a/Grid/tensors/Tensor_SIMT.h +++ b/Grid/tensors/Tensor_SIMT.h @@ -66,7 +66,7 @@ void coalescedWriteNonTemporal(vobj & __restrict__ vec,const vobj & __restrict__ #ifndef GRID_SYCL -// Use the scalar as our own complex on GPU +// Use the scalar as our own complex on GPU ... thrust::complex or std::complex template = 0> accelerator_inline typename vsimd::scalar_type coalescedRead(const vsimd & __restrict__ vec,int lane=acceleratorSIMTlane(vsimd::Nsimd())) @@ -96,6 +96,8 @@ void coalescedWrite(vsimd & __restrict__ vec, p[lane]=extracted; } #else +// For SyCL have option to use GpuComplex from inside the vector type in SIMT loops +// Faster for some reason template = 0> accelerator_inline typename vsimd::vector_type::datum coalescedRead(const vsimd & __restrict__ vec,int lane=acceleratorSIMTlane(vsimd::Nsimd())) diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h index 2b7bf53a..f1a694fb 100644 --- a/Grid/threads/Accelerator.h +++ b/Grid/threads/Accelerator.h @@ -456,7 +456,7 @@ accelerator_inline void acceleratorSynchronise(void) __syncwarp(); #endif #ifdef GRID_SYCL - // No barrier call on SYCL?? // Option get __spir:: stuff to do warp barrier + cl::sycl::detail::workGroupBarrier(); #endif #ifdef GRID_HIP __syncthreads(); diff --git a/benchmarks/Benchmark_dwf_fp32.cc b/benchmarks/Benchmark_dwf_fp32.cc index cb86177e..03f3ee61 100644 --- a/benchmarks/Benchmark_dwf_fp32.cc +++ b/benchmarks/Benchmark_dwf_fp32.cc @@ -53,7 +53,7 @@ int main (int argc, char ** argv) int threads = GridThread::GetThreads(); Coordinate latt4 = GridDefaultLatt(); - int Ls=8; + int Ls=16; for(int i=0;i> Ls; diff --git a/configure.ac b/configure.ac index fb0c78fc..5f165412 100644 --- a/configure.ac +++ b/configure.ac @@ -140,12 +140,23 @@ AC_ARG_ENABLE([gparity], [ac_GPARITY=${enable_gparity}], [ac_GPARITY=yes]) AM_CONDITIONAL(BUILD_GPARITY, [ test "${ac_GPARITY}X" == "yesX" ]) + +AC_ARG_ENABLE([zmobius], + [AC_HELP_STRING([--enable-zmobius=yes|no], [enable Zmobius support])], + [ac_ZMOBIUS=${enable_zmobius}], [ac_ZMOBIUS=yes]) + +AM_CONDITIONAL(BUILD_ZMOBIUS, [ test "${ac_ZMOBIUS}X" == "yesX" ]) + + case ${ac_FERMION_REPS} in yes) AC_DEFINE([ENABLE_FERMION_REPS],[1],[non QCD fermion reps]);; esac case ${ac_GPARITY} in yes) AC_DEFINE([ENABLE_GPARITY],[1],[fermion actions with GPARITY BCs]);; esac +case ${ac_ZMOBIUS} in + yes) AC_DEFINE([ENABLE_ZMOBIUS],[1],[Zmobius fermion actions]);; +esac ############### Nc AC_ARG_ENABLE([Nc], [AC_HELP_STRING([--enable-Nc=2|3|4], [enable number of colours])], From 1eea9d73b984252c69869f65991f2f33d6e93193 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 3 Mar 2021 23:50:01 +0100 Subject: [PATCH 13/86] Pass serial RNG around --- Grid/qcd/action/ActionBase.h | 2 +- Grid/qcd/action/gauge/GaugeImplTypes.h | 2 +- Grid/qcd/action/gauge/PlaqPlusRectangleAction.h | 2 +- Grid/qcd/action/gauge/WilsonGaugeAction.h | 3 +-- .../qcd/action/pseudofermion/ExactOneFlavourRatio.h | 2 +- .../pseudofermion/OneFlavourEvenOddRational.h | 11 ++++++----- .../pseudofermion/OneFlavourEvenOddRationalRatio.h | 7 +++++-- Grid/qcd/action/pseudofermion/OneFlavourRational.h | 7 +++++-- .../action/pseudofermion/OneFlavourRationalRatio.h | 7 +++++-- Grid/qcd/action/pseudofermion/TwoFlavour.h | 2 +- Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h | 2 +- .../action/pseudofermion/TwoFlavourEvenOddRatio.h | 2 +- Grid/qcd/action/pseudofermion/TwoFlavourRatio.h | 2 +- Grid/qcd/action/scalar/ScalarAction.h | 2 +- Grid/qcd/action/scalar/ScalarImpl.h | 4 ++-- Grid/qcd/action/scalar/ScalarInteractionAction.h | 2 +- Grid/qcd/hmc/HMC.h | 2 +- Grid/qcd/hmc/integrators/Integrator.h | 13 ++++++------- benchmarks/Benchmark_dwf_fp32.cc | 2 +- 19 files changed, 42 insertions(+), 34 deletions(-) diff --git a/Grid/qcd/action/ActionBase.h b/Grid/qcd/action/ActionBase.h index bff21d1d..17980ee0 100644 --- a/Grid/qcd/action/ActionBase.h +++ b/Grid/qcd/action/ActionBase.h @@ -41,7 +41,7 @@ class Action public: bool is_smeared = false; // Heatbath? - virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG) = 0; // refresh pseudofermions + virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) = 0; // refresh pseudofermions virtual RealD S(const GaugeField& U) = 0; // evaluate the action virtual void deriv(const GaugeField& U, GaugeField& dSdU) = 0; // evaluate the action derivative virtual std::string action_name() = 0; // return the action name diff --git a/Grid/qcd/action/gauge/GaugeImplTypes.h b/Grid/qcd/action/gauge/GaugeImplTypes.h index 55a20eca..2499e0e9 100644 --- a/Grid/qcd/action/gauge/GaugeImplTypes.h +++ b/Grid/qcd/action/gauge/GaugeImplTypes.h @@ -96,7 +96,7 @@ public: /////////////////////////////////////////////////////////// // Move these to another class // HMC auxiliary functions - static inline void generate_momenta(Field &P, GridParallelRNG &pRNG) + static inline void generate_momenta(Field &P, GridSerialRNG & sRNG, GridParallelRNG &pRNG) { // Zbigniew Srocinsky thesis: // diff --git a/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h b/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h index 639aca19..7690092d 100644 --- a/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h +++ b/Grid/qcd/action/gauge/PlaqPlusRectangleAction.h @@ -49,7 +49,7 @@ public: virtual std::string action_name(){return "PlaqPlusRectangleAction";} - virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {}; // noop as no pseudoferms + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {}; // noop as no pseudoferms virtual std::string LogParameters(){ std::stringstream sstream; diff --git a/Grid/qcd/action/gauge/WilsonGaugeAction.h b/Grid/qcd/action/gauge/WilsonGaugeAction.h index 40d600d2..f535b54f 100644 --- a/Grid/qcd/action/gauge/WilsonGaugeAction.h +++ b/Grid/qcd/action/gauge/WilsonGaugeAction.h @@ -54,8 +54,7 @@ public: return sstream.str(); } - virtual void refresh(const GaugeField &U, - GridParallelRNG &pRNG){}; // noop as no pseudoferms + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG){}; // noop as no pseudoferms virtual RealD S(const GaugeField &U) { RealD plaq = WilsonLoops::avgPlaquette(U); diff --git a/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h b/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h index 9fc0a3b0..576a8cf6 100644 --- a/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h +++ b/Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h @@ -124,7 +124,7 @@ NAMESPACE_BEGIN(Grid); // // As a check of rational require \Phi^dag M_{EOFA} \Phi == eta^dag M^-1/2^dag M M^-1/2 eta = eta^dag eta // - virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG) + virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { Lop.ImportGauge(U); Rop.ImportGauge(U); diff --git a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h index 56dff94d..656e9b2f 100644 --- a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h +++ b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h @@ -1,4 +1,3 @@ - /************************************************************************************* Grid physics library, www.github.com/paboyle/Grid @@ -43,8 +42,7 @@ NAMESPACE_BEGIN(Grid); // template -class OneFlavourEvenOddRationalPseudoFermionAction - : public Action { +class OneFlavourEvenOddRationalPseudoFermionAction : public Action { public: INHERIT_IMPL_TYPES(Impl); @@ -103,7 +101,7 @@ public: return sstream.str(); } - virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { // P(phi) = e^{- phi^dag (MpcdagMpc)^-1/2 phi} // = e^{- phi^dag (MpcdagMpc)^-1/4 (MpcdagMpc)^-1/4 phi} // Phi = MpcdagMpc^{1/4} eta @@ -156,7 +154,10 @@ public: msCG(Mpc, PhiOdd, Y); - if ( (rand()%param.BoundsCheckFreq)==0 ) { + auto grid = FermOp.FermionGrid(); + auto r=rand(); + grid->Broadcast(0,r); + if ( (r%param.BoundsCheckFreq)==0 ) { FermionField gauss(FermOp.FermionRedBlackGrid()); gauss = PhiOdd; HighBoundCheck(Mpc,gauss,param.hi); diff --git a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h index e5f0b602..e968b8e4 100644 --- a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h +++ b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h @@ -101,7 +101,7 @@ NAMESPACE_BEGIN(Grid); } - virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { // S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi // @@ -170,7 +170,10 @@ NAMESPACE_BEGIN(Grid); msCG_M(MdagM,X,Y); // Randomly apply rational bounds checks. - if ( (rand()%param.BoundsCheckFreq)==0 ) { + auto grid = NumOp.FermionGrid(); + auto r=rand(); + grid->Broadcast(0,r); + if ( (r%param.BoundsCheckFreq)==0 ) { FermionField gauss(NumOp.FermionRedBlackGrid()); gauss = PhiOdd; HighBoundCheck(MdagM,gauss,param.hi); diff --git a/Grid/qcd/action/pseudofermion/OneFlavourRational.h b/Grid/qcd/action/pseudofermion/OneFlavourRational.h index f6c823c9..aa647445 100644 --- a/Grid/qcd/action/pseudofermion/OneFlavourRational.h +++ b/Grid/qcd/action/pseudofermion/OneFlavourRational.h @@ -98,7 +98,7 @@ NAMESPACE_BEGIN(Grid); - virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { // P(phi) = e^{- phi^dag (MdagM)^-1/2 phi} @@ -142,7 +142,10 @@ NAMESPACE_BEGIN(Grid); msCG(MdagMOp,Phi,Y); - if ( (rand()%param.BoundsCheckFreq)==0 ) { + auto grid = FermOp.FermionGrid(); + auto r=rand(); + grid->Broadcast(0,r); + if ( (r%param.BoundsCheckFreq)==0 ) { FermionField gauss(FermOp.FermionGrid()); gauss = Phi; HighBoundCheck(MdagMOp,gauss,param.hi); diff --git a/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h b/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h index 5fae2fe9..128c869a 100644 --- a/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h +++ b/Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h @@ -95,7 +95,7 @@ NAMESPACE_BEGIN(Grid); } - virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { // S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi // @@ -156,7 +156,10 @@ NAMESPACE_BEGIN(Grid); msCG_M(MdagM,X,Y); // Randomly apply rational bounds checks. - if ( (rand()%param.BoundsCheckFreq)==0 ) { + auto grid = NumOp.FermionGrid(); + auto r=rand(); + grid->Broadcast(0,r); + if ( (r%param.BoundsCheckFreq)==0 ) { FermionField gauss(NumOp.FermionGrid()); gauss = Phi; HighBoundCheck(MdagM,gauss,param.hi); diff --git a/Grid/qcd/action/pseudofermion/TwoFlavour.h b/Grid/qcd/action/pseudofermion/TwoFlavour.h index f905a675..2ac97ddd 100644 --- a/Grid/qcd/action/pseudofermion/TwoFlavour.h +++ b/Grid/qcd/action/pseudofermion/TwoFlavour.h @@ -73,7 +73,7 @@ public: ////////////////////////////////////////////////////////////////////////////////////// // Push the gauge field in to the dops. Assume any BC's and smearing already applied ////////////////////////////////////////////////////////////////////////////////////// - virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { // P(phi) = e^{- phi^dag (MdagM)^-1 phi} // Phi = Mdag eta // P(eta) = e^{- eta^dag eta} diff --git a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h index a3cf8f08..2e5208a8 100644 --- a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h +++ b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOdd.h @@ -77,7 +77,7 @@ public: ////////////////////////////////////////////////////////////////////////////////////// // Push the gauge field in to the dops. Assume any BC's and smearing already applied ////////////////////////////////////////////////////////////////////////////////////// - virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { // P(phi) = e^{- phi^dag (MpcdagMpc)^-1 phi} // Phi = McpDag eta diff --git a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h index d1d6f336..da628c75 100644 --- a/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h +++ b/Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h @@ -84,7 +84,7 @@ NAMESPACE_BEGIN(Grid); } - virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { // P(phi) = e^{- phi^dag Vpc (MpcdagMpc)^-1 Vpcdag phi} // diff --git a/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h b/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h index 4d72faba..f584706d 100644 --- a/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h +++ b/Grid/qcd/action/pseudofermion/TwoFlavourRatio.h @@ -64,7 +64,7 @@ public: return sstream.str(); } - virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) { + virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) { // P(phi) = e^{- phi^dag V (MdagM)^-1 Vdag phi} // diff --git a/Grid/qcd/action/scalar/ScalarAction.h b/Grid/qcd/action/scalar/ScalarAction.h index 34fc4fac..8b4f4f79 100644 --- a/Grid/qcd/action/scalar/ScalarAction.h +++ b/Grid/qcd/action/scalar/ScalarAction.h @@ -55,7 +55,7 @@ public: } virtual std::string action_name() {return "ScalarAction";} - virtual void refresh(const Field &U, GridParallelRNG &pRNG) {} // noop as no pseudoferms + virtual void refresh(const Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {} // noop as no pseudoferms virtual RealD S(const Field &p) { return (mass_square * 0.5 + Nd) * ScalarObs::sumphisquared(p) + diff --git a/Grid/qcd/action/scalar/ScalarImpl.h b/Grid/qcd/action/scalar/ScalarImpl.h index 403ea573..13bd6c90 100644 --- a/Grid/qcd/action/scalar/ScalarImpl.h +++ b/Grid/qcd/action/scalar/ScalarImpl.h @@ -27,7 +27,7 @@ public: typedef Field FermionField; typedef Field PropagatorField; - static inline void generate_momenta(Field& P, GridParallelRNG& pRNG){ + static inline void generate_momenta(Field& P, GridSerialRNG &sRNG, GridParallelRNG& pRNG){ RealD scale = ::sqrt(HMC_MOMENTUM_DENOMINATOR); // CPS/UKQCD momentum rescaling gaussian(pRNG, P); P *= scale; @@ -151,7 +151,7 @@ public: out = one / out; } - static inline void generate_momenta(Field &P, GridParallelRNG &pRNG) + static inline void generate_momenta(Field &P, GridSerialRNG & sRNG, GridParallelRNG &pRNG) { RealD scale = ::sqrt(HMC_MOMENTUM_DENOMINATOR); // CPS/UKQCD momentum rescaling #ifndef USE_FFT_ACCELERATION diff --git a/Grid/qcd/action/scalar/ScalarInteractionAction.h b/Grid/qcd/action/scalar/ScalarInteractionAction.h index 5a5f9251..e04dd486 100644 --- a/Grid/qcd/action/scalar/ScalarInteractionAction.h +++ b/Grid/qcd/action/scalar/ScalarInteractionAction.h @@ -77,7 +77,7 @@ public: virtual std::string action_name() { return "ScalarAction"; } - virtual void refresh(const Field &U, GridParallelRNG &pRNG) {} + virtual void refresh(const Field &U, GridSerialRNG & sRNG, GridParallelRNG &pRNG) {} virtual RealD S(const Field &p) { diff --git a/Grid/qcd/hmc/HMC.h b/Grid/qcd/hmc/HMC.h index f168b69a..44674ea5 100644 --- a/Grid/qcd/hmc/HMC.h +++ b/Grid/qcd/hmc/HMC.h @@ -139,7 +139,7 @@ private: // Evolution ///////////////////////////////////////////////////////// RealD evolve_hmc_step(Field &U) { - TheIntegrator.refresh(U, pRNG); // set U and initialize P and phi's + TheIntegrator.refresh(U, sRNG, pRNG); // set U and initialize P and phi's RealD H0 = TheIntegrator.S(U); // initial state action diff --git a/Grid/qcd/hmc/integrators/Integrator.h b/Grid/qcd/hmc/integrators/Integrator.h index 77b7de52..aa28c6c8 100644 --- a/Grid/qcd/hmc/integrators/Integrator.h +++ b/Grid/qcd/hmc/integrators/Integrator.h @@ -236,10 +236,9 @@ public: // over the representations struct _refresh { template - void operator()(std::vector*> repr_set, Repr& Rep, - GridParallelRNG& pRNG) { + void operator()(std::vector*> repr_set, Repr& Rep, GridSerialRNG & sRNG, GridParallelRNG& pRNG) { for (int a = 0; a < repr_set.size(); ++a){ - repr_set.at(a)->refresh(Rep.U, pRNG); + repr_set.at(a)->refresh(Rep.U, sRNG, pRNG); std::cout << GridLogDebug << "Hirep refreshing pseudofermions" << std::endl; } @@ -247,12 +246,12 @@ public: } refresh_hireps{}; // Initialization of momenta and actions - void refresh(Field& U, GridParallelRNG& pRNG) + void refresh(Field& U, GridSerialRNG & sRNG, GridParallelRNG& pRNG) { assert(P.Grid() == U.Grid()); std::cout << GridLogIntegrator << "Integrator refresh\n"; - FieldImplementation::generate_momenta(P, pRNG); + FieldImplementation::generate_momenta(P, sRNG, pRNG); // Update the smeared fields, can be implemented as observer // necessary to keep the fields updated even after a reject @@ -269,11 +268,11 @@ public: // get gauge field from the SmearingPolicy and // based on the boolean is_smeared in actionID Field& Us = Smearer.get_U(as[level].actions.at(actionID)->is_smeared); - as[level].actions.at(actionID)->refresh(Us, pRNG); + as[level].actions.at(actionID)->refresh(Us, sRNG, pRNG); } // Refresh the higher representation actions - as[level].apply(refresh_hireps, Representations, pRNG); + as[level].apply(refresh_hireps, Representations, sRNG, pRNG); } MomFilter->applyFilter(P); diff --git a/benchmarks/Benchmark_dwf_fp32.cc b/benchmarks/Benchmark_dwf_fp32.cc index cb86177e..03f3ee61 100644 --- a/benchmarks/Benchmark_dwf_fp32.cc +++ b/benchmarks/Benchmark_dwf_fp32.cc @@ -53,7 +53,7 @@ int main (int argc, char ** argv) int threads = GridThread::GetThreads(); Coordinate latt4 = GridDefaultLatt(); - int Ls=8; + int Ls=16; for(int i=0;i> Ls; From d4b4de8f428c625757c31c6d265c288671a0ef5b Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Thu, 4 Mar 2021 20:01:24 +0000 Subject: [PATCH 14/86] changes --- Grid/qcd/utils/BaryonUtils.h | 60 +++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index b74a5b20..1a7a4d38 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -252,7 +252,7 @@ public: const Gamma GammaB_sigma, const Gamma GammaB_nucl, const std::string op, - SpinMatrixField &stn_corr); + SpinMatrixField &xts_corr); }; //This computes a baryon contraction on a lattice site, including the spin-trace of the correlation matrix template @@ -1435,6 +1435,7 @@ void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, int c_x = (ie_x < 3 ? (ie_x+2)%3 : (7-ie_x)%3 ); //epsilon[ie_x][2]; //c' int eSgn_x = (ie_x < 3 ? 1 : -1); ee = Real(eSgn_s * eSgn_x); + auto ee_GD = ee * trGDq; for (int alpha_x=0; alpha_x::XiToSigmaQ2EyeSite(const mobj &Dq_loop, // GammaB * DsGDd * GammaB auto GDsGDqGDdG = GDsGDqGDd * GammaB_sigma; + Real ee; + for (int ie_s=0; ie_s < 6 ; ie_s++){ - int a_s = epsilon[ie_s][0]; //a - int b_s = epsilon[ie_s][1]; //b - int c_s = epsilon[ie_s][2]; //c + int a_s = (ie_s < 3 ? ie_s : (6-ie_s)%3 ); //epsilon[ie_s][0]; //a' + int b_s = (ie_s < 3 ? (ie_s+1)%3 : (8-ie_s)%3 ); //epsilon[ie_s][1]; //b' + int c_s = (ie_s < 3 ? (ie_s+2)%3 : (7-ie_s)%3 ); //epsilon[ie_s][2]; //c' + int eSgn_s = (ie_s < 3 ? 1 : -1); for (int ie_x=0; ie_x < 6 ; ie_x++){ - int a_x = epsilon[ie_x][0]; //a' - int b_x = epsilon[ie_x][1]; //b' - int c_x = epsilon[ie_x][2]; //c' - auto ee = epsilon_sgn[ie_s] * epsilon_sgn[ie_x]; + int a_x = (ie_x < 3 ? ie_x : (6-ie_x)%3 ); //epsilon[ie_x][0]; //a' + int b_x = (ie_x < 3 ? (ie_x+1)%3 : (8-ie_x)%3 ); //epsilon[ie_x][1]; //b' + int c_x = (ie_x < 3 ? (ie_x+2)%3 : (7-ie_x)%3 ); //epsilon[ie_x][2]; //c' + int eSgn_x = (ie_x < 3 ? 1 : -1); + ee = Real(eSgn_s * eSgn_x); for (int alpha_x=0; alpha_x::XiToSigmaEye(const PropagatorField &qq_loop, autoView( vd_tf , qd_tf , AcceleratorRead); autoView( vs_ti , qs_ti , AcceleratorRead); - bool doQ1 = (op == "Q1"); - bool doQ2 = (op == "Q2"); - Vector my_Dq_spec{Dd_spec,Ds_spec}; mobj * Dq_spec_p = &my_Dq_spec[0]; - accelerator_for(ss, grid->oSites(), grid->Nsimd(), { - auto Dq_loop = vq_loop(ss); - auto Dd_tf = vd_tf(ss); - auto Ds_ti = vs_ti(ss); - typedef decltype(coalescedRead(vcorr[0])) spinor; - spinor result=Zero(); - if(doQ1){ + if(op == "Q1"){ + accelerator_for(ss, grid->oSites(), grid->Nsimd(), { + auto Dq_loop = vq_loop(ss); + auto Dd_tf = vd_tf(ss); + auto Ds_ti = vs_ti(ss); + typedef decltype(coalescedRead(vcorr[0])) spinor; + spinor result=Zero(); XiToSigmaQ1EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[1],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); - } else if(doQ2){ - XiToSigmaQ2EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[0],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); - } else { - assert(0 && "Weak Operator not correctly specified"); - } - coalescedWrite(vcorr[ss],result); - } );//end loop over lattice sites + coalescedWrite(vcorr[ss],result); + } );//end loop over lattice sites + } else if(op == "Q2"){ + accelerator_for(ss, grid->oSites(), grid->Nsimd(), { + auto Dq_loop = vq_loop(ss); + auto Dd_tf = vd_tf(ss); + auto Ds_ti = vs_ti(ss); + typedef decltype(coalescedRead(vcorr[0])) spinor; + spinor result=Zero(); + XiToSigmaQ2EyeSite(Dq_loop,Dq_spec_p[0],Dq_spec_p[1],Dd_tf,Ds_ti,Gamma_H,GammaB_xi,GammaB_sigma,result); + coalescedWrite(vcorr[ss],result); + } );//end loop over lattice sites + } else { + assert(0 && "Weak Operator not correctly specified"); + } } From 9b15704290048b936e71c3ddd652a6eec9bf5fff Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Fri, 5 Mar 2021 10:42:32 +0000 Subject: [PATCH 15/86] tested and consitent --- Grid/qcd/utils/BaryonUtils.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index 1a7a4d38..b69865e8 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -27,7 +27,6 @@ *************************************************************************************/ /* END LEGAL */ #pragma once -//#include #include NAMESPACE_BEGIN(Grid); @@ -200,7 +199,7 @@ public: const Gamma GammaB_sigma, const Gamma GammaB_nucl, robj &result); - template + template accelerator_inline static void XiToSigmaQ1EyeSite(const mobj &Dq_loop, const mobj2 &Dd_spec, const mobj2 &Ds_spec, @@ -210,7 +209,7 @@ public: const Gamma GammaB_sigma, const Gamma GammaB_nucl, robj &result); - template + template accelerator_inline static void XiToSigmaQ2EyeSite(const mobj &Dq_loop, const mobj2 &Dd_spec, const mobj2 &Ds_spec, @@ -1395,7 +1394,7 @@ void BaryonUtils::SigmaToNucleonNonEye(const PropagatorField &qq_ti, * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ template -template +template accelerator_inline void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, const mobj2 &Dd_spec, const mobj2 &Ds_spec, @@ -1464,7 +1463,7 @@ void BaryonUtils::XiToSigmaQ1EyeSite(const mobj &Dq_loop, * Dd_tf is a quark line from t_f to t_H * Ds_ti is a quark line from t_i to t_H */ template -template +template accelerator_inline void BaryonUtils::XiToSigmaQ2EyeSite(const mobj &Dq_loop, const mobj2 &Dd_spec, const mobj2 &Ds_spec, From 7a19432e0b77fff8943ad6701f448ab366dab27f Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Fri, 5 Mar 2021 10:57:09 +0000 Subject: [PATCH 16/86] whitespace --- Grid/qcd/utils/BaryonUtils.h | 84 ++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Grid/qcd/utils/BaryonUtils.h b/Grid/qcd/utils/BaryonUtils.h index b69865e8..9d9cb508 100644 --- a/Grid/qcd/utils/BaryonUtils.h +++ b/Grid/qcd/utils/BaryonUtils.h @@ -191,34 +191,34 @@ public: robj &result); template accelerator_inline static void SigmaToNucleonQ2NonEyeSite(const mobj &Du_ti, - const mobj &Du_tf, - const mobj2 &Du_spec, - const mobj &Dd_tf, - const mobj &Ds_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - robj &result); + const mobj &Du_tf, + const mobj2 &Du_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result); template accelerator_inline static void XiToSigmaQ1EyeSite(const mobj &Dq_loop, - const mobj2 &Dd_spec, - const mobj2 &Ds_spec, - const mobj &Dd_tf, - const mobj &Ds_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - robj &result); + const mobj2 &Dd_spec, + const mobj2 &Ds_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result); template accelerator_inline static void XiToSigmaQ2EyeSite(const mobj &Dq_loop, - const mobj2 &Dd_spec, - const mobj2 &Ds_spec, - const mobj &Dd_tf, - const mobj &Ds_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - robj &result); + const mobj2 &Dd_spec, + const mobj2 &Ds_spec, + const mobj &Dd_tf, + const mobj &Ds_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + robj &result); public: template static void SigmaToNucleonEye(const PropagatorField &qq_loop, @@ -232,26 +232,26 @@ public: SpinMatrixField &stn_corr); template static void SigmaToNucleonNonEye(const PropagatorField &qq_ti, - const PropagatorField &qq_tf, - const mobj &Du_spec, - const PropagatorField &qd_tf, - const PropagatorField &qs_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - const std::string op, - SpinMatrixField &stn_corr); + const PropagatorField &qq_tf, + const mobj &Du_spec, + const PropagatorField &qd_tf, + const PropagatorField &qs_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + const std::string op, + SpinMatrixField &stn_corr); template static void XiToSigmaEye(const PropagatorField &qq_loop, - const mobj &Dd_spec, - const mobj &Ds_spec, - const PropagatorField &qd_tf, - const PropagatorField &qs_ti, - const Gamma Gamma_H, - const Gamma GammaB_sigma, - const Gamma GammaB_nucl, - const std::string op, - SpinMatrixField &xts_corr); + const mobj &Dd_spec, + const mobj &Ds_spec, + const PropagatorField &qd_tf, + const PropagatorField &qs_ti, + const Gamma Gamma_H, + const Gamma GammaB_sigma, + const Gamma GammaB_nucl, + const std::string op, + SpinMatrixField &xts_corr); }; //This computes a baryon contraction on a lattice site, including the spin-trace of the correlation matrix template From aa173e29989b93445cc28655dd7d0feadb746961 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Fri, 5 Mar 2021 10:25:33 -0500 Subject: [PATCH 17/86] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 4cbae720..fff68dc6 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,6 @@ If you want to build all the tests at once just use `make tests`. - `--enable-numa`: enable NUMA first touch optimisation - `--enable-simd=`: setup Grid for the SIMD target `` (default: `GEN`). A list of possible SIMD targets is detailed in a section below. - `--enable-gen-simd-width=`: select the size (in bytes) of the generic SIMD vector type (default: 32 bytes). -- `--enable-precision={single|double}`: set the default precision (default: `double`). **Deprecated option** - `--enable-comms=`: Use `` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below. - `--enable-rng={sitmo|ranlux48|mt19937}`: choose the RNG (default: `sitmo `). - `--disable-timers`: disable system dependent high-resolution timers. From a76cb005e007bf887e0573175f9e9806fecc61b1 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 8 Mar 2021 13:37:57 -0500 Subject: [PATCH 18/86] Update Tensor_exp.h --- Grid/tensors/Tensor_exp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/tensors/Tensor_exp.h b/Grid/tensors/Tensor_exp.h index 0a1d6389..1f637d5f 100644 --- a/Grid/tensors/Tensor_exp.h +++ b/Grid/tensors/Tensor_exp.h @@ -28,7 +28,7 @@ Author: neo #ifndef GRID_MATH_EXP_H #define GRID_MATH_EXP_H -#define DEFAULT_MAT_EXP 12 +#define DEFAULT_MAT_EXP 20 NAMESPACE_BEGIN(Grid); From 4d1ea15c79d30c961272d5020404395db44365dc Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 9 Mar 2021 04:29:37 +0100 Subject: [PATCH 19/86] More verbosity. The 16bit limit on Grid.y, Grid.z is annoying --- Grid/threads/Accelerator.cc | 3 +-- Grid/threads/Accelerator.h | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Grid/threads/Accelerator.cc b/Grid/threads/Accelerator.cc index 4bf7f395..9d9d851c 100644 --- a/Grid/threads/Accelerator.cc +++ b/Grid/threads/Accelerator.cc @@ -53,7 +53,6 @@ void acceleratorInit(void) prop = gpu_props[i]; totalDeviceMem = prop.totalGlobalMem; if ( world_rank == 0) { -#ifndef GRID_DEFAULT_GPU if ( i==rank ) { printf("AcceleratorCudaInit[%d]: ========================\n",rank); printf("AcceleratorCudaInit[%d]: Device Number : %d\n", rank,i); @@ -67,8 +66,8 @@ void acceleratorInit(void) GPU_PROP(warpSize); GPU_PROP(pciBusID); GPU_PROP(pciDeviceID); + printf("AcceleratorCudaInit[%d]: maxGridSize (%d,%d,%d)\n",rank,prop.maxGridSize[0],prop.maxGridSize[1],prop.maxGridSize[2]); } -#endif // GPU_PROP(unifiedAddressing); // GPU_PROP(l2CacheSize); // GPU_PROP(singleToDoublePrecisionPerfRatio); diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h index f1a694fb..56b85c72 100644 --- a/Grid/threads/Accelerator.h +++ b/Grid/threads/Accelerator.h @@ -178,9 +178,10 @@ void Lambda6Apply(uint64_t num1, uint64_t num2, uint64_t num3, cudaDeviceSynchronize(); \ cudaError err = cudaGetLastError(); \ if ( cudaSuccess != err ) { \ - printf("Cuda error %s \n", cudaGetErrorString( err )); \ - puts(__FILE__); \ - printf("Line %d\n",__LINE__); \ + printf("accelerator_barrier(): Cuda error %s \n", \ + cudaGetErrorString( err )); \ + printf("File %s Line %d\n",__FILE__,__LINE__); \ + fflush(stdout); \ if (acceleratorAbortOnGpuError) assert(err==cudaSuccess); \ } \ } From 6a429ee6d3eee71689f650782decb8948bf6fa77 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 9 Mar 2021 04:31:10 +0100 Subject: [PATCH 20/86] 2d loop hits Nvidia 16bit limit on large local vols --- Grid/cshift/Cshift_common.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Grid/cshift/Cshift_common.h b/Grid/cshift/Cshift_common.h index f2f39815..cf902b58 100644 --- a/Grid/cshift/Cshift_common.h +++ b/Grid/cshift/Cshift_common.h @@ -110,9 +110,11 @@ Gather_plane_extract(const Lattice &rhs, int n1=rhs.Grid()->_slice_stride[dimension]; if ( cbmask ==0x3){ -#ifdef ACCELERATOR_CSHIFT +#ifdef ACCELERATOR_CSHIFT autoView(rhs_v , rhs, AcceleratorRead); - accelerator_for2d(n,e1,b,e2,1,{ + accelerator_for(nn,e1*e2,1,{ + int n = nn%e1; + int b = nn/e1; int o = n*n1; int offset = b+n*e2; @@ -135,7 +137,9 @@ Gather_plane_extract(const Lattice &rhs, std::cout << " Dense packed buffer WARNING " < void Scatter_plane_merge(Lattice &rhs,ExtractPointerA int _slice_block = rhs.Grid()->_slice_block[dimension]; #ifdef ACCELERATOR_CSHIFT autoView( rhs_v , rhs, AcceleratorWrite); - accelerator_for2d(n,e1,b,e2,1,{ + accelerator_for(nn,e1*e2,1,{ + int n = nn%e1; + int b = nn/e1; int o = n*_slice_stride; int offset = b+n*_slice_block; merge(rhs_v[so+o+b],pointers,offset); @@ -274,7 +280,7 @@ template void Scatter_plane_merge(Lattice &rhs,ExtractPointerA // Case of SIMD split AND checker dim cannot currently be hit, except in // Test_cshift_red_black code. - // std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME + std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME std::cout<<" Unthreaded warning -- buffer is not densely packed ??"< Date: Wed, 10 Mar 2021 02:45:22 +0100 Subject: [PATCH 21/86] Clean up test --- tests/core/Test_where.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/Test_where.cc b/tests/core/Test_where.cc index 050b711b..deb29865 100644 --- a/tests/core/Test_where.cc +++ b/tests/core/Test_where.cc @@ -40,9 +40,9 @@ int main (int argc, char ** argv) int N=16; - std::vector latt_size ({N,4,4}); - std::vector simd_layout({vComplexD::Nsimd(),1,1}); - std::vector mpi_layout ({1,1,1}); + std::vector latt_size ({N,N,N,N}); + std::vector simd_layout({vComplexD::Nsimd(),1,1,1}); + std::vector mpi_layout ({1,1,1,1}); int vol = 1; int nd = latt_size.size(); @@ -69,7 +69,7 @@ int main (int argc, char ** argv) for(int t=0;t Date: Wed, 10 Mar 2021 05:40:51 -0800 Subject: [PATCH 22/86] Gives 200GF/s on SyCL/DG1 8^4, doesn't uglify develop for other platforms too badly. Easy to revert to clean more C++ stylistic code. Theres a SYCL_HACK macro I will clean up later once dpcpp evolves a central nervous systems. --- Grid/qcd/action/fermion/WilsonKernels.h | 10 +- .../WilsonKernelsHandImplementation.h | 99 ++++++++++++++++--- .../WilsonKernelsImplementation.h | 20 +++- 3 files changed, 115 insertions(+), 14 deletions(-) diff --git a/Grid/qcd/action/fermion/WilsonKernels.h b/Grid/qcd/action/fermion/WilsonKernels.h index 1bac9211..68422f28 100644 --- a/Grid/qcd/action/fermion/WilsonKernels.h +++ b/Grid/qcd/action/fermion/WilsonKernels.h @@ -49,9 +49,17 @@ public: INHERIT_IMPL_TYPES(Impl); typedef FermionOperator Base; - + typedef AcceleratorVector StencilVector; public: +#ifdef GRID_SYCL +#define SYCL_HACK +#endif +#ifdef SYCL_HACK + static void HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor *buf, + int ss,int sU,const SiteSpinor *in, SiteSpinor *out); +#endif + static void DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField &U, SiteHalfSpinor * buf, int Ls, int Nsite, const FermionField &in, FermionField &out, int interior=1,int exterior=1) ; diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h index 688cb75a..fb42fe88 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h @@ -118,15 +118,6 @@ Author: paboyle #endif -#define LOAD_CHI \ - {const SiteHalfSpinor &ref(buf[offset]); \ - Chi_00 = coalescedRead(ref()(0)(0)); \ - Chi_01 = coalescedRead(ref()(0)(1)); \ - Chi_02 = coalescedRead(ref()(0)(2)); \ - Chi_10 = coalescedRead(ref()(1)(0)); \ - Chi_11 = coalescedRead(ref()(1)(1)); \ - Chi_12 = coalescedRead(ref()(1)(2));} - #define MULT_2SPIN(A)\ {auto & ref(U[sU](A)); \ U_00=coalescedRead(ref()(0,0)); \ @@ -157,6 +148,15 @@ Author: paboyle UChi_02+= U_20*Chi_02; \ UChi_12+= U_20*Chi_12;} +#define LOAD_CHI \ + {const SiteHalfSpinor &ref(buf[offset]); \ + Chi_00 = coalescedRead(ref()(0)(0)); \ + Chi_01 = coalescedRead(ref()(0)(1)); \ + Chi_02 = coalescedRead(ref()(0)(2)); \ + Chi_10 = coalescedRead(ref()(1)(0)); \ + Chi_11 = coalescedRead(ref()(1)(1)); \ + Chi_12 = coalescedRead(ref()(1)(2));} + // hspin(0)=fspin(0)+timesI(fspin(3)); // hspin(1)=fspin(1)+timesI(fspin(2)); #define XP_PROJ \ @@ -370,7 +370,7 @@ Author: paboyle result_31-= UChi_11; \ result_32-= UChi_12; -#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \ +#define HAND_STENCIL_LEGB(PROJ,PERM,DIR,RECON) \ SE=st.GetEntry(ptype,DIR,ss); \ offset = SE->_offset; \ local = SE->_is_local; \ @@ -384,6 +384,37 @@ Author: paboyle } else { \ LOAD_CHI; \ } \ + acceleratorSynchronise(); \ + MULT_2SPIN(DIR); \ + RECON; + +#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \ + SE=&st_p[DIR+8*ss]; \ + ptype=st_perm[DIR]; \ + offset = SE->_offset; \ + local = SE->_is_local; \ + perm = SE->_permute; \ + if ( local ) { \ + LOAD_CHIMU(PERM); \ + PROJ; \ + if ( perm) { \ + PERMUTE_DIR(PERM); \ + } \ + } else { \ + LOAD_CHI; \ + } \ + acceleratorSynchronise(); \ + MULT_2SPIN(DIR); \ + RECON; + +#define HAND_STENCIL_LEGA(PROJ,PERM,DIR,RECON) \ + SE=&st_p[DIR+8*ss]; \ + ptype=st_perm[DIR]; \ + /*SE=st.GetEntry(ptype,DIR,ss);*/ \ + offset = SE->_offset; \ + perm = SE->_permute; \ + LOAD_CHIMU(PERM); \ + PROJ; \ MULT_2SPIN(DIR); \ RECON; @@ -401,10 +432,12 @@ Author: paboyle } else if ( st.same_node[DIR] ) { \ LOAD_CHI; \ } \ + acceleratorSynchronise(); \ if (local || st.same_node[DIR] ) { \ MULT_2SPIN(DIR); \ RECON; \ - } + } \ + acceleratorSynchronise(); #define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON) \ SE=st.GetEntry(ptype,DIR,ss); \ @@ -414,7 +447,8 @@ Author: paboyle MULT_2SPIN(DIR); \ RECON; \ nmu++; \ - } + } \ + acceleratorSynchronise(); #define HAND_RESULT(ss) \ { \ @@ -511,10 +545,41 @@ Author: paboyle NAMESPACE_BEGIN(Grid); + +#ifdef SYCL_HACK +template accelerator_inline void +WilsonKernels::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor *buf, + int ss,int sU,const SiteSpinor *in, SiteSpinor *out) +{ +// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc... + typedef typename Simd::scalar_type S; + typedef typename Simd::vector_type V; + typedef iSinglet vCplx; + // typedef decltype( coalescedRead( vCplx()()() )) Simt; + typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + + HAND_DECLARATIONS(Simt); + + int offset,local,perm, ptype; + StencilEntry *SE; + HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON); + HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM); + HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM); + HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM); + HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM); + HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM); + HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM); + HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM); + HAND_RESULT(ss); +} +#endif + template accelerator_inline void WilsonKernels::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int ss,int sU,const FermionFieldView &in, FermionFieldView &out) { + auto st_p = st._entries_p; + auto st_perm = st._permute_type; // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc... typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; @@ -539,6 +604,8 @@ template accelerator_inline void WilsonKernels::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int ss,int sU,const FermionFieldView &in, FermionFieldView &out) { + auto st_p = st._entries_p; + auto st_perm = st._permute_type; typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; @@ -562,6 +629,8 @@ template accelerator_inline void WilsonKernels::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int ss,int sU,const FermionFieldView &in, FermionFieldView &out) { + auto st_p = st._entries_p; + auto st_perm = st._permute_type; // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc... typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; @@ -586,6 +655,8 @@ template accelerator_inline void WilsonKernels::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int ss,int sU,const FermionFieldView &in, FermionFieldView &out) { + auto st_p = st._entries_p; + auto st_perm = st._permute_type; typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; @@ -609,6 +680,8 @@ template accelerator_inline void WilsonKernels::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int ss,int sU,const FermionFieldView &in, FermionFieldView &out) { + auto st_p = st._entries_p; + auto st_perm = st._permute_type; // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc... typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; @@ -634,6 +707,8 @@ template accelerator_inline void WilsonKernels::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf, int ss,int sU,const FermionFieldView &in, FermionFieldView &out) { + auto st_p = st._entries_p; + auto st_perm = st._permute_type; typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h index 937d13af..9228b84c 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h @@ -416,7 +416,21 @@ void WilsonKernels::DhopDirKernel( StencilImpl &st, DoubledGaugeField &U,S #undef LoopBody } -#define KERNEL_CALLNB(A) \ +#define KERNEL_CALL_TMP(A) \ + const uint64_t NN = Nsite*Ls; \ + auto U_p = & U_v[0]; \ + auto in_p = & in_v[0]; \ + auto out_p = & out_v[0]; \ + auto st_p = st_v._entries_p; \ + auto st_perm = st_v._permute_type; \ + accelerator_forNB( ss, NN, Simd::Nsimd(), { \ + int sF = ss; \ + int sU = ss/Ls; \ + WilsonKernels::A(st_perm,st_p,U_p,buf,sF,sU,in_p,out_p); \ + }); \ + accelerator_barrier(); + +#define KERNEL_CALLNB(A) \ const uint64_t NN = Nsite*Ls; \ accelerator_forNB( ss, NN, Simd::Nsimd(), { \ int sF = ss; \ @@ -445,7 +459,11 @@ void WilsonKernels::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField if( interior && exterior ) { if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALL(GenericDhopSite); return;} +#ifdef SYCL_HACK + if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL_TMP(HandDhopSiteSycl); return; } +#else if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALL(HandDhopSite); return;} +#endif #ifndef GRID_CUDA if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSite); return;} #endif From f786ff8d69534e7885a55a7090a26b4d28619209 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 10 Mar 2021 14:32:06 -0500 Subject: [PATCH 23/86] Extend test from Fionn, fails on A100 apparently --- tests/core/Test_where_extended.cc | 137 ++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/core/Test_where_extended.cc diff --git a/tests/core/Test_where_extended.cc b/tests/core/Test_where_extended.cc new file mode 100644 index 00000000..706fd7ee --- /dev/null +++ b/tests/core/Test_where_extended.cc @@ -0,0 +1,137 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/Test_poisson_fft.cc + + Copyright (C) 2015 + +Author: Azusa Yamaguchi +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include + +using namespace Grid; + ; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + int threads = GridThread::GetThreads(); + std::cout< latt_size ({N,4,4}); + std::vector simd_layout({vComplexD::Nsimd(),1,1}); + std::vector mpi_layout ({1,1,1}); + + int vol = 1; + int nd = latt_size.size(); + for(int d=0;d({45,12,81,9})); + gaussian(RNG,rn); + + RealD nn=norm2(rn); + for(int mu=0;mu seeds4({1,2,3,4}); + GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); + LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1); + + + std::cout<=tmin),tmpF,ZZF); + nA = nA + norm2(tmp2F); + InsertSlice(tmp2F, q_outF, s , 0); + } + + RealD nQO=norm2(q_outF); + std::cout <=tmin),tmpP,ZZP); + nA = nA + norm2(tmp2P); + InsertSlice(tmp2P, q_outP, s , 0); + } + + nQO=norm2(q_outP); + std::cout < Date: Thu, 11 Mar 2021 12:58:49 +0100 Subject: [PATCH 24/86] Fix inconsistent configure option AVX512 Before this change AVX512 enabled different instruction sets depending on the compiler: For Intel C++ Compiler Classic (ICC): AVX512F, AVX512CD, AVX512DQ, AVX512BW, AVX512VL i.e. Intel Xeon Skylake and newer For Intel ICX, gcc, clang: AVX512F, AVX512CD, AVX512ER, AVX512PF i.e. Intel Xeon Phi x200/x205 (KNL/KNM) With this commit AVX512 now only enables the common instruction sets supported by all CPUs supporting any AVX-512 instructions set: AVX512F and AVX512CD (called COMMON-AVX512 by icc) --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 5f165412..afd5cbad 100644 --- a/configure.ac +++ b/configure.ac @@ -444,7 +444,7 @@ case ${ax_cv_cxx_compiler_vendor} in SIMD_FLAGS='-mavx2 -mfma -mf16c';; AVX512) AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) - SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';; + SIMD_FLAGS='-mavx512f -mavx512cd';; SKL) AC_DEFINE([AVX512],[1],[AVX512 intrinsics for SkyLake Xeon]) SIMD_FLAGS='-march=skylake-avx512';; @@ -498,7 +498,7 @@ case ${ax_cv_cxx_compiler_vendor} in SIMD_FLAGS='-march=core-avx2 -xcore-avx2';; AVX512) AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) - SIMD_FLAGS='-xcore-avx512';; + SIMD_FLAGS='-xcommon-avx512';; KNC) AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner]) SIMD_FLAGS='';; From 82402c6a7cc3057dd0e4c2f6389ba2b347972cfc Mon Sep 17 00:00:00 2001 From: Peter Georg Date: Thu, 11 Mar 2021 13:08:40 +0100 Subject: [PATCH 25/86] Add simd option SKL for ICC --- configure.ac | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure.ac b/configure.ac index afd5cbad..4e5e33c8 100644 --- a/configure.ac +++ b/configure.ac @@ -499,6 +499,9 @@ case ${ax_cv_cxx_compiler_vendor} in AVX512) AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) SIMD_FLAGS='-xcommon-avx512';; + SKL) + AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) + SIMD_FLAGS='-xcore-avx512';; KNC) AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner]) SIMD_FLAGS='';; From ce1fc1f48aac5a48512b6ff70c1d85d0810d7623 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 11 Mar 2021 22:20:53 +0100 Subject: [PATCH 26/86] Possible fallback plan for Fionn's compiler bbug in nvcc --- Grid/lattice/Lattice_where.h | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/Grid/lattice/Lattice_where.h b/Grid/lattice/Lattice_where.h index 6686d1b3..e9a3f823 100644 --- a/Grid/lattice/Lattice_where.h +++ b/Grid/lattice/Lattice_where.h @@ -43,7 +43,7 @@ inline void whereWolf(Lattice &ret,const Lattice &predicate,Lattice< conformable(iftrue,predicate); conformable(iftrue,ret); - GridBase *grid=iftrue._grid; + GridBase *grid=iftrue.Grid(); typedef typename vobj::scalar_object scalar_object; typedef typename vobj::scalar_type scalar_type; @@ -52,22 +52,24 @@ inline void whereWolf(Lattice &ret,const Lattice &predicate,Lattice< const int Nsimd = grid->Nsimd(); - std::vector mask(Nsimd); - std::vector truevals (Nsimd); - std::vector falsevals(Nsimd); + Integer mask; + scalar_object trueval; + scalar_object falseval; - parallel_for(int ss=0;ssoSites(); ss++){ - - extract(iftrue._odata[ss] ,truevals); - extract(iffalse._odata[ss] ,falsevals); - extract(TensorRemove(predicate._odata[ss]),mask); - - for(int s=0;soSites(); + thread_for(ss,NN,{ + for(int l=0;l @@ -76,9 +78,9 @@ inline Lattice whereWolf(const Lattice &predicate,Lattice &ift conformable(iftrue,iffalse); conformable(iftrue,predicate); - Lattice ret(iftrue._grid); + Lattice ret(iftrue.Grid()); - where(ret,predicate,iftrue,iffalse); + whereWolf(ret,predicate,iftrue,iffalse); return ret; } From cd5891eecd45b53db3a02498d97d0bd5e29c67eb Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 11 Mar 2021 22:34:28 +0100 Subject: [PATCH 27/86] Test that fails on Cuda 11.0 --- tests/core/Test_where_extended.cc | 124 ++++++++++++++++-------------- 1 file changed, 65 insertions(+), 59 deletions(-) diff --git a/tests/core/Test_where_extended.cc b/tests/core/Test_where_extended.cc index 706fd7ee..9862b3ed 100644 --- a/tests/core/Test_where_extended.cc +++ b/tests/core/Test_where_extended.cc @@ -51,87 +51,93 @@ int main (int argc, char ** argv) } GridCartesian GRID(latt_size,simd_layout,mpi_layout); - - LatticeComplexD zz(&GRID); - LatticeInteger coor(&GRID); - LatticeComplexD rn(&GRID); - LatticeComplexD sl(&GRID); - - zz = ComplexD(0.0,0.0); - GridParallelRNG RNG(&GRID); RNG.SeedFixedIntegers(std::vector({45,12,81,9})); - gaussian(RNG,rn); - RealD nn=norm2(rn); - for(int mu=0;mu seeds4({1,2,3,4}); - GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); - LatticeInteger lcoor(UGrid); LatticeCoordinate(lcoor,Nd-1); - std::cout<=tmin),tmpF,ZZF); - nA = nA + norm2(tmp2F); - InsertSlice(tmp2F, q_outF, s , 0); + gaussian(RNG,rn); + + RealD nn=norm2(rn); + for(int mu=0;mu=tmin),tmpP,ZZP); - nA = nA + norm2(tmp2P); - InsertSlice(tmp2P, q_outP, s , 0); + zz = ComplexD(0.0,0.0); + + gaussian(RNG,rn); + + RealD nn=norm2(rn); + for(int mu=0;mu Date: Thu, 11 Mar 2021 23:54:53 +0100 Subject: [PATCH 28/86] NVCC versions found buggy added as guard --- Grid/util/CompilerCompatible.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Grid/util/CompilerCompatible.h b/Grid/util/CompilerCompatible.h index 37331668..7c4a056d 100644 --- a/Grid/util/CompilerCompatible.h +++ b/Grid/util/CompilerCompatible.h @@ -1,5 +1,16 @@ #pragma once +#if defined(__NVCC__) + +#if (__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ == 0) +#error "NVCC version 11.0 breaks on Ampere, see Github issue 346" +#endif +#if (__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ == 1) +#error "NVCC version 11.1 breaks on Ampere, see Github issue 346" +#endif + +#endif + #if defined(__clang__) #if __clang_major__ < 3 From db3ac67506bcbdbe0799632dcc1561a96d0e320f Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Fri, 12 Mar 2021 14:55:07 +0100 Subject: [PATCH 29/86] Update thread issue --- Grid/lattice/Lattice_where.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Grid/lattice/Lattice_where.h b/Grid/lattice/Lattice_where.h index e9a3f823..777f4015 100644 --- a/Grid/lattice/Lattice_where.h +++ b/Grid/lattice/Lattice_where.h @@ -52,16 +52,15 @@ inline void whereWolf(Lattice &ret,const Lattice &predicate,Lattice< const int Nsimd = grid->Nsimd(); - Integer mask; - scalar_object trueval; - scalar_object falseval; - autoView(iftrue_v,iftrue,CpuRead); autoView(iffalse_v,iffalse,CpuRead); autoView(predicate_v,predicate,CpuRead); autoView(ret_v,ret,CpuWrite); Integer NN= grid->oSites(); thread_for(ss,NN,{ + Integer mask; + scalar_object trueval; + scalar_object falseval; for(int l=0;l Date: Fri, 12 Mar 2021 09:31:17 -0500 Subject: [PATCH 30/86] updated to do list. Start adding DDHMC work items --- TODO | 75 ++++++++++++++++++++++++++++-------------------------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/TODO b/TODO index f1175560..e23e040d 100644 --- a/TODO +++ b/TODO @@ -1,3 +1,6 @@ +-- comms threads issue?? +-- Part done: Staggered kernel performance on GPU + ========================================================= General ========================================================= @@ -5,28 +8,18 @@ General - Make representations code take Gimpl - Simplify the HMCand remove modules - Lattice_arith - are the mult, mac etc.. still needed after ET engine? -- Lattice_rng -- Lattice_transfer.h -- accelerate A2Autils -- off critical path for HMC +- Lattice_rng - faster local only loop in init +- Audit: accelerate A2Autils -- off critical path for HMC ========================================================= -GPU branch code item work list +GPU work list ========================================================= -* sum_cpu promote to double during summation for increased precisoin. +* sum_cpu promote to double during summation for increased precision. * Introduce sumD & ReduceD * GPU sum is probably better currently. - * Accelerate the cshift & benchmark -* 0) Single GPU -- 128 bit integer table load in GPU code. - - ImprovedStaggered accelerate & measure perf - - Gianluca's changes to Cayley into gpu-port - - Mobius kernel fusion. -- Gianluca? - - Lebesque order reintroduction. StencilView should have pointer to it - - Lebesgue reorder in all kernels - * 3) Comms/NVlink - OpenMP tasks to run comms threads. Experiment with it - Remove explicit openMP in staggered. @@ -35,14 +28,6 @@ GPU branch code item work list - Stencil gather ?? - SIMD dirs in stencil -* 4) ET enhancements -- eval -> scalar ops in ET engine -- coalescedRead, coalescedWrite in expressions. - -* 5) Misc -- Conserved current clean up. -- multLinkProp eliminate - 8) Merge develop and test HMC 9) Gamma tables on GPU; check this. Appear to work, but no idea why. Are these done on CPU? @@ -52,7 +37,7 @@ GPU branch code item work list - Audit NAMESPACE CHANGES - Audit changes ------ +--------- Gianluca's changes - Performance impact of construct in aligned allocator??? --------- @@ -62,6 +47,33 @@ Gianluca's changes ----------------------------- DONE: ----------------------------- +===== +-- Done: Remez X^-1/2 X^-1/2 X = 1 test. + Feed in MdagM^2 as a test and take its sqrt. + Automated test that MdagM invsqrt(MdagM)invsqrt(MdagM) = 1 in HMC for bounds satisfaction. + +-- Done: Sycl Kernels into develop. Compare to existing unroll and just use. +-- Done: sRNG into refresh functions +-- Done: Tuned decomposition on CUDA into develop +-- Done: Sycl friend accessor. Const view attempt via typedef?? + + +* Done 5) Misc +- Conserved current clean up. +- multLinkProp eliminate + +* Done 0) Single GPU +- 128 bit integer table load in GPU code. + - ImprovedStaggered accelerate & measure perf + - Gianluca's changes to Cayley into gpu-port + - Mobius kernel fusion. -- Gianluca? + - Lebesque order reintroduction. StencilView should have pointer to it + - Lebesgue reorder in all kernels + +* 4) ET enhancements +- Done eval -> scalar ops in ET engine +- Done coalescedRead, coalescedWrite in expressions. + ============================================================================================= AUDIT ContractWWVV with respect to develop -- DONE - GPU accelerate EOFA -- DONE @@ -125,23 +137,6 @@ AUDIT ContractWWVV with respect to develop -- DONE - - (4) omp parallel for collapse(n) - - Only (1) has a natural mirror in accelerator_loop - - Nested loop macros get cumbersome made a generic interface for N deep -- - Don't like thread_region and thread_loop_in_region -- - Could replace with - - thread_nested(1, - for { - - } - ); - thread_nested(2, - for (){ - for (){ - - } - } - ); - - and same "in_region". ----------------------------- From 51f506553c59923fc60f3a68333b14757b8853a5 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Fri, 12 Mar 2021 15:33:04 +0100 Subject: [PATCH 31/86] Read out the local ID once, and store --- .../WilsonKernelsHandImplementation.h | 165 ++++++++++-------- 1 file changed, 96 insertions(+), 69 deletions(-) diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h index fb42fe88..0703b613 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h @@ -79,18 +79,18 @@ Author: paboyle #ifdef GRID_SIMT #define LOAD_CHIMU(ptype) \ {const SiteSpinor & ref (in[offset]); \ - Chimu_00=coalescedReadPermute(ref()(0)(0),perm); \ - Chimu_01=coalescedReadPermute(ref()(0)(1),perm); \ - Chimu_02=coalescedReadPermute(ref()(0)(2),perm); \ - Chimu_10=coalescedReadPermute(ref()(1)(0),perm); \ - Chimu_11=coalescedReadPermute(ref()(1)(1),perm); \ - Chimu_12=coalescedReadPermute(ref()(1)(2),perm); \ - Chimu_20=coalescedReadPermute(ref()(2)(0),perm); \ - Chimu_21=coalescedReadPermute(ref()(2)(1),perm); \ - Chimu_22=coalescedReadPermute(ref()(2)(2),perm); \ - Chimu_30=coalescedReadPermute(ref()(3)(0),perm); \ - Chimu_31=coalescedReadPermute(ref()(3)(1),perm); \ - Chimu_32=coalescedReadPermute(ref()(3)(2),perm); } + Chimu_00=coalescedReadPermute(ref()(0)(0),perm,lane); \ + Chimu_01=coalescedReadPermute(ref()(0)(1),perm,lane); \ + Chimu_02=coalescedReadPermute(ref()(0)(2),perm,lane); \ + Chimu_10=coalescedReadPermute(ref()(1)(0),perm,lane); \ + Chimu_11=coalescedReadPermute(ref()(1)(1),perm,lane); \ + Chimu_12=coalescedReadPermute(ref()(1)(2),perm,lane); \ + Chimu_20=coalescedReadPermute(ref()(2)(0),perm,lane); \ + Chimu_21=coalescedReadPermute(ref()(2)(1),perm,lane); \ + Chimu_22=coalescedReadPermute(ref()(2)(2),perm,lane); \ + Chimu_30=coalescedReadPermute(ref()(3)(0),perm,lane); \ + Chimu_31=coalescedReadPermute(ref()(3)(1),perm,lane); \ + Chimu_32=coalescedReadPermute(ref()(3)(2),perm,lane); } #define PERMUTE_DIR(dir) ; #else #define LOAD_CHIMU(ptype) \ @@ -119,43 +119,43 @@ Author: paboyle #endif #define MULT_2SPIN(A)\ - {auto & ref(U[sU](A)); \ - U_00=coalescedRead(ref()(0,0)); \ - U_10=coalescedRead(ref()(1,0)); \ - U_20=coalescedRead(ref()(2,0)); \ - U_01=coalescedRead(ref()(0,1)); \ - U_11=coalescedRead(ref()(1,1)); \ - U_21=coalescedRead(ref()(2,1)); \ - UChi_00 = U_00*Chi_00; \ - UChi_10 = U_00*Chi_10; \ - UChi_01 = U_10*Chi_00; \ - UChi_11 = U_10*Chi_10; \ - UChi_02 = U_20*Chi_00; \ - UChi_12 = U_20*Chi_10; \ - UChi_00+= U_01*Chi_01; \ - UChi_10+= U_01*Chi_11; \ - UChi_01+= U_11*Chi_01; \ - UChi_11+= U_11*Chi_11; \ - UChi_02+= U_21*Chi_01; \ - UChi_12+= U_21*Chi_11; \ - U_00=coalescedRead(ref()(0,2)); \ - U_10=coalescedRead(ref()(1,2)); \ - U_20=coalescedRead(ref()(2,2)); \ - UChi_00+= U_00*Chi_02; \ - UChi_10+= U_00*Chi_12; \ - UChi_01+= U_10*Chi_02; \ - UChi_11+= U_10*Chi_12; \ - UChi_02+= U_20*Chi_02; \ + {auto & ref(U[sU](A)); \ + U_00=coalescedRead(ref()(0,0),lane); \ + U_10=coalescedRead(ref()(1,0),lane); \ + U_20=coalescedRead(ref()(2,0),lane); \ + U_01=coalescedRead(ref()(0,1),lane); \ + U_11=coalescedRead(ref()(1,1),lane); \ + U_21=coalescedRead(ref()(2,1),lane); \ + UChi_00 = U_00*Chi_00; \ + UChi_10 = U_00*Chi_10; \ + UChi_01 = U_10*Chi_00; \ + UChi_11 = U_10*Chi_10; \ + UChi_02 = U_20*Chi_00; \ + UChi_12 = U_20*Chi_10; \ + UChi_00+= U_01*Chi_01; \ + UChi_10+= U_01*Chi_11; \ + UChi_01+= U_11*Chi_01; \ + UChi_11+= U_11*Chi_11; \ + UChi_02+= U_21*Chi_01; \ + UChi_12+= U_21*Chi_11; \ + U_00=coalescedRead(ref()(0,2),lane); \ + U_10=coalescedRead(ref()(1,2),lane); \ + U_20=coalescedRead(ref()(2,2),lane); \ + UChi_00+= U_00*Chi_02; \ + UChi_10+= U_00*Chi_12; \ + UChi_01+= U_10*Chi_02; \ + UChi_11+= U_10*Chi_12; \ + UChi_02+= U_20*Chi_02; \ UChi_12+= U_20*Chi_12;} #define LOAD_CHI \ {const SiteHalfSpinor &ref(buf[offset]); \ - Chi_00 = coalescedRead(ref()(0)(0)); \ - Chi_01 = coalescedRead(ref()(0)(1)); \ - Chi_02 = coalescedRead(ref()(0)(2)); \ - Chi_10 = coalescedRead(ref()(1)(0)); \ - Chi_11 = coalescedRead(ref()(1)(1)); \ - Chi_12 = coalescedRead(ref()(1)(2));} + Chi_00 = coalescedRead(ref()(0)(0),lane); \ + Chi_01 = coalescedRead(ref()(0)(1),lane); \ + Chi_02 = coalescedRead(ref()(0)(2),lane); \ + Chi_10 = coalescedRead(ref()(1)(0),lane); \ + Chi_11 = coalescedRead(ref()(1)(1),lane); \ + Chi_12 = coalescedRead(ref()(1)(2),lane);} // hspin(0)=fspin(0)+timesI(fspin(3)); // hspin(1)=fspin(1)+timesI(fspin(2)); @@ -453,35 +453,35 @@ Author: paboyle #define HAND_RESULT(ss) \ { \ SiteSpinor & ref (out[ss]); \ - coalescedWrite(ref()(0)(0),result_00); \ - coalescedWrite(ref()(0)(1),result_01); \ - coalescedWrite(ref()(0)(2),result_02); \ - coalescedWrite(ref()(1)(0),result_10); \ - coalescedWrite(ref()(1)(1),result_11); \ - coalescedWrite(ref()(1)(2),result_12); \ - coalescedWrite(ref()(2)(0),result_20); \ - coalescedWrite(ref()(2)(1),result_21); \ - coalescedWrite(ref()(2)(2),result_22); \ - coalescedWrite(ref()(3)(0),result_30); \ - coalescedWrite(ref()(3)(1),result_31); \ - coalescedWrite(ref()(3)(2),result_32); \ + coalescedWrite(ref()(0)(0),result_00,lane); \ + coalescedWrite(ref()(0)(1),result_01,lane); \ + coalescedWrite(ref()(0)(2),result_02,lane); \ + coalescedWrite(ref()(1)(0),result_10,lane); \ + coalescedWrite(ref()(1)(1),result_11,lane); \ + coalescedWrite(ref()(1)(2),result_12,lane); \ + coalescedWrite(ref()(2)(0),result_20,lane); \ + coalescedWrite(ref()(2)(1),result_21,lane); \ + coalescedWrite(ref()(2)(2),result_22,lane); \ + coalescedWrite(ref()(3)(0),result_30,lane); \ + coalescedWrite(ref()(3)(1),result_31,lane); \ + coalescedWrite(ref()(3)(2),result_32,lane); \ } #define HAND_RESULT_EXT(ss) \ { \ SiteSpinor & ref (out[ss]); \ - coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00); \ - coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01); \ - coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02); \ - coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10); \ - coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11); \ - coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12); \ - coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20); \ - coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21); \ - coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22); \ - coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30); \ - coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31); \ - coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32); \ + coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00,lane); \ + coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01,lane); \ + coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02,lane); \ + coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10,lane); \ + coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11,lane); \ + coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12,lane); \ + coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20,lane); \ + coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21,lane); \ + coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22,lane); \ + coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30,lane); \ + coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31,lane); \ + coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32,lane); \ } #define HAND_DECLARATIONS(Simd) \ @@ -558,6 +558,9 @@ WilsonKernels::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, // typedef decltype( coalescedRead( vCplx()()() )) Simt; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + HAND_DECLARATIONS(Simt); int offset,local,perm, ptype; @@ -584,6 +587,10 @@ WilsonKernels::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,Site typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + HAND_DECLARATIONS(Simt); int offset,local,perm, ptype; @@ -609,6 +616,10 @@ void WilsonKernels::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + HAND_DECLARATIONS(Simt); StencilEntry *SE; @@ -635,6 +646,10 @@ WilsonKernels::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,Si typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + HAND_DECLARATIONS(Simt); int offset,local,perm, ptype; @@ -660,6 +675,10 @@ void WilsonKernels::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldVi typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + HAND_DECLARATIONS(Simt); StencilEntry *SE; @@ -686,6 +705,10 @@ WilsonKernels::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,Si typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + HAND_DECLARATIONS(Simt); int offset, ptype; @@ -712,6 +735,10 @@ void WilsonKernels::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldVi typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt; + + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + HAND_DECLARATIONS(Simt); StencilEntry *SE; From 9c2b37218a8849ece7198fc7a25b56bde10f6b13 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 18 Mar 2021 06:24:11 -0400 Subject: [PATCH 32/86] sRNG parameter added --- Grid/qcd/utils/Metric.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Grid/qcd/utils/Metric.h b/Grid/qcd/utils/Metric.h index 10d06de8..d8ae27dc 100644 --- a/Grid/qcd/utils/Metric.h +++ b/Grid/qcd/utils/Metric.h @@ -93,13 +93,13 @@ public: GeneralisedMomenta(GridBase* grid, Metric& M): M(M), Mom(grid), AuxMom(grid), AuxField(grid){} // Correct - void MomentaDistribution(GridParallelRNG& pRNG){ + void MomentaDistribution(GridSerialRNG & sRNG, GridParallelRNG& pRNG){ // Generate a distribution for // P^dag G P // where G = M^-1 // Generate gaussian momenta - Implementation::generate_momenta(Mom, pRNG); + Implementation::generate_momenta(Mom, sRNG, pRNG); // Modify the distribution with the metric M.MSquareRoot(Mom); @@ -107,8 +107,8 @@ public: // Auxiliary momenta // do nothing if trivial, so hide in the metric MomentaField AuxMomTemp(Mom.Grid()); - Implementation::generate_momenta(AuxMom, pRNG); - Implementation::generate_momenta(AuxField, pRNG); + Implementation::generate_momenta(AuxMom, sRNG, pRNG); + Implementation::generate_momenta(AuxField, sRNG, pRNG); // Modify the distribution with the metric // Aux^dag M Aux M.MInvSquareRoot(AuxMom); // AuxMom = M^{-1/2} AuxMomTemp From 49b0af2c95bd078b6b0b40b8b28b2ea8309daf45 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 18 Mar 2021 09:10:02 -0400 Subject: [PATCH 33/86] Update of tests to compile with the sRNG addition. Audited the code conventions (again) with the CPS momentum denominator and added anti periodic in time to the Test_mobius_force.cc and tested the Test_dwf_gpforce. Promoted thesee to test full HMC hamiltonian, tr P^2/2 + phidag MdagM phi with the same pdot and Udot as audited in the Integrator.h etc... With full comments and sources for factors. --- tests/forces/Test_dwf_force_eofa.cc | 4 +- tests/forces/Test_dwf_gpforce.cc | 123 +++++++++++++++-------- tests/forces/Test_dwf_gpforce_eofa.cc | 3 +- tests/forces/Test_laplacian_force.cc | 3 +- tests/forces/Test_mobius_force.cc | 109 +++++++++++++++++--- tests/forces/Test_mobius_force_eofa.cc | 3 +- tests/forces/Test_mobius_gpforce_eofa.cc | 3 +- 7 files changed, 188 insertions(+), 60 deletions(-) diff --git a/tests/forces/Test_dwf_force_eofa.cc b/tests/forces/Test_dwf_force_eofa.cc index 80d36934..525178d0 100644 --- a/tests/forces/Test_dwf_force_eofa.cc +++ b/tests/forces/Test_dwf_force_eofa.cc @@ -86,7 +86,9 @@ int main (int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, true); - Meofa.refresh(U, RNG5); + GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds4); + Meofa.refresh(U, sRNG, RNG5 ); + RealD S = Meofa.S(U); // pdag M p // get the deriv of phidag M phi with respect to "U" diff --git a/tests/forces/Test_dwf_gpforce.cc b/tests/forces/Test_dwf_gpforce.cc index 28133cc6..1fa1c6e4 100644 --- a/tests/forces/Test_dwf_gpforce.cc +++ b/tests/forces/Test_dwf_gpforce.cc @@ -84,6 +84,13 @@ int main (int argc, char ** argv) GparityDomainWallFermionR::ImplParams params; params.twists = twists; + /* + params.boundary_phases[0] = 1.0; + params.boundary_phases[1] = 1.0; + params.boundary_phases[2] = 1.0; + params.boundary_phases[3] =- 1.0; + */ + GparityDomainWallFermionR Dw(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,params); Dw.M (phi,Mphi); @@ -96,6 +103,16 @@ int main (int argc, char ** argv) Dw.MDeriv(tmp , Mphi, phi,DaggerNo ); UdSdU=tmp; Dw.MDeriv(tmp , phi, Mphi,DaggerYes ); UdSdU=(UdSdU+tmp); + + // ***************************************************************************************** + // *** There is a funny negative sign in all derivatives. This is - UdSdU. *** + // *** *** + // *** Deriv in both Wilson gauge action and the TwoFlavour.h seems to miss a minus sign *** + // *** UdSdU is negated relative to what I think - call what is returned mUdSdU, *** + // *** and insert minus sign *** + // ***************************************************************************************** + + UdSdU = - UdSdU ; // Follow sign convention of actions in Grid. Seems crazy. FermionField Ftmp (FGrid); @@ -106,7 +123,7 @@ int main (int argc, char ** argv) RealD Hmom = 0.0; RealD Hmomprime = 0.0; LatticeColourMatrix mommu(UGrid); - LatticeColourMatrix forcemu(UGrid); + LatticeColourMatrix mUdSdUmu(UGrid); LatticeGaugeField mom(UGrid); LatticeGaugeField Uprime(UGrid); @@ -114,10 +131,20 @@ int main (int argc, char ** argv) SU::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg - Hmom -= real(sum(trace(mommu*mommu))); + // Momentum Hamiltonian is - trace(p^2)/HMC_MOM_DENOMINATOR + // + // Integrator.h: RealD H = - FieldImplementation::FieldSquareNorm(P)/HMC_MOMENTUM_DENOMINATOR; // - trace (P*P)/denom // GaugeImplTypes.h: Hloc += trace(Pmu * Pmu); + // Sign comes from a sneaky multiply by "i" in GaussianFundemantalLie algebra + // P is i P^a_\mu T^a, not Pa Ta + // + // Integrator.h: H = Hmom + sum S(action) + Hmom -= real(sum(trace(mommu*mommu)))/ HMC_MOMENTUM_DENOMINATOR; PokeIndex(mom,mommu,mu); + // -- Drops factor of "i" in the U update: U' = e^{P dt} U [ _not_ e^{iPdt}U ]. P is anti hermitian already + // -- Udot = p U + // fourth order exponential approx autoView( mom_v, mom, CpuRead); autoView( U_v , U, CpuRead); @@ -134,8 +161,8 @@ int main (int argc, char ** argv) ; }); } - std::cout << GridLogMessage <<"Initial mom hamiltonian is "<< Hmom <(mom,mu); - std::cout << GridLogMessage<< " Mommu " << norm2(mommu)<(UdSdU,mu); - std::cout << GridLogMessage<< " dsdumu " << norm2(mommu)<(UdSdU,mu); - mommu=Ta(mommu)*2.0; + mommu=Ta(mommu); // projectForce , GaugeImplTypes.h PokeIndex(UdSdU,mommu,mu); } for(int mu=0;mu(mom,mu); - std::cout << GridLogMessage<< " Mommu " << norm2(mommu)<(UdSdU,mu); - std::cout << GridLogMessage<< " dsdumu " << norm2(mommu)<(UdSdU,mu); + mUdSdUmu= PeekIndex(UdSdU,mu); mommu = PeekIndex(mom,mu); - // Update PF action density - dS = dS+trace(mommu*forcemu)*dt; + // + // Derive HMC eom: + // + // Sdot = - 2 trace( p p^dot ) / D - trace( p [ mUdSdU - h.c. ] ) = 0 + // + // + // Sdot = 0 = - 2 trace( p p^dot ) / D - 2 trace( p Ta( mUdSdU ) = 0 + // + // EOM: + // + // pdot = - D Ta( mUdSdU ) -- source of sign is the "funny sign" above + // + // dSqcd_dt = - 2.0*trace(mommu* Ta(mUdSdU) )*dt -- i.e. mUdSdU with adjoint term -> force has a 2x implicit + // + // dH_mom/dt = - 2 trace (p pdot)/Denom + // + // dH_tot / dt = 0 <= pdot = - Denom * mUdSdU + // + // dH_mom/dt = 2 trace (p mUdSdU ) + // + // True Momentum delta H has a dt^2 piece + // + // dSmom = [ trace mom*mom - trace ( (mom-Denom*f*dt)(mom-Denom*f*dt) ) ] / Denom + // = 2*trace(mom*f) dt - Denom*dt*dt * trace(f*f). + // = dSmom + dSmom2 + // - dSmom = dSmom - trace(mommu*forcemu) * dt; - dSmom2 = dSmom2 - trace(forcemu*forcemu) *(0.25* dt*dt); + dS = dS - 2.0*trace(mommu*mUdSdUmu)*dt; // U and Udagger derivs hence 2x. - // Update mom action density - mommu = mommu + forcemu*(dt*0.5); + dSmom = dSmom + 2.0*trace(mommu*mUdSdUmu) * dt; // this 2.0 coms from derivative of p^2 + + dSmom2 = dSmom2 - trace(mUdSdUmu*mUdSdUmu) * dt*dt* HMC_MOMENTUM_DENOMINATOR; // Remnant - Hmomprime -= real(sum(trace(mommu*mommu))); + // Update mom action density . Verbatim update_P in Integrator.h + mommu = mommu - mUdSdUmu * dt* HMC_MOMENTUM_DENOMINATOR;; + + Hmomprime -= real(sum(trace(mommu*mommu))) / HMC_MOMENTUM_DENOMINATOR; } @@ -199,20 +233,25 @@ int main (int argc, char ** argv) ComplexD dSm = sum(dSmom); ComplexD dSm2 = sum(dSmom2); + std::cout << GridLogMessage <<"dSm "<< dSm< CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, true); - Meofa.refresh(U, RNG5); + GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds4); + Meofa.refresh(U, sRNG, RNG5); RealD S = Meofa.S(U); // pdag M p // get the deriv of phidag M phi with respect to "U" diff --git a/tests/forces/Test_laplacian_force.cc b/tests/forces/Test_laplacian_force.cc index 18508860..dbaf1cbd 100644 --- a/tests/forces/Test_laplacian_force.cc +++ b/tests/forces/Test_laplacian_force.cc @@ -46,6 +46,7 @@ int main (int argc, char ** argv) std::vector seeds({1,2,3,4}); + GridSerialRNG sRNG; sRNG.SeedFixedIntegers({4,5,6,7}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({15,91,21,3})); @@ -67,7 +68,7 @@ int main (int argc, char ** argv) LaplacianAdjointField Laplacian(&Grid, CG, LapPar, Kappa); GeneralisedMomenta LaplacianMomenta(&Grid, Laplacian); LaplacianMomenta.M.ImportGauge(U); - LaplacianMomenta.MomentaDistribution(pRNG);// fills the Momenta with the correct distr + LaplacianMomenta.MomentaDistribution(sRNG,pRNG);// fills the Momenta with the correct distr std::cout << std::setprecision(15); diff --git a/tests/forces/Test_mobius_force.cc b/tests/forces/Test_mobius_force.cc index ba7bc363..d2326a81 100644 --- a/tests/forces/Test_mobius_force.cc +++ b/tests/forces/Test_mobius_force.cc @@ -69,7 +69,14 @@ int main (int argc, char ** argv) RealD M5=1.8; RealD b=0.5; RealD c=0.5; - MobiusFermionR Ddwf(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c); + + WilsonImplParams p; + p.boundary_phases[0] = 1.0; + p.boundary_phases[1] = 1.0; + p.boundary_phases[2] = 1.0; + p.boundary_phases[3] =- 1.0; + + MobiusFermionR Ddwf(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c,p); Ddwf.M (phi,Mphi); ComplexD S = innerProduct(Mphi,Mphi); // pdag MdagM p @@ -82,24 +89,44 @@ int main (int argc, char ** argv) Ddwf.MDeriv(tmp , Mphi, phi,DaggerNo ); UdSdU=tmp; Ddwf.MDeriv(tmp , phi, Mphi,DaggerYes ); UdSdU=(UdSdU+tmp); + // ***************************************************************************************** + // *** There is a funny negative sign in all derivatives. This is - UdSdU. *** + // *** *** + // *** Deriv in both Wilson gauge action and the TwoFlavour.h seems to miss a minus sign *** + // *** UdSdU is negated relative to what I think - call what is returned mUdSdU, *** + // *** and insert minus sign *** + // ***************************************************************************************** + + UdSdU = - UdSdU ; // Follow sign convention of actions in Grid. Seems crazy. + LatticeFermion Ftmp (FGrid); //////////////////////////////////// // Modify the gauge field a little //////////////////////////////////// - RealD dt = 0.0001; + RealD dt = 0.001; + RealD Hmom = 0.0; + RealD Hmomprime = 0.0; LatticeColourMatrix mommu(UGrid); - LatticeColourMatrix forcemu(UGrid); + LatticeColourMatrix mUdSdUmu(UGrid); LatticeGaugeField mom(UGrid); LatticeGaugeField Uprime(UGrid); for(int mu=0;mu::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg - PokeIndex(mom,mommu,mu); + // Momentum Hamiltonian is - trace(p^2)/HMC_MOM_DENOMINATOR + // + // Integrator.h: RealD H = - FieldImplementation::FieldSquareNorm(P)/HMC_MOMENTUM_DENOMINATOR; // - trace (P*P)/denom // GaugeImplTypes.h: Hloc += trace(Pmu * Pmu); + // Sign comes from a sneaky multiply by "i" in GaussianFundemantalLie algebra + // P is i P^a_\mu T^a, not Pa Ta + // + // Integrator.h: H = Hmom + sum S(action) + Hmom -= real(sum(trace(mommu*mommu)))/ HMC_MOMENTUM_DENOMINATOR; + // fourth order exponential approx autoView( U_v , U, CpuRead); autoView( mom_v, mom, CpuRead); @@ -115,6 +142,7 @@ int main (int argc, char ** argv) ; }); } + std::cout << GridLogMessage <<"Initial mom hamiltonian is "<< Hmom <(UdSdU,mu); - mommu=Ta(mommu)*2.0; + mommu=Ta(mommu); PokeIndex(UdSdU,mommu,mu); } for(int mu=0;mu(UdSdU,mu); + + mUdSdUmu= PeekIndex(UdSdU,mu); mommu = PeekIndex(mom,mu); - // Update PF action density - dS = dS+trace(mommu*forcemu)*dt; + // + // Derive HMC eom: + // + // Sdot = - 2 trace( p p^dot ) / D - trace( p [ mUdSdU - h.c. ] ) = 0 + // + // + // Sdot = 0 = - 2 trace( p p^dot ) / D - 2 trace( p Ta( mUdSdU ) = 0 + // + // EOM: + // + // pdot = - D Ta( mUdSdU ) -- source of sign is the "funny sign" above + // + // dSqcd_dt = - 2.0*trace(mommu* Ta(mUdSdU) )*dt -- i.e. mUdSdU with adjoint term -> force has a 2x implicit + // + // dH_mom/dt = - 2 trace (p pdot)/Denom + // + // dH_tot / dt = 0 <= pdot = - Denom * mUdSdU + // + // dH_mom/dt = 2 trace (p mUdSdU ) + // + // True Momentum delta H has a dt^2 piece + // + // dSmom = [ trace mom*mom - trace ( (mom-Denom*f*dt)(mom-Denom*f*dt) ) ] / Denom + // = 2*trace(mom*f) dt - Denom*dt*dt * trace(f*f). + // = dSmom + dSmom2 + // + + dS = dS - 2.0*trace(mommu*mUdSdUmu)*dt; // U and Udagger derivs hence 2x. + + dSmom = dSmom + 2.0*trace(mommu*mUdSdUmu) * dt; // this 2.0 coms from derivative of p^2 + + dSmom2 = dSmom2 - trace(mUdSdUmu*mUdSdUmu) * dt*dt* HMC_MOMENTUM_DENOMINATOR; // Remnant + + mommu = mommu - mUdSdUmu * dt* HMC_MOMENTUM_DENOMINATOR;; + + Hmomprime -= real(sum(trace(mommu*mommu))) / HMC_MOMENTUM_DENOMINATOR; } ComplexD dSpred = sum(dS); + ComplexD dSm = sum(dSmom); + ComplexD dSm2 = sum(dSmom2); - std::cout << GridLogMessage << " -- S "< CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, false); - Meofa.refresh(U, RNG5); + GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds4); + Meofa.refresh(U, sRNG, RNG5 ); RealD S = Meofa.S(U); // pdag M p // get the deriv of phidag M phi with respect to "U" diff --git a/tests/forces/Test_mobius_gpforce_eofa.cc b/tests/forces/Test_mobius_gpforce_eofa.cc index 9c80b2aa..7f114615 100644 --- a/tests/forces/Test_mobius_gpforce_eofa.cc +++ b/tests/forces/Test_mobius_gpforce_eofa.cc @@ -93,7 +93,8 @@ int main (int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, CG, CG, CG, CG, Params, false); - Meofa.refresh(U, RNG5); + GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds4); + Meofa.refresh(U, sRNG, RNG5 ); RealD S = Meofa.S(U); // pdag M p // get the deriv of phidag M phi with respect to "U" From 15c50a7442527a7962e53cdc4a6bb3369f41501c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 18 Mar 2021 15:40:42 -0400 Subject: [PATCH 34/86] Explicit instantiate the template function --- Grid/util/Init.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc index 55d8c5bf..bfbc464d 100644 --- a/Grid/util/Init.cc +++ b/Grid/util/Init.cc @@ -153,6 +153,9 @@ void GridCmdOptionIntVector(const std::string &str,VectorInt & vec) return; } +template void GridCmdOptionIntVector(const std::string &str,std::vector & vec); +template void GridCmdOptionIntVector(const std::string &str,Coordinate & vec); + void GridCmdOptionInt(std::string &str,int & val) { std::stringstream ss(str); From 8bdadbadaca9cffd8d7616cc9deed07187b262f1 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 18 Mar 2021 15:41:14 -0400 Subject: [PATCH 35/86] Cold start --- HMC/Mobius2p1fRHMC.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HMC/Mobius2p1fRHMC.cc b/HMC/Mobius2p1fRHMC.cc index 82ca4d37..b958d548 100644 --- a/HMC/Mobius2p1fRHMC.cc +++ b/HMC/Mobius2p1fRHMC.cc @@ -56,12 +56,12 @@ int main(int argc, char **argv) { MD.trajL = 1.0; HMCparameters HMCparams; - HMCparams.StartTrajectory = 30; + HMCparams.StartTrajectory = 0; HMCparams.Trajectories = 200; HMCparams.NoMetropolisUntil= 0; // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n"; - // HMCparams.StartingType =std::string("ColdStart"); - HMCparams.StartingType =std::string("CheckpointStart"); + HMCparams.StartingType =std::string("ColdStart"); + // HMCparams.StartingType =std::string("CheckpointStart"); HMCparams.MD = MD; HMCWrapper TheHMC(HMCparams); From bb89a82a07be478d50060efd5c76cf71622af870 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 29 Mar 2021 20:01:15 +0200 Subject: [PATCH 36/86] Staggered coalseced read --- Grid/qcd/action/fermion/StaggeredImpl.h | 16 ++-- Grid/qcd/action/fermion/WilsonImpl.h | 35 ++++++++- .../StaggeredKernelsImplementation.h | 76 ++++++++++--------- 3 files changed, 82 insertions(+), 45 deletions(-) diff --git a/Grid/qcd/action/fermion/StaggeredImpl.h b/Grid/qcd/action/fermion/StaggeredImpl.h index 8adf45a4..f44d12f4 100644 --- a/Grid/qcd/action/fermion/StaggeredImpl.h +++ b/Grid/qcd/action/fermion/StaggeredImpl.h @@ -72,19 +72,23 @@ public: StaggeredImpl(const ImplParams &p = ImplParams()) : Params(p){}; - static accelerator_inline void multLink(SiteSpinor &phi, + template + static accelerator_inline void multLink(_Spinor &phi, const SiteDoubledGaugeField &U, - const SiteSpinor &chi, + const _Spinor &chi, int mu) { - mult(&phi(), &U(mu), &chi()); + auto UU = coalescedRead(U(mu)); + mult(&phi(), &UU, &chi()); } - static accelerator_inline void multLinkAdd(SiteSpinor &phi, + template + static accelerator_inline void multLinkAdd(_Spinor &phi, const SiteDoubledGaugeField &U, - const SiteSpinor &chi, + const _Spinor &chi, int mu) { - mac(&phi(), &U(mu), &chi()); + auto UU = coalescedRead(U(mu)); + mac(&phi(), &UU, &chi()); } template diff --git a/Grid/qcd/action/fermion/WilsonImpl.h b/Grid/qcd/action/fermion/WilsonImpl.h index 94676b6b..2ff6feba 100644 --- a/Grid/qcd/action/fermion/WilsonImpl.h +++ b/Grid/qcd/action/fermion/WilsonImpl.h @@ -184,18 +184,22 @@ public: mat = TraceIndex(P); } - inline void extractLinkField(std::vector &mat, DoubledGaugeField &Uds){ + inline void extractLinkField(std::vector &mat, DoubledGaugeField &Uds) + { for (int mu = 0; mu < Nd; mu++) mat[mu] = PeekIndex(Uds, mu); } - - inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu){ - + inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu) + { +#undef USE_OLD_INSERT_FORCE int Ls=Btilde.Grid()->_fdimensions[0]; + autoView( mat_v , mat, AcceleratorWrite); +#ifdef USE_OLD_INSERT_FORCE GaugeLinkField tmp(mat.Grid()); tmp = Zero(); { + const int Nsimd = SiteSpinor::Nsimd(); autoView( tmp_v , tmp, AcceleratorWrite); autoView( Btilde_v , Btilde, AcceleratorRead); autoView( Atilde_v , Atilde, AcceleratorRead); @@ -208,6 +212,29 @@ public: }); } PokeIndex(mat,tmp,mu); +#else + { + const int Nsimd = SiteSpinor::Nsimd(); + autoView( Btilde_v , Btilde, AcceleratorRead); + autoView( Atilde_v , Atilde, AcceleratorRead); + accelerator_for(sss,mat.Grid()->oSites(),Nsimd,{ + int sU=sss; + typedef decltype(coalescedRead(mat_v[sU](mu)() )) ColorMatrixType; + ColorMatrixType sum; + zeroit(sum); + for(int s=0;s_is_local ) { \ - if (SE->_permute) { \ - chi_p = χ \ - permute(chi, in[SE->_offset], ptype); \ - } else { \ - chi_p = &in[SE->_offset]; \ - } \ + int perm= SE->_permute; \ + chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\ } else { \ - chi_p = &buf[SE->_offset]; \ + chi = coalescedRead(buf[SE->_offset],lane); \ } \ - multLink(Uchi, U[sU], *chi_p, Dir); + acceleratorSynchronise(); \ + multLink(Uchi, U[sU], chi, Dir); #define GENERIC_STENCIL_LEG_INT(U,Dir,skew,multLink) \ SE = st.GetEntry(ptype, Dir+skew, sF); \ if (SE->_is_local ) { \ - if (SE->_permute) { \ - chi_p = χ \ - permute(chi, in[SE->_offset], ptype); \ - } else { \ - chi_p = &in[SE->_offset]; \ - } \ + int perm= SE->_permute; \ + chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane);\ } else if ( st.same_node[Dir] ) { \ - chi_p = &buf[SE->_offset]; \ + chi = coalescedRead(buf[SE->_offset],lane); \ } \ if (SE->_is_local || st.same_node[Dir] ) { \ - multLink(Uchi, U[sU], *chi_p, Dir); \ + multLink(Uchi, U[sU], chi, Dir); \ } #define GENERIC_STENCIL_LEG_EXT(U,Dir,skew,multLink) \ SE = st.GetEntry(ptype, Dir+skew, sF); \ if ((!SE->_is_local) && (!st.same_node[Dir]) ) { \ nmu++; \ - chi_p = &buf[SE->_offset]; \ - multLink(Uchi, U[sU], *chi_p, Dir); \ + chi = coalescedRead(buf[SE->_offset],lane); \ + multLink(Uchi, U[sU], chi, Dir); \ } template @@ -84,12 +77,14 @@ void StaggeredKernels::DhopSiteGeneric(StencilView &st, SiteSpinor *buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out, int dag) { - const SiteSpinor *chi_p; - SiteSpinor chi; - SiteSpinor Uchi; + typedef decltype(coalescedRead(in[0])) calcSpinor; + calcSpinor chi; + calcSpinor Uchi; StencilEntry *SE; int ptype; int skew; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); // for(int s=0;s::DhopSiteGeneric(StencilView &st, if ( dag ) { Uchi = - Uchi; } - vstream(out[sF], Uchi); + coalescedWrite(out[sF], Uchi,lane); } }; @@ -130,13 +125,16 @@ template accelerator_inline void StaggeredKernels::DhopSiteGenericInt(StencilView &st, DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU, SiteSpinor *buf, int sF, int sU, - const FermionFieldView &in, FermionFieldView &out,int dag) { - const SiteSpinor *chi_p; - SiteSpinor chi; - SiteSpinor Uchi; + const FermionFieldView &in, FermionFieldView &out,int dag) +{ + typedef decltype(coalescedRead(in[0])) calcSpinor; + calcSpinor chi; + calcSpinor Uchi; StencilEntry *SE; int ptype; int skew ; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); // for(int s=0;s::DhopSiteGenericInt(StencilView &st, if ( dag ) { Uchi = - Uchi; } - vstream(out[sF], Uchi); + coalescedWrite(out[sF], Uchi,lane); } }; @@ -178,14 +176,17 @@ template accelerator_inline void StaggeredKernels::DhopSiteGenericExt(StencilView &st, DoubledGaugeFieldView &U, DoubledGaugeFieldView &UUU, SiteSpinor *buf, int sF, int sU, - const FermionFieldView &in, FermionFieldView &out,int dag) { - const SiteSpinor *chi_p; - // SiteSpinor chi; - SiteSpinor Uchi; + const FermionFieldView &in, FermionFieldView &out,int dag) +{ + typedef decltype(coalescedRead(in[0])) calcSpinor; + calcSpinor chi; + calcSpinor Uchi; StencilEntry *SE; int ptype; int nmu=0; int skew ; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); // for(int s=0;s::DhopSiteGenericExt(StencilView &st, GENERIC_STENCIL_LEG_EXT(UUU,Zm,skew,Impl::multLinkAdd); GENERIC_STENCIL_LEG_EXT(UUU,Tm,skew,Impl::multLinkAdd); } - if ( nmu ) { - if ( dag ) { - out[sF] = out[sF] - Uchi; + if ( nmu ) { + auto _out = coalescedRead(out[sF],lane); + if ( dag ) { + coalescedWrite(out[sF], _out-Uchi,lane); } else { - out[sF] = out[sF] + Uchi; + coalescedWrite(out[sF], _out+Uchi,lane); } } } @@ -261,6 +263,8 @@ void StaggeredKernels::DhopImproved(StencilImpl &st, LebesgueOrder &lo, GridBase *FGrid=in.Grid(); GridBase *UGrid=U.Grid(); typedef StaggeredKernels ThisKernel; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); autoView( UUU_v , UUU, AcceleratorRead); autoView( U_v , U, AcceleratorRead); autoView( in_v , in, AcceleratorRead); @@ -301,6 +305,8 @@ void StaggeredKernels::DhopNaive(StencilImpl &st, LebesgueOrder &lo, GridBase *FGrid=in.Grid(); GridBase *UGrid=U.Grid(); typedef StaggeredKernels ThisKernel; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); autoView( UUU_v , U, AcceleratorRead); autoView( U_v , U, AcceleratorRead); autoView( in_v , in, AcceleratorRead); From e9479929570e3b6e000f60b6368d7981666fac7c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 29 Mar 2021 20:04:06 +0200 Subject: [PATCH 37/86] Improved force terms --- Grid/tensors/Tensor_outer.h | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Grid/tensors/Tensor_outer.h b/Grid/tensors/Tensor_outer.h index 4902c22f..a32a2a91 100644 --- a/Grid/tensors/Tensor_outer.h +++ b/Grid/tensors/Tensor_outer.h @@ -34,6 +34,16 @@ NAMESPACE_BEGIN(Grid); // outerProduct Scalar x Scalar -> Scalar // Vector x Vector -> Matrix /////////////////////////////////////////////////////////////////////////////////////// +template = 0> +accelerator_inline CC outerProduct(const CC &l, const CC& r) +{ + return l*conj(r); +} +template = 0> +accelerator_inline RR outerProduct(const RR &l, const RR& r) +{ + return l*r; +} template accelerator_inline auto outerProduct (const iVector& lhs,const iVector& rhs) -> iMatrix @@ -57,17 +67,6 @@ auto outerProduct (const iScalar& lhs,const iScalar& rhs) -> iScalar = 0> -accelerator_inline CC outerProduct(const CC &l, const CC& r) -{ - return l*conj(r); -} -template = 0> -accelerator_inline RR outerProduct(const RR &l, const RR& r) -{ - return l*r; -} - NAMESPACE_END(Grid); #endif From a7fb25adf66996f5de6861228774c803ef87bd4a Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 29 Mar 2021 21:44:14 +0200 Subject: [PATCH 38/86] Make Cshift fields static to avoid repeated reallocaate overhead --- Grid/cshift/Cshift_mpi.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Grid/cshift/Cshift_mpi.h b/Grid/cshift/Cshift_mpi.h index 375d004e..7e93e260 100644 --- a/Grid/cshift/Cshift_mpi.h +++ b/Grid/cshift/Cshift_mpi.h @@ -122,8 +122,8 @@ template void Cshift_comms(Lattice &ret,const Lattice &r assert(shift_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension]; - cshiftVector send_buf(buffer_size); - cshiftVector recv_buf(buffer_size); + static cshiftVector send_buf; send_buf.resize(buffer_size); + static cshiftVector recv_buf; recv_buf.resize(buffer_size); int cb= (cbmask==0x2)? Odd : Even; int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb); @@ -198,8 +198,8 @@ template void Cshift_comms_simd(Lattice &ret,const Lattice_slice_nblock[dimension]*grid->_slice_block[dimension]; // int words = sizeof(vobj)/sizeof(vector_type); - std::vector > send_buf_extract(Nsimd); - std::vector > recv_buf_extract(Nsimd); + static std::vector > send_buf_extract; send_buf_extract.resize(Nsimd); + static std::vector > recv_buf_extract; recv_buf_extract.resize(Nsimd); scalar_object * recv_buf_extract_mpi; scalar_object * send_buf_extract_mpi; @@ -294,8 +294,8 @@ template void Cshift_comms(Lattice &ret,const Lattice &r assert(shift_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension]; - cshiftVector send_buf_v(buffer_size); - cshiftVector recv_buf_v(buffer_size); + static cshiftVector send_buf_v; send_buf_v.resize(buffer_size); + static cshiftVector recv_buf_v; recv_buf_v.resize(buffer_size); vobj *send_buf; vobj *recv_buf; { @@ -381,8 +381,8 @@ template void Cshift_comms_simd(Lattice &ret,const Lattice_slice_nblock[dimension]*grid->_slice_block[dimension]; // int words = sizeof(vobj)/sizeof(vector_type); - std::vector > send_buf_extract(Nsimd); - std::vector > recv_buf_extract(Nsimd); + static std::vector > send_buf_extract; send_buf_extract.resize(Nsimd); + static std::vector > recv_buf_extract; recv_buf_extract.resize(Nsimd); scalar_object * recv_buf_extract_mpi; scalar_object * send_buf_extract_mpi; { From addeb621a7015cb690a08d4650e8f2b10568fbf7 Mon Sep 17 00:00:00 2001 From: Andrew Zhen Ning Yong Date: Tue, 6 Apr 2021 13:45:37 +0100 Subject: [PATCH 39/86] Implemented tadpole operator for Shamir action. --- .../implementation/CayleyFermion5DImplementation.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h b/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h index f11e9c44..c3e0f821 100644 --- a/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h +++ b/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h @@ -880,11 +880,23 @@ void CayleyFermion5D::SeqConservedCurrent(PropagatorField &q_in, } std::vector G_s(Ls,1.0); + Integer sign = 1; // sign flip for vector/tadpole if ( curr_type == Current::Axial ) { for(int s=0;s_b; + auto c=this->_c; + if ( b == 1 && c == 0 ) { + sign = -1; + } + else { + std::cerr << "Error: Tadpole implementation currently unavailable for non-Shamir actions." << std::endl; + assert(b==1 && c==0); + } + } for(int s=0;s::SeqConservedCurrent(PropagatorField &q_in, tmp = Cshift(tmp,mu,1); Impl::multLinkField(Utmp,this->Umu,tmp,mu); - tmp = G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop + tmp = sign*G_s[s]*( Utmp*ph - gmu*Utmp*ph ); // Forward hop tmp = where((lcoor>=tmin),tmp,zz); // Mask the time L_Q = where((lcoor<=tmax),tmp,zz); // Position of current complicated From 980e721f6e25864cadfea3611a9c1a052d8d05c4 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 13 Apr 2021 09:33:01 -0400 Subject: [PATCH 40/86] Update MetaData.h --- Grid/parallelIO/MetaData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/parallelIO/MetaData.h b/Grid/parallelIO/MetaData.h index d30ba523..af8b3f76 100644 --- a/Grid/parallelIO/MetaData.h +++ b/Grid/parallelIO/MetaData.h @@ -128,7 +128,7 @@ inline void MachineCharacteristics(FieldMetaData &header) std::time_t t = std::time(nullptr); std::tm tm_ = *std::localtime(&t); std::ostringstream oss; - // oss << std::put_time(&tm_, "%c %Z"); + oss << std::put_time(&tm_, "%c %Z"); header.creation_date = oss.str(); header.archive_date = header.creation_date; From 86e11743ca3bb52ebfbedce8584b95b7be6f7fe0 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 20 Apr 2021 10:19:11 -0400 Subject: [PATCH 41/86] set twists --- tests/forces/Test_gp_rect_force.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/forces/Test_gp_rect_force.cc b/tests/forces/Test_gp_rect_force.cc index 98ebb2fa..e277ea6b 100644 --- a/tests/forces/Test_gp_rect_force.cc +++ b/tests/forces/Test_gp_rect_force.cc @@ -29,7 +29,6 @@ Author: paboyle using namespace std; using namespace Grid; - ; @@ -59,6 +58,10 @@ int main (int argc, char ** argv) double beta = 1.0; double c1 = 0.331; + const int nu = 1; + std::vector twists(Nd,0); + twists[nu] = 1; + ConjugateGimplD::setDirections(twists); ConjugatePlaqPlusRectangleActionR Action(beta,c1); //ConjugateWilsonGaugeActionR Action(beta); //WilsonGaugeActionR Action(beta); From dbe210dd53405303cf57374c9b658902cbf8072a Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sun, 25 Apr 2021 10:25:59 -0400 Subject: [PATCH 42/86] Open the ens_id --- Grid/parallelIO/NerscIO.h | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Grid/parallelIO/NerscIO.h b/Grid/parallelIO/NerscIO.h index 3ebdf0cc..1ffda074 100644 --- a/Grid/parallelIO/NerscIO.h +++ b/Grid/parallelIO/NerscIO.h @@ -205,11 +205,22 @@ public: std::cout< + static inline void writeConfiguration(Lattice &Umu, + std::string file, + std::string ens_id = std::string("UKQCD"), + std::string ens_label = std::string("DWF")) + { + writeConfiguration(Umu,file,0,1,ens_id,ens_label); + } template static inline void writeConfiguration(Lattice &Umu, std::string file, int two_row, - int bits32) + int bits32, + std::string ens_id = std::string("UKQCD"), + std::string ens_label = std::string("DWF")) { typedef vLorentzColourMatrixD vobj; typedef typename vobj::scalar_object sobj; @@ -219,8 +230,8 @@ public: // Following should become arguments /////////////////////////////////////////// header.sequence_number = 1; - header.ensemble_id = "UKQCD"; - header.ensemble_label = "DWF"; + header.ensemble_id = ens_id; + header.ensemble_label = ens_label; typedef LorentzColourMatrixD fobj3D; typedef LorentzColour2x3D fobj2D; @@ -232,7 +243,7 @@ public: GaugeStats Stats; Stats(Umu,header); MachineCharacteristics(header); - uint64_t offset; + uint64_t offset; // Sod it -- always write 3x3 double header.floating_point = std::string("IEEE64BIG"); From 955a8113ded53fc55c204fc107832933ad120c2c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sun, 25 Apr 2021 10:36:38 -0400 Subject: [PATCH 43/86] Expose label only to reduce number of parameters --- Grid/parallelIO/NerscIO.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Grid/parallelIO/NerscIO.h b/Grid/parallelIO/NerscIO.h index 1ffda074..04aae5d8 100644 --- a/Grid/parallelIO/NerscIO.h +++ b/Grid/parallelIO/NerscIO.h @@ -209,7 +209,6 @@ public: template static inline void writeConfiguration(Lattice &Umu, std::string file, - std::string ens_id = std::string("UKQCD"), std::string ens_label = std::string("DWF")) { writeConfiguration(Umu,file,0,1,ens_id,ens_label); @@ -219,7 +218,6 @@ public: std::string file, int two_row, int bits32, - std::string ens_id = std::string("UKQCD"), std::string ens_label = std::string("DWF")) { typedef vLorentzColourMatrixD vobj; @@ -230,7 +228,7 @@ public: // Following should become arguments /////////////////////////////////////////// header.sequence_number = 1; - header.ensemble_id = ens_id; + header.ensemble_id = std::string("UKQCD"); header.ensemble_label = ens_label; typedef LorentzColourMatrixD fobj3D; From d45c868656b4e28452946f90a6e71cdf12c21cf2 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sun, 25 Apr 2021 10:53:34 -0400 Subject: [PATCH 44/86] Change interface --- Grid/parallelIO/NerscIO.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/parallelIO/NerscIO.h b/Grid/parallelIO/NerscIO.h index 04aae5d8..99011e25 100644 --- a/Grid/parallelIO/NerscIO.h +++ b/Grid/parallelIO/NerscIO.h @@ -211,7 +211,7 @@ public: std::string file, std::string ens_label = std::string("DWF")) { - writeConfiguration(Umu,file,0,1,ens_id,ens_label); + writeConfiguration(Umu,file,0,1,ens_label); } template static inline void writeConfiguration(Lattice &Umu, From 8cd4263974060f9af3b002604d9036e2552cc307 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sun, 25 Apr 2021 22:20:37 -0400 Subject: [PATCH 45/86] Tests compile --- tests/debug/Test_heatbath_dwf_eofa.cc | 6 ++++-- tests/debug/Test_heatbath_dwf_eofa_gparity.cc | 7 +++++-- tests/debug/Test_heatbath_mobius_eofa.cc | 6 ++++-- tests/debug/Test_heatbath_mobius_eofa_gparity.cc | 6 ++++-- tests/forces/Test_momentum_filter.cc | 4 +++- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/debug/Test_heatbath_dwf_eofa.cc b/tests/debug/Test_heatbath_dwf_eofa.cc index 9d453a96..e1c18021 100644 --- a/tests/debug/Test_heatbath_dwf_eofa.cc +++ b/tests/debug/Test_heatbath_dwf_eofa.cc @@ -66,7 +66,9 @@ int main(int argc, char** argv) // Set up RNGs std::vector seeds4({1, 2, 3, 4}); std::vector seeds5({5, 6, 7, 8}); + GridSerialRNG sRNG; GridParallelRNG RNG5(FGrid); + sRNG.SeedFixedIntegers(seeds5); RNG5.SeedFixedIntegers(seeds5); GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); @@ -84,7 +86,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, false); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu,sRNG, RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } @@ -94,7 +96,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, true); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu,sRNG, RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } diff --git a/tests/debug/Test_heatbath_dwf_eofa_gparity.cc b/tests/debug/Test_heatbath_dwf_eofa_gparity.cc index 22cc1e90..7eabfc65 100644 --- a/tests/debug/Test_heatbath_dwf_eofa_gparity.cc +++ b/tests/debug/Test_heatbath_dwf_eofa_gparity.cc @@ -74,6 +74,9 @@ int main(int argc, char** argv) RNG5.SeedFixedIntegers(seeds5); GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); + GridSerialRNG sRNG; + RNG4.SeedFixedIntegers(seeds4); + sRNG.SeedFixedIntegers(seeds5); // Random gauge field LatticeGaugeField Umu(UGrid); @@ -90,7 +93,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, false); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu,sRNG, RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } @@ -100,7 +103,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, true); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu,sRNG, RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } diff --git a/tests/debug/Test_heatbath_mobius_eofa.cc b/tests/debug/Test_heatbath_mobius_eofa.cc index 4cf4bf53..48806642 100644 --- a/tests/debug/Test_heatbath_mobius_eofa.cc +++ b/tests/debug/Test_heatbath_mobius_eofa.cc @@ -68,8 +68,10 @@ int main(int argc, char** argv) // Set up RNGs std::vector seeds4({1, 2, 3, 4}); std::vector seeds5({5, 6, 7, 8}); + GridSerialRNG sRNG; GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5); + sRNG.SeedFixedIntegers(seeds5); GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); @@ -86,7 +88,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, false); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu, sRNG,RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } @@ -96,7 +98,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, true); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu, sRNG,RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } diff --git a/tests/debug/Test_heatbath_mobius_eofa_gparity.cc b/tests/debug/Test_heatbath_mobius_eofa_gparity.cc index 2fcb4b9f..52447e5e 100644 --- a/tests/debug/Test_heatbath_mobius_eofa_gparity.cc +++ b/tests/debug/Test_heatbath_mobius_eofa_gparity.cc @@ -73,7 +73,9 @@ int main(int argc, char** argv) std::vector seeds4({1, 2, 3, 4}); std::vector seeds5({5, 6, 7, 8}); GridParallelRNG RNG5(FGrid); + GridSerialRNG sRNG; RNG5.SeedFixedIntegers(seeds5); + sRNG.SeedFixedIntegers(seeds5); GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); @@ -91,7 +93,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, false); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu, sRNG, RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } @@ -101,7 +103,7 @@ int main(int argc, char** argv) ConjugateGradient CG(1.0e-12, 5000); ExactOneFlavourRatioPseudoFermionAction Meofa(Lop, Rop, CG, Params, true); - Meofa.refresh(Umu, RNG5); + Meofa.refresh(Umu, sRNG, RNG5); printf(" = %1.15e\n", Meofa.S(Umu)); } diff --git a/tests/forces/Test_momentum_filter.cc b/tests/forces/Test_momentum_filter.cc index 856ea0f2..794b5fa0 100644 --- a/tests/forces/Test_momentum_filter.cc +++ b/tests/forces/Test_momentum_filter.cc @@ -61,7 +61,9 @@ int main (int argc, char ** argv) std::vector seeds({1,2,3,4}); GridParallelRNG pRNG(&Grid); + GridSerialRNG sRNG; pRNG.SeedFixedIntegers(seeds); + sRNG.SeedFixedIntegers(seeds); typedef PeriodicGimplR Gimpl; typedef WilsonGaugeAction GaugeAction; @@ -115,7 +117,7 @@ int main (int argc, char ** argv) integrator.setMomentumFilter(filter); - integrator.refresh(U, pRNG); //doesn't actually change the gauge field + integrator.refresh(U, sRNG, pRNG); //doesn't actually change the gauge field //Check the momentum is zero on the boundary const auto &P = integrator.getMomentum(); From 009ccd581ede8faf0ba748fa49a1757419106e23 Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Mon, 26 Apr 2021 10:36:33 +0100 Subject: [PATCH 46/86] bugfix 3D stout smearing --- Grid/qcd/smearing/StoutSmearing.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Grid/qcd/smearing/StoutSmearing.h b/Grid/qcd/smearing/StoutSmearing.h index ed2ccdb6..629f81e2 100644 --- a/Grid/qcd/smearing/StoutSmearing.h +++ b/Grid/qcd/smearing/StoutSmearing.h @@ -89,11 +89,12 @@ public: SmearBase->smear(C, U); for (int mu = 0; mu < Nd; mu++) { - if( mu == OrthogDim ) + Umu = peekLorentz(U, mu); + if( mu == OrthogDim ){ tmp = 1.0; // Don't smear in the orthogonal direction + } else { tmp = peekLorentz(C, mu); - Umu = peekLorentz(U, mu); iq_mu = Ta( tmp * adj(Umu)); // iq_mu = Ta(Omega_mu) to match the signs with the paper From cf2923d5ddb9c190ebb78efadab281e5a06ba247 Mon Sep 17 00:00:00 2001 From: Felix Erben Date: Tue, 27 Apr 2021 16:53:37 +0100 Subject: [PATCH 47/86] Jamie's fix --- Grid/qcd/smearing/StoutSmearing.h | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Grid/qcd/smearing/StoutSmearing.h b/Grid/qcd/smearing/StoutSmearing.h index 629f81e2..6ee78e8c 100644 --- a/Grid/qcd/smearing/StoutSmearing.h +++ b/Grid/qcd/smearing/StoutSmearing.h @@ -85,22 +85,18 @@ public: std::cout << GridLogDebug << "Stout smearing started\n"; - // Smear the configurations + // C contains the staples multiplied by some rho + u_smr = U ; // set the smeared field to the current gauge field SmearBase->smear(C, U); for (int mu = 0; mu < Nd; mu++) { + if( mu == OrthogDim ) continue ; + // u_smr = exp(iQ_mu)*U_mu apart from Orthogdim Umu = peekLorentz(U, mu); - if( mu == OrthogDim ){ - tmp = 1.0; // Don't smear in the orthogonal direction - } - else { - tmp = peekLorentz(C, mu); - iq_mu = Ta( - tmp * - adj(Umu)); // iq_mu = Ta(Omega_mu) to match the signs with the paper - exponentiate_iQ(tmp, iq_mu); - } - pokeLorentz(u_smr, tmp * Umu, mu); // u_smr = exp(iQ_mu)*U_mu + tmp = peekLorentz(C, mu); + iq_mu = Ta( tmp * adj(Umu)); + exponentiate_iQ(tmp, iq_mu); + pokeLorentz(u_smr, tmp * Umu, mu); } std::cout << GridLogDebug << "Stout smearing completed\n"; }; From 834f536b5f426aa0b3a334a89b37d8da39fb4238 Mon Sep 17 00:00:00 2001 From: u61464 Date: Tue, 4 May 2021 08:40:18 -0700 Subject: [PATCH 48/86] Fastest option on SyCL is now std::complex --- Grid/tensors/Tensor_SIMT.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Grid/tensors/Tensor_SIMT.h b/Grid/tensors/Tensor_SIMT.h index 672f385f..0a7d3382 100644 --- a/Grid/tensors/Tensor_SIMT.h +++ b/Grid/tensors/Tensor_SIMT.h @@ -65,7 +65,8 @@ void coalescedWriteNonTemporal(vobj & __restrict__ vec,const vobj & __restrict__ #else -#ifndef GRID_SYCL +//#ifndef GRID_SYCL +#if 1 // Use the scalar as our own complex on GPU ... thrust::complex or std::complex template = 0> accelerator_inline typename vsimd::scalar_type From 8cfc7342cde4b93d1de8f41a6f909ddd6d5d351f Mon Sep 17 00:00:00 2001 From: u61464 Date: Wed, 5 May 2021 14:17:18 -0700 Subject: [PATCH 49/86] staggered hand unroll read coalesce --- Grid/qcd/action/fermion/Fermion.h | 6 - Grid/qcd/action/fermion/FermionOperatorImpl.h | 5 +- .../implementation/StaggeredKernelsAsm.h | 5 +- .../implementation/StaggeredKernelsHand.h | 203 +++++++++--------- 4 files changed, 106 insertions(+), 113 deletions(-) diff --git a/Grid/qcd/action/fermion/Fermion.h b/Grid/qcd/action/fermion/Fermion.h index 16252340..09777204 100644 --- a/Grid/qcd/action/fermion/Fermion.h +++ b/Grid/qcd/action/fermion/Fermion.h @@ -291,12 +291,6 @@ typedef ImprovedStaggeredFermion5D ImprovedStaggeredFermion5DR; typedef ImprovedStaggeredFermion5D ImprovedStaggeredFermion5DF; typedef ImprovedStaggeredFermion5D ImprovedStaggeredFermion5DD; -#ifndef GRID_CUDA -typedef ImprovedStaggeredFermion5D ImprovedStaggeredFermionVec5dR; -typedef ImprovedStaggeredFermion5D ImprovedStaggeredFermionVec5dF; -typedef ImprovedStaggeredFermion5D ImprovedStaggeredFermionVec5dD; -#endif - NAMESPACE_END(Grid); //////////////////// diff --git a/Grid/qcd/action/fermion/FermionOperatorImpl.h b/Grid/qcd/action/fermion/FermionOperatorImpl.h index 9345c0e6..56aaca12 100644 --- a/Grid/qcd/action/fermion/FermionOperatorImpl.h +++ b/Grid/qcd/action/fermion/FermionOperatorImpl.h @@ -183,7 +183,8 @@ NAMESPACE_CHECK(ImplStaggered); ///////////////////////////////////////////////////////////////////////////// // Single flavour one component spinors with colour index. 5d vec ///////////////////////////////////////////////////////////////////////////// -#include -NAMESPACE_CHECK(ImplStaggered5dVec); +// Deprecate Vec5d +//#include +//NAMESPACE_CHECK(ImplStaggered5dVec); diff --git a/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h b/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h index 63fd2a2f..e9cacbcf 100644 --- a/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h +++ b/Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h @@ -680,7 +680,8 @@ void StaggeredKernels::DhopSiteAsm(StencilView &st, gauge2 =(uint64_t)&UU[sU]( Z ); \ gauge3 =(uint64_t)&UU[sU]( T ); - +#undef STAG_VEC5D +#ifdef STAG_VEC5D // This is the single precision 5th direction vectorised kernel #include template <> void StaggeredKernels::DhopSiteAsm(StencilView &st, @@ -790,7 +791,7 @@ template <> void StaggeredKernels::DhopSiteAsm(StencilView #endif } - +#endif #define PERMUTE_DIR3 __asm__ ( \ diff --git a/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h b/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h index 6bcb22b4..2b6087bc 100644 --- a/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h +++ b/Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h @@ -32,25 +32,50 @@ Author: paboyle NAMESPACE_BEGIN(Grid); -#define LOAD_CHI(b) \ +#ifdef GRID_SIMT + +#define LOAD_CHI(ptype,b) \ + const SiteSpinor & ref (b[offset]); \ + Chi_0=coalescedReadPermute(ref()()(0),perm,lane); \ + Chi_1=coalescedReadPermute(ref()()(1),perm,lane); \ + Chi_2=coalescedReadPermute(ref()()(2),perm,lane); + +#define LOAD_CHI_COMMS(b) \ const SiteSpinor & ref (b[offset]); \ - Chi_0=ref()()(0);\ - Chi_1=ref()()(1);\ - Chi_2=ref()()(2); + Chi_0=coalescedRead(ref()()(0),lane); \ + Chi_1=coalescedRead(ref()()(1),lane); \ + Chi_2=coalescedRead(ref()()(2),lane); + +#define PERMUTE_DIR(dir) ; +#else +#define LOAD_CHI(ptype,b) LOAD_CHI_COMMS(b) + +#define LOAD_CHI_COMMS(b) \ + const SiteSpinor & ref (b[offset]); \ + Chi_0=ref()()(0); \ + Chi_1=ref()()(1); \ + Chi_2=ref()()(2); + +#define PERMUTE_DIR(dir) \ + permute##dir(Chi_0,Chi_0); \ + permute##dir(Chi_1,Chi_1); \ + permute##dir(Chi_2,Chi_2); + +#endif // To splat or not to splat depends on the implementation #define MULT(A,UChi) \ auto & ref(U[sU](A)); \ - Impl::loadLinkElement(U_00,ref()(0,0)); \ - Impl::loadLinkElement(U_10,ref()(1,0)); \ - Impl::loadLinkElement(U_20,ref()(2,0)); \ - Impl::loadLinkElement(U_01,ref()(0,1)); \ - Impl::loadLinkElement(U_11,ref()(1,1)); \ - Impl::loadLinkElement(U_21,ref()(2,1)); \ - Impl::loadLinkElement(U_02,ref()(0,2)); \ - Impl::loadLinkElement(U_12,ref()(1,2)); \ - Impl::loadLinkElement(U_22,ref()(2,2)); \ + U_00=coalescedRead(ref()(0,0),lane); \ + U_10=coalescedRead(ref()(1,0),lane); \ + U_20=coalescedRead(ref()(2,0),lane); \ + U_01=coalescedRead(ref()(0,1),lane); \ + U_11=coalescedRead(ref()(1,1),lane); \ + U_21=coalescedRead(ref()(2,1),lane); \ + U_02=coalescedRead(ref()(0,2),lane); \ + U_12=coalescedRead(ref()(1,2),lane); \ + U_22=coalescedRead(ref()(2,2),lane); \ UChi ## _0 = U_00*Chi_0; \ UChi ## _1 = U_10*Chi_0;\ UChi ## _2 = U_20*Chi_0;\ @@ -63,15 +88,15 @@ NAMESPACE_BEGIN(Grid); #define MULT_ADD(U,A,UChi) \ auto & ref(U[sU](A)); \ - Impl::loadLinkElement(U_00,ref()(0,0)); \ - Impl::loadLinkElement(U_10,ref()(1,0)); \ - Impl::loadLinkElement(U_20,ref()(2,0)); \ - Impl::loadLinkElement(U_01,ref()(0,1)); \ - Impl::loadLinkElement(U_11,ref()(1,1)); \ - Impl::loadLinkElement(U_21,ref()(2,1)); \ - Impl::loadLinkElement(U_02,ref()(0,2)); \ - Impl::loadLinkElement(U_12,ref()(1,2)); \ - Impl::loadLinkElement(U_22,ref()(2,2)); \ + U_00=coalescedRead(ref()(0,0),lane); \ + U_10=coalescedRead(ref()(1,0),lane); \ + U_20=coalescedRead(ref()(2,0),lane); \ + U_01=coalescedRead(ref()(0,1),lane); \ + U_11=coalescedRead(ref()(1,1),lane); \ + U_21=coalescedRead(ref()(2,1),lane); \ + U_02=coalescedRead(ref()(0,2),lane); \ + U_12=coalescedRead(ref()(1,2),lane); \ + U_22=coalescedRead(ref()(2,2),lane); \ UChi ## _0 += U_00*Chi_0; \ UChi ## _1 += U_10*Chi_0;\ UChi ## _2 += U_20*Chi_0;\ @@ -83,24 +108,18 @@ NAMESPACE_BEGIN(Grid); UChi ## _2 += U_22*Chi_2; -#define PERMUTE_DIR(dir) \ - permute##dir(Chi_0,Chi_0); \ - permute##dir(Chi_1,Chi_1); \ - permute##dir(Chi_2,Chi_2); - - #define HAND_STENCIL_LEG_BASE(Dir,Perm,skew) \ SE=st.GetEntry(ptype,Dir+skew,sF); \ offset = SE->_offset; \ local = SE->_is_local; \ perm = SE->_permute; \ if ( local ) { \ - LOAD_CHI(in); \ + LOAD_CHI(Perm,in); \ if ( perm) { \ PERMUTE_DIR(Perm); \ } \ } else { \ - LOAD_CHI(buf); \ + LOAD_CHI_COMMS(buf); \ } #define HAND_STENCIL_LEG_BEGIN(Dir,Perm,skew,even) \ @@ -116,19 +135,18 @@ NAMESPACE_BEGIN(Grid); } - #define HAND_STENCIL_LEG_INT(U,Dir,Perm,skew,even) \ SE=st.GetEntry(ptype,Dir+skew,sF); \ offset = SE->_offset; \ local = SE->_is_local; \ perm = SE->_permute; \ if ( local ) { \ - LOAD_CHI(in); \ + LOAD_CHI(Perm,in); \ if ( perm) { \ PERMUTE_DIR(Perm); \ } \ } else if ( st.same_node[Dir] ) { \ - LOAD_CHI(buf); \ + LOAD_CHI_COMMS(buf); \ } \ if (local || st.same_node[Dir] ) { \ MULT_ADD(U,Dir,even); \ @@ -140,10 +158,32 @@ NAMESPACE_BEGIN(Grid); local = SE->_is_local; \ if ((!local) && (!st.same_node[Dir]) ) { \ nmu++; \ - { LOAD_CHI(buf); } \ + { LOAD_CHI_COMMS(buf); } \ { MULT_ADD(U,Dir,even); } \ } +#define HAND_DECLARATIONS(Simd) \ + Simd even_0; \ + Simd even_1; \ + Simd even_2; \ + Simd odd_0; \ + Simd odd_1; \ + Simd odd_2; \ + \ + Simd Chi_0; \ + Simd Chi_1; \ + Simd Chi_2; \ + \ + Simd U_00; \ + Simd U_10; \ + Simd U_20; \ + Simd U_01; \ + Simd U_11; \ + Simd U_21; \ + Simd U_02; \ + Simd U_12; \ + Simd U_22; + template template accelerator_inline @@ -155,28 +195,14 @@ void StaggeredKernels::DhopSiteHand(StencilView &st, typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - Simd even_0; // 12 regs on knc - Simd even_1; - Simd even_2; - Simd odd_0; // 12 regs on knc - Simd odd_1; - Simd odd_2; - Simd Chi_0; // two spinor; 6 regs - Simd Chi_1; - Simd Chi_2; - - Simd U_00; // two rows of U matrix - Simd U_10; - Simd U_20; - Simd U_01; - Simd U_11; - Simd U_21; // 2 reg left. - Simd U_02; - Simd U_12; - Simd U_22; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + typedef decltype( coalescedRead( in[0]()()(0) )) Simt; + HAND_DECLARATIONS(Simt); - SiteSpinor result; + typedef decltype( coalescedRead( in[0] )) calcSiteSpinor; + calcSiteSpinor result; int offset,local,perm, ptype; StencilEntry *SE; @@ -215,7 +241,7 @@ void StaggeredKernels::DhopSiteHand(StencilView &st, result()()(1) = even_1 + odd_1; result()()(2) = even_2 + odd_2; } - vstream(out[sF],result); + coalescedWrite(out[sF],result); } } @@ -230,28 +256,13 @@ void StaggeredKernels::DhopSiteHandInt(StencilView &st, typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - Simd even_0; // 12 regs on knc - Simd even_1; - Simd even_2; - Simd odd_0; // 12 regs on knc - Simd odd_1; - Simd odd_2; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + typedef decltype( coalescedRead( in[0]()()(0) )) Simt; + HAND_DECLARATIONS(Simt); - Simd Chi_0; // two spinor; 6 regs - Simd Chi_1; - Simd Chi_2; - - Simd U_00; // two rows of U matrix - Simd U_10; - Simd U_20; - Simd U_01; - Simd U_11; - Simd U_21; // 2 reg left. - Simd U_02; - Simd U_12; - Simd U_22; - - SiteSpinor result; + typedef decltype( coalescedRead( in[0] )) calcSiteSpinor; + calcSiteSpinor result; int offset, ptype, local, perm; StencilEntry *SE; @@ -261,8 +272,8 @@ void StaggeredKernels::DhopSiteHandInt(StencilView &st, // int sF=s+LLs*sU; { - even_0 = Zero(); even_1 = Zero(); even_2 = Zero(); - odd_0 = Zero(); odd_1 = Zero(); odd_2 = Zero(); + zeroit(even_0); zeroit(even_1); zeroit(even_2); + zeroit(odd_0); zeroit(odd_1); zeroit(odd_2); skew = 0; HAND_STENCIL_LEG_INT(U,Xp,3,skew,even); @@ -294,7 +305,7 @@ void StaggeredKernels::DhopSiteHandInt(StencilView &st, result()()(1) = even_1 + odd_1; result()()(2) = even_2 + odd_2; } - vstream(out[sF],result); + coalescedWrite(out[sF],result); } } @@ -309,28 +320,13 @@ void StaggeredKernels::DhopSiteHandExt(StencilView &st, typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; - Simd even_0; // 12 regs on knc - Simd even_1; - Simd even_2; - Simd odd_0; // 12 regs on knc - Simd odd_1; - Simd odd_2; + const int Nsimd = SiteHalfSpinor::Nsimd(); + const int lane=acceleratorSIMTlane(Nsimd); + typedef decltype( coalescedRead( in[0]()()(0) )) Simt; + HAND_DECLARATIONS(Simt); - Simd Chi_0; // two spinor; 6 regs - Simd Chi_1; - Simd Chi_2; - - Simd U_00; // two rows of U matrix - Simd U_10; - Simd U_20; - Simd U_01; - Simd U_11; - Simd U_21; // 2 reg left. - Simd U_02; - Simd U_12; - Simd U_22; - - SiteSpinor result; + typedef decltype( coalescedRead( in[0] )) calcSiteSpinor; + calcSiteSpinor result; int offset, ptype, local; StencilEntry *SE; @@ -340,8 +336,8 @@ void StaggeredKernels::DhopSiteHandExt(StencilView &st, // int sF=s+LLs*sU; { - even_0 = Zero(); even_1 = Zero(); even_2 = Zero(); - odd_0 = Zero(); odd_1 = Zero(); odd_2 = Zero(); + zeroit(even_0); zeroit(even_1); zeroit(even_2); + zeroit(odd_0); zeroit(odd_1); zeroit(odd_2); int nmu=0; skew = 0; HAND_STENCIL_LEG_EXT(U,Xp,3,skew,even); @@ -374,7 +370,7 @@ void StaggeredKernels::DhopSiteHandExt(StencilView &st, result()()(1) = even_1 + odd_1; result()()(2) = even_2 + odd_2; } - out[sF] = out[sF] + result; + coalescedWrite(out[sF] , out(sF)+ result); } } } @@ -397,6 +393,7 @@ void StaggeredKernels::DhopSiteHandExt(StencilView &st, const FermionFieldView &in, FermionFieldView &out, int dag); \ */ #undef LOAD_CHI +#undef HAND_DECLARATIONS NAMESPACE_END(Grid); From 0e27e3847d6252c7a950e59517a8af0bf1e15549 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 3 Jun 2021 04:24:19 +0000 Subject: [PATCH 50/86] Remove synch --- Grid/threads/Accelerator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h index 56b85c72..b76d6d1c 100644 --- a/Grid/threads/Accelerator.h +++ b/Grid/threads/Accelerator.h @@ -457,7 +457,7 @@ accelerator_inline void acceleratorSynchronise(void) __syncwarp(); #endif #ifdef GRID_SYCL - cl::sycl::detail::workGroupBarrier(); + //cl::sycl::detail::workGroupBarrier(); #endif #ifdef GRID_HIP __syncthreads(); From ca10bfa1c7f6615bcc13322ee5e050c6e4b37ad8 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Fri, 4 Jun 2021 11:12:22 +0100 Subject: [PATCH 51/86] removing Travis CI constantly failing due to overtime (no way we can compile Grid on free time anymore) --- .travis.yml | 56 ----------------------------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3a0e1e35..00000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ -language: cpp - -cache: - directories: - - clang - -matrix: - include: - - os: osx - osx_image: xcode8.3 - compiler: clang - -before_install: - - export GRIDDIR=`pwd` - - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]] && [ ! -e clang/bin ]; then wget $CLANG_LINK; tar -xf `basename $CLANG_LINK`; mkdir clang; mv clang+*/* clang/; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export PATH="${GRIDDIR}/clang/bin:${PATH}"; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc openssl; fi - -install: - - export CWD=`pwd` - - echo $CWD - - export CC=$CC$VERSION - - export CXX=$CXX$VERSION - - echo $PATH - - which autoconf - - autoconf --version - - which automake - - automake --version - - which $CC - - $CC --version - - which $CXX - - $CXX --version - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export LDFLAGS='-L/usr/local/lib'; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export EXTRACONF='--with-openssl=/usr/local/opt/openssl'; fi - -script: - - ./bootstrap.sh - - mkdir build - - cd build - - mkdir lime - - cd lime - - mkdir build - - cd build - - wget http://usqcd-software.github.io/downloads/c-lime/lime-1.3.2.tar.gz - - tar xf lime-1.3.2.tar.gz - - cd lime-1.3.2 - - ./configure --prefix=$CWD/build/lime/install - - make -j4 - - make install - - cd $CWD/build - - ../configure --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install ${EXTRACONF} - - make -j4 - - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals - - make check From 92def28bd3331153da2b8a2414f471e4f7831a4c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sun, 6 Jun 2021 04:52:05 -0400 Subject: [PATCH 52/86] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fff68dc6..88b922a5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Grid [![Teamcity status](http://ci.cliath.ph.ed.ac.uk/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:GridBasedSoftware_Grid)),branch:name:develop)/statusIcon.svg)](http://ci.cliath.ph.ed.ac.uk/project.html?projectId=GridBasedSoftware_Grid&tab=projectOverview) [![Travis status](https://travis-ci.org/paboyle/Grid.svg?branch=develop)](https://travis-ci.org/paboyle/Grid) +# Grid [![Teamcity status](http://ci.cliath.ph.ed.ac.uk/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:GridBasedSoftware_Grid)),branch:name:develop)/statusIcon.svg)](http://ci.cliath.ph.ed.ac.uk/project.html?projectId=GridBasedSoftware_Grid&tab=projectOverview) **Data parallel C++ mathematical object library.** From 4c5440fb0678b3a936ebe95f2c891d90b62feaaf Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 15 Jun 2021 21:45:07 +0000 Subject: [PATCH 53/86] const happy for sycl --- Grid/tensors/Tensor_extract_merge.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Grid/tensors/Tensor_extract_merge.h b/Grid/tensors/Tensor_extract_merge.h index f1ded209..ea619d0f 100644 --- a/Grid/tensors/Tensor_extract_merge.h +++ b/Grid/tensors/Tensor_extract_merge.h @@ -1,5 +1,5 @@ /************************************************************************************* - +n Grid physics library, www.github.com/paboyle/Grid Source file: ./lib/tensors/Tensor_extract_merge.h @@ -153,7 +153,7 @@ void insertLane(int lane, vobj & __restrict__ vec,const typename vobj::scalar_ob // Extract to a bunch of scalar object pointers of different scalar type, with offset. Useful for precision change //////////////////////////////////////////////////////////////////////// template accelerator -void extract(const vobj &vec,ExtractPointerArray &extracted, int offset) +void extract(const vobj &vec,const ExtractPointerArray &extracted, int offset) { typedef typename GridTypeMapper::scalar_type sobj_scalar_type; typedef typename GridTypeMapper::scalar_type scalar_type; @@ -181,7 +181,7 @@ void extract(const vobj &vec,ExtractPointerArray &extracted, int offset) // Merge bunch of scalar object pointers of different scalar type, with offset. Useful for precision change //////////////////////////////////////////////////////////////////////// template accelerator -void merge(vobj &vec,ExtractPointerArray &extracted, int offset) +void merge(vobj &vec,const ExtractPointerArray &extracted, int offset) { typedef typename GridTypeMapper::scalar_type sobj_scalar_type; typedef typename GridTypeMapper::scalar_type scalar_type; From 6cd9224dd78aca959d3997479287cf943832d79c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 16 Jun 2021 17:10:55 +0000 Subject: [PATCH 54/86] SYCL comms buffer allocate --- Grid/communicator/SharedMemoryMPI.cc | 54 +++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index 466f6a1e..786122fa 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -35,6 +35,9 @@ Author: Christoph Lehner #endif #ifdef GRID_HIP #include +#endif +#ifdef GRID_SYCl + #endif NAMESPACE_BEGIN(Grid); @@ -446,7 +449,46 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) //////////////////////////////////////////////////////////////////////////////////////////// // Hugetlbfs mapping intended //////////////////////////////////////////////////////////////////////////////////////////// -#if defined(GRID_CUDA) ||defined(GRID_HIP) +#if defined(GRID_CUDA) ||defined(GRID_HIP) || defined(GRID_SYCL) + +#if defined(GRID_SYCL) +void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) +{ + void * ShmCommBuf ; + assert(_ShmSetup==1); + assert(_ShmAlloc==0); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // allocate the pointer array for shared windows for our group + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + MPI_Barrier(WorldShmComm); + WorldShmCommBufs.resize(WorldShmSize); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Each MPI rank should allocate our own buffer + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + ShmCommBuf = acceleratorAllocDevice(bytes); + + if (ShmCommBuf == (void *)NULL ) { + std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl; + exit(EXIT_FAILURE); + } + // if ( WorldRank == 0 ){ + if ( 1 ){ + std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes + << "bytes at "<< std::hex<< ShmCommBuf < Date: Tue, 22 Jun 2021 17:56:10 +0000 Subject: [PATCH 55/86] Force reqd subgroup size fo SYCL --- Grid/threads/Accelerator.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h index b76d6d1c..c0af1019 100644 --- a/Grid/threads/Accelerator.h +++ b/Grid/threads/Accelerator.h @@ -257,11 +257,14 @@ accelerator_inline int acceleratorSIMTlane(int Nsimd) { unsigned long nt=acceleratorThreads(); \ unsigned long unum1 = num1; \ unsigned long unum2 = num2; \ + if(nt < 8)nt=8; \ cl::sycl::range<3> local {nt,1,nsimd}; \ cl::sycl::range<3> global{unum1,unum2,nsimd}; \ cgh.parallel_for( \ cl::sycl::nd_range<3>(global,local), \ - [=] (cl::sycl::nd_item<3> item) /*mutable*/ { \ + [=] (cl::sycl::nd_item<3> item) /*mutable*/ \ + [[intel::reqd_sub_group_size(8)]] \ + { \ auto iter1 = item.get_global_id(0); \ auto iter2 = item.get_global_id(1); \ auto lane = item.get_global_id(2); \ From 29a22ae603a3cf18d2ebeba2eb5aabcf27fe3e5d Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 22 Jun 2021 17:57:20 +0000 Subject: [PATCH 56/86] Simpler SYCL setup --- Grid/communicator/SharedMemoryMPI.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index 786122fa..caa03a60 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -473,13 +473,13 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) std::cerr << " SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl; exit(EXIT_FAILURE); } - // if ( WorldRank == 0 ){ - if ( 1 ){ - std::cout << WorldRank << header " SharedMemoryMPI.cc acceleratorAllocDevice "<< bytes - << "bytes at "<< std::hex<< ShmCommBuf < Date: Thu, 5 Aug 2021 18:33:20 -0400 Subject: [PATCH 57/86] Check is wrong (HtoD / DtoH) --- Grid/communicator/SharedMemoryMPI.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index caa03a60..d7b41cee 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -844,7 +844,7 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm) } #endif - SharedMemoryTest(); + //SharedMemoryTest(); } ////////////////////////////////////////////////////////////////// // On node barrier From fe5aaf7677c11e917af03bd0cbbc14a29b67bd2d Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 9 Aug 2021 04:06:30 -0700 Subject: [PATCH 58/86] Make comms benchmark same as Benchmark_comms_host_device --- benchmarks/Benchmark_ITT.cc | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc index 032535b3..7311dfc4 100644 --- a/benchmarks/Benchmark_ITT.cc +++ b/benchmarks/Benchmark_ITT.cc @@ -133,15 +133,15 @@ public: std::vector xbuf(8); std::vector rbuf(8); - Grid.ShmBufferFreeAll(); + //Grid.ShmBufferFreeAll(); + uint64_t bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); for(int d=0;d<8;d++){ - xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); - rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + xbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes); + rbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes); // bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); // bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); } - int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); int ncomm; double dbytes; std::vector times(Nloop); @@ -152,7 +152,7 @@ public: dbytes=0; ncomm=0; - thread_for(dir,8,{ + for(int dir=0;dir<8;dir++) { double tbytes; int mu =dir % 4; @@ -168,15 +168,16 @@ public: int comm_proc = mpi_layout[mu]-1; Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); } - tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, - (void *)&rbuf[dir][0], recv_from_rank, - bytes,dir); + Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, + (void *)&rbuf[dir][0], recv_from_rank, + bytes); + tbytes = bytes; thread_critical { ncomm++; dbytes+=tbytes; } } - }); + }; Grid.Barrier(); double stop=usecond(); t_time[i] = stop-start; // microseconds @@ -196,8 +197,12 @@ public: << "\t\t"<< bidibytes/timestat.mean<< " " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " " << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl; - } - } + for(int d=0;d<8;d++){ + acceleratorFreeDevice(xbuf[d]); + acceleratorFreeDevice(rbuf[d]); + } + } + } return; } @@ -281,7 +286,6 @@ public: uint64_t lmax=32; -#define NLOOP (1000*lmax*lmax*lmax*lmax/lat/lat/lat/lat) GridSerialRNG sRNG; sRNG.SeedFixedIntegers(std::vector({45,12,81,9})); for(int lat=8;lat<=lmax;lat+=8){ From 75030637cca1f60173ca760b09dd37852a88299d Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 10 Aug 2021 05:16:30 -0700 Subject: [PATCH 59/86] Improved comms benchmark, same as benchmark_comms_host_device --- benchmarks/Benchmark_ITT.cc | 73 ++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc index 7311dfc4..81d1acd4 100644 --- a/benchmarks/Benchmark_ITT.cc +++ b/benchmarks/Benchmark_ITT.cc @@ -144,23 +144,19 @@ public: int ncomm; double dbytes; - std::vector times(Nloop); - for(int i=0;i1 ) { - dbytes=0; - ncomm=0; + std::vector times(Nloop); + for(int i=0;i1 ) { - + dbytes=0; + double start=usecond(); int xmit_to_rank; int recv_from_rank; + if ( dir == mu ) { int comm_proc=1; Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); @@ -171,42 +167,35 @@ public: Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, (void *)&rbuf[dir][0], recv_from_rank, bytes); - tbytes = bytes; - thread_critical { - ncomm++; - dbytes+=tbytes; - } + dbytes+=bytes; + + double stop=usecond(); + t_time[i] = stop-start; // microseconds + } - }; - Grid.Barrier(); - double stop=usecond(); - t_time[i] = stop-start; // microseconds + timestat.statistics(t_time); + + dbytes=dbytes*ppn; + double xbytes = dbytes*0.5; + double bidibytes = dbytes; + + std::cout<1) ) { + if ( do_comms ) { std::cout< Date: Tue, 10 Aug 2021 05:35:15 -0700 Subject: [PATCH 60/86] Level 0 IPC set up --- Grid/communicator/SharedMemoryMPI.cc | 56 +++++++++++++++++++++++++--- Grid/threads/Accelerator.cc | 5 ++- Grid/threads/Accelerator.h | 7 ++++ 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index caa03a60..554f338b 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -73,6 +73,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm) WorldNodes = WorldSize/WorldShmSize; assert( (WorldNodes * WorldShmSize) == WorldSize ); + // FIXME: Check all WorldShmSize are the same ? ///////////////////////////////////////////////////////////////////// @@ -451,7 +452,8 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) //////////////////////////////////////////////////////////////////////////////////////////// #if defined(GRID_CUDA) ||defined(GRID_HIP) || defined(GRID_SYCL) -#if defined(GRID_SYCL) +//if defined(GRID_SYCL) +#if 0 void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) { void * ShmCommBuf ; @@ -488,7 +490,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) } #endif -#if defined(GRID_CUDA) ||defined(GRID_HIP) +#if defined(GRID_CUDA) ||defined(GRID_HIP) ||defined(GRID_SYCL) void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) { void * ShmCommBuf ; @@ -511,8 +513,16 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) /////////////////////////////////////////////////////////////////////////////////////////////////////////// // Each MPI rank should allocate our own buffer /////////////////////////////////////////////////////////////////////////////////////////////////////////// + auto zeDevice = cl::sycl::get_native(theGridAccelerator->get_device()); + auto zeContext= cl::sycl::get_native(theGridAccelerator->get_context()); +#ifdef GRID_SYCL_LEVEL_ZERO_IPC + ze_device_mem_alloc_desc_t zeDesc = {}; + zeMemAllocDevice(zeContext,&zeDesc,bytes,2*1024*1024,zeDevice,&ShmCommBuf); + std::cout << WorldRank << header " SharedMemoryMPI.cc zeMemAllocDevice "<< bytes + << "bytes at "<< std::hex<< ShmCommBuf < #include + +#define GRID_SYCL_LEVEL_ZERO_IPC + +#ifdef GRID_SYCL_LEVEL_ZERO_IPC +#include +#include +#endif NAMESPACE_BEGIN(Grid); extern cl::sycl::queue *theGridAccelerator; From 5d29e175d82edc92af07ca9b708ec7f187fc006d Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 10 Aug 2021 18:25:43 +0100 Subject: [PATCH 61/86] Typo fix --- Grid/communicator/SharedMemoryMPI.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index d230b2a5..11788744 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -513,9 +513,9 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) /////////////////////////////////////////////////////////////////////////////////////////////////////////// // Each MPI rank should allocate our own buffer /////////////////////////////////////////////////////////////////////////////////////////////////////////// +#ifdef GRID_SYCL_LEVEL_ZERO_IPC auto zeDevice = cl::sycl::get_native(theGridAccelerator->get_device()); auto zeContext= cl::sycl::get_native(theGridAccelerator->get_context()); -#ifdef GRID_SYCL_LEVEL_ZERO_IPC ze_device_mem_alloc_desc_t zeDesc = {}; zeMemAllocDevice(zeContext,&zeDesc,bytes,2*1024*1024,zeDevice,&ShmCommBuf); std::cout << WorldRank << header " SharedMemoryMPI.cc zeMemAllocDevice "<< bytes From ffbdd91e0e2770f0e12f9296e5e9991e52799f4d Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Fri, 20 Aug 2021 01:15:00 +0100 Subject: [PATCH 62/86] Apple happiness --- Grid/util/Init.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc index bfbc464d..3b661524 100644 --- a/Grid/util/Init.cc +++ b/Grid/util/Init.cc @@ -52,10 +52,12 @@ Author: paboyle #include -#ifdef __APPLE__ static int +#ifdef __APPLE__ feenableexcept (unsigned int excepts) { +#if 0 + // Fails on Apple M1 static fenv_t fenv; unsigned int new_excepts = excepts & FE_ALL_EXCEPT; unsigned int old_excepts; // previous masks @@ -70,6 +72,7 @@ feenableexcept (unsigned int excepts) iold_excepts = (int) old_excepts; return ( fesetenv (&fenv) ? -1 : iold_excepts ); +#endif } #endif From 7163b31a26bba90241480fe444a3584cd66f5893 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Fri, 20 Aug 2021 01:15:23 +0100 Subject: [PATCH 63/86] Examples --- scripts/filelist | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/filelist b/scripts/filelist index 8e29ba88..c3f0cf3f 100755 --- a/scripts/filelist +++ b/scripts/filelist @@ -82,3 +82,18 @@ for f in $TESTS; do echo >> Make.inc done cd .. + +# examples Make.inc +cd $home/examples/ +echo> Make.inc +TESTS=`ls *.cc` +TESTLIST=`echo ${TESTS} | sed s/.cc//g ` +echo bin_PROGRAMS = ${TESTLIST} > Make.inc +echo >> Make.inc +for f in $TESTS; do + BNAME=`basename $f .cc` + echo ${BNAME}_SOURCES=$f >> Make.inc + echo ${BNAME}_LDADD='$(top_builddir)/Grid/libGrid.a'>> Make.inc + echo >> Make.inc +done +cd .. From 40098424c75cb693cf92ebafa758efac8b25053a Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sun, 22 Aug 2021 14:17:12 +0100 Subject: [PATCH 64/86] Examples --- Makefile.am | 2 +- configure.ac | 1 + examples/Example_Laplacian.cc | 395 ++++++++++++++++++++++++++++++++++ examples/Makefile.am | 6 + 4 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 examples/Example_Laplacian.cc create mode 100644 examples/Makefile.am diff --git a/Makefile.am b/Makefile.am index 33b25026..d2a1a326 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,5 @@ # additional include paths necessary to compile the C++ library -SUBDIRS = Grid HMC benchmarks tests +SUBDIRS = Grid HMC benchmarks tests examples include $(top_srcdir)/doxygen.inc diff --git a/configure.ac b/configure.ac index 4e5e33c8..721d890e 100644 --- a/configure.ac +++ b/configure.ac @@ -815,6 +815,7 @@ AC_CONFIG_FILES(tests/smearing/Makefile) AC_CONFIG_FILES(tests/qdpxx/Makefile) AC_CONFIG_FILES(tests/testu01/Makefile) AC_CONFIG_FILES(benchmarks/Makefile) +AC_CONFIG_FILES(examples/Makefile) AC_OUTPUT echo "" diff --git a/examples/Example_Laplacian.cc b/examples/Example_Laplacian.cc new file mode 100644 index 00000000..f6bbf7cf --- /dev/null +++ b/examples/Example_Laplacian.cc @@ -0,0 +1,395 @@ +#include +using namespace Grid; + +/* +///////////////////////////////////////////////////////////////////////////////////////////// +// Grid/algorithms/SparseMatrix.h: Interface defining what I expect of a general sparse matrix, such as a Fermion action +///////////////////////////////////////////////////////////////////////////////////////////// +template class SparseMatrixBase { +public: + virtual GridBase *Grid(void) =0; + + virtual void M (const Field &in, Field &out)=0; + virtual void Mdag (const Field &in, Field &out)=0; + virtual void MdagM(const Field &in, Field &out) { + Field tmp (in.Grid()); + M(in,tmp); + Mdag(tmp,out); + } + virtual void Mdiag (const Field &in, Field &out)=0; + virtual void Mdir (const Field &in, Field &out,int dir, int disp)=0; + virtual void MdirAll (const Field &in, std::vector &out)=0; +}; +*/ + +const std::vector directions ({Xdir,Ydir,Zdir,Xdir,Ydir,Zdir}); +const std::vector displacements({1,1,1,-1,-1,-1}); + +template class FreeLaplacianCshift : public SparseMatrixBase +{ +public: + GridBase *grid; + FreeLaplacianCshift(GridBase *_grid) + { + grid=_grid; + }; + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &in, Field &out) + { + out = Zero(); + for(int mu=0;mu &out) {assert(0);}; // Unimplemented need only for multigrid +}; + +template class CovariantLaplacianCshift : public SparseMatrixBase +{ +public: + INHERIT_GIMPL_TYPES(Gimpl); + + GridBase *grid; + GaugeField U; + + CovariantLaplacianCshift(GaugeField &_U) : + grid(_U.Grid()), + U(_U) { }; + + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &in, Field &out) + { + out=Zero(); + for(int mu=0;mu(U, mu); // NB: Inefficent + out = out + Gimpl::CovShiftForward(Umu,mu,in); + out = out + Gimpl::CovShiftBackward(Umu,mu,in); + out = out - 2.0*in; + } + }; + virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian + virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid + virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid + virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid +}; + + +#define LEG_LOAD(Dir) \ + SE = st.GetEntry(ptype, Dir, ss); \ + if (SE->_is_local ) { \ + int perm= SE->_permute; \ + chi = coalescedReadPermute(in[SE->_offset],ptype,perm,lane); \ + } else { \ + chi = coalescedRead(buf[SE->_offset],lane); \ + } \ + acceleratorSynchronise(); + +template class FreeLaplacianStencil : public SparseMatrixBase +{ +public: + typedef typename Field::vector_object siteObject; + typedef CartesianStencil StencilImpl; + + GridBase *grid; + StencilImpl Stencil; + SimpleCompressor Compressor; + + FreeLaplacianStencil(GridBase *_grid) + : Stencil (_grid,6,Even,directions,displacements,0), grid(_grid) + { }; + + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &_in, Field &_out) + { + + /////////////////////////////////////////////// + // Halo exchange for this geometry of stencil + /////////////////////////////////////////////// + Stencil.HaloExchange(_in, Compressor); + + /////////////////////////////////// + // Arithmetic expressions + /////////////////////////////////// + StencilEntry *SE; + + // Views; device friendly/accessible pointers + auto st = Stencil.View(AcceleratorRead); + auto buf = st.CommBuf(); + autoView( in , _in , AcceleratorRead); + autoView( out , _out , AcceleratorWrite); + + typedef typename Field::vector_object vobj; + typedef decltype(coalescedRead(in[0])) calcObj; + + const int Nsimd = vobj::Nsimd(); + const uint64_t NN = grid->oSites(); + const int lane=acceleratorSIMTlane(Nsimd); + + accelerator_for( ss, NN, Nsimd, { + + const int lane=acceleratorSIMTlane(Nsimd); + + calcObj chi; + calcObj res; + int ptype; + + res = coalescedRead(in[ss])*(-6.0); + LEG_LOAD(0); res = res + chi; + LEG_LOAD(1); res = res + chi; + LEG_LOAD(2); res = res + chi; + LEG_LOAD(3); res = res + chi; + LEG_LOAD(4); res = res + chi; + LEG_LOAD(5); res = res + chi; + + coalescedWrite(out[ss], res,lane); + + }); + }; + virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian + virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid + virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid + virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid +}; + +template class CovariantLaplacianStencil : public SparseMatrixBase +{ +public: + INHERIT_GIMPL_TYPES(Gimpl); + + typedef typename Field::vector_object siteObject; + + template using iImplDoubledGaugeField = iVector >, Nds>; + typedef iImplDoubledGaugeField SiteDoubledGaugeField; + typedef Lattice DoubledGaugeField; + + typedef CartesianStencil StencilImpl; + + GridBase *grid; + StencilImpl Stencil; + SimpleCompressor Compressor; + DoubledGaugeField Uds; + CovariantLaplacianStencil(GaugeField &Umu) + : + grid(Umu.Grid()), + Stencil (grid,6,Even,directions,displacements,0), + Uds(grid) + { + for (int mu = 0; mu < Nd; mu++) { + auto U = PeekIndex(Umu, mu); + PokeIndex(Uds, U, mu ); + U = adj(Cshift(U, mu, -1)); + PokeIndex(Uds, U, mu + 4); + } + }; + + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &_in, Field &_out) + { + /////////////////////////////////////////////// + // Halo exchange for this geometry of stencil + /////////////////////////////////////////////// + Stencil.HaloExchange(_in, Compressor); + + /////////////////////////////////// + // Arithmetic expressions + /////////////////////////////////// + auto st = Stencil.View(AcceleratorRead); + auto buf = st.CommBuf(); + StencilEntry *SE; + + autoView( in , _in , AcceleratorRead); + autoView( out , _out , AcceleratorWrite); + autoView( U , Uds , AcceleratorRead); + + typedef typename Field::vector_object vobj; + typedef decltype(coalescedRead(in[0])) calcObj; + typedef decltype(coalescedRead(U[0](0))) calcLink; + + const int Nsimd = vobj::Nsimd(); + const uint64_t NN = grid->oSites(); + const int lane=acceleratorSIMTlane(Nsimd); + accelerator_for( ss, NN, Nsimd, { + + const int lane=acceleratorSIMTlane(Nsimd); + + calcObj chi; + calcObj res; + calcObj Uchi; + calcLink UU; + int ptype; + + res = coalescedRead(in[ss])*(-6.0); + +#define LEG_LOAD_MULT(leg,polarisation) \ + UU = coalescedRead(U[ss](polarisation)); \ + LEG_LOAD(leg); \ + mult(&Uchi(), &UU, &chi()); \ + res = res + Uchi; + + LEG_LOAD_MULT(0,Xp); + LEG_LOAD_MULT(1,Yp); + LEG_LOAD_MULT(2,Zp); + LEG_LOAD_MULT(3,Xm); + LEG_LOAD_MULT(4,Ym); + LEG_LOAD_MULT(5,Zm); + + coalescedWrite(out[ss], res,lane); + }); + }; + virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian + virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid + virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid + virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid +}; + +#undef LEG_LOAD_MULT +#undef LEG_LOAD + +int main(int argc, char ** argv) +{ + Grid_init(&argc, &argv); + + typedef LatticeColourVector Field; + + auto latt_size = GridDefaultLatt(); + auto simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + auto mpi_layout = GridDefaultMpi(); + + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridParallelRNG RNG(&Grid); RNG.SeedFixedIntegers(std::vector({45,12,81,9})); + + FreeLaplacianCshift FLcs(&Grid); + FreeLaplacianStencil FLst(&Grid); + + LatticeGaugeField U(&Grid); + + SU::ColdConfiguration(RNG,U); + + std::cout << " Gauge field has norm " < CLcs(U); + CovariantLaplacianStencil CLst(U); + + Field in(&Grid); gaussian(RNG,in); + Field out_FLcs(&Grid); + Field out_FLst(&Grid); + Field out_CLcs(&Grid); + Field out_CLst(&Grid); + Field diff(&Grid); + + //////////////////////////////////////////////////////// + // First test: in free field these should all agree + //////////////////////////////////////////////////////// + FLcs.M(in,out_FLcs); + FLst.M(in,out_FLst); + CLcs.M(in,out_CLcs); + CLst.M(in,out_CLst); + + std:: cout << "******************************************************************" <::RandomGaugeTransform(RNG,U_GT,g); // Unit gauge + + Field in_GT(&Grid); + Field out_GT(&Grid); + + Field out_CLcs_GT(&Grid); + Field out_CLst_GT(&Grid); + + CovariantLaplacianCshift CLcs_GT(U_GT); + CovariantLaplacianStencil CLst_GT(U_GT); + + in_GT = g*in; + out_GT = g*out_FLcs; + + // Check M^GT_xy in_GT = g(x) M_xy g^dag(y) g(y) in = g(x) out(x) + CLcs_GT.M(in_GT,out_CLcs_GT); + CLst_GT.M(in_GT,out_CLst_GT); + + diff = out_CLcs_GT - out_GT; + std:: cout << " Difference between Gauge xformed result and covariant Cshift Laplacian in xformed gauge = " < dim_mask({1,1,1,0}); // 3d FFT + FFT theFFT(&Grid); + Field out(&Grid); + Field F_out(&Grid); + Field F_in(&Grid); + + // FFT the random input vector + theFFT.FFT_dim_mask(F_in,in,dim_mask,FFT::forward); + + // Convolution theorem: multiply by Fourier representation of (discrete) Laplacian to apply diff op + LatticeComplexD lap(&Grid); lap = Zero(); + LatticeComplexD kmu(&Grid); + ComplexD ci(0.0,1.0); + for(int mu=0;mu<3;mu++) { + + RealD TwoPiL = M_PI * 2.0/ latt_size[mu]; + + LatticeCoordinate(kmu,mu); + kmu = TwoPiL * kmu; + + // (e^ik_mu + e^-ik_mu - 2) = 2( cos kmu - 1) ~ 2 (1 - k_mu^2/2 -1 ) = - k_mu^2 + O(k^4) + lap = lap + 2.0*cos(kmu) - 2.0; + + } + F_out = lap * F_in; + + // Inverse FFT the result + theFFT.FFT_dim_mask(out,F_out,dim_mask,FFT::backward); + + std::cout<<"Fourier xformed (in) "< Date: Sun, 22 Aug 2021 18:25:07 +0100 Subject: [PATCH 65/86] Extra examples / solutions --- examples/Example_Laplacian_inverter.cc | 122 ++++++++++++++++++++++++ examples/Example_Laplacian_smearing.cc | 127 +++++++++++++++++++++++++ examples/Example_Laplacian_solver.cc | 127 +++++++++++++++++++++++++ 3 files changed, 376 insertions(+) create mode 100644 examples/Example_Laplacian_inverter.cc create mode 100644 examples/Example_Laplacian_smearing.cc create mode 100644 examples/Example_Laplacian_solver.cc diff --git a/examples/Example_Laplacian_inverter.cc b/examples/Example_Laplacian_inverter.cc new file mode 100644 index 00000000..1235d2b8 --- /dev/null +++ b/examples/Example_Laplacian_inverter.cc @@ -0,0 +1,122 @@ +#include +using namespace Grid; + +// Function used for Chebyshev smearing +// +Real MomentumSmearing(Real p2) +{ + return (1 - 4.0*p2) * exp(-p2/4); +} +Real DistillationSmearing(Real p2) +{ + if ( p2 > 0.5 ) return 0.0; + else return 1.0; +} + +// Flip sign to make prop to p^2, not -p^2 relative to last example +template class CovariantLaplacianCshift : public SparseMatrixBase +{ +public: + INHERIT_GIMPL_TYPES(Gimpl); + + GridBase *grid; + GaugeField U; + + CovariantLaplacianCshift(GaugeField &_U) : + grid(_U.Grid()), + U(_U) { }; + + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &in, Field &out) + { + out=Zero(); + for(int mu=0;mu(U, mu); // NB: Inefficent + out = out - Gimpl::CovShiftForward(Umu,mu,in); + out = out - Gimpl::CovShiftBackward(Umu,mu,in); + out = out + 2.0*in; + } + }; + virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian + virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid + virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid + virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid +}; + + + +int main(int argc, char ** argv) +{ + Grid_init(&argc, &argv); + + typedef LatticeColourVector Field; + + auto latt_size = GridDefaultLatt(); + auto simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + auto mpi_layout = GridDefaultMpi(); + + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridParallelRNG RNG(&Grid); RNG.SeedFixedIntegers(std::vector({45,12,81,9})); + + + LatticeGaugeField U(&Grid); + + SU::ColdConfiguration(RNG,U); + + typedef CovariantLaplacianCshift Laplacian_t; + Laplacian_t Laplacian(U); + + ColourVector ColourKronecker; + ColourKronecker = Zero(); + ColourKronecker()()(0) = 1.0; + + Coordinate site({latt_size[0]/2, + latt_size[1]/2, + latt_size[2]/2, + 0}); + + Field kronecker(&Grid); + kronecker = Zero(); + pokeSite(ColourKronecker,kronecker,site); + + + Field psi(&Grid), chi(&Grid); + + ////////////////////////////////////// + // Classic Wuppertal smearing + ////////////////////////////////////// + + Integer Iterations = 80; + Real width = 2.0; + Real coeff = (width*width) / Real(4*Iterations); + + chi=kronecker; + // chi = (1-p^2/2N)^N kronecker + for(int n = 0; n < Iterations; ++n) { + Laplacian.M(chi,psi); + chi = chi - coeff*psi; + } + + std::cout << " Wuppertal smeared operator is chi = \n" << chi < HermOp(Laplacian); + + Chebyshev ChebySmear(lo,hi,20,DistillationSmearing); + // Chebyshev ChebySmear(lo,hi,20,MomentumSmearing); + { + std::ofstream of("chebysmear"); + ChebySmear.csv(of); + } + + ChebySmear(HermOp,kronecker,chi); + + std::cout << " Chebyshev smeared operator is chi = \n" << chi < +using namespace Grid; + +// Function used for Chebyshev smearing +// +Real MomentumSmearing(Real p2) +{ + return (1 - 4.0*p2) * exp(-p2/4); +} +Real DistillationSmearing(Real p2) +{ + if ( p2 > 0.5 ) return 0.0; + else return 1.0; +} + +// Flip sign to make prop to p^2, not -p^2 relative to last example +template class CovariantLaplacianCshift : public SparseMatrixBase +{ +public: + INHERIT_GIMPL_TYPES(Gimpl); + + GridBase *grid; + GaugeField U; + + CovariantLaplacianCshift(GaugeField &_U) : + grid(_U.Grid()), + U(_U) { }; + + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &in, Field &out) + { + out=Zero(); + for(int mu=0;mu(U, mu); // NB: Inefficent + out = out - Gimpl::CovShiftForward(Umu,mu,in); + out = out - Gimpl::CovShiftBackward(Umu,mu,in); + out = out + 2.0*in; + } + }; + virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian + virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid + virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid + virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid +}; + + + +int main(int argc, char ** argv) +{ + Grid_init(&argc, &argv); + + typedef LatticeColourVector Field; + + auto latt_size = GridDefaultLatt(); + auto simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + auto mpi_layout = GridDefaultMpi(); + + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridParallelRNG RNG(&Grid); RNG.SeedFixedIntegers(std::vector({45,12,81,9})); + + + LatticeGaugeField U(&Grid); + + SU::ColdConfiguration(RNG,U); + + typedef CovariantLaplacianCshift Laplacian_t; + Laplacian_t Laplacian(U); + + + ColourVector ColourKronecker; + ColourKronecker = Zero(); + ColourKronecker()()(0) = 1.0; + + Coordinate site({latt_size[0]/2, + latt_size[1]/2, + latt_size[2]/2, + 0}); + + Field kronecker(&Grid); + kronecker = Zero(); + pokeSite(ColourKronecker,kronecker,site); + + + Field psi(&Grid), chi(&Grid); + + ////////////////////////////////////// + // Classic Wuppertal smearing + ////////////////////////////////////// + + Integer Iterations = 80; + Real width = 2.0; + Real coeff = (width*width) / Real(4*Iterations); + + chi=kronecker; + // chi = (1-p^2/2N)^N kronecker + for(int n = 0; n < Iterations; ++n) { + Laplacian.M(chi,psi); + chi = chi - coeff*psi; + } + + std::cout << " Wuppertal smeared operator is chi = \n" << chi < HermOp(Laplacian); + + std::cout << " Checking spectral range of our POSITIVE definite operator \n"; + PowerMethod PM; + PM(HermOp,kronecker); + + Chebyshev ChebySmear(lo,hi,20,DistillationSmearing); + // Chebyshev ChebySmear(lo,hi,20,MomentumSmearing); + { + std::ofstream of("chebysmear"); + ChebySmear.csv(of); + } + + ChebySmear(HermOp,kronecker,chi); + + std::cout << " Chebyshev smeared operator is chi = \n" << chi < +using namespace Grid; + +template +void SimpleConjugateGradient(LinearOperatorBase &HPDop,const Field &b, Field &x) +{ + RealD cp, c, alpha, d, beta, ssq, qq; + RealD Tolerance=1.0e-10; + int MaxIterations=10000; + + Field p(b), mmp(b), r(b); + + HPDop.HermOpAndNorm(x, mmp, d, beta); + + r = b - mmp; + p = r; + + alpha = norm2(p); + cp = alpha; + ssq = norm2(b); + + RealD rsq = Tolerance * Tolerance * ssq; + + for (int k = 1; k <= MaxIterations; k++) { + c = cp; + + HPDop.HermOp(p, mmp); + + ComplexD dc = innerProduct(p,mmp); + d = dc.real(); + alpha = c / d; + + cp = axpy_norm(r, -alpha, mmp, r); + beta = cp / c; + + x = x + alpha* p ; + p = r + beta* p ; + + std::cout << "iteration "< class CovariantLaplacianCshift : public SparseMatrixBase +{ +public: + INHERIT_GIMPL_TYPES(Gimpl); + + GridBase *grid; + GaugeField U; + RealD m2=1.0e-2; + CovariantLaplacianCshift(GaugeField &_U) : + grid(_U.Grid()), + U(_U) { }; + + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &in, Field &out) + { + out=Zero(); + for(int mu=0;mu(U, mu); // NB: Inefficent + out = out - Gimpl::CovShiftForward(Umu,mu,in); + out = out - Gimpl::CovShiftBackward(Umu,mu,in); + out = out + 2.0*in + m2*in; + } + }; + virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian + virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid + virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid + virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid +}; + + + +int main(int argc, char ** argv) +{ + Grid_init(&argc, &argv); + + typedef LatticeColourVector Field; + + auto latt_size = GridDefaultLatt(); + auto simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + auto mpi_layout = GridDefaultMpi(); + + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridParallelRNG RNG(&Grid); RNG.SeedFixedIntegers(std::vector({45,12,81,9})); + + + LatticeGaugeField U(&Grid); + + SU::ColdConfiguration(RNG,U); + + typedef CovariantLaplacianCshift Laplacian_t; + Laplacian_t Laplacian(U); + + + ColourVector ColourKronecker; + ColourKronecker = Zero(); + ColourKronecker()()(0) = 1.0; + + Coordinate site({0,0,0,0}); // Point source at origin + + Field kronecker(&Grid); + kronecker = Zero(); + pokeSite(ColourKronecker,kronecker,site); + + Field psi(&Grid); psi=Zero(); + + HermitianLinearOperator HermOp(Laplacian); + SimpleConjugateGradient(HermOp, kronecker,psi); + + Field r(&Grid); + Laplacian.M(psi,r); + r=kronecker-r; + + std::cout << "True residual "<< norm2(r) < Date: Sun, 22 Aug 2021 18:28:39 +0100 Subject: [PATCH 66/86] Remove the file --- examples/Example_Laplacian_inverter.cc | 122 ------------------------- 1 file changed, 122 deletions(-) delete mode 100644 examples/Example_Laplacian_inverter.cc diff --git a/examples/Example_Laplacian_inverter.cc b/examples/Example_Laplacian_inverter.cc deleted file mode 100644 index 1235d2b8..00000000 --- a/examples/Example_Laplacian_inverter.cc +++ /dev/null @@ -1,122 +0,0 @@ -#include -using namespace Grid; - -// Function used for Chebyshev smearing -// -Real MomentumSmearing(Real p2) -{ - return (1 - 4.0*p2) * exp(-p2/4); -} -Real DistillationSmearing(Real p2) -{ - if ( p2 > 0.5 ) return 0.0; - else return 1.0; -} - -// Flip sign to make prop to p^2, not -p^2 relative to last example -template class CovariantLaplacianCshift : public SparseMatrixBase -{ -public: - INHERIT_GIMPL_TYPES(Gimpl); - - GridBase *grid; - GaugeField U; - - CovariantLaplacianCshift(GaugeField &_U) : - grid(_U.Grid()), - U(_U) { }; - - virtual GridBase *Grid(void) { return grid; }; - - virtual void M (const Field &in, Field &out) - { - out=Zero(); - for(int mu=0;mu(U, mu); // NB: Inefficent - out = out - Gimpl::CovShiftForward(Umu,mu,in); - out = out - Gimpl::CovShiftBackward(Umu,mu,in); - out = out + 2.0*in; - } - }; - virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian - virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid - virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid - virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid -}; - - - -int main(int argc, char ** argv) -{ - Grid_init(&argc, &argv); - - typedef LatticeColourVector Field; - - auto latt_size = GridDefaultLatt(); - auto simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); - auto mpi_layout = GridDefaultMpi(); - - GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridParallelRNG RNG(&Grid); RNG.SeedFixedIntegers(std::vector({45,12,81,9})); - - - LatticeGaugeField U(&Grid); - - SU::ColdConfiguration(RNG,U); - - typedef CovariantLaplacianCshift Laplacian_t; - Laplacian_t Laplacian(U); - - ColourVector ColourKronecker; - ColourKronecker = Zero(); - ColourKronecker()()(0) = 1.0; - - Coordinate site({latt_size[0]/2, - latt_size[1]/2, - latt_size[2]/2, - 0}); - - Field kronecker(&Grid); - kronecker = Zero(); - pokeSite(ColourKronecker,kronecker,site); - - - Field psi(&Grid), chi(&Grid); - - ////////////////////////////////////// - // Classic Wuppertal smearing - ////////////////////////////////////// - - Integer Iterations = 80; - Real width = 2.0; - Real coeff = (width*width) / Real(4*Iterations); - - chi=kronecker; - // chi = (1-p^2/2N)^N kronecker - for(int n = 0; n < Iterations; ++n) { - Laplacian.M(chi,psi); - chi = chi - coeff*psi; - } - - std::cout << " Wuppertal smeared operator is chi = \n" << chi < HermOp(Laplacian); - - Chebyshev ChebySmear(lo,hi,20,DistillationSmearing); - // Chebyshev ChebySmear(lo,hi,20,MomentumSmearing); - { - std::ofstream of("chebysmear"); - ChebySmear.csv(of); - } - - ChebySmear(HermOp,kronecker,chi); - - std::cout << " Chebyshev smeared operator is chi = \n" << chi < Date: Sun, 22 Aug 2021 18:40:55 +0100 Subject: [PATCH 67/86] Fail on non-apple --- Grid/util/Init.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc index 3b661524..c0bfc906 100644 --- a/Grid/util/Init.cc +++ b/Grid/util/Init.cc @@ -52,8 +52,8 @@ Author: paboyle #include -static int #ifdef __APPLE__ +static int feenableexcept (unsigned int excepts) { #if 0 From 5b3c530aa778b23ae9383a5b7a63fd20f5d9c2cb Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 23 Aug 2021 15:30:45 +0100 Subject: [PATCH 68/86] Return value --- Grid/util/Init.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc index c0bfc906..ab2d2399 100644 --- a/Grid/util/Init.cc +++ b/Grid/util/Init.cc @@ -73,6 +73,7 @@ feenableexcept (unsigned int excepts) iold_excepts = (int) old_excepts; return ( fesetenv (&fenv) ? -1 : iold_excepts ); #endif + return 0; } #endif From 0d588b95f4e1847024c4eea0094f7fc0dc36ad28 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 23 Aug 2021 23:14:26 +0100 Subject: [PATCH 69/86] Bug fix to Example_Laplacian test --- Grid/threads/Accelerator.h | 6 ++++++ examples/Example_Laplacian.cc | 9 +++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h index 2c9d15ba..a8c91aa8 100644 --- a/Grid/threads/Accelerator.h +++ b/Grid/threads/Accelerator.h @@ -39,6 +39,10 @@ Author: paboyle #ifdef HAVE_MM_MALLOC_H #include #endif +#ifdef __APPLE__ +// no memalign +inline void *memalign(size_t align, size_t bytes) { return malloc(bytes); } +#endif NAMESPACE_BEGIN(Grid); @@ -419,6 +423,8 @@ inline void acceleratorMemSet(void *base,int value,size_t bytes) { hipMemset(bas #undef GRID_SIMT + + #define accelerator #define accelerator_inline strong_inline #define accelerator_for(iterator,num,nsimd, ... ) thread_for(iterator, num, { __VA_ARGS__ }); diff --git a/examples/Example_Laplacian.cc b/examples/Example_Laplacian.cc index f6bbf7cf..fa8466cf 100644 --- a/examples/Example_Laplacian.cc +++ b/examples/Example_Laplacian.cc @@ -116,7 +116,6 @@ public: /////////////////////////////////// // Arithmetic expressions /////////////////////////////////// - StencilEntry *SE; // Views; device friendly/accessible pointers auto st = Stencil.View(AcceleratorRead); @@ -129,10 +128,11 @@ public: const int Nsimd = vobj::Nsimd(); const uint64_t NN = grid->oSites(); - const int lane=acceleratorSIMTlane(Nsimd); accelerator_for( ss, NN, Nsimd, { + StencilEntry *SE; + const int lane=acceleratorSIMTlane(Nsimd); calcObj chi; @@ -202,7 +202,6 @@ public: /////////////////////////////////// auto st = Stencil.View(AcceleratorRead); auto buf = st.CommBuf(); - StencilEntry *SE; autoView( in , _in , AcceleratorRead); autoView( out , _out , AcceleratorWrite); @@ -214,9 +213,11 @@ public: const int Nsimd = vobj::Nsimd(); const uint64_t NN = grid->oSites(); - const int lane=acceleratorSIMTlane(Nsimd); + accelerator_for( ss, NN, Nsimd, { + StencilEntry *SE; + const int lane=acceleratorSIMTlane(Nsimd); calcObj chi; From 114920b8de4048b250206b4bd3ac469cd7fffd95 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 25 Aug 2021 12:24:17 +0100 Subject: [PATCH 70/86] Some example clean up --- examples/Example_Laplacian_smearing.cc | 4 ++-- examples/Example_Laplacian_solver.cc | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/Example_Laplacian_smearing.cc b/examples/Example_Laplacian_smearing.cc index 327c5ca9..9780e8a0 100644 --- a/examples/Example_Laplacian_smearing.cc +++ b/examples/Example_Laplacian_smearing.cc @@ -112,8 +112,8 @@ int main(int argc, char ** argv) PowerMethod PM; PM(HermOp,kronecker); - Chebyshev ChebySmear(lo,hi,20,DistillationSmearing); - // Chebyshev ChebySmear(lo,hi,20,MomentumSmearing); + // Chebyshev ChebySmear(lo,hi,20,DistillationSmearing); + Chebyshev ChebySmear(lo,hi,20,MomentumSmearing); { std::ofstream of("chebysmear"); ChebySmear.csv(of); diff --git a/examples/Example_Laplacian_solver.cc b/examples/Example_Laplacian_solver.cc index 88275df8..4dc00280 100644 --- a/examples/Example_Laplacian_solver.cc +++ b/examples/Example_Laplacian_solver.cc @@ -15,8 +15,7 @@ void SimpleConjugateGradient(LinearOperatorBase &HPDop,const Field &b, Fi r = b - mmp; p = r; - alpha = norm2(p); - cp = alpha; + cp = alpha = norm2(p); ssq = norm2(b); RealD rsq = Tolerance * Tolerance * ssq; @@ -26,11 +25,12 @@ void SimpleConjugateGradient(LinearOperatorBase &HPDop,const Field &b, Fi HPDop.HermOp(p, mmp); - ComplexD dc = innerProduct(p,mmp); - d = dc.real(); + d = real(innerProduct(p,mmp)); + alpha = c / d; - cp = axpy_norm(r, -alpha, mmp, r); + r = r - alpha *mmp; + cp = norm2(r); beta = cp / c; x = x + alpha* p ; @@ -121,7 +121,9 @@ int main(int argc, char ** argv) r=kronecker-r; std::cout << "True residual "<< norm2(r) < Date: Mon, 30 Aug 2021 20:32:11 -0400 Subject: [PATCH 71/86] Some sample code --- examples/Example_Mobius_spectrum.cc | 334 ++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 examples/Example_Mobius_spectrum.cc diff --git a/examples/Example_Mobius_spectrum.cc b/examples/Example_Mobius_spectrum.cc new file mode 100644 index 00000000..dd84a336 --- /dev/null +++ b/examples/Example_Mobius_spectrum.cc @@ -0,0 +1,334 @@ +/************************************************************************************* + Grid physics library, www.github.com/paboyle/Grid + + Copyright (C) 2021 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include + +using namespace std; +using namespace Grid; + +// Flip sign to make prop to p^2, not -p^2 relative to last example +template class CovariantLaplacianCshift : public SparseMatrixBase +{ +public: + INHERIT_GIMPL_TYPES(Gimpl); + + GridBase *grid; + GaugeField U; + + CovariantLaplacianCshift(GaugeField &_U) : + grid(_U.Grid()), + U(_U) { }; + + virtual GridBase *Grid(void) { return grid; }; + + virtual void M (const Field &in, Field &out) + { + out=Zero(); + for(int mu=0;mu(U, mu); // NB: Inefficent + out = out - Gimpl::CovShiftForward(Umu,mu,in); + out = out - Gimpl::CovShiftBackward(Umu,mu,in); + out = out + 2.0*in; + } + }; + virtual void Mdag (const Field &in, Field &out) { M(in,out);}; // Laplacian is hermitian + virtual void Mdiag (const Field &in, Field &out) {assert(0);}; // Unimplemented need only for multigrid + virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);}; // Unimplemented need only for multigrid + virtual void MdirAll (const Field &in, std::vector &out) {assert(0);}; // Unimplemented need only for multigrid +}; + +void MakePhase(Coordinate mom,LatticeComplex &phase) +{ + GridBase *grid = phase.Grid(); + auto latt_size = grid->GlobalDimensions(); + ComplexD ci(0.0,1.0); + phase=Zero(); + + LatticeComplex coor(phase.Grid()); + for(int mu=0;mu +void GaussianSmear(LatticeGaugeField &U,Field &unsmeared,Field &smeared) +{ + typedef CovariantLaplacianCshift Laplacian_t; + Laplacian_t Laplacian(U); + + Integer Iterations = 40; + Real width = 2.0; + Real coeff = (width*width) / Real(4*Iterations); + + Field tmp(U.Grid()); + smeared=unsmeared; + // chi = (1-p^2/2N)^N kronecker + for(int n = 0; n < Iterations; ++n) { + Laplacian.M(smeared,tmp); + smeared = smeared - coeff*tmp; + std::cout << " smear iter " << n<<" " < +void Solve(Action &D,LatticePropagator &source,LatticePropagator &propagator) +{ + GridBase *UGrid = D.GaugeGrid(); + GridBase *FGrid = D.FermionGrid(); + + LatticeFermion src4 (UGrid); + LatticeFermion src5 (FGrid); + LatticeFermion result5(FGrid); + LatticeFermion result4(UGrid); + + ConjugateGradient CG(1.0e-8,100000); + SchurRedBlackDiagMooeeSolve schur(CG); + ZeroGuesser ZG; // Could be a DeflatedGuesser if have eigenvectors + for(int s=0;s(src4,source,s,c); + + D.ImportPhysicalFermionSource(src4,src5); + + result5=Zero(); + schur(D,src5,result5,ZG); + std::cout<(propagator,result4,s,c); + } + } +} + +class MesonFile: Serializable { +public: + GRID_SERIALIZABLE_CLASS_MEMBERS(MesonFile, std::vector >, data); +}; + +void MesonTrace(std::string file,LatticePropagator &q1,LatticePropagator &q2,LatticeComplex &phase) +{ + const int nchannel=4; + Gamma::Algebra Gammas[nchannel][2] = { + {Gamma::Algebra::Gamma5 ,Gamma::Algebra::Gamma5}, + {Gamma::Algebra::GammaTGamma5,Gamma::Algebra::GammaTGamma5}, + {Gamma::Algebra::GammaTGamma5,Gamma::Algebra::Gamma5}, + {Gamma::Algebra::Gamma5 ,Gamma::Algebra::GammaTGamma5} + }; + + Gamma G5(Gamma::Algebra::Gamma5); + + LatticeComplex meson_CF(q1.Grid()); + MesonFile MF; + + for(int ch=0;ch meson_T; + sliceSum(meson_CF,meson_T, Tdir); + + int nt=meson_T.size(); + + std::vector corr(nt); + for(int t=0;t seeds4({1,2,3,4}); + GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); + + LatticeGaugeField Umu(UGrid); + std::string config; + if( argc > 1 && argv[1][0] != '-' ) + { + std::cout<::ColdConfiguration(Umu); + // SU::HotConfiguration(RNG4,Umu); + config="HotConfig"; + } + + std::vector masses({ 0.03,0.04,0.45} ); // u/d, s, c ?? + + int nmass = masses.size(); + + std::vector FermActs; + + std::cout< PointProps(nmass,UGrid); + std::vector GaussProps(nmass,UGrid); + std::vector Z2Props (nmass,UGrid); + + for(int m=0;m Date: Mon, 30 Aug 2021 21:15:39 -0400 Subject: [PATCH 72/86] Comment update --- examples/Example_Mobius_spectrum.cc | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/examples/Example_Mobius_spectrum.cc b/examples/Example_Mobius_spectrum.cc index dd84a336..f4cd3335 100644 --- a/examples/Example_Mobius_spectrum.cc +++ b/examples/Example_Mobius_spectrum.cc @@ -1,33 +1,13 @@ -/************************************************************************************* - Grid physics library, www.github.com/paboyle/Grid +/* + * Warning: This code illustrative only: not well tested, and not meant for production use + * without regression / tests being applied + */ - Copyright (C) 2021 - -Author: Peter Boyle - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution directory - *************************************************************************************/ - /* END LEGAL */ #include using namespace std; using namespace Grid; -// Flip sign to make prop to p^2, not -p^2 relative to last example template class CovariantLaplacianCshift : public SparseMatrixBase { public: From 381d8797d0225bef118fe390882ea725f5b32cea Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sat, 11 Sep 2021 23:05:02 +0100 Subject: [PATCH 73/86] Drop half prec comms for now --- Grid/qcd/action/fermion/Fermion.h | 72 +++++++++++++++---------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/Grid/qcd/action/fermion/Fermion.h b/Grid/qcd/action/fermion/Fermion.h index 09777204..59fc17d5 100644 --- a/Grid/qcd/action/fermion/Fermion.h +++ b/Grid/qcd/action/fermion/Fermion.h @@ -115,9 +115,9 @@ typedef WilsonFermion WilsonFermionR; typedef WilsonFermion WilsonFermionF; typedef WilsonFermion WilsonFermionD; -typedef WilsonFermion WilsonFermionRL; -typedef WilsonFermion WilsonFermionFH; -typedef WilsonFermion WilsonFermionDF; +//typedef WilsonFermion WilsonFermionRL; +//typedef WilsonFermion WilsonFermionFH; +//typedef WilsonFermion WilsonFermionDF; typedef WilsonFermion WilsonAdjFermionR; typedef WilsonFermion WilsonAdjFermionF; @@ -158,41 +158,41 @@ typedef DomainWallFermion DomainWallFermionR; typedef DomainWallFermion DomainWallFermionF; typedef DomainWallFermion DomainWallFermionD; -typedef DomainWallFermion DomainWallFermionRL; -typedef DomainWallFermion DomainWallFermionFH; -typedef DomainWallFermion DomainWallFermionDF; +//typedef DomainWallFermion DomainWallFermionRL; +//typedef DomainWallFermion DomainWallFermionFH; +//typedef DomainWallFermion DomainWallFermionDF; typedef DomainWallEOFAFermion DomainWallEOFAFermionR; typedef DomainWallEOFAFermion DomainWallEOFAFermionF; typedef DomainWallEOFAFermion DomainWallEOFAFermionD; -typedef DomainWallEOFAFermion DomainWallEOFAFermionRL; -typedef DomainWallEOFAFermion DomainWallEOFAFermionFH; -typedef DomainWallEOFAFermion DomainWallEOFAFermionDF; +//typedef DomainWallEOFAFermion DomainWallEOFAFermionRL; +//typedef DomainWallEOFAFermion DomainWallEOFAFermionFH; +//typedef DomainWallEOFAFermion DomainWallEOFAFermionDF; typedef MobiusFermion MobiusFermionR; typedef MobiusFermion MobiusFermionF; typedef MobiusFermion MobiusFermionD; -typedef MobiusFermion MobiusFermionRL; -typedef MobiusFermion MobiusFermionFH; -typedef MobiusFermion MobiusFermionDF; +//typedef MobiusFermion MobiusFermionRL; +//typedef MobiusFermion MobiusFermionFH; +//typedef MobiusFermion MobiusFermionDF; typedef MobiusEOFAFermion MobiusEOFAFermionR; typedef MobiusEOFAFermion MobiusEOFAFermionF; typedef MobiusEOFAFermion MobiusEOFAFermionD; -typedef MobiusEOFAFermion MobiusEOFAFermionRL; -typedef MobiusEOFAFermion MobiusEOFAFermionFH; -typedef MobiusEOFAFermion MobiusEOFAFermionDF; +//typedef MobiusEOFAFermion MobiusEOFAFermionRL; +//typedef MobiusEOFAFermion MobiusEOFAFermionFH; +//typedef MobiusEOFAFermion MobiusEOFAFermionDF; typedef ZMobiusFermion ZMobiusFermionR; typedef ZMobiusFermion ZMobiusFermionF; typedef ZMobiusFermion ZMobiusFermionD; -typedef ZMobiusFermion ZMobiusFermionRL; -typedef ZMobiusFermion ZMobiusFermionFH; -typedef ZMobiusFermion ZMobiusFermionDF; +//typedef ZMobiusFermion ZMobiusFermionRL; +//typedef ZMobiusFermion ZMobiusFermionFH; +//typedef ZMobiusFermion ZMobiusFermionDF; // Ls vectorised typedef ScaledShamirFermion ScaledShamirFermionR; @@ -235,49 +235,49 @@ typedef WilsonFermion GparityWilsonFermionR; typedef WilsonFermion GparityWilsonFermionF; typedef WilsonFermion GparityWilsonFermionD; -typedef WilsonFermion GparityWilsonFermionRL; -typedef WilsonFermion GparityWilsonFermionFH; -typedef WilsonFermion GparityWilsonFermionDF; +//typedef WilsonFermion GparityWilsonFermionRL; +//typedef WilsonFermion GparityWilsonFermionFH; +//typedef WilsonFermion GparityWilsonFermionDF; typedef DomainWallFermion GparityDomainWallFermionR; typedef DomainWallFermion GparityDomainWallFermionF; typedef DomainWallFermion GparityDomainWallFermionD; -typedef DomainWallFermion GparityDomainWallFermionRL; -typedef DomainWallFermion GparityDomainWallFermionFH; -typedef DomainWallFermion GparityDomainWallFermionDF; +//typedef DomainWallFermion GparityDomainWallFermionRL; +//typedef DomainWallFermion GparityDomainWallFermionFH; +//typedef DomainWallFermion GparityDomainWallFermionDF; typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionR; typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionF; typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionD; -typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionRL; -typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionFH; -typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionDF; +//typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionRL; +//typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionFH; +//typedef DomainWallEOFAFermion GparityDomainWallEOFAFermionDF; typedef WilsonTMFermion GparityWilsonTMFermionR; typedef WilsonTMFermion GparityWilsonTMFermionF; typedef WilsonTMFermion GparityWilsonTMFermionD; -typedef WilsonTMFermion GparityWilsonTMFermionRL; -typedef WilsonTMFermion GparityWilsonTMFermionFH; -typedef WilsonTMFermion GparityWilsonTMFermionDF; +//typedef WilsonTMFermion GparityWilsonTMFermionRL; +//typedef WilsonTMFermion GparityWilsonTMFermionFH; +//typedef WilsonTMFermion GparityWilsonTMFermionDF; typedef MobiusFermion GparityMobiusFermionR; typedef MobiusFermion GparityMobiusFermionF; typedef MobiusFermion GparityMobiusFermionD; -typedef MobiusFermion GparityMobiusFermionRL; -typedef MobiusFermion GparityMobiusFermionFH; -typedef MobiusFermion GparityMobiusFermionDF; +//typedef MobiusFermion GparityMobiusFermionRL; +//typedef MobiusFermion GparityMobiusFermionFH; +//typedef MobiusFermion GparityMobiusFermionDF; typedef MobiusEOFAFermion GparityMobiusEOFAFermionR; typedef MobiusEOFAFermion GparityMobiusEOFAFermionF; typedef MobiusEOFAFermion GparityMobiusEOFAFermionD; -typedef MobiusEOFAFermion GparityMobiusEOFAFermionRL; -typedef MobiusEOFAFermion GparityMobiusEOFAFermionFH; -typedef MobiusEOFAFermion GparityMobiusEOFAFermionDF; +//typedef MobiusEOFAFermion GparityMobiusEOFAFermionRL; +//typedef MobiusEOFAFermion GparityMobiusEOFAFermionFH; +//typedef MobiusEOFAFermion GparityMobiusEOFAFermionDF; typedef ImprovedStaggeredFermion ImprovedStaggeredFermionR; typedef ImprovedStaggeredFermion ImprovedStaggeredFermionF; From d1b0b7f5c64f7d051b278ec7c67eda802906e223 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sat, 11 Sep 2021 23:05:40 +0100 Subject: [PATCH 74/86] Half prec comms dropping --- Grid/qcd/action/fermion/GparityWilsonImpl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Grid/qcd/action/fermion/GparityWilsonImpl.h b/Grid/qcd/action/fermion/GparityWilsonImpl.h index 9dca403b..fd627aed 100644 --- a/Grid/qcd/action/fermion/GparityWilsonImpl.h +++ b/Grid/qcd/action/fermion/GparityWilsonImpl.h @@ -327,8 +327,8 @@ typedef GparityWilsonImpl Gparit typedef GparityWilsonImpl GparityWilsonImplF; // Float typedef GparityWilsonImpl GparityWilsonImplD; // Double -typedef GparityWilsonImpl GparityWilsonImplRL; // Real.. whichever prec -typedef GparityWilsonImpl GparityWilsonImplFH; // Float -typedef GparityWilsonImpl GparityWilsonImplDF; // Double +//typedef GparityWilsonImpl GparityWilsonImplRL; // Real.. whichever prec +//typedef GparityWilsonImpl GparityWilsonImplFH; // Float +//typedef GparityWilsonImpl GparityWilsonImplDF; // Double NAMESPACE_END(Grid); From 73b944c1526491b5625ca8c19b341f135d62427f Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sat, 11 Sep 2021 23:07:18 +0100 Subject: [PATCH 75/86] Drop half prec comms for now. --- Grid/qcd/action/fermion/WilsonImpl.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Grid/qcd/action/fermion/WilsonImpl.h b/Grid/qcd/action/fermion/WilsonImpl.h index 2ff6feba..2685796d 100644 --- a/Grid/qcd/action/fermion/WilsonImpl.h +++ b/Grid/qcd/action/fermion/WilsonImpl.h @@ -243,17 +243,17 @@ typedef WilsonImpl WilsonImplR typedef WilsonImpl WilsonImplF; // Float typedef WilsonImpl WilsonImplD; // Double -typedef WilsonImpl WilsonImplRL; // Real.. whichever prec -typedef WilsonImpl WilsonImplFH; // Float -typedef WilsonImpl WilsonImplDF; // Double +//typedef WilsonImpl WilsonImplRL; // Real.. whichever prec +//typedef WilsonImpl WilsonImplFH; // Float +//typedef WilsonImpl WilsonImplDF; // Double typedef WilsonImpl ZWilsonImplR; // Real.. whichever prec typedef WilsonImpl ZWilsonImplF; // Float typedef WilsonImpl ZWilsonImplD; // Double -typedef WilsonImpl ZWilsonImplRL; // Real.. whichever prec -typedef WilsonImpl ZWilsonImplFH; // Float -typedef WilsonImpl ZWilsonImplDF; // Double +//typedef WilsonImpl ZWilsonImplRL; // Real.. whichever prec +//typedef WilsonImpl ZWilsonImplFH; // Float +//typedef WilsonImpl ZWilsonImplDF; // Double typedef WilsonImpl WilsonAdjImplR; // Real.. whichever prec typedef WilsonImpl WilsonAdjImplF; // Float From 4c88104a73871551050484fac521d49aec8a6de9 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Sat, 11 Sep 2021 23:08:05 +0100 Subject: [PATCH 76/86] Fix compile warns --- .../fermion/implementation/CayleyFermion5DImplementation.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h b/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h index c3e0f821..1ed66bda 100644 --- a/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h +++ b/Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h @@ -880,7 +880,7 @@ void CayleyFermion5D::SeqConservedCurrent(PropagatorField &q_in, } std::vector G_s(Ls,1.0); - Integer sign = 1; // sign flip for vector/tadpole + RealD sign = 1; // sign flip for vector/tadpole if ( curr_type == Current::Axial ) { for(int s=0;s::SeqConservedCurrent(PropagatorField &q_in, for(int s=0;sbs[s]; From 8195890640c37324757b864b472d17bc23408402 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 14 Sep 2021 05:00:17 +0100 Subject: [PATCH 77/86] Force MPI over NVLINK --- Grid/communicator/Communicator_base.h | 6 ++++++ Grid/communicator/Communicator_mpi3.cc | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Grid/communicator/Communicator_base.h b/Grid/communicator/Communicator_base.h index a15f9789..4510b03e 100644 --- a/Grid/communicator/Communicator_base.h +++ b/Grid/communicator/Communicator_base.h @@ -35,6 +35,12 @@ Author: Peter Boyle NAMESPACE_BEGIN(Grid); +#ifdef GRID_MPI3_SHM_NVLINK +const bool Stencil_force_mpi = true; +#else +const bool Stencil_force_mpi = false; +#endif + class CartesianCommunicator : public SharedMemory { public: diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc index 5713fe35..a6eeeebc 100644 --- a/Grid/communicator/Communicator_mpi3.cc +++ b/Grid/communicator/Communicator_mpi3.cc @@ -370,7 +370,7 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector Date: Tue, 14 Sep 2021 05:00:44 +0100 Subject: [PATCH 78/86] Dirac cuda 11.4 happy ; force host for functions accessing mult table ET runs these on host BEFORE lodging result in AST for kernel --- Grid/qcd/spin/Dirac.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Grid/qcd/spin/Dirac.h b/Grid/qcd/spin/Dirac.h index d03e0939..2f2a9732 100644 --- a/Grid/qcd/spin/Dirac.h +++ b/Grid/qcd/spin/Dirac.h @@ -40,7 +40,7 @@ See the full license in the file "LICENSE" in the top level distribution directo NAMESPACE_BEGIN(Grid); // Dirac algebra adjoint operator (not in to overload other adj) -accelerator_inline Gamma adj(const Gamma &g) +inline Gamma adj(const Gamma &g) { return Gamma (Gamma::adj[g.g]); } @@ -48,7 +48,7 @@ accelerator_inline Gamma adj(const Gamma &g) // Dirac algebra mutliplication operator -accelerator_inline Gamma operator*(const Gamma &g1, const Gamma &g2) +inline Gamma operator*(const Gamma &g1, const Gamma &g2) { return Gamma (Gamma::mul[g1.g][g2.g]); } From 7440cde92f25f2a9176b725910c15d748dbbd27c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 14 Sep 2021 05:04:56 +0100 Subject: [PATCH 79/86] No half prec comms; coalesced access on GPU --- Grid/qcd/action/fermion/WilsonCompressor.h | 98 +++++++++++----------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/Grid/qcd/action/fermion/WilsonCompressor.h b/Grid/qcd/action/fermion/WilsonCompressor.h index 0760bcba..e0e08c1c 100644 --- a/Grid/qcd/action/fermion/WilsonCompressor.h +++ b/Grid/qcd/action/fermion/WilsonCompressor.h @@ -68,11 +68,12 @@ public: /*****************************************************/ /* Compress includes precision change if mpi data is not same */ /*****************************************************/ - template - accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) const { - _SiteHalfSpinor tmp; - projector::Proj(tmp,in,mu,dag); - vstream(buf[o],tmp); + accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const { + typedef decltype(coalescedRead(buf)) sobj; + sobj sp; + auto sin = coalescedRead(in); + projector::Proj(sp,sin,mu,dag); + coalescedWrite(buf,sp); } /*****************************************************/ @@ -82,13 +83,18 @@ public: const SiteHalfSpinor * __restrict__ vp0, const SiteHalfSpinor * __restrict__ vp1, Integer type,Integer o) const { +#ifdef GRID_SIMT + exchangeSIMT(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type); +#else SiteHalfSpinor tmp1; SiteHalfSpinor tmp2; exchange(tmp1,tmp2,vp0[o],vp1[o],type); vstream(mp[2*o ],tmp1); vstream(mp[2*o+1],tmp2); +#endif } + /*****************************************************/ /* Have a decompression step if mpi data is not same */ /*****************************************************/ @@ -105,6 +111,28 @@ public: const SiteSpinor * __restrict__ in, Integer j,Integer k, Integer m,Integer type) const { +#ifdef GRID_SIMT + typedef SiteSpinor vobj; + typedef SiteHalfSpinor hvobj; + typedef decltype(coalescedRead(*in)) sobj; + typedef decltype(coalescedRead(*out0)) hsobj; + + unsigned int Nsimd = vobj::Nsimd(); + unsigned int mask = Nsimd >> (type + 1); + int lane = acceleratorSIMTlane(Nsimd); + int j0 = lane &(~mask); // inner coor zero + int j1 = lane |(mask) ; // inner coor one + const vobj *vp0 = &in[k]; + const vobj *vp1 = &in[m]; + const vobj *vp = (lane&mask) ? vp1:vp0; + auto sa = coalescedRead(*vp,j0); + auto sb = coalescedRead(*vp,j1); + hsobj psa, psb; + projector::Proj(psa,sa,mu,dag); + projector::Proj(psb,sb,mu,dag); + coalescedWrite(out0[j],psa); + coalescedWrite(out1[j],psb); +#else SiteHalfSpinor temp1, temp2; SiteHalfSpinor temp3, temp4; projector::Proj(temp1,in[k],mu,dag); @@ -112,6 +140,7 @@ public: exchange(temp3,temp4,temp1,temp2,type); vstream(out0[j],temp3); vstream(out1[j],temp4); +#endif } /*****************************************************/ @@ -121,6 +150,7 @@ public: }; +#if 0 template class WilsonCompressorTemplate< _HCspinor, _Hspinor, _Spinor, projector, typename std::enable_if::value>::type > @@ -149,13 +179,23 @@ public: /*****************************************************/ /* Compress includes precision change if mpi data is not same */ /*****************************************************/ - template - accelerator_inline void Compress(_SiteHalfSpinor *buf,Integer o,const _SiteSpinor &in) const { - _SiteHalfSpinor hsp; + accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const { + SiteHalfSpinor hsp; SiteHalfCommSpinor *hbuf = (SiteHalfCommSpinor *)buf; projector::Proj(hsp,in,mu,dag); precisionChange((vComplexLow *)&hbuf[o],(vComplexHigh *)&hsp,Nw); } + accelerator_inline void Compress(SiteHalfSpinor &buf,const SiteSpinor &in) const { +#ifdef GRID_SIMT + typedef decltype(coalescedRead(buf)) sobj; + sobj sp; + auto sin = coalescedRead(in); + projector::Proj(sp,sin,mu,dag); + coalescedWrite(buf,sp); +#else + projector::Proj(buf,in,mu,dag); +#endif + } /*****************************************************/ /* Exchange includes precision change if mpi data is not same */ @@ -203,6 +243,7 @@ public: accelerator_inline bool DecompressionStep(void) const { return true; } }; +#endif #define DECLARE_PROJ(Projector,Compressor,spProj) \ class Projector { \ @@ -253,33 +294,8 @@ public: typedef typename Base::View_type View_type; typedef typename Base::StencilVector StencilVector; - double timer0; - double timer1; - double timer2; - double timer3; - double timer4; - double timer5; - double timer6; - uint64_t callsi; - void ZeroCountersi(void) - { - timer0=0; - timer1=0; - timer2=0; - timer3=0; - timer4=0; - timer5=0; - timer6=0; - callsi=0; - } - void Reporti(int calls) - { - if ( timer0 ) std::cout << GridLogMessage << " timer0 (HaloGatherOpt) " < surface_list; @@ -321,26 +337,18 @@ public: { std::vector > reqs; this->HaloExchangeOptGather(source,compress); - double t1=usecond(); // Asynchronous MPI calls multidirectional, Isend etc... // Non-overlapped directions within a thread. Asynchronous calls except MPI3, threaded up to comm threads ways. this->Communicate(); - double t2=usecond(); timer1 += t2-t1; this->CommsMerge(compress); - double t3=usecond(); timer2 += t3-t2; this->CommsMergeSHM(compress); - double t4=usecond(); timer3 += t4-t3; } template void HaloExchangeOptGather(const Lattice &source,compressor &compress) { this->Prepare(); - double t0=usecond(); this->HaloGatherOpt(source,compress); - double t1=usecond(); - timer0 += t1-t0; - callsi++; } template @@ -352,12 +360,9 @@ public: typedef typename compressor::SiteHalfSpinor SiteHalfSpinor; typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor; - this->mpi3synctime_g-=usecond(); this->_grid->StencilBarrier(); - this->mpi3synctime_g+=usecond(); assert(source.Grid()==this->_grid); - this->halogtime-=usecond(); this->u_comm_offset=0; @@ -393,7 +398,6 @@ public: } this->face_table_computed=1; assert(this->u_comm_offset==this->_unified_buffer_size); - this->halogtime+=usecond(); accelerator_barrier(); } From a7b943b33e262d528df5074e95e1f45b5a9b059c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 14 Sep 2021 05:05:33 +0100 Subject: [PATCH 80/86] Remove half prec comms --- .../instantiation/generate_instantiations.sh | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/Grid/qcd/action/fermion/instantiation/generate_instantiations.sh b/Grid/qcd/action/fermion/instantiation/generate_instantiations.sh index 72a9eaf9..d7553cdb 100755 --- a/Grid/qcd/action/fermion/instantiation/generate_instantiations.sh +++ b/Grid/qcd/action/fermion/instantiation/generate_instantiations.sh @@ -9,8 +9,6 @@ STAG5_IMPL_LIST="" WILSON_IMPL_LIST=" \ WilsonImplF \ WilsonImplD \ - WilsonImplFH \ - WilsonImplDF \ WilsonAdjImplF \ WilsonAdjImplD \ WilsonTwoIndexSymmetricImplF \ @@ -18,26 +16,17 @@ WILSON_IMPL_LIST=" \ WilsonTwoIndexAntiSymmetricImplF \ WilsonTwoIndexAntiSymmetricImplD \ GparityWilsonImplF \ - GparityWilsonImplD \ - GparityWilsonImplFH \ - GparityWilsonImplDF" + GparityWilsonImplD " DWF_IMPL_LIST=" \ WilsonImplF \ WilsonImplD \ - WilsonImplFH \ - WilsonImplDF \ ZWilsonImplF \ - ZWilsonImplD \ - ZWilsonImplFH \ - ZWilsonImplDF " + ZWilsonImplD " GDWF_IMPL_LIST=" \ GparityWilsonImplF \ - GparityWilsonImplD \ - GparityWilsonImplFH \ - GparityWilsonImplDF" - + GparityWilsonImplD " IMPL_LIST="$STAG_IMPL_LIST $WILSON_IMPL_LIST $DWF_IMPL_LIST $GDWF_IMPL_LIST" From d5835c0222aca3962bff100c0331b79f30c75254 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 14 Sep 2021 15:04:14 +0100 Subject: [PATCH 81/86] Switch to coalesced stencil face gather --- Grid/stencil/SimpleCompressor.h | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/Grid/stencil/SimpleCompressor.h b/Grid/stencil/SimpleCompressor.h index 2ce48369..1150b234 100644 --- a/Grid/stencil/SimpleCompressor.h +++ b/Grid/stencil/SimpleCompressor.h @@ -3,20 +3,48 @@ NAMESPACE_BEGIN(Grid); +template +accelerator_inline void exchangeSIMT(vobj &mp0,vobj &mp1,const vobj &vp0,const vobj &vp1,Integer type) +{ + typedef decltype(coalescedRead(mp0)) sobj; + unsigned int Nsimd = vobj::Nsimd(); + unsigned int mask = Nsimd >> (type + 1); + int lane = acceleratorSIMTlane(Nsimd); + int j0 = lane &(~mask); // inner coor zero + int j1 = lane |(mask) ; // inner coor one + const vobj *vpa = &vp0; + const vobj *vpb = &vp1; + const vobj *vp = (lane&mask) ? (vpb) : (vpa); + auto sa = coalescedRead(vp[0],j0); + auto sb = coalescedRead(vp[0],j1); + coalescedWrite(mp0,sa); + coalescedWrite(mp1,sb); +} + template class SimpleCompressor { public: void Point(int) {}; accelerator_inline int CommDatumSize(void) const { return sizeof(vobj); } accelerator_inline bool DecompressionStep(void) const { return false; } - template accelerator_inline void Compress(cobj *buf,int o,const cobj &in) const { buf[o]=in; } + accelerator_inline void Compress(vobj &buf,const vobj &in) const { + coalescedWrite(buf,coalescedRead(in)); + } accelerator_inline void Exchange(vobj *mp,vobj *vp0,vobj *vp1,Integer type,Integer o) const { +#ifdef GRID_SIMT + exchangeSIMT(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type); +#else exchange(mp[2*o],mp[2*o+1],vp0[o],vp1[o],type); +#endif } accelerator_inline void Decompress(vobj *out,vobj *in, int o) const { assert(0); } accelerator_inline void CompressExchange(vobj *out0,vobj *out1,const vobj *in, - int j,int k, int m,int type) const { + int j,int k, int m,int type) const { +#ifdef GRID_SIMT + exchangeSIMT(out0[j],out1[j],in[k],in[m],type); +#else exchange(out0[j],out1[j],in[k],in[m],type); +#endif } // For cshift. Cshift should drop compressor coupling altogether // because I had to decouple the code from the Stencil anyway From 65ef4ec29fd628bedcfe3c6399dcc642ad8e18a0 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 14 Sep 2021 15:05:01 +0100 Subject: [PATCH 82/86] Move tables to device memory --- Grid/stencil/Stencil.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grid/stencil/Stencil.cc b/Grid/stencil/Stencil.cc index 5b1bb2ea..c1b33baa 100644 --- a/Grid/stencil/Stencil.cc +++ b/Grid/stencil/Stencil.cc @@ -30,7 +30,7 @@ NAMESPACE_BEGIN(Grid); void Gather_plane_table_compute (GridBase *grid,int dimension,int plane,int cbmask, - int off,Vector > & table) + int off,std::vector > & table) { table.resize(0); From 7efdb3cd2b84b83bd8508d682071d6c7ecf144f9 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 14 Sep 2021 15:06:06 +0100 Subject: [PATCH 83/86] Remove half prec comms --- benchmarks/Benchmark_dwf.cc | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc index d7b49122..707f330c 100644 --- a/benchmarks/Benchmark_dwf.cc +++ b/benchmarks/Benchmark_dwf.cc @@ -236,34 +236,6 @@ int main (int argc, char ** argv) Dw.Report(); } - DomainWallFermionRL DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5); - if (0) { - FGrid->Barrier(); - DwH.ZeroCounters(); - DwH.Dhop(src,result,0); - double t0=usecond(); - for(int i=0;iBarrier(); - - double volume=Ls; for(int mu=0;mu Date: Tue, 14 Sep 2021 15:06:29 +0100 Subject: [PATCH 84/86] Remove half prec comms --- benchmarks/Benchmark_gparity.cc | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/benchmarks/Benchmark_gparity.cc b/benchmarks/Benchmark_gparity.cc index 2045d650..ce84ecbc 100644 --- a/benchmarks/Benchmark_gparity.cc +++ b/benchmarks/Benchmark_gparity.cc @@ -2,7 +2,6 @@ #include using namespace std; using namespace Grid; - ; template struct scal { @@ -118,30 +117,6 @@ int main (int argc, char ** argv) Dw.Report(); } - std::cout << GridLogMessage<< "* SINGLE/HALF"<Barrier(); - DwH.ZeroCounters(); - DwH.Dhop(src,result,0); - double t0=usecond(); - for(int i=0;iBarrier(); - - double volume=Ls; for(int mu=0;mu -#include -#include - -#ifndef AVX512 -#ifndef QPX -#ifndef A64FX -#ifndef A64FXFIXEDSIZE -#include -#endif -#endif -#endif -#endif - -NAMESPACE_BEGIN(Grid); - -#include "impl.h" -template class WilsonKernels; - -NAMESPACE_END(Grid); diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplDF/WilsonTMFermionInstantiationWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplDF/WilsonTMFermionInstantiationWilsonImplDF.cc deleted file mode 120000 index d5789bcf..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplDF/WilsonTMFermionInstantiationWilsonImplDF.cc +++ /dev/null @@ -1 +0,0 @@ -../WilsonTMFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplDF/impl.h b/Grid/qcd/action/fermion/instantiation/WilsonImplDF/impl.h deleted file mode 100644 index 2adc6136..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplDF/impl.h +++ /dev/null @@ -1 +0,0 @@ -#define IMPLEMENTATION WilsonImplDF diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/CayleyFermion5DInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/CayleyFermion5DInstantiationWilsonImplFH.cc deleted file mode 120000 index cb1db625..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/CayleyFermion5DInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../CayleyFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/ContinuedFractionFermion5DInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/ContinuedFractionFermion5DInstantiationWilsonImplFH.cc deleted file mode 120000 index c2d4b8fc..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/ContinuedFractionFermion5DInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../ContinuedFractionFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/DomainWallEOFAFermionInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/DomainWallEOFAFermionInstantiationWilsonImplFH.cc deleted file mode 120000 index 2f550a2b..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/DomainWallEOFAFermionInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../DomainWallEOFAFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/MobiusEOFAFermionInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/MobiusEOFAFermionInstantiationWilsonImplFH.cc deleted file mode 120000 index 7a8f1172..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/MobiusEOFAFermionInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../MobiusEOFAFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/PartialFractionFermion5DInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/PartialFractionFermion5DInstantiationWilsonImplFH.cc deleted file mode 120000 index 7f4cea71..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/PartialFractionFermion5DInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../PartialFractionFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonCloverFermionInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonCloverFermionInstantiationWilsonImplFH.cc deleted file mode 120000 index 9cc05107..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonCloverFermionInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../WilsonCloverFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonFermion5DInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonFermion5DInstantiationWilsonImplFH.cc deleted file mode 120000 index 804d0884..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonFermion5DInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../WilsonFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonFermionInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonFermionInstantiationWilsonImplFH.cc deleted file mode 120000 index 5f6ab65e..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonFermionInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../WilsonFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonKernelsInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonKernelsInstantiationWilsonImplFH.cc deleted file mode 100644 index f0b15e3b..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonKernelsInstantiationWilsonImplFH.cc +++ /dev/null @@ -1,51 +0,0 @@ -/************************************************************************************* - -Grid physics library, www.github.com/paboyle/Grid - -Source file: ./lib/qcd/action/fermion/WilsonKernels.cc - -Copyright (C) 2015, 2020 - -Author: Peter Boyle -Author: Peter Boyle -Author: paboyle -Author: Nils Meyer Regensburg University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -See the full license in the file "LICENSE" in the top level distribution -directory -*************************************************************************************/ -/* END LEGAL */ -#include -#include -#include - -#ifndef AVX512 -#ifndef QPX -#ifndef A64FX -#ifndef A64FXFIXEDSIZE -#include -#endif -#endif -#endif -#endif - -NAMESPACE_BEGIN(Grid); - -#include "impl.h" -template class WilsonKernels; - -NAMESPACE_END(Grid); diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonTMFermionInstantiationWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonTMFermionInstantiationWilsonImplFH.cc deleted file mode 120000 index d5789bcf..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/WilsonTMFermionInstantiationWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../WilsonTMFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/impl.h b/Grid/qcd/action/fermion/instantiation/WilsonImplFH/impl.h deleted file mode 100644 index e442863d..00000000 --- a/Grid/qcd/action/fermion/instantiation/WilsonImplFH/impl.h +++ /dev/null @@ -1 +0,0 @@ -#define IMPLEMENTATION WilsonImplFH diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/CayleyFermion5DInstantiationZWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/CayleyFermion5DInstantiationZWilsonImplDF.cc deleted file mode 120000 index cb1db625..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/CayleyFermion5DInstantiationZWilsonImplDF.cc +++ /dev/null @@ -1 +0,0 @@ -../CayleyFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/ContinuedFractionFermion5DInstantiationZWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/ContinuedFractionFermion5DInstantiationZWilsonImplDF.cc deleted file mode 120000 index c2d4b8fc..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/ContinuedFractionFermion5DInstantiationZWilsonImplDF.cc +++ /dev/null @@ -1 +0,0 @@ -../ContinuedFractionFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/DomainWallEOFAFermionInstantiationZWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/DomainWallEOFAFermionInstantiationZWilsonImplDF.cc deleted file mode 120000 index 2f550a2b..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/DomainWallEOFAFermionInstantiationZWilsonImplDF.cc +++ /dev/null @@ -1 +0,0 @@ -../DomainWallEOFAFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/MobiusEOFAFermionInstantiationZWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/MobiusEOFAFermionInstantiationZWilsonImplDF.cc deleted file mode 120000 index 7a8f1172..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/MobiusEOFAFermionInstantiationZWilsonImplDF.cc +++ /dev/null @@ -1 +0,0 @@ -../MobiusEOFAFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/PartialFractionFermion5DInstantiationZWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/PartialFractionFermion5DInstantiationZWilsonImplDF.cc deleted file mode 120000 index 7f4cea71..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/PartialFractionFermion5DInstantiationZWilsonImplDF.cc +++ /dev/null @@ -1 +0,0 @@ -../PartialFractionFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonFermion5DInstantiationZWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonFermion5DInstantiationZWilsonImplDF.cc deleted file mode 120000 index 804d0884..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonFermion5DInstantiationZWilsonImplDF.cc +++ /dev/null @@ -1 +0,0 @@ -../WilsonFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonKernelsInstantiationZWilsonImplDF.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonKernelsInstantiationZWilsonImplDF.cc deleted file mode 100644 index f0b15e3b..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/WilsonKernelsInstantiationZWilsonImplDF.cc +++ /dev/null @@ -1,51 +0,0 @@ -/************************************************************************************* - -Grid physics library, www.github.com/paboyle/Grid - -Source file: ./lib/qcd/action/fermion/WilsonKernels.cc - -Copyright (C) 2015, 2020 - -Author: Peter Boyle -Author: Peter Boyle -Author: paboyle -Author: Nils Meyer Regensburg University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -See the full license in the file "LICENSE" in the top level distribution -directory -*************************************************************************************/ -/* END LEGAL */ -#include -#include -#include - -#ifndef AVX512 -#ifndef QPX -#ifndef A64FX -#ifndef A64FXFIXEDSIZE -#include -#endif -#endif -#endif -#endif - -NAMESPACE_BEGIN(Grid); - -#include "impl.h" -template class WilsonKernels; - -NAMESPACE_END(Grid); diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/impl.h b/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/impl.h deleted file mode 100644 index 7daf76ef..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplDF/impl.h +++ /dev/null @@ -1 +0,0 @@ -#define IMPLEMENTATION ZWilsonImplDF diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/CayleyFermion5DInstantiationZWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/CayleyFermion5DInstantiationZWilsonImplFH.cc deleted file mode 120000 index cb1db625..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/CayleyFermion5DInstantiationZWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../CayleyFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/ContinuedFractionFermion5DInstantiationZWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/ContinuedFractionFermion5DInstantiationZWilsonImplFH.cc deleted file mode 120000 index c2d4b8fc..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/ContinuedFractionFermion5DInstantiationZWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../ContinuedFractionFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/DomainWallEOFAFermionInstantiationZWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/DomainWallEOFAFermionInstantiationZWilsonImplFH.cc deleted file mode 120000 index 2f550a2b..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/DomainWallEOFAFermionInstantiationZWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../DomainWallEOFAFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/MobiusEOFAFermionInstantiationZWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/MobiusEOFAFermionInstantiationZWilsonImplFH.cc deleted file mode 120000 index 7a8f1172..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/MobiusEOFAFermionInstantiationZWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../MobiusEOFAFermionInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/PartialFractionFermion5DInstantiationZWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/PartialFractionFermion5DInstantiationZWilsonImplFH.cc deleted file mode 120000 index 7f4cea71..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/PartialFractionFermion5DInstantiationZWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../PartialFractionFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonFermion5DInstantiationZWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonFermion5DInstantiationZWilsonImplFH.cc deleted file mode 120000 index 804d0884..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonFermion5DInstantiationZWilsonImplFH.cc +++ /dev/null @@ -1 +0,0 @@ -../WilsonFermion5DInstantiation.cc.master \ No newline at end of file diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonKernelsInstantiationZWilsonImplFH.cc b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonKernelsInstantiationZWilsonImplFH.cc deleted file mode 100644 index f0b15e3b..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/WilsonKernelsInstantiationZWilsonImplFH.cc +++ /dev/null @@ -1,51 +0,0 @@ -/************************************************************************************* - -Grid physics library, www.github.com/paboyle/Grid - -Source file: ./lib/qcd/action/fermion/WilsonKernels.cc - -Copyright (C) 2015, 2020 - -Author: Peter Boyle -Author: Peter Boyle -Author: paboyle -Author: Nils Meyer Regensburg University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -See the full license in the file "LICENSE" in the top level distribution -directory -*************************************************************************************/ -/* END LEGAL */ -#include -#include -#include - -#ifndef AVX512 -#ifndef QPX -#ifndef A64FX -#ifndef A64FXFIXEDSIZE -#include -#endif -#endif -#endif -#endif - -NAMESPACE_BEGIN(Grid); - -#include "impl.h" -template class WilsonKernels; - -NAMESPACE_END(Grid); diff --git a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/impl.h b/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/impl.h deleted file mode 100644 index 7eb490db..00000000 --- a/Grid/qcd/action/fermion/instantiation/ZWilsonImplFH/impl.h +++ /dev/null @@ -1 +0,0 @@ -#define IMPLEMENTATION ZWilsonImplFH diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h index 58cebed3..ca581804 100644 --- a/Grid/stencil/Stencil.h +++ b/Grid/stencil/Stencil.h @@ -57,27 +57,22 @@ NAMESPACE_BEGIN(Grid); /////////////////////////////////////////////////////////////////// void Gather_plane_table_compute (GridBase *grid,int dimension,int plane,int cbmask, - int off,Vector > & table); + int off,std::vector > & table); template -void Gather_plane_simple_table (Vector >& table,const Lattice &rhs,cobj *buffer,compressor &compress, int off,int so) __attribute__((noinline)); +void Gather_plane_simple_table (commVector >& table,const Lattice &rhs,cobj *buffer,compressor &compress, int off,int so) __attribute__((noinline)); template -void Gather_plane_simple_table (Vector >& table,const Lattice &rhs,cobj *buffer,compressor &compress, int off,int so) +void Gather_plane_simple_table (commVector >& table,const Lattice &rhs,cobj *buffer,compressor &compress, int off,int so) { int num=table.size(); std::pair *table_v = & table[0]; auto rhs_v = rhs.View(AcceleratorRead); accelerator_forNB( i,num, vobj::Nsimd(), { - typedef decltype(coalescedRead(buffer[0])) compressed_t; - compressed_t tmp_c; - uint64_t o = table_v[i].first; - compress.Compress(&tmp_c,0,rhs_v(so+table_v[i].second)); - coalescedWrite(buffer[off+o],tmp_c); + compress.Compress(buffer[off+table_v[i].first],rhs_v[so+table_v[i].second]); }); rhs_v.ViewClose(); -// Further optimisatoin: i) software prefetch the first element of the next table entry, prefetch the table } /////////////////////////////////////////////////////////////////// @@ -85,10 +80,10 @@ void Gather_plane_simple_table (Vector >& table,const Lattice /////////////////////////////////////////////////////////////////// template void Gather_plane_exchange_table(const Lattice &rhs, - Vector pointers,int dimension,int plane,int cbmask,compressor &compress,int type) __attribute__((noinline)); + commVector pointers,int dimension,int plane,int cbmask,compressor &compress,int type) __attribute__((noinline)); template -void Gather_plane_exchange_table(Vector >& table,const Lattice &rhs, +void Gather_plane_exchange_table(commVector >& table,const Lattice &rhs, Vector pointers,int dimension,int plane,int cbmask, compressor &compress,int type) { @@ -100,7 +95,7 @@ void Gather_plane_exchange_table(Vector >& table,const Lattic auto p0=&pointers[0][0]; auto p1=&pointers[1][0]; auto tp=&table[0]; - accelerator_forNB(j, num, 1, { + accelerator_forNB(j, num, vobj::Nsimd(), { compress.CompressExchange(p0,p1, &rhs_v[0], j, so+tp[2*j ].second, so+tp[2*j+1].second, @@ -266,10 +261,11 @@ public: } int face_table_computed; - std::vector > > face_table ; + std::vector > > face_table ; Vector surface_list; stencilVector _entries; // Resident in managed memory + commVector _entries_device; // Resident in managed memory std::vector Packets; std::vector Mergers; std::vector MergersSHM; @@ -342,7 +338,7 @@ public: void *shm = (void *) _grid->ShmBufferTranslate(recv_from_rank,this->u_recv_buf_p); - if ( shm==NULL ) return 0; + if ( (shm==NULL) || Stencil_force_mpi ) return 0; return 1; } @@ -609,13 +605,14 @@ public: template void CommsMerge(decompressor decompress,std::vector &mm,std::vector &dd) { + mergetime-=usecond(); for(int i=0;i_ostride[dimension]; // base offset for start of plane if ( !face_table_computed ) { face_table.resize(face_idx+1); - Gather_plane_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,face_table[face_idx]); + std::vector > face_table_host ; + Gather_plane_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,face_table_host); + face_table[face_idx].resize(face_table_host.size()); + acceleratorCopyToDevice(&face_table_host[0], + &face_table[face_idx][0], + face_table[face_idx].size()*sizeof(face_table_host[0])); } // int rank = _grid->_processor; @@ -1062,20 +1064,20 @@ public: } send_buf = (cobj *)_grid->ShmBufferTranslate(xmit_to_rank,recv_buf); - if ( send_buf==NULL ) { + if ( (send_buf==NULL) || Stencil_force_mpi ) { send_buf = this->u_send_buf_p; } // Find out if we get the direct copy. void *success = (void *) _grid->ShmBufferTranslate(recv_from_rank,this->u_send_buf_p); - if (success==NULL) { + if ((success==NULL)||Stencil_force_mpi) { // we found a packet that comes from MPI and contributes to this leg of stencil shm_receive_only = 0; } gathertime-=usecond(); assert(send_buf!=NULL); - Gather_plane_simple_table(face_table[face_idx],rhs,send_buf,compress,u_comm_offset,so); face_idx++; + Gather_plane_simple_table(face_table[face_idx],rhs,send_buf,compress,u_comm_offset,so); face_idx++; gathertime+=usecond(); if ( compress.DecompressionStep() ) { @@ -1172,11 +1174,18 @@ public: if ( !face_table_computed ) { face_table.resize(face_idx+1); - Gather_plane_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,face_table[face_idx]); + std::vector > face_table_host ; + + Gather_plane_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset,face_table_host); + face_table[face_idx].resize(face_table_host.size()); + acceleratorCopyToDevice(&face_table_host[0], + &face_table[face_idx][0], + face_table[face_idx].size()*sizeof(face_table_host[0])); } gathermtime-=usecond(); - Gather_plane_exchange_table(face_table[face_idx],rhs,spointers,dimension,sx,cbmask,compress,permute_type); face_idx++; + Gather_plane_exchange_table(face_table[face_idx],rhs,spointers,dimension,sx,cbmask,compress,permute_type); + face_idx++; gathermtime+=usecond(); //spointers[0] -- low @@ -1208,7 +1217,7 @@ public: // shm == receive pointer if offnode // shm == Translate[send pointer] if on node -- my view of his send pointer cobj *shm = (cobj *) _grid->ShmBufferTranslate(recv_from_rank,sp); - if (shm==NULL) { + if ((shm==NULL)||Stencil_force_mpi) { shm = rp; // we found a packet that comes from MPI and contributes to this shift. // is_same_node is only used in the WilsonStencil, and gets set for this point in the stencil.