From 6252ffaf76b0f841c35321a77a1330e61214deeb Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 3 Apr 2023 18:25:22 -0400 Subject: [PATCH 1/6] No unified --- systems/Crusher/config-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/Crusher/config-command b/systems/Crusher/config-command index d310ff55..2176fc6b 100644 --- a/systems/Crusher/config-command +++ b/systems/Crusher/config-command @@ -1,7 +1,7 @@ CLIME=`spack find --paths c-lime@2-3-9 | grep c-lime| cut -c 15-` ../../configure --enable-comms=mpi-auto \ --with-lime=$CLIME \ ---enable-unified=yes \ +--enable-unified=no \ --enable-shm=nvlink \ --enable-tracing=timer \ --enable-accelerator=hip \ From 6b979f0a69b2c460c6cad9d9782b77467f61cd66 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 4 Apr 2023 23:13:17 -0400 Subject: [PATCH 2/6] Dirichlet improvements that I failed to commit --- .../GeneralEvenOddRationalRatioMixedPrec.h | 47 +++++++------------ .../OneFlavourEvenOddRationalRatio.h | 8 ++-- HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc | 8 ++-- HMC/Mobius2p1f_EOFA_96I_hmc.cc | 10 +--- systems/Crusher/sourceme.sh | 4 +- 5 files changed, 25 insertions(+), 52 deletions(-) diff --git a/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h b/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h index 6fe32e15..104e3376 100644 --- a/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h +++ b/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h @@ -38,19 +38,15 @@ NAMESPACE_BEGIN(Grid); // cf. GeneralEvenOddRational.h for details ///////////////////////////////////////////////////////////////////////////////////////////////////////////// - template + template class GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction : public GeneralEvenOddRatioRationalPseudoFermionAction { private: - typedef typename ImplD2::FermionField FermionFieldD2; typedef typename ImplD::FermionField FermionFieldD; typedef typename ImplF::FermionField FermionFieldF; FermionOperator & NumOpD; FermionOperator & DenOpD; - FermionOperator & NumOpD2; - FermionOperator & DenOpD2; - FermionOperator & NumOpF; FermionOperator & DenOpF; @@ -64,40 +60,31 @@ NAMESPACE_BEGIN(Grid); ConjugateGradientMultiShift msCG(MaxIter, approx); msCG(schurOp,in, out); #else - SchurDifferentiableOperator schurOpD2(numerator ? NumOpD2 : DenOpD2); + SchurDifferentiableOperator schurOpD(numerator ? NumOpD : DenOpD); SchurDifferentiableOperator schurOpF(numerator ? NumOpF : DenOpF); - FermionFieldD2 inD2(NumOpD2.FermionRedBlackGrid()); - FermionFieldD2 outD2(NumOpD2.FermionRedBlackGrid()); + FermionFieldD inD(NumOpD.FermionRedBlackGrid()); + FermionFieldD outD(NumOpD.FermionRedBlackGrid()); // Action better with higher precision? - ConjugateGradientMultiShiftMixedPrec msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); - precisionChange(inD2,in); - std::cout << "msCG single solve "< msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); + msCG(schurOpD, in, out); #endif } virtual void multiShiftInverse(bool numerator, const MultiShiftFunction &approx, const Integer MaxIter, const FermionFieldD &in, std::vector &out_elems, FermionFieldD &out){ - SchurDifferentiableOperator schurOpD2(numerator ? NumOpD2 : DenOpD2); + SchurDifferentiableOperator schurOpD(numerator ? NumOpD : DenOpD); SchurDifferentiableOperator schurOpF (numerator ? NumOpF : DenOpF); - FermionFieldD2 inD2(NumOpD2.FermionRedBlackGrid()); - FermionFieldD2 outD2(NumOpD2.FermionRedBlackGrid()); - std::vector out_elemsD2(out_elems.size(),NumOpD2.FermionRedBlackGrid()); - ConjugateGradientMultiShiftMixedPrecCleanup msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); - precisionChange(inD2,in); - std::cout << "msCG in "< out_elemsD(out_elems.size(),NumOpD.FermionRedBlackGrid()); + ConjugateGradientMultiShiftMixedPrecCleanup msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); + msCG(schurOpD, in, out_elems, out); } //Allow derived classes to override the gauge import virtual void ImportGauge(const typename ImplD::GaugeField &Ud){ typename ImplF::GaugeField Uf(NumOpF.GaugeGrid()); - typename ImplD2::GaugeField Ud2(NumOpD2.GaugeGrid()); + typename ImplD::GaugeField Ud2(NumOpD.GaugeGrid()); precisionChange(Uf, Ud); precisionChange(Ud2, Ud); @@ -109,20 +96,18 @@ NAMESPACE_BEGIN(Grid); NumOpF.ImportGauge(Uf); DenOpF.ImportGauge(Uf); - NumOpD2.ImportGauge(Ud2); - DenOpD2.ImportGauge(Ud2); + NumOpD.ImportGauge(Ud2); + DenOpD.ImportGauge(Ud2); } public: GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction(FermionOperator &_NumOpD, FermionOperator &_DenOpD, FermionOperator &_NumOpF, FermionOperator &_DenOpF, - FermionOperator &_NumOpD2, FermionOperator &_DenOpD2, const RationalActionParams & p, Integer _ReliableUpdateFreq ) : GeneralEvenOddRatioRationalPseudoFermionAction(_NumOpD, _DenOpD, p), ReliableUpdateFreq(_ReliableUpdateFreq), NumOpD(_NumOpD), DenOpD(_DenOpD), - NumOpF(_NumOpF), DenOpF(_DenOpF), - NumOpD2(_NumOpD2), DenOpD2(_DenOpD2) + NumOpF(_NumOpF), DenOpF(_DenOpF) {} virtual std::string action_name(){return "GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction";} diff --git a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h index 302c3a5d..d1e5b69d 100644 --- a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h +++ b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h @@ -67,9 +67,9 @@ NAMESPACE_BEGIN(Grid); virtual std::string action_name(){return "OneFlavourEvenOddRatioRationalPseudoFermionAction";} }; - template + template class OneFlavourEvenOddRatioRationalMixedPrecPseudoFermionAction - : public GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction { + : public GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction { public: typedef OneFlavourRationalParams Params; private: @@ -91,11 +91,9 @@ NAMESPACE_BEGIN(Grid); FermionOperator &_DenOp, FermionOperator &_NumOpF, FermionOperator &_DenOpF, - FermionOperator &_NumOpD2, - FermionOperator &_DenOpD2, const Params & p, Integer ReliableUpdateFreq ) : - GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction(_NumOp, _DenOp,_NumOpF, _DenOpF,_NumOpD2, _DenOpD2, transcribe(p),ReliableUpdateFreq){} + GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction(_NumOp, _DenOp,_NumOpF, _DenOpF, transcribe(p),ReliableUpdateFreq){} virtual std::string action_name(){return "OneFlavourEvenOddRatioRationalPseudoFermionAction";} }; diff --git a/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc b/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc index 5572d11f..792b23f9 100644 --- a/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc +++ b/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc @@ -451,7 +451,7 @@ int main(int argc, char **argv) { #define MIXED_PRECISION #ifdef MIXED_PRECISION - std::vector *> Bdys; + std::vector *> Bdys; #else std::vector *> Bdys; #endif @@ -526,15 +526,13 @@ int main(int argc, char **argv) { Quotients.push_back (new TwoFlavourEvenOddRatioPseudoFermionAction(*Numerators[h],*Denominators[h],*MPCG[h],*ActionMPCG[h],CG)); } else { #ifdef MIXED_PRECISION - Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( + Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( *Numerators[h],*Denominators[h], *NumeratorsF[h],*DenominatorsF[h], - *Numerators[h],*Denominators[h], OFRp, SP_iters) ); - Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( + Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( *Numerators[h],*Denominators[h], *NumeratorsF[h],*DenominatorsF[h], - *Numerators[h],*Denominators[h], OFRp, SP_iters) ); #else Bdys.push_back( new GeneralEvenOddRatioRationalPseudoFermionAction(*Numerators[h],*Denominators[h],OFRp)); diff --git a/HMC/Mobius2p1f_EOFA_96I_hmc.cc b/HMC/Mobius2p1f_EOFA_96I_hmc.cc index 6e7fb3cd..f4c7db79 100644 --- a/HMC/Mobius2p1f_EOFA_96I_hmc.cc +++ b/HMC/Mobius2p1f_EOFA_96I_hmc.cc @@ -164,11 +164,6 @@ int main(int argc, char **argv) { typedef MobiusEOFAFermionF FermionEOFAActionF; typedef typename FermionActionF::FermionField FermionFieldF; - typedef WilsonImplD2 FermionImplPolicyD2; - typedef MobiusFermionD2 FermionActionD2; - typedef MobiusEOFAFermionD2 FermionEOFAActionD2; - typedef typename FermionActionD2::FermionField FermionFieldD2; - typedef Grid::XmlReader Serialiser; //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: @@ -272,7 +267,6 @@ int main(int argc, char **argv) { // temporarily need a gauge field LatticeGaugeFieldD U(GridPtr); U=Zero(); LatticeGaugeFieldF UF(GridPtrF); UF=Zero(); - LatticeGaugeFieldD2 UD2(GridPtrF); UD2=Zero(); std::cout << GridLogMessage << " Running the HMC "<< std::endl; TheHMC.ReadCommandLine(argc,argv); // params on CML or from param file @@ -394,15 +388,13 @@ int main(int argc, char **argv) { std::vector Denominators; std::vector NumeratorsF; std::vector DenominatorsF; - std::vector NumeratorsD2; - std::vector DenominatorsD2; std::vector *> Quotients; std::vector ActionMPCG; std::vector MPCG; #define MIXED_PRECISION #ifdef MIXED_PRECISION - std::vector *> Bdys; + std::vector *> Bdys; #else std::vector *> Bdys; #endif diff --git a/systems/Crusher/sourceme.sh b/systems/Crusher/sourceme.sh index ac248b93..3fc39c63 100644 --- a/systems/Crusher/sourceme.sh +++ b/systems/Crusher/sourceme.sh @@ -5,8 +5,8 @@ module load emacs #module load gperftools module load PrgEnv-gnu module load rocm/5.3.0 -module load cray-mpich/8.1.16 -#module load cray-mpich/8.1.17 +#module load cray-mpich/8.1.16 +module load cray-mpich/8.1.17 module load gmp module load cray-fftw module load craype-accel-amd-gfx90a From 18ce23aa75f366269eb01e705a9301648b87c26b Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Thu, 6 Apr 2023 11:30:48 +0100 Subject: [PATCH 3/6] Fix NEON SIMD --- Grid/simd/Grid_neon.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Grid/simd/Grid_neon.h b/Grid/simd/Grid_neon.h index f53575cb..16ee4f80 100644 --- a/Grid/simd/Grid_neon.h +++ b/Grid/simd/Grid_neon.h @@ -320,7 +320,7 @@ struct Conj{ struct TimesMinusI{ //Complex single - inline float32x4_t operator()(float32x4_t in, float32x4_t ret){ + inline float32x4_t operator()(float32x4_t in){ // ar ai br bi -> ai -ar ai -br float32x4_t r0, r1; r0 = vnegq_f32(in); // -ar -ai -br -bi @@ -328,7 +328,7 @@ struct TimesMinusI{ return vtrn1q_f32(r1, r0); // ar -ai br -bi } //Complex double - inline float64x2_t operator()(float64x2_t in, float64x2_t ret){ + inline float64x2_t operator()(float64x2_t in){ // a ib -> b -ia float64x2_t tmp; tmp = vnegq_f64(in); @@ -338,7 +338,7 @@ struct TimesMinusI{ struct TimesI{ //Complex single - inline float32x4_t operator()(float32x4_t in, float32x4_t ret){ + inline float32x4_t operator()(float32x4_t in){ // ar ai br bi -> -ai ar -bi br float32x4_t r0, r1; r0 = vnegq_f32(in); // -ar -ai -br -bi @@ -346,7 +346,7 @@ struct TimesI{ return vtrn1q_f32(r1, in); // -ai ar -bi br } //Complex double - inline float64x2_t operator()(float64x2_t in, float64x2_t ret){ + inline float64x2_t operator()(float64x2_t in){ // a ib -> -b ia float64x2_t tmp; tmp = vnegq_f64(in); From bd76b47fbf63f62b6a6191bb1a8a2577eb1baba3 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Fri, 7 Apr 2023 11:44:48 +0100 Subject: [PATCH 4/6] Update CI badge in README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4af52d78..ad1494d8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Grid [![Teamcity status](http://ci.cliath.ph.ed.ac.uk/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:GridBasedSoftware_Grid)),branch:name:develop)/statusIcon.svg)](http://ci.cliath.ph.ed.ac.uk/project.html?projectId=GridBasedSoftware_Grid&tab=projectOverview) +# Grid + +[![Teamcity status](https://ci.dev.dirac.ed.ac.uk/guestAuth/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:GridBasedSoftware_Grid)),branch:default:true)/statusIcon.svg)](https://ci.dev.dirac.ed.ac.uk/project/GridBasedSoftware_Grid?mode=builds) **Data parallel C++ mathematical object library.** From 4072408b6fdd155737aaeaa697340ec02b588b83 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Fri, 7 Apr 2023 11:45:28 +0100 Subject: [PATCH 5/6] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ad1494d8..29b99671 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ # Grid +**Data parallel C++ mathematical object library.** [![Teamcity status](https://ci.dev.dirac.ed.ac.uk/guestAuth/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:GridBasedSoftware_Grid)),branch:default:true)/statusIcon.svg)](https://ci.dev.dirac.ed.ac.uk/project/GridBasedSoftware_Grid?mode=builds) -**Data parallel C++ mathematical object library.** - License: GPL v2. Last update June 2017. From f9b41a84d2a816f52df8335f44d27c271731ffc2 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Fri, 7 Apr 2023 15:42:45 -0400 Subject: [PATCH 6/6] Trajectory runs to completion on Crusher within wall clock time --- .../ConjugateGradientMultiShiftCleanup.h | 16 ++++++++-------- .../ConjugateGradientMultiShiftMixedPrec.h | 2 +- .../GeneralEvenOddRationalRatioMixedPrec.h | 13 +++++-------- HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc | 8 ++++---- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/Grid/algorithms/iterative/ConjugateGradientMultiShiftCleanup.h b/Grid/algorithms/iterative/ConjugateGradientMultiShiftCleanup.h index 17714f09..23baff61 100644 --- a/Grid/algorithms/iterative/ConjugateGradientMultiShiftCleanup.h +++ b/Grid/algorithms/iterative/ConjugateGradientMultiShiftCleanup.h @@ -166,16 +166,16 @@ public: rsqf[s] =rsq[s]; std::cout< schurOp(numerator ? NumOpD : DenOpD); ConjugateGradientMultiShift msCG(MaxIter, approx); msCG(schurOp,in, out); @@ -70,9 +71,10 @@ NAMESPACE_BEGIN(Grid); msCG(schurOpD, in, out); #endif } + //Force evaluation virtual void multiShiftInverse(bool numerator, const MultiShiftFunction &approx, const Integer MaxIter, const FermionFieldD &in, std::vector &out_elems, FermionFieldD &out){ SchurDifferentiableOperator schurOpD(numerator ? NumOpD : DenOpD); - SchurDifferentiableOperator schurOpF (numerator ? NumOpF : DenOpF); + SchurDifferentiableOperator schurOpF(numerator ? NumOpF : DenOpF); FermionFieldD inD(NumOpD.FermionRedBlackGrid()); FermionFieldD outD(NumOpD.FermionRedBlackGrid()); @@ -84,20 +86,15 @@ NAMESPACE_BEGIN(Grid); virtual void ImportGauge(const typename ImplD::GaugeField &Ud){ typename ImplF::GaugeField Uf(NumOpF.GaugeGrid()); - typename ImplD::GaugeField Ud2(NumOpD.GaugeGrid()); precisionChange(Uf, Ud); - precisionChange(Ud2, Ud); - std::cout << "Importing "< hasenbusch({ light_mass, 0.005, 0.0145, 0.045, 0.108, 0.25, 0.51 , pv_mass }); // Updated // std::vector hasenbusch({ light_mass, 0.0145, 0.045, 0.108, 0.25, 0.51 , 0.75 , pv_mass }); - int SP_iters=10000; + int SP_iters=9000; RationalActionParams OFRp; // Up/down OFRp.lo = 6.0e-5; @@ -362,12 +362,12 @@ int main(int argc, char **argv) { // Probably dominates the force - back to EOFA. OneFlavourRationalParams SFRp; - SFRp.lo = 0.25; + SFRp.lo = 0.1; SFRp.hi = 25.0; SFRp.MaxIter = 10000; - SFRp.tolerance= 1.0e-5; + SFRp.tolerance= 1.0e-8; SFRp.mdtolerance= 2.0e-4; - SFRp.degree = 8; + SFRp.degree = 12; SFRp.precision= 50; MobiusEOFAFermionD Strange_Op_L (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);