From 6252ffaf76b0f841c35321a77a1330e61214deeb Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 3 Apr 2023 18:25:22 -0400 Subject: [PATCH 1/2] No unified --- systems/Crusher/config-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/Crusher/config-command b/systems/Crusher/config-command index d310ff55..2176fc6b 100644 --- a/systems/Crusher/config-command +++ b/systems/Crusher/config-command @@ -1,7 +1,7 @@ CLIME=`spack find --paths c-lime@2-3-9 | grep c-lime| cut -c 15-` ../../configure --enable-comms=mpi-auto \ --with-lime=$CLIME \ ---enable-unified=yes \ +--enable-unified=no \ --enable-shm=nvlink \ --enable-tracing=timer \ --enable-accelerator=hip \ From 6b979f0a69b2c460c6cad9d9782b77467f61cd66 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 4 Apr 2023 23:13:17 -0400 Subject: [PATCH 2/2] Dirichlet improvements that I failed to commit --- .../GeneralEvenOddRationalRatioMixedPrec.h | 47 +++++++------------ .../OneFlavourEvenOddRationalRatio.h | 8 ++-- HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc | 8 ++-- HMC/Mobius2p1f_EOFA_96I_hmc.cc | 10 +--- systems/Crusher/sourceme.sh | 4 +- 5 files changed, 25 insertions(+), 52 deletions(-) diff --git a/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h b/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h index 6fe32e15..104e3376 100644 --- a/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h +++ b/Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h @@ -38,19 +38,15 @@ NAMESPACE_BEGIN(Grid); // cf. GeneralEvenOddRational.h for details ///////////////////////////////////////////////////////////////////////////////////////////////////////////// - template + template class GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction : public GeneralEvenOddRatioRationalPseudoFermionAction { private: - typedef typename ImplD2::FermionField FermionFieldD2; typedef typename ImplD::FermionField FermionFieldD; typedef typename ImplF::FermionField FermionFieldF; FermionOperator & NumOpD; FermionOperator & DenOpD; - FermionOperator & NumOpD2; - FermionOperator & DenOpD2; - FermionOperator & NumOpF; FermionOperator & DenOpF; @@ -64,40 +60,31 @@ NAMESPACE_BEGIN(Grid); ConjugateGradientMultiShift msCG(MaxIter, approx); msCG(schurOp,in, out); #else - SchurDifferentiableOperator schurOpD2(numerator ? NumOpD2 : DenOpD2); + SchurDifferentiableOperator schurOpD(numerator ? NumOpD : DenOpD); SchurDifferentiableOperator schurOpF(numerator ? NumOpF : DenOpF); - FermionFieldD2 inD2(NumOpD2.FermionRedBlackGrid()); - FermionFieldD2 outD2(NumOpD2.FermionRedBlackGrid()); + FermionFieldD inD(NumOpD.FermionRedBlackGrid()); + FermionFieldD outD(NumOpD.FermionRedBlackGrid()); // Action better with higher precision? - ConjugateGradientMultiShiftMixedPrec msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); - precisionChange(inD2,in); - std::cout << "msCG single solve "< msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); + msCG(schurOpD, in, out); #endif } virtual void multiShiftInverse(bool numerator, const MultiShiftFunction &approx, const Integer MaxIter, const FermionFieldD &in, std::vector &out_elems, FermionFieldD &out){ - SchurDifferentiableOperator schurOpD2(numerator ? NumOpD2 : DenOpD2); + SchurDifferentiableOperator schurOpD(numerator ? NumOpD : DenOpD); SchurDifferentiableOperator schurOpF (numerator ? NumOpF : DenOpF); - FermionFieldD2 inD2(NumOpD2.FermionRedBlackGrid()); - FermionFieldD2 outD2(NumOpD2.FermionRedBlackGrid()); - std::vector out_elemsD2(out_elems.size(),NumOpD2.FermionRedBlackGrid()); - ConjugateGradientMultiShiftMixedPrecCleanup msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); - precisionChange(inD2,in); - std::cout << "msCG in "< out_elemsD(out_elems.size(),NumOpD.FermionRedBlackGrid()); + ConjugateGradientMultiShiftMixedPrecCleanup msCG(MaxIter, approx, NumOpF.FermionRedBlackGrid(), schurOpF, ReliableUpdateFreq); + msCG(schurOpD, in, out_elems, out); } //Allow derived classes to override the gauge import virtual void ImportGauge(const typename ImplD::GaugeField &Ud){ typename ImplF::GaugeField Uf(NumOpF.GaugeGrid()); - typename ImplD2::GaugeField Ud2(NumOpD2.GaugeGrid()); + typename ImplD::GaugeField Ud2(NumOpD.GaugeGrid()); precisionChange(Uf, Ud); precisionChange(Ud2, Ud); @@ -109,20 +96,18 @@ NAMESPACE_BEGIN(Grid); NumOpF.ImportGauge(Uf); DenOpF.ImportGauge(Uf); - NumOpD2.ImportGauge(Ud2); - DenOpD2.ImportGauge(Ud2); + NumOpD.ImportGauge(Ud2); + DenOpD.ImportGauge(Ud2); } public: GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction(FermionOperator &_NumOpD, FermionOperator &_DenOpD, FermionOperator &_NumOpF, FermionOperator &_DenOpF, - FermionOperator &_NumOpD2, FermionOperator &_DenOpD2, const RationalActionParams & p, Integer _ReliableUpdateFreq ) : GeneralEvenOddRatioRationalPseudoFermionAction(_NumOpD, _DenOpD, p), ReliableUpdateFreq(_ReliableUpdateFreq), NumOpD(_NumOpD), DenOpD(_DenOpD), - NumOpF(_NumOpF), DenOpF(_DenOpF), - NumOpD2(_NumOpD2), DenOpD2(_DenOpD2) + NumOpF(_NumOpF), DenOpF(_DenOpF) {} virtual std::string action_name(){return "GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction";} diff --git a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h index 302c3a5d..d1e5b69d 100644 --- a/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h +++ b/Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h @@ -67,9 +67,9 @@ NAMESPACE_BEGIN(Grid); virtual std::string action_name(){return "OneFlavourEvenOddRatioRationalPseudoFermionAction";} }; - template + template class OneFlavourEvenOddRatioRationalMixedPrecPseudoFermionAction - : public GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction { + : public GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction { public: typedef OneFlavourRationalParams Params; private: @@ -91,11 +91,9 @@ NAMESPACE_BEGIN(Grid); FermionOperator &_DenOp, FermionOperator &_NumOpF, FermionOperator &_DenOpF, - FermionOperator &_NumOpD2, - FermionOperator &_DenOpD2, const Params & p, Integer ReliableUpdateFreq ) : - GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction(_NumOp, _DenOp,_NumOpF, _DenOpF,_NumOpD2, _DenOpD2, transcribe(p),ReliableUpdateFreq){} + GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction(_NumOp, _DenOp,_NumOpF, _DenOpF, transcribe(p),ReliableUpdateFreq){} virtual std::string action_name(){return "OneFlavourEvenOddRatioRationalPseudoFermionAction";} }; diff --git a/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc b/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc index 5572d11f..792b23f9 100644 --- a/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc +++ b/HMC/Mobius2p1f_DD_EOFA_96I_mshift.cc @@ -451,7 +451,7 @@ int main(int argc, char **argv) { #define MIXED_PRECISION #ifdef MIXED_PRECISION - std::vector *> Bdys; + std::vector *> Bdys; #else std::vector *> Bdys; #endif @@ -526,15 +526,13 @@ int main(int argc, char **argv) { Quotients.push_back (new TwoFlavourEvenOddRatioPseudoFermionAction(*Numerators[h],*Denominators[h],*MPCG[h],*ActionMPCG[h],CG)); } else { #ifdef MIXED_PRECISION - Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( + Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( *Numerators[h],*Denominators[h], *NumeratorsF[h],*DenominatorsF[h], - *Numerators[h],*Denominators[h], OFRp, SP_iters) ); - Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( + Bdys.push_back( new GeneralEvenOddRatioRationalMixedPrecPseudoFermionAction( *Numerators[h],*Denominators[h], *NumeratorsF[h],*DenominatorsF[h], - *Numerators[h],*Denominators[h], OFRp, SP_iters) ); #else Bdys.push_back( new GeneralEvenOddRatioRationalPseudoFermionAction(*Numerators[h],*Denominators[h],OFRp)); diff --git a/HMC/Mobius2p1f_EOFA_96I_hmc.cc b/HMC/Mobius2p1f_EOFA_96I_hmc.cc index 6e7fb3cd..f4c7db79 100644 --- a/HMC/Mobius2p1f_EOFA_96I_hmc.cc +++ b/HMC/Mobius2p1f_EOFA_96I_hmc.cc @@ -164,11 +164,6 @@ int main(int argc, char **argv) { typedef MobiusEOFAFermionF FermionEOFAActionF; typedef typename FermionActionF::FermionField FermionFieldF; - typedef WilsonImplD2 FermionImplPolicyD2; - typedef MobiusFermionD2 FermionActionD2; - typedef MobiusEOFAFermionD2 FermionEOFAActionD2; - typedef typename FermionActionD2::FermionField FermionFieldD2; - typedef Grid::XmlReader Serialiser; //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: @@ -272,7 +267,6 @@ int main(int argc, char **argv) { // temporarily need a gauge field LatticeGaugeFieldD U(GridPtr); U=Zero(); LatticeGaugeFieldF UF(GridPtrF); UF=Zero(); - LatticeGaugeFieldD2 UD2(GridPtrF); UD2=Zero(); std::cout << GridLogMessage << " Running the HMC "<< std::endl; TheHMC.ReadCommandLine(argc,argv); // params on CML or from param file @@ -394,15 +388,13 @@ int main(int argc, char **argv) { std::vector Denominators; std::vector NumeratorsF; std::vector DenominatorsF; - std::vector NumeratorsD2; - std::vector DenominatorsD2; std::vector *> Quotients; std::vector ActionMPCG; std::vector MPCG; #define MIXED_PRECISION #ifdef MIXED_PRECISION - std::vector *> Bdys; + std::vector *> Bdys; #else std::vector *> Bdys; #endif diff --git a/systems/Crusher/sourceme.sh b/systems/Crusher/sourceme.sh index ac248b93..3fc39c63 100644 --- a/systems/Crusher/sourceme.sh +++ b/systems/Crusher/sourceme.sh @@ -5,8 +5,8 @@ module load emacs #module load gperftools module load PrgEnv-gnu module load rocm/5.3.0 -module load cray-mpich/8.1.16 -#module load cray-mpich/8.1.17 +#module load cray-mpich/8.1.16 +module load cray-mpich/8.1.17 module load gmp module load cray-fftw module load craype-accel-amd-gfx90a