diff --git a/Grid/qcd/hmc/HMC.h b/Grid/qcd/hmc/HMC.h index d4739fb0..c56ae6ad 100644 --- a/Grid/qcd/hmc/HMC.h +++ b/Grid/qcd/hmc/HMC.h @@ -283,12 +283,13 @@ public: std::cout << GridLogHMC << "Total time for trajectory (s): " << (t1-t0)/1e6 << std::endl; TheIntegrator.print_timer(); - + + TheIntegrator.Smearer.set_Field(Ucur); for (int obs = 0; obs < Observables.size(); obs++) { std::cout << GridLogDebug << "Observables # " << obs << std::endl; std::cout << GridLogDebug << "Observables total " << Observables.size() << std::endl; std::cout << GridLogDebug << "Observables pointer " << Observables[obs] << std::endl; - Observables[obs]->TrajectoryComplete(traj + 1, Ucur, sRNG, pRNG); + Observables[obs]->TrajectoryComplete(traj + 1, TheIntegrator.Smearer, sRNG, pRNG); } std::cout << GridLogHMC << ":::::::::::::::::::::::::::::::::::::::::::" << std::endl; } diff --git a/Grid/qcd/hmc/checkpointers/BaseCheckpointer.h b/Grid/qcd/hmc/checkpointers/BaseCheckpointer.h index c09fdeeb..72f5e39b 100644 --- a/Grid/qcd/hmc/checkpointers/BaseCheckpointer.h +++ b/Grid/qcd/hmc/checkpointers/BaseCheckpointer.h @@ -35,13 +35,16 @@ class CheckpointerParameters : Serializable { public: GRID_SERIALIZABLE_CLASS_MEMBERS(CheckpointerParameters, std::string, config_prefix, + std::string, smeared_prefix, std::string, rng_prefix, int, saveInterval, + bool, saveSmeared, std::string, format, ); - CheckpointerParameters(std::string cf = "cfg", std::string rn = "rng", + CheckpointerParameters(std::string cf = "cfg", std::string sf="cfg_smr" , std::string rn = "rng", int savemodulo = 1, const std::string &f = "IEEE64BIG") : config_prefix(cf), + smeared_prefix(sf), rng_prefix(rn), saveInterval(savemodulo), format(f){}; @@ -61,13 +64,21 @@ template class BaseHmcCheckpointer : public HmcObservable { public: void build_filenames(int traj, CheckpointerParameters &Params, - std::string &conf_file, std::string &rng_file) { + std::string &conf_file, + std::string &smear_file, + std::string &rng_file) { { std::ostringstream os; os << Params.rng_prefix << "." << traj; rng_file = os.str(); } + { + std::ostringstream os; + os << Params.smeared_prefix << "." << traj; + smear_file = os.str(); + } + { std::ostringstream os; os << Params.config_prefix << "." << traj; @@ -84,6 +95,11 @@ public: } virtual void initialize(const CheckpointerParameters &Params) = 0; + virtual void TrajectoryComplete(int traj, + typename Impl::Field &U, + GridSerialRNG &sRNG, + GridParallelRNG &pRNG) { assert(0); } ; // HMC should pass the smart config with smeared and unsmeared + virtual void CheckpointRestore(int traj, typename Impl::Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) = 0; diff --git a/Grid/qcd/hmc/checkpointers/BinaryCheckpointer.h b/Grid/qcd/hmc/checkpointers/BinaryCheckpointer.h index ef9e6194..a91ec67f 100644 --- a/Grid/qcd/hmc/checkpointers/BinaryCheckpointer.h +++ b/Grid/qcd/hmc/checkpointers/BinaryCheckpointer.h @@ -61,11 +61,14 @@ public: fout.close(); } - void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { + void TrajectoryComplete(int traj, + ConfigurationBase &SmartConfig, + GridSerialRNG &sRNG, GridParallelRNG &pRNG) + { if ((traj % Params.saveInterval) == 0) { - std::string config, rng; - this->build_filenames(traj, Params, config, rng); + std::string config, rng, smr; + this->build_filenames(traj, Params, config, smr, rng); uint32_t nersc_csum; uint32_t scidac_csuma; @@ -74,9 +77,15 @@ public: BinarySimpleUnmunger munge; truncate(rng); BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); - truncate(config); + std::cout << GridLogMessage << "Written Binary RNG " << rng + << " checksum " << std::hex + << nersc_csum <<"/" + << scidac_csuma <<"/" + << scidac_csumb + << std::dec << std::endl; - BinaryIO::writeLatticeObject(U, config, munge, 0, Params.format, + truncate(config); + BinaryIO::writeLatticeObject(SmartConfig.get_U(false), config, munge, 0, Params.format, nersc_csum,scidac_csuma,scidac_csumb); std::cout << GridLogMessage << "Written Binary Configuration " << config @@ -85,6 +94,18 @@ public: << scidac_csuma <<"/" << scidac_csumb << std::dec << std::endl; + + if ( Params.saveSmeared ) { + truncate(smr); + BinaryIO::writeLatticeObject(SmartConfig.get_U(true), smr, munge, 0, Params.format, + nersc_csum,scidac_csuma,scidac_csumb); + std::cout << GridLogMessage << "Written Binary Smeared Configuration " << smr + << " checksum " << std::hex + << nersc_csum <<"/" + << scidac_csuma <<"/" + << scidac_csumb + << std::dec << std::endl; + } } }; diff --git a/Grid/qcd/hmc/checkpointers/ILDGCheckpointer.h b/Grid/qcd/hmc/checkpointers/ILDGCheckpointer.h index 1bb8aa1a..12d391ef 100644 --- a/Grid/qcd/hmc/checkpointers/ILDGCheckpointer.h +++ b/Grid/qcd/hmc/checkpointers/ILDGCheckpointer.h @@ -69,17 +69,27 @@ public: } } - void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG, + void TrajectoryComplete(int traj, + ConfigurationBase &SmartConfig, + GridSerialRNG &sRNG, GridParallelRNG &pRNG) { if ((traj % Params.saveInterval) == 0) { - std::string config, rng; + std::string config, rng, smr; this->build_filenames(traj, Params, config, rng); - GridBase *grid = U.Grid(); + GridBase *grid = SmartConfig.get_U(false).Grid(); uint32_t nersc_csum,scidac_csuma,scidac_csumb; BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); + std::cout << GridLogMessage << "Written BINARY RNG " << rng + << " checksum " << std::hex + << nersc_csum<<"/" + << scidac_csuma<<"/" + << scidac_csumb + << std::dec << std::endl; + + IldgWriter _IldgWriter(grid->IsBoss()); _IldgWriter.open(config); - _IldgWriter.writeConfiguration(U, traj, config, config); + _IldgWriter.writeConfiguration(SmartConfig.get_U(false), traj, config, config); _IldgWriter.close(); std::cout << GridLogMessage << "Written ILDG Configuration on " << config @@ -88,6 +98,21 @@ public: << scidac_csuma<<"/" << scidac_csumb << std::dec << std::endl; + + if ( Params.saveSmeared ) { + IldgWriter _IldgWriter(grid->IsBoss()); + _IldgWriter.open(smr); + _IldgWriter.writeConfiguration(SmartConfig.get_U(true), traj, config, config); + _IldgWriter.close(); + + std::cout << GridLogMessage << "Written ILDG Configuration on " << smr + << " checksum " << std::hex + << nersc_csum<<"/" + << scidac_csuma<<"/" + << scidac_csumb + << std::dec << std::endl; + } + } }; diff --git a/Grid/qcd/hmc/checkpointers/NerscCheckpointer.h b/Grid/qcd/hmc/checkpointers/NerscCheckpointer.h index 4534e4c4..79424300 100644 --- a/Grid/qcd/hmc/checkpointers/NerscCheckpointer.h +++ b/Grid/qcd/hmc/checkpointers/NerscCheckpointer.h @@ -52,23 +52,29 @@ public: Params.format = "IEEE64BIG"; // fixed, overwrite any other choice } - void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG, - GridParallelRNG &pRNG) { + virtual void TrajectoryComplete(int traj, + ConfigurationBase &SmartConfig, + GridSerialRNG &sRNG, + GridParallelRNG &pRNG) + { if ((traj % Params.saveInterval) == 0) { - std::string config, rng; - this->build_filenames(traj, Params, config, rng); - + std::string config, rng, smr; + this->build_filenames(traj, Params, config, smr, rng); + int precision32 = 1; int tworow = 0; NerscIO::writeRNGState(sRNG, pRNG, rng); - NerscIO::writeConfiguration(U, config, tworow, precision32); + NerscIO::writeConfiguration(SmartConfig.get_U(false), config, tworow, precision32); + if ( Params.saveSmeared ) { + NerscIO::writeConfiguration(SmartConfig.get_U(true), smr, tworow, precision32); + } } }; void CheckpointRestore(int traj, GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { - std::string config, rng; - this->build_filenames(traj, Params, config, rng); + std::string config, rng, smr; + this->build_filenames(traj, Params, config, smr, rng ); this->check_filename(rng); this->check_filename(config); diff --git a/Grid/qcd/hmc/checkpointers/ScidacCheckpointer.h b/Grid/qcd/hmc/checkpointers/ScidacCheckpointer.h index 986585ea..259a44b0 100644 --- a/Grid/qcd/hmc/checkpointers/ScidacCheckpointer.h +++ b/Grid/qcd/hmc/checkpointers/ScidacCheckpointer.h @@ -70,19 +70,37 @@ class ScidacHmcCheckpointer : public BaseHmcCheckpointer { } } - void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, + void TrajectoryComplete(int traj, + ConfigurationBase &SmartConfig, + GridSerialRNG &sRNG, GridParallelRNG &pRNG) { if ((traj % Params.saveInterval) == 0) { - std::string config, rng; - this->build_filenames(traj, Params, config, rng); - GridBase *grid = U.Grid(); + std::string config, rng,smr; + this->build_filenames(traj, Params, config, smr, rng); + GridBase *grid = SmartConfig.get_U(false).Grid(); uint32_t nersc_csum,scidac_csuma,scidac_csumb; BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); - ScidacWriter _ScidacWriter(grid->IsBoss()); - _ScidacWriter.open(config); - _ScidacWriter.writeScidacFieldRecord(U, MData); - _ScidacWriter.close(); + std::cout << GridLogMessage << "Written Binary RNG " << rng + << " checksum " << std::hex + << nersc_csum <<"/" + << scidac_csuma <<"/" + << scidac_csumb + << std::dec << std::endl; + + { + ScidacWriter _ScidacWriter(grid->IsBoss()); + _ScidacWriter.open(config); + _ScidacWriter.writeScidacFieldRecord(SmartConfig.get_U(false), MData); + _ScidacWriter.close(); + } + + if ( Params.saveSmeared ) { + ScidacWriter _ScidacWriter(grid->IsBoss()); + _ScidacWriter.open(smr); + _ScidacWriter.writeScidacFieldRecord(SmartConfig.get_U(true), MData); + _ScidacWriter.close(); + } std::cout << GridLogMessage << "Written Scidac Configuration on " << config << std::endl; } }; diff --git a/Grid/qcd/hmc/integrators/Integrator.h b/Grid/qcd/hmc/integrators/Integrator.h index 4f3d8f32..4dd5a634 100644 --- a/Grid/qcd/hmc/integrators/Integrator.h +++ b/Grid/qcd/hmc/integrators/Integrator.h @@ -132,14 +132,10 @@ public: double start_force = usecond(); - std::cout << GridLogMessage << "AuditForce["<deriv_timer_start(); as[level].actions.at(a)->deriv(Smearer, force); // deriv should NOT include Ta as[level].actions.at(a)->deriv_timer_stop(); - std::cout << GridLogMessage << "AuditForce["<action_name(); force = FieldImplementation::projectForce(force); // Ta for gauge fields diff --git a/Grid/qcd/observables/hmc_observable.h b/Grid/qcd/observables/hmc_observable.h index c28c376d..277ded28 100644 --- a/Grid/qcd/observables/hmc_observable.h +++ b/Grid/qcd/observables/hmc_observable.h @@ -34,6 +34,13 @@ NAMESPACE_BEGIN(Grid); template class HmcObservable { public: + virtual void TrajectoryComplete(int traj, + ConfigurationBase &SmartConfig, + GridSerialRNG &sRNG, + GridParallelRNG &pRNG) + { + TrajectoryComplete(traj,SmartConfig.get_U(false),sRNG,pRNG); // Unsmeared observable + }; virtual void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, diff --git a/Grid/qcd/observables/plaquette.h b/Grid/qcd/observables/plaquette.h index f038e455..9f67c7cb 100644 --- a/Grid/qcd/observables/plaquette.h +++ b/Grid/qcd/observables/plaquette.h @@ -42,6 +42,18 @@ public: // necessary for HmcObservable compatibility typedef typename Impl::Field Field; + virtual void TrajectoryComplete(int traj, + ConfigurationBase &SmartConfig, + GridSerialRNG &sRNG, + GridParallelRNG &pRNG) + { + std::cout << GridLogMessage << "+++++++++++++++++++"<