From 4fe182e5a7c4b1d1dddc022706a71f1c0432cda5 Mon Sep 17 00:00:00 2001 From: Guido Cossu Date: Sun, 6 Aug 2017 10:46:19 +0100 Subject: [PATCH] Added high level HMC support for overriding default SIMD lane decomposition --- lib/cartesian/Cartesian_base.h | 23 ++++---- lib/qcd/hmc/HMCResourceManager.h | 14 ++++- lib/qcd/hmc/HMC_GridModules.h | 92 +++++++++++++++++++++-------- lib/util/Init.cc | 2 +- tests/hmc/Test_hmc_EOMobiusRatio.cc | 13 ++-- 5 files changed, 98 insertions(+), 46 deletions(-) diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h index 0db6ce0d..f4f9a269 100644 --- a/lib/cartesian/Cartesian_base.h +++ b/lib/cartesian/Cartesian_base.h @@ -185,17 +185,18 @@ public: //////////////////////////////////////////////////////////////// void show_decomposition(){ - std::cout << GridLogMessage << "Full Dimensions : " << _fdimensions << std::endl; - std::cout << GridLogMessage << "Global Dimensions : " << _gdimensions << std::endl; - std::cout << GridLogMessage << "Local Dimensions : " << _ldimensions << std::endl; - std::cout << GridLogMessage << "Reduced Dimensions : " << _rdimensions << std::endl; - std::cout << GridLogMessage << "Outer strides : " << _ostride << std::endl; - std::cout << GridLogMessage << "Inner strides : " << _istride << std::endl; - std::cout << GridLogMessage << "iSites : " << _isites << std::endl; - std::cout << GridLogMessage << "oSites : " << _osites << std::endl; - std::cout << GridLogMessage << "lSites : " << lSites() << std::endl; - std::cout << GridLogMessage << "gSites : " << gSites() << std::endl; - std::cout << GridLogMessage << "Nd : " << _ndimension << std::endl; + std::cout << GridLogMessage << "\tFull Dimensions : " << _fdimensions << std::endl; + std::cout << GridLogMessage << "\tSIMD layout : " << _simd_layout << std::endl; + std::cout << GridLogMessage << "\tGlobal Dimensions : " << _gdimensions << std::endl; + std::cout << GridLogMessage << "\tLocal Dimensions : " << _ldimensions << std::endl; + std::cout << GridLogMessage << "\tReduced Dimensions : " << _rdimensions << std::endl; + std::cout << GridLogMessage << "\tOuter strides : " << _ostride << std::endl; + std::cout << GridLogMessage << "\tInner strides : " << _istride << std::endl; + std::cout << GridLogMessage << "\tiSites : " << _isites << std::endl; + std::cout << GridLogMessage << "\toSites : " << _osites << std::endl; + std::cout << GridLogMessage << "\tlSites : " << lSites() << std::endl; + std::cout << GridLogMessage << "\tgSites : " << gSites() << std::endl; + std::cout << GridLogMessage << "\tNd : " << _ndimension << std::endl; } //////////////////////////////////////////////////////////////// diff --git a/lib/qcd/hmc/HMCResourceManager.h b/lib/qcd/hmc/HMCResourceManager.h index cf0000ed..3e20a8c1 100644 --- a/lib/qcd/hmc/HMCResourceManager.h +++ b/lib/qcd/hmc/HMCResourceManager.h @@ -165,7 +165,7 @@ class HMCResourceManager { // Grids ////////////////////////////////////////////////////////////// - void AddGrid(std::string s, GridModule& M) { + void AddGrid(const std::string s, GridModule& M) { // Check for name clashes auto search = Grids.find(s); if (search != Grids.end()) { @@ -174,14 +174,24 @@ class HMCResourceManager { exit(1); } Grids[s] = std::move(M); + std::cout << GridLogMessage << "::::::::::::::::::::::::::::::::::::::::" < Mod; AddGrid(s, Mod); } + // Add a named grid set, 4d shortcut + tweak simd lanes + void AddFourDimGrid(const std::string s, const std::vector simd_decomposition) { + GridFourDimModule Mod(simd_decomposition); + AddGrid(s, Mod); + } GridCartesian* GetCartesian(std::string s = "") { diff --git a/lib/qcd/hmc/HMC_GridModules.h b/lib/qcd/hmc/HMC_GridModules.h index 8331c02b..0f34e9a7 100644 --- a/lib/qcd/hmc/HMC_GridModules.h +++ b/lib/qcd/hmc/HMC_GridModules.h @@ -33,28 +33,29 @@ directory namespace Grid { // Resources -// Modules for grids +// Modules for grids // Introduce another namespace HMCModules? -class GridModuleParameters: Serializable{ +class GridModuleParameters: Serializable{ public: GRID_SERIALIZABLE_CLASS_MEMBERS(GridModuleParameters, std::string, lattice, std::string, mpi); - std::vector getLattice(){return strToVec(lattice);} - std::vector getMpi() {return strToVec(mpi);} + std::vector getLattice() const {return strToVec(lattice);} + std::vector getMpi() const {return strToVec(mpi);} - void check(){ - if (getLattice().size() != getMpi().size()) { - std::cout << GridLogError + + void check() const { + if (getLattice().size() != getMpi().size() ) { + std::cout << GridLogError << "Error in GridModuleParameters: lattice and mpi dimensions " "do not match" << std::endl; exit(1); } - } + } template GridModuleParameters(Reader& Reader, std::string n = "LatticeGrid"):name(n) { @@ -75,51 +76,94 @@ private: // Lower level class class GridModule { public: - GridCartesian* get_full() { + GridCartesian* get_full() { std::cout << GridLogDebug << "Getting cartesian in module"<< std::endl; return grid_.get(); } - GridRedBlackCartesian* get_rb() { + GridRedBlackCartesian* get_rb() { std::cout << GridLogDebug << "Getting rb-cartesian in module"<< std::endl; return rbgrid_.get(); } void set_full(GridCartesian* grid) { grid_.reset(grid); } void set_rb(GridRedBlackCartesian* rbgrid) { rbgrid_.reset(rbgrid); } + void show_full_decomposition(){ grid_->show_decomposition(); } + void show_rb_decomposition(){ rbgrid_->show_decomposition(); } protected: std::unique_ptr grid_; std::unique_ptr rbgrid_; - + }; //////////////////////////////////// // Classes for the user //////////////////////////////////// // Note: the space time grid should be out of the QCD namespace -template< class vector_type> -class GridFourDimModule : public GridModule { - public: - GridFourDimModule() { +template +class GridFourDimModule : public GridModule +{ +public: + GridFourDimModule() + { using namespace QCD; set_full(SpaceTimeGrid::makeFourDimGrid( - GridDefaultLatt(), GridDefaultSimd(4, vector_type::Nsimd()), + GridDefaultLatt(), + GridDefaultSimd(4, vector_type::Nsimd()), GridDefaultMpi())); set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get())); } - GridFourDimModule(GridModuleParameters Params) { + GridFourDimModule(const std::vector tweak_simd) + { + using namespace QCD; + if (tweak_simd.size() != 4) + { + std::cout << GridLogError + << "Error in GridFourDimModule: SIMD size different from 4" + << std::endl; + exit(1); + } + + // Checks that the product agrees with the expectation + int simd_sum = 1; + for (auto &n : tweak_simd) + simd_sum *= n; + std::cout << GridLogDebug << "TweakSIMD: " << tweak_simd << " Sum: " << simd_sum << std::endl; + + if (simd_sum == vector_type::Nsimd()) + { + set_full(SpaceTimeGrid::makeFourDimGrid( + GridDefaultLatt(), + tweak_simd, + GridDefaultMpi())); + set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get())); + } + else + { + std::cout << GridLogError + << "Error in GridFourDimModule: SIMD lanes must sum to " + << vector_type::Nsimd() + << std::endl; + } + } + + GridFourDimModule(const GridModuleParameters Params) + { using namespace QCD; - Params.check(); std::vector lattice_v = Params.getLattice(); std::vector mpi_v = Params.getMpi(); - if (lattice_v.size() == 4) { + if (lattice_v.size() == 4) + { set_full(SpaceTimeGrid::makeFourDimGrid( - lattice_v, GridDefaultSimd(4, vector_type::Nsimd()), + lattice_v, + GridDefaultSimd(4, vector_type::Nsimd()), mpi_v)); set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get())); - } else { - std::cout << GridLogError - << "Error in GridFourDimModule: lattice dimension different from 4" - << std::endl; + } + else + { + std::cout << GridLogError + << "Error in GridFourDimModule: lattice dimension different from 4" + << std::endl; exit(1); } } diff --git a/lib/util/Init.cc b/lib/util/Init.cc index fe3b1734..35a569ba 100644 --- a/lib/util/Init.cc +++ b/lib/util/Init.cc @@ -377,7 +377,7 @@ void Grid_init(int *argc,char ***argv) std::cout << GridLogDebug << "Requesting "<< CartesianCommunicator::MAX_MPI_SHM_BYTES <<" byte stencil comms buffers "< - //FermionParameters(Reader& Reader){ - // read(Reader, "Mobius", *this); - //} - }; @@ -113,9 +107,12 @@ int main(int argc, char **argv) { bool ApplySmearing = MyParams.Mobius.ApplySmearing; + // Use this if you want to tweak the default decomposition + std::vector simd_lanes({2,2,1,1}); - // Grid from the command line - TheHMC.Resources.AddFourDimGrid("gauge"); + // Grid from the command line arguments --grid and --mpi + // drop the simd_lanes argument to fall back to the default decomposition for the SIMD lanes + TheHMC.Resources.AddFourDimGrid("gauge", simd_lanes); // Possibile to create the module by hand // hardcoding parameters or using a Reader