mirror of
https://github.com/paboyle/Grid.git
synced 2025-04-09 21:50:45 +01:00
Hadrons: meson field parallel IO
This commit is contained in:
parent
6e5a2b7922
commit
cc2780bea3
@ -37,6 +37,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
|
|||||||
#include <Grid/Hadrons/Modules/MSolver/A2AVectors.hpp>
|
#include <Grid/Hadrons/Modules/MSolver/A2AVectors.hpp>
|
||||||
#include <Grid/Hadrons/Modules/MContraction/A2AMesonFieldKernels.hpp>
|
#include <Grid/Hadrons/Modules/MContraction/A2AMesonFieldKernels.hpp>
|
||||||
|
|
||||||
|
#define MF_PARALLEL_IO
|
||||||
|
|
||||||
BEGIN_HADRONS_NAMESPACE
|
BEGIN_HADRONS_NAMESPACE
|
||||||
|
|
||||||
/******************************************************************************
|
/******************************************************************************
|
||||||
@ -99,6 +101,7 @@ private:
|
|||||||
std::string momphName_;
|
std::string momphName_;
|
||||||
std::vector<Gamma::Algebra> gamma_;
|
std::vector<Gamma::Algebra> gamma_;
|
||||||
std::vector<std::vector<double>> mom_;
|
std::vector<std::vector<double>> mom_;
|
||||||
|
std::vector<std::pair<unsigned int, unsigned int>> nodeFile_;
|
||||||
};
|
};
|
||||||
|
|
||||||
MODULE_REGISTER(A2AMesonField, ARG(TA2AMesonField<FIMPL>), MContraction);
|
MODULE_REGISTER(A2AMesonField, ARG(TA2AMesonField<FIMPL>), MContraction);
|
||||||
@ -311,14 +314,30 @@ void TA2AMesonField<FImpl>::execute(void)
|
|||||||
// IO
|
// IO
|
||||||
if (!par().output.empty())
|
if (!par().output.empty())
|
||||||
{
|
{
|
||||||
double blockSize, ioTime;
|
double blockSize, ioTime;
|
||||||
|
unsigned int myRank = env().getGrid()->ThisRank(),
|
||||||
|
nRank = env().getGrid()->RankCount();
|
||||||
|
|
||||||
LOG(Message) << "Writing block to disk" << std::endl;
|
LOG(Message) << "Writing block to disk" << std::endl;
|
||||||
ioTime = -getDTimer("IO: write block");
|
ioTime = -getDTimer("IO: write block");
|
||||||
startTimer("IO: total");
|
startTimer("IO: total");
|
||||||
for(int m = 0; m < nmom; m++)
|
makeFileDir(filename(0, 0), env().getGrid());
|
||||||
for(int g = 0; g < ngamma; g++)
|
#ifdef MF_PARALLEL_IO
|
||||||
|
env().getGrid()->Barrier();
|
||||||
|
nodeFile_.clear();
|
||||||
|
for(int f = myRank; f < nmom*ngamma; f += nRank)
|
||||||
{
|
{
|
||||||
|
std::pair<unsigned int, unsigned int> file;
|
||||||
|
|
||||||
|
file.first = f/ngamma;
|
||||||
|
file.second = f % ngamma;
|
||||||
|
nodeFile_.push_back(file);
|
||||||
|
}
|
||||||
|
// parallel IO
|
||||||
|
for (auto &f: nodeFile_)
|
||||||
|
{
|
||||||
|
auto m = f.first, g = f.second;
|
||||||
|
|
||||||
if ((i == 0) and (j == 0))
|
if ((i == 0) and (j == 0))
|
||||||
{
|
{
|
||||||
startTimer("IO: file creation");
|
startTimer("IO: file creation");
|
||||||
@ -329,6 +348,29 @@ void TA2AMesonField<FImpl>::execute(void)
|
|||||||
saveBlock(mfBlock, m, g, i, j);
|
saveBlock(mfBlock, m, g, i, j);
|
||||||
stopTimer("IO: write block");
|
stopTimer("IO: write block");
|
||||||
}
|
}
|
||||||
|
env().getGrid()->Barrier();
|
||||||
|
#else
|
||||||
|
// serial IO
|
||||||
|
for(int m = 0; m < nmom; m++)
|
||||||
|
for(int g = 0; g < ngamma; g++)
|
||||||
|
{
|
||||||
|
if ((i == 0) and (j == 0))
|
||||||
|
{
|
||||||
|
startTimer("IO: file creation");
|
||||||
|
if (env().getGrid()->IsBoss())
|
||||||
|
{
|
||||||
|
initFile(m, g);
|
||||||
|
}
|
||||||
|
stopTimer("IO: file creation");
|
||||||
|
}
|
||||||
|
startTimer("IO: write block");
|
||||||
|
if (env().getGrid()->IsBoss())
|
||||||
|
{
|
||||||
|
saveBlock(mfBlock, m, g, i, j);
|
||||||
|
}
|
||||||
|
stopTimer("IO: write block");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
stopTimer("IO: total");
|
stopTimer("IO: total");
|
||||||
blockSize = static_cast<double>(nmom*ngamma*nt*N_ii*N_jj*sizeof(Complex));
|
blockSize = static_cast<double>(nmom*ngamma*nt*N_ii*N_jj*sizeof(Complex));
|
||||||
ioTime += getDTimer("IO: write block");
|
ioTime += getDTimer("IO: write block");
|
||||||
@ -372,30 +414,25 @@ void TA2AMesonField<FImpl>::initFile(unsigned int m, unsigned int g)
|
|||||||
{
|
{
|
||||||
#ifdef HAVE_HDF5
|
#ifdef HAVE_HDF5
|
||||||
std::string f = filename(m, g);
|
std::string f = filename(m, g);
|
||||||
GridBase *grid = env().getGrid();
|
|
||||||
auto &v = envGet(std::vector<FermionField>, par().v);
|
auto &v = envGet(std::vector<FermionField>, par().v);
|
||||||
auto &w = envGet(std::vector<FermionField>, par().w);
|
auto &w = envGet(std::vector<FermionField>, par().w);
|
||||||
int nt = env().getDim().back();
|
int nt = env().getDim().back();
|
||||||
int N_i = w.size();
|
int N_i = w.size();
|
||||||
int N_j = v.size();
|
int N_j = v.size();
|
||||||
|
|
||||||
makeFileDir(f, grid);
|
Hdf5Writer writer(f);
|
||||||
if (grid->IsBoss())
|
std::vector<hsize_t> dim = {static_cast<hsize_t>(nt),
|
||||||
{
|
static_cast<hsize_t>(N_i),
|
||||||
Hdf5Writer writer(f);
|
static_cast<hsize_t>(N_j)};
|
||||||
std::vector<hsize_t> dim = {static_cast<hsize_t>(nt),
|
H5NS::DataSpace dataspace(dim.size(), dim.data());
|
||||||
static_cast<hsize_t>(N_i),
|
H5NS::DataSet dataset;
|
||||||
static_cast<hsize_t>(N_j)};
|
|
||||||
H5NS::DataSpace dataspace(dim.size(), dim.data());
|
push(writer, ioname(m, g));
|
||||||
H5NS::DataSet dataset;
|
write(writer, "momentum", mom_[m]);
|
||||||
|
write(writer, "gamma", gamma_[g]);
|
||||||
push(writer, ioname(m, g));
|
auto &group = writer.getGroup();
|
||||||
write(writer, "momentum", mom_[m]);
|
dataset = group.createDataSet("mesonField", Hdf5Type<Complex>::type(),
|
||||||
write(writer, "gamma", gamma_[g]);
|
dataspace);
|
||||||
auto &group = writer.getGroup();
|
|
||||||
dataset = group.createDataSet("mesonField", Hdf5Type<Complex>::type(),
|
|
||||||
dataspace);
|
|
||||||
}
|
|
||||||
#else
|
#else
|
||||||
HADRONS_ERROR(Implementation, "meson field I/O needs HDF5 library");
|
HADRONS_ERROR(Implementation, "meson field I/O needs HDF5 library");
|
||||||
#endif
|
#endif
|
||||||
@ -407,34 +444,29 @@ void TA2AMesonField<FImpl>::saveBlock(const MesonField &mf,
|
|||||||
unsigned int i, unsigned int j)
|
unsigned int i, unsigned int j)
|
||||||
{
|
{
|
||||||
#ifdef HAVE_HDF5
|
#ifdef HAVE_HDF5
|
||||||
std::string f = filename(m, g);
|
std::string f = filename(m, g);
|
||||||
GridBase *grid = env().getGrid();
|
Hdf5Reader reader(f);
|
||||||
|
hsize_t nt = mf.dimension(2),
|
||||||
|
Ni = mf.dimension(3),
|
||||||
|
Nj = mf.dimension(4);
|
||||||
|
std::vector<hsize_t> count = {nt, Ni, Nj},
|
||||||
|
offset = {0, static_cast<hsize_t>(i),
|
||||||
|
static_cast<hsize_t>(j)},
|
||||||
|
stride = {1, 1, 1},
|
||||||
|
block = {1, 1, 1};
|
||||||
|
H5NS::DataSpace memspace(count.size(), count.data()), dataspace;
|
||||||
|
H5NS::DataSet dataset;
|
||||||
|
size_t shift;
|
||||||
|
|
||||||
if (grid->IsBoss())
|
push(reader, ioname(m, g));
|
||||||
{
|
auto &group = reader.getGroup();
|
||||||
Hdf5Reader reader(f);
|
dataset = group.openDataSet("mesonField");
|
||||||
hsize_t nt = mf.dimension(2),
|
dataspace = dataset.getSpace();
|
||||||
Ni = mf.dimension(3),
|
dataspace.selectHyperslab(H5S_SELECT_SET, count.data(), offset.data(),
|
||||||
Nj = mf.dimension(4);
|
stride.data(), block.data());
|
||||||
std::vector<hsize_t> count = {nt, Ni, Nj},
|
shift = (m*mf.dimension(1) + g)*nt*Ni*Nj;
|
||||||
offset = {0, static_cast<hsize_t>(i),
|
dataset.write(mf.data() + shift, Hdf5Type<Complex>::type(), memspace,
|
||||||
static_cast<hsize_t>(j)},
|
dataspace);
|
||||||
stride = {1, 1, 1},
|
|
||||||
block = {1, 1, 1};
|
|
||||||
H5NS::DataSpace memspace(count.size(), count.data()), dataspace;
|
|
||||||
H5NS::DataSet dataset;
|
|
||||||
size_t shift;
|
|
||||||
|
|
||||||
push(reader, ioname(m, g));
|
|
||||||
auto &group = reader.getGroup();
|
|
||||||
dataset = group.openDataSet("mesonField");
|
|
||||||
dataspace = dataset.getSpace();
|
|
||||||
dataspace.selectHyperslab(H5S_SELECT_SET, count.data(), offset.data(),
|
|
||||||
stride.data(), block.data());
|
|
||||||
shift = (m*mf.dimension(1) + g)*nt*Ni*Nj;
|
|
||||||
dataset.write(mf.data() + shift, Hdf5Type<Complex>::type(), memspace,
|
|
||||||
dataspace);
|
|
||||||
}
|
|
||||||
#else
|
#else
|
||||||
HADRONS_ERROR(Implementation, "meson field I/O needs HDF5 library");
|
HADRONS_ERROR(Implementation, "meson field I/O needs HDF5 library");
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user