mirror of
https://github.com/paboyle/Grid.git
synced 2024-11-10 07:55:35 +00:00
Hadrons: dedicated IO class for A2A matrices
This commit is contained in:
parent
0bb532f72b
commit
2940c9bcfd
97
Hadrons/A2AMatrix.hpp
Normal file
97
Hadrons/A2AMatrix.hpp
Normal file
@ -0,0 +1,97 @@
|
||||
#ifndef A2A_Matrix_hpp_
|
||||
#define A2A_Matrix_hpp_
|
||||
|
||||
#include <Hadrons/Global.hpp>
|
||||
|
||||
BEGIN_HADRONS_NAMESPACE
|
||||
|
||||
template <typename T, typename MetadataType>
|
||||
class A2AMatrixIo
|
||||
{
|
||||
public:
|
||||
A2AMatrixIo(void) = default;
|
||||
A2AMatrixIo(std::string filename, std::string dataname,
|
||||
const unsigned int nt, const unsigned int ni,
|
||||
const unsigned int nj, const unsigned int blockSize);
|
||||
~A2AMatrixIo(void) = default;
|
||||
void initFile(const MetadataType &d);
|
||||
void saveBlock(const T *data, const unsigned int i, const unsigned int j);
|
||||
private:
|
||||
std::string filename_, dataname_;
|
||||
unsigned int nt_, ni_, nj_, blockSize_;
|
||||
};
|
||||
|
||||
template <typename T, typename MetadataType>
|
||||
A2AMatrixIo<T, MetadataType>::A2AMatrixIo(std::string filename,
|
||||
std::string dataname,
|
||||
const unsigned int nt,
|
||||
const unsigned int ni,
|
||||
const unsigned int nj,
|
||||
const unsigned int blockSize)
|
||||
: filename_(filename), dataname_(dataname)
|
||||
, nt_(nt), ni_(ni), nj_(nj), blockSize_(blockSize)
|
||||
{}
|
||||
|
||||
template <typename T, typename MetadataType>
|
||||
void A2AMatrixIo<T, MetadataType>::initFile(const MetadataType &d)
|
||||
{
|
||||
#ifdef HAVE_HDF5
|
||||
std::vector<hsize_t> dim = {static_cast<hsize_t>(nt_),
|
||||
static_cast<hsize_t>(ni_),
|
||||
static_cast<hsize_t>(nj_)},
|
||||
chunk = {static_cast<hsize_t>(nt_),
|
||||
static_cast<hsize_t>(blockSize_),
|
||||
static_cast<hsize_t>(blockSize_)};
|
||||
H5NS::DataSpace dataspace(dim.size(), dim.data());
|
||||
H5NS::DataSet dataset;
|
||||
H5NS::DSetCreatPropList plist;
|
||||
|
||||
// create empty file just with metadata
|
||||
{
|
||||
Hdf5Writer writer(filename_);
|
||||
write(writer, dataname_, d);
|
||||
}
|
||||
|
||||
// create the dataset
|
||||
Hdf5Writer writer(filename_);
|
||||
|
||||
push(writer, dataname_);
|
||||
auto &group = writer.getGroup();
|
||||
plist.setChunk(chunk.size(), chunk.data());
|
||||
dataset = group.createDataSet("data", Hdf5Type<T>::type(), dataspace, plist);
|
||||
#else
|
||||
HADRONS_ERROR(Implementation, "all-to-all matrix I/O needs HDF5 library");
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T, typename MetadataType>
|
||||
void A2AMatrixIo<T, MetadataType>::saveBlock(const T *data,
|
||||
const unsigned int i,
|
||||
const unsigned int j)
|
||||
{
|
||||
#ifdef HAVE_HDF5
|
||||
Hdf5Reader reader(filename_);
|
||||
std::vector<hsize_t> count = {nt_, blockSize_, blockSize_},
|
||||
offset = {0, static_cast<hsize_t>(i),
|
||||
static_cast<hsize_t>(j)},
|
||||
stride = {1, 1, 1},
|
||||
block = {1, 1, 1};
|
||||
H5NS::DataSpace memspace(count.size(), count.data()), dataspace;
|
||||
H5NS::DataSet dataset;
|
||||
size_t shift;
|
||||
|
||||
push(reader, dataname_);
|
||||
auto &group = reader.getGroup();
|
||||
dataset = group.openDataSet("data");
|
||||
dataspace = dataset.getSpace();
|
||||
dataspace.selectHyperslab(H5S_SELECT_SET, count.data(), offset.data(),
|
||||
stride.data(), block.data());
|
||||
dataset.write(data, Hdf5Type<T>::type(), memspace, dataspace);
|
||||
#else
|
||||
HADRONS_ERROR(Implementation, "all-to-all matrix I/O needs HDF5 library");
|
||||
#endif
|
||||
}
|
||||
|
||||
END_HADRONS_NAMESPACE
|
||||
|
||||
#endif // A2A_Matrix_hpp_
|
||||
@ -15,7 +15,9 @@ libHadrons_adir = $(includedir)/Hadrons
|
||||
nobase_libHadrons_a_HEADERS = \
|
||||
$(modules_hpp) \
|
||||
A2AVectors.hpp \
|
||||
A2AMatrix.hpp \
|
||||
Application.hpp \
|
||||
DilutedNoise.hpp \
|
||||
EigenPack.hpp \
|
||||
Environment.hpp \
|
||||
Exceptions.hpp \
|
||||
|
||||
@ -34,6 +34,7 @@ See the full license in the file "LICENSE" in the top level distribution directo
|
||||
#include <Hadrons/Module.hpp>
|
||||
#include <Hadrons/ModuleFactory.hpp>
|
||||
#include <Hadrons/A2AVectors.hpp>
|
||||
#include <Hadrons/A2AMatrix.hpp>
|
||||
#include <Hadrons/Modules/MSolver/A2AVectors.hpp>
|
||||
#include <Hadrons/Modules/MContraction/A2AMesonFieldKernels.hpp>
|
||||
|
||||
@ -62,6 +63,14 @@ class A2AMesonFieldPar: Serializable
|
||||
std::vector<std::string>, mom);
|
||||
};
|
||||
|
||||
class A2AMesonFieldMetadata: Serializable
|
||||
{
|
||||
public:
|
||||
GRID_SERIALIZABLE_CLASS_MEMBERS(A2AMesonFieldMetadata,
|
||||
std::vector<RealF>, momentum,
|
||||
Gamma::Algebra, gamma);
|
||||
};
|
||||
|
||||
template <typename FImpl>
|
||||
class TA2AMesonField : public Module<A2AMesonFieldPar>
|
||||
{
|
||||
@ -70,6 +79,13 @@ public:
|
||||
SOLVER_TYPE_ALIASES(FImpl,);
|
||||
typedef Eigen::TensorMap<Eigen::Tensor<Complex, 5, Eigen::RowMajor>> MesonField;
|
||||
typedef Eigen::TensorMap<Eigen::Tensor<MF_IO_TYPE, 5, Eigen::RowMajor>> MesonFieldIo;
|
||||
typedef A2AMatrixIo<MF_IO_TYPE, A2AMesonFieldMetadata> MatrixIo;
|
||||
struct IoHelper
|
||||
{
|
||||
MatrixIo io;
|
||||
A2AMesonFieldMetadata metadata;
|
||||
size_t offset;
|
||||
};
|
||||
public:
|
||||
// constructor
|
||||
TA2AMesonField(const std::string name);
|
||||
@ -86,16 +102,13 @@ private:
|
||||
// IO
|
||||
std::string ioname(unsigned int m, unsigned int g) const;
|
||||
std::string filename(unsigned int m, unsigned int g) const;
|
||||
void initFile(unsigned int m, unsigned int g);
|
||||
void saveBlock(const MesonFieldIo &mf,
|
||||
unsigned int m, unsigned int g,
|
||||
unsigned int i, unsigned int j);
|
||||
void saveBlock(MF_IO_TYPE *data, IoHelper &h, unsigned int i, unsigned int j);
|
||||
private:
|
||||
bool hasPhase_{false};
|
||||
std::string momphName_;
|
||||
std::vector<Gamma::Algebra> gamma_;
|
||||
std::vector<std::vector<Real>> mom_;
|
||||
std::vector<std::pair<unsigned int, unsigned int>> nodeFile_;
|
||||
std::vector<IoHelper> nodeIo_;
|
||||
};
|
||||
|
||||
MODULE_REGISTER(A2AMesonField, ARG(TA2AMesonField<FIMPL>), MContraction);
|
||||
@ -331,29 +344,25 @@ void TA2AMesonField<FImpl>::execute(void)
|
||||
makeFileDir(filename(0, 0), env().getGrid());
|
||||
#ifdef MF_PARALLEL_IO
|
||||
env().getGrid()->Barrier();
|
||||
nodeFile_.clear();
|
||||
nodeIo_.clear();
|
||||
for(int f = myRank; f < nmom*ngamma; f += nRank)
|
||||
{
|
||||
std::pair<unsigned int, unsigned int> file;
|
||||
const unsigned int m = f/ngamma, g = f % ngamma;
|
||||
IoHelper h;
|
||||
|
||||
file.first = f/ngamma;
|
||||
file.second = f % ngamma;
|
||||
nodeFile_.push_back(file);
|
||||
h.io = MatrixIo(filename(m, g), ioname(m, g), nt, N_i, N_j, block);
|
||||
for (auto pmu: mom_[m])
|
||||
{
|
||||
h.metadata.momentum.push_back(pmu);
|
||||
}
|
||||
h.metadata.gamma = gamma_[g];
|
||||
h.offset = (m*ngamma + g)*nt*block*block;
|
||||
nodeIo_.push_back(h);
|
||||
}
|
||||
// parallel IO
|
||||
for (auto &f: nodeFile_)
|
||||
for (auto &h: nodeIo_)
|
||||
{
|
||||
auto m = f.first, g = f.second;
|
||||
|
||||
if ((i == 0) and (j == 0))
|
||||
{
|
||||
startTimer("IO: file creation");
|
||||
initFile(m, g);
|
||||
stopTimer("IO: file creation");
|
||||
}
|
||||
startTimer("IO: write block");
|
||||
saveBlock(mfBlock, m, g, i, j);
|
||||
stopTimer("IO: write block");
|
||||
saveBlock(mfBlock.data(), h, i, j);
|
||||
}
|
||||
env().getGrid()->Barrier();
|
||||
#else
|
||||
@ -361,21 +370,16 @@ void TA2AMesonField<FImpl>::execute(void)
|
||||
for(int m = 0; m < nmom; m++)
|
||||
for(int g = 0; g < ngamma; g++)
|
||||
{
|
||||
if ((i == 0) and (j == 0))
|
||||
IoHelper h;
|
||||
|
||||
h.io = MatrixIo(filename(m, g), ioname(m, g), nt, N_i, N_j, block);
|
||||
for (auto pmu: mom_[m])
|
||||
{
|
||||
startTimer("IO: file creation");
|
||||
if (env().getGrid()->IsBoss())
|
||||
{
|
||||
initFile(m, g);
|
||||
h.metadata.momentum.push_back(pmu);
|
||||
}
|
||||
stopTimer("IO: file creation");
|
||||
}
|
||||
startTimer("IO: write block");
|
||||
if (env().getGrid()->IsBoss())
|
||||
{
|
||||
saveBlock(mfBlock, m, g, i, j);
|
||||
}
|
||||
stopTimer("IO: write block");
|
||||
h.metadata.gamma = gamma_[g];
|
||||
h.offset = (m*ngamma + g)*nt*block*block;
|
||||
saveBlock(mfBlock.data(), h, i, j);
|
||||
}
|
||||
#endif
|
||||
stopTimer("IO: total");
|
||||
@ -412,71 +416,18 @@ std::string TA2AMesonField<FImpl>::filename(unsigned int m, unsigned int g) cons
|
||||
}
|
||||
|
||||
template <typename FImpl>
|
||||
void TA2AMesonField<FImpl>::initFile(unsigned int m, unsigned int g)
|
||||
{
|
||||
#ifdef HAVE_HDF5
|
||||
std::string f = filename(m, g);
|
||||
auto &v = envGet(std::vector<FermionField>, par().v);
|
||||
auto &w = envGet(std::vector<FermionField>, par().w);
|
||||
int nt = env().getDim().back();
|
||||
int N_i = w.size();
|
||||
int N_j = v.size();
|
||||
|
||||
Hdf5Writer writer(f);
|
||||
std::vector<hsize_t> dim = {static_cast<hsize_t>(nt),
|
||||
static_cast<hsize_t>(N_i),
|
||||
static_cast<hsize_t>(N_j)},
|
||||
chunk = {static_cast<hsize_t>(nt),
|
||||
static_cast<hsize_t>(par().block),
|
||||
static_cast<hsize_t>(par().block)};
|
||||
H5NS::DataSpace dataspace(dim.size(), dim.data());
|
||||
H5NS::DataSet dataset;
|
||||
H5NS::DSetCreatPropList plist;
|
||||
|
||||
push(writer, ioname(m, g));
|
||||
write(writer, "momentum", mom_[m]);
|
||||
write(writer, "gamma", gamma_[g]);
|
||||
auto &group = writer.getGroup();
|
||||
plist.setChunk(chunk.size(), chunk.data());
|
||||
dataset = group.createDataSet("mesonField", Hdf5Type<MF_IO_TYPE>::type(),
|
||||
dataspace, plist);
|
||||
#else
|
||||
HADRONS_ERROR(Implementation, "meson field I/O needs HDF5 library");
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename FImpl>
|
||||
void TA2AMesonField<FImpl>::saveBlock(const MesonFieldIo &mf,
|
||||
unsigned int m, unsigned int g,
|
||||
void TA2AMesonField<FImpl>::saveBlock(MF_IO_TYPE *data, IoHelper &h,
|
||||
unsigned int i, unsigned int j)
|
||||
{
|
||||
#ifdef HAVE_HDF5
|
||||
std::string f = filename(m, g);
|
||||
Hdf5Reader reader(f);
|
||||
hsize_t nt = mf.dimension(2),
|
||||
Ni = mf.dimension(3),
|
||||
Nj = mf.dimension(4);
|
||||
std::vector<hsize_t> count = {nt, Ni, Nj},
|
||||
offset = {0, static_cast<hsize_t>(i),
|
||||
static_cast<hsize_t>(j)},
|
||||
stride = {1, 1, 1},
|
||||
block = {1, 1, 1};
|
||||
H5NS::DataSpace memspace(count.size(), count.data()), dataspace;
|
||||
H5NS::DataSet dataset;
|
||||
size_t shift;
|
||||
|
||||
push(reader, ioname(m, g));
|
||||
auto &group = reader.getGroup();
|
||||
dataset = group.openDataSet("mesonField");
|
||||
dataspace = dataset.getSpace();
|
||||
dataspace.selectHyperslab(H5S_SELECT_SET, count.data(), offset.data(),
|
||||
stride.data(), block.data());
|
||||
shift = (m*mf.dimension(1) + g)*nt*Ni*Nj;
|
||||
dataset.write(mf.data() + shift, Hdf5Type<MF_IO_TYPE>::type(), memspace,
|
||||
dataspace);
|
||||
#else
|
||||
HADRONS_ERROR(Implementation, "meson field I/O needs HDF5 library");
|
||||
#endif
|
||||
if ((i == 0) and (j == 0))
|
||||
{
|
||||
startTimer("IO: file creation");
|
||||
h.io.initFile(h.metadata);
|
||||
stopTimer("IO: file creation");
|
||||
}
|
||||
startTimer("IO: write block");
|
||||
h.io.saveBlock(data + h.offset, i, j);
|
||||
stopTimer("IO: write block");
|
||||
}
|
||||
|
||||
END_MODULE_NAMESPACE
|
||||
|
||||
Loading…
Reference in New Issue
Block a user