portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2024-11-10 07:55:35 +00:00
Code Releases Activity

Roll over to MPI IO for parallel IO

Browse Source
This commit is contained in:
Peter Boyle 2017-06-01 17:37:26 -04:00
parent 21421656ab
commit 1a1f6d55f9
1 changed files with 52 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
lib/parallelIO/NerscIO.h
View File

@ -30,12 +30,6 @@
#ifndef GRID_NERSC_IO_H #ifndef GRID_NERSC_IO_H
#define GRID_NERSC_IO_H #define GRID_NERSC_IO_H
#undef PARALLEL_READ
#undef SERIAL_READ
#define MPI_READ
#define PARALLEL_WRITE
#include <algorithm> #include <algorithm>
#include <iostream> #include <iostream>
#include <iomanip> #include <iomanip>
@ -133,10 +127,6 @@ namespace Grid {
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
// Utilities ; these are QCD aware // Utilities ; these are QCD aware
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
inline void NerscChecksum(uint32_t *buf,uint32_t buf_size_bytes,uint32_t &csum)
{
BinaryIO::Uint32Checksum(buf,buf_size_bytes,csum);
}
inline void reconstruct3(LorentzColourMatrix & cm) inline void reconstruct3(LorentzColourMatrix & cm)
{ {
const int x=0; const int x=0;
@ -151,43 +141,38 @@ namespace Grid {
template<class fobj,class sobj> template<class fobj,class sobj>
struct NerscSimpleMunger{ struct NerscSimpleMunger{
void operator()(fobj &in, sobj &out, uint32_t &csum) { void operator()(fobj &in, sobj &out) {
for (int mu = 0; mu < Nd; mu++) { for (int mu = 0; mu < Nd; mu++) {
for (int i = 0; i < Nc; i++) { for (int i = 0; i < Nc; i++) {
for (int j = 0; j < Nc; j++) { for (int j = 0; j < Nc; j++) {
out(mu)()(i, j) = in(mu)()(i, j); out(mu)()(i, j) = in(mu)()(i, j);
} }}
}
} }
NerscChecksum((uint32_t *)&in, sizeof(in), csum);
}; };
}; };
template <class fobj, class sobj> template <class fobj, class sobj>
struct NerscSimpleUnmunger { struct NerscSimpleUnmunger {
void operator()(sobj &in, fobj &out, uint32_t &csum) {
void operator()(sobj &in, fobj &out) {
for (int mu = 0; mu < Nd; mu++) { for (int mu = 0; mu < Nd; mu++) {
for (int i = 0; i < Nc; i++) { for (int i = 0; i < Nc; i++) {
for (int j = 0; j < Nc; j++) { for (int j = 0; j < Nc; j++) {
out(mu)()(i, j) = in(mu)()(i, j); out(mu)()(i, j) = in(mu)()(i, j);
} }}
}
} }
NerscChecksum((uint32_t *)&out, sizeof(out), csum);
}; };
}; };
template<class fobj,class sobj> template<class fobj,class sobj>
struct Nersc3x2munger{ struct Nersc3x2munger{
void operator() (fobj &in,sobj &out,uint32_t &csum){
NerscChecksum((uint32_t *)&in,sizeof(in),csum);
void operator() (fobj &in,sobj &out){
for(int mu=0;mu<4;mu++){ for(int mu=0;mu<4;mu++){
for(int i=0;i<2;i++){ for(int i=0;i<2;i++){
for(int j=0;j<3;j++){ for(int j=0;j<3;j++){
out(mu)()(i,j) = in(mu)(i)(j); out(mu)()(i,j) = in(mu)(i)(j);
}} }}
} }
reconstruct3(out); reconstruct3(out);
} }
@ -196,18 +181,13 @@ namespace Grid {
template<class fobj,class sobj> template<class fobj,class sobj>
struct Nersc3x2unmunger{ struct Nersc3x2unmunger{
void operator() (sobj &in,fobj &out,uint32_t &csum){ void operator() (sobj &in,fobj &out){
for(int mu=0;mu<4;mu++){ for(int mu=0;mu<4;mu++){
for(int i=0;i<2;i++){ for(int i=0;i<2;i++){
for(int j=0;j<3;j++){ for(int j=0;j<3;j++){
out(mu)(i)(j) = in(mu)()(i,j); out(mu)(i)(j) = in(mu)()(i,j);
}} }}
} }
NerscChecksum((uint32_t *)&out,sizeof(out),csum);
} }
}; };
@ -333,9 +313,9 @@ namespace Grid {
// Now the meat: the object readers // Now the meat: the object readers
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
template<class vsimd> template<class vsimd>
static inline void readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,NerscField& header,std::string file) static inline void readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,NerscField& header,std::string file)
{ {
typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField; typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
GridBase *grid = Umu._grid; GridBase *grid = Umu._grid;
@ -354,62 +334,22 @@ namespace Grid {
// depending on datatype, set up munger; // depending on datatype, set up munger;
// munger is a function of <floating point, Real, data_type> // munger is a function of <floating point, Real, data_type>
if ( header.data_type == std::string("4D_SU3_GAUGE") ) { if ( header.data_type == std::string("4D_SU3_GAUGE") ) {
if ( ieee32 || ieee32big ) {
#ifdef PARALLEL_READ
csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>, LorentzColour2x3F>
(Umu,file,Nersc3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format);
#endif
#ifdef MPI_READ
csum=BinaryIO::readObjectMPI<iLorentzColourMatrix<vsimd>, LorentzColour2x3F>
(Umu,file,Nersc3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format);
#endif
#ifdef SERIAL_READ
csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>, LorentzColour2x3F>
(Umu,file,Nersc3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format);
#endif
}
if ( ieee64 || ieee64big ) {
#ifdef PARALLEL_READ
csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>, LorentzColour2x3D>
(Umu,file,Nersc3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format);
#endif
#ifdef MPI_READ
csum=BinaryIO::readObjectMPI<iLorentzColourMatrix<vsimd>, LorentzColour2x3D>
(Umu,file,Nersc3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format);
#endif
#ifdef SERIAL_READ
csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>, LorentzColour2x3D>
(Umu,file,Nersc3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format);
#endif
}
} else if ( header.data_type == std::string("4D_SU3_GAUGE_3x3") ) {
if ( ieee32 || ieee32big ) { if ( ieee32 || ieee32big ) {
#ifdef PARALLEL_READ csum=BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>, LorentzColour2x3F>
csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>,LorentzColourMatrixF> (Umu,file,Nersc3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format);
(Umu,file,NerscSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format);
#endif
#ifdef MPI_READ
csum=BinaryIO::readObjectMPI<iLorentzColourMatrix<vsimd>,LorentzColourMatrixF>
(Umu,file,NerscSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format);
#endif
#ifdef SERIAL_READ
csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>,LorentzColourMatrixF>
(Umu,file,NerscSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format);
#endif
} }
if ( ieee64 || ieee64big ) { if ( ieee64 || ieee64big ) {
#ifdef PARALLEL_READ csum=BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>, LorentzColour2x3D>
csum=BinaryIO::readObjectParallel<iLorentzColourMatrix<vsimd>,LorentzColourMatrixD> (Umu,file,Nersc3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format);
}
} else if ( header.data_type == std::string("4D_SU3_GAUGE_3x3") ) {
if ( ieee32 || ieee32big ) {
csum=BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>,LorentzColourMatrixF>
(Umu,file,NerscSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format);
}
if ( ieee64 || ieee64big ) {
csum=BinaryIO::readLatticeObject<iLorentzColourMatrix<vsimd>,LorentzColourMatrixD>
(Umu,file,NerscSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format); (Umu,file,NerscSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format);
#endif
#ifdef MPI_READ
csum=BinaryIO::readObjectMPI<iLorentzColourMatrix<vsimd>,LorentzColourMatrixD>
(Umu,file,NerscSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format);
#endif
#ifdef SERIAL_READ
csum=BinaryIO::readObjectSerial<iLorentzColourMatrix<vsimd>,LorentzColourMatrixD>
(Umu,file,NerscSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format);
#endif
} }
} else { } else {
assert(0); assert(0);
@ -434,14 +374,14 @@ namespace Grid {
std::cerr << " plaqs " << clone.plaquette << " " << header.plaquette << std::endl; std::cerr << " plaqs " << clone.plaquette << " " << header.plaquette << std::endl;
std::cerr << " trace " << clone.link_trace<< " " << header.link_trace<< std::endl; std::cerr << " trace " << clone.link_trace<< " " << header.link_trace<< std::endl;
std::cerr << " csum " <<std::hex<< csum << " " << header.checksum<< std::dec<< std::endl; std::cerr << " csum " <<std::hex<< csum << " " << header.checksum<< std::dec<< std::endl;
// exit(0); exit(0);
}
// assert(fabs(clone.plaquette -header.plaquette ) < 1.0e-5 );
// assert(fabs(clone.link_trace-header.link_trace) < 1.0e-6 );
// assert(csum == header.checksum );
// std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
} }
assert(fabs(clone.plaquette -header.plaquette ) < 1.0e-5 );
assert(fabs(clone.link_trace-header.link_trace) < 1.0e-6 );
assert(csum == header.checksum );
std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
}
template<class vsimd> template<class vsimd>
static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,std::string file, int two_row,int bits32) static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,std::string file, int two_row,int bits32)
@ -466,41 +406,29 @@ namespace Grid {
NerscStatistics<GaugeField>(Umu,header); NerscStatistics<GaugeField>(Umu,header);
NerscMachineCharacteristics(header); NerscMachineCharacteristics(header);
uint32_t csum;
int offset; int offset;
truncate(file); truncate(file);
if ( two_row ) { if ( two_row ) {
header.floating_point = std::string("IEEE64BIG"); header.floating_point = std::string("IEEE64BIG");
header.data_type = std::string("4D_SU3_GAUGE"); header.data_type = std::string("4D_SU3_GAUGE");
Nersc3x2unmunger<fobj2D,sobj> munge; Nersc3x2unmunger<fobj2D,sobj> munge;
BinaryIO::Uint32Checksum<vobj,fobj2D>(Umu, munge,header.checksum);
offset = writeHeader(header,file); offset = writeHeader(header,file);
#ifdef PARALLEL_WRITE header.checksum=BinaryIO::writeLatticeObject<vobj,fobj2D>(Umu,file,munge,offset,header.floating_point);
csum=BinaryIO::writeObjectParallel<vobj,fobj2D>(Umu,file,munge,offset,header.floating_point); writeHeader(header,file);
#else
csum=BinaryIO::writeObjectSerial<vobj,fobj2D>(Umu,file,munge,offset,header.floating_point);
#endif
} else { } else {
header.floating_point = std::string("IEEE64BIG"); header.floating_point = std::string("IEEE64BIG");
header.data_type = std::string("4D_SU3_GAUGE_3x3"); header.data_type = std::string("4D_SU3_GAUGE_3x3");
NerscSimpleUnmunger<fobj3D,sobj> munge; NerscSimpleUnmunger<fobj3D,sobj> munge;
BinaryIO::Uint32Checksum<vobj,fobj3D>(Umu, munge,header.checksum);
offset = writeHeader(header,file); offset = writeHeader(header,file);
#ifdef PARALLEL_WRITE header.checksum=BinaryIO::writeLatticeObject<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point);
csum=BinaryIO::writeObjectParallel<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point); writeHeader(header,file);
#else
csum=BinaryIO::writeObjectSerial<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point);
#endif
} }
std::cout<<GridLogMessage <<"Written NERSC Configuration on "<< file << " checksum "
std::cout<<GridLogMessage <<"Written NERSC Configuration on "<< file << " checksum "<<std::hex<<csum<< std::dec<<" plaq "<< header.plaquette <<std::endl; <<std::hex<<header.checksum
<<std::dec<<" plaq "<< header.plaquette <<std::endl;
} }
/////////////////////////////// ///////////////////////////////
// RNG state // RNG state
/////////////////////////////// ///////////////////////////////
@ -521,7 +449,6 @@ namespace Grid {
header.plaquette=0.0; header.plaquette=0.0;
NerscMachineCharacteristics(header); NerscMachineCharacteristics(header);
uint32_t csum;
int offset; int offset;
#ifdef RNG_RANLUX #ifdef RNG_RANLUX
@ -539,11 +466,13 @@ namespace Grid {
truncate(file); truncate(file);
offset = writeHeader(header,file); offset = writeHeader(header,file);
csum=BinaryIO::writeRNGSerial(serial,parallel,file,offset); header.checksum = BinaryIO::writeRNG(serial,parallel,file,offset);
header.checksum = csum;
offset = writeHeader(header,file); offset = writeHeader(header,file);
std::cout<<GridLogMessage <<"Written NERSC RNG STATE "<<file<< " checksum "<<std::hex<<csum<<std::dec<<std::endl; std::cout<<GridLogMessage
<<"Written NERSC RNG STATE "<<file<< " checksum "
<<std::hex<<header.checksum
<<std::dec<<std::endl;
} }
@ -575,7 +504,7 @@ namespace Grid {
// depending on datatype, set up munger; // depending on datatype, set up munger;
// munger is a function of <floating point, Real, data_type> // munger is a function of <floating point, Real, data_type>
uint32_t csum=BinaryIO::readRNGSerial(serial,parallel,file,offset); uint32_t csum=BinaryIO::readRNG(serial,parallel,file,offset);
if ( csum != header.checksum ) { if ( csum != header.checksum ) {
std::cerr << "checksum mismatch "<<std::hex<< csum <<" "<<header.checksum<<std::dec<<std::endl; std::cerr << "checksum mismatch "<<std::hex<< csum <<" "<<header.checksum<<std::dec<<std::endl;