portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2025-04-25 21:25:56 +01:00
Code Releases Activity

Added MPI version

Browse Source
This commit is contained in:
paboyle 2017-05-30 23:41:07 +01:00
parent d38a4de36c
commit 1e429a0d57
1 changed files with 144 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
lib/parallelIO/BinaryIO.h
View File

@ -250,6 +250,149 @@ class BinaryIO {
return csum;
}
template<class vobj,class fobj,class munger>
static inline uint32_t readObjectMPI(Lattice<vobj> &Umu,std::string file,munger munge,int offset,const std::string &format)
{
typedef typename vobj::scalar_object sobj;
GridBase *grid = Umu._grid;
std::cout<< GridLogMessage<< "MPI read I/O "<< file<< std::endl;
GridStopWatch timer; timer.Start();
Umu = zero;
uint32_t csum=0;
uint64_t bytes=0;
int ndim = grid->Dimensions();
int nrank = grid->ProcessorCount();
int myrank = grid->ThisRank();
std::vector<int> psizes = grid->ProcessorGrid();
std::vector<int> pcoor = grid->ThisProcessorCoor();
std::vector<int> gLattice= grid->GlobalDimensions();
std::vector<int> lLattice= grid->LocalDimensions();
std::vector<int> distribs(ndim,MPI_DISTRIBUTE_BLOCK);
std::vector<int> dargs (ndim,MPI_DISTRIBUTE_DFLT_DARG);
std::vector<int> lStart(ndim);
std::vector<int> gStart(ndim);
// Flatten the file
int lsites = grid->lSites();
std::vector<sobj> scalardata(lsites);
std::vector<fobj> iodata(lsites); // Munge, checksum, byte order in here
for(int d=0;d<ndim;d++){
gStart[d] = lLattice[d]*pcoor[d];
lStart[d] = 0;
}
MPI_Datatype mpiObject;
MPI_Datatype fileArray;
MPI_Datatype localArray;
MPI_Datatype mpiword;
MPI_Offset disp = offset;
MPI_File fh ;
MPI_Status status;
int numword;
if ( sizeof( typename vobj::Realified::scalar_type ) == sizeof(float ) ) {
numword = sizeof(fobj)/sizeof(float);
mpiword = MPI_FLOAT;
} else {
numword = sizeof(fobj)/sizeof(double);
mpiword = MPI_DOUBLE;
}
bytes = sizeof(fobj)*lsites*nrank;
//////////////////////////////////////////////////////////////////////////////
// Sobj in MPI phrasing
//////////////////////////////////////////////////////////////////////////////
int ierr;
ierr = MPI_Type_contiguous(numword,mpiword,&mpiObject);
assert(ierr==0);
ierr=MPI_Type_commit(&mpiObject);
//////////////////////////////////////////////////////////////////////////////
// File global array data type
//////////////////////////////////////////////////////////////////////////////
ierr=MPI_Type_create_subarray(ndim,&gLattice[0],&lLattice[0],&gStart[0],MPI_ORDER_FORTRAN, mpiObject,&fileArray);
assert(ierr==0);
ierr=MPI_Type_commit(&fileArray);
assert(ierr==0);
//////////////////////////////////////////////////////////////////////////////
// local lattice array
//////////////////////////////////////////////////////////////////////////////
ierr=MPI_Type_create_subarray(ndim,&lLattice[0],&lLattice[0],&lStart[0],MPI_ORDER_FORTRAN, mpiObject,&localArray);
assert(ierr==0);
ierr=MPI_Type_commit(&localArray);
assert(ierr==0);
//////////////////////////////////////////////////////////////////////////////
// Do the MPI I/O read
//////////////////////////////////////////////////////////////////////////////
// std::cout << "MPI IO read from " <<file<<std::endl;
ierr=MPI_File_open(grid->communicator, file.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
assert(ierr==0);
ierr=MPI_File_set_view(fh, disp, mpiObject, fileArray, "native", MPI_INFO_NULL);
// std::cout<< "MPI File set view returned " <<ierr<<std::endl;
/*
if ( ierr ) {
char buf[MPI_MAX_ERROR_STRING];
int blen;
MPI_Error_string(ierr,buf,&blen);
std::cout << " Error string " <<buf<<std::endl;
}
*/
assert(ierr==0);
ierr=MPI_File_read_all(fh, &iodata[0], 1, localArray, &status);
assert(ierr==0);
//////////////////////////////////////////////////////////////////////////////
// Finish up MPI I/O
//////////////////////////////////////////////////////////////////////////////
MPI_File_close(&fh);
MPI_Type_free(&fileArray);
MPI_Type_free(&localArray);
//////////////////////////////////////////////////////////////////////////////
// Byte order
//////////////////////////////////////////////////////////////////////////////
int ieee32big = (format == std::string("IEEE32BIG"));
int ieee32 = (format == std::string("IEEE32"));
int ieee64big = (format == std::string("IEEE64BIG"));
int ieee64 = (format == std::string("IEEE64"));
if (ieee32big) be32toh_v((void *)&iodata[0], sizeof(fobj)*lsites);
if (ieee32) le32toh_v((void *)&iodata[0], sizeof(fobj)*lsites);
if (ieee64big) be64toh_v((void *)&iodata[0], sizeof(fobj)*lsites);
if (ieee64) le64toh_v((void *)&iodata[0], sizeof(fobj)*lsites);
//////////////////////////////////////////////////////////////////////////////
// Munge [ .e.g 3rd row recon ]
//////////////////////////////////////////////////////////////////////////////
for(int x=0;x<lsites;x++) munge(iodata[x], scalardata[x], csum);
//////////////////////////////////////////////////////////////////////////////
// Safety check
//////////////////////////////////////////////////////////////////////////////
grid->GlobalSum(csum);
grid->Barrier();
vectorizeFromLexOrdArray(scalardata,Umu);
timer.Stop();
std::cout<<GridLogMessage<<"readObjectMPI: read "<< bytes <<" bytes in "<<timer.Elapsed() <<" "
<< (double)bytes/ (double)timer.useconds() <<" MB/s " <<std::endl;
return csum;
}
template<class vobj,class fobj,class munger>
static inline uint32_t writeObjectSerial(Lattice<vobj> &Umu,std::string file,munger munge,int offset,
const std::string & format)
@ -597,7 +740,7 @@ class BinaryIO {
for(int c=0;c<chunk;c++) munge(fileObj[c],siteObj[c],csum);
}
}
// Possibly do transport through pt2pt
for(int cc=0;cc<chunk;cc+=lstrip){