|
|
|
@ -272,8 +272,10 @@ class GridLimeReader : public BinaryIO {
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class GridLimeWriter : public BinaryIO {
|
|
|
|
|
class GridLimeWriter : public BinaryIO
|
|
|
|
|
{
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////
|
|
|
|
|
// FIXME: format for RNG? Now just binary out instead
|
|
|
|
|
// FIXME: collective calls or not ?
|
|
|
|
@ -282,17 +284,24 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
FILE *File;
|
|
|
|
|
LimeWriter *LimeW;
|
|
|
|
|
std::string filename;
|
|
|
|
|
|
|
|
|
|
bool boss_node;
|
|
|
|
|
GridLimeWriter( bool isboss = true) {
|
|
|
|
|
boss_node = isboss;
|
|
|
|
|
}
|
|
|
|
|
void open(const std::string &_filename) {
|
|
|
|
|
filename= _filename;
|
|
|
|
|
File = fopen(filename.c_str(), "w");
|
|
|
|
|
LimeW = limeCreateWriter(File); assert(LimeW != NULL );
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
File = fopen(filename.c_str(), "w");
|
|
|
|
|
LimeW = limeCreateWriter(File); assert(LimeW != NULL );
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
/////////////////////////////////////////////
|
|
|
|
|
// Close the file
|
|
|
|
|
/////////////////////////////////////////////
|
|
|
|
|
void close(void) {
|
|
|
|
|
fclose(File);
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
fclose(File);
|
|
|
|
|
}
|
|
|
|
|
// limeDestroyWriter(LimeW);
|
|
|
|
|
}
|
|
|
|
|
///////////////////////////////////////////////////////
|
|
|
|
@ -300,10 +309,12 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
///////////////////////////////////////////////////////
|
|
|
|
|
int createLimeRecordHeader(std::string message, int MB, int ME, size_t PayloadSize)
|
|
|
|
|
{
|
|
|
|
|
LimeRecordHeader *h;
|
|
|
|
|
h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
|
|
|
|
|
assert(limeWriteRecordHeader(h, LimeW) >= 0);
|
|
|
|
|
limeDestroyHeader(h);
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
LimeRecordHeader *h;
|
|
|
|
|
h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
|
|
|
|
|
assert(limeWriteRecordHeader(h, LimeW) >= 0);
|
|
|
|
|
limeDestroyHeader(h);
|
|
|
|
|
}
|
|
|
|
|
return LIME_SUCCESS;
|
|
|
|
|
}
|
|
|
|
|
////////////////////////////////////////////
|
|
|
|
@ -312,28 +323,31 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
template<class serialisable_object>
|
|
|
|
|
void writeLimeObject(int MB,int ME,serialisable_object &object,std::string object_name,std::string record_name)
|
|
|
|
|
{
|
|
|
|
|
std::string xmlstring;
|
|
|
|
|
{
|
|
|
|
|
XmlWriter WR("","");
|
|
|
|
|
write(WR,object_name,object);
|
|
|
|
|
xmlstring = WR.XmlString();
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
std::string xmlstring;
|
|
|
|
|
{
|
|
|
|
|
XmlWriter WR("","");
|
|
|
|
|
write(WR,object_name,object);
|
|
|
|
|
xmlstring = WR.XmlString();
|
|
|
|
|
}
|
|
|
|
|
// std::cout << "WriteLimeObject" << record_name <<std::endl;
|
|
|
|
|
uint64_t nbytes = xmlstring.size();
|
|
|
|
|
// std::cout << " xmlstring "<< nbytes<< " " << xmlstring <<std::endl;
|
|
|
|
|
int err;
|
|
|
|
|
LimeRecordHeader *h = limeCreateHeader(MB, ME,const_cast<char *>(record_name.c_str()), nbytes);
|
|
|
|
|
assert(h!= NULL);
|
|
|
|
|
|
|
|
|
|
err=limeWriteRecordHeader(h, LimeW); assert(err>=0);
|
|
|
|
|
err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
|
|
|
|
|
err=limeWriterCloseRecord(LimeW); assert(err>=0);
|
|
|
|
|
limeDestroyHeader(h);
|
|
|
|
|
}
|
|
|
|
|
// std::cout << "WriteLimeObject" << record_name <<std::endl;
|
|
|
|
|
uint64_t nbytes = xmlstring.size();
|
|
|
|
|
// std::cout << " xmlstring "<< nbytes<< " " << xmlstring <<std::endl;
|
|
|
|
|
int err;
|
|
|
|
|
LimeRecordHeader *h = limeCreateHeader(MB, ME,const_cast<char *>(record_name.c_str()), nbytes);
|
|
|
|
|
assert(h!= NULL);
|
|
|
|
|
|
|
|
|
|
err=limeWriteRecordHeader(h, LimeW); assert(err>=0);
|
|
|
|
|
err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
|
|
|
|
|
err=limeWriterCloseRecord(LimeW); assert(err>=0);
|
|
|
|
|
limeDestroyHeader(h);
|
|
|
|
|
// std::cout << " File offset is now"<<ftello(File) << std::endl;
|
|
|
|
|
}
|
|
|
|
|
////////////////////////////////////////////
|
|
|
|
|
////////////////////////////////////////////////////
|
|
|
|
|
// Write a generic lattice field and csum
|
|
|
|
|
////////////////////////////////////////////
|
|
|
|
|
// This routine is Collectively called by all nodes
|
|
|
|
|
// in communicator used by the field._grid
|
|
|
|
|
////////////////////////////////////////////////////
|
|
|
|
|
template<class vobj>
|
|
|
|
|
void writeLimeLatticeBinaryObject(Lattice<vobj> &field,std::string record_name)
|
|
|
|
|
{
|
|
|
|
@ -352,6 +366,8 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
////////////////////////////////////////////////////////////////////
|
|
|
|
|
|
|
|
|
|
GridBase *grid = field._grid;
|
|
|
|
|
assert(boss_node == field._grid->IsBoss() );
|
|
|
|
|
|
|
|
|
|
////////////////////////////////////////////
|
|
|
|
|
// Create record header
|
|
|
|
|
////////////////////////////////////////////
|
|
|
|
@ -359,8 +375,10 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
int err;
|
|
|
|
|
uint32_t nersc_csum,scidac_csuma,scidac_csumb;
|
|
|
|
|
uint64_t PayloadSize = sizeof(sobj) * grid->_gsites;
|
|
|
|
|
createLimeRecordHeader(record_name, 0, 0, PayloadSize);
|
|
|
|
|
fflush(File);
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
createLimeRecordHeader(record_name, 0, 0, PayloadSize);
|
|
|
|
|
fflush(File);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// std::cout << "W sizeof(sobj)" <<sizeof(sobj)<<std::endl;
|
|
|
|
|
// std::cout << "W Gsites " <<field._grid->_gsites<<std::endl;
|
|
|
|
@ -369,17 +387,15 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
////////////////////////////////////////////////
|
|
|
|
|
// Check all nodes agree on file position
|
|
|
|
|
////////////////////////////////////////////////
|
|
|
|
|
uint64_t offset1 = ftello(File);
|
|
|
|
|
|
|
|
|
|
uint64_t compare = offset1;
|
|
|
|
|
grid->Broadcast(0,(void *)&compare,sizeof(compare));
|
|
|
|
|
|
|
|
|
|
assert(compare == offset1 );
|
|
|
|
|
uint64_t offset1;
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
offset1 = ftello(File);
|
|
|
|
|
}
|
|
|
|
|
grid->Broadcast(0,(void *)&offset1,sizeof(offset1));
|
|
|
|
|
|
|
|
|
|
///////////////////////////////////////////
|
|
|
|
|
// Write by other means into the binary record
|
|
|
|
|
// The above is collective. Write by other means into the binary record
|
|
|
|
|
///////////////////////////////////////////
|
|
|
|
|
|
|
|
|
|
std::string format = getFormatString<vobj>();
|
|
|
|
|
BinarySimpleMunger<sobj,sobj> munge;
|
|
|
|
|
BinaryIO::writeLatticeObject<vobj,sobj>(field, filename, munge, offset1, format,nersc_csum,scidac_csuma,scidac_csumb);
|
|
|
|
@ -387,21 +403,19 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
///////////////////////////////////////////
|
|
|
|
|
// Wind forward and close the record
|
|
|
|
|
///////////////////////////////////////////
|
|
|
|
|
fseek(File,0,SEEK_END);
|
|
|
|
|
uint64_t offset2 = ftello(File); // std::cout << " now at offset "<<offset2 << std::endl;
|
|
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
|
// Must synchronise the nodes so no race between nodes
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
|
grid->Barrier();
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
fseek(File,0,SEEK_END);
|
|
|
|
|
uint64_t offset2 = ftello(File); // std::cout << " now at offset "<<offset2 << std::endl;
|
|
|
|
|
assert( (offset2-offset1) == PayloadSize);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
|
// Check MPI-2 I/O did what we expect to file
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
|
assert( (offset2-offset1) == PayloadSize);
|
|
|
|
|
|
|
|
|
|
err=limeWriterCloseRecord(LimeW); assert(err>=0);
|
|
|
|
|
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
err=limeWriterCloseRecord(LimeW); assert(err>=0);
|
|
|
|
|
}
|
|
|
|
|
////////////////////////////////////////
|
|
|
|
|
// Write checksum element, propagaing forward from the BinaryIO
|
|
|
|
|
// Always pair a checksum with a binary object, and close message
|
|
|
|
@ -411,21 +425,26 @@ class GridLimeWriter : public BinaryIO {
|
|
|
|
|
std::stringstream streamb; streamb << std::hex << scidac_csumb;
|
|
|
|
|
checksum.suma= streama.str();
|
|
|
|
|
checksum.sumb= streamb.str();
|
|
|
|
|
// std::cout << GridLogMessage<<" writing scidac checksums "<<std::hex<<scidac_csuma<<"/"<<scidac_csumb<<std::dec<<std::endl;
|
|
|
|
|
writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
|
|
|
|
|
if ( boss_node ) {
|
|
|
|
|
writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class ScidacWriter : public GridLimeWriter {
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
|
|
template<class SerialisableUserFile>
|
|
|
|
|
void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
|
|
|
|
|
{
|
|
|
|
|
scidacFile _scidacFile(grid);
|
|
|
|
|
writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
|
|
|
|
|
writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
|
|
|
|
|
}
|
|
|
|
|
ScidacWriter(bool isboss =true ) : GridLimeWriter(isboss) { };
|
|
|
|
|
|
|
|
|
|
template<class SerialisableUserFile>
|
|
|
|
|
void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
|
|
|
|
|
{
|
|
|
|
|
scidacFile _scidacFile(grid);
|
|
|
|
|
if ( this->boss_node ) {
|
|
|
|
|
writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
|
|
|
|
|
writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
////////////////////////////////////////////////
|
|
|
|
|
// Write generic lattice field in scidac format
|
|
|
|
|
////////////////////////////////////////////////
|
|
|
|
@ -446,9 +465,12 @@ class ScidacWriter : public GridLimeWriter {
|
|
|
|
|
//////////////////////////////////////////////
|
|
|
|
|
// Fill the Lime file record by record
|
|
|
|
|
//////////////////////////////////////////////
|
|
|
|
|
writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message
|
|
|
|
|
writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
|
|
|
|
|
writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
|
|
|
|
|
if ( this->boss_node ) {
|
|
|
|
|
writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message
|
|
|
|
|
writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
|
|
|
|
|
writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
|
|
|
|
|
}
|
|
|
|
|
// Collective call
|
|
|
|
|
writeLimeLatticeBinaryObject(field,std::string(ILDG_BINARY_DATA)); // Closes message with checksum
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
@ -515,6 +537,8 @@ class ScidacReader : public GridLimeReader {
|
|
|
|
|
|
|
|
|
|
class IldgWriter : public ScidacWriter {
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
|
|
IldgWriter(bool isboss) : ScidacWriter(isboss) {};
|
|
|
|
|
|
|
|
|
|
///////////////////////////////////
|
|
|
|
|
// A little helper
|
|
|
|
|