From 47c7159177a6a341bcd273d592e0b0c75346ffb9 Mon Sep 17 00:00:00 2001 From: Guido Cossu Date: Mon, 24 Oct 2016 21:57:54 +0100 Subject: [PATCH] ILDG reader/writer works Fill the xml header with the required information, todo. --- lib/Grid.h | 1 + lib/parallelIO/BinaryIO.h | 46 +++++---- lib/parallelIO/IldgIO.h | 115 +++++++++++++++------ lib/parallelIO/IldgIOtypes.h | 18 ++-- lib/parallelIO/NerscIO.h | 7 +- lib/qcd/hmc/ILDGCheckpointer.h | 121 +++++++++++++++++++++++ tests/hmc/Test_hmc_WilsonGauge_Binary.cc | 2 +- 7 files changed, 248 insertions(+), 62 deletions(-) create mode 100644 lib/qcd/hmc/ILDGCheckpointer.h diff --git a/lib/Grid.h b/lib/Grid.h index 5db4a0d2..03921400 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -87,6 +87,7 @@ Author: paboyle #include #include +#include #include #include diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h index 3b2a8912..2b7747ba 100644 --- a/lib/parallelIO/BinaryIO.h +++ b/lib/parallelIO/BinaryIO.h @@ -449,10 +449,13 @@ class BinaryIO { return csum; } - - template - static inline uint32_t readObjectParallel(Lattice &Umu,std::string file,munger munge,int offset,const std::string &format) - { + template + static inline uint32_t readObjectParallel(Lattice &Umu, + std::string file, + munger munge, + int offset, + const std::string &format, + ILDGtype ILDG = ILDGtype()) { typedef typename vobj::scalar_object sobj; GridBase *grid = Umu._grid; @@ -518,9 +521,10 @@ class BinaryIO { int myrank = grid->ThisRank(); int iorank = grid->RankFromProcessorCoor(ioproc); - if ( IOnode ) { - fin.open(file,std::ios::binary|std::ios::in); - } + if (!ILDG.is_ILDG) + if ( IOnode ) { + fin.open(file,std::ios::binary|std::ios::in); + } ////////////////////////////////////////////////////////// // Find the location of each site and send to primary node @@ -562,8 +566,15 @@ class BinaryIO { //////////////////////////////// if (myrank == iorank) { - fin.seekg(offset+g_idx*sizeof(fileObj)); - fin.read((char *)&fileObj,sizeof(fileObj)); + if (ILDG.is_ILDG){ + // use C-LIME to populate the record + size_t sizeFO = sizeof(fileObj); + limeReaderSeek(ILDG.LR, g_idx*sizeFO, SEEK_SET); + int status = limeReaderReadData((void *)&fileObj, &sizeFO, ILDG.LR); + } else{ + fin.seekg(offset+g_idx*sizeof(fileObj)); + fin.read((char *)&fileObj,sizeof(fileObj)); + } bytes+=sizeof(fileObj); if(ieee32big) be32toh_v((void *)&fileObj,sizeof(fileObj)); @@ -681,7 +692,7 @@ class BinaryIO { // Ideally one reader/writer per xy plane and read these contiguously // with comms from nominated I/O nodes. std::ofstream fout; - if (!ILDG.is_ILDG){ + if (!ILDG.is_ILDG) if (IOnode){ fout.open(file, std::ios::binary | std::ios::in | std::ios::out); if (!fout.is_open()) { @@ -690,7 +701,7 @@ class BinaryIO { exit(0); } } - } + ////////////////////////////////////////////////////////// // Find the location of each site and send to primary node @@ -753,12 +764,13 @@ class BinaryIO { if (ieee64big) htobe64_v((void *)&fileObj, sizeof(fileObj)); if (ieee64) htole64_v((void *)&fileObj, sizeof(fileObj)); - if (ILDG.is_ILDG){ - size_t sizeFO = sizeof(fileObj); - int status = limeWriteRecordData((char*)&fileObj, &sizeFO, ILDG.LW); - } else{ - fout.seekp(offset + g_idx * sizeof(fileObj)); - fout.write((char *)&fileObj, sizeof(fileObj)); + if (ILDG.is_ILDG) { + size_t sizeFO = sizeof(fileObj); + limeWriterSeek(ILDG.LW, g_idx*sizeFO, SEEK_SET); + int status = limeWriteRecordData((void *)&fileObj, &sizeFO, ILDG.LW); + } else { + fout.seekp(offset + g_idx * sizeof(fileObj)); + fout.write((char *)&fileObj, sizeof(fileObj)); } bytes += sizeof(fileObj); } diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h index e723d3f8..ad8fcfd3 100644 --- a/lib/parallelIO/IldgIO.h +++ b/lib/parallelIO/IldgIO.h @@ -87,7 +87,7 @@ struct ILDGMunger { }; template -struct ILDGSimpleUnmunger { +struct ILDGUnmunger { void operator()(sobj &in, fobj &out, uint32_t &csum) { for (int mu = 0; mu < 4; mu++) { for (int i = 0; i < 3; i++) { @@ -103,20 +103,45 @@ struct ILDGSimpleUnmunger { //////////////////////////////////////////////////////////////////////////////// // Write and read from fstream; compute header offset for payload //////////////////////////////////////////////////////////////////////////////// +enum ILDGstate {ILDGread, ILDGwrite}; + class ILDGIO : public BinaryIO { - FILE *outFile; + FILE *File; LimeWriter *LimeW; LimeRecordHeader *LimeHeader; + LimeReader *LimeR; + std::string filename; + public: - ILDGIO(std::string file) { - outFile = fopen(file.c_str(), "w"); - // check if opened correctly + ILDGIO(std::string file, ILDGstate RW) { + filename = file; + if (RW == ILDGwrite){ + File = fopen(file.c_str(), "w"); + // check if opened correctly - LimeW = limeCreateWriter(outFile); + LimeW = limeCreateWriter(File); + } else { + File = fopen(file.c_str(), "r"); + // check if opened correctly + + LimeR = limeCreateReader(File); + } } - ~ILDGIO() { fclose(outFile); } + ~ILDGIO() { fclose(File); } + + int createHeader(std::string message, int MB, int ME, size_t PayloadSize, LimeWriter* L){ + LimeRecordHeader *h; + h = limeCreateHeader(MB, ME, const_cast(message.c_str()), PayloadSize); + int status = limeWriteRecordHeader(h, L); + if (status < 0) { + std::cerr << "ILDG Header error\n"; + return status; + } + limeDestroyHeader(h); + return LIME_SUCCESS; + } unsigned int writeHeader(ILDGField &header) { // write header in LIME @@ -131,60 +156,86 @@ class ILDGIO : public BinaryIO { // save the xml header here // use the xml_writer to c++ streams in pugixml // and convert to char message - // limeWriteRecordData(message, &nbytes, LimeW); + limeWriteRecordData(message, &nbytes, LimeW); limeWriterCloseRecord(LimeW); return 0; } - unsigned int readHeader(std::string file, GridBase *grid, ILDGField &field) { + unsigned int readHeader(ILDGField &header) { return 0; } template - int readConfiguration(Lattice > &Umu, - ILDGField &header, std::string file) { + uint32_t readConfiguration(Lattice > &Umu) { typedef Lattice > GaugeField; + typedef LorentzColourMatrixD sobjd; + typedef LorentzColourMatrixF sobjf; + typedef iLorentzColourMatrix itype; + typedef LorentzColourMatrix sobj; + GridBase *grid = Umu._grid; - return 0; + ILDGField header; + readHeader(header); + + // now just the conf, ignore the header + std::string format = std::string("IEEE64BIG"); + do {limeReaderNextRecord(LimeR);} + while (strncmp(limeReaderType(LimeR), "ildg-binary-data",16)); + + n_uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration) + + + ILDGtype ILDGt(true, LimeR); + // this is special for double prec data, just for the moment + uint32_t csum = BinaryIO::readObjectParallel< itype, sobjd >( + Umu, filename, ILDGMunger(), 0, format, ILDGt); + + // Check configuration + // todo + + return csum; } template - int writeConfiguration(Lattice > &Umu, - ILDGField &header, std::string file) { + uint32_t writeConfiguration(Lattice > &Umu, std::string format) { typedef Lattice > GaugeField; typedef iLorentzColourMatrix vobj; typedef typename vobj::scalar_object sobj; typedef LorentzColourMatrixD fobj; - ILDGSimpleUnmunger munge; + ILDGField header; + // fill the header + header.floating_point = format; + + ILDGUnmunger munge; unsigned int offset = writeHeader(header); BinaryIO::Uint32Checksum(Umu, munge, header.checksum); - // Write record header - LimeRecordHeader *h; - std::cout << GridLogDebug << "ILDG Creating Header" << std::endl; - char message[] = "ildg-binary-data"; - h = limeCreateHeader(1, 1, message, strlen(message)); - - std::cout << GridLogDebug << "ILDG Writing Header" << std::endl; - int status = limeWriteRecordHeader(h, LimeW); - - if (status < 0) { - std::cerr << "ILDG Header error\n"; - return 1; - } - - limeDestroyHeader(h); + // Write data record header + n_uint64_t PayloadSize = sizeof(fobj) * Umu._grid->_gsites; + createHeader("ildg-binary-data", 0, 1, PayloadSize, LimeW); ILDGtype ILDGt(true, LimeW); uint32_t csum = BinaryIO::writeObjectParallel( - Umu, file, munge, offset, header.floating_point, ILDGt); + Umu, filename, munge, 0, header.floating_point, ILDGt); limeWriterCloseRecord(LimeW); - return 0; + // Last record + // the logical file name LNF + // look into documentation on how to generate this string + std::string LNF = "empty"; + + + PayloadSize = sizeof(LNF); + createHeader("ildg-binary-lfn", 1 , 1, PayloadSize, LimeW); + limeWriteRecordData(const_cast(LNF.c_str()), &PayloadSize, LimeW); + + limeWriterCloseRecord(LimeW); + + return csum; } // format for RNG? diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h index bbee2a53..455fef41 100644 --- a/lib/parallelIO/IldgIOtypes.h +++ b/lib/parallelIO/IldgIOtypes.h @@ -33,14 +33,16 @@ extern "C" { // for linkage namespace Grid { - struct ILDGtype{ - bool is_ILDG; - LimeWriter* LW; - - ILDGtype(bool is, LimeWriter* L):is_ILDG(is),LW(L){} - ILDGtype():is_ILDG(false),LW(NULL){} - }; - + struct ILDGtype{ + bool is_ILDG; + LimeWriter* LW; + LimeReader* LR; + + ILDGtype(bool is, LimeWriter* L):is_ILDG(is),LW(L),LR(NULL){} + ILDGtype(bool is, LimeReader* L):is_ILDG(is),LW(NULL),LR(L){} + ILDGtype():is_ILDG(false),LW(NULL),LR(NULL){} + }; + diff --git a/lib/parallelIO/NerscIO.h b/lib/parallelIO/NerscIO.h index 91e622db..0c64f414 100644 --- a/lib/parallelIO/NerscIO.h +++ b/lib/parallelIO/NerscIO.h @@ -424,8 +424,8 @@ static inline void writeConfiguration(Lattice > &Umu //int csum1=BinaryIO::writeObjectSerial(Umu,file1,munge,offset,header.floating_point); - std::cout << GridLogMessage << " TESTING PARALLEL WRITE offsets " << offset1 << " "<< offset << std::endl; - std::cout << GridLogMessage << " TESTING PARALLEL WRITE csums " << csum1 << " "< > &Umu NerscSimpleUnmunger munge; BinaryIO::Uint32Checksum(Umu, munge,header.checksum); offset = writeHeader(header,file); - // csum=BinaryIO::writeObjectSerial(Umu,file,munge,offset,header.floating_point); csum=BinaryIO::writeObjectParallel(Umu,file,munge,offset,header.floating_point); } - std::cout< +#include +#include + +namespace Grid { +namespace QCD { + +// Only for Gauge fields +template +class ILDGHmcCheckpointer + : public HmcObservable { + private: + std::string configStem; + std::string rngStem; + int SaveInterval; + std::string format; + + public: + INHERIT_GIMPL_TYPES(Implementation); // + + ILDGHmcCheckpointer(std::string cf, std::string rn, int savemodulo, + std::string form = "IEEE64BIG") { + configStem = cf; + rngStem = rn; + SaveInterval = savemodulo; + format = form; + + // check here that the format is valid + int ieee32big = (format == std::string("IEEE32BIG")); + int ieee32 = (format == std::string("IEEE32")); + int ieee64big = (format == std::string("IEEE64BIG")); + int ieee64 = (format == std::string("IEEE64")); + + if (!(ieee64big ^ ieee32 ^ ieee32big ^ ieee64)) { + std::cout << GridLogMessage << "Invalid format: " << format << std::endl; + exit(0); + } + }; + + void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG, + GridParallelRNG &pRNG) { + if ((traj % SaveInterval) == 0) { + std::string rng; + { + std::ostringstream os; + os << rngStem << "." << traj; + rng = os.str(); + } + std::string config; + { + std::ostringstream os; + os << configStem << "." << traj; + config = os.str(); + } + + ILDGIO IO(config, ILDGwrite); + BinaryIO::writeRNGSerial(sRNG, pRNG, rng, 0); + uint32_t csum = IO.writeConfiguration(U, format); + + std::cout << GridLogMessage << "Written ILDG Configuration on " << config + << " checksum " << std::hex << csum << std::dec << std::endl; + } + }; + + void CheckpointRestore(int traj, GaugeField &U, GridSerialRNG &sRNG, + GridParallelRNG &pRNG) { + std::string rng; + { + std::ostringstream os; + os << rngStem << "." << traj; + rng = os.str(); + } + std::string config; + { + std::ostringstream os; + os << configStem << "." << traj; + config = os.str(); + } + + ILDGIO IO(config, ILDGread); + BinaryIO::readRNGSerial(sRNG, pRNG, rng, 0); + uint32_t csum = IO.readConfiguration(U);// format from the header + + std::cout << GridLogMessage << "Read ILDG Configuration from " << config + << " checksum " << std::hex << csum << std::dec << std::endl; + }; +}; +} +} +#endif diff --git a/tests/hmc/Test_hmc_WilsonGauge_Binary.cc b/tests/hmc/Test_hmc_WilsonGauge_Binary.cc index aed66d20..65b055ef 100644 --- a/tests/hmc/Test_hmc_WilsonGauge_Binary.cc +++ b/tests/hmc/Test_hmc_WilsonGauge_Binary.cc @@ -81,7 +81,7 @@ class HmcRunner : public BinaryHmcRunner { TheAction.push_back(Level1); // Add observables - BinaryHmcCheckpointer Checkpoint( + ILDGHmcCheckpointer Checkpoint( HMCPar.conf_prefix, HMCPar.rng_prefix, HMCPar.SaveInterval, HMCPar.format); // Can implement also a specific function in the hmcrunner // AddCheckpoint (...) that takes the same parameters + a string/tag