diff --git a/benchmarks/Benchmark_IO.cc b/benchmarks/Benchmark_IO.cc new file mode 100644 index 00000000..c8f2692a --- /dev/null +++ b/benchmarks/Benchmark_IO.cc @@ -0,0 +1,109 @@ +#ifdef HAVE_LIME +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + +#define MSG cout << GridLogMessage +#define SEP \ +"=============================================================================" +#ifndef BENCH_IO_LMAX +#define BENCH_IO_LMAX 40 +#endif + +typedef function WriterFn; +typedef function ReaderFn; + +string filestem(const int l) +{ + return "iobench_l" + to_string(l); +} + +void limeWrite(const string filestem, LatticeFermion &vec) +{ + emptyUserRecord record; + ScidacWriter binWriter; + + binWriter.open(filestem + ".bin"); + binWriter.writeScidacFieldRecord(vec, record); + binWriter.close(); +} + +void limeRead(LatticeFermion &vec, const string filestem) +{ + emptyUserRecord record; + ScidacReader binReader; + + binReader.open(filestem + ".bin"); + binReader.readScidacFieldRecord(vec, record); + binReader.close(); +} + +void writeBenchmark(const int l, const WriterFn &write) +{ + auto mpi = GridDefaultMpi(); + auto simd = GridDefaultSimd(Nd, vComplex::Nsimd()); + vector latt = {l*mpi[0], l*mpi[1], l*mpi[2], l*mpi[3]}; + unique_ptr gPt(SpaceTimeGrid::makeFourDimGrid(latt, simd, mpi)); + GridCartesian *g = gPt.get(); + GridParallelRNG rng(g); + LatticeFermion vec(g); + emptyUserRecord record; + ScidacWriter binWriter; + + cout << "-- Local volume " << l << "^4" << endl; + random(rng, vec); + write(filestem(l), vec); +} + +void readBenchmark(const int l, const ReaderFn &read) +{ + auto mpi = GridDefaultMpi(); + auto simd = GridDefaultSimd(Nd, vComplex::Nsimd()); + vector latt = {l*mpi[0], l*mpi[1], l*mpi[2], l*mpi[3]}; + unique_ptr gPt(SpaceTimeGrid::makeFourDimGrid(latt, simd, mpi)); + GridCartesian *g = gPt.get(); + LatticeFermion vec(g); + emptyUserRecord record; + ScidacReader binReader; + + cout << "-- Local volume " << l << "^4" << endl; + read(vec, filestem(l)); +} + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + auto simd = GridDefaultSimd(Nd,vComplex::Nsimd()); + auto mpi = GridDefaultMpi(); + + int64_t threads = GridThread::GetThreads(); + MSG << "Grid is setup to use " << threads << " threads" << endl; + MSG << SEP << endl; + MSG << "Benchmark Lime write" << endl; + MSG << SEP << endl; + for (int l = 4; l <= BENCH_IO_LMAX; l += 2) + { + writeBenchmark(l, limeWrite); + } + + MSG << "Benchmark Lime read" << endl; + MSG << SEP << endl; + for (int l = 4; l <= BENCH_IO_LMAX; l += 2) + { + readBenchmark(l, limeRead); + } + + Grid_finalize(); + + return EXIT_SUCCESS; +} +#else +#include +int main (int argc, char ** argv) +{ + return EXIT_SUCCESS; +} +#endif diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h index 39acf0e0..45fd522e 100644 --- a/lib/parallelIO/BinaryIO.h +++ b/lib/parallelIO/BinaryIO.h @@ -372,7 +372,7 @@ class BinaryIO { std::cout << GridLogMessage <<"IOobject: C++ read I/O " << file << " : " << iodata.size() * sizeof(fobj) << " bytes" << std::endl; std::ifstream fin; - fin.open(file, std::ios::binary | std::ios::in); + fin.open(file, std::ios::binary | std::ios::in); if (control & BINARYIO_MASTER_APPEND) { fin.seekg(-sizeof(fobj), fin.end); @@ -453,11 +453,15 @@ class BinaryIO { std::ofstream fout; fout.exceptions ( std::fstream::failbit | std::fstream::badbit ); try { - fout.open(file,std::ios::binary|std::ios::out|std::ios::in); + if (offset) { // Must already exist and contain data + fout.open(file,std::ios::binary|std::ios::out|std::ios::in); + } else { // Allow create + fout.open(file,std::ios::binary|std::ios::out); + } } catch (const std::fstream::failure& exc) { std::cout << GridLogError << "Error in opening the file " << file << " for output" <_gsites; + uint64_t PayloadSize = sizeof(sobj) * grid->_gsites; createLimeRecordHeader(record_name, 0, 0, PayloadSize); - + fflush(File); + // std::cout << "W sizeof(sobj)" <_gsites<Broadcast(0,(void *)&compare,sizeof(compare)); + + assert(compare == offset1 ); /////////////////////////////////////////// // Write by other means into the binary record /////////////////////////////////////////// - uint64_t offset1 = ftello(File); // std::cout << " Writing to offset "<(); BinarySimpleMunger munge; BinaryIO::writeLatticeObject(field, filename, munge, offset1, format,nersc_csum,scidac_csuma,scidac_csumb); @@ -380,7 +390,15 @@ class GridLimeWriter : public BinaryIO { fseek(File,0,SEEK_END); uint64_t offset2 = ftello(File); // std::cout << " now at offset "<Barrier(); + + ///////////////////////////////////////////////////////////// + // Check MPI-2 I/O did what we expect to file + ///////////////////////////////////////////////////////////// + assert( (offset2-offset1) == PayloadSize); err=limeWriterCloseRecord(LimeW); assert(err>=0); diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h index 5b397e14..281b20f2 100644 --- a/lib/parallelIO/IldgIOtypes.h +++ b/lib/parallelIO/IldgIOtypes.h @@ -136,8 +136,9 @@ struct scidacRecord : Serializable { int, typesize, int, datacount); - scidacRecord() { version =1.0; } - + scidacRecord() + : version(1.0), recordtype(0), colors(0), spins(0), typesize(0), datacount(0) + {} }; //////////////////////// diff --git a/lib/parallelIO/MetaData.h b/lib/parallelIO/MetaData.h index ccc8b18f..55254786 100644 --- a/lib/parallelIO/MetaData.h +++ b/lib/parallelIO/MetaData.h @@ -81,18 +81,16 @@ namespace Grid { std::string, creation_date, std::string, archive_date, std::string, floating_point); - FieldMetaData(void) { - nd=4; - dimension.resize(4); - boundary.resize(4); - scidac_checksuma=0; - scidac_checksumb=0; - checksum=0; - } + // WARNING: non-initialised values might lead to twisted parallel IO + // issues, std::string are fine because they initliase to size 0 + // as per C++ standard. + FieldMetaData(void) + : nd(4), dimension(4,0), boundary(4, ""), data_start(0), + link_trace(0.), plaquette(0.), checksum(0), + scidac_checksuma(0), scidac_checksumb(0), sequence_number(0) + {} }; - - namespace QCD { using namespace Grid;