diff --git a/configure.ac b/configure.ac index 0f3a3018..33a94004 100644 --- a/configure.ac +++ b/configure.ac @@ -130,6 +130,13 @@ AC_CHECK_LIB([fftw3],[fftw_execute], CXXFLAGS=$CXXFLAGS_CPY LDFLAGS=$LDFLAGS_CPY +AC_CHECK_LIB([lime],[limeCreateReader], + [LIBS="$LIBS -llime"], + [AC_MSG_ERROR(C-LIME library was not found in your system. +Please install or provide the correct path to your installation [default search path ~/lime/] +Info at: http://usqcd.jlab.org/usqcd-docs/c-lime/)]) + + ############### SIMD instruction selection AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=SSE4|AVX|AVXFMA4|AVXFMA|AVX2|AVX512|AVX512MIC|IMCI|KNL|KNC],\ [Select instructions to be SSE4.0, AVX 1.0, AVX 2.0+FMA, AVX 512, IMCI])],\ diff --git a/lib/Grid.h b/lib/Grid.h index d2e12d29..5db4a0d2 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -76,10 +76,13 @@ Author: paboyle #include #include #include -#include #include +#include +#include +#include #include + #include #include diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h index abba1880..3b2a8912 100644 --- a/lib/parallelIO/BinaryIO.h +++ b/lib/parallelIO/BinaryIO.h @@ -348,8 +348,8 @@ class BinaryIO { grid->GlobalIndexToGlobalCoor(gidx, gcoor); grid->GlobalCoorToRankIndex(rank, o_idx, i_idx, gcoor); int l_idx = parallel.generator_idx(o_idx, i_idx); - std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx - << " i_idx " << i_idx << " rank " << rank << std::endl; + //std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx + // << " i_idx " << i_idx << " rank " << rank << std::endl; if (rank == grid->ThisRank()) { parallel.GetState(saved, l_idx); } @@ -357,7 +357,6 @@ class BinaryIO { grid->Barrier(); // necessary? if (grid->IsBoss()) { - std::cout << "Saved: " << saved << std::endl; Uint32Checksum((uint32_t *)&saved[0], bytes, csum); fout.write((char *)&saved[0], bytes); } @@ -422,12 +421,11 @@ class BinaryIO { grid->GlobalIndexToGlobalCoor(gidx,gcoor); grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor); int l_idx=parallel.generator_idx(o_idx,i_idx); - std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx - << " i_idx " << i_idx << " rank " << rank << std::endl; + //std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx + // << " i_idx " << i_idx << " rank " << rank << std::endl; if ( grid->IsBoss() ) { fin.read((char *)&saved[0],bytes); - std::cout << "Saved: " << saved << std::endl; Uint32Checksum((uint32_t *)&saved[0],bytes,csum); } @@ -608,7 +606,8 @@ class BinaryIO { static inline uint32_t writeObjectParallel(Lattice &Umu, std::string file, munger munge, int offset, - const std::string &format) { + const std::string &format, + ILDGtype ILDG = ILDGtype()) { typedef typename vobj::scalar_object sobj; GridBase *grid = Umu._grid; @@ -682,13 +681,15 @@ class BinaryIO { // Ideally one reader/writer per xy plane and read these contiguously // with comms from nominated I/O nodes. std::ofstream fout; - if (IOnode){ - fout.open(file, std::ios::binary | std::ios::in | std::ios::out); - if (!fout.is_open()) { - std::cout << GridLogMessage << "writeObjectParallel: Error opening file " << file - << std::endl; - exit(0); - } + if (!ILDG.is_ILDG){ + if (IOnode){ + fout.open(file, std::ios::binary | std::ios::in | std::ios::out); + if (!fout.is_open()) { + std::cout << GridLogMessage << "writeObjectParallel: Error opening file " << file + << std::endl; + exit(0); + } + } } ////////////////////////////////////////////////////////// @@ -752,8 +753,13 @@ class BinaryIO { if (ieee64big) htobe64_v((void *)&fileObj, sizeof(fileObj)); if (ieee64) htole64_v((void *)&fileObj, sizeof(fileObj)); - fout.seekp(offset + g_idx * sizeof(fileObj)); - fout.write((char *)&fileObj, sizeof(fileObj)); + if (ILDG.is_ILDG){ + size_t sizeFO = sizeof(fileObj); + int status = limeWriteRecordData((char*)&fileObj, &sizeFO, ILDG.LW); + } else{ + fout.seekp(offset + g_idx * sizeof(fileObj)); + fout.write((char *)&fileObj, sizeof(fileObj)); + } bytes += sizeof(fileObj); } } diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h new file mode 100644 index 00000000..e723d3f8 --- /dev/null +++ b/lib/parallelIO/IldgIO.h @@ -0,0 +1,194 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./lib/parallelIO/IldgIO.h + +Copyright (C) 2015 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_ILDG_IO_H +#define GRID_ILDG_IO_H + +#include +#include +#include +#include +#include + +#include +#include +#include + +extern "C" { // for linkage +#include "lime.h" +} + +namespace Grid { +namespace QCD { + +inline void ILDGGrid(GridBase *grid, ILDGField &header) { + assert(grid->_ndimension == 4); // emit error if not + header.dimension.resize(4); + header.boundary.resize(4); + for (int d = 0; d < 4; d++) { + header.dimension[d] = grid->_fdimensions[d]; + // Read boundary conditions from ... ? + header.boundary[d] = std::string("periodic"); + } +} + +inline void ILDGChecksum(uint32_t *buf, uint32_t buf_size_bytes, + uint32_t &csum) { + BinaryIO::Uint32Checksum(buf, buf_size_bytes, csum); +} + +////////////////////////////////////////////////////////////////////// +// Utilities ; these are QCD aware +////////////////////////////////////////////////////////////////////// +template +inline void ILDGStatistics(GaugeField &data, ILDGField &header) { + // How to convert data precision etc... + header.link_trace = Grid::QCD::WilsonLoops::linkTrace(data); + header.plaquette = Grid::QCD::WilsonLoops::avgPlaquette(data); + // header.polyakov = +} + +// Forcing QCD here +template +struct ILDGMunger { + void operator()(fobj &in, sobj &out, uint32_t &csum) { + for (int mu = 0; mu < 4; mu++) { + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + out(mu)()(i, j) = in(mu)()(i, j); + } + } + } + ILDGChecksum((uint32_t *)&in, sizeof(in), csum); + }; +}; + +template +struct ILDGSimpleUnmunger { + void operator()(sobj &in, fobj &out, uint32_t &csum) { + for (int mu = 0; mu < 4; mu++) { + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + out(mu)()(i, j) = in(mu)()(i, j); + } + } + } + ILDGChecksum((uint32_t *)&out, sizeof(out), csum); + }; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Write and read from fstream; compute header offset for payload +//////////////////////////////////////////////////////////////////////////////// +class ILDGIO : public BinaryIO { + FILE *outFile; + LimeWriter *LimeW; + LimeRecordHeader *LimeHeader; + + public: + ILDGIO(std::string file) { + outFile = fopen(file.c_str(), "w"); + // check if opened correctly + + LimeW = limeCreateWriter(outFile); + } + + ~ILDGIO() { fclose(outFile); } + + unsigned int writeHeader(ILDGField &header) { + // write header in LIME + n_uint64_t nbytes; + int MB_flag = 1, ME_flag = 0; + + char message[] = "ildg-format"; + nbytes = strlen(message); + LimeHeader = limeCreateHeader(MB_flag, ME_flag, message, nbytes); + limeWriteRecordHeader(LimeHeader, LimeW); + limeDestroyHeader(LimeHeader); + // save the xml header here + // use the xml_writer to c++ streams in pugixml + // and convert to char message + // limeWriteRecordData(message, &nbytes, LimeW); + limeWriterCloseRecord(LimeW); + + return 0; + } + + unsigned int readHeader(std::string file, GridBase *grid, ILDGField &field) { + return 0; + } + + template + int readConfiguration(Lattice > &Umu, + ILDGField &header, std::string file) { + typedef Lattice > GaugeField; + + return 0; + } + + template + int writeConfiguration(Lattice > &Umu, + ILDGField &header, std::string file) { + typedef Lattice > GaugeField; + typedef iLorentzColourMatrix vobj; + typedef typename vobj::scalar_object sobj; + typedef LorentzColourMatrixD fobj; + + ILDGSimpleUnmunger munge; + unsigned int offset = writeHeader(header); + + BinaryIO::Uint32Checksum(Umu, munge, header.checksum); + + // Write record header + LimeRecordHeader *h; + std::cout << GridLogDebug << "ILDG Creating Header" << std::endl; + char message[] = "ildg-binary-data"; + h = limeCreateHeader(1, 1, message, strlen(message)); + + std::cout << GridLogDebug << "ILDG Writing Header" << std::endl; + int status = limeWriteRecordHeader(h, LimeW); + + if (status < 0) { + std::cerr << "ILDG Header error\n"; + return 1; + } + + limeDestroyHeader(h); + + ILDGtype ILDGt(true, LimeW); + uint32_t csum = BinaryIO::writeObjectParallel( + Umu, file, munge, offset, header.floating_point, ILDGt); + + limeWriterCloseRecord(LimeW); + + return 0; + } + + // format for RNG? +}; +} +} +#endif diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h new file mode 100644 index 00000000..bbee2a53 --- /dev/null +++ b/lib/parallelIO/IldgIOtypes.h @@ -0,0 +1,74 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./lib/parallelIO/IldgIO.h + +Copyright (C) 2015 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_ILDGTYPES_IO_H +#define GRID_ILDGTYPES_IO_H + +extern "C" { // for linkage +#include "lime.h" +} + +namespace Grid { + + struct ILDGtype{ + bool is_ILDG; + LimeWriter* LW; + + ILDGtype(bool is, LimeWriter* L):is_ILDG(is),LW(L){} + ILDGtype():is_ILDG(false),LW(NULL){} + }; + + + + + class ILDGField { + public: + // header strings (not in order) + std::vector dimension; + std::vector boundary; + int data_start; + std::string hdr_version; + std::string storage_format; + // Checks on data + double link_trace; + double plaquette; + uint32_t checksum; + unsigned int sequence_number; + std::string data_type; + std::string ensemble_id ; + std::string ensemble_label ; + std::string creator ; + std::string creator_hardware ; + std::string creation_date ; + std::string archive_date ; + std::string floating_point; + }; + + + + +} +#endif diff --git a/lib/parallelIO/NerscIO.h b/lib/parallelIO/NerscIO.h index 6fdf83ef..91e622db 100644 --- a/lib/parallelIO/NerscIO.h +++ b/lib/parallelIO/NerscIO.h @@ -397,9 +397,7 @@ static inline void writeConfiguration(Lattice > &Umu typedef LorentzColourMatrixD fobj3D; typedef LorentzColour2x3D fobj2D; - //typedef LorentzColourMatrixF fobj3f; - //typedef LorentzColour2x3F fobj2f; - + GridBase *grid = Umu._grid; NerscGrid(grid,header);