Tweaked format and memory use on Xml format. Still crashes (out of memory) on large read on my laptop

Grid/serialisation/TextIO.h

@@ -150,8 +150,8 @@ namespace Grid
       NumElements *= d;
     }
     buf.resize( NumElements );
-    for( auto i = 0; i < NumElements; i++ )
+    for( auto &x : buf )
-      read(s, buf[i]);
+      read(s, x);
   }
 }
 

Grid/serialisation/XmlIO.h

@@ -125,24 +125,28 @@ namespace Grid
   template <typename U>
   void XmlWriter::writeDefault(const std::string &s, const std::vector<U> &x)
   {
-    std::vector<size_t> dims(1);
+    push(s);
-    dims[0] = x.size();
+    for( auto &u : x )
-    writeMultiDim(s, dims, &x[0], dims[0]);
+    {
+      write("elem", u);
+    }
+    pop();
   }
 
   template <typename U>
   void XmlWriter::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements)
   {
     push(s);
-    if( Dimensions.size() > 1 )
+    size_t count = 1;
-    {
+    const size_t Rank = Dimensions.size();
-      for( auto d : Dimensions )
+    write("rank", Rank );
-        write("dim", d);
+    for( auto d : Dimensions ) {
+      write("dim", d);
+      count *= d;
     }
+    assert( count == NumElements && "XmlIO : element count doesn't match dimensions" );
     while (NumElements--)
-    {
       write("elem", *pDataRowMajor++);
-    }
     pop();
   }
 
@@ -162,38 +166,46 @@ namespace Grid
   template <typename U>
   void XmlReader::readDefault(const std::string &s, std::vector<U> &output)
   {
-    std::vector<size_t> dims;
-    readMultiDim(s, output, dims);
-    assert( dims.size() == 1 && dims[0] == output.size() && "XmlIO: Expected 1D vector" );
-  }
-
-  template <typename U>
-  void XmlReader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
-  {
-    unsigned int i = 0;
-    unsigned int Rank = 0;
     if (!push(s))
     {
       std::cout << GridLogWarning << "XML: cannot open node '" << s << "'";
       std::cout << std::endl;
     } else {
-      while (node_.child("dim"))
+      for(unsigned int i = 0; node_.child("elem"); )
       {
-        dim.resize(Rank + 1);
+        output.resize(i + 1);
-        read("dim", dim[Rank]);
+        read("elem", output[i++]);
-        node_.child("dim").set_name("dim-done");
+        node_.child("elem").set_name("elem-done");
-        Rank++;
+      }
-      }
+      pop();
-      while (node_.child("elem"))
+    }
-      {
+  }
-        buf.resize(i + 1);
+
-        read("elem", buf[i]);
+  template <typename U>
+  void XmlReader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
+  {
+    if (!push(s))
+    {
+      std::cout << GridLogWarning << "XML: cannot open node '" << s << "'";
+      std::cout << std::endl;
+    } else {
+      size_t Rank;
+      read("rank", Rank);
+      dim.resize( Rank );
+      size_t NumElements = 1;
+      for( auto &d : dim )
+      {
+        read("dim", d);
+        node_.child("dim").set_name("dim-done");
+        NumElements *= d;
+      }
+      buf.resize( NumElements );
+      for( auto &x : buf )
+      {
+        read("elem", x);
         node_.child("elem").set_name("elem-done");
-        i++;
       }
       pop();
-      if( Rank == 0 )
-        dim.push_back(i);
     }
   }
 }

tests/IO/Test_serialisation.cc

@@ -113,7 +113,7 @@ typedef Eigen::Tensor<TestScalar, 3, Eigen::StorageOptions::RowMajor> TensorRank
 typedef Eigen::TensorFixedSize<TestScalar, Eigen::Sizes<9,4,2>, Eigen::StorageOptions::RowMajor> Tensor_9_4_2;
 typedef std::vector<Tensor_9_4_2> aTensor_9_4_2;
 typedef Eigen::TensorFixedSize<SpinColourVector, Eigen::Sizes<6,5>> LSCTensor;
-#ifdef DEBUG
+#ifndef DEBUG
 typedef Eigen::TensorFixedSize<iMatrix<iVector<iMatrix<iVector<LorentzColourMatrix,5>,2>,7>,3>, Eigen::Sizes<2,4,11,10,9>, Eigen::StorageOptions::RowMajor> LCMTensor;
 #endif
 
@@ -193,7 +193,7 @@ void EigenHdf5IOTest(const char * pszExtension)
   }
   TensorWriteRead ( LSCTensor )
   TensorWriteReadLarge( PerambIOTestClass )
-#ifdef DEBUG
+#ifndef DEBUG
   std::cout << "sizeof( LCMTensor ) = " << sizeof( LCMTensor ) / 1024 / 1024 << " MB" << std::endl;
   TensorWriteReadLarge ( LCMTensor )
   // Also write > 4GB of complex numbers (I suspect this will fail inside Hdf5)
@@ -295,12 +295,12 @@ int main(int argc,char **argv)
   std::cout << "\n==== detailed Hdf5 tensor tests (Grid::EigenIO)" << std::endl;
   EigenHdf5IOTest<Hdf5Writer, Hdf5Reader>(".h5");
 #endif
-  std::cout << "\n==== detailed xml tensor tests (Grid::EigenIO)" << std::endl;
-  EigenHdf5IOTest<XmlWriter, XmlReader>(".xml");
   std::cout << "\n==== detailed binary tensor tests (Grid::EigenIO)" << std::endl;
   EigenHdf5IOTest<BinaryWriter, BinaryReader>(".bin");
   std::cout << "\n==== detailed text tensor tests (Grid::EigenIO)" << std::endl;
   EigenHdf5IOTest<TextWriter, TextReader>(".dat");
+  std::cout << "\n==== detailed xml tensor tests (Grid::EigenIO)" << std::endl;
+  EigenHdf5IOTest<XmlWriter, XmlReader>(".xml");
 
   std::cout << "\n==== vector flattening/reconstruction" << std::endl;
   typedef std::vector<std::vector<std::vector<double>>> vec3d;