Memor footprint monitor

Grid/GridCore.h

@@ -56,6 +56,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/communicator/Communicator.h> 
 #include <Grid/communicator/RingAllReduce.h>
 #include <Grid/cartesian/Cartesian.h>
+#include <Grid/perfmon/HostMemory.h>
 #include <Grid/tensors/Tensors.h>      
 #include <Grid/lattice/Lattice.h>      
 #include <Grid/cshift/Cshift.h>       

Grid/algorithms/iterative/ConjugateGradientMixedPrecBatched.h

@@ -98,6 +98,8 @@ public:
 
     GridStopWatch InnerCGtimer;
     GridStopWatch PrecChangeTimer;
+    GridStopWatch OuterResidualTimer;
+    GridStopWatch PatchupTimer;
     
     int cb = src_d_in[0].Checkerboard();
     
@@ -138,16 +140,19 @@ public:
     CG_f.ErrorOnNoConverge = false;
     
     //Optionally clone the single precision operator onto split-communicator partitions
-    GridStopWatch SplitTimer;
+    GridStopWatch SplitCloneTimer;
+    SplitTimers   splitTimers;
     SplitOperator<FieldF> *split = nullptr;
     int partitions;
     Coordinate layout = BatchedSolverSplitLayout(SinglePrecGrid,BatchedSplit,BatchedSplitNode,NBatch,partitions);
     if ( partitions > 1 ) {
-      SplitTimer.Start();
+      SplitCloneTimer.Start();
       split = Linop_f.SplitClone(layout);
-      SplitTimer.Stop();
+      SplitCloneTimer.Stop();
       if ( split == nullptr ) {
         std::cout << GridLogMessage << "MixedPrecisionConjugateGradientBatched: operator cannot be split; serial inner solves" << std::endl;
+      } else {
+        HostMemoryReport(DoublePrecGrid,GridLogMessage,"MixedPrecisionConjugateGradientBatched: after split clone");
       }
     }
     
@@ -156,13 +161,16 @@ public:
     for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
       std::cout << GridLogMessage << std::endl;
       std::cout << GridLogMessage << "Outer iteration " << outer_iter << std::endl;
+      HostMemoryReport(DoublePrecGrid,GridLogMessage,"MixedPrecisionConjugateGradientBatched: outer iteration "+std::to_string(outer_iter));
       
       bool allConverged = true;
       
       for (int i=0; i<NBatch; i++) {
         //Compute double precision rsd and also new RHS vector.
+        OuterResidualTimer.Start();
         Linop_d.HermOp(sol_d[i], tmp_d);
         norm[i] = axpy_norm(src_d[i], -1., tmp_d, src_d_in[i]); //src_d is residual vector
+        OuterResidualTimer.Stop();
         
         std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Outer iteration " << outer_iter <<" solve " << i << " residual "<< norm[i] << " target "<< stop[i] <<std::endl;
 
@@ -191,7 +199,7 @@ public:
       }
 
       if ( split != nullptr ) {
-        InnerSplitSolves(*split, CG_f, src_f, sol_f, TotalInnerIterations, InnerCGtimer, SplitTimer);
+        InnerSplitSolves(*split, CG_f, src_f, sol_f, TotalInnerIterations, InnerCGtimer, splitTimers);
       }
 
       for (int i=0; i<NBatch; i++) {
@@ -213,17 +221,21 @@ public:
 
     }
 
+    SplitCloneTimer.Start();
     delete split;
+    SplitCloneTimer.Stop();
     
     //Final trial CG
     std::cout << GridLogMessage << std::endl;
     std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Starting final patch-up double-precision solve"<<std::endl;
     
+    PatchupTimer.Start();
     for (int i=0; i<NBatch; i++) {
       ConjugateGradient<FieldD> CG_d(Tolerance, MaxPatchupIterations);
       CG_d(Linop_d, src_d_in[i], sol_d[i]);
       TotalFinalStepIterations[i] += CG_d.IterationsToComplete;
     }
+    PatchupTimer.Stop();
 
     TotalTimer.Stop();
 
@@ -232,12 +244,37 @@ public:
       std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: solve " << i << " Inner CG iterations " << TotalInnerIterations[i] << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations[i] << std::endl;
     }
     std::cout << GridLogMessage << std::endl;
-    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << " Split setup and transfer " << SplitTimer.Elapsed() << std::endl;
+    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
+    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Outer residual " << OuterResidualTimer.Elapsed() << " Patch-up " << PatchupTimer.Elapsed() << std::endl;
+    if ( partitions > 1 ) {
+      std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Split clone and delete " << SplitCloneTimer.Elapsed()
+               << " Grid_split " << splitTimers.Split.Elapsed() << " (" << splitTimers.SplitCalls << " calls)"
+               << " Grid_unsplit " << splitTimers.Unsplit.Elapsed() << " (" << splitTimers.UnsplitCalls << " calls)"
+               << " Staging copies " << splitTimers.Staging.Elapsed()
+               << " Host to device " << splitTimers.H2D.Elapsed()
+               << " Iteration count sum " << splitTimers.Reduce.Elapsed() << std::endl;
+    }
+    double accounted = PrecChangeTimer.useconds() + InnerCGtimer.useconds() + OuterResidualTimer.useconds()
+                     + PatchupTimer.useconds() + SplitCloneTimer.useconds() + splitTimers.Split.useconds()
+                     + splitTimers.Unsplit.useconds() + splitTimers.Staging.useconds() + splitTimers.H2D.useconds()
+                     + splitTimers.Reduce.useconds();
+    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Unaccounted " << (TotalTimer.useconds()-accounted)/1.0e6 << " s" << std::endl;
     
   }
 
 private:
 
+  // Coarse breakdown of the split path outside the inner CG
+  struct SplitTimers {
+    GridStopWatch Split;
+    GridStopWatch Unsplit;
+    GridStopWatch Staging;
+    GridStopWatch H2D;
+    GridStopWatch Reduce;
+    int SplitCalls   = 0;
+    int UnsplitCalls = 0;
+  };
+
   ////////////////////////////////////////////////////////////////////////////////////////
   // One inner solve per partition, in groups of Partitions right-hand sides. The last
   // group is zero-padded; a zero source returns at once from CG. Collective.
@@ -248,7 +285,7 @@ private:
                         std::vector<FieldF> &sol_f,
                         std::vector<Integer> &TotalInnerIterations,
                         GridStopWatch &InnerCGtimer,
-                        GridStopWatch &SplitTimer)
+                        SplitTimers &timers)
   {
     int NBatch = src_f.size();
     int P      = split.Partitions;
@@ -263,6 +300,7 @@ private:
     for(int g=0;g<NBatch;g+=P){
 
       // Gather the group; zero-pad past the end of the batch
+      timers.Staging.Start();
       for(int p=0;p<P;p++){
         group_src[p].Checkerboard() = cb;
         group_sol[p].Checkerboard() = cb;
@@ -274,22 +312,34 @@ private:
           group_sol[p] = Zero();
         }
       }
+      timers.Staging.Stop();
 
       // The initial guess (e.g. from the guesser) travels with the source
-      SplitTimer.Start();
+      timers.Split.Start();
       Grid_split(group_src,s_src);
       Grid_split(group_sol,s_sol);
-      SplitTimer.Stop();
+      timers.Split.Stop();
+      timers.SplitCalls += 2;
+
+      // Grid_split leaves its output on the host; move it now so the copy is not in the CG time
+      timers.H2D.Start();
+      {
+        autoView(s_src_v, s_src, AcceleratorRead);
+        autoView(s_sol_v, s_sol, AcceleratorRead);
+      }
+      timers.H2D.Stop();
 
       InnerCGtimer.Start();
       CG_f(*split.Linop,s_src,s_sol);
       InnerCGtimer.Stop();
 
-      SplitTimer.Start();
+      timers.Unsplit.Start();
       Grid_unsplit(group_sol,s_sol);
-      SplitTimer.Stop();
+      timers.Unsplit.Stop();
+      timers.UnsplitCalls += 1;
 
       // One iteration count per partition, contributed by the partition's rank 0 only
+      timers.Reduce.Start();
       for(int p=0;p<P;p++){
         iters[p] = 0;
       }
@@ -297,11 +347,15 @@ private:
         iters[split.Partition] = CG_f.IterationsToComplete;
       }
       SinglePrecGrid->GlobalSumVector(&iters[0],P);
+      timers.Reduce.Stop();
 
+      // Includes the host to device copy of group_sol left by Grid_unsplit
+      timers.Staging.Start();
       for(int p=0;p<P && g+p<NBatch;p++){
         sol_f[g+p] = group_sol[p];
         TotalInnerIterations[g+p] += iters[p];
       }
+      timers.Staging.Stop();
     }
   }
 };

Grid/lattice/Lattice_transfer.h

@@ -1637,17 +1637,35 @@ void Grid_split(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
     ratio[d] = full_grid->_processors[d]/ split_grid->_processors[d];
   }
 
+  // Coarse phase timers, reported on GridLogPerformance
+  double t_start   = usecond();
+  double t_d2h     = 0;
+  double t_unvec   = 0;
+  double t_a2a     = 0;
+  double t_reorder = 0;
+  double t_vec     = 0;
+  double t0;
+
   uint64_t lsites = full_grid->lSites();
   uint64_t     sz = lsites * nvector;
   std::vector<Sobj> tmpdata(sz);
   std::vector<Sobj> alldata(sz);
   std::vector<Sobj> scalardata(lsites); 
+  double t_alloc = usecond() - t_start;
 
   for(int v=0;v<nvector;v++){
+    // Open and close a host view first so any device to host copy is timed on its own
+    t0 = usecond();
+    {
+      autoView(full_v, full[v], CpuRead);
+    }
+    t_d2h += usecond() - t0;
+    t0 = usecond();
     unvectorizeToLexOrdArray(scalardata,full[v]);    
     thread_for(site,lsites,{
       alldata[v*lsites+site] = scalardata[site];
     });
+    t_unvec += usecond() - t0;
   }
 
   int nvec = nvector; // Counts down to 1 as we collapse dims
@@ -1657,11 +1675,14 @@ void Grid_split(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
 
     if ( ratio[d] != 1 ) {
 
+      t0 = usecond();
       full_grid ->AllToAll(d,alldata,tmpdata);
       if ( split_grid->_processors[d] > 1 ) {
 	alldata=tmpdata;
 	split_grid->AllToAll(d,alldata,tmpdata);
       }
+      t_a2a += usecond() - t0;
+      t0 = usecond();
 
       auto rdims = ldims; 
       auto     M = ratio[d];
@@ -1701,12 +1722,29 @@ void Grid_split(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
 	  }
 	}
       });
+      t_reorder += usecond() - t0;
       ldims[d]*= ratio[d];
       lsites  *= ratio[d];
 
     }
   }
+  t0 = usecond();
   vectorizeFromLexOrdArray(alldata,split);    
+  t_vec = usecond() - t0;
+  double t_total = usecond() - t_start;
+
+  // Host to device copy of split happens lazily at its first accelerator use; not counted here
+  std::cout << GridLogPerformance << "Grid_split: " << nvector << " x " << sz/nvector*sizeof(Sobj)/1.0e6
+            << " MB/rank total " << t_total/1.0e6 << " s : alloc " << t_alloc/1.0e6
+            << " d2h " << t_d2h/1.0e6
+            << " unvectorise " << t_unvec/1.0e6
+            << " alltoall " << t_a2a/1.0e6
+            << " reorder " << t_reorder/1.0e6
+            << " vectorise " << t_vec/1.0e6 << std::endl;
+  // Staging vectors are still live here, so this is the high-water point of the call
+  if ( GridLogPerformance.isActive() ) {
+    HostMemoryReport(full_grid,GridLogPerformance,"Grid_split");
+  }
 }
 
 template<class Vobj>
@@ -1763,13 +1801,29 @@ void Grid_unsplit(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
     ratio[d] = full_grid->_processors[d]/ split_grid->_processors[d];
   }
 
+  // Coarse phase timers, reported on GridLogPerformance
+  double t_start   = usecond();
+  double t_a2a     = 0;
+  double t_reorder = 0;
+  double t_vec     = 0;
+  double t0;
+
   uint64_t lsites = full_grid->lSites();
   uint64_t     sz = lsites * nvector;
   std::vector<Sobj> tmpdata(sz);
   std::vector<Sobj> alldata(sz);
   std::vector<Sobj> scalardata(lsites); 
+  double t_alloc = usecond() - t_start;
 
+  // Open and close a host view first so any device to host copy is timed on its own
+  t0 = usecond();
+  {
+    autoView(split_v, split, CpuRead);
+  }
+  double t_d2h = usecond() - t0;
+  t0 = usecond();
   unvectorizeToLexOrdArray(alldata,split);    
+  double t_unvec = usecond() - t0;
 
   /////////////////////////////////////////////////////////////////
   // Start from split grid and work towards full grid
@@ -1794,6 +1848,7 @@ void Grid_unsplit(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
       int fvol   = lsites;
       int chunk  = (nvec*fvol)/sP;          GRID_ASSERT(chunk*sP == nvec*fvol);
 	
+      t0 = usecond();
       {
 	// Loop over reordered data post A2A
 	thread_for(c, chunk,{
@@ -1821,17 +1876,22 @@ void Grid_unsplit(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
         });
       }
 
+      t_reorder += usecond() - t0;
+
+      t0 = usecond();
       if ( split_grid->_processors[d] > 1 ) {
 	split_grid->AllToAll(d,tmpdata,alldata);
 	tmpdata=alldata;
       }
       full_grid ->AllToAll(d,tmpdata,alldata);
+      t_a2a += usecond() - t0;
       rdims[d]/= M;
       rsites  /= M;
       nvec    *= M;       // Increase nvec by subdivision factor
     }
   }
 
+  t0 = usecond();
   lsites = full_grid->lSites();
   for(int v=0;v<nvector;v++){
     thread_for(site, lsites,{
@@ -1839,6 +1899,21 @@ void Grid_unsplit(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
     });
     vectorizeFromLexOrdArray(scalardata,full[v]);    
   }
+  t_vec = usecond() - t0;
+  double t_total = usecond() - t_start;
+
+  // Host to device copies of full happen lazily at their first accelerator use; not counted here
+  std::cout << GridLogPerformance << "Grid_unsplit: " << nvector << " x " << sz/nvector*sizeof(Sobj)/1.0e6
+            << " MB/rank total " << t_total/1.0e6 << " s : alloc " << t_alloc/1.0e6
+            << " d2h " << t_d2h/1.0e6
+            << " unvectorise " << t_unvec/1.0e6
+            << " alltoall " << t_a2a/1.0e6
+            << " reorder " << t_reorder/1.0e6
+            << " vectorise " << t_vec/1.0e6 << std::endl;
+  // Staging vectors are still live here, so this is the high-water point of the call
+  if ( GridLogPerformance.isActive() ) {
+    HostMemoryReport(full_grid,GridLogPerformance,"Grid_unsplit");
+  }
 }
 
 //////////////////////////////////////////////////////

Grid/perfmon/HostMemory.h

@@ -0,0 +1,227 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./lib/perfmon/HostMemory.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+#include <sys/resource.h>
+#include <unistd.h>
+#include <cctype>
+#include <fstream>
+#ifdef __APPLE__
+#include <mach/mach.h>
+#endif
+
+NAMESPACE_BEGIN(Grid);
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Host memory as the OOM killer sees it, plus the device allocator footprint.
+// Targets are Apple or Linux; on Apple only the RSS and device fields are filled.
+//
+//  RSS      : this process, now and its high-water mark (getrusage).
+//  Node     : MemAvailable and MemTotal from /proc/meminfo (Linux).
+//  Cgroup   : the nearest enclosing memory cgroup with a finite limit (the one the batch
+//             system enforces), its usage now, its high-water mark and the limit (Linux,
+//             v2 or v1 hierarchy). Cgroup usage includes page cache, tmpfs and kernel memory
+//             charged to the job, none of which appear in any process RSS.
+//  Device   : MemoryManager Lattice footprint on the device and its free-block cache. Device
+//             allocations outside the MemoryManager (comms buffers) are not included.
+//
+// All values in GB; -1 where the source does not exist on this platform or kernel.
+/////////////////////////////////////////////////////////////////////////////////////////////
+struct HostMemoryStatus
+{
+  RealD RSS           = -1;
+  RealD RSSPeak       = -1;
+  RealD NodeAvailable = -1;
+  RealD NodeTotal     = -1;
+  RealD CgroupCurrent = -1;
+  RealD CgroupPeak    = -1;
+  RealD CgroupLimit   = -1;
+  RealD DeviceLattice = -1;
+  RealD DeviceCache   = -1;
+};
+
+#ifndef __APPLE__ // Linux
+
+// One number from a file; -1 if the file is absent or holds no number (e.g. "max")
+inline RealD HostMemoryReadBytes(const std::string &file)
+{
+  std::ifstream f(file);
+  if ( !f.good() ) {
+    return -1;
+  }
+  std::string word;
+  f >> word;
+  if ( word.empty() || !isdigit(word[0]) ) {
+    return -1;
+  }
+  return std::stod(word);
+}
+
+// Value of a "Key: N kB" line in /proc/meminfo, in bytes; -1 if absent
+inline RealD HostMemoryMeminfo(const std::string &key)
+{
+  std::ifstream f("/proc/meminfo");
+  std::string name;
+  RealD value;
+  std::string unit;
+  while ( f >> name >> value >> unit ) {
+    if ( name == key + ":" ) {
+      return value*1024.0;
+    }
+  }
+  return -1;
+}
+
+// Fill the cgroup fields from the nearest enclosing memory cgroup with a finite limit, or
+// from this process's own cgroup (limit left at -1) if no ancestor has one
+inline void HostMemoryCgroup(HostMemoryStatus &s)
+{
+  std::ifstream f("/proc/self/cgroup");
+  std::string line;
+  std::string root;
+  std::string path;
+  std::string current;
+  std::string peak;
+  std::string limit;
+  while ( std::getline(f,line) ) {
+    size_t c1 = line.find(':');
+    size_t c2 = line.find(':',c1+1);
+    if ( c1 == std::string::npos || c2 == std::string::npos ) {
+      continue;
+    }
+    std::string controllers = line.substr(c1+1,c2-c1-1);
+    // v2 unified hierarchy: "0::/path"
+    if ( controllers.empty() && root.empty() ) {
+      root    = "/sys/fs/cgroup";
+      path    = line.substr(c2+1);
+      current = "memory.current";
+      peak    = "memory.peak";
+      limit   = "memory.max";
+    }
+    // v1 memory controller: "N:memory:/path", takes precedence
+    if ( (","+controllers+",").find(",memory,") != std::string::npos ) {
+      root    = "/sys/fs/cgroup/memory";
+      path    = line.substr(c2+1);
+      current = "memory.usage_in_bytes";
+      peak    = "memory.max_usage_in_bytes";
+      limit   = "memory.limit_in_bytes";
+      break;
+    }
+  }
+  if ( root.empty() ) {
+    return;
+  }
+
+  // v1 reports "no limit" as a number near 2^63
+  const RealD unlimited = 1.0e18;
+  std::string dir = path;
+  while ( true ) {
+    std::string base = root + dir + "/";
+    RealD lim = HostMemoryReadBytes(base+limit);
+    if ( lim > 0 && lim < unlimited ) {
+      s.CgroupCurrent = HostMemoryReadBytes(base+current)/1.0e9;
+      s.CgroupPeak    = HostMemoryReadBytes(base+peak)/1.0e9;
+      s.CgroupLimit   = lim/1.0e9;
+      return;
+    }
+    if ( dir.empty() || dir == "/" ) {
+      break;
+    }
+    dir = dir.substr(0,dir.rfind('/'));
+  }
+  std::string base = root + path + "/";
+  s.CgroupCurrent = HostMemoryReadBytes(base+current)/1.0e9;
+  s.CgroupPeak    = HostMemoryReadBytes(base+peak)/1.0e9;
+}
+
+#endif
+
+// This rank only; no communication
+inline HostMemoryStatus HostMemoryQuery(void)
+{
+  HostMemoryStatus s;
+
+  struct rusage ru;
+  getrusage(RUSAGE_SELF,&ru);
+#ifdef __APPLE__
+  s.RSSPeak = ru.ru_maxrss/1.0e9;                // bytes on macOS
+  mach_task_basic_info_data_t info;
+  mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT;
+  if ( task_info(mach_task_self(),MACH_TASK_BASIC_INFO,(task_info_t)&info,&count) == KERN_SUCCESS ) {
+    s.RSS = info.resident_size/1.0e9;
+  }
+#else // Linux
+  s.RSSPeak = ru.ru_maxrss*1024.0/1.0e9;         // kilobytes on Linux
+  std::ifstream statm("/proc/self/statm");
+  long pages    = 0;
+  long resident = 0;
+  if ( statm >> pages >> resident ) {
+    s.RSS = resident*(RealD)sysconf(_SC_PAGESIZE)/1.0e9;
+  }
+  RealD avail = HostMemoryMeminfo("MemAvailable");
+  RealD total = HostMemoryMeminfo("MemTotal");
+  if ( avail >= 0 ) {
+    s.NodeAvailable = avail/1.0e9;
+  }
+  if ( total >= 0 ) {
+    s.NodeTotal = total/1.0e9;
+  }
+  HostMemoryCgroup(s);
+#endif
+
+  s.DeviceLattice = MemoryManager::DeviceBytes/1.0e9;
+  s.DeviceCache   = MemoryManager::DeviceCacheBytes()/1.0e9;
+  return s;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Collective over grid. One line on log: maxima over ranks, except node MemAvailable which
+// is the minimum over ranks (the tightest node).
+/////////////////////////////////////////////////////////////////////////////////////////////
+inline void HostMemoryReport(GridBase *grid,GridLogger &log,const std::string &label)
+{
+  HostMemoryStatus s = HostMemoryQuery();
+  RealD minus_available = -s.NodeAvailable;
+  grid->GlobalMax(s.RSS);
+  grid->GlobalMax(s.RSSPeak);
+  grid->GlobalMax(minus_available);
+  grid->GlobalMax(s.NodeTotal);
+  grid->GlobalMax(s.CgroupCurrent);
+  grid->GlobalMax(s.CgroupPeak);
+  grid->GlobalMax(s.CgroupLimit);
+  grid->GlobalMax(s.DeviceLattice);
+  grid->GlobalMax(s.DeviceCache);
+  std::cout << log << "HOSTMEM " << label
+            << " : rank RSS " << s.RSS << " peak " << s.RSSPeak
+            << " ; node available " << -minus_available << " of " << s.NodeTotal
+            << " ; cgroup " << s.CgroupCurrent << " peak " << s.CgroupPeak << " limit " << s.CgroupLimit
+            << " ; device lattice " << s.DeviceLattice << " cache " << s.DeviceCache
+            << " GB" << std::endl;
+}
+
+NAMESPACE_END(Grid);

Grid/qcd/action/fermion/FermionOperator.h

@@ -70,6 +70,8 @@ public:
     GRID_ASSERT(U != nullptr);
     GRID_ASSERT(mpi_split.size() == Nd);
 
+    double t0 = usecond();
+
     SplitOperator<FermionField> *split = new SplitOperator<FermionField>();
 
     split->GaugeGrid   = new GridCartesian(U->FullDimensions(),U->_simd_layout,mpi_split,*U);
@@ -85,6 +87,7 @@ public:
       split->FermionGrid   = split->GaugeGrid;
       split->FermionRBGrid = split->GaugeRBGrid;
     }
+    std::cout << GridLogPerformance << "MakeSplitGrids: " << (usecond()-t0)/1.0e6 << " s" << std::endl;
     return split;
   }
 

Grid/qcd/action/fermion/WilsonFermion5D.h

@@ -192,9 +192,14 @@ public:
   // Redistribute this operator's doubled gauge field onto clone's split grid. Collective.
   void SplitDoubledGaugeInto(WilsonFermion5D<Impl> &clone)
   {
+    double t0 = usecond();
     DoubledGaugeField sUmu(clone.GaugeGrid());
     Grid_split(Umu,sUmu);
+    double t1 = usecond();
     clone.ImportDoubledGauge(sUmu);
+    double t2 = usecond();
+    std::cout << GridLogPerformance << "SplitDoubledGaugeInto: Grid_split " << (t1-t0)/1.0e6
+              << " s import " << (t2-t1)/1.0e6 << " s" << std::endl;
   }
 
   ///////////////////////////////////////////////////////////////

tests/solver/Test_split_mobius_batched.cc

@@ -93,6 +93,7 @@ void ReportMemory(GridBase *grid,const std::string &phase)
             << " : host RSS " << rss << " GB, peak " << peak
             << " GB; allocator cache host " << hostcache << " GB, device " << devcache
             << " GB (max over ranks)" << std::endl;
+  HostMemoryReport(grid,GridLogMessage,phase);
 }
 
 typedef LatticeFermionD FieldD;