Split testing

systems/Frontier/split_batched_mobius.job

@@ -0,0 +1,119 @@
+#!/bin/bash -l
+#SBATCH --job-name=split-batched-mobius
+#SBATCH --nodes=4
+#SBATCH --ntasks-per-node=8
+#SBATCH --cpus-per-task=7
+#SBATCH --gpus-per-node=8
+#SBATCH --time=1:30:00
+#SBATCH --account=phy157_dwf
+#SBATCH --gpu-bind=none
+#SBATCH --exclusive
+#SBATCH --mem=0
+#SBATCH -S 0
+##SBATCH -q debug
+
+##############################################################################
+# MixedPrecisionConjugateGradientBatched with split inner solves, at production
+# size: Mobius 64^3x128, Ls=12, mass 0.026, b=1.5 c=0.5 M5=1.8, antiperiodic
+# in time, Hadrons default SchurDiagMooeeOperator.
+#
+# 4 nodes = 32 GCDs, --mpi 2.2.2.4 (local 32^4).  --batched-solver-split node
+# makes each node one partition (P=4), so Nbatch=4 is one round with no
+# zero padding.  The driver solves the same batch twice from zero guesses:
+#   UNSPLIT  inner CG on all 32 GCDs, one rhs after another (existing path)
+#   SPLIT    four independent inner CGs, one per node
+# and prints per-rhs iterations, true residuals and wall clock for each.
+#
+# Build (in the Frontier build tree, after ./scripts/filelist in the source
+# root so the new test is in tests/solver/Make.inc):
+#   make -C tests/solver Test_split_mobius_batched
+#
+# Memory, per GCD (evictable Lattice fields, so bounded by --device-mem):
+#   double RB 5d field 1.21 GB, float 0.60 GB
+#   driver src+sol 9.7, solver batch copies 9.7, split partition fields and
+#   inner CG on the 4x larger split local volume ~20, gauge copies ~5
+#   => ~45 GB, hence --device-mem 40000 as in pvdagm_mixed_precision.job.
+# Host memory: the device is an inclusive cache, so host RSS per rank is the whole
+# ~45 GB working set (plus transient Grid_split staging): ~400 GB of a node's 512 GB.
+# The driver prints MEMORY lines (host RSS, peak, allocator caches) at each phase.
+# Grid_split stages 2 x 4.8 GB per rank on the host per transfer.
+##############################################################################
+
+LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
+RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
+mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"
+
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3 7 6 5 4)
+export NUMA_MAP=(3 3 1 1 2 2 0 0)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
+export HIP_VISIBLE_DEVICES=\$GPU
+unset ROCR_VISIBLE_DEVICES
+if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
+exec numactl -m \$NUMA -N \$NUMA \$*
+EOF
+chmod +x ./select_gpu
+
+# Source tree and build of THIS branch (feature/splitGridCGBatch) on Frontier
+root=$LUSTRE/SplitCG/Grid/systems/Frontier
+source $root/sourceme-rocm7.2.sh
+
+export OMP_NUM_THREADS=7
+ulimit -c 0                             # no 22 GB GPU core dumps
+export FI_MR_CACHE_MONITOR=kdreg2       # site default; device-buffer MPI on Slingshot
+export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_OFI_NIC_POLICY=GPU
+# No GRID_ALLOC_NCACHE_LARGE here. With --enable-unified=no every Lattice lives in host
+# memory (the device is an inclusive cache), so a deep large-allocation cache keeps freed
+# host blocks on top of the full working set: measured on the laptop at 64 entries it
+# held about as much again as the working set, enough to OOM a 4-node run of this job.
+module load libfabric
+
+BIN=$root/tests/solver/Test_split_mobius_batched
+OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 40000 --comms-overlap"
+vol=64.64.64.128
+MPI_GEOM=2.2.2.4
+
+# A thermalised 64^3x128 NERSC configuration.  Leave empty for a hot start:
+# fine for plumbing, but mass 0.026 on a hot field is not a physical
+# iteration count and may run long.
+CONFIG=
+
+PHYS="--Ls 12 --mass 0.026 --M5 1.8 --b 1.5 --c 0.5 --nbatch 4 --tol 1e-8"
+if [ -n "$CONFIG" ]; then PHYS="$PHYS --config $CONFIG"; fi
+
+export GRID_STDOUT_ROOT=$RUNDIR/split
+srun -N4 -n32 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi $MPI_GEOM --grid $vol $OPTS \
+     $PHYS --batched-solver-split node \
+     --debug-stdout --log Error,Warning,Message,Performance \
+     > log.split 2>&1
+echo "exit $?"
+
+##############################################################################
+# Readout.  Device OOM appears only on stderr as "hipMalloc failed".
+##############################################################################
+f=$(grep -l "Memory access fault\|NO_TRANSLATION\|hipMalloc failed\|out of memory\|illegal memory access\|GRID_ASSERT" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
+if [ -n "$f" ]; then
+  echo "FAULT in $f"; tail -20 $f | cut -c1-160
+fi
+grep -h "MEMORY\|BatchedSolverSplit\|Inner CG iterations\|Total time\|true residual\|SUMMARY" \
+     $GRID_STDOUT_ROOT/0/Grid.stdout.0 2>/dev/null | cut -c1-200
+
+##############################################################################
+# What to read.
+#
+# BatchedSolverSplit line: the partition layout must equal the node layout
+# (ShmGrid), e.g. [1 2 2 2] of [2 2 2 4] : 4 partitions.  A "straddle node
+# boundaries" warning means the rank reordering did not give node-aligned
+# blocks; the result is still correct but the inner CG talks off node.
+#
+# UNSPLIT vs SPLIT: per-rhs inner iterations should agree closely (the same
+# solves, different reduction order); the SUMMARY lines give the wall clock.
+# The gain is the strong-scaling loss of the 32-GCD inner CG against four
+# 8-GCD CGs with node-local halos.  "Split setup and transfer" in the Total
+# time line prices the clone (grids, gauge split) and the Grid_split /
+# Grid_unsplit traffic, which is host-staged.
+##############################################################################

tests/forces/ForceTest.h

@@ -0,0 +1,138 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/forces/ForceTest.h
+
+    Copyright (C) 2022
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#pragma once
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Finite-difference check of an action's force: refresh, S1 = S(U), step the links by eps
+// along a random momentum P, take the force at the midpoint, step again, S2 = S(U'').
+// The force predicts S2 - S1 to O(eps^3).
+//
+// Runs on a copy of U, so the caller's field is unchanged. Returns S2 - S1 - dSpred for
+// the caller to assert on; halving eps should reduce it by about 8.
+/////////////////////////////////////////////////////////////////////////////////////////////
+template<class Gimpl>
+RealD ForceTest(Action<LatticeGaugeField> &action,LatticeGaugeField & Uin,MomentumFilterBase<LatticeGaugeField> &Filter,RealD eps=0.005)
+{
+  GridBase *UGrid = Uin.Grid();
+
+  std::vector<int> seeds({1,2,3,5});
+  GridSerialRNG            sRNG;         sRNG.SeedFixedIntegers(seeds);
+  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds);
+
+  LatticeColourMatrix Pmu(UGrid);
+  LatticeGaugeField P(UGrid);
+  LatticeGaugeField UdSdU(UGrid);
+  LatticeGaugeField U(UGrid);
+  U = Uin;
+
+  std::cout << GridLogMessage << "*********************************************************"<<std::endl;
+  std::cout << GridLogMessage << " Force test for "<<action.action_name()<<" eps "<<eps<<std::endl;
+  std::cout << GridLogMessage << "*********************************************************"<<std::endl;
+
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+  std::cout << GridLogMessage << " Refresh "<<action.action_name()<<std::endl;
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+
+  Gimpl::generate_momenta(P,sRNG,RNG4);
+  Filter.applyFilter(P);
+
+  action.refresh(U,sRNG,RNG4);
+
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+  std::cout << GridLogMessage << " Action "<<action.action_name()<<std::endl;
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+
+  RealD S1 = action.S(U);
+
+  Gimpl::update_field(P,U,eps);
+
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+  std::cout << GridLogMessage << " Derivative "<<action.action_name()<<std::endl;
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+  action.deriv(U,UdSdU);
+  UdSdU = Ta(UdSdU);
+  Filter.applyFilter(UdSdU);
+
+  DumpSliceNorm("Force",UdSdU,Nd-1);
+
+  Gimpl::update_field(P,U,eps);
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+  std::cout << GridLogMessage << " Action "<<action.action_name()<<std::endl;
+  std::cout << GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+
+  RealD S2 = action.S(U);
+
+  // Use the derivative
+  LatticeComplex dS(UGrid); dS = Zero();
+  for(int mu=0;mu<Nd;mu++){
+    auto UdSdUmu = PeekIndex<LorentzIndex>(UdSdU,mu);
+    Pmu= PeekIndex<LorentzIndex>(P,mu);
+    dS = dS - trace(Pmu*UdSdUmu)*eps*2.0*2.0;
+  }
+  ComplexD dSpred    = sum(dS);
+  RealD diff =  S2-S1-dSpred.real();
+
+  std::cout<< GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"<<std::endl;
+  std::cout<< GridLogMessage << "S1 : "<< S1    <<std::endl;
+  std::cout<< GridLogMessage << "S2 : "<< S2    <<std::endl;
+  std::cout<< GridLogMessage << "dS : "<< S2-S1 <<std::endl;
+  std::cout<< GridLogMessage << "dSpred : "<< dSpred.real() <<std::endl;
+  std::cout<< GridLogMessage << "diff : "<< diff<<std::endl;
+  std::cout<< GridLogMessage << "*********************************************************"<<std::endl;
+  std::cout<< GridLogMessage << "Done" <<std::endl;
+  std::cout << GridLogMessage << "*********************************************************"<<std::endl;
+  return diff;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// ForceTest at eps0, eps0/2, ... (neps values). A correct force leaves an O(eps^3)
+// discrepancy, so successive ratios approach 8 as eps shrinks; an error in the force adds
+// an O(eps) part and drives them towards 2. A strongly curved action needs a smaller eps0
+// before the ratios settle. Returns the ratio for the smallest pair.
+/////////////////////////////////////////////////////////////////////////////////////////////
+template<class Gimpl>
+RealD ForceTestScaling(Action<LatticeGaugeField> &action,LatticeGaugeField &U,MomentumFilterBase<LatticeGaugeField> &Filter,const std::string &name,
+                       RealD eps0=0.01,int neps=4)
+{
+  std::vector<RealD> eps(neps);
+  std::vector<RealD> d(neps);
+  for(int i=0;i<neps;i++){
+    eps[i] = eps0/(1<<i);
+    d[i]   = ForceTest<Gimpl>(action,U,Filter,eps[i]);
+  }
+  for(int i=0;i<neps;i++){
+    std::cout << GridLogMessage << name << ": eps " << eps[i] << " discrepancy " << d[i];
+    if ( i > 0 ) {
+      std::cout << "  ratio to previous " << d[i-1]/d[i];
+    }
+    std::cout << std::endl;
+  }
+  RealD scale = d[neps-2]/d[neps-1];
+  std::cout << GridLogMessage << name << ": smallest-eps ratio " << scale << " (expect ~8)" << std::endl;
+  return scale;
+}

tests/solver/Test_split_mixedprec_batched.cc

@@ -0,0 +1,352 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/solver/Test_split_mixedprec_batched.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+/////////////////////////////////////////////////////////////////////////////////////////////
+// MixedPrecisionConjugateGradientBatched with inner solves on split-communicator
+// partitions (--batched-solver-split), against the same solver unsplit.
+//
+//   mpirun -n 2 ./Test_split_mixedprec_batched --grid 8.8.8.8 --mpi 1.1.1.2 --batched-solver-split 1.1.1.1
+//
+// With no --batched-solver-split the partitions are single ranks (1.1.1.1).
+// NBATCH is deliberately not a multiple of the partition count, so the last group is
+// zero-padded.
+/////////////////////////////////////////////////////////////////////////////////////////////
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+
+const int   NBATCH    = 3;
+const RealD TOLERANCE = 1.0e-8;
+
+template<class Field>
+RealD RelativeDifference(const Field &a,const Field &b)
+{
+  Field diff(a.Grid());
+  diff = a - b;
+  return std::sqrt(norm2(diff)/norm2(b));
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// The solver assumes vector p of a group lands in the partition whose
+// GridSplitVectorIndex is p: fill vector p with the constant p+1, split, and check
+// every partition sees its own index.
+/////////////////////////////////////////////////////////////////////////////////////////////
+void CheckPartitionOrder(GridCartesian *UGrid,const Coordinate &layout)
+{
+  GridCartesian SGrid(UGrid->FullDimensions(),UGrid->_simd_layout,layout,*UGrid);
+  int P     = UGrid->ProcessorCount()/SGrid.ProcessorCount();
+  int index = GridSplitVectorIndex(UGrid,&SGrid);
+
+  std::vector<LatticeComplexF> full(P,UGrid);
+  for(int p=0;p<P;p++){
+    full[p] = ComplexF(p+1,0.0);
+  }
+  LatticeComplexF split(&SGrid);
+  Grid_split(full,split);
+
+  LatticeComplexF expect(&SGrid);
+  expect = ComplexF(index+1,0.0);
+  RealD err = norm2(split - expect);
+
+  std::cout << GridLogMessage << "Partition order: vector index " << index << " error " << err << std::endl;
+  GRID_ASSERT(err == 0.0);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// For one pair of double/float linear operators:
+//  (1) red-black-aware split/unsplit round trip is exact;
+//  (2) the split clone's HermOp agrees with the full-grid HermOp;
+//  (3) the batched solver with split inner solves agrees with the unsplit solver;
+//  (4) the split solution has true residual at the requested tolerance.
+/////////////////////////////////////////////////////////////////////////////////////////////
+template<class FieldD,class FieldF>
+void CheckSplitBatched(const std::string &name,
+                       LinearOperatorBase<FieldD> &Linop_d,
+                       LinearOperatorBase<FieldF> &Linop_f,
+                       GridBase *grid_f,
+                       std::vector<FieldD> &src,
+                       const Coordinate &layout)
+{
+  std::cout << GridLogMessage << "==================================================" << std::endl;
+  std::cout << GridLogMessage << name << std::endl;
+  std::cout << GridLogMessage << "==================================================" << std::endl;
+
+  int cb = src[0].Checkerboard();
+
+  SplitOperator<FieldF> *split = Linop_f.SplitClone(layout);
+  GRID_ASSERT(split != nullptr);
+  int P = split->Partitions;
+
+  std::vector<FieldF> full(P,grid_f);
+  std::vector<FieldF> back(P,grid_f);
+  std::vector<FieldF> Mfull(P,grid_f);
+  for(int p=0;p<P;p++){
+    full[p].Checkerboard()  = cb;
+    back[p].Checkerboard()  = cb;
+    Mfull[p].Checkerboard() = cb;
+    precisionChange(full[p],src[p%src.size()]);
+  }
+
+  // (1) round trip
+  FieldF s_in(split->FieldGrid);
+  FieldF s_out(split->FieldGrid);
+  Grid_split(full,s_in);
+  Grid_unsplit(back,s_in);
+  for(int p=0;p<P;p++){
+    RealD err = norm2(back[p] - full[p]);
+    std::cout << GridLogMessage << name << ": split/unsplit round trip " << p << " error " << err << std::endl;
+    GRID_ASSERT(err == 0.0);
+  }
+
+  // (2) operator
+  split->Linop->HermOp(s_in,s_out);
+  Grid_unsplit(back,s_out);
+  for(int p=0;p<P;p++){
+    Linop_f.HermOp(full[p],Mfull[p]);
+    RealD err = RelativeDifference(back[p],Mfull[p]);
+    std::cout << GridLogMessage << name << ": split HermOp vs full HermOp " << p << " relative difference " << err << std::endl;
+    GRID_ASSERT(err < 1.0e-6);
+  }
+  delete split;
+
+  // (3) solver, unsplit then split
+  int NBatch = src.size();
+  std::vector<FieldD> sol_ref(NBatch,src[0].Grid());
+  std::vector<FieldD> sol_split(NBatch,src[0].Grid());
+  for(int i=0;i<NBatch;i++){
+    sol_ref[i].Checkerboard()   = cb;
+    sol_split[i].Checkerboard() = cb;
+    sol_ref[i]   = Zero();
+    sol_split[i] = Zero();
+  }
+
+  MixedPrecisionConjugateGradientBatched<FieldD,FieldF> mCG(TOLERANCE,10000,50,1000,grid_f,Linop_f,Linop_d);
+
+  std::cout << GridLogMessage << name << ": unsplit batched solve" << std::endl;
+  mCG.BatchedSplit     = Coordinate();
+  mCG.BatchedSplitNode = false;
+  mCG(src,sol_ref);
+
+  std::cout << GridLogMessage << name << ": split batched solve, partition layout " << layout << std::endl;
+  mCG.BatchedSplit = layout;
+  mCG(src,sol_split);
+
+  FieldD Msol(src[0].Grid());
+  Msol.Checkerboard() = cb;
+  for(int i=0;i<NBatch;i++){
+    RealD diff = RelativeDifference(sol_split[i],sol_ref[i]);
+    Linop_d.HermOp(sol_split[i],Msol);
+    RealD resid = RelativeDifference(Msol,src[i]);
+    std::cout << GridLogMessage << name << ": rhs " << i
+              << " split vs unsplit " << diff
+              << " true residual " << resid << std::endl;
+    GRID_ASSERT(diff  < 1.0e-6);
+    GRID_ASSERT(resid < 10.0*TOLERANCE);
+  }
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  const int Ls = 8;
+
+  Coordinate layout = GridDefaultBatchedSolverSplit();
+  if ( layout.size() == 0 ) {
+    layout = Coordinate(Nd,1);
+  }
+
+  GridCartesian         *UGrid_d   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_d);
+  GridCartesian         *FGrid_d   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_d);
+  GridRedBlackCartesian *FrbGrid_d = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_d);
+
+  GridCartesian         *UGrid_f   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexF::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid_f = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_f);
+  GridCartesian         *FGrid_f   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_f);
+  GridRedBlackCartesian *FrbGrid_f = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_f);
+
+  CheckPartitionOrder(UGrid_f,layout);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+  GridParallelRNG RNG4(UGrid_d);
+  GridParallelRNG RNG5(FGrid_d);
+  RNG4.SeedFixedIntegers(seeds4);
+  RNG5.SeedFixedIntegers(seeds5);
+
+  LatticeGaugeFieldD Umu_d(UGrid_d);
+  LatticeGaugeFieldF Umu_f(UGrid_f);
+  SU<Nc>::HotConfiguration(RNG4,Umu_d);
+  precisionChange(Umu_f,Umu_d);
+
+  // Antiperiodic in time for some actions, so boundary phases must travel with the links
+  WilsonImplParams antiperiodic;
+  antiperiodic.boundary_phases[Nd-1] = -1.0;
+
+  // Sources: odd checkerboard (Schur) and full lattice (MdagM), 4d and 5d
+  std::vector<LatticeFermionD> src4_o(NBATCH,UrbGrid_d);
+  std::vector<LatticeFermionD> src5_o(NBATCH,FrbGrid_d);
+  std::vector<LatticeFermionD> src5(NBATCH,FGrid_d);
+  LatticeFermionD tmp4(UGrid_d);
+  LatticeFermionD tmp5(FGrid_d);
+  for(int i=0;i<NBATCH;i++){
+    random(RNG4,tmp4);
+    random(RNG5,tmp5);
+    pickCheckerboard(Odd,src4_o[i],tmp4);
+    pickCheckerboard(Odd,src5_o[i],tmp5);
+    src5[i] = tmp5;
+  }
+
+  //////////////////////////////////////////
+  // Wilson
+  //////////////////////////////////////////
+  {
+    RealD mass = 0.1;
+    WilsonFermionD Dd(Umu_d,*UGrid_d,*UrbGrid_d,mass);
+    WilsonFermionF Df(Umu_f,*UGrid_f,*UrbGrid_f,mass);
+    SchurDiagMooeeOperator<WilsonFermionD,LatticeFermionD> Ld(Dd);
+    SchurDiagMooeeOperator<WilsonFermionF,LatticeFermionF> Lf(Df);
+    CheckSplitBatched("Wilson SchurDiagMooee",Ld,Lf,UrbGrid_f,src4_o,layout);
+  }
+
+  //////////////////////////////////////////
+  // Wilson clover
+  //////////////////////////////////////////
+  {
+    RealD mass  = 0.1;
+    RealD csw_r = 1.0;
+    RealD csw_t = 1.0;
+    WilsonCloverFermionD Dd(Umu_d,*UGrid_d,*UrbGrid_d,mass,csw_r,csw_t);
+    WilsonCloverFermionF Df(Umu_f,*UGrid_f,*UrbGrid_f,mass,csw_r,csw_t);
+    SchurDiagMooeeOperator<WilsonCloverFermionD,LatticeFermionD> Ld(Dd);
+    SchurDiagMooeeOperator<WilsonCloverFermionF,LatticeFermionF> Lf(Df);
+    CheckSplitBatched("WilsonClover SchurDiagMooee",Ld,Lf,UrbGrid_f,src4_o,layout);
+  }
+
+  //////////////////////////////////////////
+  // Compact Wilson clover, antiperiodic
+  //////////////////////////////////////////
+  {
+    RealD mass  = 0.1;
+    RealD csw_r = 1.0;
+    RealD csw_t = 1.0;
+    RealD cF    = 1.0;
+    WilsonAnisotropyCoefficients anis;
+    CompactWilsonCloverFermionD Dd(Umu_d,*UGrid_d,*UrbGrid_d,mass,csw_r,csw_t,cF,anis,antiperiodic);
+    CompactWilsonCloverFermionF Df(Umu_f,*UGrid_f,*UrbGrid_f,mass,csw_r,csw_t,cF,anis,antiperiodic);
+    SchurDiagMooeeOperator<CompactWilsonCloverFermionD,LatticeFermionD> Ld(Dd);
+    SchurDiagMooeeOperator<CompactWilsonCloverFermionF,LatticeFermionF> Lf(Df);
+    CheckSplitBatched("CompactWilsonClover SchurDiagMooee antiperiodic",Ld,Lf,UrbGrid_f,src4_o,layout);
+  }
+
+  //////////////////////////////////////////
+  // Domain wall
+  //////////////////////////////////////////
+  {
+    RealD mass = 0.1;
+    RealD M5   = 1.8;
+    DomainWallFermionD Dd(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5);
+    DomainWallFermionF Df(Umu_f,*FGrid_f,*FrbGrid_f,*UGrid_f,*UrbGrid_f,mass,M5);
+    SchurDiagMooeeOperator<DomainWallFermionD,LatticeFermionD> Ld(Dd);
+    SchurDiagMooeeOperator<DomainWallFermionF,LatticeFermionF> Lf(Df);
+    CheckSplitBatched("DomainWall SchurDiagMooee",Ld,Lf,FrbGrid_f,src5_o,layout);
+  }
+
+  //////////////////////////////////////////
+  // Mobius, antiperiodic, with unequal
+  // masses; all wrapper kinds
+  //////////////////////////////////////////
+  {
+    RealD mass = 0.1;
+    RealD M5   = 1.8;
+    RealD b    = 1.5;
+    RealD c    = 0.5;
+    MobiusFermionD Dd(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5,b,c,antiperiodic);
+    MobiusFermionF Df(Umu_f,*FGrid_f,*FrbGrid_f,*UGrid_f,*UrbGrid_f,mass,M5,b,c,antiperiodic);
+    Dd.SetMass(0.1,0.12);
+    Df.SetMass(0.1,0.12);
+    {
+      SchurDiagMooeeOperator<MobiusFermionD,LatticeFermionD> Ld(Dd);
+      SchurDiagMooeeOperator<MobiusFermionF,LatticeFermionF> Lf(Df);
+      CheckSplitBatched("Mobius SchurDiagMooee antiperiodic",Ld,Lf,FrbGrid_f,src5_o,layout);
+    }
+    {
+      SchurDiagOneOperator<MobiusFermionD,LatticeFermionD> Ld(Dd);
+      SchurDiagOneOperator<MobiusFermionF,LatticeFermionF> Lf(Df);
+      CheckSplitBatched("Mobius SchurDiagOne antiperiodic",Ld,Lf,FrbGrid_f,src5_o,layout);
+    }
+    {
+      SchurDiagTwoOperator<MobiusFermionD,LatticeFermionD> Ld(Dd);
+      SchurDiagTwoOperator<MobiusFermionF,LatticeFermionF> Lf(Df);
+      CheckSplitBatched("Mobius SchurDiagTwo antiperiodic",Ld,Lf,FrbGrid_f,src5_o,layout);
+    }
+    {
+      MdagMLinearOperator<MobiusFermionD,LatticeFermionD> Ld(Dd);
+      MdagMLinearOperator<MobiusFermionF,LatticeFermionF> Lf(Df);
+      CheckSplitBatched("Mobius MdagM antiperiodic",Ld,Lf,FGrid_f,src5,layout);
+    }
+  }
+
+  //////////////////////////////////////////
+  // ZMobius
+  //////////////////////////////////////////
+  {
+    RealD mass = 0.1;
+    RealD M5   = 1.8;
+    RealD b    = 1.0;
+    RealD c    = 0.0;
+    std::vector<ComplexD> gamma(Ls);
+    for(int s=0;s<Ls;s++){
+      gamma[s] = ComplexD(1.0+0.05*s, (s%2) ? 0.02 : -0.02);
+    }
+    ZMobiusFermionD Dd(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5,gamma,b,c);
+    ZMobiusFermionF Df(Umu_f,*FGrid_f,*FrbGrid_f,*UGrid_f,*UrbGrid_f,mass,M5,gamma,b,c);
+    SchurDiagMooeeOperator<ZMobiusFermionD,LatticeFermionD> Ld(Dd);
+    SchurDiagMooeeOperator<ZMobiusFermionF,LatticeFermionF> Lf(Df);
+    CheckSplitBatched("ZMobius SchurDiagMooee",Ld,Lf,FrbGrid_f,src5_o,layout);
+  }
+
+  //////////////////////////////////////////
+  // A derived operator must not inherit
+  // its parent's SplitClone
+  //////////////////////////////////////////
+  {
+    RealD mass = 0.1;
+    RealD mu   = 0.1;
+    WilsonTMFermionF Df(Umu_f,*UGrid_f,*UrbGrid_f,mass,mu);
+    SchurDiagMooeeOperator<WilsonTMFermionF,LatticeFermionF> Lf(Df);
+    SplitOperator<LatticeFermionF> *split = Lf.SplitClone(layout);
+    std::cout << GridLogMessage << "WilsonTM SplitClone refused: " << (split == nullptr) << std::endl;
+    GRID_ASSERT(split == nullptr);
+  }
+
+  std::cout << GridLogMessage << "Test_split_mixedprec_batched: all checks passed" << std::endl;
+
+  Grid_finalize();
+}

tests/solver/Test_split_mobius_batched.cc

@@ -0,0 +1,236 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/solver/Test_split_mobius_batched.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Production-size timing of MixedPrecisionConjugateGradientBatched for Mobius with the
+// Hadrons default SchurDiagMooeeOperator: the same batch solved without and then with
+// split inner solves (--batched-solver-split), reporting wall clock, per-rhs iterations
+// and true residuals for each.
+//
+//   --Ls 12 --mass 0.026 --M5 1.8 --b 1.5 --c 0.5 --nbatch 4 --tol 1e-8
+//   --config <NERSC file>    (omit for a hot configuration: timing only, not physics)
+//   --nounsplit              (skip the reference unsplit solve)
+//   --repeat N               (split solve N times; host RSS must not grow between them)
+//
+// MEMORY lines report host RSS (current and peak) and allocator cache sizes, maximum over
+// ranks, at each phase: with --enable-unified=no every Lattice lives in host memory.
+//
+// Only one solution vector is kept, so the driver's own footprint is two batches of
+// double red-black 5d fields; the solver adds about as much again.
+/////////////////////////////////////////////////////////////////////////////////////////////
+#include <Grid/Grid.h>
+#include <sys/resource.h>
+#ifdef __APPLE__
+#include <mach/mach.h>
+#endif
+
+using namespace std;
+using namespace Grid;
+
+// Host memory of this process in GB: current resident set and its high-water mark
+void HostRSS(RealD &current,RealD &peak)
+{
+  struct rusage ru;
+  getrusage(RUSAGE_SELF,&ru);
+#ifdef __APPLE__
+  peak = ru.ru_maxrss/1.0e9;              // bytes on macOS
+  mach_task_basic_info_data_t info;
+  mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT;
+  task_info(mach_task_self(),MACH_TASK_BASIC_INFO,(task_info_t)&info,&count);
+  current = info.resident_size/1.0e9;
+#else
+  peak = ru.ru_maxrss*1024.0/1.0e9;       // kilobytes on Linux
+  long pages = 0;
+  long resident = 0;
+  FILE *f = fopen("/proc/self/statm","r");
+  if ( f ) {
+    if ( fscanf(f,"%ld %ld",&pages,&resident) != 2 ) {
+      resident = 0;
+    }
+    fclose(f);
+  }
+  current = resident*(RealD)sysconf(_SC_PAGESIZE)/1.0e9;
+#endif
+}
+
+// Largest values over ranks: host RSS now and at peak, and the allocator caches
+void ReportMemory(GridBase *grid,const std::string &phase)
+{
+  RealD rss;
+  RealD peak;
+  HostRSS(rss,peak);
+  RealD hostcache = MemoryManager::HostCacheBytes()/1.0e9;
+  RealD devcache  = MemoryManager::DeviceCacheBytes()/1.0e9;
+  grid->GlobalMax(rss);
+  grid->GlobalMax(peak);
+  grid->GlobalMax(hostcache);
+  grid->GlobalMax(devcache);
+  std::cout << GridLogMessage << "MEMORY " << phase
+            << " : host RSS " << rss << " GB, peak " << peak
+            << " GB; allocator cache host " << hostcache << " GB, device " << devcache
+            << " GB (max over ranks)" << std::endl;
+}
+
+typedef LatticeFermionD FieldD;
+typedef LatticeFermionF FieldF;
+
+template<class T>
+T CmdOption(int argc,char **argv,const std::string &name,T def)
+{
+  T val = def;
+  if ( GridCmdOptionExists(argv,argv+argc,name) ) {
+    std::stringstream ss(GridCmdOptionPayload(argv,argv+argc,name));
+    ss >> val;
+  }
+  return val;
+}
+
+void SolveAndReport(const std::string &label,
+                    MixedPrecisionConjugateGradientBatched<FieldD,FieldF> &mCG,
+                    LinearOperatorBase<FieldD> &Linop_d,
+                    std::vector<FieldD> &src,
+                    std::vector<FieldD> &sol)
+{
+  int nbatch = src.size();
+  for(int i=0;i<nbatch;i++){
+    sol[i].Checkerboard() = src[i].Checkerboard();
+    sol[i] = Zero();
+  }
+
+  std::cout << GridLogMessage << "==================================================" << std::endl;
+  std::cout << GridLogMessage << label << " batched solve, nbatch " << nbatch << std::endl;
+  std::cout << GridLogMessage << "==================================================" << std::endl;
+
+  RealD t0 = usecond();
+  mCG(src,sol);
+  RealD t1 = usecond();
+
+  FieldD Msol(src[0].Grid());
+  Msol.Checkerboard() = src[0].Checkerboard();
+  RealD worst = 0.0;
+  for(int i=0;i<nbatch;i++){
+    Linop_d.HermOp(sol[i],Msol);
+    Msol = Msol - src[i];
+    RealD resid = std::sqrt(norm2(Msol)/norm2(src[i]));
+    worst = std::max(worst,resid);
+    std::cout << GridLogMessage << label << ": rhs " << i << " true residual " << resid << std::endl;
+  }
+  std::cout << GridLogMessage << label << ": SUMMARY wall clock " << (t1-t0)/1.0e6
+            << " s for " << nbatch << " rhs, " << (t1-t0)/1.0e6/nbatch
+            << " s/rhs, worst true residual " << worst << std::endl;
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int         Ls        = CmdOption<int>        (argc,argv,"--Ls",12);
+  RealD       mass      = CmdOption<RealD>      (argc,argv,"--mass",0.026);
+  RealD       M5        = CmdOption<RealD>      (argc,argv,"--M5",1.8);
+  RealD       b         = CmdOption<RealD>      (argc,argv,"--b",1.5);
+  RealD       c         = CmdOption<RealD>      (argc,argv,"--c",0.5);
+  int         nbatch    = CmdOption<int>        (argc,argv,"--nbatch",4);
+  RealD       tol       = CmdOption<RealD>      (argc,argv,"--tol",1.0e-8);
+  std::string config    = CmdOption<std::string>(argc,argv,"--config",std::string(""));
+  bool        unsplit   = !GridCmdOptionExists(argv,argv+argc,"--nounsplit");
+  int         repeat    = CmdOption<int>        (argc,argv,"--repeat",1);
+
+  std::cout << GridLogMessage << "Mobius Ls " << Ls << " mass " << mass << " M5 " << M5
+            << " b " << b << " c " << c << " nbatch " << nbatch << " tol " << tol << std::endl;
+
+  GridCartesian         *UGrid_d   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_d);
+  GridCartesian         *FGrid_d   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_d);
+  GridRedBlackCartesian *FrbGrid_d = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_d);
+
+  GridCartesian         *UGrid_f   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexF::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid_f = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_f);
+  GridCartesian         *FGrid_f   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_f);
+  GridRedBlackCartesian *FrbGrid_f = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_f);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+  GridParallelRNG RNG4(UGrid_d);
+  GridParallelRNG RNG5(FGrid_d);
+  RNG4.SeedFixedIntegers(seeds4);
+  RNG5.SeedFixedIntegers(seeds5);
+
+  LatticeGaugeFieldD Umu_d(UGrid_d);
+  LatticeGaugeFieldF Umu_f(UGrid_f);
+  if ( config.size() ) {
+    FieldMetaData header;
+    NerscIO::readConfiguration(Umu_d,header,config);
+  } else {
+    std::cout << GridLogMessage << "No --config: hot configuration, timing only" << std::endl;
+    SU<Nc>::HotConfiguration(RNG4,Umu_d);
+  }
+  precisionChange(Umu_f,Umu_d);
+  ReportMemory(UGrid_d,"gauge field ready");
+
+  // Antiperiodic in time, as in production
+  WilsonImplParams params;
+  params.boundary_phases[Nd-1] = -1.0;
+
+  MobiusFermionD Dd(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5,b,c,params);
+  MobiusFermionF Df(Umu_f,*FGrid_f,*FrbGrid_f,*UGrid_f,*UrbGrid_f,mass,M5,b,c,params);
+  SchurDiagMooeeOperator<MobiusFermionD,FieldD> Linop_d(Dd);
+  SchurDiagMooeeOperator<MobiusFermionF,FieldF> Linop_f(Df);
+
+  std::vector<FieldD> src(nbatch,FrbGrid_d);
+  std::vector<FieldD> sol(nbatch,FrbGrid_d);
+  {
+    FieldD tmp(FGrid_d);
+    for(int i=0;i<nbatch;i++){
+      random(RNG5,tmp);
+      pickCheckerboard(Odd,src[i],tmp);
+    }
+  }
+
+  ReportMemory(UGrid_d,"operators and sources ready");
+
+  MixedPrecisionConjugateGradientBatched<FieldD,FieldF> mCG(tol,10000,50,10000,FrbGrid_f,Linop_f,Linop_d);
+
+  Coordinate split     = mCG.BatchedSplit;
+  bool       splitnode = mCG.BatchedSplitNode;
+
+  if ( unsplit ) {
+    mCG.BatchedSplit     = Coordinate();
+    mCG.BatchedSplitNode = false;
+    SolveAndReport("UNSPLIT",mCG,Linop_d,src,sol);
+    ReportMemory(UGrid_d,"after unsplit solve");
+  }
+
+  mCG.BatchedSplit     = split;
+  mCG.BatchedSplitNode = splitnode;
+  // Repeated split solves expose allocations not released between calls
+  for(int r=0;r<repeat;r++){
+    SolveAndReport("SPLIT",mCG,Linop_d,src,sol);
+    ReportMemory(UGrid_d,"after split solve "+std::to_string(r));
+  }
+
+  Grid_finalize();
+}