diff --git a/systems/Frontier/split_batched_mobius.job b/systems/Frontier/split_batched_mobius.job new file mode 100644 index 000000000..5e5ac0a06 --- /dev/null +++ b/systems/Frontier/split_batched_mobius.job @@ -0,0 +1,119 @@ +#!/bin/bash -l +#SBATCH --job-name=split-batched-mobius +#SBATCH --nodes=4 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=1:30:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -S 0 +##SBATCH -q debug + +############################################################################## +# MixedPrecisionConjugateGradientBatched with split inner solves, at production +# size: Mobius 64^3x128, Ls=12, mass 0.026, b=1.5 c=0.5 M5=1.8, antiperiodic +# in time, Hadrons default SchurDiagMooeeOperator. +# +# 4 nodes = 32 GCDs, --mpi 2.2.2.4 (local 32^4). --batched-solver-split node +# makes each node one partition (P=4), so Nbatch=4 is one round with no +# zero padding. The driver solves the same batch twice from zero guesses: +# UNSPLIT inner CG on all 32 GCDs, one rhs after another (existing path) +# SPLIT four independent inner CGs, one per node +# and prints per-rhs iterations, true residuals and wall clock for each. +# +# Build (in the Frontier build tree, after ./scripts/filelist in the source +# root so the new test is in tests/solver/Make.inc): +# make -C tests/solver Test_split_mobius_batched +# +# Memory, per GCD (evictable Lattice fields, so bounded by --device-mem): +# double RB 5d field 1.21 GB, float 0.60 GB +# driver src+sol 9.7, solver batch copies 9.7, split partition fields and +# inner CG on the 4x larger split local volume ~20, gauge copies ~5 +# => ~45 GB, hence --device-mem 40000 as in pvdagm_mixed_precision.job. +# Host memory: the device is an inclusive cache, so host RSS per rank is the whole +# ~45 GB working set (plus transient Grid_split staging): ~400 GB of a node's 512 GB. +# The driver prints MEMORY lines (host RSS, peak, allocator caches) at each phase. +# Grid_split stages 2 x 4.8 GB per rank on the host per transfer. +############################################################################## + +LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle +RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID} +mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR" + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +export HIP_VISIBLE_DEVICES=\$GPU +unset ROCR_VISIBLE_DEVICES +if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi +exec numactl -m \$NUMA -N \$NUMA \$* +EOF +chmod +x ./select_gpu + +# Source tree and build of THIS branch (feature/splitGridCGBatch) on Frontier +root=$LUSTRE/SplitCG/Grid/systems/Frontier +source $root/sourceme-rocm7.2.sh + +export OMP_NUM_THREADS=7 +ulimit -c 0 # no 22 GB GPU core dumps +export FI_MR_CACHE_MONITOR=kdreg2 # site default; device-buffer MPI on Slingshot +export MPICH_GPU_SUPPORT_ENABLED=1 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_OFI_NIC_POLICY=GPU +# No GRID_ALLOC_NCACHE_LARGE here. With --enable-unified=no every Lattice lives in host +# memory (the device is an inclusive cache), so a deep large-allocation cache keeps freed +# host blocks on top of the full working set: measured on the laptop at 64 entries it +# held about as much again as the working set, enough to OOM a 4-node run of this job. +module load libfabric + +BIN=$root/tests/solver/Test_split_mobius_batched +OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 40000 --comms-overlap" +vol=64.64.64.128 +MPI_GEOM=2.2.2.4 + +# A thermalised 64^3x128 NERSC configuration. Leave empty for a hot start: +# fine for plumbing, but mass 0.026 on a hot field is not a physical +# iteration count and may run long. +CONFIG= + +PHYS="--Ls 12 --mass 0.026 --M5 1.8 --b 1.5 --c 0.5 --nbatch 4 --tol 1e-8" +if [ -n "$CONFIG" ]; then PHYS="$PHYS --config $CONFIG"; fi + +export GRID_STDOUT_ROOT=$RUNDIR/split +srun -N4 -n32 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi $MPI_GEOM --grid $vol $OPTS \ + $PHYS --batched-solver-split node \ + --debug-stdout --log Error,Warning,Message,Performance \ + > log.split 2>&1 +echo "exit $?" + +############################################################################## +# Readout. Device OOM appears only on stderr as "hipMalloc failed". +############################################################################## +f=$(grep -l "Memory access fault\|NO_TRANSLATION\|hipMalloc failed\|out of memory\|illegal memory access\|GRID_ASSERT" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1) +if [ -n "$f" ]; then + echo "FAULT in $f"; tail -20 $f | cut -c1-160 +fi +grep -h "MEMORY\|BatchedSolverSplit\|Inner CG iterations\|Total time\|true residual\|SUMMARY" \ + $GRID_STDOUT_ROOT/0/Grid.stdout.0 2>/dev/null | cut -c1-200 + +############################################################################## +# What to read. +# +# BatchedSolverSplit line: the partition layout must equal the node layout +# (ShmGrid), e.g. [1 2 2 2] of [2 2 2 4] : 4 partitions. A "straddle node +# boundaries" warning means the rank reordering did not give node-aligned +# blocks; the result is still correct but the inner CG talks off node. +# +# UNSPLIT vs SPLIT: per-rhs inner iterations should agree closely (the same +# solves, different reduction order); the SUMMARY lines give the wall clock. +# The gain is the strong-scaling loss of the 32-GCD inner CG against four +# 8-GCD CGs with node-local halos. "Split setup and transfer" in the Total +# time line prices the clone (grids, gauge split) and the Grid_split / +# Grid_unsplit traffic, which is host-staged. +############################################################################## diff --git a/tests/forces/ForceTest.h b/tests/forces/ForceTest.h new file mode 100644 index 000000000..9531358f0 --- /dev/null +++ b/tests/forces/ForceTest.h @@ -0,0 +1,138 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/forces/ForceTest.h + + Copyright (C) 2022 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#pragma once + +///////////////////////////////////////////////////////////////////////////////////////////// +// Finite-difference check of an action's force: refresh, S1 = S(U), step the links by eps +// along a random momentum P, take the force at the midpoint, step again, S2 = S(U''). +// The force predicts S2 - S1 to O(eps^3). +// +// Runs on a copy of U, so the caller's field is unchanged. Returns S2 - S1 - dSpred for +// the caller to assert on; halving eps should reduce it by about 8. +///////////////////////////////////////////////////////////////////////////////////////////// +template +RealD ForceTest(Action &action,LatticeGaugeField & Uin,MomentumFilterBase &Filter,RealD eps=0.005) +{ + GridBase *UGrid = Uin.Grid(); + + std::vector seeds({1,2,3,5}); + GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds); + GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds); + + LatticeColourMatrix Pmu(UGrid); + LatticeGaugeField P(UGrid); + LatticeGaugeField UdSdU(UGrid); + LatticeGaugeField U(UGrid); + U = Uin; + + std::cout << GridLogMessage << "*********************************************************"<(UdSdU,mu); + Pmu= PeekIndex(P,mu); + dS = dS - trace(Pmu*UdSdUmu)*eps*2.0*2.0; + } + ComplexD dSpred = sum(dS); + RealD diff = S2-S1-dSpred.real(); + + std::cout<< GridLogMessage << "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"< +RealD ForceTestScaling(Action &action,LatticeGaugeField &U,MomentumFilterBase &Filter,const std::string &name, + RealD eps0=0.01,int neps=4) +{ + std::vector eps(neps); + std::vector d(neps); + for(int i=0;i(action,U,Filter,eps[i]); + } + for(int i=0;i 0 ) { + std::cout << " ratio to previous " << d[i-1]/d[i]; + } + std::cout << std::endl; + } + RealD scale = d[neps-2]/d[neps-1]; + std::cout << GridLogMessage << name << ": smallest-eps ratio " << scale << " (expect ~8)" << std::endl; + return scale; +} diff --git a/tests/solver/Test_split_mixedprec_batched.cc b/tests/solver/Test_split_mixedprec_batched.cc new file mode 100644 index 000000000..4f327be0a --- /dev/null +++ b/tests/solver/Test_split_mixedprec_batched.cc @@ -0,0 +1,352 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/solver/Test_split_mixedprec_batched.cc + + Copyright (C) 2026 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory +*************************************************************************************/ +/* END LEGAL */ +///////////////////////////////////////////////////////////////////////////////////////////// +// MixedPrecisionConjugateGradientBatched with inner solves on split-communicator +// partitions (--batched-solver-split), against the same solver unsplit. +// +// mpirun -n 2 ./Test_split_mixedprec_batched --grid 8.8.8.8 --mpi 1.1.1.2 --batched-solver-split 1.1.1.1 +// +// With no --batched-solver-split the partitions are single ranks (1.1.1.1). +// NBATCH is deliberately not a multiple of the partition count, so the last group is +// zero-padded. +///////////////////////////////////////////////////////////////////////////////////////////// +#include + +using namespace std; +using namespace Grid; + +const int NBATCH = 3; +const RealD TOLERANCE = 1.0e-8; + +template +RealD RelativeDifference(const Field &a,const Field &b) +{ + Field diff(a.Grid()); + diff = a - b; + return std::sqrt(norm2(diff)/norm2(b)); +} + +///////////////////////////////////////////////////////////////////////////////////////////// +// The solver assumes vector p of a group lands in the partition whose +// GridSplitVectorIndex is p: fill vector p with the constant p+1, split, and check +// every partition sees its own index. +///////////////////////////////////////////////////////////////////////////////////////////// +void CheckPartitionOrder(GridCartesian *UGrid,const Coordinate &layout) +{ + GridCartesian SGrid(UGrid->FullDimensions(),UGrid->_simd_layout,layout,*UGrid); + int P = UGrid->ProcessorCount()/SGrid.ProcessorCount(); + int index = GridSplitVectorIndex(UGrid,&SGrid); + + std::vector full(P,UGrid); + for(int p=0;p +void CheckSplitBatched(const std::string &name, + LinearOperatorBase &Linop_d, + LinearOperatorBase &Linop_f, + GridBase *grid_f, + std::vector &src, + const Coordinate &layout) +{ + std::cout << GridLogMessage << "==================================================" << std::endl; + std::cout << GridLogMessage << name << std::endl; + std::cout << GridLogMessage << "==================================================" << std::endl; + + int cb = src[0].Checkerboard(); + + SplitOperator *split = Linop_f.SplitClone(layout); + GRID_ASSERT(split != nullptr); + int P = split->Partitions; + + std::vector full(P,grid_f); + std::vector back(P,grid_f); + std::vector Mfull(P,grid_f); + for(int p=0;pFieldGrid); + FieldF s_out(split->FieldGrid); + Grid_split(full,s_in); + Grid_unsplit(back,s_in); + for(int p=0;pLinop->HermOp(s_in,s_out); + Grid_unsplit(back,s_out); + for(int p=0;p sol_ref(NBatch,src[0].Grid()); + std::vector sol_split(NBatch,src[0].Grid()); + for(int i=0;i mCG(TOLERANCE,10000,50,1000,grid_f,Linop_f,Linop_d); + + std::cout << GridLogMessage << name << ": unsplit batched solve" << std::endl; + mCG.BatchedSplit = Coordinate(); + mCG.BatchedSplitNode = false; + mCG(src,sol_ref); + + std::cout << GridLogMessage << name << ": split batched solve, partition layout " << layout << std::endl; + mCG.BatchedSplit = layout; + mCG(src,sol_split); + + FieldD Msol(src[0].Grid()); + Msol.Checkerboard() = cb; + for(int i=0;i seeds4({1,2,3,4}); + std::vector seeds5({5,6,7,8}); + GridParallelRNG RNG4(UGrid_d); + GridParallelRNG RNG5(FGrid_d); + RNG4.SeedFixedIntegers(seeds4); + RNG5.SeedFixedIntegers(seeds5); + + LatticeGaugeFieldD Umu_d(UGrid_d); + LatticeGaugeFieldF Umu_f(UGrid_f); + SU::HotConfiguration(RNG4,Umu_d); + precisionChange(Umu_f,Umu_d); + + // Antiperiodic in time for some actions, so boundary phases must travel with the links + WilsonImplParams antiperiodic; + antiperiodic.boundary_phases[Nd-1] = -1.0; + + // Sources: odd checkerboard (Schur) and full lattice (MdagM), 4d and 5d + std::vector src4_o(NBATCH,UrbGrid_d); + std::vector src5_o(NBATCH,FrbGrid_d); + std::vector src5(NBATCH,FGrid_d); + LatticeFermionD tmp4(UGrid_d); + LatticeFermionD tmp5(FGrid_d); + for(int i=0;i Ld(Dd); + SchurDiagMooeeOperator Lf(Df); + CheckSplitBatched("Wilson SchurDiagMooee",Ld,Lf,UrbGrid_f,src4_o,layout); + } + + ////////////////////////////////////////// + // Wilson clover + ////////////////////////////////////////// + { + RealD mass = 0.1; + RealD csw_r = 1.0; + RealD csw_t = 1.0; + WilsonCloverFermionD Dd(Umu_d,*UGrid_d,*UrbGrid_d,mass,csw_r,csw_t); + WilsonCloverFermionF Df(Umu_f,*UGrid_f,*UrbGrid_f,mass,csw_r,csw_t); + SchurDiagMooeeOperator Ld(Dd); + SchurDiagMooeeOperator Lf(Df); + CheckSplitBatched("WilsonClover SchurDiagMooee",Ld,Lf,UrbGrid_f,src4_o,layout); + } + + ////////////////////////////////////////// + // Compact Wilson clover, antiperiodic + ////////////////////////////////////////// + { + RealD mass = 0.1; + RealD csw_r = 1.0; + RealD csw_t = 1.0; + RealD cF = 1.0; + WilsonAnisotropyCoefficients anis; + CompactWilsonCloverFermionD Dd(Umu_d,*UGrid_d,*UrbGrid_d,mass,csw_r,csw_t,cF,anis,antiperiodic); + CompactWilsonCloverFermionF Df(Umu_f,*UGrid_f,*UrbGrid_f,mass,csw_r,csw_t,cF,anis,antiperiodic); + SchurDiagMooeeOperator Ld(Dd); + SchurDiagMooeeOperator Lf(Df); + CheckSplitBatched("CompactWilsonClover SchurDiagMooee antiperiodic",Ld,Lf,UrbGrid_f,src4_o,layout); + } + + ////////////////////////////////////////// + // Domain wall + ////////////////////////////////////////// + { + RealD mass = 0.1; + RealD M5 = 1.8; + DomainWallFermionD Dd(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5); + DomainWallFermionF Df(Umu_f,*FGrid_f,*FrbGrid_f,*UGrid_f,*UrbGrid_f,mass,M5); + SchurDiagMooeeOperator Ld(Dd); + SchurDiagMooeeOperator Lf(Df); + CheckSplitBatched("DomainWall SchurDiagMooee",Ld,Lf,FrbGrid_f,src5_o,layout); + } + + ////////////////////////////////////////// + // Mobius, antiperiodic, with unequal + // masses; all wrapper kinds + ////////////////////////////////////////// + { + RealD mass = 0.1; + RealD M5 = 1.8; + RealD b = 1.5; + RealD c = 0.5; + MobiusFermionD Dd(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5,b,c,antiperiodic); + MobiusFermionF Df(Umu_f,*FGrid_f,*FrbGrid_f,*UGrid_f,*UrbGrid_f,mass,M5,b,c,antiperiodic); + Dd.SetMass(0.1,0.12); + Df.SetMass(0.1,0.12); + { + SchurDiagMooeeOperator Ld(Dd); + SchurDiagMooeeOperator Lf(Df); + CheckSplitBatched("Mobius SchurDiagMooee antiperiodic",Ld,Lf,FrbGrid_f,src5_o,layout); + } + { + SchurDiagOneOperator Ld(Dd); + SchurDiagOneOperator Lf(Df); + CheckSplitBatched("Mobius SchurDiagOne antiperiodic",Ld,Lf,FrbGrid_f,src5_o,layout); + } + { + SchurDiagTwoOperator Ld(Dd); + SchurDiagTwoOperator Lf(Df); + CheckSplitBatched("Mobius SchurDiagTwo antiperiodic",Ld,Lf,FrbGrid_f,src5_o,layout); + } + { + MdagMLinearOperator Ld(Dd); + MdagMLinearOperator Lf(Df); + CheckSplitBatched("Mobius MdagM antiperiodic",Ld,Lf,FGrid_f,src5,layout); + } + } + + ////////////////////////////////////////// + // ZMobius + ////////////////////////////////////////// + { + RealD mass = 0.1; + RealD M5 = 1.8; + RealD b = 1.0; + RealD c = 0.0; + std::vector gamma(Ls); + for(int s=0;s Ld(Dd); + SchurDiagMooeeOperator Lf(Df); + CheckSplitBatched("ZMobius SchurDiagMooee",Ld,Lf,FrbGrid_f,src5_o,layout); + } + + ////////////////////////////////////////// + // A derived operator must not inherit + // its parent's SplitClone + ////////////////////////////////////////// + { + RealD mass = 0.1; + RealD mu = 0.1; + WilsonTMFermionF Df(Umu_f,*UGrid_f,*UrbGrid_f,mass,mu); + SchurDiagMooeeOperator Lf(Df); + SplitOperator *split = Lf.SplitClone(layout); + std::cout << GridLogMessage << "WilsonTM SplitClone refused: " << (split == nullptr) << std::endl; + GRID_ASSERT(split == nullptr); + } + + std::cout << GridLogMessage << "Test_split_mixedprec_batched: all checks passed" << std::endl; + + Grid_finalize(); +} diff --git a/tests/solver/Test_split_mobius_batched.cc b/tests/solver/Test_split_mobius_batched.cc new file mode 100644 index 000000000..faf5eadfb --- /dev/null +++ b/tests/solver/Test_split_mobius_batched.cc @@ -0,0 +1,236 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/solver/Test_split_mobius_batched.cc + + Copyright (C) 2026 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory +*************************************************************************************/ +/* END LEGAL */ +///////////////////////////////////////////////////////////////////////////////////////////// +// Production-size timing of MixedPrecisionConjugateGradientBatched for Mobius with the +// Hadrons default SchurDiagMooeeOperator: the same batch solved without and then with +// split inner solves (--batched-solver-split), reporting wall clock, per-rhs iterations +// and true residuals for each. +// +// --Ls 12 --mass 0.026 --M5 1.8 --b 1.5 --c 0.5 --nbatch 4 --tol 1e-8 +// --config (omit for a hot configuration: timing only, not physics) +// --nounsplit (skip the reference unsplit solve) +// --repeat N (split solve N times; host RSS must not grow between them) +// +// MEMORY lines report host RSS (current and peak) and allocator cache sizes, maximum over +// ranks, at each phase: with --enable-unified=no every Lattice lives in host memory. +// +// Only one solution vector is kept, so the driver's own footprint is two batches of +// double red-black 5d fields; the solver adds about as much again. +///////////////////////////////////////////////////////////////////////////////////////////// +#include +#include +#ifdef __APPLE__ +#include +#endif + +using namespace std; +using namespace Grid; + +// Host memory of this process in GB: current resident set and its high-water mark +void HostRSS(RealD ¤t,RealD &peak) +{ + struct rusage ru; + getrusage(RUSAGE_SELF,&ru); +#ifdef __APPLE__ + peak = ru.ru_maxrss/1.0e9; // bytes on macOS + mach_task_basic_info_data_t info; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + task_info(mach_task_self(),MACH_TASK_BASIC_INFO,(task_info_t)&info,&count); + current = info.resident_size/1.0e9; +#else + peak = ru.ru_maxrss*1024.0/1.0e9; // kilobytes on Linux + long pages = 0; + long resident = 0; + FILE *f = fopen("/proc/self/statm","r"); + if ( f ) { + if ( fscanf(f,"%ld %ld",&pages,&resident) != 2 ) { + resident = 0; + } + fclose(f); + } + current = resident*(RealD)sysconf(_SC_PAGESIZE)/1.0e9; +#endif +} + +// Largest values over ranks: host RSS now and at peak, and the allocator caches +void ReportMemory(GridBase *grid,const std::string &phase) +{ + RealD rss; + RealD peak; + HostRSS(rss,peak); + RealD hostcache = MemoryManager::HostCacheBytes()/1.0e9; + RealD devcache = MemoryManager::DeviceCacheBytes()/1.0e9; + grid->GlobalMax(rss); + grid->GlobalMax(peak); + grid->GlobalMax(hostcache); + grid->GlobalMax(devcache); + std::cout << GridLogMessage << "MEMORY " << phase + << " : host RSS " << rss << " GB, peak " << peak + << " GB; allocator cache host " << hostcache << " GB, device " << devcache + << " GB (max over ranks)" << std::endl; +} + +typedef LatticeFermionD FieldD; +typedef LatticeFermionF FieldF; + +template +T CmdOption(int argc,char **argv,const std::string &name,T def) +{ + T val = def; + if ( GridCmdOptionExists(argv,argv+argc,name) ) { + std::stringstream ss(GridCmdOptionPayload(argv,argv+argc,name)); + ss >> val; + } + return val; +} + +void SolveAndReport(const std::string &label, + MixedPrecisionConjugateGradientBatched &mCG, + LinearOperatorBase &Linop_d, + std::vector &src, + std::vector &sol) +{ + int nbatch = src.size(); + for(int i=0;i (argc,argv,"--Ls",12); + RealD mass = CmdOption (argc,argv,"--mass",0.026); + RealD M5 = CmdOption (argc,argv,"--M5",1.8); + RealD b = CmdOption (argc,argv,"--b",1.5); + RealD c = CmdOption (argc,argv,"--c",0.5); + int nbatch = CmdOption (argc,argv,"--nbatch",4); + RealD tol = CmdOption (argc,argv,"--tol",1.0e-8); + std::string config = CmdOption(argc,argv,"--config",std::string("")); + bool unsplit = !GridCmdOptionExists(argv,argv+argc,"--nounsplit"); + int repeat = CmdOption (argc,argv,"--repeat",1); + + std::cout << GridLogMessage << "Mobius Ls " << Ls << " mass " << mass << " M5 " << M5 + << " b " << b << " c " << c << " nbatch " << nbatch << " tol " << tol << std::endl; + + GridCartesian *UGrid_d = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()), GridDefaultMpi()); + GridRedBlackCartesian *UrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_d); + GridCartesian *FGrid_d = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_d); + GridRedBlackCartesian *FrbGrid_d = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_d); + + GridCartesian *UGrid_f = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexF::Nsimd()), GridDefaultMpi()); + GridRedBlackCartesian *UrbGrid_f = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_f); + GridCartesian *FGrid_f = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_f); + GridRedBlackCartesian *FrbGrid_f = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_f); + + std::vector seeds4({1,2,3,4}); + std::vector seeds5({5,6,7,8}); + GridParallelRNG RNG4(UGrid_d); + GridParallelRNG RNG5(FGrid_d); + RNG4.SeedFixedIntegers(seeds4); + RNG5.SeedFixedIntegers(seeds5); + + LatticeGaugeFieldD Umu_d(UGrid_d); + LatticeGaugeFieldF Umu_f(UGrid_f); + if ( config.size() ) { + FieldMetaData header; + NerscIO::readConfiguration(Umu_d,header,config); + } else { + std::cout << GridLogMessage << "No --config: hot configuration, timing only" << std::endl; + SU::HotConfiguration(RNG4,Umu_d); + } + precisionChange(Umu_f,Umu_d); + ReportMemory(UGrid_d,"gauge field ready"); + + // Antiperiodic in time, as in production + WilsonImplParams params; + params.boundary_phases[Nd-1] = -1.0; + + MobiusFermionD Dd(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5,b,c,params); + MobiusFermionF Df(Umu_f,*FGrid_f,*FrbGrid_f,*UGrid_f,*UrbGrid_f,mass,M5,b,c,params); + SchurDiagMooeeOperator Linop_d(Dd); + SchurDiagMooeeOperator Linop_f(Df); + + std::vector src(nbatch,FrbGrid_d); + std::vector sol(nbatch,FrbGrid_d); + { + FieldD tmp(FGrid_d); + for(int i=0;i mCG(tol,10000,50,10000,FrbGrid_f,Linop_f,Linop_d); + + Coordinate split = mCG.BatchedSplit; + bool splitnode = mCG.BatchedSplitNode; + + if ( unsplit ) { + mCG.BatchedSplit = Coordinate(); + mCG.BatchedSplitNode = false; + SolveAndReport("UNSPLIT",mCG,Linop_d,src,sol); + ReportMemory(UGrid_d,"after unsplit solve"); + } + + mCG.BatchedSplit = split; + mCG.BatchedSplitNode = splitnode; + // Repeated split solves expose allocations not released between calls + for(int r=0;r