diff --git a/Grid/algorithms/SplitOperator.h b/Grid/algorithms/SplitOperator.h new file mode 100644 index 000000000..dae9c571f --- /dev/null +++ b/Grid/algorithms/SplitOperator.h @@ -0,0 +1,178 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./lib/algorithms/SplitOperator.h + + Copyright (C) 2026 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory +*************************************************************************************/ +/* END LEGAL */ +#pragma once + +NAMESPACE_BEGIN(Grid); + +template class LinearOperatorBase; +template class CheckerBoardedSparseMatrixBase; + +///////////////////////////////////////////////////////////////////////////////////////////// +// A copy of an operator living on grids whose communicator is split into independent +// partitions, together with those grids. Produced by SplitClone(). +// +// Owns everything it points to. Destruction is the reverse of creation: linear operator, +// matrix, then grids, so no object outlives a grid it references. For 4d operators the +// fermion grids are the gauge grids and are deleted once. +///////////////////////////////////////////////////////////////////////////////////////////// +template +class SplitOperator +{ +public: + GridCartesian *GaugeGrid = nullptr; + GridRedBlackCartesian *GaugeRBGrid = nullptr; + GridCartesian *FermionGrid = nullptr; + GridRedBlackCartesian *FermionRBGrid = nullptr; + + int Partition = 0; // Grid_split vector index held by this rank's partition + int Partitions = 1; // number of partitions + + CheckerBoardedSparseMatrixBase *Matrix = nullptr; + LinearOperatorBase *Linop = nullptr; + GridBase *FieldGrid = nullptr; // grid of Linop's fields + + SplitOperator(void) {}; + + SplitOperator(const SplitOperator &) = delete; + + SplitOperator &operator=(const SplitOperator &) = delete; + + ~SplitOperator(void) + { + delete Linop; + delete Matrix; + if ( FermionRBGrid != GaugeRBGrid ) { + delete FermionRBGrid; + } + if ( FermionGrid != GaugeGrid ) { + delete FermionGrid; + } + delete GaugeRBGrid; + delete GaugeGrid; + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////// +// Index of the vector that Grid_split(std::vector full, Field split) delivers to this +// rank's partition. Grid_split orders partitions lexicographically with the first +// dimension fastest; the split communicator's own rank (srank) uses the reversed MPI +// convention, so the two differ whenever more than one dimension is split. +///////////////////////////////////////////////////////////////////////////////////////////// +inline int GridSplitVectorIndex(GridBase *full,GridBase *split) +{ + int nd = full->_ndimension; + GRID_ASSERT(split->_ndimension == nd); + + Coordinate scoor(nd); + Coordinate ssize(nd); + for(int d=0;dThisProcessorCoor()[d] / split->ProcessorGrid()[d]; + ssize[d] = full->ProcessorGrid()[d] / split->ProcessorGrid()[d]; + } + int index; + Lexicographic::IndexFromCoor(scoor,index,ssize); + return index; +} + +///////////////////////////////////////////////////////////////////////////////////////////// +// Partition MPI layout for a batched solve on grid, from a request as given on the command +// line (--batched-solver-split). Uses the trailing dimensions of grid's processor and shm +// layouts, so it works for 4d and 5d grids. Returns grid's own processor layout (one +// partition) when no split is requested. Asserts divisibility; warns when partitions +// straddle nodes, when node boundaries are not visible, or when there are more partitions +// than right-hand sides. +///////////////////////////////////////////////////////////////////////////////////////////// +inline Coordinate BatchedSolverSplitLayout(GridBase *grid, + const Coordinate &request, + bool node, + int nbatch, + int &partitions) +{ + int nd = GridDefaultMpi().size(); + int pad = grid->_ndimension - nd; + GRID_ASSERT(pad >= 0); + GRID_ASSERT(grid->ShmGrid().size() == grid->_ndimension); + + Coordinate processors(nd); + Coordinate shm(nd); + for(int d=0;dProcessorGrid()[pad+d]; + shm[d] = grid->ShmGrid()[pad+d]; + } + + Coordinate split(nd); + if ( node ) { + split = shm; + } else if ( request.size() == 0 ) { + split = processors; + } else { + GRID_ASSERT(request.size() == nd); + split = request; + } + + partitions = 1; + for(int d=0;d nbatch ) { + std::cout << GridLogWarning << "BatchedSolverSplit: " << partitions << " partitions for " + << nbatch << " right-hand sides; the extra partitions only solve zero padding" << std::endl; + } + return split; +} + +NAMESPACE_END(Grid);