diff --git a/Grid/algorithms/Algorithms.h b/Grid/algorithms/Algorithms.h index ef147c53..b716c48f 100644 --- a/Grid/algorithms/Algorithms.h +++ b/Grid/algorithms/Algorithms.h @@ -48,6 +48,12 @@ Author: Peter Boyle #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/Grid/algorithms/CoarsenedMatrix.h b/Grid/algorithms/CoarsenedMatrix.h index 8af8d7ac..a6c6c030 100644 --- a/Grid/algorithms/CoarsenedMatrix.h +++ b/Grid/algorithms/CoarsenedMatrix.h @@ -211,6 +211,7 @@ namespace Grid { for(int b=0;b compressor; + Stencil.HaloExchange(in,compressor); + + auto point = [dir, disp](){ + if(dir == 0 and disp == 0) + return 8; + else + return (4 * dir + 1 - disp) / 2; + }(); + + parallel_for(int ss=0;ssoSites();ss++){ + siteVector res = zero; + siteVector nbr; + int ptype; + StencilEntry *SE; + + SE=Stencil.GetEntry(ptype,point,ss); + + if(SE->_is_local&&SE->_permute) { + permute(nbr,in._odata[SE->_offset],ptype); + } else if(SE->_is_local) { + nbr = in._odata[SE->_offset]; + } else { + nbr = Stencil.CommBuf()[SE->_offset]; + } + + res = res + A[point]._odata[ss]*nbr; + + vstream(out._odata[ss],res); + } + }; + + void Mdiag(const CoarseVector &in, CoarseVector &out){ + Mdir(in, out, 0, 0); // use the self coupling (= last) point of the stencil + }; CoarsenedMatrix(GridCartesian &CoarseGrid) : @@ -417,7 +463,7 @@ namespace Grid { std::cout< + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H +#define GRID_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H + +namespace Grid { + +template +class CommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction { + public: + bool ErrorOnNoConverge; // Throw an assert when CAGMRES fails to converge, + // defaults to true + + RealD Tolerance; + + Integer MaxIterations; + Integer RestartLength; + Integer MaxNumberOfRestarts; + Integer IterationCount; // Number of iterations the CAGMRES took to finish, + // filled in upon completion + + GridStopWatch MatrixTimer; + GridStopWatch LinalgTimer; + GridStopWatch QrTimer; + GridStopWatch CompSolutionTimer; + + Eigen::MatrixXcd H; + + std::vector> y; + std::vector> gamma; + std::vector> c; + std::vector> s; + + CommunicationAvoidingGeneralisedMinimalResidual(RealD tol, + Integer maxit, + Integer restart_length, + bool err_on_no_conv = true) + : Tolerance(tol) + , MaxIterations(maxit) + , RestartLength(restart_length) + , MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1)) + , ErrorOnNoConverge(err_on_no_conv) + , H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base + , y(RestartLength + 1, 0.) + , gamma(RestartLength + 1, 0.) + , c(RestartLength + 1, 0.) + , s(RestartLength + 1, 0.) {}; + + void operator()(LinearOperatorBase &LinOp, const Field &src, Field &psi) { + + std::cout << GridLogWarning << "This algorithm currently doesn't differ from regular GMRES" << std::endl; + + psi.checkerboard = src.checkerboard; + conformable(psi, src); + + RealD guess = norm2(psi); + assert(std::isnan(guess) == 0); + + RealD cp; + RealD ssq = norm2(src); + RealD rsq = Tolerance * Tolerance * ssq; + + Field r(src._grid); + + std::cout << std::setprecision(4) << std::scientific; + std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: guess " << guess << std::endl; + std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: src " << ssq << std::endl; + + MatrixTimer.Reset(); + LinalgTimer.Reset(); + QrTimer.Reset(); + CompSolutionTimer.Reset(); + + GridStopWatch SolverTimer; + SolverTimer.Start(); + + IterationCount = 0; + + for (int k=0; k &LinOp, const Field &src, Field &psi, RealD rsq) { + + RealD cp = 0; + + Field w(src._grid); + Field r(src._grid); + + // this should probably be made a class member so that it is only allocated once, not in every restart + std::vector v(RestartLength + 1, src._grid); for (auto &elem : v) elem = zero; + + MatrixTimer.Start(); + LinOp.Op(psi, w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + r = src - w; + + gamma[0] = sqrt(norm2(r)); + + v[0] = (1. / gamma[0]) * r; + LinalgTimer.Stop(); + + for (int i=0; i &LinOp, std::vector &v, Field &w, int iter) { + + MatrixTimer.Start(); + LinOp.Op(v[iter], w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + for (int i = 0; i <= iter; ++i) { + H(iter, i) = innerProduct(v[i], w); + w = w - H(iter, i) * v[i]; + } + + H(iter, iter + 1) = sqrt(norm2(w)); + v[iter + 1] = (1. / H(iter, iter + 1)) * w; + LinalgTimer.Stop(); + } + + void qrUpdate(int iter) { + + QrTimer.Start(); + for (int i = 0; i < iter ; ++i) { + auto tmp = -s[i] * H(iter, i) + c[i] * H(iter, i + 1); + H(iter, i) = std::conj(c[i]) * H(iter, i) + std::conj(s[i]) * H(iter, i + 1); + H(iter, i + 1) = tmp; + } + + // Compute new Givens Rotation + ComplexD nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1))); + c[iter] = H(iter, iter) / nu; + s[iter] = H(iter, iter + 1) / nu; + + // Apply new Givens rotation + H(iter, iter) = nu; + H(iter, iter + 1) = 0.; + + gamma[iter + 1] = -s[iter] * gamma[iter]; + gamma[iter] = std::conj(c[iter]) * gamma[iter]; + QrTimer.Stop(); + } + + void computeSolution(std::vector const &v, Field &psi, int iter) { + + CompSolutionTimer.Start(); + for (int i = iter; i >= 0; i--) { + y[i] = gamma[i]; + for (int k = i + 1; k <= iter; k++) + y[i] = y[i] - H(k, i) * y[k]; + y[i] = y[i] / H(i, i); + } + + for (int i = 0; i <= iter; i++) + psi = psi + v[i] * y[i]; + CompSolutionTimer.Stop(); + } +}; +} +#endif diff --git a/Grid/algorithms/iterative/FlexibleCommunicationAvoidingGeneralisedMinimalResidual.h b/Grid/algorithms/iterative/FlexibleCommunicationAvoidingGeneralisedMinimalResidual.h new file mode 100644 index 00000000..db857248 --- /dev/null +++ b/Grid/algorithms/iterative/FlexibleCommunicationAvoidingGeneralisedMinimalResidual.h @@ -0,0 +1,256 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./lib/algorithms/iterative/FlexibleCommunicationAvoidingGeneralisedMinimalResidual.h + +Copyright (C) 2015 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_FLEXIBLE_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H +#define GRID_FLEXIBLE_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H + +namespace Grid { + +template +class FlexibleCommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction { + public: + bool ErrorOnNoConverge; // Throw an assert when FCAGMRES fails to converge, + // defaults to true + + RealD Tolerance; + + Integer MaxIterations; + Integer RestartLength; + Integer MaxNumberOfRestarts; + Integer IterationCount; // Number of iterations the FCAGMRES took to finish, + // filled in upon completion + + GridStopWatch MatrixTimer; + GridStopWatch PrecTimer; + GridStopWatch LinalgTimer; + GridStopWatch QrTimer; + GridStopWatch CompSolutionTimer; + + Eigen::MatrixXcd H; + + std::vector> y; + std::vector> gamma; + std::vector> c; + std::vector> s; + + LinearFunction &Preconditioner; + + FlexibleCommunicationAvoidingGeneralisedMinimalResidual(RealD tol, + Integer maxit, + LinearFunction &Prec, + Integer restart_length, + bool err_on_no_conv = true) + : Tolerance(tol) + , MaxIterations(maxit) + , RestartLength(restart_length) + , MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1)) + , ErrorOnNoConverge(err_on_no_conv) + , H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base + , y(RestartLength + 1, 0.) + , gamma(RestartLength + 1, 0.) + , c(RestartLength + 1, 0.) + , s(RestartLength + 1, 0.) + , Preconditioner(Prec) {}; + + void operator()(LinearOperatorBase &LinOp, const Field &src, Field &psi) { + + std::cout << GridLogWarning << "This algorithm currently doesn't differ from regular FGMRES" << std::endl; + + psi.checkerboard = src.checkerboard; + conformable(psi, src); + + RealD guess = norm2(psi); + assert(std::isnan(guess) == 0); + + RealD cp; + RealD ssq = norm2(src); + RealD rsq = Tolerance * Tolerance * ssq; + + Field r(src._grid); + + std::cout << std::setprecision(4) << std::scientific; + std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: guess " << guess << std::endl; + std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: src " << ssq << std::endl; + + PrecTimer.Reset(); + MatrixTimer.Reset(); + LinalgTimer.Reset(); + QrTimer.Reset(); + CompSolutionTimer.Reset(); + + GridStopWatch SolverTimer; + SolverTimer.Start(); + + IterationCount = 0; + + for (int k=0; k &LinOp, const Field &src, Field &psi, RealD rsq) { + + RealD cp = 0; + + Field w(src._grid); + Field r(src._grid); + + // these should probably be made class members so that they are only allocated once, not in every restart + std::vector v(RestartLength + 1, src._grid); for (auto &elem : v) elem = zero; + std::vector z(RestartLength + 1, src._grid); for (auto &elem : z) elem = zero; + + MatrixTimer.Start(); + LinOp.Op(psi, w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + r = src - w; + + gamma[0] = sqrt(norm2(r)); + + v[0] = (1. / gamma[0]) * r; + LinalgTimer.Stop(); + + for (int i=0; i &LinOp, std::vector &v, std::vector &z, Field &w, int iter) { + + PrecTimer.Start(); + Preconditioner(v[iter], z[iter]); + PrecTimer.Stop(); + + MatrixTimer.Start(); + LinOp.Op(z[iter], w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + for (int i = 0; i <= iter; ++i) { + H(iter, i) = innerProduct(v[i], w); + w = w - H(iter, i) * v[i]; + } + + H(iter, iter + 1) = sqrt(norm2(w)); + v[iter + 1] = (1. / H(iter, iter + 1)) * w; + LinalgTimer.Stop(); + } + + void qrUpdate(int iter) { + + QrTimer.Start(); + for (int i = 0; i < iter ; ++i) { + auto tmp = -s[i] * H(iter, i) + c[i] * H(iter, i + 1); + H(iter, i) = std::conj(c[i]) * H(iter, i) + std::conj(s[i]) * H(iter, i + 1); + H(iter, i + 1) = tmp; + } + + // Compute new Givens Rotation + ComplexD nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1))); + c[iter] = H(iter, iter) / nu; + s[iter] = H(iter, iter + 1) / nu; + + // Apply new Givens rotation + H(iter, iter) = nu; + H(iter, iter + 1) = 0.; + + gamma[iter + 1] = -s[iter] * gamma[iter]; + gamma[iter] = std::conj(c[iter]) * gamma[iter]; + QrTimer.Stop(); + } + + void computeSolution(std::vector const &z, Field &psi, int iter) { + + CompSolutionTimer.Start(); + for (int i = iter; i >= 0; i--) { + y[i] = gamma[i]; + for (int k = i + 1; k <= iter; k++) + y[i] = y[i] - H(k, i) * y[k]; + y[i] = y[i] / H(i, i); + } + + for (int i = 0; i <= iter; i++) + psi = psi + z[i] * y[i]; + CompSolutionTimer.Stop(); + } +}; +} +#endif diff --git a/Grid/algorithms/iterative/FlexibleGeneralisedMinimalResidual.h b/Grid/algorithms/iterative/FlexibleGeneralisedMinimalResidual.h new file mode 100644 index 00000000..efc8c787 --- /dev/null +++ b/Grid/algorithms/iterative/FlexibleGeneralisedMinimalResidual.h @@ -0,0 +1,254 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./lib/algorithms/iterative/FlexibleGeneralisedMinimalResidual.h + +Copyright (C) 2015 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H +#define GRID_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H + +namespace Grid { + +template +class FlexibleGeneralisedMinimalResidual : public OperatorFunction { + public: + bool ErrorOnNoConverge; // Throw an assert when FGMRES fails to converge, + // defaults to true + + RealD Tolerance; + + Integer MaxIterations; + Integer RestartLength; + Integer MaxNumberOfRestarts; + Integer IterationCount; // Number of iterations the FGMRES took to finish, + // filled in upon completion + + GridStopWatch MatrixTimer; + GridStopWatch PrecTimer; + GridStopWatch LinalgTimer; + GridStopWatch QrTimer; + GridStopWatch CompSolutionTimer; + + Eigen::MatrixXcd H; + + std::vector> y; + std::vector> gamma; + std::vector> c; + std::vector> s; + + LinearFunction &Preconditioner; + + FlexibleGeneralisedMinimalResidual(RealD tol, + Integer maxit, + LinearFunction &Prec, + Integer restart_length, + bool err_on_no_conv = true) + : Tolerance(tol) + , MaxIterations(maxit) + , RestartLength(restart_length) + , MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1)) + , ErrorOnNoConverge(err_on_no_conv) + , H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base + , y(RestartLength + 1, 0.) + , gamma(RestartLength + 1, 0.) + , c(RestartLength + 1, 0.) + , s(RestartLength + 1, 0.) + , Preconditioner(Prec) {}; + + void operator()(LinearOperatorBase &LinOp, const Field &src, Field &psi) { + + psi.checkerboard = src.checkerboard; + conformable(psi, src); + + RealD guess = norm2(psi); + assert(std::isnan(guess) == 0); + + RealD cp; + RealD ssq = norm2(src); + RealD rsq = Tolerance * Tolerance * ssq; + + Field r(src._grid); + + std::cout << std::setprecision(4) << std::scientific; + std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: guess " << guess << std::endl; + std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: src " << ssq << std::endl; + + PrecTimer.Reset(); + MatrixTimer.Reset(); + LinalgTimer.Reset(); + QrTimer.Reset(); + CompSolutionTimer.Reset(); + + GridStopWatch SolverTimer; + SolverTimer.Start(); + + IterationCount = 0; + + for (int k=0; k &LinOp, const Field &src, Field &psi, RealD rsq) { + + RealD cp = 0; + + Field w(src._grid); + Field r(src._grid); + + // these should probably be made class members so that they are only allocated once, not in every restart + std::vector v(RestartLength + 1, src._grid); for (auto &elem : v) elem = zero; + std::vector z(RestartLength + 1, src._grid); for (auto &elem : z) elem = zero; + + MatrixTimer.Start(); + LinOp.Op(psi, w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + r = src - w; + + gamma[0] = sqrt(norm2(r)); + + v[0] = (1. / gamma[0]) * r; + LinalgTimer.Stop(); + + for (int i=0; i &LinOp, std::vector &v, std::vector &z, Field &w, int iter) { + + PrecTimer.Start(); + Preconditioner(v[iter], z[iter]); + PrecTimer.Stop(); + + MatrixTimer.Start(); + LinOp.Op(z[iter], w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + for (int i = 0; i <= iter; ++i) { + H(iter, i) = innerProduct(v[i], w); + w = w - H(iter, i) * v[i]; + } + + H(iter, iter + 1) = sqrt(norm2(w)); + v[iter + 1] = (1. / H(iter, iter + 1)) * w; + LinalgTimer.Stop(); + } + + void qrUpdate(int iter) { + + QrTimer.Start(); + for (int i = 0; i < iter ; ++i) { + auto tmp = -s[i] * H(iter, i) + c[i] * H(iter, i + 1); + H(iter, i) = std::conj(c[i]) * H(iter, i) + std::conj(s[i]) * H(iter, i + 1); + H(iter, i + 1) = tmp; + } + + // Compute new Givens Rotation + ComplexD nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1))); + c[iter] = H(iter, iter) / nu; + s[iter] = H(iter, iter + 1) / nu; + + // Apply new Givens rotation + H(iter, iter) = nu; + H(iter, iter + 1) = 0.; + + gamma[iter + 1] = -s[iter] * gamma[iter]; + gamma[iter] = std::conj(c[iter]) * gamma[iter]; + QrTimer.Stop(); + } + + void computeSolution(std::vector const &z, Field &psi, int iter) { + + CompSolutionTimer.Start(); + for (int i = iter; i >= 0; i--) { + y[i] = gamma[i]; + for (int k = i + 1; k <= iter; k++) + y[i] = y[i] - H(k, i) * y[k]; + y[i] = y[i] / H(i, i); + } + + for (int i = 0; i <= iter; i++) + psi = psi + z[i] * y[i]; + CompSolutionTimer.Stop(); + } +}; +} +#endif diff --git a/Grid/algorithms/iterative/GeneralisedMinimalResidual.h b/Grid/algorithms/iterative/GeneralisedMinimalResidual.h new file mode 100644 index 00000000..10636234 --- /dev/null +++ b/Grid/algorithms/iterative/GeneralisedMinimalResidual.h @@ -0,0 +1,242 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./lib/algorithms/iterative/GeneralisedMinimalResidual.h + +Copyright (C) 2015 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_GENERALISED_MINIMAL_RESIDUAL_H +#define GRID_GENERALISED_MINIMAL_RESIDUAL_H + +namespace Grid { + +template +class GeneralisedMinimalResidual : public OperatorFunction { + public: + bool ErrorOnNoConverge; // Throw an assert when GMRES fails to converge, + // defaults to true + + RealD Tolerance; + + Integer MaxIterations; + Integer RestartLength; + Integer MaxNumberOfRestarts; + Integer IterationCount; // Number of iterations the GMRES took to finish, + // filled in upon completion + + GridStopWatch MatrixTimer; + GridStopWatch LinalgTimer; + GridStopWatch QrTimer; + GridStopWatch CompSolutionTimer; + + Eigen::MatrixXcd H; + + std::vector> y; + std::vector> gamma; + std::vector> c; + std::vector> s; + + GeneralisedMinimalResidual(RealD tol, + Integer maxit, + Integer restart_length, + bool err_on_no_conv = true) + : Tolerance(tol) + , MaxIterations(maxit) + , RestartLength(restart_length) + , MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1)) + , ErrorOnNoConverge(err_on_no_conv) + , H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base + , y(RestartLength + 1, 0.) + , gamma(RestartLength + 1, 0.) + , c(RestartLength + 1, 0.) + , s(RestartLength + 1, 0.) {}; + + void operator()(LinearOperatorBase &LinOp, const Field &src, Field &psi) { + + psi.checkerboard = src.checkerboard; + conformable(psi, src); + + RealD guess = norm2(psi); + assert(std::isnan(guess) == 0); + + RealD cp; + RealD ssq = norm2(src); + RealD rsq = Tolerance * Tolerance * ssq; + + Field r(src._grid); + + std::cout << std::setprecision(4) << std::scientific; + std::cout << GridLogIterative << "GeneralisedMinimalResidual: guess " << guess << std::endl; + std::cout << GridLogIterative << "GeneralisedMinimalResidual: src " << ssq << std::endl; + + MatrixTimer.Reset(); + LinalgTimer.Reset(); + QrTimer.Reset(); + CompSolutionTimer.Reset(); + + GridStopWatch SolverTimer; + SolverTimer.Start(); + + IterationCount = 0; + + for (int k=0; k &LinOp, const Field &src, Field &psi, RealD rsq) { + + RealD cp = 0; + + Field w(src._grid); + Field r(src._grid); + + // this should probably be made a class member so that it is only allocated once, not in every restart + std::vector v(RestartLength + 1, src._grid); for (auto &elem : v) elem = zero; + + MatrixTimer.Start(); + LinOp.Op(psi, w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + r = src - w; + + gamma[0] = sqrt(norm2(r)); + + v[0] = (1. / gamma[0]) * r; + LinalgTimer.Stop(); + + for (int i=0; i &LinOp, std::vector &v, Field &w, int iter) { + + MatrixTimer.Start(); + LinOp.Op(v[iter], w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + for (int i = 0; i <= iter; ++i) { + H(iter, i) = innerProduct(v[i], w); + w = w - H(iter, i) * v[i]; + } + + H(iter, iter + 1) = sqrt(norm2(w)); + v[iter + 1] = (1. / H(iter, iter + 1)) * w; + LinalgTimer.Stop(); + } + + void qrUpdate(int iter) { + + QrTimer.Start(); + for (int i = 0; i < iter ; ++i) { + auto tmp = -s[i] * H(iter, i) + c[i] * H(iter, i + 1); + H(iter, i) = std::conj(c[i]) * H(iter, i) + std::conj(s[i]) * H(iter, i + 1); + H(iter, i + 1) = tmp; + } + + // Compute new Givens Rotation + ComplexD nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1))); + c[iter] = H(iter, iter) / nu; + s[iter] = H(iter, iter + 1) / nu; + + // Apply new Givens rotation + H(iter, iter) = nu; + H(iter, iter + 1) = 0.; + + gamma[iter + 1] = -s[iter] * gamma[iter]; + gamma[iter] = std::conj(c[iter]) * gamma[iter]; + QrTimer.Stop(); + } + + void computeSolution(std::vector const &v, Field &psi, int iter) { + + CompSolutionTimer.Start(); + for (int i = iter; i >= 0; i--) { + y[i] = gamma[i]; + for (int k = i + 1; k <= iter; k++) + y[i] = y[i] - H(k, i) * y[k]; + y[i] = y[i] / H(i, i); + } + + for (int i = 0; i <= iter; i++) + psi = psi + v[i] * y[i]; + CompSolutionTimer.Stop(); + } +}; +} +#endif diff --git a/Grid/algorithms/iterative/MinimalResidual.h b/Grid/algorithms/iterative/MinimalResidual.h new file mode 100644 index 00000000..fa1912cf --- /dev/null +++ b/Grid/algorithms/iterative/MinimalResidual.h @@ -0,0 +1,156 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./lib/algorithms/iterative/MinimalResidual.h + +Copyright (C) 2015 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_MINIMAL_RESIDUAL_H +#define GRID_MINIMAL_RESIDUAL_H + +namespace Grid { + +template class MinimalResidual : public OperatorFunction { + public: + bool ErrorOnNoConverge; // throw an assert when the MR fails to converge. + // Defaults true. + RealD Tolerance; + Integer MaxIterations; + RealD overRelaxParam; + Integer IterationsToComplete; // Number of iterations the MR took to finish. + // Filled in upon completion + + MinimalResidual(RealD tol, Integer maxit, Real ovrelparam = 1.0, bool err_on_no_conv = true) + : Tolerance(tol), MaxIterations(maxit), overRelaxParam(ovrelparam), ErrorOnNoConverge(err_on_no_conv){}; + + void operator()(LinearOperatorBase &Linop, const Field &src, Field &psi) { + + psi.checkerboard = src.checkerboard; + conformable(psi, src); + + Complex a, c; + Real d; + + Field Mr(src); + Field r(src); + + // Initial residual computation & set up + RealD guess = norm2(psi); + assert(std::isnan(guess) == 0); + + RealD ssq = norm2(src); + RealD rsq = Tolerance * Tolerance * ssq; + + Linop.Op(psi, Mr); + + r = src - Mr; + + RealD cp = norm2(r); + + std::cout << std::setprecision(4) << std::scientific; + std::cout << GridLogIterative << "MinimalResidual: guess " << guess << std::endl; + std::cout << GridLogIterative << "MinimalResidual: src " << ssq << std::endl; + std::cout << GridLogIterative << "MinimalResidual: mp " << d << std::endl; + std::cout << GridLogIterative << "MinimalResidual: cp,r " << cp << std::endl; + + if (cp <= rsq) { + return; + } + + std::cout << GridLogIterative << "MinimalResidual: k=0 residual " << cp << " target " << rsq << std::endl; + + GridStopWatch LinalgTimer; + GridStopWatch MatrixTimer; + GridStopWatch SolverTimer; + + SolverTimer.Start(); + int k; + for (k = 1; k <= MaxIterations; k++) { + + MatrixTimer.Start(); + Linop.Op(r, Mr); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + + c = innerProduct(Mr, r); + + d = norm2(Mr); + + a = c / d; + + a = a * overRelaxParam; + + psi = psi + r * a; + + r = r - Mr * a; + + cp = norm2(r); + + LinalgTimer.Stop(); + + std::cout << GridLogIterative << "MinimalResidual: Iteration " << k + << " residual " << cp << " target " << rsq << std::endl; + std::cout << GridLogDebug << "a = " << a << " c = " << c << " d = " << d << std::endl; + + // Stopping condition + if (cp <= rsq) { + SolverTimer.Stop(); + + Linop.Op(psi, Mr); + r = src - Mr; + + RealD srcnorm = sqrt(ssq); + RealD resnorm = sqrt(norm2(r)); + RealD true_residual = resnorm / srcnorm; + + std::cout << GridLogMessage << "MinimalResidual Converged on iteration " << k + << " computed residual " << sqrt(cp / ssq) + << " true residual " << true_residual + << " target " << Tolerance << std::endl; + + std::cout << GridLogMessage << "MR Time elapsed: Total " << SolverTimer.Elapsed() << std::endl; + std::cout << GridLogMessage << "MR Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl; + std::cout << GridLogMessage << "MR Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl; + + if (ErrorOnNoConverge) + assert(true_residual / Tolerance < 10000.0); + + IterationsToComplete = k; + + return; + } + } + + std::cout << GridLogMessage << "MinimalResidual did NOT converge" + << std::endl; + + if (ErrorOnNoConverge) + assert(0); + + IterationsToComplete = k; + } +}; +} // namespace Grid +#endif diff --git a/Grid/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h b/Grid/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h new file mode 100644 index 00000000..04113684 --- /dev/null +++ b/Grid/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h @@ -0,0 +1,273 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./lib/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h + +Copyright (C) 2015 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_MIXED_PRECISION_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H +#define GRID_MIXED_PRECISION_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H + +namespace Grid { + +template::value == 2, int>::type = 0, typename std::enable_if< getPrecision::value == 1, int>::type = 0> +class MixedPrecisionFlexibleGeneralisedMinimalResidual : public OperatorFunction { + public: + bool ErrorOnNoConverge; // Throw an assert when MPFGMRES fails to converge, + // defaults to true + + RealD Tolerance; + + Integer MaxIterations; + Integer RestartLength; + Integer MaxNumberOfRestarts; + Integer IterationCount; // Number of iterations the MPFGMRES took to finish, + // filled in upon completion + + GridStopWatch MatrixTimer; + GridStopWatch PrecTimer; + GridStopWatch LinalgTimer; + GridStopWatch QrTimer; + GridStopWatch CompSolutionTimer; + GridStopWatch ChangePrecTimer; + + Eigen::MatrixXcd H; + + std::vector> y; + std::vector> gamma; + std::vector> c; + std::vector> s; + + GridBase* SinglePrecGrid; + + LinearFunction &Preconditioner; + + MixedPrecisionFlexibleGeneralisedMinimalResidual(RealD tol, + Integer maxit, + GridBase * sp_grid, + LinearFunction &Prec, + Integer restart_length, + bool err_on_no_conv = true) + : Tolerance(tol) + , MaxIterations(maxit) + , RestartLength(restart_length) + , MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1)) + , ErrorOnNoConverge(err_on_no_conv) + , H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base + , y(RestartLength + 1, 0.) + , gamma(RestartLength + 1, 0.) + , c(RestartLength + 1, 0.) + , s(RestartLength + 1, 0.) + , SinglePrecGrid(sp_grid) + , Preconditioner(Prec) {}; + + void operator()(LinearOperatorBase &LinOp, const FieldD &src, FieldD &psi) { + + psi.checkerboard = src.checkerboard; + conformable(psi, src); + + RealD guess = norm2(psi); + assert(std::isnan(guess) == 0); + + RealD cp; + RealD ssq = norm2(src); + RealD rsq = Tolerance * Tolerance * ssq; + + FieldD r(src._grid); + + std::cout << std::setprecision(4) << std::scientific; + std::cout << GridLogIterative << "MPFGMRES: guess " << guess << std::endl; + std::cout << GridLogIterative << "MPFGMRES: src " << ssq << std::endl; + + PrecTimer.Reset(); + MatrixTimer.Reset(); + LinalgTimer.Reset(); + QrTimer.Reset(); + CompSolutionTimer.Reset(); + ChangePrecTimer.Reset(); + + GridStopWatch SolverTimer; + SolverTimer.Start(); + + IterationCount = 0; + + for (int k=0; k &LinOp, const FieldD &src, FieldD &psi, RealD rsq) { + + RealD cp = 0; + + FieldD w(src._grid); + FieldD r(src._grid); + + // these should probably be made class members so that they are only allocated once, not in every restart + std::vector v(RestartLength + 1, src._grid); for (auto &elem : v) elem = zero; + std::vector z(RestartLength + 1, src._grid); for (auto &elem : z) elem = zero; + + MatrixTimer.Start(); + LinOp.Op(psi, w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + r = src - w; + + gamma[0] = sqrt(norm2(r)); + + v[0] = (1. / gamma[0]) * r; + LinalgTimer.Stop(); + + for (int i=0; i &LinOp, std::vector &v, std::vector &z, FieldD &w, int iter) { + + FieldF v_f(SinglePrecGrid); + FieldF z_f(SinglePrecGrid); + + ChangePrecTimer.Start(); + precisionChange(v_f, v[iter]); + precisionChange(z_f, z[iter]); + ChangePrecTimer.Stop(); + + PrecTimer.Start(); + Preconditioner(v_f, z_f); + PrecTimer.Stop(); + + ChangePrecTimer.Start(); + precisionChange(z[iter], z_f); + ChangePrecTimer.Stop(); + + MatrixTimer.Start(); + LinOp.Op(z[iter], w); + MatrixTimer.Stop(); + + LinalgTimer.Start(); + for (int i = 0; i <= iter; ++i) { + H(iter, i) = innerProduct(v[i], w); + w = w - H(iter, i) * v[i]; + } + + H(iter, iter + 1) = sqrt(norm2(w)); + v[iter + 1] = (1. / H(iter, iter + 1)) * w; + LinalgTimer.Stop(); + } + + void qrUpdate(int iter) { + + QrTimer.Start(); + for (int i = 0; i < iter ; ++i) { + auto tmp = -s[i] * H(iter, i) + c[i] * H(iter, i + 1); + H(iter, i) = std::conj(c[i]) * H(iter, i) + std::conj(s[i]) * H(iter, i + 1); + H(iter, i + 1) = tmp; + } + + // Compute new Givens Rotation + ComplexD nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1))); + c[iter] = H(iter, iter) / nu; + s[iter] = H(iter, iter + 1) / nu; + + // Apply new Givens rotation + H(iter, iter) = nu; + H(iter, iter + 1) = 0.; + + gamma[iter + 1] = -s[iter] * gamma[iter]; + gamma[iter] = std::conj(c[iter]) * gamma[iter]; + QrTimer.Stop(); + } + + void computeSolution(std::vector const &z, FieldD &psi, int iter) { + + CompSolutionTimer.Start(); + for (int i = iter; i >= 0; i--) { + y[i] = gamma[i]; + for (int k = i + 1; k <= iter; k++) + y[i] = y[i] - H(k, i) * y[k]; + y[i] = y[i] / H(i, i); + } + + for (int i = 0; i <= iter; i++) + psi = psi + z[i] * y[i]; + CompSolutionTimer.Stop(); + } +}; +} +#endif diff --git a/Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h b/Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h index fd11352e..c723c4a9 100644 --- a/Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h +++ b/Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h @@ -139,8 +139,11 @@ namespace Grid { MatTimer.Start(); Linop.HermOpAndNorm(psi,Az,zAz,zAAz); MatTimer.Stop(); + + LinalgTimer.Start(); r=src-Az; - + LinalgTimer.Stop(); + ///////////////////// // p = Prec(r) ///////////////////// @@ -152,8 +155,10 @@ namespace Grid { Linop.HermOp(z,tmp); MatTimer.Stop(); + LinalgTimer.Start(); ttmp=tmp; tmp=tmp-r; + LinalgTimer.Stop(); /* std::cout< &logstreams); +extern GridLogger GridLogMG; extern GridLogger GridLogIRL; extern GridLogger GridLogSolver; extern GridLogger GridLogError; diff --git a/Grid/parallelIO/BinaryIO.cc b/Grid/parallelIO/BinaryIO.cc new file mode 100644 index 00000000..221a7fe8 --- /dev/null +++ b/Grid/parallelIO/BinaryIO.cc @@ -0,0 +1,3 @@ +#include + +int Grid::BinaryIO::latticeWriteMaxRetry = -1; diff --git a/Grid/parallelIO/BinaryIO.h b/Grid/parallelIO/BinaryIO.h index a60fe962..1895dc3e 100644 --- a/Grid/parallelIO/BinaryIO.h +++ b/Grid/parallelIO/BinaryIO.h @@ -81,6 +81,7 @@ inline void removeWhitespace(std::string &key) /////////////////////////////////////////////////////////////////////////////////////////////////// class BinaryIO { public: + static int latticeWriteMaxRetry; ///////////////////////////////////////////////////////////////////////////// // more byte manipulation helpers @@ -209,10 +210,10 @@ PARALLEL_CRITICAL static inline void le32toh_v(void *file_object,uint64_t bytes) { uint32_t *fp = (uint32_t *)file_object; - uint32_t f; uint64_t count = bytes/sizeof(uint32_t); parallel_for(uint64_t i=0;i>8) | ((f&0xFF000000UL)>>24) ; @@ -234,10 +235,9 @@ PARALLEL_CRITICAL static inline void le64toh_v(void *file_object,uint64_t bytes) { uint64_t *fp = (uint64_t *)file_object; - uint64_t f,g; - uint64_t count = bytes/sizeof(uint64_t); parallel_for(uint64_t i=0;i>8) | ((f&0xFF000000UL)>>24) ; @@ -348,7 +348,8 @@ PARALLEL_CRITICAL int ieee32 = (format == std::string("IEEE32")); int ieee64big = (format == std::string("IEEE64BIG")); int ieee64 = (format == std::string("IEEE64")); - + assert(ieee64||ieee32|ieee64big||ieee32big); + assert((ieee64+ieee32+ieee64big+ieee32big)==1); ////////////////////////////////////////////////////////////////////////////// // Do the I/O ////////////////////////////////////////////////////////////////////////////// @@ -370,7 +371,7 @@ PARALLEL_CRITICAL #endif } else { std::cout << GridLogMessage <<"IOobject: C++ read I/O " << file << " : " - << iodata.size() * sizeof(fobj) << " bytes" << std::endl; + << iodata.size() * sizeof(fobj) << " bytes and offset " << offset << std::endl; std::ifstream fin; fin.open(file, std::ios::binary | std::ios::in); if (control & BINARYIO_MASTER_APPEND) @@ -582,7 +583,9 @@ PARALLEL_CRITICAL typedef typename vobj::scalar_object sobj; typedef typename vobj::Realified::scalar_type word; word w=0; GridBase *grid = Umu._grid; - uint64_t lsites = grid->lSites(); + uint64_t lsites = grid->lSites(), offsetCopy = offset; + int attemptsLeft = std::max(0, BinaryIO::latticeWriteMaxRetry); + bool checkWrite = (BinaryIO::latticeWriteMaxRetry >= 0); std::vector scalardata(lsites); std::vector iodata(lsites); // Munge, checksum, byte order in here @@ -597,9 +600,35 @@ PARALLEL_CRITICAL grid->Barrier(); timer.Stop(); + while (attemptsLeft >= 0) + { + grid->Barrier(); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); + if (checkWrite) + { + std::vector ckiodata(lsites); + uint32_t cknersc_csum, ckscidac_csuma, ckscidac_csumb; + uint64_t ckoffset = offsetCopy; - IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC, - nersc_csum,scidac_csuma,scidac_csumb); + std::cout << GridLogMessage << "writeLatticeObject: read back object" << std::endl; + grid->Barrier(); + IOobject(w,grid,ckiodata,file,ckoffset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC, + cknersc_csum,ckscidac_csuma,ckscidac_csumb); + if ((cknersc_csum != nersc_csum) or (ckscidac_csuma != scidac_csuma) or (ckscidac_csumb != scidac_csumb)) + { + std::cout << GridLogMessage << "writeLatticeObject: read test checksum failure, re-writing (" << attemptsLeft << " attempt(s) remaining)" << std::endl; + offset = offsetCopy; + } + else + { + std::cout << GridLogMessage << "writeLatticeObject: read test checksum correct" << std::endl; + break; + } + } + attemptsLeft--; + } + std::cout<(); @@ -233,21 +237,52 @@ class GridLimeReader : public BinaryIO { // std::cout << " ReadLatticeObject from offset "< munge; BinaryIO::readLatticeObject< vobj, sobj >(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); - std::cout << GridLogMessage << "SciDAC checksum A " << std::hex << scidac_csuma << std::dec << std::endl; - std::cout << GridLogMessage << "SciDAC checksum B " << std::hex << scidac_csumb << std::dec << std::endl; + std::cout << GridLogMessage << "SciDAC checksum A " << std::hex << scidac_csuma << std::dec << std::endl; + std::cout << GridLogMessage << "SciDAC checksum B " << std::hex << scidac_csumb << std::dec << std::endl; ///////////////////////////////////////////// // Insist checksum is next record ///////////////////////////////////////////// - readLimeObject(scidacChecksum_,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM)); - + readScidacChecksum(scidacChecksum_,FieldNormMetaData_); ///////////////////////////////////////////// // Verify checksums ///////////////////////////////////////////// + if(FieldNormMetaData_.norm2 != 0.0){ + RealD n2ck = norm2(field); + // std::cout << GridLogMessage << "checking field norm: metadata "< xmlc(nbytes+1,'\0'); + limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR); + std::string xmlstring = std::string(&xmlc[0]); + XmlReader RD(xmlstring, true, ""); + if ( !strncmp(limeReaderType(LimeR), field_norm_str.c_str(),strlen(field_norm_str.c_str()) ) ) { + // std::cout << "FieldNormMetaData "<IsBoss() ); + FieldNormMetaData FNMD; FNMD.norm2 = norm2(field); + //////////////////////////////////////////// // Create record header //////////////////////////////////////////// @@ -448,6 +485,7 @@ class GridLimeWriter : public BinaryIO checksum.suma= streama.str(); checksum.sumb= streamb.str(); if ( boss_node ) { + writeLimeObject(0,0,FNMD,std::string(GRID_FIELD_NORM),std::string(GRID_FIELD_NORM)); writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM)); } } @@ -625,6 +663,12 @@ class IldgWriter : public ScidacWriter { assert(header.nd==4); assert(header.nd==header.dimension.size()); + ////////////////////////////////////////////////////////////////////////////// + // Field norm tests + ////////////////////////////////////////////////////////////////////////////// + FieldNormMetaData FieldNormMetaData_; + FieldNormMetaData_.norm2 = norm2(Umu); + ////////////////////////////////////////////////////////////////////////////// // Fill the USQCD info field ////////////////////////////////////////////////////////////////////////////// @@ -633,11 +677,12 @@ class IldgWriter : public ScidacWriter { info.plaq = header.plaquette; info.linktr = header.link_trace; - std::cout << GridLogMessage << " Writing config; IldgIO "< munge; @@ -846,6 +898,13 @@ class IldgReader : public GridLimeReader { //////////////////////////////////////////////////////////// // Really really want to mandate a scidac checksum //////////////////////////////////////////////////////////// + if ( found_FieldNormMetaData ) { + RealD nn = norm2(Umu); + GRID_FIELD_NORM_CHECK(FieldNormMetaData_,nn); + std::cout << GridLogMessage<<"FieldNormMetaData matches " << std::endl; + } else { + std::cout << GridLogWarning<<"FieldNormMetaData not found. " << std::endl; + } if ( found_scidacChecksum ) { FieldMetaData_.scidac_checksuma = stoull(scidacChecksum_.suma,0,16); FieldMetaData_.scidac_checksumb = stoull(scidacChecksum_.sumb,0,16); diff --git a/Grid/parallelIO/MetaData.h b/Grid/parallelIO/MetaData.h index 55254786..4cc93d03 100644 --- a/Grid/parallelIO/MetaData.h +++ b/Grid/parallelIO/MetaData.h @@ -56,6 +56,10 @@ namespace Grid { //////////////////////////////////////////////////////////////////////////////// // header specification/interpretation //////////////////////////////////////////////////////////////////////////////// + class FieldNormMetaData : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(FieldNormMetaData, double, norm2); + }; class FieldMetaData : Serializable { public: diff --git a/Grid/perfmon/Timer.h b/Grid/perfmon/Timer.h index 07c5febd..ce1b5d76 100644 --- a/Grid/perfmon/Timer.h +++ b/Grid/perfmon/Timer.h @@ -64,16 +64,20 @@ inline std::ostream& operator<< (std::ostream & stream, const GridMillisecs & no { GridSecs second(1); auto secs = now/second ; - auto subseconds = now%second ; + auto subseconds = now%second ; + auto fill = stream.fill(); stream << secs<<"."< + Author: Antonin Portelli + Author: James Harrison This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -30,8 +32,9 @@ namespace Grid{ namespace QCD{ + template - class QedGimpl + class QedGImpl { public: typedef S Simd; @@ -43,27 +46,27 @@ namespace QCD{ typedef iImplGaugeLink SiteLink; typedef iImplGaugeField SiteField; - typedef SiteField SiteComplex; + typedef SiteLink SiteComplex; typedef Lattice LinkField; typedef Lattice Field; typedef Field ComplexField; }; - typedef QedGimpl QedGimplR; + typedef QedGImpl QedGImplR; - template + template class Photon { public: - INHERIT_GIMPL_TYPES(Gimpl); + INHERIT_GIMPL_TYPES(GImpl); + typedef typename SiteGaugeLink::scalar_object ScalarSite; + typedef typename ScalarSite::scalar_type ScalarComplex; GRID_SERIALIZABLE_ENUM(Gauge, undef, feynman, 1, coulomb, 2, landau, 3); - GRID_SERIALIZABLE_ENUM(ZmScheme, undef, qedL, 1, qedTL, 2, qedInf, 3); + GRID_SERIALIZABLE_ENUM(ZmScheme, undef, qedL, 1, qedTL, 2); public: - Photon(Gauge gauge, ZmScheme zmScheme); - Photon(Gauge gauge, ZmScheme zmScheme, std::vector improvements); - Photon(Gauge gauge, ZmScheme zmScheme, Real G0); - Photon(Gauge gauge, ZmScheme zmScheme, std::vector improvements, Real G0); + Photon(GridBase *grid, Gauge gauge, ZmScheme zmScheme, std::vector improvement); + Photon(GridBase *grid, Gauge gauge, ZmScheme zmScheme); virtual ~Photon(void) = default; void FreePropagator(const GaugeField &in, GaugeField &out); void MomentumSpacePropagator(const GaugeField &in, GaugeField &out); @@ -73,345 +76,255 @@ namespace QCD{ const GaugeLinkField &weight); void UnitField(GaugeField &out); private: - void infVolPropagator(GaugeLinkField &out); - void invKHatSquared(GaugeLinkField &out); + void makeSpatialNorm(LatticeInteger &spNrm); + void makeKHat(std::vector &khat); + void makeInvKHatSquared(GaugeLinkField &out); void zmSub(GaugeLinkField &out); + void transverseProjectSpatial(GaugeField &out); + void gaugeTransform(GaugeField &out); private: - Gauge gauge_; - ZmScheme zmScheme_; - std::vector improvement_; - Real G0_; + GridBase *grid_; + Gauge gauge_; + ZmScheme zmScheme_; + std::vector improvement_; }; - typedef Photon PhotonR; + typedef Photon PhotonR; - template - Photon::Photon(Gauge gauge, ZmScheme zmScheme) - : gauge_(gauge), zmScheme_(zmScheme), improvement_(std::vector()), - G0_(0.15493339023106021408483720810737508876916113364521) - {} - - template - Photon::Photon(Gauge gauge, ZmScheme zmScheme, + template + Photon::Photon(GridBase *grid, Gauge gauge, ZmScheme zmScheme, std::vector improvements) - : gauge_(gauge), zmScheme_(zmScheme), improvement_(improvements), - G0_(0.15493339023106021408483720810737508876916113364521) + : grid_(grid), gauge_(gauge), zmScheme_(zmScheme), improvement_(improvements) {} - template - Photon::Photon(Gauge gauge, ZmScheme zmScheme, Real G0) - : gauge_(gauge), zmScheme_(zmScheme), improvement_(std::vector()), G0_(G0) + template + Photon::Photon(GridBase *grid, Gauge gauge, ZmScheme zmScheme) + : Photon(grid, gauge, zmScheme, std::vector()) {} - template - Photon::Photon(Gauge gauge, ZmScheme zmScheme, - std::vector improvements, Real G0) - : gauge_(gauge), zmScheme_(zmScheme), improvement_(improvements), G0_(G0) - {} - - template - void Photon::FreePropagator (const GaugeField &in,GaugeField &out) + template + void Photon::FreePropagator(const GaugeField &in, GaugeField &out) { - FFT theFFT(in._grid); + FFT theFFT(dynamic_cast(grid_)); + GaugeField in_k(grid_); + GaugeField prop_k(grid_); - GaugeField in_k(in._grid); - GaugeField prop_k(in._grid); - - theFFT.FFT_all_dim(in_k,in,FFT::forward); - MomentumSpacePropagator(prop_k,in_k); - theFFT.FFT_all_dim(out,prop_k,FFT::backward); + theFFT.FFT_all_dim(in_k, in, FFT::forward); + MomentumSpacePropagator(prop_k, in_k); + theFFT.FFT_all_dim(out, prop_k, FFT::backward); } - template - void Photon::infVolPropagator(GaugeLinkField &out) + template + void Photon::makeSpatialNorm(LatticeInteger &spNrm) { - auto *grid = dynamic_cast(out._grid); - LatticeReal xmu(grid); - GaugeLinkField one(grid); - const unsigned int nd = grid->_ndimension; - std::vector &l = grid->_fdimensions; - std::vector x0(nd,0); - TComplex Tone = Complex(1.0,0.0); - TComplex Tzero = Complex(G0_,0.0); - FFT fft(grid); + LatticeInteger coor(grid_); + std::vector l = grid_->FullDimensions(); + + spNrm = zero; + for(int mu = 0; mu < grid_->Nd() - 1; mu++) + { + LatticeCoordinate(coor, mu); + coor = where(coor < Integer(l[mu]/2), coor, coor - Integer(l[mu])); + spNrm = spNrm + coor*coor; + } + } + + template + void Photon::makeKHat(std::vector &khat) + { + const unsigned int nd = grid_->Nd(); + std::vector l = grid_->FullDimensions(); + Complex ci(0., 1.); + + khat.resize(nd, grid_); + for (unsigned int mu = 0; mu < nd; ++mu) + { + Real piL = M_PI/l[mu]; + + LatticeCoordinate(khat[mu], mu); + khat[mu] = exp(piL*ci*khat[mu])*2.*sin(piL*khat[mu]); + } + } + + template + void Photon::makeInvKHatSquared(GaugeLinkField &out) + { + std::vector khat; + GaugeLinkField lone(grid_); + const unsigned int nd = grid_->Nd(); + std::vector zm(nd, 0); + ScalarSite one = ScalarComplex(1., 0.), z = ScalarComplex(0., 0.); - one = Complex(1.0,0.0); out = zero; + makeKHat(khat); for(int mu = 0; mu < nd; mu++) { - LatticeCoordinate(xmu,mu); - Real lo2 = l[mu]/2.0; - xmu = where(xmu < lo2, xmu, xmu-double(l[mu])); - out = out + toComplex(4*M_PI*M_PI*xmu*xmu); + out = out + khat[mu]*conjugate(khat[mu]); } - pokeSite(Tone, out, x0); - out = one/out; - pokeSite(Tzero, out, x0); - fft.FFT_all_dim(out, out, FFT::forward); + lone = ScalarComplex(1., 0.); + pokeSite(one, out, zm); + out = lone/out; + pokeSite(z, out, zm); } - template - void Photon::invKHatSquared(GaugeLinkField &out) + template + void Photon::zmSub(GaugeLinkField &out) { - GridBase *grid = out._grid; - GaugeLinkField kmu(grid), one(grid); - const unsigned int nd = grid->_ndimension; - std::vector &l = grid->_fdimensions; - std::vector zm(nd,0); - TComplex Tone = Complex(1.0,0.0); - TComplex Tzero= Complex(0.0,0.0); - - one = Complex(1.0,0.0); - out = zero; - for(int mu = 0; mu < nd; mu++) - { - Real twoPiL = M_PI*2./l[mu]; - - LatticeCoordinate(kmu,mu); - kmu = 2.*sin(.5*twoPiL*kmu); - out = out + kmu*kmu; - } - pokeSite(Tone, out, zm); - out = one/out; - pokeSite(Tzero, out, zm); - } - - template - void Photon::zmSub(GaugeLinkField &out) - { - GridBase *grid = out._grid; - const unsigned int nd = grid->_ndimension; - std::vector &l = grid->_fdimensions; - switch (zmScheme_) { case ZmScheme::qedTL: { - std::vector zm(nd,0); - TComplex Tzero = Complex(0.0,0.0); - - pokeSite(Tzero, out, zm); + std::vector zm(grid_->Nd(), 0); + ScalarSite z = ScalarComplex(0., 0.); + pokeSite(z, out, zm); break; } case ZmScheme::qedL: { - LatticeInteger spNrm(grid), coor(grid); - GaugeLinkField z(grid); - - spNrm = zero; - for(int d = 0; d < grid->_ndimension - 1; d++) - { - LatticeCoordinate(coor,d); - coor = where(coor < Integer(l[d]/2), coor, coor-Integer(l[d])); - spNrm = spNrm + coor*coor; - } - out = where(spNrm == Integer(0), 0.*out, out); + LatticeInteger spNrm(grid_); - // IR improvement + makeSpatialNorm(spNrm); + out = where(spNrm == Integer(0), 0.*out, out); for(int i = 0; i < improvement_.size(); i++) { - Real f = sqrt(improvement_[i]+1); - out = where(spNrm == Integer(i+1), f*out, out); + Real f = sqrt(improvement_[i] + 1); + out = where(spNrm == Integer(i + 1), f*out, out); } + break; } default: + assert(0); break; } } - template - void Photon::MomentumSpacePropagator(const GaugeField &in, - GaugeField &out) + template + void Photon::transverseProjectSpatial(GaugeField &out) { - GridBase *grid = out._grid; - LatticeComplex momProp(grid); - - switch (zmScheme_) + const unsigned int nd = grid_->Nd(); + GaugeLinkField invKHat(grid_), cst(grid_), spdiv(grid_); + LatticeInteger spNrm(grid_); + std::vector khat, a(nd, grid_), aProj(nd, grid_); + + invKHat = zero; + makeSpatialNorm(spNrm); + makeKHat(khat); + for (unsigned int mu = 0; mu < nd; ++mu) { - case ZmScheme::qedTL: - case ZmScheme::qedL: + a[mu] = peekLorentz(out, mu); + if (mu < nd - 1) { - invKHatSquared(momProp); - zmSub(momProp); - break; + invKHat += khat[mu]*conjugate(khat[mu]); } - case ZmScheme::qedInf: - { - infVolPropagator(momProp); + } + cst = ScalarComplex(1., 0.); + invKHat = where(spNrm == Integer(0), cst, invKHat); + invKHat = cst/invKHat; + cst = zero; + invKHat = where(spNrm == Integer(0), cst, invKHat); + spdiv = zero; + for (unsigned int nu = 0; nu < nd - 1; ++nu) + { + spdiv += conjugate(khat[nu])*a[nu]; + } + spdiv *= invKHat; + for (unsigned int mu = 0; mu < nd; ++mu) + { + aProj[mu] = a[mu] - khat[mu]*spdiv; + pokeLorentz(out, aProj[mu], mu); + } + } + + template + void Photon::gaugeTransform(GaugeField &out) + { + switch (gauge_) + { + case Gauge::feynman: + break; + case Gauge::coulomb: + transverseProjectSpatial(out); + break; + case Gauge::landau: + assert(0); break; - } default: + assert(0); break; } + } + + template + void Photon::MomentumSpacePropagator(const GaugeField &in, + GaugeField &out) + { + LatticeComplex momProp(grid_); + + makeInvKHatSquared(momProp); + zmSub(momProp); out = in*momProp; } - template - void Photon::StochasticWeight(GaugeLinkField &weight) + template + void Photon::StochasticWeight(GaugeLinkField &weight) { - auto *grid = dynamic_cast(weight._grid); - const unsigned int nd = grid->_ndimension; - std::vector latt_size = grid->_fdimensions; - - switch (zmScheme_) + const unsigned int nd = grid_->Nd(); + std::vector l = grid_->FullDimensions(); + Integer vol = 1; + + for(unsigned int mu = 0; mu < nd; mu++) { - case ZmScheme::qedTL: - case ZmScheme::qedL: - { - Integer vol = 1; - for(int d = 0; d < nd; d++) - { - vol = vol * latt_size[d]; - } - invKHatSquared(weight); - weight = sqrt(vol)*sqrt(weight); - zmSub(weight); - break; - } - case ZmScheme::qedInf: - { - infVolPropagator(weight); - weight = sqrt(real(weight)); - break; - } - default: - break; + vol = vol*l[mu]; } + makeInvKHatSquared(weight); + weight = sqrt(vol)*sqrt(weight); + zmSub(weight); } - template - void Photon::StochasticField(GaugeField &out, GridParallelRNG &rng) + template + void Photon::StochasticField(GaugeField &out, GridParallelRNG &rng) { - auto *grid = dynamic_cast(out._grid); - GaugeLinkField weight(grid); + GaugeLinkField weight(grid_); StochasticWeight(weight); StochasticField(out, rng, weight); } - template - void Photon::StochasticField(GaugeField &out, GridParallelRNG &rng, + template + void Photon::StochasticField(GaugeField &out, GridParallelRNG &rng, const GaugeLinkField &weight) { - auto *grid = dynamic_cast(out._grid); - const unsigned int nd = grid->_ndimension; - GaugeLinkField r(grid); - GaugeField aTilde(grid); - FFT fft(grid); + const unsigned int nd = grid_->Nd(); + GaugeLinkField r(grid_); + GaugeField aTilde(grid_); + FFT fft(dynamic_cast(grid_)); - switch (zmScheme_) + for(unsigned int mu = 0; mu < nd; mu++) { - case ZmScheme::qedTL: - case ZmScheme::qedL: - { - for(int mu = 0; mu < nd; mu++) - { - gaussian(rng, r); - r = weight*r; - pokeLorentz(aTilde, r, mu); - } - break; - } - case ZmScheme::qedInf: - { - Complex shift(1., 1.); // This needs to be a GaugeLink element? - for(int mu = 0; mu < nd; mu++) - { - bernoulli(rng, r); - r = weight*(2.*r - shift); - pokeLorentz(aTilde, r, mu); - } - break; - } - default: - break; + gaussian(rng, r); + r = weight*r; + pokeLorentz(aTilde, r, mu); } - + gaugeTransform(aTilde); fft.FFT_all_dim(out, aTilde, FFT::backward); - out = real(out); } - template - void Photon::UnitField(GaugeField &out) + template + void Photon::UnitField(GaugeField &out) { - auto *grid = dynamic_cast(out._grid); - const unsigned int nd = grid->_ndimension; - GaugeLinkField r(grid); + const unsigned int nd = grid_->Nd(); + GaugeLinkField r(grid_); - r = Complex(1.0,0.0); - - for(int mu = 0; mu < nd; mu++) + r = ScalarComplex(1., 0.); + for(unsigned int mu = 0; mu < nd; mu++) { pokeLorentz(out, r, mu); } - out = real(out); } -// template -// void Photon::FeynmanGaugeMomentumSpacePropagator_L(GaugeField &out, -// const GaugeField &in) -// { -// -// FeynmanGaugeMomentumSpacePropagator_TL(out,in); -// -// GridBase *grid = out._grid; -// LatticeInteger coor(grid); -// GaugeField zz(grid); zz=zero; -// -// // xyzt -// for(int d = 0; d < grid->_ndimension-1;d++){ -// LatticeCoordinate(coor,d); -// out = where(coor==Integer(0),zz,out); -// } -// } -// -// template -// void Photon::FeynmanGaugeMomentumSpacePropagator_TL(GaugeField &out, -// const GaugeField &in) -// { -// -// // what type LatticeComplex -// GridBase *grid = out._grid; -// int nd = grid->_ndimension; -// -// typedef typename GaugeField::vector_type vector_type; -// typedef typename GaugeField::scalar_type ScalComplex; -// typedef Lattice > LatComplex; -// -// std::vector latt_size = grid->_fdimensions; -// -// LatComplex denom(grid); denom= zero; -// LatComplex one(grid); one = ScalComplex(1.0,0.0); -// LatComplex kmu(grid); -// -// ScalComplex ci(0.0,1.0); -// // momphase = n * 2pi / L -// for(int mu=0;mu zero_mode(nd,0); -// TComplexD Tone = ComplexD(1.0,0.0); -// TComplexD Tzero= ComplexD(0.0,0.0); -// -// pokeSite(Tone,denom,zero_mode); -// -// denom= one/denom; -// -// pokeSite(Tzero,denom,zero_mode); -// -// out = zero; -// out = in*denom; -// }; }} #endif diff --git a/Grid/qcd/utils/LinalgUtils.h b/Grid/qcd/utils/LinalgUtils.h index 5eaf1c2a..04a224e5 100644 --- a/Grid/qcd/utils/LinalgUtils.h +++ b/Grid/qcd/utils/LinalgUtils.h @@ -173,6 +173,39 @@ void G5R5(Lattice &z,const Lattice &x) } } } +} -}} +// I explicitly need these outside the QCD namespace +template +void G5C(Lattice &z, const Lattice &x) +{ + GridBase *grid = x._grid; + z.checkerboard = x.checkerboard; + conformable(x, z); + + QCD::Gamma G5(QCD::Gamma::Algebra::Gamma5); + z = G5 * x; +} + +template +void G5C(Lattice> &z, const Lattice> &x) +{ + GridBase *grid = x._grid; + z.checkerboard = x.checkerboard; + conformable(x, z); + + static_assert(nbasis % 2 == 0, ""); + int nb = nbasis / 2; + + parallel_for(int ss = 0; ss < grid->oSites(); ss++) { + for(int n = 0; n < nb; ++n) { + z._odata[ss](n) = x._odata[ss](n); + } + for(int n = nb; n < nbasis; ++n) { + z._odata[ss](n) = -x._odata[ss](n); + } + } +} + +} #endif diff --git a/Grid/qcd/utils/WilsonLoops.h b/Grid/qcd/utils/WilsonLoops.h index 6cf34e0c..d4790df2 100644 --- a/Grid/qcd/utils/WilsonLoops.h +++ b/Grid/qcd/utils/WilsonLoops.h @@ -6,10 +6,12 @@ Copyright (C) 2015 -Author: Azusa Yamaguchi -Author: Peter Boyle -Author: neo -Author: paboyle + Author: Azusa Yamaguchi + Author: Peter Boyle + Author: neo + Author: paboyle + Author: James Harrison + Author: Antonin Portelli This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -645,6 +647,184 @@ static void StapleMult(GaugeMat &staple, const GaugeLorentz &Umu, int mu) { } } } + + ////////////////////////////////////////////////// + // Wilson loop of size (R1, R2), oriented in mu,nu plane + ////////////////////////////////////////////////// + static void wilsonLoop(GaugeMat &wl, const std::vector &U, + const int Rmu, const int Rnu, + const int mu, const int nu) { + wl = U[nu]; + + for(int i = 0; i < Rnu-1; i++){ + wl = Gimpl::CovShiftForward(U[nu], nu, wl); + } + + for(int i = 0; i < Rmu; i++){ + wl = Gimpl::CovShiftForward(U[mu], mu, wl); + } + + for(int i = 0; i < Rnu; i++){ + wl = Gimpl::CovShiftBackward(U[nu], nu, wl); + } + + for(int i = 0; i < Rmu; i++){ + wl = Gimpl::CovShiftBackward(U[mu], mu, wl); + } + } + ////////////////////////////////////////////////// + // trace of Wilson Loop oriented in mu,nu plane + ////////////////////////////////////////////////// + static void traceWilsonLoop(LatticeComplex &wl, + const std::vector &U, + const int Rmu, const int Rnu, + const int mu, const int nu) { + GaugeMat sp(U[0]._grid); + wilsonLoop(sp, U, Rmu, Rnu, mu, nu); + wl = trace(sp); + } + ////////////////////////////////////////////////// + // sum over all planes of Wilson loop + ////////////////////////////////////////////////// + static void siteWilsonLoop(LatticeComplex &Wl, + const std::vector &U, + const int R1, const int R2) { + LatticeComplex siteWl(U[0]._grid); + Wl = zero; + for (int mu = 1; mu < U[0]._grid->_ndimension; mu++) { + for (int nu = 0; nu < mu; nu++) { + traceWilsonLoop(siteWl, U, R1, R2, mu, nu); + Wl = Wl + siteWl; + traceWilsonLoop(siteWl, U, R2, R1, mu, nu); + Wl = Wl + siteWl; + } + } + } + ////////////////////////////////////////////////// + // sum over planes of Wilson loop with length R1 + // in the time direction + ////////////////////////////////////////////////// + static void siteTimelikeWilsonLoop(LatticeComplex &Wl, + const std::vector &U, + const int R1, const int R2) { + LatticeComplex siteWl(U[0]._grid); + + int ndim = U[0]._grid->_ndimension; + + Wl = zero; + for (int nu = 0; nu < ndim - 1; nu++) { + traceWilsonLoop(siteWl, U, R1, R2, ndim-1, nu); + Wl = Wl + siteWl; + } + } + ////////////////////////////////////////////////// + // sum Wilson loop over all planes orthogonal to the time direction + ////////////////////////////////////////////////// + static void siteSpatialWilsonLoop(LatticeComplex &Wl, + const std::vector &U, + const int R1, const int R2) { + LatticeComplex siteWl(U[0]._grid); + + Wl = zero; + for (int mu = 1; mu < U[0]._grid->_ndimension - 1; mu++) { + for (int nu = 0; nu < mu; nu++) { + traceWilsonLoop(siteWl, U, R1, R2, mu, nu); + Wl = Wl + siteWl; + traceWilsonLoop(siteWl, U, R2, R1, mu, nu); + Wl = Wl + siteWl; + } + } + } + ////////////////////////////////////////////////// + // sum over all x,y,z,t and over all planes of Wilson loop + ////////////////////////////////////////////////// + static Real sumWilsonLoop(const GaugeLorentz &Umu, + const int R1, const int R2) { + std::vector U(4, Umu._grid); + + for (int mu = 0; mu < Umu._grid->_ndimension; mu++) { + U[mu] = PeekIndex(Umu, mu); + } + + LatticeComplex Wl(Umu._grid); + + siteWilsonLoop(Wl, U, R1, R2); + + TComplex Tp = sum(Wl); + Complex p = TensorRemove(Tp); + return p.real(); + } + ////////////////////////////////////////////////// + // sum over all x,y,z,t and over all planes of timelike Wilson loop + ////////////////////////////////////////////////// + static Real sumTimelikeWilsonLoop(const GaugeLorentz &Umu, + const int R1, const int R2) { + std::vector U(4, Umu._grid); + + for (int mu = 0; mu < Umu._grid->_ndimension; mu++) { + U[mu] = PeekIndex(Umu, mu); + } + + LatticeComplex Wl(Umu._grid); + + siteTimelikeWilsonLoop(Wl, U, R1, R2); + + TComplex Tp = sum(Wl); + Complex p = TensorRemove(Tp); + return p.real(); + } + ////////////////////////////////////////////////// + // sum over all x,y,z,t and over all planes of spatial Wilson loop + ////////////////////////////////////////////////// + static Real sumSpatialWilsonLoop(const GaugeLorentz &Umu, + const int R1, const int R2) { + std::vector U(4, Umu._grid); + + for (int mu = 0; mu < Umu._grid->_ndimension; mu++) { + U[mu] = PeekIndex(Umu, mu); + } + + LatticeComplex Wl(Umu._grid); + + siteSpatialWilsonLoop(Wl, U, R1, R2); + + TComplex Tp = sum(Wl); + Complex p = TensorRemove(Tp); + return p.real(); + } + ////////////////////////////////////////////////// + // average over all x,y,z,t and over all planes of Wilson loop + ////////////////////////////////////////////////// + static Real avgWilsonLoop(const GaugeLorentz &Umu, + const int R1, const int R2) { + int ndim = Umu._grid->_ndimension; + Real sumWl = sumWilsonLoop(Umu, R1, R2); + Real vol = Umu._grid->gSites(); + Real faces = 1.0 * ndim * (ndim - 1); + return sumWl / vol / faces / Nc; // Nc dependent... FIXME + } + ////////////////////////////////////////////////// + // average over all x,y,z,t and over all planes of timelike Wilson loop + ////////////////////////////////////////////////// + static Real avgTimelikeWilsonLoop(const GaugeLorentz &Umu, + const int R1, const int R2) { + int ndim = Umu._grid->_ndimension; + Real sumWl = sumTimelikeWilsonLoop(Umu, R1, R2); + Real vol = Umu._grid->gSites(); + Real faces = 1.0 * (ndim - 1); + return sumWl / vol / faces / Nc; // Nc dependent... FIXME + } + ////////////////////////////////////////////////// + // average over all x,y,z,t and over all planes of spatial Wilson loop + ////////////////////////////////////////////////// + static Real avgSpatialWilsonLoop(const GaugeLorentz &Umu, + const int R1, const int R2) { + int ndim = Umu._grid->_ndimension; + Real sumWl = sumSpatialWilsonLoop(Umu, R1, R2); + Real vol = Umu._grid->gSites(); + Real faces = 1.0 * (ndim - 1) * (ndim - 2); + return sumWl / vol / faces / Nc; // Nc dependent... FIXME + } }; typedef WilsonLoops ColourWilsonLoops; diff --git a/Grid/serialisation/Hdf5IO.h b/Grid/serialisation/Hdf5IO.h index ec26612a..59804240 100644 --- a/Grid/serialisation/Hdf5IO.h +++ b/Grid/serialisation/Hdf5IO.h @@ -123,9 +123,12 @@ namespace Grid if (flatx.size() > dataSetThres_) { - H5NS::DataSet dataSet; + H5NS::DataSet dataSet; + H5NS::DSetCreatPropList plist; - dataSet = group_.createDataSet(s, Hdf5Type::type(), dataSpace); + plist.setChunk(dim.size(), dim.data()); + plist.setFletcher32(); + dataSet = group_.createDataSet(s, Hdf5Type::type(), dataSpace, plist); dataSet.write(flatx.data(), Hdf5Type::type()); } else diff --git a/Hadrons/A2AMatrix.hpp b/Hadrons/A2AMatrix.hpp index e224a95e..ed2f5d36 100644 --- a/Hadrons/A2AMatrix.hpp +++ b/Hadrons/A2AMatrix.hpp @@ -442,6 +442,7 @@ void A2AMatrixIo::initFile(const MetadataType &d, const unsigned int chunkSiz push(reader, dataname_); auto &group = reader.getGroup(); plist.setChunk(chunk.size(), chunk.data()); + plist.setFletcher32(); dataset = group.createDataSet(HADRONS_A2AM_NAME, Hdf5Type::type(), dataspace, plist); #else HADRONS_ERROR(Implementation, "all-to-all matrix I/O needs HDF5 library"); @@ -502,14 +503,12 @@ void A2AMatrixIo::load(Vec &v, double *tRead) H5NS::DataSet dataset; H5NS::DataSpace dataspace; H5NS::CompType datatype; - H5NS::DSetCreatPropList plist; push(reader, dataname_); auto &group = reader.getGroup(); dataset = group.openDataSet(HADRONS_A2AM_NAME); datatype = dataset.getCompType(); dataspace = dataset.getSpace(); - plist = dataset.getCreatePlist(); hdim.resize(dataspace.getSimpleExtentNdims()); dataspace.getSimpleExtentDims(hdim.data()); if ((nt_*ni_*nj_ != 0) and diff --git a/Hadrons/Application.cc b/Hadrons/Application.cc index 44579646..d04c9a50 100644 --- a/Hadrons/Application.cc +++ b/Hadrons/Application.cc @@ -108,6 +108,9 @@ void Application::run(void) HADRONS_ERROR(Definition, "run id is empty"); } LOG(Message) << "RUN ID '" << getPar().runId << "'" << std::endl; + BinaryIO::latticeWriteMaxRetry = getPar().parallelWriteMaxRetry; + LOG(Message) << "Attempt(s) for resilient parallel I/O: " + << BinaryIO::latticeWriteMaxRetry << std::endl; vm().setRunId(getPar().runId); vm().printContent(); env().printContent(); diff --git a/Hadrons/Application.hpp b/Hadrons/Application.hpp index 432fe757..3578c919 100644 --- a/Hadrons/Application.hpp +++ b/Hadrons/Application.hpp @@ -56,7 +56,9 @@ public: TrajRange, trajCounter, VirtualMachine::GeneticPar, genetic, std::string, runId, - std::string, graphFile); + std::string, graphFile, + int, parallelWriteMaxRetry); + GlobalPar(void): parallelWriteMaxRetry{-1} {} }; public: // constructors diff --git a/Hadrons/Modules/MGauge/StochEm.cc b/Hadrons/Modules/MGauge/StochEm.cc index 6f8bf55e..574387e4 100644 --- a/Hadrons/Modules/MGauge/StochEm.cc +++ b/Hadrons/Modules/MGauge/StochEm.cc @@ -70,7 +70,7 @@ void TStochEm::execute(void) LOG(Message) << "Generating stochastic EM potential..." << std::endl; std::vector improvements = strToVec(par().improvement); - PhotonR photon(par().gauge, par().zmScheme, improvements, par().G0_qedInf); + PhotonR photon(envGetGrid(EmField), par().gauge, par().zmScheme, improvements); auto &a = envGet(EmField, getName()); auto &w = envGet(EmComp, "_" + getName() + "_weight"); diff --git a/Hadrons/Modules/MGauge/StochEm.hpp b/Hadrons/Modules/MGauge/StochEm.hpp index a3f8cc96..b549387b 100644 --- a/Hadrons/Modules/MGauge/StochEm.hpp +++ b/Hadrons/Modules/MGauge/StochEm.hpp @@ -47,8 +47,7 @@ public: GRID_SERIALIZABLE_CLASS_MEMBERS(StochEmPar, PhotonR::Gauge, gauge, PhotonR::ZmScheme, zmScheme, - std::string, improvement, - Real, G0_qedInf); + std::string, improvement); }; class TStochEm: public Module diff --git a/Hadrons/Modules/MGauge/UnitEm.cc b/Hadrons/Modules/MGauge/UnitEm.cc index d2ecad5e..97da8224 100644 --- a/Hadrons/Modules/MGauge/UnitEm.cc +++ b/Hadrons/Modules/MGauge/UnitEm.cc @@ -62,7 +62,7 @@ void TUnitEm::setup(void) // execution /////////////////////////////////////////////////////////////////// void TUnitEm::execute(void) { - PhotonR photon(0, 0); // Just chose arbitrary input values here + PhotonR photon(envGetGrid(EmField), 0, 0); // Just chose arbitrary input values here auto &a = envGet(EmField, getName()); LOG(Message) << "Generating unit EM potential..." << std::endl; photon.UnitField(a); diff --git a/tests/IO/Test_ildg_io.cc b/tests/IO/Test_ildg_io.cc index 55dd93b8..cb5efed2 100644 --- a/tests/IO/Test_ildg_io.cc +++ b/tests/IO/Test_ildg_io.cc @@ -53,7 +53,6 @@ int main (int argc, char ** argv) GridCartesian Fine(latt_size,simd_layout,mpi_layout); GridCartesian Coarse(clatt_size,simd_layout,mpi_layout); - GridParallelRNG pRNGa(&Fine); GridParallelRNG pRNGb(&Fine); GridSerialRNG sRNGa; @@ -94,6 +93,27 @@ int main (int argc, char ** argv) _IldgReader.close(); Umu_diff = Umu - Umu_saved; + std::cout < + +using namespace Grid; +using namespace QCD; + +typedef PeriodicGaugeImpl QedPeriodicGImplR; +typedef PhotonR::GaugeField EmField; +typedef PhotonR::GaugeLinkField EmComp; + +const int NCONFIGS = 20; +const int NWILSON = 10; + +int main(int argc, char *argv[]) +{ + // initialization + Grid_init(&argc, &argv); + std::cout << GridLogMessage << "Grid initialized" << std::endl; + + // QED stuff + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(4, vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian grid(latt_size,simd_layout,mpi_layout); + GridParallelRNG pRNG(&grid); + PhotonR photon(&grid, PhotonR::Gauge::coulomb, PhotonR::ZmScheme::qedL); + EmField a(&grid); + EmField expA(&grid); + + Complex imag_unit(0, 1); + + Real wlA; + std::vector logWlAvg(NWILSON, 0.0), logWlTime(NWILSON, 0.0), logWlSpace(NWILSON, 0.0); + + pRNG.SeedFixedIntegers({1, 2, 3, 4}); + + std::cout << GridLogMessage << "Wilson loop calculation beginning" << std::endl; + for(int ic = 0; ic < NCONFIGS; ic++){ + std::cout << GridLogMessage << "Configuration " << ic < zm; + + std::cout << GridLogMessage << "Total zero-mode norm 2 " + << std::sqrt(norm2(sum(a))) << std::endl; + + std::cout << GridLogMessage << "Spatial zero-mode norm 2" << std::endl; + sliceSum(a, zm, grid.Nd() - 1); + for (unsigned int t = 0; t < latt_size.back(); ++t) + { + std::cout << GridLogMessage << "t = " << t << " " << std::sqrt(norm2(zm[t])) << std::endl; + } + + // Calculate divergence + EmComp diva(&grid), amu(&grid); + + diva = zero; + for (unsigned int mu = 0; mu < grid.Nd(); ++mu) + { + amu = peekLorentz(a, mu); + diva += amu - Cshift(amu, mu, -1); + if (mu == grid.Nd() - 2) + { + std::cout << GridLogMessage << "Spatial divergence norm 2 " << std::sqrt(norm2(diva)) << std::endl; + } + } + std::cout << GridLogMessage << "Total divergence norm 2 " << std::sqrt(norm2(diva)) << std::endl; + + // Calculate Wilson loops + for(int iw=1; iw<=NWILSON; iw++){ + wlA = WilsonLoops::avgWilsonLoop(expA, iw, iw) * 3; + logWlAvg[iw-1] -= 2*log(wlA); + wlA = WilsonLoops::avgTimelikeWilsonLoop(expA, iw, iw) * 3; + logWlTime[iw-1] -= 2*log(wlA); + wlA = WilsonLoops::avgSpatialWilsonLoop(expA, iw, iw) * 3; + logWlSpace[iw-1] -= 2*log(wlA); + } + } + std::cout << GridLogMessage << "Wilson loop calculation completed" << std::endl; + + // Calculate Wilson loops + // From A. Portelli's PhD thesis: + // size -2*log(W) + // 1 0.500000000(1) + // 2 1.369311535(1) + // 3 2.305193057(1) + // 4 3.261483854(1) + // 5 4.228829967(1) + // 6 5.203604529(1) + // 7 6.183728249(1) + // 8 7.167859805(1) + // 9 8.155091868(1) + // 10 9.144788116(1) + + for(int iw=1; iw<=10; iw++){ + std::cout << GridLogMessage << iw << 'x' << iw << " Wilson loop" << std::endl; + std::cout << GridLogMessage << "-2*log(W) average: " << logWlAvg[iw-1]/NCONFIGS << std::endl; + std::cout << GridLogMessage << "-2*log(W) timelike: " << logWlTime[iw-1]/NCONFIGS << std::endl; + std::cout << GridLogMessage << "-2*log(W) spatial: " << logWlSpace[iw-1]/NCONFIGS << std::endl; + } + + // epilogue + std::cout << GridLogMessage << "Grid is finalizing now" << std::endl; + Grid_finalize(); + + return EXIT_SUCCESS; +} diff --git a/tests/solver/Test_multigrid_common.h b/tests/solver/Test_multigrid_common.h new file mode 100644 index 00000000..add833f2 --- /dev/null +++ b/tests/solver/Test_multigrid_common.h @@ -0,0 +1,670 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/solver/Test_multigrid_common.h + + Copyright (C) 2015-2018 + + Author: Daniel Richtmann + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ +/* END LEGAL */ +#ifndef GRID_TEST_MULTIGRID_COMMON_H +#define GRID_TEST_MULTIGRID_COMMON_H + +namespace Grid { + +// TODO: Can think about having one parameter struct per level and then a +// vector of these structs. How well would that work together with the +// serialization strategy of Grid? + +// clang-format off +struct MultiGridParams : Serializable { +public: + GRID_SERIALIZABLE_CLASS_MEMBERS(MultiGridParams, + int, nLevels, + std::vector>, blockSizes, // size == nLevels - 1 + std::vector, smootherTol, // size == nLevels - 1 + std::vector, smootherMaxOuterIter, // size == nLevels - 1 + std::vector, smootherMaxInnerIter, // size == nLevels - 1 + bool, kCycle, + std::vector, kCycleTol, // size == nLevels - 1 + std::vector, kCycleMaxOuterIter, // size == nLevels - 1 + std::vector, kCycleMaxInnerIter, // size == nLevels - 1 + double, coarseSolverTol, + int, coarseSolverMaxOuterIter, + int, coarseSolverMaxInnerIter); + + // constructor with default values + MultiGridParams(int _nLevels = 2, + std::vector> _blockSizes = {{4, 4, 4, 4}}, + std::vector _smootherTol = {1e-14}, + std::vector _smootherMaxOuterIter = {4}, + std::vector _smootherMaxInnerIter = {4}, + bool _kCycle = true, + std::vector _kCycleTol = {1e-1}, + std::vector _kCycleMaxOuterIter = {2}, + std::vector _kCycleMaxInnerIter = {5}, + double _coarseSolverTol = 5e-2, + int _coarseSolverMaxOuterIter = 10, + int _coarseSolverMaxInnerIter = 500) + : nLevels(_nLevels) + , blockSizes(_blockSizes) + , smootherTol(_smootherTol) + , smootherMaxOuterIter(_smootherMaxOuterIter) + , smootherMaxInnerIter(_smootherMaxInnerIter) + , kCycle(_kCycle) + , kCycleTol(_kCycleTol) + , kCycleMaxOuterIter(_kCycleMaxOuterIter) + , kCycleMaxInnerIter(_kCycleMaxInnerIter) + , coarseSolverTol(_coarseSolverTol) + , coarseSolverMaxOuterIter(_coarseSolverMaxOuterIter) + , coarseSolverMaxInnerIter(_coarseSolverMaxInnerIter) + {} +}; +// clang-format on + +void checkParameterValidity(MultiGridParams const ¶ms) { + + auto correctSize = params.nLevels - 1; + + assert(correctSize == params.blockSizes.size()); + assert(correctSize == params.smootherTol.size()); + assert(correctSize == params.smootherMaxOuterIter.size()); + assert(correctSize == params.smootherMaxInnerIter.size()); + assert(correctSize == params.kCycleTol.size()); + assert(correctSize == params.kCycleMaxOuterIter.size()); + assert(correctSize == params.kCycleMaxInnerIter.size()); +} + +struct LevelInfo { +public: + std::vector> Seeds; + std::vector Grids; + std::vector PRNGs; + + LevelInfo(GridCartesian *FineGrid, MultiGridParams const &mgParams) { + + auto nCoarseLevels = mgParams.blockSizes.size(); + + assert(nCoarseLevels == mgParams.nLevels - 1); + + // set up values for finest grid + Grids.push_back(FineGrid); + Seeds.push_back({1, 2, 3, 4}); + PRNGs.push_back(GridParallelRNG(Grids.back())); + PRNGs.back().SeedFixedIntegers(Seeds.back()); + + // set up values for coarser grids + for(int level = 1; level < mgParams.nLevels; ++level) { + auto Nd = Grids[level - 1]->_ndimension; + auto tmp = Grids[level - 1]->_fdimensions; + assert(tmp.size() == Nd); + + Seeds.push_back(std::vector(Nd)); + + for(int d = 0; d < Nd; ++d) { + tmp[d] /= mgParams.blockSizes[level - 1][d]; + Seeds[level][d] = (level)*Nd + d + 1; + } + + Grids.push_back(QCD::SpaceTimeGrid::makeFourDimGrid(tmp, Grids[level - 1]->_simd_layout, GridDefaultMpi())); + PRNGs.push_back(GridParallelRNG(Grids[level])); + + PRNGs[level].SeedFixedIntegers(Seeds[level]); + } + + std::cout << GridLogMessage << "Constructed " << mgParams.nLevels << " levels" << std::endl; + + for(int level = 0; level < mgParams.nLevels; ++level) { + std::cout << GridLogMessage << "level = " << level << ":" << std::endl; + Grids[level]->show_decomposition(); + } + } +}; + +template class MultiGridPreconditionerBase : public LinearFunction { +public: + virtual ~MultiGridPreconditionerBase() = default; + virtual void setup() = 0; + virtual void operator()(Field const &in, Field &out) = 0; + virtual void runChecks(RealD tolerance) = 0; + virtual void reportTimings() = 0; +}; + +template +class MultiGridPreconditioner : public MultiGridPreconditionerBase> { +public: + ///////////////////////////////////////////// + // Type Definitions + ///////////////////////////////////////////// + + // clang-format off + typedef Aggregation Aggregates; + typedef CoarsenedMatrix CoarseDiracMatrix; + typedef typename Aggregates::CoarseVector CoarseVector; + typedef typename Aggregates::siteVector CoarseSiteVector; + typedef Matrix FineDiracMatrix; + typedef typename Aggregates::FineField FineVector; + typedef MultiGridPreconditioner, nBasis, nCoarserLevels - 1, CoarseDiracMatrix> NextPreconditionerLevel; + // clang-format on + + ///////////////////////////////////////////// + // Member Data + ///////////////////////////////////////////// + + int _CurrentLevel; + int _NextCoarserLevel; + + MultiGridParams &_MultiGridParams; + LevelInfo & _LevelInfo; + + FineDiracMatrix & _FineMatrix; + FineDiracMatrix & _SmootherMatrix; + Aggregates _Aggregates; + CoarseDiracMatrix _CoarseMatrix; + + std::unique_ptr _NextPreconditionerLevel; + + GridStopWatch _SetupTotalTimer; + GridStopWatch _SetupCreateSubspaceTimer; + GridStopWatch _SetupProjectToChiralitiesTimer; + GridStopWatch _SetupCoarsenOperatorTimer; + GridStopWatch _SetupNextLevelTimer; + GridStopWatch _SolveTotalTimer; + GridStopWatch _SolveRestrictionTimer; + GridStopWatch _SolveProlongationTimer; + GridStopWatch _SolveSmootherTimer; + GridStopWatch _SolveNextLevelTimer; + + ///////////////////////////////////////////// + // Member Functions + ///////////////////////////////////////////// + + MultiGridPreconditioner(MultiGridParams &mgParams, LevelInfo &LvlInfo, FineDiracMatrix &FineMat, FineDiracMatrix &SmootherMat) + : _CurrentLevel(mgParams.nLevels - (nCoarserLevels + 1)) // _Level = 0 corresponds to finest + , _NextCoarserLevel(_CurrentLevel + 1) // incremented for instances on coarser levels + , _MultiGridParams(mgParams) + , _LevelInfo(LvlInfo) + , _FineMatrix(FineMat) + , _SmootherMatrix(SmootherMat) + , _Aggregates(_LevelInfo.Grids[_NextCoarserLevel], _LevelInfo.Grids[_CurrentLevel], 0) + , _CoarseMatrix(*_LevelInfo.Grids[_NextCoarserLevel]) { + + _NextPreconditionerLevel + = std::unique_ptr(new NextPreconditionerLevel(_MultiGridParams, _LevelInfo, _CoarseMatrix, _CoarseMatrix)); + + resetTimers(); + } + + void setup() { + + _SetupTotalTimer.Start(); + + static_assert((nBasis & 0x1) == 0, "MG Preconditioner only supports an even number of basis vectors"); + int nb = nBasis / 2; + + MdagMLinearOperator fineMdagMOp(_FineMatrix); + + _SetupCreateSubspaceTimer.Start(); + _Aggregates.CreateSubspace(_LevelInfo.PRNGs[_CurrentLevel], fineMdagMOp, nb); + _SetupCreateSubspaceTimer.Stop(); + + _SetupProjectToChiralitiesTimer.Start(); + FineVector tmp1(_Aggregates.subspace[0]._grid); + FineVector tmp2(_Aggregates.subspace[0]._grid); + for(int n = 0; n < nb; n++) { + auto tmp1 = _Aggregates.subspace[n]; + G5C(tmp2, _Aggregates.subspace[n]); + axpby(_Aggregates.subspace[n], 0.5, 0.5, tmp1, tmp2); + axpby(_Aggregates.subspace[n + nb], 0.5, -0.5, tmp1, tmp2); + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Chirally doubled vector " << n << ". " + << "norm2(vec[" << n << "]) = " << norm2(_Aggregates.subspace[n]) << ". " + << "norm2(vec[" << n + nb << "]) = " << norm2(_Aggregates.subspace[n + nb]) << std::endl; + } + _SetupProjectToChiralitiesTimer.Stop(); + + _SetupCoarsenOperatorTimer.Start(); + _CoarseMatrix.CoarsenOperator(_LevelInfo.Grids[_CurrentLevel], fineMdagMOp, _Aggregates); + _SetupCoarsenOperatorTimer.Stop(); + + _SetupNextLevelTimer.Start(); + _NextPreconditionerLevel->setup(); + _SetupNextLevelTimer.Stop(); + + _SetupTotalTimer.Stop(); + } + + virtual void operator()(FineVector const &in, FineVector &out) { + + conformable(_LevelInfo.Grids[_CurrentLevel], in._grid); + conformable(in, out); + + // TODO: implement a W-cycle + if(_MultiGridParams.kCycle) + kCycle(in, out); + else + vCycle(in, out); + } + + void vCycle(FineVector const &in, FineVector &out) { + + _SolveTotalTimer.Start(); + + RealD inputNorm = norm2(in); + + CoarseVector coarseSrc(_LevelInfo.Grids[_NextCoarserLevel]); + CoarseVector coarseSol(_LevelInfo.Grids[_NextCoarserLevel]); + coarseSol = zero; + + FineVector fineTmp(in._grid); + + auto maxSmootherIter = _MultiGridParams.smootherMaxOuterIter[_CurrentLevel] * _MultiGridParams.smootherMaxInnerIter[_CurrentLevel]; + + TrivialPrecon fineTrivialPreconditioner; + FlexibleGeneralisedMinimalResidual fineFGMRES(_MultiGridParams.smootherTol[_CurrentLevel], + maxSmootherIter, + fineTrivialPreconditioner, + _MultiGridParams.smootherMaxInnerIter[_CurrentLevel], + false); + + MdagMLinearOperator fineMdagMOp(_FineMatrix); + MdagMLinearOperator fineSmootherMdagMOp(_SmootherMatrix); + + _SolveRestrictionTimer.Start(); + _Aggregates.ProjectToSubspace(coarseSrc, in); + _SolveRestrictionTimer.Stop(); + + _SolveNextLevelTimer.Start(); + (*_NextPreconditionerLevel)(coarseSrc, coarseSol); + _SolveNextLevelTimer.Stop(); + + _SolveProlongationTimer.Start(); + _Aggregates.PromoteFromSubspace(coarseSol, out); + _SolveProlongationTimer.Stop(); + + fineMdagMOp.Op(out, fineTmp); + fineTmp = in - fineTmp; + auto r = norm2(fineTmp); + auto residualAfterCoarseGridCorrection = std::sqrt(r / inputNorm); + + _SolveSmootherTimer.Start(); + fineFGMRES(fineSmootherMdagMOp, in, out); + _SolveSmootherTimer.Stop(); + + fineMdagMOp.Op(out, fineTmp); + fineTmp = in - fineTmp; + r = norm2(fineTmp); + auto residualAfterPostSmoother = std::sqrt(r / inputNorm); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": V-cycle: Input norm = " << std::sqrt(inputNorm) + << " Coarse residual = " << residualAfterCoarseGridCorrection << " Post-Smoother residual = " << residualAfterPostSmoother + << std::endl; + + _SolveTotalTimer.Stop(); + } + + void kCycle(FineVector const &in, FineVector &out) { + + _SolveTotalTimer.Start(); + + RealD inputNorm = norm2(in); + + CoarseVector coarseSrc(_LevelInfo.Grids[_NextCoarserLevel]); + CoarseVector coarseSol(_LevelInfo.Grids[_NextCoarserLevel]); + coarseSol = zero; + + FineVector fineTmp(in._grid); + + auto smootherMaxIter = _MultiGridParams.smootherMaxOuterIter[_CurrentLevel] * _MultiGridParams.smootherMaxInnerIter[_CurrentLevel]; + auto kCycleMaxIter = _MultiGridParams.kCycleMaxOuterIter[_CurrentLevel] * _MultiGridParams.kCycleMaxInnerIter[_CurrentLevel]; + + TrivialPrecon fineTrivialPreconditioner; + FlexibleGeneralisedMinimalResidual fineFGMRES(_MultiGridParams.smootherTol[_CurrentLevel], + smootherMaxIter, + fineTrivialPreconditioner, + _MultiGridParams.smootherMaxInnerIter[_CurrentLevel], + false); + FlexibleGeneralisedMinimalResidual coarseFGMRES(_MultiGridParams.kCycleTol[_CurrentLevel], + kCycleMaxIter, + *_NextPreconditionerLevel, + _MultiGridParams.kCycleMaxInnerIter[_CurrentLevel], + false); + + MdagMLinearOperator fineMdagMOp(_FineMatrix); + MdagMLinearOperator fineSmootherMdagMOp(_SmootherMatrix); + MdagMLinearOperator coarseMdagMOp(_CoarseMatrix); + + _SolveRestrictionTimer.Start(); + _Aggregates.ProjectToSubspace(coarseSrc, in); + _SolveRestrictionTimer.Stop(); + + _SolveNextLevelTimer.Start(); + coarseFGMRES(coarseMdagMOp, coarseSrc, coarseSol); + _SolveNextLevelTimer.Stop(); + + _SolveProlongationTimer.Start(); + _Aggregates.PromoteFromSubspace(coarseSol, out); + _SolveProlongationTimer.Stop(); + + fineMdagMOp.Op(out, fineTmp); + fineTmp = in - fineTmp; + auto r = norm2(fineTmp); + auto residualAfterCoarseGridCorrection = std::sqrt(r / inputNorm); + + _SolveSmootherTimer.Start(); + fineFGMRES(fineSmootherMdagMOp, in, out); + _SolveSmootherTimer.Stop(); + + fineMdagMOp.Op(out, fineTmp); + fineTmp = in - fineTmp; + r = norm2(fineTmp); + auto residualAfterPostSmoother = std::sqrt(r / inputNorm); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": K-cycle: Input norm = " << std::sqrt(inputNorm) + << " Coarse residual = " << residualAfterCoarseGridCorrection << " Post-Smoother residual = " << residualAfterPostSmoother + << std::endl; + + _SolveTotalTimer.Stop(); + } + + void runChecks(RealD tolerance) { + + std::vector fineTmps(7, _LevelInfo.Grids[_CurrentLevel]); + std::vector coarseTmps(4, _LevelInfo.Grids[_NextCoarserLevel]); + + MdagMLinearOperator fineMdagMOp(_FineMatrix); + MdagMLinearOperator coarseMdagMOp(_CoarseMatrix); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": MG correctness check: 0 == (M - (Mdiag + Σ_μ Mdir_μ)) * v" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + + random(_LevelInfo.PRNGs[_CurrentLevel], fineTmps[0]); + + fineMdagMOp.Op(fineTmps[0], fineTmps[1]); // M * v + fineMdagMOp.OpDiag(fineTmps[0], fineTmps[2]); // Mdiag * v + + fineTmps[4] = zero; + for(int dir = 0; dir < 4; dir++) { // Σ_μ Mdir_μ * v + for(auto disp : {+1, -1}) { + fineMdagMOp.OpDir(fineTmps[0], fineTmps[3], dir, disp); + fineTmps[4] = fineTmps[4] + fineTmps[3]; + } + } + + fineTmps[5] = fineTmps[2] + fineTmps[4]; // (Mdiag + Σ_μ Mdir_μ) * v + + fineTmps[6] = fineTmps[1] - fineTmps[5]; + auto deviation = std::sqrt(norm2(fineTmps[6]) / norm2(fineTmps[1])); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": norm2(M * v) = " << norm2(fineTmps[1]) << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": norm2(Mdiag * v) = " << norm2(fineTmps[2]) << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": norm2(Σ_μ Mdir_μ * v) = " << norm2(fineTmps[4]) << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": norm2((Mdiag + Σ_μ Mdir_μ) * v) = " << norm2(fineTmps[5]) << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": relative deviation = " << deviation; + + if(deviation > tolerance) { + std::cout << " > " << tolerance << " -> check failed" << std::endl; + abort(); + } else { + std::cout << " < " << tolerance << " -> check passed" << std::endl; + } + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": MG correctness check: 0 == (1 - P R) v" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + + for(auto i = 0; i < _Aggregates.subspace.size(); ++i) { + _Aggregates.ProjectToSubspace(coarseTmps[0], _Aggregates.subspace[i]); // R v_i + _Aggregates.PromoteFromSubspace(coarseTmps[0], fineTmps[0]); // P R v_i + + fineTmps[1] = _Aggregates.subspace[i] - fineTmps[0]; // v_i - P R v_i + deviation = std::sqrt(norm2(fineTmps[1]) / norm2(_Aggregates.subspace[i])); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Vector " << i << ": norm2(v_i) = " << norm2(_Aggregates.subspace[i]) + << " | norm2(R v_i) = " << norm2(coarseTmps[0]) << " | norm2(P R v_i) = " << norm2(fineTmps[0]) + << " | relative deviation = " << deviation; + + if(deviation > tolerance) { + std::cout << " > " << tolerance << " -> check failed" << std::endl; + abort(); + } else { + std::cout << " < " << tolerance << " -> check passed" << std::endl; + } + } + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": MG correctness check: 0 == (1 - R P) v_c" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + + random(_LevelInfo.PRNGs[_NextCoarserLevel], coarseTmps[0]); + + _Aggregates.PromoteFromSubspace(coarseTmps[0], fineTmps[0]); // P v_c + _Aggregates.ProjectToSubspace(coarseTmps[1], fineTmps[0]); // R P v_c + + coarseTmps[2] = coarseTmps[0] - coarseTmps[1]; // v_c - R P v_c + deviation = std::sqrt(norm2(coarseTmps[2]) / norm2(coarseTmps[0])); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": norm2(v_c) = " << norm2(coarseTmps[0]) + << " | norm2(R P v_c) = " << norm2(coarseTmps[1]) << " | norm2(P v_c) = " << norm2(fineTmps[0]) + << " | relative deviation = " << deviation; + + if(deviation > tolerance) { + std::cout << " > " << tolerance << " -> check failed" << std::endl; + abort(); + } else { + std::cout << " < " << tolerance << " -> check passed" << std::endl; + } + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": MG correctness check: 0 == (R D P - D_c) v_c" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + + random(_LevelInfo.PRNGs[_NextCoarserLevel], coarseTmps[0]); + + _Aggregates.PromoteFromSubspace(coarseTmps[0], fineTmps[0]); // P v_c + fineMdagMOp.Op(fineTmps[0], fineTmps[1]); // D P v_c + _Aggregates.ProjectToSubspace(coarseTmps[1], fineTmps[1]); // R D P v_c + + coarseMdagMOp.Op(coarseTmps[0], coarseTmps[2]); // D_c v_c + + coarseTmps[3] = coarseTmps[1] - coarseTmps[2]; // R D P v_c - D_c v_c + deviation = std::sqrt(norm2(coarseTmps[3]) / norm2(coarseTmps[1])); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": norm2(R D P v_c) = " << norm2(coarseTmps[1]) + << " | norm2(D_c v_c) = " << norm2(coarseTmps[2]) << " | relative deviation = " << deviation; + + if(deviation > tolerance) { + std::cout << " > " << tolerance << " -> check failed" << std::endl; + abort(); + } else { + std::cout << " < " << tolerance << " -> check passed" << std::endl; + } + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": MG correctness check: 0 == |(Im(v_c^dag D_c^dag D_c v_c)|" << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": **************************************************" << std::endl; + + random(_LevelInfo.PRNGs[_NextCoarserLevel], coarseTmps[0]); + + coarseMdagMOp.Op(coarseTmps[0], coarseTmps[1]); // D_c v_c + coarseMdagMOp.AdjOp(coarseTmps[1], coarseTmps[2]); // D_c^dag D_c v_c + + auto dot = innerProduct(coarseTmps[0], coarseTmps[2]); //v_c^dag D_c^dag D_c v_c + deviation = std::abs(imag(dot)) / std::abs(real(dot)); + + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Re(v_c^dag D_c^dag D_c v_c) = " << real(dot) + << " | Im(v_c^dag D_c^dag D_c v_c) = " << imag(dot) << " | relative deviation = " << deviation; + + if(deviation > tolerance) { + std::cout << " > " << tolerance << " -> check failed" << std::endl; + abort(); + } else { + std::cout << " < " << tolerance << " -> check passed" << std::endl; + } + + _NextPreconditionerLevel->runChecks(tolerance); + } + + void reportTimings() { + + // clang-format off + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Sum total " << _SetupTotalTimer.Elapsed() + _SolveTotalTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Setup total " << _SetupTotalTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Setup create subspace " << _SetupCreateSubspaceTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Setup project chiral " << _SetupProjectToChiralitiesTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Setup coarsen operator " << _SetupCoarsenOperatorTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Setup next level " << _SetupNextLevelTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Solve total " << _SolveTotalTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Solve restriction " << _SolveRestrictionTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Solve prolongation " << _SolveProlongationTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Solve smoother " << _SolveSmootherTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Solve next level " << _SolveNextLevelTimer.Elapsed() << std::endl; + // clang-format on + + _NextPreconditionerLevel->reportTimings(); + } + + void resetTimers() { + + _SetupTotalTimer.Reset(); + _SetupCreateSubspaceTimer.Reset(); + _SetupProjectToChiralitiesTimer.Reset(); + _SetupCoarsenOperatorTimer.Reset(); + _SetupNextLevelTimer.Reset(); + _SolveTotalTimer.Reset(); + _SolveRestrictionTimer.Reset(); + _SolveProlongationTimer.Reset(); + _SolveSmootherTimer.Reset(); + _SolveNextLevelTimer.Reset(); + + _NextPreconditionerLevel->resetTimers(); + } +}; + +// Specialization for the coarsest level +template +class MultiGridPreconditioner : public MultiGridPreconditionerBase> { +public: + ///////////////////////////////////////////// + // Type Definitions + ///////////////////////////////////////////// + + typedef Matrix FineDiracMatrix; + typedef Lattice FineVector; + + ///////////////////////////////////////////// + // Member Data + ///////////////////////////////////////////// + + int _CurrentLevel; + + MultiGridParams &_MultiGridParams; + LevelInfo & _LevelInfo; + + FineDiracMatrix &_FineMatrix; + FineDiracMatrix &_SmootherMatrix; + + GridStopWatch _SolveTotalTimer; + GridStopWatch _SolveSmootherTimer; + + ///////////////////////////////////////////// + // Member Functions + ///////////////////////////////////////////// + + MultiGridPreconditioner(MultiGridParams &mgParams, LevelInfo &LvlInfo, FineDiracMatrix &FineMat, FineDiracMatrix &SmootherMat) + : _CurrentLevel(mgParams.nLevels - (0 + 1)) + , _MultiGridParams(mgParams) + , _LevelInfo(LvlInfo) + , _FineMatrix(FineMat) + , _SmootherMatrix(SmootherMat) { + + resetTimers(); + } + + void setup() {} + + virtual void operator()(FineVector const &in, FineVector &out) { + + _SolveTotalTimer.Start(); + + conformable(_LevelInfo.Grids[_CurrentLevel], in._grid); + conformable(in, out); + + auto coarseSolverMaxIter = _MultiGridParams.coarseSolverMaxOuterIter * _MultiGridParams.coarseSolverMaxInnerIter; + + // On the coarsest level we only have what I above call the fine level, no coarse one + TrivialPrecon fineTrivialPreconditioner; + FlexibleGeneralisedMinimalResidual fineFGMRES( + _MultiGridParams.coarseSolverTol, coarseSolverMaxIter, fineTrivialPreconditioner, _MultiGridParams.coarseSolverMaxInnerIter, false); + + MdagMLinearOperator fineMdagMOp(_FineMatrix); + + _SolveSmootherTimer.Start(); + fineFGMRES(fineMdagMOp, in, out); + _SolveSmootherTimer.Stop(); + + _SolveTotalTimer.Stop(); + } + + void runChecks(RealD tolerance) {} + + void reportTimings() { + + // clang-format off + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Solve total " << _SolveTotalTimer.Elapsed() << std::endl; + std::cout << GridLogMG << " Level " << _CurrentLevel << ": Time elapsed: Solve smoother " << _SolveSmootherTimer.Elapsed() << std::endl; + // clang-format on + } + + void resetTimers() { + + _SolveTotalTimer.Reset(); + _SolveSmootherTimer.Reset(); + } +}; + +template +using NLevelMGPreconditioner = MultiGridPreconditioner; + +template +std::unique_ptr>> +createMGInstance(MultiGridParams &mgParams, LevelInfo &levelInfo, Matrix &FineMat, Matrix &SmootherMat) { + +#define CASE_FOR_N_LEVELS(nLevels) \ + case nLevels: \ + return std::unique_ptr>( \ + new NLevelMGPreconditioner(mgParams, levelInfo, FineMat, SmootherMat)); \ + break; + + switch(mgParams.nLevels) { + CASE_FOR_N_LEVELS(2); + CASE_FOR_N_LEVELS(3); + CASE_FOR_N_LEVELS(4); + default: + std::cout << GridLogError << "We currently only support nLevels ∈ {2, 3, 4}" << std::endl; + exit(EXIT_FAILURE); + break; + } +#undef CASE_FOR_N_LEVELS +} + +} +#endif diff --git a/tests/solver/Test_staggered_cagmres_unprec.cc b/tests/solver/Test_staggered_cagmres_unprec.cc new file mode 100644 index 00000000..b82ecaeb --- /dev/null +++ b/tests/solver/Test_staggered_cagmres_unprec.cc @@ -0,0 +1,72 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_staggered_cagmres_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + typedef typename ImprovedStaggeredFermionR::FermionField FermionField; + typedef typename ImprovedStaggeredFermionR::ComplexField ComplexField; + typename ImprovedStaggeredFermionR::ImplParams params; + + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Ds); + CommunicationAvoidingGeneralisedMinimalResidual CAGMRES(1.0e-8, 10000, 25); + CAGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_staggered_fcagmres_prec.cc b/tests/solver/Test_staggered_fcagmres_prec.cc new file mode 100644 index 00000000..7685585b --- /dev/null +++ b/tests/solver/Test_staggered_fcagmres_prec.cc @@ -0,0 +1,75 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_staggered_fcagmres_prec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + typedef typename ImprovedStaggeredFermionR::FermionField FermionField; + typedef typename ImprovedStaggeredFermionR::ComplexField ComplexField; + typename ImprovedStaggeredFermionR::ImplParams params; + + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Ds); + + TrivialPrecon simple; + + FlexibleCommunicationAvoidingGeneralisedMinimalResidual FCAGMRES(1.0e-8, 10000, simple, 25); + FCAGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_staggered_fgmres_prec.cc b/tests/solver/Test_staggered_fgmres_prec.cc new file mode 100644 index 00000000..30905e35 --- /dev/null +++ b/tests/solver/Test_staggered_fgmres_prec.cc @@ -0,0 +1,75 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_staggered_fgmres_prec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + typedef typename ImprovedStaggeredFermionR::FermionField FermionField; + typedef typename ImprovedStaggeredFermionR::ComplexField ComplexField; + typename ImprovedStaggeredFermionR::ImplParams params; + + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Ds); + + TrivialPrecon simple; + + FlexibleGeneralisedMinimalResidual FGMRES(1.0e-8, 10000, simple, 25); + FGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_staggered_gmres_unprec.cc b/tests/solver/Test_staggered_gmres_unprec.cc new file mode 100644 index 00000000..d65b0b31 --- /dev/null +++ b/tests/solver/Test_staggered_gmres_unprec.cc @@ -0,0 +1,72 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_staggered_gmres_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + typedef typename ImprovedStaggeredFermionR::FermionField FermionField; + typedef typename ImprovedStaggeredFermionR::ComplexField ComplexField; + typename ImprovedStaggeredFermionR::ImplParams params; + + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Ds); + GeneralisedMinimalResidual GMRES(1.0e-8, 10000, 25); + GMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_staggered_mr_unprec.cc b/tests/solver/Test_staggered_mr_unprec.cc new file mode 100644 index 00000000..ca60edb4 --- /dev/null +++ b/tests/solver/Test_staggered_mr_unprec.cc @@ -0,0 +1,72 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_staggered_mr_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + typedef typename ImprovedStaggeredFermionR::FermionField FermionField; + typedef typename ImprovedStaggeredFermionR::ComplexField ComplexField; + typename ImprovedStaggeredFermionR::ImplParams params; + + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Ds); + MinimalResidual MR(1.0e-8,10000,0.8); + MR(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilson_cagmres_unprec.cc b/tests/solver/Test_wilson_cagmres_unprec.cc new file mode 100644 index 00000000..46f9e6a6 --- /dev/null +++ b/tests/solver/Test_wilson_cagmres_unprec.cc @@ -0,0 +1,65 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilson_cagmres_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + LatticeFermion src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + LatticeFermion result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dw); + CommunicationAvoidingGeneralisedMinimalResidual CAGMRES(1.0e-8, 10000, 25); + CAGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilson_fcagmres_prec.cc b/tests/solver/Test_wilson_fcagmres_prec.cc new file mode 100644 index 00000000..f802984f --- /dev/null +++ b/tests/solver/Test_wilson_fcagmres_prec.cc @@ -0,0 +1,68 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilson_fcagmres_prec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + LatticeFermion src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + LatticeFermion result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dw); + + TrivialPrecon simple; + + FlexibleCommunicationAvoidingGeneralisedMinimalResidual FCAGMRES(1.0e-8, 10000, simple, 25); + FCAGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilson_fgmres_prec.cc b/tests/solver/Test_wilson_fgmres_prec.cc new file mode 100644 index 00000000..f55516da --- /dev/null +++ b/tests/solver/Test_wilson_fgmres_prec.cc @@ -0,0 +1,68 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilson_fgmres_prec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + LatticeFermion src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + LatticeFermion result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dw); + + TrivialPrecon simple; + + FlexibleGeneralisedMinimalResidual FGMRES(1.0e-8, 10000, simple, 25); + FGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilson_gmres_unprec.cc b/tests/solver/Test_wilson_gmres_unprec.cc new file mode 100644 index 00000000..443f7ebc --- /dev/null +++ b/tests/solver/Test_wilson_gmres_unprec.cc @@ -0,0 +1,65 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilson_gmres_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + LatticeFermion src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + LatticeFermion result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dw); + GeneralisedMinimalResidual GMRES(1.0e-8, 10000, 25); + GMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilson_mg.cc b/tests/solver/Test_wilson_mg.cc new file mode 100644 index 00000000..1609c1fc --- /dev/null +++ b/tests/solver/Test_wilson_mg.cc @@ -0,0 +1,114 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/solver/Test_wilson_mg.cc + + Copyright (C) 2015-2018 + + Author: Daniel Richtmann + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ +/* END LEGAL */ + +#include +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + +int main(int argc, char **argv) { + + Grid_init(&argc, &argv); + + GridCartesian * FGrid = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi()); + GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(FGrid); + + std::vector fSeeds({1, 2, 3, 4}); + GridParallelRNG fPRNG(FGrid); + fPRNG.SeedFixedIntegers(fSeeds); + + // clang-format off + LatticeFermion src(FGrid); gaussian(fPRNG, src); + LatticeFermion result(FGrid); result = zero; + LatticeGaugeField Umu(FGrid); SU3::HotConfiguration(fPRNG, Umu); + // clang-format on + + RealD mass = -0.25; + + MultiGridParams mgParams; + std::string inputXml{"./mg_params.xml"}; + + if(GridCmdOptionExists(argv, argv + argc, "--inputxml")) { + inputXml = GridCmdOptionPayload(argv, argv + argc, "--inputxml"); + assert(inputXml.length() != 0); + } + + { + XmlWriter writer("mg_params_template.xml"); + write(writer, "Params", mgParams); + std::cout << GridLogMessage << "Written mg_params_template.xml" << std::endl; + + XmlReader reader(inputXml); + read(reader, "Params", mgParams); + std::cout << GridLogMessage << "Read in " << inputXml << std::endl; + } + + checkParameterValidity(mgParams); + std::cout << mgParams << std::endl; + + LevelInfo levelInfo(FGrid, mgParams); + + // Note: We do chiral doubling, so actually only nbasis/2 full basis vectors are used + const int nbasis = 40; + + WilsonFermionR Dw(Umu, *FGrid, *FrbGrid, mass); + + MdagMLinearOperator MdagMOpDw(Dw); + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Testing Multigrid for Wilson" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + TrivialPrecon TrivialPrecon; + auto MGPreconDw = createMGInstance(mgParams, levelInfo, Dw, Dw); + + MGPreconDw->setup(); + + if(GridCmdOptionExists(argv, argv + argc, "--runchecks")) { + RealD toleranceForMGChecks = (getPrecision::value == 1) ? 1e-6 : 1e-13; + MGPreconDw->runChecks(toleranceForMGChecks); + } + + std::vector>> solversDw; + + solversDw.emplace_back(new ConjugateGradient(1.0e-12, 50000, false)); + solversDw.emplace_back(new FlexibleGeneralisedMinimalResidual(1.0e-12, 50000, TrivialPrecon, 100, false)); + solversDw.emplace_back(new FlexibleGeneralisedMinimalResidual(1.0e-12, 50000, *MGPreconDw, 100, false)); + + for(auto const &solver : solversDw) { + std::cout << std::endl << "Starting with a new solver" << std::endl; + result = zero; + (*solver)(MdagMOpDw, src, result); + } + + MGPreconDw->reportTimings(); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilson_mg_mp.cc b/tests/solver/Test_wilson_mg_mp.cc new file mode 100644 index 00000000..0cd51227 --- /dev/null +++ b/tests/solver/Test_wilson_mg_mp.cc @@ -0,0 +1,166 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/solver/Test_wilson_mg_mp.cc + + Copyright (C) 2015-2018 + + Author: Daniel Richtmann + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ +/* END LEGAL */ + +#include +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + +int main(int argc, char **argv) { + + Grid_init(&argc, &argv); + + // clang-format off + GridCartesian *FGrid_d = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd, vComplexD::Nsimd()), GridDefaultMpi()); + GridCartesian *FGrid_f = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd, vComplexF::Nsimd()), GridDefaultMpi()); + GridRedBlackCartesian *FrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(FGrid_d); + GridRedBlackCartesian *FrbGrid_f = SpaceTimeGrid::makeFourDimRedBlackGrid(FGrid_f); + // clang-format on + + std::vector fSeeds({1, 2, 3, 4}); + GridParallelRNG fPRNG(FGrid_d); + fPRNG.SeedFixedIntegers(fSeeds); + + // clang-format off + LatticeFermionD src_d(FGrid_d); gaussian(fPRNG, src_d); + LatticeFermionD resultMGD_d(FGrid_d); resultMGD_d = zero; + LatticeFermionD resultMGF_d(FGrid_d); resultMGF_d = zero; + LatticeGaugeFieldD Umu_d(FGrid_d); SU3::HotConfiguration(fPRNG, Umu_d); + LatticeGaugeFieldF Umu_f(FGrid_f); precisionChange(Umu_f, Umu_d); + // clang-format on + + RealD mass = -0.25; + + MultiGridParams mgParams; + std::string inputXml{"./mg_params.xml"}; + + if(GridCmdOptionExists(argv, argv + argc, "--inputxml")) { + inputXml = GridCmdOptionPayload(argv, argv + argc, "--inputxml"); + assert(inputXml.length() != 0); + } + + { + XmlWriter writer("mg_params_template.xml"); + write(writer, "Params", mgParams); + std::cout << GridLogMessage << "Written mg_params_template.xml" << std::endl; + + XmlReader reader(inputXml); + read(reader, "Params", mgParams); + std::cout << GridLogMessage << "Read in " << inputXml << std::endl; + } + + checkParameterValidity(mgParams); + std::cout << mgParams << std::endl; + + LevelInfo levelInfo_d(FGrid_d, mgParams); + LevelInfo levelInfo_f(FGrid_f, mgParams); + + // Note: We do chiral doubling, so actually only nbasis/2 full basis vectors are used + const int nbasis = 40; + + WilsonFermionD Dw_d(Umu_d, *FGrid_d, *FrbGrid_d, mass); + WilsonFermionF Dw_f(Umu_f, *FGrid_f, *FrbGrid_f, mass); + + MdagMLinearOperator MdagMOpDw_d(Dw_d); + MdagMLinearOperator MdagMOpDw_f(Dw_f); + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Testing single-precision Multigrid for Wilson" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + auto MGPreconDw_f = createMGInstance(mgParams, levelInfo_f, Dw_f, Dw_f); + + MGPreconDw_f->setup(); + + if(GridCmdOptionExists(argv, argv + argc, "--runchecks")) { + MGPreconDw_f->runChecks(1e-6); + } + + MixedPrecisionFlexibleGeneralisedMinimalResidual MPFGMRESPREC(1.0e-12, 50000, FGrid_f, *MGPreconDw_f, 100, false); + + std::cout << std::endl << "Starting with a new solver" << std::endl; + MPFGMRESPREC(MdagMOpDw_d, src_d, resultMGF_d); + + MGPreconDw_f->reportTimings(); + + if(GridCmdOptionExists(argv, argv + argc, "--docomparison")) { + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Testing double-precision Multigrid for Wilson" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + auto MGPreconDw_d = createMGInstance(mgParams, levelInfo_d, Dw_d, Dw_d); + + MGPreconDw_d->setup(); + + if(GridCmdOptionExists(argv, argv + argc, "--runchecks")) { + MGPreconDw_d->runChecks(1e-13); + } + + FlexibleGeneralisedMinimalResidual FGMRESPREC(1.0e-12, 50000, *MGPreconDw_d, 100, false); + + std::cout << std::endl << "Starting with a new solver" << std::endl; + FGMRESPREC(MdagMOpDw_d, src_d, resultMGD_d); + + MGPreconDw_d->reportTimings(); + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Comparing single-precision Multigrid with double-precision one for Wilson" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + LatticeFermionD diffFullSolver(FGrid_d); + + RealD deviationFullSolver = axpy_norm(diffFullSolver, -1.0, resultMGF_d, resultMGD_d); + + // clang-format off + LatticeFermionF src_f(FGrid_f); precisionChange(src_f, src_d); + LatticeFermionF resMGF_f(FGrid_f); resMGF_f = zero; + LatticeFermionD resMGD_d(FGrid_d); resMGD_d = zero; + // clang-format on + + (*MGPreconDw_f)(src_f, resMGF_f); + (*MGPreconDw_d)(src_d, resMGD_d); + + LatticeFermionD diffOnlyMG(FGrid_d); + LatticeFermionD resMGF_d(FGrid_d); + precisionChange(resMGF_d, resMGF_f); + + RealD deviationOnlyPrec = axpy_norm(diffOnlyMG, -1.0, resMGF_d, resMGD_d); + + // clang-format off + std::cout << GridLogMessage << "Absolute difference between FGMRES preconditioned by double and single precicision MG: " << deviationFullSolver << std::endl; + std::cout << GridLogMessage << "Relative deviation between FGMRES preconditioned by double and single precicision MG: " << deviationFullSolver / norm2(resultMGD_d) << std::endl; + std::cout << GridLogMessage << "Absolute difference between one iteration of MG Prec in double and single precision: " << deviationOnlyPrec << std::endl; + std::cout << GridLogMessage << "Relative deviation between one iteration of MG Prec in double and single precision: " << deviationOnlyPrec / norm2(resMGD_d) << std::endl; + // clang-format on + } + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilson_mr_unprec.cc b/tests/solver/Test_wilson_mr_unprec.cc new file mode 100644 index 00000000..976130d3 --- /dev/null +++ b/tests/solver/Test_wilson_mr_unprec.cc @@ -0,0 +1,65 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilson_mr_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + LatticeFermion src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + LatticeFermion result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dw); + MinimalResidual MR(1.0e-8,10000,0.8); + MR(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilsonclover_cagmres_unprec.cc b/tests/solver/Test_wilsonclover_cagmres_unprec.cc new file mode 100644 index 00000000..3ecdf738 --- /dev/null +++ b/tests/solver/Test_wilsonclover_cagmres_unprec.cc @@ -0,0 +1,71 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilsonclover_cagmres_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + typedef typename WilsonCloverFermionR::FermionField FermionField; + typename WilsonCloverFermionR::ImplParams params; + WilsonAnisotropyCoefficients anis; + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dwc); + CommunicationAvoidingGeneralisedMinimalResidual CAGMRES(1.0e-8, 10000, 25); + CAGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilsonclover_fcagmres_prec.cc b/tests/solver/Test_wilsonclover_fcagmres_prec.cc new file mode 100644 index 00000000..3cbbfc02 --- /dev/null +++ b/tests/solver/Test_wilsonclover_fcagmres_prec.cc @@ -0,0 +1,74 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilsonclover_fcagmres_prec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + typedef typename WilsonCloverFermionR::FermionField FermionField; + typename WilsonCloverFermionR::ImplParams params; + WilsonAnisotropyCoefficients anis; + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dwc); + + TrivialPrecon simple; + + FlexibleCommunicationAvoidingGeneralisedMinimalResidual FCAGMRES(1.0e-8, 10000, simple, 25); + FCAGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilsonclover_fgmres_prec.cc b/tests/solver/Test_wilsonclover_fgmres_prec.cc new file mode 100644 index 00000000..7ad0fa24 --- /dev/null +++ b/tests/solver/Test_wilsonclover_fgmres_prec.cc @@ -0,0 +1,74 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilsonclover_fgmres_prec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + typedef typename WilsonCloverFermionR::FermionField FermionField; + typename WilsonCloverFermionR::ImplParams params; + WilsonAnisotropyCoefficients anis; + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dwc); + + TrivialPrecon simple; + + FlexibleGeneralisedMinimalResidual FGMRES(1.0e-8, 10000, simple, 25); + FGMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilsonclover_gmres_unprec.cc b/tests/solver/Test_wilsonclover_gmres_unprec.cc new file mode 100644 index 00000000..a9fe7181 --- /dev/null +++ b/tests/solver/Test_wilsonclover_gmres_unprec.cc @@ -0,0 +1,71 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilsonclover_gmres_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + typedef typename WilsonCloverFermionR::FermionField FermionField; + typename WilsonCloverFermionR::ImplParams params; + WilsonAnisotropyCoefficients anis; + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dwc); + GeneralisedMinimalResidual GMRES(1.0e-8, 10000, 25); + GMRES(HermOp,src,result); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilsonclover_mg.cc b/tests/solver/Test_wilsonclover_mg.cc new file mode 100644 index 00000000..e749aacb --- /dev/null +++ b/tests/solver/Test_wilsonclover_mg.cc @@ -0,0 +1,117 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/solver/Test_wilsonclover_mg.cc + + Copyright (C) 2015-2018 + + Author: Daniel Richtmann + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ +/* END LEGAL */ + +#include +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + +int main(int argc, char **argv) { + + Grid_init(&argc, &argv); + + GridCartesian * FGrid = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi()); + GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(FGrid); + + std::vector fSeeds({1, 2, 3, 4}); + GridParallelRNG fPRNG(FGrid); + fPRNG.SeedFixedIntegers(fSeeds); + + // clang-format off + LatticeFermion src(FGrid); gaussian(fPRNG, src); + LatticeFermion result(FGrid); result = zero; + LatticeGaugeField Umu(FGrid); SU3::HotConfiguration(fPRNG, Umu); + // clang-format on + + RealD mass = -0.25; + RealD csw_r = 1.0; + RealD csw_t = 1.0; + + MultiGridParams mgParams; + std::string inputXml{"./mg_params.xml"}; + + if(GridCmdOptionExists(argv, argv + argc, "--inputxml")) { + inputXml = GridCmdOptionPayload(argv, argv + argc, "--inputxml"); + assert(inputXml.length() != 0); + } + + { + XmlWriter writer("mg_params_template.xml"); + write(writer, "Params", mgParams); + std::cout << GridLogMessage << "Written mg_params_template.xml" << std::endl; + + XmlReader reader(inputXml); + read(reader, "Params", mgParams); + std::cout << GridLogMessage << "Read in " << inputXml << std::endl; + } + + checkParameterValidity(mgParams); + std::cout << mgParams << std::endl; + + LevelInfo levelInfo(FGrid, mgParams); + + // Note: We do chiral doubling, so actually only nbasis/2 full basis vectors are used + const int nbasis = 40; + + WilsonCloverFermionR Dwc(Umu, *FGrid, *FrbGrid, mass, csw_r, csw_t); + + MdagMLinearOperator MdagMOpDwc(Dwc); + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Testing Multigrid for Wilson Clover" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + TrivialPrecon TrivialPrecon; + auto MGPreconDwc = createMGInstance(mgParams, levelInfo, Dwc, Dwc); + + MGPreconDwc->setup(); + + if(GridCmdOptionExists(argv, argv + argc, "--runchecks")) { + RealD toleranceForMGChecks = (getPrecision::value == 1) ? 1e-6 : 1e-13; + MGPreconDwc->runChecks(toleranceForMGChecks); + } + + std::vector>> solversDwc; + + solversDwc.emplace_back(new ConjugateGradient(1.0e-12, 50000, false)); + solversDwc.emplace_back(new FlexibleGeneralisedMinimalResidual(1.0e-12, 50000, TrivialPrecon, 100, false)); + solversDwc.emplace_back(new FlexibleGeneralisedMinimalResidual(1.0e-12, 50000, *MGPreconDwc, 100, false)); + + for(auto const &solver : solversDwc) { + std::cout << std::endl << "Starting with a new solver" << std::endl; + result = zero; + (*solver)(MdagMOpDwc, src, result); + std::cout << std::endl; + } + + MGPreconDwc->reportTimings(); + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilsonclover_mg_mp.cc b/tests/solver/Test_wilsonclover_mg_mp.cc new file mode 100644 index 00000000..d9ed1d33 --- /dev/null +++ b/tests/solver/Test_wilsonclover_mg_mp.cc @@ -0,0 +1,169 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/solver/Test_wilsonclover_mg_mp.cc + + Copyright (C) 2015-2018 + + Author: Daniel Richtmann + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ +/* END LEGAL */ + +#include +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + +int main(int argc, char **argv) { + + Grid_init(&argc, &argv); + + // clang-format off + GridCartesian *FGrid_d = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd, vComplexD::Nsimd()), GridDefaultMpi()); + GridCartesian *FGrid_f = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd, vComplexF::Nsimd()), GridDefaultMpi()); + GridRedBlackCartesian *FrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(FGrid_d); + GridRedBlackCartesian *FrbGrid_f = SpaceTimeGrid::makeFourDimRedBlackGrid(FGrid_f); + // clang-format on + + std::vector fSeeds({1, 2, 3, 4}); + GridParallelRNG fPRNG(FGrid_d); + fPRNG.SeedFixedIntegers(fSeeds); + + // clang-format off + LatticeFermionD src_d(FGrid_d); gaussian(fPRNG, src_d); + LatticeFermionD resultMGD_d(FGrid_d); resultMGD_d = zero; + LatticeFermionD resultMGF_d(FGrid_d); resultMGF_d = zero; + LatticeGaugeFieldD Umu_d(FGrid_d); SU3::HotConfiguration(fPRNG, Umu_d); + LatticeGaugeFieldF Umu_f(FGrid_f); precisionChange(Umu_f, Umu_d); + // clang-format on + + RealD mass = -0.25; + RealD csw_r = 1.0; + RealD csw_t = 1.0; + + MultiGridParams mgParams; + std::string inputXml{"./mg_params.xml"}; + + if(GridCmdOptionExists(argv, argv + argc, "--inputxml")) { + inputXml = GridCmdOptionPayload(argv, argv + argc, "--inputxml"); + assert(inputXml.length() != 0); + } + + { + XmlWriter writer("mg_params_template.xml"); + write(writer, "Params", mgParams); + std::cout << GridLogMessage << "Written mg_params_template.xml" << std::endl; + + XmlReader reader(inputXml); + read(reader, "Params", mgParams); + std::cout << GridLogMessage << "Read in " << inputXml << std::endl; + } + + checkParameterValidity(mgParams); + std::cout << mgParams << std::endl; + + LevelInfo levelInfo_d(FGrid_d, mgParams); + LevelInfo levelInfo_f(FGrid_f, mgParams); + + // Note: We do chiral doubling, so actually only nbasis/2 full basis vectors are used + const int nbasis = 40; + + WilsonCloverFermionD Dwc_d(Umu_d, *FGrid_d, *FrbGrid_d, mass, csw_r, csw_t); + WilsonCloverFermionF Dwc_f(Umu_f, *FGrid_f, *FrbGrid_f, mass, csw_r, csw_t); + + MdagMLinearOperator MdagMOpDwc_d(Dwc_d); + MdagMLinearOperator MdagMOpDwc_f(Dwc_f); + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Testing single-precision Multigrid for Wilson Clover" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + auto MGPreconDwc_f = createMGInstance(mgParams, levelInfo_f, Dwc_f, Dwc_f); + + MGPreconDwc_f->setup(); + + if(GridCmdOptionExists(argv, argv + argc, "--runchecks")) { + MGPreconDwc_f->runChecks(1e-6); + } + + MixedPrecisionFlexibleGeneralisedMinimalResidual MPFGMRESPREC( + 1.0e-12, 50000, FGrid_f, *MGPreconDwc_f, 100, false); + + std::cout << std::endl << "Starting with a new solver" << std::endl; + MPFGMRESPREC(MdagMOpDwc_d, src_d, resultMGF_d); + + MGPreconDwc_f->reportTimings(); + + if(GridCmdOptionExists(argv, argv + argc, "--docomparison")) { + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Testing double-precision Multigrid for Wilson Clover" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + auto MGPreconDwc_d = createMGInstance(mgParams, levelInfo_d, Dwc_d, Dwc_d); + + MGPreconDwc_d->setup(); + + if(GridCmdOptionExists(argv, argv + argc, "--runchecks")) { + MGPreconDwc_d->runChecks(1e-13); + } + + FlexibleGeneralisedMinimalResidual FGMRESPREC(1.0e-12, 50000, *MGPreconDwc_d, 100, false); + + std::cout << std::endl << "Starting with a new solver" << std::endl; + FGMRESPREC(MdagMOpDwc_d, src_d, resultMGD_d); + + MGPreconDwc_d->reportTimings(); + + std::cout << GridLogMessage << "**************************************************" << std::endl; + std::cout << GridLogMessage << "Comparing single-precision Multigrid with double-precision one for Wilson Clover" << std::endl; + std::cout << GridLogMessage << "**************************************************" << std::endl; + + LatticeFermionD diffFullSolver(FGrid_d); + + RealD deviationFullSolver = axpy_norm(diffFullSolver, -1.0, resultMGF_d, resultMGD_d); + + // clang-format off + LatticeFermionF src_f(FGrid_f); precisionChange(src_f, src_d); + LatticeFermionF resMGF_f(FGrid_f); resMGF_f = zero; + LatticeFermionD resMGD_d(FGrid_d); resMGD_d = zero; + // clang-format on + + (*MGPreconDwc_f)(src_f, resMGF_f); + (*MGPreconDwc_d)(src_d, resMGD_d); + + LatticeFermionD diffOnlyMG(FGrid_d); + LatticeFermionD resMGF_d(FGrid_d); + precisionChange(resMGF_d, resMGF_f); + + RealD deviationOnlyPrec = axpy_norm(diffOnlyMG, -1.0, resMGF_d, resMGD_d); + + // clang-format off + std::cout << GridLogMessage << "Absolute difference between FGMRES preconditioned by double and single precicision MG: " << deviationFullSolver << std::endl; + std::cout << GridLogMessage << "Relative deviation between FGMRES preconditioned by double and single precicision MG: " << deviationFullSolver / norm2(resultMGD_d) << std::endl; + std::cout << GridLogMessage << "Absolute difference between one iteration of MG Prec in double and single precision: " << deviationOnlyPrec << std::endl; + std::cout << GridLogMessage << "Relative deviation between one iteration of MG Prec in double and single precision: " << deviationOnlyPrec / norm2(resMGD_d) << std::endl; + // clang-format on + } + + Grid_finalize(); +} diff --git a/tests/solver/Test_wilsonclover_mr_unprec.cc b/tests/solver/Test_wilsonclover_mr_unprec.cc new file mode 100644 index 00000000..e3aa8838 --- /dev/null +++ b/tests/solver/Test_wilsonclover_mr_unprec.cc @@ -0,0 +1,71 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/solver/Test_wilsonclover_mr_unprec.cc + +Copyright (C) 2015-2018 + +Author: Daniel Richtmann + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); + + std::vector seeds({1,2,3,4}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); + + typedef typename WilsonCloverFermionR::FermionField FermionField; + typename WilsonCloverFermionR::ImplParams params; + WilsonAnisotropyCoefficients anis; + + FermionField src(&Grid); random(pRNG,src); + RealD nrm = norm2(src); + FermionField result(&Grid); result=zero; + LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu); + + double volume=1; + for(int mu=0;mu HermOp(Dwc); + MinimalResidual MR(1.0e-8,10000,0.8); + MR(HermOp,src,result); + + Grid_finalize(); +}