forked from portelli/lattice-benchmarks
Merge remote-tracking branch 'upstream/main' into main
This commit is contained in:
@ -30,8 +30,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
using namespace std;
|
||||
using namespace Grid;
|
||||
|
||||
template <class d>
|
||||
struct scal
|
||||
template <class d> struct scal
|
||||
{
|
||||
d internal;
|
||||
};
|
||||
@ -69,13 +68,11 @@ int main(int argc, char **argv)
|
||||
json["single_site_flops"] = single_site_flops;
|
||||
|
||||
GridCartesian *UGrid = SpaceTimeGrid::makeFourDimGrid(
|
||||
GridDefaultLatt(), GridDefaultSimd(Nd, vComplexF::Nsimd()),
|
||||
GridDefaultMpi());
|
||||
GridRedBlackCartesian *UrbGrid =
|
||||
SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
|
||||
GridDefaultLatt(), GridDefaultSimd(Nd, vComplexF::Nsimd()), GridDefaultMpi());
|
||||
GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
|
||||
|
||||
GridCartesian *FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
|
||||
GridRedBlackCartesian *FrbGrid =
|
||||
SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
|
||||
GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
|
||||
|
||||
json["grid"] = FGrid->FullDimensions().toVector();
|
||||
json["local_grid"] = FGrid->LocalDimensions().toVector();
|
||||
@ -83,11 +80,10 @@ int main(int argc, char **argv)
|
||||
std::cout << GridLogMessage << "Making s innermost grids" << std::endl;
|
||||
GridCartesian *sUGrid =
|
||||
SpaceTimeGrid::makeFourDimDWFGrid(GridDefaultLatt(), GridDefaultMpi());
|
||||
GridRedBlackCartesian *sUrbGrid =
|
||||
SpaceTimeGrid::makeFourDimRedBlackGrid(sUGrid);
|
||||
|
||||
GridRedBlackCartesian *sUrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(sUGrid);
|
||||
GridCartesian *sFGrid = SpaceTimeGrid::makeFiveDimDWFGrid(Ls, UGrid);
|
||||
GridRedBlackCartesian *sFrbGrid =
|
||||
SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(Ls, UGrid);
|
||||
GridRedBlackCartesian *sFrbGrid = SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(Ls, UGrid);
|
||||
|
||||
std::vector<int> seeds4({1, 2, 3, 4});
|
||||
std::vector<int> seeds5({5, 6, 7, 8});
|
||||
@ -150,8 +146,7 @@ int main(int argc, char **argv)
|
||||
{
|
||||
U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
|
||||
}
|
||||
std::cout << GridLogMessage << "Setting up Cshift based reference "
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "Setting up Cshift based reference " << std::endl;
|
||||
|
||||
if (1)
|
||||
{
|
||||
@ -200,54 +195,45 @@ int main(int argc, char **argv)
|
||||
json["ranks"] = NP;
|
||||
json["nodes"] = NN;
|
||||
|
||||
std::cout
|
||||
<< GridLogMessage
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
std::cout
|
||||
<< GridLogMessage
|
||||
<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm"
|
||||
<< std::endl;
|
||||
std::cout
|
||||
<< GridLogMessage
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
std::cout
|
||||
<< GridLogMessage
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "* Benchmarking DomainWallFermionR::Dhop "
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Vectorising space-time by "
|
||||
<< vComplexF::Nsimd() << std::endl;
|
||||
std::cout << GridLogMessage << "* VComplexF size is " << sizeof(vComplexF)
|
||||
<< " B" << std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "* Benchmarking DomainWallFermionR::Dhop " << std::endl;
|
||||
std::cout << GridLogMessage << "* Vectorising space-time by " << vComplexF::Nsimd()
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* VComplexF size is " << sizeof(vComplexF) << " B"
|
||||
<< std::endl;
|
||||
|
||||
if (sizeof(RealF) == 4)
|
||||
std::cout << GridLogMessage << "* SINGLE precision " << std::endl;
|
||||
if (sizeof(RealF) == 8)
|
||||
std::cout << GridLogMessage << "* DOUBLE precision " << std::endl;
|
||||
#ifdef GRID_OMP
|
||||
if (WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute)
|
||||
std::cout << GridLogMessage << "* Using Overlapped Comms/Compute"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using Overlapped Comms/Compute" << std::endl;
|
||||
if (WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute)
|
||||
std::cout << GridLogMessage << "* Using sequential comms compute"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using sequential comms compute" << std::endl;
|
||||
#endif
|
||||
if (WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric)
|
||||
std::cout << GridLogMessage << "* Using GENERIC Nc WilsonKernels"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using GENERIC Nc WilsonKernels" << std::endl;
|
||||
if (WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll)
|
||||
std::cout << GridLogMessage << "* Using Nc=3 WilsonKernels"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using Nc=3 WilsonKernels" << std::endl;
|
||||
if (WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm)
|
||||
std::cout << GridLogMessage << "* Using Asm Nc=3 WilsonKernels"
|
||||
<< std::endl;
|
||||
std::cout
|
||||
<< GridLogMessage
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "* Using Asm Nc=3 WilsonKernels" << std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "*****************************************************************"
|
||||
<< std::endl;
|
||||
|
||||
DomainWallFermionF Dw(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mass, M5);
|
||||
int ncall = 300;
|
||||
@ -277,16 +263,13 @@ int main(int argc, char **argv)
|
||||
auto simdwidth = sizeof(vComplex);
|
||||
|
||||
// RF: Nd Wilson * Ls, Nd gauge * Ls, Nc colors
|
||||
double data_rf = volume * ((2 * Nd + 1) * Nd * Nc + 2 * Nd * Nc * Nc) *
|
||||
simdwidth / nsimd * ncall / (1024. * 1024. * 1024.);
|
||||
double data_rf = volume * ((2 * Nd + 1) * Nd * Nc + 2 * Nd * Nc * Nc) * simdwidth /
|
||||
nsimd * ncall / (1024. * 1024. * 1024.);
|
||||
|
||||
// mem: Nd Wilson * Ls, Nd gauge, Nc colors
|
||||
double data_mem =
|
||||
(volume * (2 * Nd + 1) * Nd * Nc + (volume / Ls) * 2 * Nd * Nc * Nc) *
|
||||
simdwidth / nsimd * ncall / (1024. * 1024. * 1024.);
|
||||
|
||||
std::cout << GridLogMessage << "Called Dw " << ncall << " times in "
|
||||
<< t1 - t0 << " us" << std::endl;
|
||||
(volume * (2 * Nd + 1) * Nd * Nc + (volume / Ls) * 2 * Nd * Nc * Nc) * simdwidth /
|
||||
nsimd * ncall / (1024. * 1024. * 1024.);
|
||||
|
||||
json["Dw"]["calls"] = ncall;
|
||||
json["Dw"]["time"] = t1 - t0;
|
||||
@ -296,15 +279,16 @@ int main(int argc, char **argv)
|
||||
json["Dw"]["RF"] = 1000000. * data_rf / ((t1 - t0));
|
||||
json["Dw"]["mem"] = 1000000. * data_mem / ((t1 - t0));
|
||||
|
||||
// std::cout<<GridLogMessage << "norm result "<<
|
||||
// norm2(result)<<std::endl; std::cout<<GridLogMessage << "norm ref "<<
|
||||
// norm2(ref)<<std::endl;
|
||||
std::cout << GridLogMessage << "mflop/s = " << flops / (t1 - t0)
|
||||
std::cout << GridLogMessage << "Called Dw " << ncall << " times in " << t1 - t0
|
||||
<< " us" << std::endl;
|
||||
// std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
|
||||
// std::cout<<GridLogMessage << "norm ref "<< norm2(ref)<<std::endl;
|
||||
std::cout << GridLogMessage << "mflop/s = " << flops / (t1 - t0) << std::endl;
|
||||
std::cout << GridLogMessage << "mflop/s per rank = " << flops / (t1 - t0) / NP
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "mflop/s per rank = " << flops / (t1 - t0) / NP << std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "mflop/s per node = " << flops / (t1 - t0) / NN << std::endl;
|
||||
std::cout << GridLogMessage << "mflop/s per node = " << flops / (t1 - t0) / NN
|
||||
<< std::endl;
|
||||
|
||||
std::cout << GridLogMessage
|
||||
<< "RF GiB/s (base 2) = " << 1000000. * data_rf / ((t1 - t0))
|
||||
<< std::endl;
|
||||
@ -381,13 +365,12 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// dump=1;
|
||||
Dw.Dhop(src, result, 1);
|
||||
std::cout
|
||||
<< GridLogMessage
|
||||
<< "Compare to naive wilson implementation Dag to verify correctness"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "Called DwDag" << std::endl;
|
||||
std::cout << GridLogMessage << "norm dag result " << norm2(result)
|
||||
|
||||
std::cout << GridLogMessage
|
||||
<< "Compare to naive wilson implementation Dag to verify correctness"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "Called DwDag" << std::endl;
|
||||
std::cout << GridLogMessage << "norm dag result " << norm2(result) << std::endl;
|
||||
std::cout << GridLogMessage << "norm dag ref " << norm2(ref) << std::endl;
|
||||
err = ref - result;
|
||||
std::cout << GridLogMessage << "norm dag diff " << norm2(err) << std::endl;
|
||||
@ -405,8 +388,7 @@ int main(int argc, char **argv)
|
||||
LatticeFermionF r_o(FrbGrid);
|
||||
LatticeFermionF r_eo(FGrid);
|
||||
|
||||
std::cout << GridLogMessage
|
||||
<< "Calling Deo and Doe and //assert Deo+Doe == Dunprec"
|
||||
std::cout << GridLogMessage << "Calling Deo and Doe and //assert Deo+Doe == Dunprec"
|
||||
<< std::endl;
|
||||
pickCheckerboard(Even, src_e, src);
|
||||
pickCheckerboard(Odd, src_o, src);
|
||||
@ -416,37 +398,32 @@ int main(int argc, char **argv)
|
||||
|
||||
// S-direction is INNERMOST and takes no part in the parity.
|
||||
std::cout << GridLogMessage
|
||||
<< "*********************************************************"
|
||||
<< std::endl;
|
||||
|
||||
<< "*********************************************************" << std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "* Benchmarking DomainWallFermionF::DhopEO "
|
||||
<< "* Benchmarking DomainWallFermionF::DhopEO " << std::endl;
|
||||
std::cout << GridLogMessage << "* Vectorising space-time by " << vComplexF::Nsimd()
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Vectorising space-time by "
|
||||
<< vComplexF::Nsimd() << std::endl;
|
||||
|
||||
if (sizeof(RealF) == 4)
|
||||
std::cout << GridLogMessage << "* SINGLE precision " << std::endl;
|
||||
if (sizeof(RealF) == 8)
|
||||
std::cout << GridLogMessage << "* DOUBLE precision " << std::endl;
|
||||
#ifdef GRID_OMP
|
||||
if (WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute)
|
||||
std::cout << GridLogMessage << "* Using Overlapped Comms/Compute"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using Overlapped Comms/Compute" << std::endl;
|
||||
if (WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute)
|
||||
std::cout << GridLogMessage << "* Using sequential comms compute"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using sequential comms compute" << std::endl;
|
||||
#endif
|
||||
if (WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric)
|
||||
std::cout << GridLogMessage << "* Using GENERIC Nc WilsonKernels"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using GENERIC Nc WilsonKernels" << std::endl;
|
||||
if (WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll)
|
||||
std::cout << GridLogMessage << "* Using Nc=3 WilsonKernels"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using Nc=3 WilsonKernels" << std::endl;
|
||||
if (WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm)
|
||||
std::cout << GridLogMessage << "* Using Asm Nc=3 WilsonKernels"
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Using Asm Nc=3 WilsonKernels" << std::endl;
|
||||
std::cout << GridLogMessage
|
||||
<< "*********************************************************"
|
||||
<< std::endl;
|
||||
<< "*********************************************************" << std::endl;
|
||||
|
||||
{
|
||||
Dw.ZeroCounters();
|
||||
FGrid->Barrier();
|
||||
@ -478,12 +455,12 @@ int main(int argc, char **argv)
|
||||
json["Deo"]["mflops_per_rank"] = flops / (t1 - t0) / NP;
|
||||
json["Deo"]["mflops_per_node"] = flops / (t1 - t0) / NN;
|
||||
|
||||
std::cout << GridLogMessage << "Deo mflop/s = " << flops / (t1 - t0)
|
||||
std::cout << GridLogMessage << "Deo mflop/s = " << flops / (t1 - t0) << std::endl;
|
||||
std::cout << GridLogMessage << "Deo mflop/s per rank " << flops / (t1 - t0) / NP
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "Deo mflop/s per rank "
|
||||
<< flops / (t1 - t0) / NP << std::endl;
|
||||
std::cout << GridLogMessage << "Deo mflop/s per node "
|
||||
<< flops / (t1 - t0) / NN << std::endl;
|
||||
std::cout << GridLogMessage << "Deo mflop/s per node " << flops / (t1 - t0) / NN
|
||||
<< std::endl;
|
||||
|
||||
Dw.Report();
|
||||
}
|
||||
Dw.DhopEO(src_o, r_e, DaggerNo);
|
||||
@ -510,18 +487,16 @@ int main(int argc, char **argv)
|
||||
|
||||
pickCheckerboard(Even, src_e, err);
|
||||
pickCheckerboard(Odd, src_o, err);
|
||||
std::cout << GridLogMessage << "norm diff even " << norm2(src_e)
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "norm diff odd " << norm2(src_o)
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "norm diff even " << norm2(src_e) << std::endl;
|
||||
std::cout << GridLogMessage << "norm diff odd " << norm2(src_o) << std::endl;
|
||||
|
||||
assert(norm2(src_e) < 1.0e-4);
|
||||
assert(norm2(src_o) < 1.0e-4);
|
||||
|
||||
if (!json_filename.empty())
|
||||
{
|
||||
std::cout << GridLogMessage << "writing benchmark results to "
|
||||
<< json_filename << std::endl;
|
||||
std::cout << GridLogMessage << "writing benchmark results to " << json_filename
|
||||
<< std::endl;
|
||||
|
||||
int me = 0;
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &me);
|
||||
|
||||
Reference in New Issue
Block a user