Compare commits

...

6 Commits

View File

@ -264,6 +264,85 @@ class Benchmark
return; return;
} }
static void PointToPoint(void)
{
int Nloop = 200;
Coordinate simd_layout = GridDefaultSimd(Nd, vComplexD::Nsimd());
Coordinate mpi_layout = GridDefaultMpi();
std::cout << GridLogMessage << "Benchmarking point-to-point latency" << std::endl;
grid_small_sep();
grid_printf("from to mean(usec) err min\n");
int lat = 8; // dummy lattice size. Not actually used.
Coordinate latt_size({lat * mpi_layout[0], lat * mpi_layout[1], lat * mpi_layout[2],
lat * mpi_layout[3]});
GridCartesian Grid(latt_size, simd_layout, mpi_layout);
int ranks;
int me;
MPI_Comm_size(Grid.communicator, &ranks);
MPI_Comm_rank(Grid.communicator, &me);
assert(ranks == Grid._Nprocessors);
assert(me == Grid._processor);
int bytes = 8;
void *buf_from = acceleratorAllocDevice(bytes);
void *buf_to = acceleratorAllocDevice(bytes);
nlohmann::json json_p2p;
for (int from = 0; from < ranks; ++from)
for (int to = 0; to < ranks; ++to)
{
if (from == to)
continue;
std::vector<double> t_time(Nloop);
time_statistics timestat;
MPI_Status status;
for (int i = 0; i < Nloop; ++i)
{
double start = usecond();
if (from == me)
{
auto err = MPI_Send(buf_from, bytes, MPI_CHAR, to, 0, Grid.communicator);
assert(err == MPI_SUCCESS);
err = MPI_Recv(buf_to, bytes, MPI_CHAR, to, 0, Grid.communicator, &status);
assert(err == MPI_SUCCESS);
}
if (to == me)
{
auto err =
MPI_Recv(buf_to, bytes, MPI_CHAR, from, 0, Grid.communicator, &status);
assert(err == MPI_SUCCESS);
err = MPI_Send(buf_from, bytes, MPI_CHAR, from, 0, Grid.communicator);
assert(err == MPI_SUCCESS);
}
double stop = usecond();
t_time[i] = stop - start;
}
// important: only the 'from' rank has a trustworthy time
MPI_Bcast(t_time.data(), Nloop, MPI_DOUBLE, from, Grid.communicator);
timestat.statistics(t_time);
grid_printf("%2d %2d %15.2f %15.1f %15.2f\n", from, to, timestat.mean,
timestat.err, timestat.min);
nlohmann::json tmp;
tmp["from"] = from;
tmp["to"] = to;
tmp["time_usec"] = timestat.mean;
tmp["time_usec_error"] = timestat.err;
tmp["time_usec_max"] = timestat.min;
json_p2p.push_back(tmp);
}
json_results["latency"] = json_p2p;
acceleratorFreeDevice(buf_from);
acceleratorFreeDevice(buf_to);
}
static void Memory(void) static void Memory(void)
{ {
const int Nvec = 8; const int Nvec = 8;
@ -525,8 +604,6 @@ class Benchmark
FGrid->Broadcast(0, &ncall, sizeof(ncall)); FGrid->Broadcast(0, &ncall, sizeof(ncall));
Dw.ZeroCounters();
time_statistics timestat; time_statistics timestat;
std::vector<double> t_time(ncall); std::vector<double> t_time(ncall);
for (uint64_t i = 0; i < ncall; i++) for (uint64_t i = 0; i < ncall; i++)
@ -721,7 +798,6 @@ class Benchmark
uint64_t ncall = 500; uint64_t ncall = 500;
FGrid->Broadcast(0, &ncall, sizeof(ncall)); FGrid->Broadcast(0, &ncall, sizeof(ncall));
Ds.ZeroCounters();
time_statistics timestat; time_statistics timestat;
std::vector<double> t_time(ncall); std::vector<double> t_time(ncall);
@ -808,6 +884,7 @@ int main(int argc, char **argv)
int do_su4 = 1; int do_su4 = 1;
int do_memory = 1; int do_memory = 1;
int do_comms = 1; int do_comms = 1;
int do_p2p = 1;
int do_flops = 1; int do_flops = 1;
int Ls = 1; int Ls = 1;
@ -843,6 +920,14 @@ int main(int argc, char **argv)
Benchmark::Comms(); Benchmark::Comms();
} }
if (do_p2p)
{
grid_big_sep();
std::cout << GridLogMessage << " Point-to-Point benchmark " << std::endl;
grid_big_sep();
Benchmark::PointToPoint();
}
if (do_flops) if (do_flops)
{ {
Ls = 1; Ls = 1;