From 233150d93f7c9829a2a00d6da087d85c00ae161d Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 24 Apr 2025 11:40:46 -0400 Subject: [PATCH] Bug fix for no accelerator aware MPI, thanks Shuhei for finding it. --- Grid/lattice/PaddedCell.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Grid/lattice/PaddedCell.h b/Grid/lattice/PaddedCell.h index bdb0301c..0340698c 100644 --- a/Grid/lattice/PaddedCell.h +++ b/Grid/lattice/PaddedCell.h @@ -510,7 +510,6 @@ public: grid->SendToRecvFromBegin(fwd_req, (void *)&hsend_buf[d*buffer_size], xmit_to_rank, (void *)&hrecv_buf[d*buffer_size], recv_from_rank, bytes, tag); - acceleratorCopyToDevice(&hrecv_buf[d*buffer_size],&recv_buf[d*buffer_size],bytes); #endif t_comms+=usecond()-t; } @@ -531,7 +530,6 @@ public: grid->SendToRecvFromBegin(bwd_req, (void *)&hsend_buf[(d+depth)*buffer_size], recv_from_rank, (void *)&hrecv_buf[(d+depth)*buffer_size], xmit_to_rank, bytes,tag); - acceleratorCopyToDevice(&hrecv_buf[(d+depth)*buffer_size],&recv_buf[(d+depth)*buffer_size],bytes); #endif t_comms+=usecond()-t; } @@ -555,8 +553,13 @@ public: t=usecond(); grid->CommsComplete(fwd_req); +#ifndef ACCELERATOR_AWARE_MPI + for ( int d=0;d < depth ; d ++ ) { + acceleratorCopyToDevice(&hrecv_buf[d*buffer_size],&recv_buf[d*buffer_size],bytes); + } +#endif t_comms+= usecond() - t; - + t=usecond(); for ( int d=0;d < depth ; d ++ ) { ScatterSlice(recv_buf,to,nld-depth+d,dimension,plane*buffer_size); plane++; @@ -565,6 +568,11 @@ public: t=usecond(); grid->CommsComplete(bwd_req); +#ifndef ACCELERATOR_AWARE_MPI + for ( int d=0;d < depth ; d ++ ) { + acceleratorCopyToDevice(&hrecv_buf[(d+depth)*buffer_size],&recv_buf[(d+depth)*buffer_size],bytes); + } +#endif t_comms+= usecond() - t; t=usecond();