diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc index 01335b41..305a3a9b 100644 --- a/Grid/communicator/Communicator_mpi3.cc +++ b/Grid/communicator/Communicator_mpi3.cc @@ -389,7 +389,6 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vectorShmBufferTranslate(dest,recv); assert(shm!=NULL); acceleratorCopyDeviceToDeviceAsynch(xmit,shm,bytes); - acceleratorCopySynchronise(); // MPI prob slower } if ( CommunicatorPolicy == CommunicatorPolicySequential ) { @@ -405,6 +404,7 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector status(nreq); + acceleratorCopySynchronise(); int ierr = MPI_Waitall(nreq,&list[0],&status[0]); assert(ierr==0); list.resize(0); diff --git a/Grid/lattice/Lattice_reduction_gpu.h b/Grid/lattice/Lattice_reduction_gpu.h index d8a47ae1..c2875052 100644 --- a/Grid/lattice/Lattice_reduction_gpu.h +++ b/Grid/lattice/Lattice_reduction_gpu.h @@ -42,7 +42,6 @@ void getNumBlocksAndThreads(const Iterator n, const size_t sizeofsobj, Iterator std::cout << GridLogDebug << "\twarpSize = " << warpSize << std::endl; std::cout << GridLogDebug << "\tsharedMemPerBlock = " << sharedMemPerBlock << std::endl; std::cout << GridLogDebug << "\tmaxThreadsPerBlock = " << maxThreadsPerBlock << std::endl; - std::cout << GridLogDebug << "\tmaxThreadsPerBlock = " << warpSize << std::endl; std::cout << GridLogDebug << "\tmultiProcessorCount = " << multiProcessorCount << std::endl; if (warpSize != WARP_SIZE) { @@ -52,6 +51,10 @@ void getNumBlocksAndThreads(const Iterator n, const size_t sizeofsobj, Iterator // let the number of threads in a block be a multiple of 2, starting from warpSize threads = warpSize; + if ( threads*sizeofsobj > sharedMemPerBlock ) { + std::cout << GridLogError << "The object is too large for the shared memory." << std::endl; + exit(EXIT_FAILURE); + } while( 2*threads*sizeofsobj < sharedMemPerBlock && 2*threads <= maxThreadsPerBlock ) threads *= 2; // keep all the streaming multiprocessors busy blocks = nextPow2(multiProcessorCount); diff --git a/Grid/threads/Accelerator.h b/Grid/threads/Accelerator.h index 2c08b76b..cec0600f 100644 --- a/Grid/threads/Accelerator.h +++ b/Grid/threads/Accelerator.h @@ -95,6 +95,7 @@ void acceleratorInit(void); ////////////////////////////////////////////// #ifdef GRID_CUDA + #include #ifdef __CUDA_ARCH__