diff --git a/Grid/algorithms/FFT.h b/Grid/algorithms/FFT.h index 5b5ba223a..79396b55b 100644 --- a/Grid/algorithms/FFT.h +++ b/Grid/algorithms/FFT.h @@ -428,8 +428,13 @@ static void FFT_dim_execute( scalar *rbuf_v = &rbuf[0]; scalar *pgbuf_v = &pgbuf[0]; - // deterministic ceil-pad slots (never read back, but keeps padded FFT lines finite) - acceleratorMemSet(sbuf_v, 0, nbuf*sizeof(scalar)); + // Pad slots (olin in [Nperp, Oloc*P)) are never packed, so they would carry + // garbage device memory into the FFT. Zero them so the padded lines stay + // finite -- but ONLY when padding is actually present. In the common + // Nperp % P == 0 case pack writes every sbuf entry bijectively, so skip the + // whole-buffer memset and its device sync entirely. + if ( (int64_t)Oloc*P != Nperp ) + acceleratorMemSet(sbuf_v, 0, nbuf*sizeof(scalar)); const Coordinate ldims = grid->_ldimensions; const Coordinate rdims = grid->_rdimensions; diff --git a/tests/core/Test_fft_prop.cc b/tests/core/Test_fft_prop.cc index 39961242c..72bcbd4a2 100644 --- a/tests/core/Test_fft_prop.cc +++ b/tests/core/Test_fft_prop.cc @@ -33,43 +33,77 @@ using namespace Grid; template void bench(GridCartesian *grid, std::string name) { - LatticeComplexD C(grid); - LatticeComplexD coor(grid); - - ComplexD ci(0.0,1.0); - Coordinate p({1,2,3,4}); - Coordinate latt_size = grid->_fdimensions; std::cout<<"*************************************************"<({1,2,3,4})); + LatticeObject S(grid); gaussian(RNG,S); + + typedef typename LatticeObject::vector_object vobj; + const int nrep = 10; + + // Correctness + Parseval pass (unplanned), also the WARMUP that absorbs + // FFTW/allocator/MPI first-touch so the two timed passes below are both warm + // and the comparison is fair (order confound removed). + { + LatticeObject Stilde(grid); Stilde=S; + FFT theFFT(grid); + std::cout << " norm2(s) "< theFFT(grid); // one-time plan build (all dims, fwd+bwd) + tc+= usecond(); + t_build = tc/1.e6; + double tt= -usecond(); + for(int r=0;r plan create+destroy "<<(t_unplanned - t_planned)<<" s/call" + <<" (one-time build "<(&GRID,std::string("LatticeComplexD")); bench(&GRID,std::string("LatticeColourMatrixD")); + bench(&GRID,std::string("LatticeFermionD")); // Ncomp=12, the FreePropagator/Fourier-precon path bench(&GRID,std::string("LatticePropagatorD")); Grid_finalize();