portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-09-04 16:59:36 +01:00
Code Releases Activity

Improved FFT -- 1.5-2x when there are 2-6 ranks in a given axis of the cartesian communicator.

Browse Source 
Barrel shift -> all to all (x2) and distributed FFT work fully load balanced without redundant work.
There is little more I can do now on FFT. Comms dominated and running distributed work dividing bandwidth optimal RingAllToAll

/ccs/home/paboyle/ParallelIO/systems/Frontier/tests/core/Test_fft_prop --mpi 3.6.4.4 --grid 48.48.48.96 --accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem
32000 --log Error,Warning,Message,Performance

*************************************************
 Benchmarking FFT of LatticeFermionD on plane wave
*************************************************
Grid : Performance : 0.501524 s :  FFT took     0.001311 s (transpose P=3)
Grid : Performance : 0.501531 s :  FFT pack     5.9e-05 s
Grid : Performance : 0.501533 s :  FFT alltoall 0.000828 s
Grid : Performance : 0.501534 s :  FFT reorder  0.000204 s
Grid : Performance : 0.501535 s :  FFT kernels  1e-05 s
Grid : Performance : 0.501536 s :  FFT unpack   5e-05 s
Grid : Performance : 0.509992 s :  FFT took     0.001829 s (transpose P=6)
Grid : Performance : 0.510000 s :  FFT pack     6e-05 s
Grid : Performance : 0.510002 s :  FFT alltoall 0.001436 s
Grid : Performance : 0.510003 s :  FFT reorder  0.000202 s
Grid : Performance : 0.510005 s :  FFT kernels  9e-06 s
Grid : Performance : 0.510006 s :  FFT unpack   5.3e-05 s
Grid : Performance : 0.517690 s :  FFT took     0.001599 s (transpose P=4)
Grid : Performance : 0.517698 s :  FFT pack     6e-05 s
Grid : Performance : 0.517700 s :  FFT alltoall 0.001258 s
Grid : Performance : 0.517701 s :  FFT reorder  0.0002 s
Grid : Performance : 0.517702 s :  FFT kernels  9e-06 s
Grid : Performance : 0.517703 s :  FFT unpack   4.9e-05 s
Grid : Performance : 0.524858 s :  FFT took     0.001561 s (transpose P=4)
Grid : Performance : 0.524865 s :  FFT pack     5.8e-05 s
Grid : Performance : 0.524867 s :  FFT alltoall 0.001213 s
Grid : Performance : 0.524868 s :  FFT reorder  0.000209 s
Grid : Performance : 0.524869 s :  FFT kernels  8e-06 s
Grid : Performance : 0.524870 s :  FFT unpack   4.9e-05 s
*************************************************
 FFT of [48 48 48 96] LatticeFermionD took 0.030916 s
*************************************************
This commit is contained in:
Peter Boyle
2026-09-03 17:33:32 -04:00
parent e767694b82
commit 357ede3664
5 changed files with 295 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/debug/Test_ring_allreduce.cc
+24
View File
@@ -204,6 +204,30 @@ int main(int argc, char **argv)
}
}
// T8: single-dimension ALL-TO-ALL. Fingerprint block j (destined for coord j)
// on the rank at coord m with a tag encoding the (source,dest) coordinate pair,
// src*Pd+dst; after the exchange block i (from coord i) must carry the tag the
// coord-i rank prepared for me, i.e. Fill(e, i*Pd + m). Analytic reference, so
// a coordinate<->rank relabelling bug (wrong ShiftedRanks partner) shows up as a
// mismatched source coordinate rather than silently passing.
{
int Nd=grid->_ndimension;
for(int d=0; d<Nd; d++){
int Pd=grid->_processors[d]; if ( Pd==1 ) continue;
int m=grid->_processor_coor[d];
uint64_t chunk=1009, n=chunk*(uint64_t)Pd;
std::vector<ComplexD> sh(n), rh(n), ref(n);
for(int j=0;j<Pd;j++) for(uint64_t e=0;e<chunk;e++) sh[j*chunk+e]=Fill<ComplexD>(e, m*Pd+j);
for(int i=0;i<Pd;i++) for(uint64_t e=0;e<chunk;e++) ref[i*chunk+e]=Fill<ComplexD>(e, i*Pd+m);
deviceVector<ComplexD> sd(n), rd(n);
acceleratorCopyToDevice(&sh[0],&sd[0],n*sizeof(ComplexD));
CartesianRingAllToAll(grid,&sd[0],&rd[0],chunk,d);
acceleratorCopyFromDevice(&rd[0],&rh[0],n*sizeof(ComplexD));
RealD diff=(memcmp(&rh[0],&ref[0],n*sizeof(ComplexD))!=0)?1.0:0.0; grid->GlobalSum(diff);
Report("T8 CartesianRingAllToAll(dim="+std::to_string(d)+") bitwise == analytic reference, P_d="+std::to_string(Pd), diff==0.0);
}
}
// T4 timing at 16 MB of ComplexF (the dense-apply size at 12 RHS is 13.3 MB)
{
uint64_t n = 2*1024*1024;