diff --git a/lib/lattice/Lattice_transfer.h b/lib/lattice/Lattice_transfer.h index dd03fb4f..3d9289d6 100644 --- a/lib/lattice/Lattice_transfer.h +++ b/lib/lattice/Lattice_transfer.h @@ -890,50 +890,85 @@ void Grid_split(std::vector > & full,Lattice & split) if ( ratio[d] != 1 ) { full_grid ->AllToAll(d,alldata,tmpdata); - // std::cout << GridLogMessage << "Grid_split: dim " <_processors[d]<_processors[d] > 1 ) { + alldata=tmpdata; + split_grid->AllToAll(d,alldata,tmpdata); + } - auto rdims = ldims; rdims[d] *= ratio[d]; - auto rsites= lsites*ratio[d]; - for(int v=0;v_processors[d]; + int fP = full_grid->_processors[d]; - for(int r=0;r_processors[d] > 1 ) { - tmpdata = alldata; - split_grid->AllToAll(d,tmpdata,alldata); - } } } vectorizeFromLexOrdArray(alldata,split); @@ -1008,55 +1043,84 @@ void Grid_unsplit(std::vector > & full,Lattice & split) std::vector rcoor(ndim); int nvec = 1; - lsites = split_grid->lSites(); - std::vector ldims = split_grid->_ldimensions; + uint64_t rsites = split_grid->lSites(); + std::vector rdims = split_grid->_ldimensions; - // for(int d=ndim-1;d>=0;d--){ for(int d=0;d_processors[d]; + int fP = full_grid->_processors[d]; - if ( split_grid->_processors[d] > 1 ) { - tmpdata = alldata; - split_grid->AllToAll(d,tmpdata,alldata); - } + int M = ratio[d]; + auto ldims = rdims; ldims[d] /= M; // Decrease local dims by same factor + auto lsites= rsites/M; // Decreases rsites by M - ////////////////////////////////////////// - //Local volume for this dimension is expanded by ratio of processor extents - // Number of vectors is decreased by same factor - // Rearrange to lexico for bigger volume - ////////////////////////////////////////// - auto rsites= lsites/ratio[d]; - auto rdims = ldims; rdims[d]/=ratio[d]; + int fvol = lsites; + int svol = rsites; + int chunk = (nvec*fvol)/sP; + int cL = (nvec*ldims[d])/sP; + + for(int c=0;c= tmpdata.size() ) { - // rsite, rcoor --> smaller local volume - // lsite, lcoor --> bigger original (single node?) volume - // For loop over each site within smaller subvol - for(int rsite=0;rsiteAllToAll(d,tmpdata,alldata); + if ( split_grid->_processors[d] > 1 ) { + split_grid->AllToAll(d,tmpdata,alldata); + tmpdata=alldata; + } + full_grid ->AllToAll(d,tmpdata,alldata); + + rdims[d]/= M; + rsites /= M; + nvec *= M; // Increase nvec by subdivision factor + } } } @@ -1064,12 +1128,13 @@ void Grid_unsplit(std::vector > & full,Lattice & split) for(int v=0;v