diff --git a/.travis.yml b/.travis.yml index de9c54d3..ae3efda8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,6 @@ matrix: - os: osx osx_image: xcode7.2 compiler: clang - - os: osx - osx_image: xcode7.2 - compiler: gcc - env: VERSION=-5 - compiler: gcc addons: apt: diff --git a/Makefile.am b/Makefile.am index 470de83b..818f0983 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,5 +3,10 @@ SUBDIRS = lib benchmarks tests include $(top_srcdir)/doxygen.inc +tests: all + $(MAKE) -C tests tests + +.PHONY: tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL) + AM_CXXFLAGS += -I$(top_builddir)/include ACLOCAL_AMFLAGS = -I m4 diff --git a/README b/README deleted file mode 100644 index 17e92fa0..00000000 --- a/README +++ /dev/null @@ -1,44 +0,0 @@ -This library provides data parallel C++ container classes with internal memory layout -that is transformed to map efficiently to SIMD architectures. CSHIFT facilities -are provided, similar to HPF and cmfortran, and user control is given over the mapping of -array indices to both MPI tasks and SIMD processing elements. - -* Identically shaped arrays then be processed with perfect data parallelisation. -* Such identically shapped arrays are called conformable arrays. - -The transformation is based on the observation that Cartesian array processing involves -identical processing to be performed on different regions of the Cartesian array. - -The library will (eventually) both geometrically decompose into MPI tasks and across SIMD lanes. - -Data parallel array operations can then be specified with a SINGLE data parallel paradigm, but -optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a significant simplification -for most programmers. - -The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture. -Presently SSE2 (128 bit) AVX, AVX2 (256 bit) and IMCI and AVX512 (512 bit) targets are supported. - -These are presented as - - vRealF, vRealD, vComplexF, vComplexD - -internal vector data types. These may be useful in themselves for other programmers. -The corresponding scalar types are named - - RealF, RealD, ComplexF, ComplexD - -MPI parallelism is UNIMPLEMENTED and for now only OpenMP and SIMD parallelism is present in the library. - - You can give `configure' initial values for configuration parameters -by setting variables in the command line or in the environment. Here -is are examples: - - ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -msse4" --enable-simd=SSE4 - - ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -mavx" --enable-simd=AVX1 - - ./configure CXX=clang++ CXXFLAGS="-std=c++11 -O3 -mavx2" --enable-simd=AVX2 - - ./configure CXX=icpc CXXFLAGS="-std=c++11 -O3 -mmic" --enable-simd=AVX512 --host=none - - diff --git a/README b/README new file mode 120000 index 00000000..42061c01 --- /dev/null +++ b/README @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/README.md b/README.md index 2a0acad6..f4a376f1 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,27 @@ **Data parallel C++ mathematical object library.** -Please send all pull requests to the `develop` branch. - License: GPL v2. -Last update 2016/08/03. +Last update Nov 2016. + +_Please do not send pull requests to the `master` branch which is reserved for releases._ + +### Bug report + +_To help us tracking and solving more efficiently issues with Grid, please report problems using the issue system of GitHub rather than sending emails to Grid developers._ + +When you file an issue, please go though the following checklist: + +1. Check that the code is pointing to the `HEAD` of `develop` or any commit in `master` which is tagged with a version number. +2. Give a description of the target platform (CPU, network, compiler). Please give the full CPU part description, using for example `cat /proc/cpuinfo | grep 'model name' | uniq` (Linux) or `sysctl machdep.cpu.brand_string` (macOS) and the full output the `--version` option of your compiler. +3. Give the exact `configure` command used. +4. Attach `config.log`. +5. Attach `config.summary`. +6. Attach the output of `make V=1`. +7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example. + + ### Description This library provides data parallel C++ container classes with internal memory layout @@ -29,7 +45,7 @@ are provided, similar to HPF and cmfortran, and user control is given over the m array indices to both MPI tasks and SIMD processing elements. * Identically shaped arrays then be processed with perfect data parallelisation. -* Such identically shapped arrays are called conformable arrays. +* Such identically shaped arrays are called conformable arrays. The transformation is based on the observation that Cartesian array processing involves identical processing to be performed on different regions of the Cartesian array. @@ -42,7 +58,7 @@ optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a signifi for most programmers. The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture. -Presently SSE4 (128 bit) AVX, AVX2 (256 bit) and IMCI and AVX512 (512 bit) targets are supported (ARM NEON and BG/Q QPX on the way). +Presently SSE4 (128 bit) AVX, AVX2, QPX (256 bit), IMCI, and AVX512 (512 bit) targets are supported (ARM NEON on the way). These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. These may be useful in themselves for other programmers. The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`. @@ -50,7 +66,7 @@ The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `Compl MPI, OpenMP, and SIMD parallelism are present in the library. Please see https://arxiv.org/abs/1512.03487 for more detail. -### Installation +### Quick start First, start by cloning the repository: ``` bash @@ -71,12 +87,10 @@ mkdir build; cd build ../configure --enable-precision=double --enable-simd=AVX --enable-comms=mpi-auto --prefix= ``` -where `--enable-precision=` set the default precision (`single` or `double`), -`--enable-simd=` set the SIMD type (see possible values below), `--enable- -comms=` set the protocol used for communications (`none`, `mpi`, `mpi-auto` or -`shmem`), and `` should be replaced by the prefix path where you want to -install Grid. The `mpi-auto` communication option set `configure` to determine -automatically how to link to MPI. Other options are available, use `configure +where `--enable-precision=` set the default precision, +`--enable-simd=` set the SIMD type, `--enable- +comms=`, and `` should be replaced by the prefix path where you want to +install Grid. Other options are detailed in the next section, you can also use `configure --help` to display them. Like with any other program using GNU autotool, the `CXX`, `CXXFLAGS`, `LDFLAGS`, ... environment variables can be modified to customise the build. @@ -92,25 +106,88 @@ To minimise the build time, only the tests at the root of the `tests` directory ``` bash make -C tests/ tests ``` +If you want to build all the tests at once just use `make tests`. + +### Build configuration options + +- `--prefix=`: installation prefix for Grid. +- `--with-gmp=`: look for GMP in the UNIX prefix `` +- `--with-mpfr=`: look for MPFR in the UNIX prefix `` +- `--with-fftw=`: look for FFTW in the UNIX prefix `` +- `--enable-lapack[=]`: enable LAPACK support in Lanczos eigensolver. A UNIX prefix containing the library can be specified (optional). +- `--enable-mkl[=]`: use Intel MKL for FFT (and LAPACK if enabled) routines. A UNIX prefix containing the library can be specified (optional). +- `--enable-numa`: ??? +- `--enable-simd=`: setup Grid for the SIMD target `` (default: `GEN`). A list of possible SIMD targets is detailed in a section below. +- `--enable-precision={single|double}`: set the default precision (default: `double`). +- `--enable-precision=`: Use `` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below. +- `--enable-rng={ranlux48|mt19937}`: choose the RNG (default: `ranlux48 `). +- `--disable-timers`: disable system dependent high-resolution timers. +- `--enable-chroma`: enable Chroma regression tests. + +### Possible communication interfaces + +The following options can be use with the `--enable-comms=` option to target different communication interfaces: + +| `` | Description | +| -------------- | ------------------------------------------------------------- | +| `none` | no communications | +| `mpi[-auto]` | MPI communications | +| `mpi3[-auto]` | MPI communications using MPI 3 shared memory | +| `mpi3l[-auto]` | MPI communications using MPI 3 shared memory and leader model | +| `shmem ` | Cray SHMEM communications | + +For the MPI interfaces the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names). ### Possible SIMD types The following options can be use with the `--enable-simd=` option to target different SIMD instruction sets: -| String | Description | +| `` | Description | | ----------- | -------------------------------------- | | `GEN` | generic portable vector code | | `SSE4` | SSE 4.2 (128 bit) | | `AVX` | AVX (256 bit) | -| `AVXFMA4` | AVX (256 bit) + FMA | +| `AVXFMA` | AVX (256 bit) + FMA | +| `AVXFMA4` | AVX (256 bit) + FMA4 | | `AVX2` | AVX 2 (256 bit) | | `AVX512` | AVX 512 bit | -| `AVX512MIC` | AVX 512 bit for Intel MIC architecture | -| `ICMI` | Intel ICMI instructions (512 bit) | +| `QPX` | QPX (256 bit) | Alternatively, some CPU codenames can be directly used: -| String | Description | +| `` | Description | | ----------- | -------------------------------------- | -| `KNC` | [Intel Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) | -| `KNL` | [Intel Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) | \ No newline at end of file +| `KNC` | [Intel Xeon Phi codename Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) | +| `KNL` | [Intel Xeon Phi codename Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) | +| `BGQ` | Blue Gene/Q | + +#### Notes: +- We currently support AVX512 only for the Intel compiler. Support for GCC and clang will appear in future versions of Grid when the AVX512 support within GCC and clang will be more advanced. +- For BG/Q only [bgclang](http://trac.alcf.anl.gov/projects/llvm-bgq) is supported. We do not presently plan to support more compilers for this platform. +- BG/Q performances are currently rather poor. This is being investigated for future versions. + +### Build setup for Intel Knights Landing platform + +The following configuration is recommended for the Intel Knights Landing platform: + +``` bash +../configure --enable-precision=double\ + --enable-simd=KNL \ + --enable-comms=mpi-auto \ + --with-gmp= \ + --with-mpfr= \ + --enable-mkl \ + CXX=icpc MPICXX=mpiicpc +``` + +where `` is the UNIX prefix where GMP and MPFR are installed. If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use: + +``` bash +../configure --enable-precision=double\ + --enable-simd=KNL \ + --enable-comms=mpi \ + --with-gmp= \ + --with-mpfr= \ + --enable-mkl \ + CXX=CC CC=cc +``` \ No newline at end of file diff --git a/VERSION b/VERSION index c12f9497..e7abbba7 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,6 @@ -Version : 0.5.0 +Version : 0.6.0 - AVX512, AVX2, AVX, SSE good - Clang 3.5 and above, ICPC v16 and above, GCC 4.9 and above +- MPI and MPI3 +- HiRep, Smearing, Generic gauge group diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc index d0cd96c7..969a2a42 100644 --- a/benchmarks/Benchmark_comms.cc +++ b/benchmarks/Benchmark_comms.cc @@ -42,15 +42,14 @@ int main (int argc, char ** argv) int Nloop=10; int nmu=0; - for(int mu=0;mu<4;mu++) if (mpi_layout[mu]>1) nmu++; + for(int mu=0;mu1) nmu++; + std::cout< latt_size ({lat*mpi_layout[0], @@ -125,7 +124,7 @@ int main (int argc, char ** argv) std::cout< latt_size ({lat,lat,lat,lat}); @@ -194,128 +193,169 @@ int main (int argc, char ** argv) } } -#if 0 + Nloop=100; std::cout< latt_size ({lat,lat,lat,lat}); + std::vector latt_size ({lat*mpi_layout[0], + lat*mpi_layout[1], + lat*mpi_layout[2], + lat*mpi_layout[3]}); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - std::vector > xbuf(8,std::vector(lat*lat*lat*Ls)); - std::vector > rbuf(8,std::vector(lat*lat*lat*Ls)); - + std::vector xbuf(8); + std::vector rbuf(8); + Grid.ShmBufferFreeAll(); + for(int d=0;d<8;d++){ + xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + } int ncomm; int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); + double start=usecond(); + for(int i=0;i empty; - std::vector > requests_fwd(Nd,empty); - std::vector > requests_bwd(Nd,empty); + std::vector requests; - for(int mu=0;mu<4;mu++){ ncomm=0; - if (mpi_layout[mu]>1 ) { - ncomm++; - - int comm_proc; - int xmit_to_rank; - int recv_from_rank; - - comm_proc=1; - Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); - Grid.SendToRecvFromInit(requests_fwd[mu], - (void *)&xbuf[mu][0], - xmit_to_rank, - (void *)&rbuf[mu][0], - recv_from_rank, - bytes); - - comm_proc = mpi_layout[mu]-1; - Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); - Grid.SendToRecvFromInit(requests_bwd[mu], - (void *)&xbuf[mu+4][0], - xmit_to_rank, - (void *)&rbuf[mu+4][0], - recv_from_rank, - bytes); - - } - } - - { - double start=usecond(); - for(int i=0;i1 ) { - for(int mu=0;mu<4;mu++){ + ncomm++; + int comm_proc=1; + int xmit_to_rank; + int recv_from_rank; - if (mpi_layout[mu]>1 ) { - - Grid.SendToRecvFromBegin(requests_fwd[mu]); - Grid.SendToRecvFromComplete(requests_fwd[mu]); - Grid.SendToRecvFromBegin(requests_bwd[mu]); - Grid.SendToRecvFromComplete(requests_bwd[mu]); - } - } - Grid.Barrier(); - } + Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); + Grid.StencilSendToRecvFromBegin(requests, + (void *)&xbuf[mu][0], + xmit_to_rank, + (void *)&rbuf[mu][0], + recv_from_rank, + bytes); - double stop=usecond(); - - double dbytes = bytes; - double xbytes = Nloop*dbytes*2.0*ncomm; - double rbytes = xbytes; - double bidibytes = xbytes+rbytes; - - double time = stop-start; - - std::cout<1 ) { - - Grid.SendToRecvFromBegin(requests_fwd[mu]); - Grid.SendToRecvFromBegin(requests_bwd[mu]); - Grid.SendToRecvFromComplete(requests_fwd[mu]); - Grid.SendToRecvFromComplete(requests_bwd[mu]); - } } - Grid.Barrier(); } - - double stop=usecond(); - - double dbytes = bytes; - double xbytes = Nloop*dbytes*2.0*ncomm; - double rbytes = xbytes; - double bidibytes = xbytes+rbytes; - - double time = stop-start; - - std::cout< latt_size ({lat*mpi_layout[0], + lat*mpi_layout[1], + lat*mpi_layout[2], + lat*mpi_layout[3]}); + + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + + std::vector xbuf(8); + std::vector rbuf(8); + Grid.ShmBufferFreeAll(); + for(int d=0;d<8;d++){ + xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + } + + int ncomm; + int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); + + double start=usecond(); + for(int i=0;i requests; + + ncomm=0; + for(int mu=0;mu<4;mu++){ + + if (mpi_layout[mu]>1 ) { + + ncomm++; + int comm_proc=1; + int xmit_to_rank; + int recv_from_rank; + + Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); + Grid.StencilSendToRecvFromBegin(requests, + (void *)&xbuf[mu][0], + xmit_to_rank, + (void *)&rbuf[mu][0], + recv_from_rank, + bytes); + // Grid.StencilSendToRecvFromComplete(requests); + // requests.resize(0); + + comm_proc = mpi_layout[mu]-1; + + Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); + Grid.StencilSendToRecvFromBegin(requests, + (void *)&xbuf[mu+4][0], + xmit_to_rank, + (void *)&rbuf[mu+4][0], + recv_from_rank, + bytes); + Grid.StencilSendToRecvFromComplete(requests); + requests.resize(0); + + } + } + Grid.Barrier(); + + } + double stop=usecond(); + + double dbytes = bytes; + double xbytes = Nloop*dbytes*2.0*ncomm; + double rbytes = xbytes; + double bidibytes = xbytes+rbytes; + + double time = stop-start; // microseconds + + std::cout< WilsonFermion5DR; typedef WilsonFermion5D WilsonFermion5DF; typedef WilsonFermion5D WilsonFermion5DD; @@ -54,15 +53,11 @@ int main (int argc, char ** argv) { Grid_init(&argc,&argv); - if( GridCmdOptionExists(argv,argv+argc,"--asynch") ){ - overlapComms = true; - } - int threads = GridThread::GetThreads(); std::cout< latt4 = GridDefaultLatt(); - const int Ls=16; + const int Ls=8; GridCartesian * UGrid = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi()); GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid); GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid); @@ -126,17 +121,24 @@ int main (int argc, char ** argv) RealD NP = UGrid->_Nprocessors; - for(int doasm=1;doasm<2;doasm++){ - - QCD::WilsonKernelsStatic::AsmOpt=doasm; - DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5); - - std::cout<Barrier(); Dw.ZeroCounters(); double t0=usecond(); for(int i=0;iBarrier(); double volume=Ls; for(int mu=0;mu::Dhop "< WilsonFermion5DR; LatticeFermion ssrc(sFGrid); LatticeFermion sref(sFGrid); @@ -179,6 +194,7 @@ int main (int argc, char ** argv) pokeSite(tmp,ssrc,site); }}}}} std::cout<Barrier(); double t0=usecond(); sDw.ZeroCounters(); for(int i=0;iBarrier(); double volume=Ls; for(int mu=0;mu 1.0e-6 ) { - std::cout << "site "<::DhopEO "<Barrier(); sDw.ZeroCounters(); + sDw.stat.init("DhopEO"); double t0=usecond(); for (int i = 0; i < ncall; i++) { sDw.DhopEO(ssrc_o, sr_e, DaggerNo); } double t1=usecond(); + FGrid->Barrier(); + sDw.stat.print(); double volume=Ls; for(int mu=0;mu1.0e-4) { + setCheckerboard(ssrc,ssrc_o); + setCheckerboard(ssrc,ssrc_e); + std::cout<< ssrc << std::endl; + } } @@ -299,13 +336,13 @@ int main (int argc, char ** argv) ref = -0.5*ref; } Dw.Dhop(src,result,1); - std::cout << GridLogMessage << "Naive wilson implementation Dag" << std::endl; + std::cout << GridLogMessage << "Compare to naive wilson implementation Dag to verify correctness" << std::endl; std::cout<Barrier(); double t0=usecond(); for(int i=0;iBarrier(); double volume=Ls; for(int mu=0;mu -Author: paboyle - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution directory - *************************************************************************************/ - /* END LEGAL */ -#include - -using namespace std; -using namespace Grid; -using namespace Grid::QCD; - -template -struct scal { - d internal; -}; - - Gamma::GammaMatrix Gmu [] = { - Gamma::GammaX, - Gamma::GammaY, - Gamma::GammaZ, - Gamma::GammaT - }; - -bool overlapComms = false; - - -int main (int argc, char ** argv) -{ - Grid_init(&argc,&argv); - - if( GridCmdOptionExists(argv,argv+argc,"--asynch") ){ - overlapComms = true; - } - - int threads = GridThread::GetThreads(); - std::cout< latt4 = GridDefaultLatt(); - const int Ls=16; - GridCartesian * UGrid = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi()); - GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid); - GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid); - GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid); - - std::vector seeds4({1,2,3,4}); - std::vector seeds5({5,6,7,8}); - - GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); - GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5); - - LatticeFermion src (FGrid); random(RNG5,src); - LatticeFermion result(FGrid); result=zero; - LatticeFermion ref(FGrid); ref=zero; - LatticeFermion tmp(FGrid); - LatticeFermion err(FGrid); - - ColourMatrix cm = Complex(1.0,0.0); - - LatticeGaugeField Umu(UGrid); - random(RNG4,Umu); - - LatticeGaugeField Umu5d(FGrid); - - // replicate across fifth dimension - for(int ss=0;ssoSites();ss++){ - for(int s=0;s U(4,FGrid); - for(int mu=0;mu(Umu5d,mu); - } - - if (1) - { - ref = zero; - for(int mu=0;mu_Nprocessors; - - - QCD::WilsonKernelsStatic::AsmOpt=1; - - DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,params); - - std::cout< seeds({1,2,3,4}); RealD mass = 0.1; + std::cout << GridLogMessage<< "*****************************************************************" < - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution directory - *************************************************************************************/ - /* END LEGAL */ -#include - - -using namespace Grid; -using namespace Grid::QCD; - - -int bench(std::ofstream &os, std::vector &latt4,int Ls); - -int main(int argc,char **argv) -{ - Grid_init(&argc,&argv); - std::ofstream os("zmm.dat"); - - os << "#V Ls Lxy Lzt C++ Asm OMP L1 " < grid({L,L,m*L,m*L}); - std::cout << GridLogMessage <<"\t"; - for(int i=0;i<4;i++) { - std::cout << grid[i]<<"x"; - } - std::cout << Ls<<"\t\t"; - bench(os,grid,Ls); - } - } - } -} - -int bench(std::ofstream &os, std::vector &latt4,int Ls) -{ - - GridCartesian * UGrid = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi()); - GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid); - GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid); - GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid); - - std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); - std::vector mpi_layout = GridDefaultMpi(); - int threads = GridThread::GetThreads(); - - std::vector seeds4({1,2,3,4}); - std::vector seeds5({5,6,7,8}); - - GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds4); - - LatticeFermion src (FGrid); - LatticeFermion tmp (FGrid); - LatticeFermion srce(FrbGrid); - - LatticeFermion resulto(FrbGrid); resulto=zero; - LatticeFermion resulta(FrbGrid); resulta=zero; - LatticeFermion junk(FrbGrid); junk=zero; - LatticeFermion diff(FrbGrid); - LatticeGaugeField Umu(UGrid); - - double mfc, mfa, mfo, mfl1; - - GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); - GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5); - random(RNG5,src); -#if 1 - random(RNG4,Umu); -#else - int mmu=2; - std::vector U(4,UGrid); - for(int mu=0;mu(Umu,mu); - if ( mu!=mmu ) U[mu] = zero; - if ( mu==mmu ) U[mu] = 1.0; - PokeIndex(Umu,U[mu],mu); - } -#endif - pickCheckerboard(Even,srce,src); - - RealD mass=0.1; - RealD M5 =1.8; - DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5); - - int ncall=50; - double t0=usecond(); - for(int i=0;i + +Grid::vRealD add(const Grid::vRealD &x, const Grid::vRealD &y) +{ + return x+y; +} + +Grid::vRealD sub(const Grid::vRealD &x, const Grid::vRealD &y) +{ + return x-y; +} diff --git a/benchmarks/simple_su3_expr.cc b/benchmarks/simple_su3_expr.cc index 11a4816e..0c6092f6 100644 --- a/benchmarks/simple_su3_expr.cc +++ b/benchmarks/simple_su3_expr.cc @@ -25,7 +25,7 @@ Author: Peter Boyle See the full license in the file "LICENSE" in the top level distribution directory *************************************************************************************/ /* END LEGAL */ -#include +#include using namespace std; using namespace Grid; diff --git a/benchmarks/simple_su3_test.cc b/benchmarks/simple_su3_test.cc index e5cca98d..79437c23 100644 --- a/benchmarks/simple_su3_test.cc +++ b/benchmarks/simple_su3_test.cc @@ -25,7 +25,7 @@ Author: Peter Boyle See the full license in the file "LICENSE" in the top level distribution directory *************************************************************************************/ /* END LEGAL */ -#include +#include using namespace std; using namespace Grid; diff --git a/bootstrap.sh b/bootstrap.sh index f847b7ab..7361161a 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,18 +1,12 @@ #!/usr/bin/env bash EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2' -FFTW_URL=http://www.fftw.org/fftw-3.3.4.tar.gz echo "-- deploying Eigen source..." wget ${EIGEN_URL} --no-check-certificate ./scripts/update_eigen.sh `basename ${EIGEN_URL}` rm `basename ${EIGEN_URL}` -echo "-- copying fftw prototypes..." -wget ${FFTW_URL} -./scripts/update_fftw.sh `basename ${FFTW_URL}` -rm `basename ${FFTW_URL}` - echo '-- generating Make.inc files...' ./scripts/filelist echo '-- generating configure script...' diff --git a/configure.ac b/configure.ac index 50f20bb2..e4aaab8f 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.63]) -AC_INIT([Grid], [0.5.1-dev], [https://github.com/paboyle/Grid], [Grid]) +AC_INIT([Grid], [0.6.0], [https://github.com/paboyle/Grid], [Grid]) AC_CANONICAL_BUILD AC_CANONICAL_HOST AC_CANONICAL_TARGET @@ -9,22 +9,33 @@ AC_CONFIG_SRCDIR([lib/Grid.h]) AC_CONFIG_HEADERS([lib/Config.h]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) - ############### Checks for programs -AC_LANG(C++) CXXFLAGS="-O3 $CXXFLAGS" AC_PROG_CXX AC_PROG_RANLIB -############ openmp ############### +############### Get compiler informations +AC_LANG([C++]) +AX_CXX_COMPILE_STDCXX_11([noext],[mandatory]) +AX_COMPILER_VENDOR +AC_DEFINE_UNQUOTED([CXX_COMP_VENDOR],["$ax_cv_cxx_compiler_vendor"], + [vendor of C++ compiler that will compile the code]) +AX_GXX_VERSION +AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"], + [version of g++ that will compile the code]) + +############### Checks for typedefs, structures, and compiler characteristics +AC_TYPE_SIZE_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T + +############### OpenMP AC_OPENMP - ac_openmp=no - if test "${OPENMP_CXXFLAGS}X" != "X"; then -ac_openmp=yes -AM_CXXFLAGS="$OPENMP_CXXFLAGS $AM_CXXFLAGS" -AM_LDFLAGS="$OPENMP_CXXFLAGS $AM_LDFLAGS" + ac_openmp=yes + AM_CXXFLAGS="$OPENMP_CXXFLAGS $AM_CXXFLAGS" + AM_LDFLAGS="$OPENMP_CXXFLAGS $AM_LDFLAGS" fi ############### Checks for header files @@ -37,12 +48,7 @@ AC_CHECK_HEADERS(execinfo.h) AC_CHECK_DECLS([ntohll],[], [], [[#include ]]) AC_CHECK_DECLS([be64toh],[], [], [[#include ]]) -############### Checks for typedefs, structures, and compiler characteristics -AC_TYPE_SIZE_T -AC_TYPE_UINT32_T -AC_TYPE_UINT64_T - -############### GMP and MPFR ################# +############### GMP and MPFR AC_ARG_WITH([gmp], [AS_HELP_STRING([--with-gmp=prefix], [try this for a non-standard install prefix of the GMP library])], @@ -54,10 +60,17 @@ AC_ARG_WITH([mpfr], [AM_CXXFLAGS="-I$with_mpfr/include $AM_CXXFLAGS"] [AM_LDFLAGS="-L$with_mpfr/lib $AM_LDFLAGS"]) -################## lapack #################### +############### FFTW3 +AC_ARG_WITH([fftw], + [AS_HELP_STRING([--with-fftw=prefix], + [try this for a non-standard install prefix of the FFTW3 library])], + [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"] + [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"]) + +############### lapack AC_ARG_ENABLE([lapack], [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])], - [ac_LAPACK=${enable_lapack}],[ac_LAPACK=no]) + [ac_LAPACK=${enable_lapack}], [ac_LAPACK=no]) case ${ac_LAPACK} in no) @@ -67,59 +80,83 @@ case ${ac_LAPACK} in *) AM_CXXFLAGS="-I$ac_LAPACK/include $AM_CXXFLAGS" AM_LDFLAGS="-L$ac_LAPACK/lib $AM_LDFLAGS" - AC_DEFINE([USE_LAPACK],[1],[use LAPACK]) + AC_DEFINE([USE_LAPACK],[1],[use LAPACK]);; esac -################## FFTW3 #################### -AC_ARG_WITH([fftw], - [AS_HELP_STRING([--with-fftw=prefix], - [try this for a non-standard install prefix of the FFTW3 library])], - [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"] - [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"]) +############### MKL +AC_ARG_ENABLE([mkl], + [AC_HELP_STRING([--enable-mkl=yes|no|prefix], [enable Intel MKL for LAPACK & FFTW])], + [ac_MKL=${enable_mkl}], [ac_MKL=no]) -################ Get compiler informations -AC_LANG([C++]) -AX_CXX_COMPILE_STDCXX_11([noext],[mandatory]) -AX_COMPILER_VENDOR -AC_DEFINE_UNQUOTED([CXX_COMP_VENDOR],["$ax_cv_cxx_compiler_vendor"], - [vendor of C++ compiler that will compile the code]) -AX_GXX_VERSION -AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"], - [version of g++ that will compile the code]) +case ${ac_MKL} in + no) + ;; + yes) + AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);; + *) + AM_CXXFLAGS="-I$ac_MKL/include $AM_CXXFLAGS" + AM_LDFLAGS="-L$ac_MKL/lib $AM_LDFLAGS" + AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);; +esac + +############### first-touch +AC_ARG_ENABLE([numa], + [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], + [ac_NUMA=${enable_NUMA}],[ac_NUMA=no]) + +case ${ac_NUMA} in + no) + ;; + yes) + AC_DEFINE([GRID_NUMA],[1],[First touch numa locality]);; + *) + AC_DEFINE([GRID_NUMA],[1],[First touch numa locality]);; +esac ############### Checks for library functions CXXFLAGS_CPY=$CXXFLAGS LDFLAGS_CPY=$LDFLAGS CXXFLAGS="$AM_CXXFLAGS $CXXFLAGS" LDFLAGS="$AM_LDFLAGS $LDFLAGS" + AC_CHECK_FUNCS([gettimeofday]) -AC_CHECK_LIB([gmp],[__gmpf_init], - [AC_CHECK_LIB([mpfr],[mpfr_init], - [AC_DEFINE([HAVE_LIBMPFR], [1], [Define to 1 if you have the `MPFR' library (-lmpfr).])] - [have_mpfr=true] - [LIBS="$LIBS -lmpfr"], - [AC_MSG_ERROR([MPFR library not found])])] - [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library (-lgmp).])] - [have_gmp=true] - [LIBS="$LIBS -lgmp"], - [AC_MSG_WARN([**** GMP library not found, Grid can still compile but RHMC will not work ****])]) + +if test "${ac_MKL}x" != "nox"; then + AC_SEARCH_LIBS([mkl_set_interface_layer], [mkl_rt], [], + [AC_MSG_ERROR("MKL enabled but library not found")]) +fi + +AC_SEARCH_LIBS([__gmpf_init], [gmp], + [AC_SEARCH_LIBS([mpfr_init], [mpfr], + [AC_DEFINE([HAVE_LIBMPFR], [1], + [Define to 1 if you have the `MPFR' library])] + [have_mpfr=true], [AC_MSG_ERROR([MPFR library not found])])] + [AC_DEFINE([HAVE_LIBGMP], [1], [Define to 1 if you have the `GMP' library])] + [have_gmp=true]) if test "${ac_LAPACK}x" != "nox"; then - AC_CHECK_LIB([lapack],[LAPACKE_sbdsdc],[], - [AC_MSG_ERROR("LAPACK enabled but library not found")]) -fi -AC_CHECK_LIB([fftw3],[fftw_execute], - [AC_DEFINE([HAVE_FFTW],[1],[Define to 1 if you have the `FFTW' library (-lfftw3).])] - [have_fftw=true] - [LIBS="$LIBS -lfftw3 -lfftw3f"], - [AC_MSG_WARN([**** FFTW library not found, Grid can still compile but FFT-based routines will not work ****])]) + AC_SEARCH_LIBS([LAPACKE_sbdsdc], [lapack], [], + [AC_MSG_ERROR("LAPACK enabled but library not found")]) +fi + +AC_SEARCH_LIBS([fftw_execute], [fftw3], + [AC_SEARCH_LIBS([fftwf_execute], [fftw3f], [], + [AC_MSG_ERROR("single precision FFTW library not found")])] + [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])] + [have_fftw=true]) + CXXFLAGS=$CXXFLAGS_CPY LDFLAGS=$LDFLAGS_CPY ############### SIMD instruction selection -AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=SSE4|AVX|AVXFMA4|AVX2|AVX512|AVX512MIC|IMCI|KNL|KNC],\ - [Select instructions to be SSE4.0, AVX 1.0, AVX 2.0+FMA, AVX 512, IMCI])],\ - [ac_SIMD=${enable_simd}],[ac_SIMD=GEN]) +AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=code], + [select SIMD target (cf. README.md)])], [ac_SIMD=${enable_simd}], [ac_SIMD=GEN]) + +AC_ARG_ENABLE([gen-simd-width], + [AS_HELP_STRING([--enable-gen-simd-width=size], + [size (in bytes) of the generic SIMD vectors (default: 32)])], + [ac_gen_simd_width=$enable_gen_simd_width], + [ac_gen_simd_width=32]) case ${ax_cv_cxx_compiler_vendor} in clang|gnu) @@ -133,17 +170,26 @@ case ${ax_cv_cxx_compiler_vendor} in AVXFMA4) AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4]) SIMD_FLAGS='-mavx -mfma4';; + AVXFMA) + AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA3]) + SIMD_FLAGS='-mavx -mfma';; AVX2) AC_DEFINE([AVX2],[1],[AVX2 intrinsics]) SIMD_FLAGS='-mavx2 -mfma';; - AVX512|AVX512MIC|KNL) + AVX512) AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';; - IMCI|KNC) + KNC) AC_DEFINE([IMCI],[1],[IMCI intrinsics for Knights Corner]) SIMD_FLAGS='';; + KNL) + AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) + SIMD_FLAGS='-march=knl';; GEN) - AC_DEFINE([GENERIC_VEC],[1],[generic vector code]) + AC_DEFINE([GEN],[1],[generic vector code]) + AC_DEFINE_UNQUOTED([GEN_SIMD_WIDTH],[$ac_gen_simd_width], + [generic SIMD vector width (in bytes)]) + SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)" SIMD_FLAGS='';; QPX|BGQ) AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q]) @@ -159,8 +205,8 @@ case ${ax_cv_cxx_compiler_vendor} in AVX) AC_DEFINE([AVX1],[1],[AVX intrinsics]) SIMD_FLAGS='-mavx -xavx';; - AVXFMA4) - AC_DEFINE([AVXFMA4],[1],[AVX intrinsics with FMA4]) + AVXFMA) + AC_DEFINE([AVXFMA],[1],[AVX intrinsics with FMA4]) SIMD_FLAGS='-mavx -mfma';; AVX2) AC_DEFINE([AVX2],[1],[AVX2 intrinsics]) @@ -168,14 +214,17 @@ case ${ax_cv_cxx_compiler_vendor} in AVX512) AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) SIMD_FLAGS='-xcore-avx512';; - AVX512MIC|KNL) - AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing]) - SIMD_FLAGS='-xmic-avx512';; - IMCI|KNC) + KNC) AC_DEFINE([IMCI],[1],[IMCI Intrinsics for Knights Corner]) SIMD_FLAGS='';; + KNL) + AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing]) + SIMD_FLAGS='-xmic-avx512';; GEN) - AC_DEFINE([GENERIC_VEC],[1],[generic vector code]) + AC_DEFINE([GEN],[1],[generic vector code]) + AC_DEFINE([GEN_SIMD_WIDTH],[$ac_gen_simd_width], + [generic SIMD vector width (in bytes)]) + SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)" SIMD_FLAGS='';; *) AC_MSG_ERROR(["SIMD option ${ac_SIMD} not supported by the Intel compiler"]);; @@ -188,14 +237,18 @@ AM_CXXFLAGS="$SIMD_FLAGS $AM_CXXFLAGS" AM_CFLAGS="$SIMD_FLAGS $AM_CFLAGS" case ${ac_SIMD} in - AVX512|AVX512MIC|KNL) + AVX512|KNL) AC_DEFINE([TEST_ZMM],[1],[compile ZMM test]);; *) ;; esac -############### precision selection -AC_ARG_ENABLE([precision],[AC_HELP_STRING([--enable-precision=single|double],[Select default word size of Real])],[ac_PRECISION=${enable_precision}],[ac_PRECISION=double]) +############### Precision selection +AC_ARG_ENABLE([precision], + [AC_HELP_STRING([--enable-precision=single|double], + [Select default word size of Real])], + [ac_PRECISION=${enable_precision}],[ac_PRECISION=double]) + case ${ac_PRECISION} in single) AC_DEFINE([GRID_DEFAULT_PRECISION_SINGLE],[1],[GRID_DEFAULT_PRECISION is SINGLE] ) @@ -206,39 +259,56 @@ case ${ac_PRECISION} in esac ############### communication type selection -AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|shmem],[Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none]) +AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi3-auto|shmem], + [Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none]) case ${ac_COMMS} in none) - AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] ) + AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] ) + comms_type='none' ;; - mpi-auto) - AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] ) - LX_FIND_MPI - if test "x$have_CXX_mpi" = 'xno'; then AC_MSG_ERROR(["MPI not found"]); fi - AM_CXXFLAGS="$MPI_CXXFLAGS $AM_CXXFLAGS" - AM_CFLAGS="$MPI_CFLAGS $AM_CFLAGS" - AM_LDFLAGS="`echo $MPI_CXXLDFLAGS | sed -E 's/-l@<:@^ @:>@+//g'` $AM_LDFLAGS" - LIBS="`echo $MPI_CXXLDFLAGS | sed -E 's/-L@<:@^ @:>@+//g'` $LIBS" + mpi3l*) + AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] ) + comms_type='mpi3l' ;; - mpi) - AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] ) + mpi3*) + AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] ) + comms_type='mpi3' + ;; + mpi*) + AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] ) + comms_type='mpi' ;; shmem) - AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] ) + AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] ) + comms_type='shmem' ;; *) - AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); + AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]); ;; esac -AM_CONDITIONAL(BUILD_COMMS_SHMEM,[ test "X${ac_COMMS}X" == "XshmemX" ]) -AM_CONDITIONAL(BUILD_COMMS_MPI,[ test "X${ac_COMMS}X" == "XmpiX" || test "X${ac_COMMS}X" == "Xmpi-autoX" ]) -AM_CONDITIONAL(BUILD_COMMS_NONE,[ test "X${ac_COMMS}X" == "XnoneX" ]) +case ${ac_COMMS} in + *-auto) + LX_FIND_MPI + if test "x$have_CXX_mpi" = 'xno'; then AC_MSG_ERROR(["MPI not found"]); fi + AM_CXXFLAGS="$MPI_CXXFLAGS $AM_CXXFLAGS" + AM_CFLAGS="$MPI_CFLAGS $AM_CFLAGS" + AM_LDFLAGS="`echo $MPI_CXXLDFLAGS | sed -E 's/-l@<:@^ @:>@+//g'` $AM_LDFLAGS" + LIBS="`echo $MPI_CXXLDFLAGS | sed -E 's/-L@<:@^ @:>@+//g'` $LIBS";; + *) + ;; +esac + +AM_CONDITIONAL(BUILD_COMMS_SHMEM, [ test "${comms_type}X" == "shmemX" ]) +AM_CONDITIONAL(BUILD_COMMS_MPI, [ test "${comms_type}X" == "mpiX" ]) +AM_CONDITIONAL(BUILD_COMMS_MPI3, [ test "${comms_type}X" == "mpi3X" ] ) +AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] ) +AM_CONDITIONAL(BUILD_COMMS_NONE, [ test "${comms_type}X" == "noneX" ]) ############### RNG selection AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937],\ - [Select Random Number Generator to be used])],\ - [ac_RNG=${enable_rng}],[ac_RNG=ranlux48]) + [Select Random Number Generator to be used])],\ + [ac_RNG=${enable_rng}],[ac_RNG=ranlux48]) case ${ac_RNG} in ranlux48) @@ -252,10 +322,11 @@ case ${ac_RNG} in ;; esac -############### timer option +############### Timer option AC_ARG_ENABLE([timers],[AC_HELP_STRING([--enable-timers],\ - [Enable system dependent high res timers])],\ - [ac_TIMERS=${enable_timers}],[ac_TIMERS=yes]) + [Enable system dependent high res timers])],\ + [ac_TIMERS=${enable_timers}],[ac_TIMERS=yes]) + case ${ac_TIMERS} in yes) AC_DEFINE([TIMERS_ON],[1],[TIMERS_ON] ) @@ -269,7 +340,9 @@ case ${ac_TIMERS} in esac ############### Chroma regression test -AC_ARG_ENABLE([chroma],[AC_HELP_STRING([--enable-chroma],[Expect chroma compiled under c++11 ])],ac_CHROMA=yes,ac_CHROMA=no) +AC_ARG_ENABLE([chroma],[AC_HELP_STRING([--enable-chroma], + [Expect chroma compiled under c++11 ])],ac_CHROMA=yes,ac_CHROMA=no) + case ${ac_CHROMA} in yes|no) ;; @@ -277,6 +350,7 @@ case ${ac_CHROMA} in AC_MSG_ERROR([${ac_CHROMA} unsupported --enable-chroma option]); ;; esac + AM_CONDITIONAL(BUILD_CHROMA_REGRESSION,[ test "X${ac_CHROMA}X" == "XyesX" ]) ############### Doxygen @@ -313,34 +387,35 @@ AC_CONFIG_FILES(tests/qdpxx/Makefile) AC_CONFIG_FILES(benchmarks/Makefile) AC_OUTPUT -echo " -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Summary of configuration for $PACKAGE v$VERSION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ----- PLATFORM ---------------------------------------- -- architecture (build) : $build_cpu -- os (build) : $build_os -- architecture (target) : $target_cpu -- os (target) : $target_os -- compiler vendor : ${ax_cv_cxx_compiler_vendor} -- compiler version : ${ax_cv_gxx_version} +architecture (build) : $build_cpu +os (build) : $build_os +architecture (target) : $target_cpu +os (target) : $target_os +compiler vendor : ${ax_cv_cxx_compiler_vendor} +compiler version : ${ax_cv_gxx_version} ----- BUILD OPTIONS ----------------------------------- -- SIMD : ${ac_SIMD} -- Threading : ${ac_openmp} -- Communications type : ${ac_COMMS} -- Default precision : ${ac_PRECISION} -- RNG choice : ${ac_RNG} -- GMP : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi` -- LAPACK : ${ac_LAPACK} -- FFTW : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi` -- build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi` +SIMD : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG} +Threading : ${ac_openmp} +Communications type : ${comms_type} +Default precision : ${ac_PRECISION} +RNG choice : ${ac_RNG} +GMP : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi` +LAPACK : ${ac_LAPACK} +FFTW : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi` +build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi` ----- BUILD FLAGS ------------------------------------- -- CXXFLAGS: +CXXFLAGS: `echo ${AM_CXXFLAGS} ${CXXFLAGS} | tr ' ' '\n' | sed 's/^-/ -/g'` -- LDFLAGS: +LDFLAGS: `echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/ -/g'` -- LIBS: +LIBS: `echo ${LIBS} | tr ' ' '\n' | sed 's/^-/ -/g'` -------------------------------------------------------- -" +-------------------------------------------------------" > config.summary +echo "" +cat config.summary +echo "" diff --git a/doxygen.cfg.bak b/doxygen.cfg.bak deleted file mode 100644 index 473077c6..00000000 --- a/doxygen.cfg.bak +++ /dev/null @@ -1,1130 +0,0 @@ -# Doxyfile 1.3.5 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -# IMPORTANT: This file was adapted to be used in combination with GNU's -# autotools. The generated "configure" and "Makefile" set all the environment -# variables mentioned below. You should change their values by specifying -# appropriate flags to "configure", rather than replacing them with a fixed -# value. Also, see the NOTE: comments for specific details on using the -# environment variables for the settings. - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -# NOTE: These are taken from the configure.ac file. The output looks better -# when placing the version in the PROJECT_NAME. Thje PROJECT_NUMBER seems -# intended more for things like the RCS version. -PROJECT_NAME = $(PROJECT) - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = $(VERSION) - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -# NOTE: This is taken from the DX_DOCDIR variable (set by calling -# DX_INIT_DOXYGEN). The Makefile relies on it being there. -OUTPUT_DIRECTORY = $(DOCDIR) - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, -# Finnish, French, German, Greek, Hungarian, Italian, Japanese, Japanese-en -# (Japanese with English messages), Korean, Norwegian, Polish, Portuguese, -# Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is used -# as the annotated text. Otherwise, the brief description is used as-is. If left -# blank, the following values are used ("$name" is automatically replaced with the -# name of the entity): "The $name class" "The $name widget" "The $name file" -# "is" "provides" "specifies" "contains" "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = YES - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -# NOTE: This is taken from the srcdir variable, which "make distcheck" plays -# with. You may need to add subdirectory(s) here, depending on your directory -# structure. But you almost certainly want the SRCDIR itself to be stripped. -STRIP_FROM_PATH = $(SRCDIR) - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = YES - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = YES - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = YES - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = YES - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 0 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = YES - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -# NOTE: You do want to limit the search for specific sub-directories. -# If you "make distcheck" and things go wrong, there's a leftover -# ($PROJECT)-$VERSION) file containing a copy of all the sources. If you don't -# limit the search, and forget to clean up, then doxygen start seeing double... -# It is important to root all paths with SRCDIR - again, because of "make -# distcheck". So usually this would be something like "$(SRCDIR)/src". -INPUT = $(SRCDIR) - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl *.cs *.php *.php3 *.inc - -FILE_PATTERNS = - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = build gcc-bug-report doxygen.inc - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = */lib/Eigen/* build* *.inc *.md - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command , where -# is the value of the INPUT_FILTER tag, and is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = YES - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = YES - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -# NOTE: You can turn this off by running "configure --disable-doxygen-html". -GENERATE_HTML = $(GENERATE_HTML) - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -# NOTE: The Makefile depends on this being 'html'. -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -# NOTE: You can turn this off by running "configure --disable-doxygen-chm". -GENERATE_HTMLHELP = $(GENERATE_HTMLHELP) - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -# NOTE: This seems to be relative to the HTML directory. The Makefile depends -# on it ending up being directly in the OUTPUT_DIRECTORY. Since DOCDIR is -# relative, using '..' seems the only safe way to specify this. -CHM_FILE = ../$(PROJECT).chm - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -# NOTE: configure finds HHC automatically for you; if it can't, it will turn -# CHM generation off. -HHC_LOCATION = $(HHC_PATH) - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -# NOTE: You can generate the CHI by running "configure # --doxygen-enable-chi" -GENERATE_CHI = $(GENERATE_CHI) - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = YES - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = YES - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -# NOTE: This is automatically YES if you generate PS or PDF output. You can -# disable it by running "configure --disable-doxygen-ps --disable-doxygen-pdf" -GENERATE_LATEX = $(GENERATE_LATEX) - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -# NOTE: The Makefile depends on this being 'latex'. -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -# NOTE: It doesn't really matter, because the Makefile generated by Doxygen is -# not used. Explicit rules in the Automake generated Makefile handle building -# the documentation instead. -LATEX_CMD_NAME = - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -# NOTE: It doesn't really matter, because the Makefile generated by Doxygen is -# not used. Explicit rules in the Automake generated Makefile handle building -# the documentation instead. -MAKEINDEX_CMD_NAME = - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = YES - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -# NOTE: You can control this using "configure DOXYGEN_PAPER_SIZE=a4" etc. -PAPER_TYPE = $(PAPER_SIZE) - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -# NOTE: You can disable this by running "configure --disable-doxygen-pdf" -USE_PDFLATEX = $(GENERATE_PDF) - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = YES - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -# NOTE: You can enable this by running "configure --enable-doxygen-rtf" -GENERATE_RTF = $(GENERATE_RTF) - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -# NOTE: The Makefile depends on this being 'rtf'. -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = YES - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -# NOTE: You can enable this by running "configure --enable-doxygen-man" -GENERATE_MAN = $(GENERATE_MAN) - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -# NOTE: The Makefile depends on this being 'man'. -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -# NOTE: You can enable this by running "configure --enable-doxygen-xml" -GENERATE_XML = $(GENERATE_XML) - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -# NOTE: The Makefile depends on this being 'xml'. -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -# NOTE: There's no support for this in the Makefile yet. -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -# NOTE: There's no support for this in the Makefile yet. -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -# NOTE: There's no support for this in the Makefile yet. -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = YES - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = YES - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = . - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -# NOTE: This allows hiding code from Doxygen using #ifndef _DOXYGEN. -PREDEFINED = _DOXYGEN - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = EASY_TYPE EASY_DESTRUCTOR EASY_METHOD - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -# NOTE: The Makefile relies on this file, and it is good practice to provide it -# anyway. -GENERATE_TAGFILE = $(DOCDIR)/$(PROJECT).tag - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -# NOTE: Not important (since we don't PERLMOD?) -PERL_PATH = /usr/bin/false - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superseded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -# NOTE: This is computed automatically by "configure". Use the --disable-dot -# flag if you have it but don't want to use it. -HAVE_DOT = $(HAVE_DOT) - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -# NOTE: Computed automatically by "configure". If not found, HAVE_DOT is -# automatically set to NO. -DOT_PATH = $(DOT_PATH) - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes that -# lay further from the root node will be omitted. Note that setting this option to -# 1 or 2 may greatly reduce the computation time needed for large code bases. Also -# note that a graph may be further truncated if the graph's image dimensions are -# not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH and MAX_DOT_GRAPH_HEIGHT). -# If 0 is used for the depth value (the default), the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/doxygen.inc b/doxygen.inc index 420049ec..120509d2 100644 --- a/doxygen.inc +++ b/doxygen.inc @@ -158,8 +158,6 @@ DX_CLEAN_LATEX = @DX_DOCDIR@/latex endif DX_COND_latex -.PHONY: doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL) - .INTERMEDIATE: doxygen-run $(DX_PS_GOAL) $(DX_PDF_GOAL) doxygen-run: @DX_DOCDIR@/@PACKAGE@.tag diff --git a/lib/AlignedAllocator.h b/lib/AlignedAllocator.h index 2cd8263d..a8b9c53b 100644 --- a/lib/AlignedAllocator.h +++ b/lib/AlignedAllocator.h @@ -40,14 +40,6 @@ Author: Peter Boyle #include #endif -#ifdef GRID_COMMS_SHMEM -extern "C" { -#include -extern void * shmem_align(size_t, size_t); -extern void shmem_free(void *); -} -#endif - namespace Grid { //////////////////////////////////////////////////////////////////// @@ -65,28 +57,85 @@ public: typedef _Tp value_type; template struct rebind { typedef alignedAllocator<_Tp1> other; }; - alignedAllocator() throw() { } - alignedAllocator(const alignedAllocator&) throw() { } - template alignedAllocator(const alignedAllocator<_Tp1>&) throw() { } - ~alignedAllocator() throw() { } - pointer address(reference __x) const { return &__x; } - // const_pointer address(const_reference __x) const { return &__x; } - size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); } pointer allocate(size_type __n, const void* _p= 0) { +#ifdef HAVE_MM_MALLOC_H + _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128); +#else + _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp)); +#endif + + _Tp tmp; +#ifdef GRID_NUMA +#pragma omp parallel for schedule(static) + for(int i=0;i<__n;i++){ + ptr[i]=tmp; + } +#endif + return ptr; + } + + void deallocate(pointer __p, size_type) { +#ifdef HAVE_MM_MALLOC_H + _mm_free((void *)__p); +#else + free((void *)__p); +#endif + } + void construct(pointer __p, const _Tp& __val) { }; + void construct(pointer __p) { }; + void destroy(pointer __p) { }; +}; +template inline bool operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; } +template inline bool operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; } + +////////////////////////////////////////////////////////////////////////////////////////// +// MPI3 : comms must use shm region +// SHMEM: comms must use symmetric heap +////////////////////////////////////////////////////////////////////////////////////////// #ifdef GRID_COMMS_SHMEM - - _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64); - - +extern "C" { +#include +extern void * shmem_align(size_t, size_t); +extern void shmem_free(void *); +} #define PARANOID_SYMMETRIC_HEAP +#endif + +template +class commAllocator { +public: + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef _Tp* pointer; + typedef const _Tp* const_pointer; + typedef _Tp& reference; + typedef const _Tp& const_reference; + typedef _Tp value_type; + + template struct rebind { typedef commAllocator<_Tp1> other; }; + commAllocator() throw() { } + commAllocator(const commAllocator&) throw() { } + template commAllocator(const commAllocator<_Tp1>&) throw() { } + ~commAllocator() throw() { } + pointer address(reference __x) const { return &__x; } + size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); } + +#ifdef GRID_COMMS_SHMEM + pointer allocate(size_type __n, const void* _p= 0) + { +#ifdef CRAY + _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64); +#else + _Tp *ptr = (_Tp *) shmem_align(64,__n*sizeof(_Tp)); +#endif #ifdef PARANOID_SYMMETRIC_HEAP static void * bcast; static long psync[_SHMEM_REDUCE_SYNC_SIZE]; @@ -96,55 +145,47 @@ public: if ( bcast != ptr ) { std::printf("inconsistent alloc pe %d %lx %lx \n",shmem_my_pe(),bcast,ptr);std::fflush(stdout); - BACKTRACEFILE(); + // BACKTRACEFILE(); exit(0); } - assert( bcast == (void *) ptr); - #endif + return ptr; + } + void deallocate(pointer __p, size_type) { + shmem_free((void *)__p); + } #else - + pointer allocate(size_type __n, const void* _p= 0) + { #ifdef HAVE_MM_MALLOC_H _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128); #else _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp)); #endif - -#endif - _Tp tmp; -#undef FIRST_TOUCH_OPTIMISE -#ifdef FIRST_TOUCH_OPTIMISE -#pragma omp parallel for - for(int i=0;i<__n;i++){ - ptr[i]=tmp; - } -#endif return ptr; } - void deallocate(pointer __p, size_type) { -#ifdef GRID_COMMS_SHMEM - shmem_free((void *)__p); -#else #ifdef HAVE_MM_MALLOC_H _mm_free((void *)__p); #else free((void *)__p); -#endif #endif } +#endif void construct(pointer __p, const _Tp& __val) { }; void construct(pointer __p) { }; - void destroy(pointer __p) { }; }; +template inline bool operator==(const commAllocator<_Tp>&, const commAllocator<_Tp>&){ return true; } +template inline bool operator!=(const commAllocator<_Tp>&, const commAllocator<_Tp>&){ return false; } -template inline bool -operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; } - -template inline bool -operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; } +//////////////////////////////////////////////////////////////////////////////// +// Template typedefs +//////////////////////////////////////////////////////////////////////////////// +template using Vector = std::vector >; +template using commVector = std::vector >; +template using Matrix = std::vector > >; }; // namespace Grid #endif diff --git a/lib/Cshift.h b/lib/Cshift.h index bbc56cae..cd162e35 100644 --- a/lib/Cshift.h +++ b/lib/Cshift.h @@ -38,6 +38,14 @@ Author: Peter Boyle #include #endif +#ifdef GRID_COMMS_MPI3 +#include +#endif + +#ifdef GRID_COMMS_MPI3L +#include +#endif + #ifdef GRID_COMMS_SHMEM #include // uses same implementation of communicator #endif diff --git a/lib/FFT.h b/lib/FFT.h index 4cda6483..b5b31d82 100644 --- a/lib/FFT.h +++ b/lib/FFT.h @@ -29,9 +29,15 @@ Author: Peter Boyle #ifndef _GRID_FFT_H_ #define _GRID_FFT_H_ -#ifdef HAVE_FFTW +#ifdef HAVE_FFTW +#ifdef USE_MKL +#include +#else #include #endif +#endif + + namespace Grid { template struct FFTW { }; @@ -98,179 +104,199 @@ namespace Grid { #define FFTW_BACKWARD (+1) #endif - class FFT { + class FFT { private: - + GridCartesian *vgrid; GridCartesian *sgrid; - + int Nd; double flops; double flops_call; uint64_t usec; - + std::vector dimensions; std::vector processors; std::vector processor_coor; - + public: - + static const int forward=FFTW_FORWARD; static const int backward=FFTW_BACKWARD; - + double Flops(void) {return flops;} double MFlops(void) {return flops/usec;} + double USec(void) {return (double)usec;} - FFT ( GridCartesian * grid ) : - vgrid(grid), - Nd(grid->_ndimension), - dimensions(grid->_fdimensions), - processors(grid->_processors), - processor_coor(grid->_processor_coor) + FFT ( GridCartesian * grid ) : + vgrid(grid), + Nd(grid->_ndimension), + dimensions(grid->_fdimensions), + processors(grid->_processors), + processor_coor(grid->_processor_coor) { flops=0; usec =0; std::vector layout(Nd,1); sgrid = new GridCartesian(dimensions,layout,processors); }; - - ~FFT ( void) { - delete sgrid; + + ~FFT ( void) { + delete sgrid; } template - void FFT_dim(Lattice &result,const Lattice &source,int dim, int inverse){ + void FFT_dim_mask(Lattice &result,const Lattice &source,std::vector mask,int sign){ conformable(result._grid,vgrid); conformable(source._grid,vgrid); + Lattice tmp(vgrid); + tmp = source; + for(int d=0;d + void FFT_all_dim(Lattice &result,const Lattice &source,int sign){ + std::vector mask(Nd,1); + FFT_dim_mask(result,source,mask,sign); + } + + + template + void FFT_dim(Lattice &result,const Lattice &source,int dim, int sign){ +#ifndef HAVE_FFTW + assert(0); +#else + conformable(result._grid,vgrid); + conformable(source._grid,vgrid); int L = vgrid->_ldimensions[dim]; int G = vgrid->_fdimensions[dim]; - + std::vector layout(Nd,1); std::vector pencil_gd(vgrid->_fdimensions); - - pencil_gd[dim] = G*processors[dim]; - + + pencil_gd[dim] = G*processors[dim]; + // Pencil global vol LxLxGxLxL per node GridCartesian pencil_g(pencil_gd,layout,processors); - + // Construct pencils typedef typename vobj::scalar_object sobj; typedef typename sobj::scalar_type scalar; + + Lattice pgbuf(&pencil_g); + - Lattice ssource(vgrid); ssource =source; - Lattice pgsource(&pencil_g); - Lattice pgresult(&pencil_g); pgresult=zero; - -#ifndef HAVE_FFTW - assert(0); -#else typedef typename FFTW::FFTW_scalar FFTW_scalar; typedef typename FFTW::FFTW_plan FFTW_plan; - - { - int Ncomp = sizeof(sobj)/sizeof(scalar); - int Nlow = 1; - for(int d=0;d_ldimensions[d]; - } - - int rank = 1; /* 1d transforms */ - int n[] = {G}; /* 1d transforms of length G */ - int howmany = Ncomp; - int odist,idist,istride,ostride; - idist = odist = 1; /* Distance between consecutive FT's */ - istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */ - int *inembed = n, *onembed = n; - - - int sign = FFTW_FORWARD; - if (inverse) sign = FFTW_BACKWARD; - - FFTW_plan p; - { - FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[0]; - FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[0]; - p = FFTW::fftw_plan_many_dft(rank,n,howmany, - in,inembed, - istride,idist, - out,onembed, - ostride, odist, - sign,FFTW_ESTIMATE); - } - - double add,mul,fma; - FFTW::fftw_flops(p,&add,&mul,&fma); - flops_call = add+mul+2.0*fma; - - GridStopWatch timer; - - // Barrel shift and collect global pencil - for(int p=0;plSites();idx++) { - - std::vector lcoor(Nd); - sgrid->LocalIndexToLocalCoor(idx,lcoor); - - sobj s; - - peekLocalSite(s,ssource,lcoor); - - lcoor[dim]+=p*L; - - pokeLocalSite(s,pgsource,lcoor); - } - - ssource = Cshift(ssource,dim,L); - } - - // Loop over orthog coords - int NN=pencil_g.lSites(); - - GridStopWatch Timer; - Timer.Start(); - -PARALLEL_FOR_LOOP - for(int idx=0;idx lcoor(Nd); - pencil_g.LocalIndexToLocalCoor(idx,lcoor); - - if ( lcoor[dim] == 0 ) { // restricts loop to plane at lcoor[dim]==0 - FFTW_scalar *in = (FFTW_scalar *)&pgsource._odata[idx]; - FFTW_scalar *out= (FFTW_scalar *)&pgresult._odata[idx]; - FFTW::fftw_execute_dft(p,in,out); - } - } - - Timer.Stop(); - usec += Timer.useconds(); - flops+= flops_call*NN; - - int pc = processor_coor[dim]; - for(int idx=0;idxlSites();idx++) { - std::vector lcoor(Nd); - sgrid->LocalIndexToLocalCoor(idx,lcoor); - std::vector gcoor = lcoor; - // extract the result - sobj s; - gcoor[dim] = lcoor[dim]+L*pc; - peekLocalSite(s,pgresult,gcoor); - pokeLocalSite(s,result,lcoor); - } - - FFTW::fftw_destroy_plan(p); + + int Ncomp = sizeof(sobj)/sizeof(scalar); + int Nlow = 1; + for(int d=0;d_ldimensions[d]; } + + int rank = 1; /* 1d transforms */ + int n[] = {G}; /* 1d transforms of length G */ + int howmany = Ncomp; + int odist,idist,istride,ostride; + idist = odist = 1; /* Distance between consecutive FT's */ + istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */ + int *inembed = n, *onembed = n; + + scalar div; + if ( sign == backward ) div = 1.0/G; + else if ( sign == forward ) div = 1.0; + else assert(0); + + FFTW_plan p; + { + FFTW_scalar *in = (FFTW_scalar *)&pgbuf._odata[0]; + FFTW_scalar *out= (FFTW_scalar *)&pgbuf._odata[0]; + p = FFTW::fftw_plan_many_dft(rank,n,howmany, + in,inembed, + istride,idist, + out,onembed, + ostride, odist, + sign,FFTW_ESTIMATE); + } + + // Barrel shift and collect global pencil + std::vector lcoor(Nd), gcoor(Nd); + result = source; + for(int p=0;p cbuf(Nd); + sobj s; + + PARALLEL_FOR_LOOP_INTERN + for(int idx=0;idxlSites();idx++) { + sgrid->LocalIndexToLocalCoor(idx,cbuf); + peekLocalSite(s,result,cbuf); + cbuf[dim]+=p*L; + pokeLocalSite(s,pgbuf,cbuf); + } + } + result = Cshift(result,dim,L); + } + + // Loop over orthog coords + int NN=pencil_g.lSites(); + GridStopWatch timer; + timer.Start(); + PARALLEL_REGION + { + std::vector cbuf(Nd); + + PARALLEL_FOR_LOOP_INTERN + for(int idx=0;idx::fftw_execute_dft(p,in,out); + } + } + } + timer.Stop(); + + // performance counting + double add,mul,fma; + FFTW::fftw_flops(p,&add,&mul,&fma); + flops_call = add+mul+2.0*fma; + usec += timer.useconds(); + flops+= flops_call*NN; + + // writing out result + int pc = processor_coor[dim]; + PARALLEL_REGION + { + std::vector clbuf(Nd), cgbuf(Nd); + sobj s; + + PARALLEL_FOR_LOOP_INTERN + for(int idx=0;idxlSites();idx++) { + sgrid->LocalIndexToLocalCoor(idx,clbuf); + cgbuf = clbuf; + cgbuf[dim] = clbuf[dim]+L*pc; + peekLocalSite(s,pgbuf,cgbuf); + s = s * div; + pokeLocalSite(s,result,clbuf); + } + } + + // destroying plan + FFTW::fftw_destroy_plan(p); #endif - - } - }; - - } #endif diff --git a/lib/Grid.h b/lib/Grid.h index 486ee4d3..0c5983f3 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -77,11 +77,10 @@ Author: paboyle #include #include #include -#include -#include - #include +#include +#include #include #include diff --git a/lib/Init.cc b/lib/Init.cc index 34e503a4..d6d6b9f8 100644 --- a/lib/Init.cc +++ b/lib/Init.cc @@ -44,9 +44,33 @@ Author: paboyle #include #include #include +#include +#include + + +#include +#ifdef __APPLE__ +static int +feenableexcept (unsigned int excepts) +{ + static fenv_t fenv; + unsigned int new_excepts = excepts & FE_ALL_EXCEPT, + old_excepts; // previous masks + + if ( fegetenv (&fenv) ) return -1; + old_excepts = fenv.__control & FE_ALL_EXCEPT; + + // unmask + fenv.__control &= ~new_excepts; + fenv.__mxcsr &= ~(new_excepts << 7); + + return ( fesetenv (&fenv) ? -1 : old_excepts ); +} +#endif namespace Grid { + ////////////////////////////////////////////////////// // Convenience functions to access stadard command line arg // driven parallelism controls @@ -123,6 +147,13 @@ void GridCmdOptionIntVector(std::string &str,std::vector & vec) return; } +void GridCmdOptionInt(std::string &str,int & val) +{ + std::stringstream ss(str); + ss>>val; + return; +} + void GridParseLayout(char **argv,int argc, std::vector &latt, @@ -153,14 +184,12 @@ void GridParseLayout(char **argv,int argc, assert(ompthreads.size()==1); GridThread::SetThreads(ompthreads[0]); } - if( GridCmdOptionExists(argv,argv+argc,"--cores") ){ - std::vector cores(0); + int cores; arg= GridCmdOptionPayload(argv,argv+argc,"--cores"); - GridCmdOptionIntVector(arg,cores); - GridThread::SetCores(cores[0]); + GridCmdOptionInt(arg,cores); + GridThread::SetCores(cores); } - } std::string GridCmdVectorIntToString(const std::vector & vec){ @@ -169,33 +198,40 @@ std::string GridCmdVectorIntToString(const std::vector & vec){ return oss.str(); } ///////////////////////////////////////////////////////// -// +// Reinit guard ///////////////////////////////////////////////////////// +static int Grid_is_initialised = 0; + + void Grid_init(int *argc,char ***argv) { - CartesianCommunicator::Init(argc,argv); - - // Parse command line args. - GridLogger::StopWatch.Start(); std::string arg; + + //////////////////////////////////// + // Shared memory block size + //////////////////////////////////// + if( GridCmdOptionExists(*argv,*argv+*argc,"--shm") ){ + int MB; + arg= GridCmdOptionPayload(*argv,*argv+*argc,"--shm"); + GridCmdOptionInt(arg,MB); + CartesianCommunicator::MAX_MPI_SHM_BYTES = MB*1024*1024; + } + + CartesianCommunicator::Init(argc,argv); + + //////////////////////////////////// + // Logging + //////////////////////////////////// + std::vector logstreams; std::string defaultLog("Error,Warning,Message,Performance"); - GridCmdOptionCSL(defaultLog,logstreams); GridLogConfigure(logstreams); - if( GridCmdOptionExists(*argv,*argv+*argc,"--help") ){ - std::cout< -#endif + void Grid_debug_handler_init(void) { struct sigaction sa,osa; @@ -338,9 +425,9 @@ void Grid_debug_handler_init(void) sa.sa_flags = SA_SIGINFO; sigaction(SIGSEGV,&sa,NULL); sigaction(SIGTRAP,&sa,NULL); -#ifdef GRID_FPE + feenableexcept( FE_INVALID|FE_OVERFLOW|FE_DIVBYZERO); + sigaction(SIGFPE,&sa,NULL); -#endif } } diff --git a/lib/Init.h b/lib/Init.h index 25fda569..6b70d42d 100644 --- a/lib/Init.h +++ b/lib/Init.h @@ -33,6 +33,7 @@ namespace Grid { void Grid_init(int *argc,char ***argv); void Grid_finalize(void); + // internal, controled with --handle void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr); void Grid_debug_handler_init(void); @@ -44,6 +45,7 @@ namespace Grid { const std::vector &GridDefaultMpi(void); const int &GridThreads(void) ; void GridSetThreads(int t) ; + void GridLogTimestamp(int); // Common parsing chores std::string GridCmdOptionPayload(char ** begin, char ** end, const std::string & option); @@ -52,6 +54,7 @@ namespace Grid { void GridCmdOptionCSL(std::string str,std::vector & vec); void GridCmdOptionIntVector(std::string &str,std::vector & vec); + void GridParseLayout(char **argv,int argc, std::vector &latt, std::vector &simd, diff --git a/lib/Log.cc b/lib/Log.cc index 2082570d..7521657b 100644 --- a/lib/Log.cc +++ b/lib/Log.cc @@ -31,11 +31,31 @@ directory /* END LEGAL */ #include +#include + namespace Grid { + std::string demangle(const char* name) { + + int status = -4; // some arbitrary value to eliminate the compiler warning + + // enable c++11 by passing the flag -std=c++11 to g++ + std::unique_ptr res { + abi::__cxa_demangle(name, NULL, NULL, &status), + std::free + }; + + return (status==0) ? res.get() : name ; + } + GridStopWatch Logger::StopWatch; +int Logger::timestamp; std::ostream Logger::devnull(0); +void GridLogTimestamp(int on){ + Logger::Timestamp(on); +} + Colours GridLogColours(0); GridLogger GridLogError(1, "Error", GridLogColours, "RED"); GridLogger GridLogWarning(1, "Warning", GridLogColours, "YELLOW"); @@ -73,7 +93,7 @@ void GridLogConfigure(std::vector &logstreams) { //////////////////////////////////////////////////////////// void Grid_quiesce_nodes(void) { int me = 0; -#ifdef GRID_COMMS_MPI +#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPI3L) MPI_Comm_rank(MPI_COMM_WORLD, &me); #endif #ifdef GRID_COMMS_SHMEM diff --git a/lib/Log.h b/lib/Log.h index 156f52ee..d7422ca9 100644 --- a/lib/Log.h +++ b/lib/Log.h @@ -37,10 +37,11 @@ #include #endif - namespace Grid { +namespace Grid { +////////////////////////////////////////////////////////////////////////////////////////////////// // Dress the output; use std::chrono for time stamping via the StopWatch class -int Rank(void); // used for early stage debug before library init +////////////////////////////////////////////////////////////////////////////////////////////////// class Colours{ @@ -55,7 +56,6 @@ public: void Active(bool activate){ is_active=activate; - if (is_active){ colour["BLACK"] ="\033[30m"; colour["RED"] ="\033[31m"; @@ -66,21 +66,18 @@ public: colour["CYAN"] ="\033[36m"; colour["WHITE"] ="\033[37m"; colour["NORMAL"] ="\033[0;39m"; - } else { - colour["BLACK"] =""; - colour["RED"] =""; - colour["GREEN"] =""; - colour["YELLOW"]=""; - colour["BLUE"] =""; - colour["PURPLE"]=""; - colour["CYAN"] =""; - colour["WHITE"] =""; - colour["NORMAL"]=""; - } - - -}; - + } else { + colour["BLACK"] =""; + colour["RED"] =""; + colour["GREEN"] =""; + colour["YELLOW"]=""; + colour["BLUE"] =""; + colour["PURPLE"]=""; + colour["CYAN"] =""; + colour["WHITE"] =""; + colour["NORMAL"]=""; + } + }; }; @@ -88,6 +85,7 @@ class Logger { protected: Colours &Painter; int active; + static int timestamp; std::string name, topName; std::string COLOUR; @@ -99,25 +97,28 @@ public: std::string evidence() {return Painter.colour["YELLOW"];} std::string colour() {return Painter.colour[COLOUR];} - Logger(std::string topNm, int on, std::string nm, Colours& col_class, std::string col) - : active(on), - name(nm), - topName(topNm), - Painter(col_class), - COLOUR(col){} ; + Logger(std::string topNm, int on, std::string nm, Colours& col_class, std::string col) : active(on), + name(nm), + topName(topNm), + Painter(col_class), + COLOUR(col) {} ; void Active(int on) {active = on;}; int isActive(void) {return active;}; + static void Timestamp(int on) {timestamp = on;}; friend std::ostream& operator<< (std::ostream& stream, Logger& log){ if ( log.active ) { - StopWatch.Stop(); - GridTime now = StopWatch.Elapsed(); - StopWatch.Start(); stream << log.background()<< log.topName << log.background()<< " : "; stream << log.colour() < #else #include #endif +#ifdef __x86_64__ +#include +#endif namespace Grid { @@ -86,7 +89,6 @@ inline uint64_t cyclecount(void){ return tmp; } #elif defined __x86_64__ -#include inline uint64_t cyclecount(void){ return __rdtsc(); // unsigned int dummy; diff --git a/lib/Simd.h b/lib/Simd.h index 9568c555..adc2849d 100644 --- a/lib/Simd.h +++ b/lib/Simd.h @@ -237,6 +237,18 @@ namespace Grid { stream<<">"; return stream; } + inline std::ostream& operator<< (std::ostream& stream, const vInteger &o){ + int nn=vInteger::Nsimd(); + std::vector > buf(nn); + vstore(o,&buf[0]); + stream<<"<"; + for(int i=0;i"; + return stream; + } } diff --git a/lib/Stat.cc b/lib/Stat.cc new file mode 100644 index 00000000..7f2e4086 --- /dev/null +++ b/lib/Stat.cc @@ -0,0 +1,247 @@ +#include +#include +#include + + +namespace Grid { + + +bool PmuStat::pmu_initialized=false; + + +void PmuStat::init(const char *regname) +{ +#ifdef __x86_64__ + name = regname; + if (!pmu_initialized) + { + std::cout<<"initialising pmu"< > & table) +{ + table.resize(0); + int rd = grid->_rdimensions[dimension]; - template - class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal fill in. - public: - - typedef uint32_t StencilInteger; - typedef typename cobj::vector_type vector_type; - typedef typename cobj::scalar_type scalar_type; - typedef typename cobj::scalar_object scalar_object; - - ////////////////////////////////////////// - // Comms packet queue for asynch thread - ////////////////////////////////////////// - - struct Packet { - void * send_buf; - void * recv_buf; - Integer to_rank; - Integer from_rank; - Integer bytes; - volatile Integer done; - }; - - std::vector Packets; - - #define SEND_IMMEDIATE - #define SERIAL_SENDS - - void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){ - #ifdef SEND_IMMEDIATE - commtime-=usecond(); - _grid->SendToRecvFrom(xmit,to,rcv,from,bytes); - commtime+=usecond(); - #endif - Packet p; - p.send_buf = xmit; - p.recv_buf = rcv; - p.to_rank = to; - p.from_rank= from; - p.bytes = bytes; - p.done = 0; - comms_bytes+=2.0*bytes; - Packets.push_back(p); - - } - - #ifdef SERIAL_SENDS - void Communicate(void ) { - commtime-=usecond(); - for(int i=0;iSendToRecvFrom( - Packets[i].send_buf, - Packets[i].to_rank, - Packets[i].recv_buf, - Packets[i].from_rank, - Packets[i].bytes); - #endif - Packets[i].done = 1; - } - commtime+=usecond(); - } - #else - void Communicate(void ) { - typedef CartesianCommunicator::CommsRequest_t CommsRequest_t; - std::vector > reqs(Packets.size()); - commtime-=usecond(); - const int concurrency=2; - for(int i=0;iSendToRecvFromBegin(reqs[j], - Packets[j].send_buf, - Packets[j].to_rank, - Packets[j].recv_buf, - Packets[j].from_rank, - Packets[j].bytes); - #endif - } - } - for(int ii=0;iiSendToRecvFromComplete(reqs[i]); - #endif - } - } - for(int ii=0;ii rpointers; - Integer buffer_size; - Integer packet_id; - }; - - std::vector Mergers; - - void AddMerge(cobj *merge_p,std::vector &rpointers,Integer buffer_size,Integer packet_id) { - Merge m; - m.mpointer = merge_p; - m.rpointers= rpointers; - m.buffer_size = buffer_size; - m.packet_id = packet_id; - #ifdef SEND_IMMEDIATE - mergetime-=usecond(); - PARALLEL_FOR_LOOP - for(int o=0;oCheckerBoarded(dimension) ) { + cbmask = 0x3; + } + int so= plane*grid->_ostride[dimension]; // base offset for start of plane + int e1=grid->_slice_nblock[dimension]; + int e2=grid->_slice_block[dimension]; + int stride=grid->_slice_stride[dimension]; + if ( cbmask == 0x3 ) { + table.resize(e1*e2); + for(int n=0;n(bo+b,o+b); + } + } + } else { + int bo=0; + table.resize(e1*e2/2); + for(int n=0;nCheckerBoardFromOindexTable(o+b); + if ( ocb &cbmask ) { + table[bo]=std::pair(bo,o+b); bo++; } } + } + } +} - //////////////////////////////////////// - // Basic Grid and stencil info - //////////////////////////////////////// +template void +Gather_plane_simple_table (std::vector >& table,const Lattice &rhs,cobj *buffer,compressor &compress, int off,int so) +{ +PARALLEL_FOR_LOOP + for(int i=0;i _directions; - std::vector _distances; - std::vector _comm_buf_size; - std::vector _permute_type; +template +class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal fill in. + public: - // npoints x Osites() of these - // Flat vector, change layout for cache friendly. - Vector _entries; + typedef CartesianCommunicator::CommsRequest_t CommsRequest_t; + typedef uint32_t StencilInteger; + typedef typename cobj::vector_type vector_type; + typedef typename cobj::scalar_type scalar_type; + typedef typename cobj::scalar_object scalar_object; + + ////////////////////////////////////////// + // Comms packet queue for asynch thread + ////////////////////////////////////////// - inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; } + struct Packet { + void * send_buf; + void * recv_buf; + Integer to_rank; + Integer from_rank; + Integer bytes; + }; - void PrecomputeByteOffsets(void){ - for(int i=0;i<_entries.size();i++){ - if( _entries[i]._is_local ) { - _entries[i]._byte_offset = _entries[i]._offset*sizeof(vobj); - } else { - _entries[i]._byte_offset =(uint64_t)&comm_buf[0]+ _entries[i]._offset*sizeof(cobj); - } - } - }; + std::vector Packets; - inline uint64_t Touch(int ent) { - // _mm_prefetch((char *)&_entries[ent],_MM_HINT_T0); - } - inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) { - //_mm_prefetch((char *)&_entries[ent+1],_MM_HINT_T0); - local = _entries[ent]._is_local; - perm = _entries[ent]._permute; - if (perm) ptype = _permute_type[point]; - if (local) return base + _entries[ent]._byte_offset; - else return _entries[ent]._byte_offset; - } - inline uint64_t GetPFInfo(int ent,uint64_t base) { - int local = _entries[ent]._is_local; - if (local) return base + _entries[ent]._byte_offset; - else return _entries[ent]._byte_offset; - } + int face_table_computed; + std::vector > > face_table ; + + void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){ + Packet p; + p.send_buf = xmit; + p.recv_buf = rcv; + p.to_rank = to; + p.from_rank= from; + p.bytes = bytes; + comms_bytes+=2.0*bytes; + Packets.push_back(p); + } - // Comms buffers - std::vector > u_simd_send_buf; - std::vector > u_simd_recv_buf; - Vector u_send_buf; - Vector comm_buf; - int u_comm_offset; - int _unified_buffer_size; + void CommunicateBegin(std::vector > &reqs) + { + reqs.resize(Packets.size()); + commtime-=usecond(); + for(int i=0;iStencilSendToRecvFromBegin(reqs[i], + Packets[i].send_buf, + Packets[i].to_rank, + Packets[i].recv_buf, + Packets[i].from_rank, + Packets[i].bytes); + /* + }else{ + _grid->SendToRecvFromBegin(reqs[i], + Packets[i].send_buf, + Packets[i].to_rank, + Packets[i].recv_buf, + Packets[i].from_rank, + Packets[i].bytes); + } + */ + } + commtime+=usecond(); + } + void CommunicateComplete(std::vector > &reqs) + { + commtime-=usecond(); - ///////////////////////////////////////// - // Timing info; ugly; possibly temporary - ///////////////////////////////////////// + for(int i=0;iStencilSendToRecvFromComplete(reqs[i]); + // else + // _grid->SendToRecvFromComplete(reqs[i]); + } + commtime+=usecond(); + } + + /////////////////////////////////////////// + // Simd merge queue for asynch comms + /////////////////////////////////////////// + struct Merge { + cobj * mpointer; + std::vector rpointers; + Integer buffer_size; + Integer packet_id; + }; + + std::vector Mergers; + + void AddMerge(cobj *merge_p,std::vector &rpointers,Integer buffer_size,Integer packet_id) { + Merge m; + m.mpointer = merge_p; + m.rpointers= rpointers; + m.buffer_size = buffer_size; + m.packet_id = packet_id; + Mergers.push_back(m); + } + + void CommsMerge(void ) { + + for(int i=0;i _directions; + std::vector _distances; + std::vector _comm_buf_size; + std::vector _permute_type; + + // npoints x Osites() of these + // Flat vector, change layout for cache friendly. + Vector _entries; + + void PrecomputeByteOffsets(void){ + for(int i=0;i<_entries.size();i++){ + if( _entries[i]._is_local ) { + _entries[i]._byte_offset = _entries[i]._offset*sizeof(vobj); + } else { + _entries[i]._byte_offset = _entries[i]._offset*sizeof(cobj); + } + } + }; + + inline StencilEntry * GetEntry(int &ptype,int point,int osite) { ptype = _permute_type[point]; return & _entries[point+_npoints*osite]; } + inline uint64_t GetInfo(int &ptype,int &local,int &perm,int point,int ent,uint64_t base) { + uint64_t cbase = (uint64_t)&u_recv_buf_p[0]; + local = _entries[ent]._is_local; + perm = _entries[ent]._permute; + if (perm) ptype = _permute_type[point]; + if (local) { + return base + _entries[ent]._byte_offset; + } else { + return cbase + _entries[ent]._byte_offset; + } + } + inline uint64_t GetPFInfo(int ent,uint64_t base) { + uint64_t cbase = (uint64_t)&u_recv_buf_p[0]; + int local = _entries[ent]._is_local; + if (local) return base + _entries[ent]._byte_offset; + else return cbase + _entries[ent]._byte_offset; + } + + /////////////////////////////////////////////////////////// + // Unified Comms buffers for all directions + /////////////////////////////////////////////////////////// + // Vectors that live on the symmetric heap in case of SHMEM + // std::vector > u_simd_send_buf_hide; + // std::vector > u_simd_recv_buf_hide; + // commVector u_send_buf_hide; + // commVector u_recv_buf_hide; + + // These are used; either SHM objects or refs to the above symmetric heap vectors + // depending on comms target + cobj* u_recv_buf_p; + cobj* u_send_buf_p; + std::vector u_simd_send_buf; + std::vector u_simd_recv_buf; + + int u_comm_offset; + int _unified_buffer_size; + + cobj *CommBuf(void) { return u_recv_buf_p; } + + ///////////////////////////////////////// + // Timing info; ugly; possibly temporary + ///////////////////////////////////////// #define TIMING_HACK #ifdef TIMING_HACK - double jointime; - double gathertime; - double commtime; - double halogtime; - double mergetime; - double spintime; - double comms_bytes; - double gathermtime; - double splicetime; - double nosplicetime; - double calls; - - void ZeroCounters(void) { - gathertime = 0.; - jointime = 0.; - commtime = 0.; - halogtime = 0.; - mergetime = 0.; - spintime = 0.; - gathermtime = 0.; - splicetime = 0.; - nosplicetime = 0.; - comms_bytes = 0.; - calls = 0.; - }; - - void Report(void) { + double jointime; + double gathertime; + double commtime; + double halogtime; + double mergetime; + double spintime; + double comms_bytes; + double gathermtime; + double splicetime; + double nosplicetime; + double t_data; + double t_table; + double calls; + + void ZeroCounters(void) { + gathertime = 0.; + jointime = 0.; + commtime = 0.; + halogtime = 0.; + mergetime = 0.; + spintime = 0.; + gathermtime = 0.; + splicetime = 0.; + nosplicetime = 0.; + t_data = 0.0; + t_table= 0.0; + comms_bytes = 0.; + calls = 0.; + }; + + void Report(void) { #define PRINTIT(A) \ std::cout << GridLogMessage << " Stencil " << #A << " "<< A/calls< 0. ) { - std::cout << GridLogMessage << " Stencil calls "< 0. ) { + std::cout << GridLogMessage << " Stencil calls "<1.0){ + PRINTIT(comms_bytes); + PRINTIT(commtime); + std::cout << GridLogMessage << " Stencil " << comms_bytes/commtime/1000. << " GB/s "< &directions, - const std::vector &distances) - : _permute_type(npoints), _comm_buf_size(npoints) - { - _npoints = npoints; - _grid = grid; - _directions = directions; - _distances = distances; - _unified_buffer_size=0; - - int osites = _grid->oSites(); - - _entries.resize(_npoints* osites); - for(int ii=0;ii_fdimensions[dimension]; - int rd = _grid->_rdimensions[dimension]; - _permute_type[point]=_grid->PermuteType(dimension); - - _checkerboard = checkerboard; - - // the permute type - int simd_layout = _grid->_simd_layout[dimension]; - int comm_dim = _grid->_processors[dimension] >1 ; - int splice_dim = _grid->_simd_layout[dimension]>1 && (comm_dim); - int rotate_dim = _grid->_simd_layout[dimension]>2; - - assert ( (rotate_dim && comm_dim) == false) ; // Do not think spread out is supported - - int sshift[2]; - - // Underlying approach. For each local site build - // up a table containing the npoint "neighbours" and whether they - // live in lattice or a comms buffer. - if ( !comm_dim ) { - sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even); - sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd); - - if ( sshift[0] == sshift[1] ) { - Local(point,dimension,shift,0x3); - } else { - Local(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes - Local(point,dimension,shift,0x2);// both with block stride loop iteration - } - } else { // All permute extract done in comms phase prior to Stencil application - // So tables are the same whether comm_dim or splice_dim - sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even); - sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd); - - if ( sshift[0] == sshift[1] ) { - Comms(point,dimension,shift,0x3); - } else { - Comms(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes - Comms(point,dimension,shift,0x2);// both with block stride loop iteration - } - } - } - u_send_buf.resize(_unified_buffer_size); - comm_buf.resize(_unified_buffer_size); - - PrecomputeByteOffsets(); - - const int Nsimd = grid->Nsimd(); - u_simd_send_buf.resize(Nsimd); - u_simd_recv_buf.resize(Nsimd); - for(int l=0;l_fdimensions[dimension]; - int rd = _grid->_rdimensions[dimension]; - int ld = _grid->_ldimensions[dimension]; - int gd = _grid->_gdimensions[dimension]; - int ly = _grid->_simd_layout[dimension]; - - // Map to always positive shift modulo global full dimension. - int shift = (shiftpm+fd)%fd; - - // the permute type - int permute_dim =_grid->PermuteDim(dimension); - - for(int x=0;x_ostride[dimension]; - - int cb= (cbmask==0x2)? Odd : Even; - - int sshift = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb); - int sx = (x+sshift)%rd; - - int wraparound=0; - if ( (shiftpm==-1) && (sx>x) ) { - wraparound = 1; - } - if ( (shiftpm== 1) && (sxNsimd(); - - int fd = _grid->_fdimensions[dimension]; - int ld = _grid->_ldimensions[dimension]; - int rd = _grid->_rdimensions[dimension]; - int pd = _grid->_processors[dimension]; - int simd_layout = _grid->_simd_layout[dimension]; - int comm_dim = _grid->_processors[dimension] >1 ; - - assert(comm_dim==1); - int shift = (shiftpm + fd) %fd; - assert(shift>=0); - assert(shift_slice_nblock[dimension]*_grid->_slice_block[dimension]; // done in reduced dims, so SIMD factored - - _comm_buf_size[point] = buffer_size; // Size of _one_ plane. Multiple planes may be gathered and - // send to one or more remote nodes. - - int cb= (cbmask==0x2)? Odd : Even; - int sshift= _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb); - - for(int x=0;xPermuteType(dimension); - - int sx = (x+sshift)%rd; - - int offnode = 0; - if ( simd_layout > 1 ) { - - for(int i=0;i>(permute_type+1)); - int ic= (i&inner_bit)? 1:0; - int my_coor = rd*ic + x; - int nbr_coor = my_coor+sshift; - int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors - - if ( nbr_proc ) { - offnode =1; - } - } - - } else { - int comm_proc = ((x+sshift)/rd)%pd; - offnode = (comm_proc!= 0); - } - - - int wraparound=0; - if ( (shiftpm==-1) && (sx>x) && (grid->_processor_coor[dimension]==0) ) { - wraparound = 1; - } - if ( (shiftpm== 1) && (sx_processor_coor[dimension]==grid->_processors[dimension]-1) ) { - wraparound = 1; - } - if (!offnode) { - - int permute_slice=0; - CopyPlane(point,dimension,x,sx,cbmask,permute_slice,wraparound); - - } else { - - int words = buffer_size; - if (cbmask != 0x3) words=words>>1; - - int rank = grid->_processor; - int recv_from_rank; - int xmit_to_rank; - - int unified_buffer_offset = _unified_buffer_size; - _unified_buffer_size += words; - - ScatterPlane(point,dimension,x,cbmask,unified_buffer_offset,wraparound); // permute/extract/merge is done in comms phase - - } - } - } - // Routine builds up integer table for each site in _offsets, _is_local, _permute - void CopyPlane(int point, int dimension,int lplane,int rplane,int cbmask,int permute,int wrap) - { - int rd = _grid->_rdimensions[dimension]; - - if ( !_grid->CheckerBoarded(dimension) ) { - - int o = 0; // relative offset to base within plane - int ro = rplane*_grid->_ostride[dimension]; // base offset for start of plane - int lo = lplane*_grid->_ostride[dimension]; // offset in buffer - - // Simple block stride gather of SIMD objects - for(int n=0;n<_grid->_slice_nblock[dimension];n++){ - for(int b=0;b<_grid->_slice_block[dimension];b++){ - int idx=point+(lo+o+b)*_npoints; - _entries[idx]._offset =ro+o+b; - _entries[idx]._permute=permute; - _entries[idx]._is_local=1; - _entries[idx]._around_the_world=wrap; - } - o +=_grid->_slice_stride[dimension]; - } - - } else { - - int ro = rplane*_grid->_ostride[dimension]; // base offset for start of plane - int lo = lplane*_grid->_ostride[dimension]; // base offset for start of plane - int o = 0; // relative offset to base within plane - - for(int n=0;n<_grid->_slice_nblock[dimension];n++){ - for(int b=0;b<_grid->_slice_block[dimension];b++){ - - int ocb=1<<_grid->CheckerBoardFromOindex(o+b); - - if ( ocb&cbmask ) { - int idx = point+(lo+o+b)*_npoints; - _entries[idx]._offset =ro+o+b; - _entries[idx]._is_local=1; - _entries[idx]._permute=permute; - _entries[idx]._around_the_world=wrap; - } - - } - o +=_grid->_slice_stride[dimension]; - } - - } - } - // Routine builds up integer table for each site in _offsets, _is_local, _permute - void ScatterPlane (int point,int dimension,int plane,int cbmask,int offset, int wrap) - { - int rd = _grid->_rdimensions[dimension]; - - if ( !_grid->CheckerBoarded(dimension) ) { - - int so = plane*_grid->_ostride[dimension]; // base offset for start of plane - int o = 0; // relative offset to base within plane - int bo = 0; // offset in buffer - - // Simple block stride gather of SIMD objects - for(int n=0;n<_grid->_slice_nblock[dimension];n++){ - for(int b=0;b<_grid->_slice_block[dimension];b++){ - int idx=point+(so+o+b)*_npoints; - _entries[idx]._offset =offset+(bo++); - _entries[idx]._is_local=0; - _entries[idx]._permute=0; - _entries[idx]._around_the_world=wrap; - } - o +=_grid->_slice_stride[dimension]; - } - - } else { - - int so = plane*_grid->_ostride[dimension]; // base offset for start of plane - int o = 0; // relative offset to base within plane - int bo = 0; // offset in buffer - - for(int n=0;n<_grid->_slice_nblock[dimension];n++){ - for(int b=0;b<_grid->_slice_block[dimension];b++){ - - int ocb=1<<_grid->CheckerBoardFromOindex(o+b);// Could easily be a table lookup - if ( ocb & cbmask ) { - int idx = point+(so+o+b)*_npoints; - _entries[idx]._offset =offset+(bo++); - _entries[idx]._is_local=0; - _entries[idx]._permute =0; - _entries[idx]._around_the_world=wrap; - } - } - o +=_grid->_slice_stride[dimension]; - } - } - } - - - - template - void HaloExchange(const Lattice &source,compressor &compress) - { - calls++; - Mergers.resize(0); - Packets.resize(0); - HaloGather(source,compress); - this->Communicate(); - CommsMerge(); // spins - } -#if 0 - // Overlapping comms and compute typically slows down compute and is useless - // unless memory bandwidth greatly exceeds network - template - std::thread HaloExchangeBegin(const Lattice &source,compressor &compress) { - Mergers.resize(0); - Packets.resize(0); - HaloGather(source,compress); - return std::thread([&] { this->Communicate(); }); - } - void HaloExchangeComplete(std::thread &thr) - { - CommsMerge(); // spins - jointime-=usecond(); - thr.join(); - jointime+=usecond(); - } -#endif - template - void HaloGatherDir(const Lattice &source,compressor &compress,int point) - { - int dimension = _directions[point]; - int displacement = _distances[point]; - - int fd = _grid->_fdimensions[dimension]; - int rd = _grid->_rdimensions[dimension]; - - - // Map to always positive shift modulo global full dimension. - int shift = (displacement+fd)%fd; - - // int checkerboard = _grid->CheckerBoardDestination(source.checkerboard,shift); - assert (source.checkerboard== _checkerboard); - - // the permute type - int simd_layout = _grid->_simd_layout[dimension]; - int comm_dim = _grid->_processors[dimension] >1 ; - int splice_dim = _grid->_simd_layout[dimension]>1 && (comm_dim); - - // Gather phase - int sshift [2]; - if ( comm_dim ) { - sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even); - sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd); - if ( sshift[0] == sshift[1] ) { - if (splice_dim) { - splicetime-=usecond(); - GatherSimd(source,dimension,shift,0x3,compress); - splicetime+=usecond(); - } else { - nosplicetime-=usecond(); - Gather(source,dimension,shift,0x3,compress); - nosplicetime+=usecond(); - } - } else { - if(splice_dim){ - splicetime-=usecond(); - GatherSimd(source,dimension,shift,0x1,compress);// if checkerboard is unfavourable take two passes - GatherSimd(source,dimension,shift,0x2,compress);// both with block stride loop iteration - splicetime+=usecond(); - } else { - nosplicetime-=usecond(); - Gather(source,dimension,shift,0x1,compress); - Gather(source,dimension,shift,0x2,compress); - nosplicetime+=usecond(); - } - } - } - } - - template - void HaloGather(const Lattice &source,compressor &compress) - { - // conformable(source._grid,_grid); - assert(source._grid==_grid); - halogtime-=usecond(); - - assert (comm_buf.size() == _unified_buffer_size ); - u_comm_offset=0; - - // Gather all comms buffers - for(int point = 0 ; point < _npoints; point++) { - compress.Point(point); - HaloGatherDir(source,compress,point); - } - - assert(u_comm_offset==_unified_buffer_size); - halogtime+=usecond(); - } - - template - void Gather(const Lattice &rhs,int dimension,int shift,int cbmask,compressor & compress) - { - typedef typename cobj::vector_type vector_type; - typedef typename cobj::scalar_type scalar_type; - - GridBase *grid=_grid; - assert(rhs._grid==_grid); - // conformable(_grid,rhs._grid); - - int fd = _grid->_fdimensions[dimension]; - int rd = _grid->_rdimensions[dimension]; - int pd = _grid->_processors[dimension]; - int simd_layout = _grid->_simd_layout[dimension]; - int comm_dim = _grid->_processors[dimension] >1 ; - assert(simd_layout==1); - assert(comm_dim==1); - assert(shift>=0); - assert(shift_slice_nblock[dimension]*_grid->_slice_block[dimension]; - - int cb= (cbmask==0x2)? Odd : Even; - int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb); - - for(int x=0;x>1; - - int bytes = words * sizeof(cobj); - - gathertime-=usecond(); - Gather_plane_simple (rhs,u_send_buf,dimension,sx,cbmask,compress,u_comm_offset); - gathertime+=usecond(); - - int rank = _grid->_processor; - int recv_from_rank; - int xmit_to_rank; - _grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank); - assert (xmit_to_rank != _grid->ThisRank()); - assert (recv_from_rank != _grid->ThisRank()); - - // FIXME Implement asynchronous send & also avoid buffer copy - AddPacket((void *)&u_send_buf[u_comm_offset], - (void *) &comm_buf[u_comm_offset], - xmit_to_rank, - recv_from_rank, - bytes); - - u_comm_offset+=words; - } - } - } - - - template - void GatherSimd(const Lattice &rhs,int dimension,int shift,int cbmask,compressor &compress) - { - const int Nsimd = _grid->Nsimd(); - - int fd = _grid->_fdimensions[dimension]; - int rd = _grid->_rdimensions[dimension]; - int ld = _grid->_ldimensions[dimension]; - int pd = _grid->_processors[dimension]; - int simd_layout = _grid->_simd_layout[dimension]; - int comm_dim = _grid->_processors[dimension] >1 ; - - assert(comm_dim==1); - // This will not work with a rotate dim - assert(simd_layout==2); - assert(shift>=0); - assert(shiftPermuteType(dimension); - - /////////////////////////////////////////////// - // Simd direction uses an extract/merge pair - /////////////////////////////////////////////// - int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension]; - int words = sizeof(cobj)/sizeof(vector_type); - - assert(cbmask==0x3); // Fixme think there is a latent bug if not true - - int bytes = buffer_size*sizeof(scalar_object); - - std::vector rpointers(Nsimd); - std::vector spointers(Nsimd); - - /////////////////////////////////////////// - // Work out what to send where - /////////////////////////////////////////// - - int cb = (cbmask==0x2)? Odd : Even; - int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb); - - // loop over outer coord planes orthog to dim - for(int x=0;x= rd ); - - if ( any_offnode ) { - - for(int i=0;i(rhs,spointers,dimension,sx,cbmask,compress); - gathermtime+=usecond(); - - for(int i=0;i2 - // std::cout << "GatherSimd : lane 1st elem " << i << u_simd_send_buf[i ][u_comm_offset]<>(permute_type+1)); - int ic= (i&inner_bit)? 1:0; - - int my_coor = rd*ic + x; - int nbr_coor = my_coor+sshift; - int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors - int nbr_lcoor= (nbr_coor%ld); - int nbr_ic = (nbr_lcoor)/rd; // inner coord of peer - int nbr_ox = (nbr_lcoor%rd); // outer coord of peer - int nbr_lane = (i&(~inner_bit)); - - if (nbr_ic) nbr_lane|=inner_bit; - assert (sx == nbr_ox); - - auto rp = &u_simd_recv_buf[i ][u_comm_offset]; - auto sp = &u_simd_send_buf[nbr_lane][u_comm_offset]; - - void *vrp = (void *)rp; - void *vsp = (void *)sp; - - - if(nbr_proc){ - - int recv_from_rank; - int xmit_to_rank; - - _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); - - AddPacket( vsp,vrp,xmit_to_rank,recv_from_rank,bytes); - - rpointers[i] = rp; - - } else { - - rpointers[i] = sp; - - } - } - - AddMerge(&comm_buf[u_comm_offset],rpointers,buffer_size,Packets.size()-1); - - u_comm_offset +=buffer_size; - } - } - } - - }; - } + CartesianStencil(GridBase *grid, + int npoints, + int checkerboard, + const std::vector &directions, + const std::vector &distances) + : _permute_type(npoints), _comm_buf_size(npoints) + { + face_table_computed=0; + _npoints = npoints; + _grid = grid; + _directions = directions; + _distances = distances; + _unified_buffer_size=0; + + int osites = _grid->oSites(); + + _entries.resize(_npoints* osites); + for(int ii=0;ii_fdimensions[dimension]; + int rd = _grid->_rdimensions[dimension]; + _permute_type[point]=_grid->PermuteType(dimension); + + _checkerboard = checkerboard; + + // the permute type + int simd_layout = _grid->_simd_layout[dimension]; + int comm_dim = _grid->_processors[dimension] >1 ; + int splice_dim = _grid->_simd_layout[dimension]>1 && (comm_dim); + int rotate_dim = _grid->_simd_layout[dimension]>2; + + assert ( (rotate_dim && comm_dim) == false) ; // Do not think spread out is supported + + int sshift[2]; + + // Underlying approach. For each local site build + // up a table containing the npoint "neighbours" and whether they + // live in lattice or a comms buffer. + if ( !comm_dim ) { + sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even); + sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd); + + if ( sshift[0] == sshift[1] ) { + Local(point,dimension,shift,0x3); + } else { + Local(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes + Local(point,dimension,shift,0x2);// both with block stride loop iteration + } + } else { // All permute extract done in comms phase prior to Stencil application + // So tables are the same whether comm_dim or splice_dim + sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even); + sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd); + + if ( sshift[0] == sshift[1] ) { + Comms(point,dimension,shift,0x3); + } else { + Comms(point,dimension,shift,0x1);// if checkerboard is unfavourable take two passes + Comms(point,dimension,shift,0x2);// both with block stride loop iteration + } + } + } + + ///////////////////////////////////////////////////////////////////////////////// + // Try to allocate for receiving in a shared memory region, fall back to buffer + ///////////////////////////////////////////////////////////////////////////////// + const int Nsimd = grid->Nsimd(); + + _grid->ShmBufferFreeAll(); + + u_simd_send_buf.resize(Nsimd); + u_simd_recv_buf.resize(Nsimd); + + u_send_buf_p=(cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj)); + u_recv_buf_p=(cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj)); + for(int l=0;lShmBufferMalloc(_unified_buffer_size*sizeof(scalar_object)); + u_simd_send_buf[l] = (scalar_object *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(scalar_object)); + } + + PrecomputeByteOffsets(); + } + + void Local (int point, int dimension,int shiftpm,int cbmask) + { + int fd = _grid->_fdimensions[dimension]; + int rd = _grid->_rdimensions[dimension]; + int ld = _grid->_ldimensions[dimension]; + int gd = _grid->_gdimensions[dimension]; + int ly = _grid->_simd_layout[dimension]; + + // Map to always positive shift modulo global full dimension. + int shift = (shiftpm+fd)%fd; + + // the permute type + int permute_dim =_grid->PermuteDim(dimension); + + for(int x=0;x_ostride[dimension]; + + int cb= (cbmask==0x2)? Odd : Even; + + int sshift = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb); + int sx = (x+sshift)%rd; + + int wraparound=0; + if ( (shiftpm==-1) && (sx>x) ) { + wraparound = 1; + } + if ( (shiftpm== 1) && (sxNsimd(); + + int fd = _grid->_fdimensions[dimension]; + int ld = _grid->_ldimensions[dimension]; + int rd = _grid->_rdimensions[dimension]; + int pd = _grid->_processors[dimension]; + int simd_layout = _grid->_simd_layout[dimension]; + int comm_dim = _grid->_processors[dimension] >1 ; + + assert(comm_dim==1); + int shift = (shiftpm + fd) %fd; + assert(shift>=0); + assert(shift_slice_nblock[dimension]*_grid->_slice_block[dimension]; // done in reduced dims, so SIMD factored + + _comm_buf_size[point] = buffer_size; // Size of _one_ plane. Multiple planes may be gathered and + // send to one or more remote nodes. + + int cb= (cbmask==0x2)? Odd : Even; + int sshift= _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,cb); + + for(int x=0;xPermuteType(dimension); + + int sx = (x+sshift)%rd; + + int offnode = 0; + if ( simd_layout > 1 ) { + + for(int i=0;i>(permute_type+1)); + int ic= (i&inner_bit)? 1:0; + int my_coor = rd*ic + x; + int nbr_coor = my_coor+sshift; + int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors + + if ( nbr_proc ) { + offnode =1; + } + } + + } else { + int comm_proc = ((x+sshift)/rd)%pd; + offnode = (comm_proc!= 0); + } + + + int wraparound=0; + if ( (shiftpm==-1) && (sx>x) && (grid->_processor_coor[dimension]==0) ) { + wraparound = 1; + } + if ( (shiftpm== 1) && (sx_processor_coor[dimension]==grid->_processors[dimension]-1) ) { + wraparound = 1; + } + if (!offnode) { + + int permute_slice=0; + CopyPlane(point,dimension,x,sx,cbmask,permute_slice,wraparound); + + } else { + + int words = buffer_size; + if (cbmask != 0x3) words=words>>1; + + int rank = grid->_processor; + int recv_from_rank; + int xmit_to_rank; + + int unified_buffer_offset = _unified_buffer_size; + _unified_buffer_size += words; + + ScatterPlane(point,dimension,x,cbmask,unified_buffer_offset,wraparound); // permute/extract/merge is done in comms phase + + } + } + } + // Routine builds up integer table for each site in _offsets, _is_local, _permute + void CopyPlane(int point, int dimension,int lplane,int rplane,int cbmask,int permute,int wrap) + { + int rd = _grid->_rdimensions[dimension]; + + if ( !_grid->CheckerBoarded(dimension) ) { + + int o = 0; // relative offset to base within plane + int ro = rplane*_grid->_ostride[dimension]; // base offset for start of plane + int lo = lplane*_grid->_ostride[dimension]; // offset in buffer + + // Simple block stride gather of SIMD objects + for(int n=0;n<_grid->_slice_nblock[dimension];n++){ + for(int b=0;b<_grid->_slice_block[dimension];b++){ + int idx=point+(lo+o+b)*_npoints; + _entries[idx]._offset =ro+o+b; + _entries[idx]._permute=permute; + _entries[idx]._is_local=1; + _entries[idx]._around_the_world=wrap; + } + o +=_grid->_slice_stride[dimension]; + } + + } else { + + int ro = rplane*_grid->_ostride[dimension]; // base offset for start of plane + int lo = lplane*_grid->_ostride[dimension]; // base offset for start of plane + int o = 0; // relative offset to base within plane + + for(int n=0;n<_grid->_slice_nblock[dimension];n++){ + for(int b=0;b<_grid->_slice_block[dimension];b++){ + + int ocb=1<<_grid->CheckerBoardFromOindex(o+b); + + if ( ocb&cbmask ) { + int idx = point+(lo+o+b)*_npoints; + _entries[idx]._offset =ro+o+b; + _entries[idx]._is_local=1; + _entries[idx]._permute=permute; + _entries[idx]._around_the_world=wrap; + } + + } + o +=_grid->_slice_stride[dimension]; + } + + } + } + // Routine builds up integer table for each site in _offsets, _is_local, _permute + void ScatterPlane (int point,int dimension,int plane,int cbmask,int offset, int wrap) + { + int rd = _grid->_rdimensions[dimension]; + + if ( !_grid->CheckerBoarded(dimension) ) { + + int so = plane*_grid->_ostride[dimension]; // base offset for start of plane + int o = 0; // relative offset to base within plane + int bo = 0; // offset in buffer + + // Simple block stride gather of SIMD objects + for(int n=0;n<_grid->_slice_nblock[dimension];n++){ + for(int b=0;b<_grid->_slice_block[dimension];b++){ + int idx=point+(so+o+b)*_npoints; + _entries[idx]._offset =offset+(bo++); + _entries[idx]._is_local=0; + _entries[idx]._permute=0; + _entries[idx]._around_the_world=wrap; + } + o +=_grid->_slice_stride[dimension]; + } + + } else { + + int so = plane*_grid->_ostride[dimension]; // base offset for start of plane + int o = 0; // relative offset to base within plane + int bo = 0; // offset in buffer + + for(int n=0;n<_grid->_slice_nblock[dimension];n++){ + for(int b=0;b<_grid->_slice_block[dimension];b++){ + + int ocb=1<<_grid->CheckerBoardFromOindex(o+b);// Could easily be a table lookup + if ( ocb & cbmask ) { + int idx = point+(so+o+b)*_npoints; + _entries[idx]._offset =offset+(bo++); + _entries[idx]._is_local=0; + _entries[idx]._permute =0; + _entries[idx]._around_the_world=wrap; + } + } + o +=_grid->_slice_stride[dimension]; + } + } + } + + template void HaloExchange(const Lattice &source,compressor &compress) + { + std::vector > reqs; + calls++; + Mergers.resize(0); + Packets.resize(0); + _grid->StencilBarrier(); + HaloGather(source,compress); + this->CommunicateBegin(reqs); + _grid->StencilBarrier(); + this->CommunicateComplete(reqs); + _grid->StencilBarrier(); + CommsMerge(); // spins + } + + template void HaloGatherDir(const Lattice &source,compressor &compress,int point,int & face_idx) + { + int dimension = _directions[point]; + int displacement = _distances[point]; + + int fd = _grid->_fdimensions[dimension]; + int rd = _grid->_rdimensions[dimension]; + + + // Map to always positive shift modulo global full dimension. + int shift = (displacement+fd)%fd; + + // int checkerboard = _grid->CheckerBoardDestination(source.checkerboard,shift); + assert (source.checkerboard== _checkerboard); + + // the permute type + int simd_layout = _grid->_simd_layout[dimension]; + int comm_dim = _grid->_processors[dimension] >1 ; + int splice_dim = _grid->_simd_layout[dimension]>1 && (comm_dim); + + // Gather phase + int sshift [2]; + if ( comm_dim ) { + sshift[0] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Even); + sshift[1] = _grid->CheckerBoardShiftForCB(_checkerboard,dimension,shift,Odd); + if ( sshift[0] == sshift[1] ) { + if (splice_dim) { + splicetime-=usecond(); + GatherSimd(source,dimension,shift,0x3,compress,face_idx); + splicetime+=usecond(); + } else { + nosplicetime-=usecond(); + Gather(source,dimension,shift,0x3,compress,face_idx); + nosplicetime+=usecond(); + } + } else { + if(splice_dim){ + splicetime-=usecond(); + GatherSimd(source,dimension,shift,0x1,compress,face_idx);// if checkerboard is unfavourable take two passes + GatherSimd(source,dimension,shift,0x2,compress,face_idx);// both with block stride loop iteration + splicetime+=usecond(); + } else { + nosplicetime-=usecond(); + Gather(source,dimension,shift,0x1,compress,face_idx); + Gather(source,dimension,shift,0x2,compress,face_idx); + nosplicetime+=usecond(); + } + } + } + } + + template + void HaloGather(const Lattice &source,compressor &compress) + { + // conformable(source._grid,_grid); + assert(source._grid==_grid); + halogtime-=usecond(); + + u_comm_offset=0; + + // Gather all comms buffers + int face_idx=0; + for(int point = 0 ; point < _npoints; point++) { + compress.Point(point); + HaloGatherDir(source,compress,point,face_idx); + } + face_table_computed=1; + + assert(u_comm_offset==_unified_buffer_size); + halogtime+=usecond(); + } + + template + void Gather(const Lattice &rhs,int dimension,int shift,int cbmask,compressor & compress,int &face_idx) + { + typedef typename cobj::vector_type vector_type; + typedef typename cobj::scalar_type scalar_type; + + GridBase *grid=_grid; + assert(rhs._grid==_grid); + // conformable(_grid,rhs._grid); + + int fd = _grid->_fdimensions[dimension]; + int rd = _grid->_rdimensions[dimension]; + int pd = _grid->_processors[dimension]; + int simd_layout = _grid->_simd_layout[dimension]; + int comm_dim = _grid->_processors[dimension] >1 ; + assert(simd_layout==1); + assert(comm_dim==1); + assert(shift>=0); + assert(shift_slice_nblock[dimension]*_grid->_slice_block[dimension]; + + int cb= (cbmask==0x2)? Odd : Even; + int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb); + + for(int x=0;x>1; + + int bytes = words * sizeof(cobj); + + gathertime-=usecond(); + int so = sx*rhs._grid->_ostride[dimension]; // base offset for start of plane + if ( !face_table_computed ) { + t_table-=usecond(); + face_table.resize(face_idx+1); + Gather_plane_simple_table_compute ((GridBase *)_grid,dimension,sx,cbmask,u_comm_offset, + face_table[face_idx]); + t_table+=usecond(); + } + + + int rank = _grid->_processor; + int recv_from_rank; + int xmit_to_rank; + _grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank); + + assert (xmit_to_rank != _grid->ThisRank()); + assert (recv_from_rank != _grid->ThisRank()); + + ///////////////////////////////////////////////////////// + // try the direct copy if possible + ///////////////////////////////////////////////////////// + + + cobj *send_buf = (cobj *)_grid->ShmBufferTranslate(xmit_to_rank,u_recv_buf_p); + if ( send_buf==NULL ) { + send_buf = u_send_buf_p; + } + // std::cout << " send_bufs "< + void GatherSimd(const Lattice &rhs,int dimension,int shift,int cbmask,compressor &compress,int & face_idx) + { + const int Nsimd = _grid->Nsimd(); + + int fd = _grid->_fdimensions[dimension]; + int rd = _grid->_rdimensions[dimension]; + int ld = _grid->_ldimensions[dimension]; + int pd = _grid->_processors[dimension]; + int simd_layout = _grid->_simd_layout[dimension]; + int comm_dim = _grid->_processors[dimension] >1 ; + + assert(comm_dim==1); + // This will not work with a rotate dim + assert(simd_layout==2); + assert(shift>=0); + assert(shiftPermuteType(dimension); + + /////////////////////////////////////////////// + // Simd direction uses an extract/merge pair + /////////////////////////////////////////////// + int buffer_size = _grid->_slice_nblock[dimension]*_grid->_slice_block[dimension]; + int words = sizeof(cobj)/sizeof(vector_type); + + assert(cbmask==0x3); // Fixme think there is a latent bug if not true + + int bytes = buffer_size*sizeof(scalar_object); + + std::vector rpointers(Nsimd); + std::vector spointers(Nsimd); + + /////////////////////////////////////////// + // Work out what to send where + /////////////////////////////////////////// + + int cb = (cbmask==0x2)? Odd : Even; + int sshift= _grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb); + + // loop over outer coord planes orthog to dim + for(int x=0;x= rd ); + + if ( any_offnode ) { + + for(int i=0;i(rhs,spointers,dimension,sx,cbmask,compress); + gathermtime+=usecond(); + + for(int i=0;i2 + // std::cout << "GatherSimd : lane 1st elem " << i << u_simd_send_buf[i ][u_comm_offset]<>(permute_type+1)); + int ic= (i&inner_bit)? 1:0; + + int my_coor = rd*ic + x; + int nbr_coor = my_coor+sshift; + int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors + int nbr_lcoor= (nbr_coor%ld); + int nbr_ic = (nbr_lcoor)/rd; // inner coord of peer + int nbr_ox = (nbr_lcoor%rd); // outer coord of peer + int nbr_lane = (i&(~inner_bit)); + + if (nbr_ic) nbr_lane|=inner_bit; + assert (sx == nbr_ox); + + auto rp = &u_simd_recv_buf[i ][u_comm_offset]; + auto sp = &u_simd_send_buf[nbr_lane][u_comm_offset]; + + if(nbr_proc){ + + int recv_from_rank; + int xmit_to_rank; + + _grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank); + + scalar_object *shm = (scalar_object *) _grid->ShmBufferTranslate(recv_from_rank,sp); + // if ((ShmDirectCopy==0)||(shm==NULL)) { + if (shm==NULL) { + shm = rp; + } + + // if Direct, StencilSendToRecvFrom will suppress copy to a peer on node + // assuming above pointer flip + AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes); + + rpointers[i] = shm; + + } else { + + rpointers[i] = sp; + + } + } + + AddMerge(&u_recv_buf_p[u_comm_offset],rpointers,buffer_size,Packets.size()-1); + + u_comm_offset +=buffer_size; + } + } + } + +}; +} #endif diff --git a/lib/Threads.h b/lib/Threads.h index 9d1295e5..2f270b73 100644 --- a/lib/Threads.h +++ b/lib/Threads.h @@ -37,11 +37,20 @@ Author: paboyle #ifdef GRID_OMP #include -#define PARALLEL_FOR_LOOP _Pragma("omp parallel for ") -#define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)") +#ifdef GRID_NUMA +#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(static)") +#define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(static)") #else -#define PARALLEL_FOR_LOOP +#define PARALLEL_FOR_LOOP _Pragma("omp parallel for schedule(runtime)") +#define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(runtime)") +#endif +#define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)") +#define PARALLEL_REGION _Pragma("omp parallel") +#else +#define PARALLEL_FOR_LOOP +#define PARALLEL_FOR_LOOP_INTERN #define PARALLEL_NESTED_LOOP2 +#define PARALLEL_REGION #endif namespace Grid { @@ -123,6 +132,22 @@ class GridThread { ThreadBarrier(); }; + static void bcopy(const void *src, void *dst, size_t len) { +#ifdef GRID_OMP +#pragma omp parallel + { + const char *c_src =(char *) src; + char *c_dest=(char *) dst; + int me,mywork,myoff; + GridThread::GetWorkBarrier(len,me, mywork,myoff); + bcopy(&c_src[myoff],&c_dest[myoff],mywork); + } +#else + bcopy(src,dst,len); +#endif + } + + }; } diff --git a/lib/algorithms/CoarsenedMatrix.h b/lib/algorithms/CoarsenedMatrix.h index 3ae0af75..fd9acc91 100644 --- a/lib/algorithms/CoarsenedMatrix.h +++ b/lib/algorithms/CoarsenedMatrix.h @@ -282,7 +282,7 @@ PARALLEL_FOR_LOOP } else if(SE->_is_local) { nbr = in._odata[SE->_offset]; } else { - nbr = Stencil.comm_buf[SE->_offset]; + nbr = Stencil.CommBuf()[SE->_offset]; } res = res + A[point]._odata[ss]*nbr; } diff --git a/lib/algorithms/iterative/ConjugateGradient.h b/lib/algorithms/iterative/ConjugateGradient.h index f340eb38..cf3872c8 100644 --- a/lib/algorithms/iterative/ConjugateGradient.h +++ b/lib/algorithms/iterative/ConjugateGradient.h @@ -154,7 +154,7 @@ class ConjugateGradient : public OperatorFunction { << LinalgTimer.Elapsed(); std::cout << std::endl; - if (ErrorOnNoConverge) assert(true_residual / Tolerance < 1000.0); + if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0); return; } diff --git a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h index b387c12d..233eb8f5 100644 --- a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h +++ b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h @@ -31,7 +31,11 @@ Author: paboyle #include //memset #ifdef USE_LAPACK -#include +void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e, + double *vl, double *vu, int *il, int *iu, double *abstol, + int *m, double *w, double *z, int *ldz, int *isuppz, + double *work, int *lwork, int *iwork, int *liwork, + int *info); #endif #include "DenseMatrix.h" #include "EigenSort.h" diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h index b69c3435..72b21ee3 100644 --- a/lib/cartesian/Cartesian_base.h +++ b/lib/cartesian/Cartesian_base.h @@ -77,15 +77,12 @@ public: // GridCartesian / GridRedBlackCartesian //////////////////////////////////////////////////////////////// virtual int CheckerBoarded(int dim)=0; - virtual int CheckerBoard(std::vector site)=0; + virtual int CheckerBoard(std::vector &site)=0; virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0; virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0; virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int cb)=0; - int CheckerBoardFromOindex (int Oindex){ - std::vector ocoor; - oCoorFromOindex(ocoor,Oindex); - return CheckerBoard(ocoor); - } + virtual int CheckerBoardFromOindex (int Oindex)=0; + virtual int CheckerBoardFromOindexTable (int Oindex)=0; ////////////////////////////////////////////////////////////////////////////////////////////// // Local layout calculations diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h index 1f8f7514..b0d20441 100644 --- a/lib/cartesian/Cartesian_full.h +++ b/lib/cartesian/Cartesian_full.h @@ -39,10 +39,17 @@ class GridCartesian: public GridBase { public: + virtual int CheckerBoardFromOindexTable (int Oindex) { + return 0; + } + virtual int CheckerBoardFromOindex (int Oindex) + { + return 0; + } virtual int CheckerBoarded(int dim){ return 0; } - virtual int CheckerBoard(std::vector site){ + virtual int CheckerBoard(std::vector &site){ return 0; } virtual int CheckerBoardDestination(int cb,int shift,int dim){ diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h index 9f5a1103..6a4300d7 100644 --- a/lib/cartesian/Cartesian_red_black.h +++ b/lib/cartesian/Cartesian_red_black.h @@ -43,12 +43,13 @@ class GridRedBlackCartesian : public GridBase public: std::vector _checker_dim_mask; int _checker_dim; + std::vector _checker_board; virtual int CheckerBoarded(int dim){ if( dim==_checker_dim) return 1; else return 0; } - virtual int CheckerBoard(std::vector site){ + virtual int CheckerBoard(std::vector &site){ int linear=0; assert(site.size()==_ndimension); for(int d=0;d<_ndimension;d++){ @@ -72,12 +73,20 @@ public: // or by looping over x,y,z and multiply rather than computing checkerboard. if ( (source_cb+ocb)&1 ) { - return (shift)/2; } else { return (shift+1)/2; } } + virtual int CheckerBoardFromOindexTable (int Oindex) { + return _checker_board[Oindex]; + } + virtual int CheckerBoardFromOindex (int Oindex) + { + std::vector ocoor; + oCoorFromOindex(ocoor,Oindex); + return CheckerBoard(ocoor); + } virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite){ if(dim != _checker_dim) return shift; @@ -169,7 +178,7 @@ public: // all elements of a simd vector must have same checkerboard. // If Ls vectorised, this must still be the case; e.g. dwf rb5d if ( _simd_layout[d]>1 ) { - if ( d != _checker_dim ) { + if ( checker_dim_mask[d] ) { assert( (_rdimensions[d]&0x1) == 0 ); } } @@ -185,6 +194,8 @@ public: _ostride[d] = _ostride[d-1]*_rdimensions[d-1]; _istride[d] = _istride[d-1]*_simd_layout[d-1]; } + + } //////////////////////////////////////////////////////////////////////////////////////////// @@ -205,6 +216,18 @@ public: _slice_nblock[d]=nblock; block = block*_rdimensions[d]; } + + //////////////////////////////////////////////// + // Create a checkerboard lookup table + //////////////////////////////////////////////// + int rvol = 1; + for(int d=0;d<_ndimension;d++){ + rvol=rvol * _rdimensions[d]; + } + _checker_board.resize(rvol); + for(int osite=0;osite<_osites;osite++){ + _checker_board[osite] = CheckerBoardFromOindex (osite); + } }; protected: diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc new file mode 100644 index 00000000..b003d867 --- /dev/null +++ b/lib/communicator/Communicator_base.cc @@ -0,0 +1,124 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./lib/communicator/Communicator_none.cc + + Copyright (C) 2015 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include "Grid.h" +namespace Grid { + +/////////////////////////////////////////////////////////////// +// Info that is setup once and indept of cartesian layout +/////////////////////////////////////////////////////////////// +void * CartesianCommunicator::ShmCommBuf; +uint64_t CartesianCommunicator::MAX_MPI_SHM_BYTES = 128*1024*1024; + +///////////////////////////////// +// Alloc, free shmem region +///////////////////////////////// +void *CartesianCommunicator::ShmBufferMalloc(size_t bytes){ + // bytes = (bytes+sizeof(vRealD))&(~(sizeof(vRealD)-1));// align up bytes + void *ptr = (void *)heap_top; + heap_top += bytes; + heap_bytes+= bytes; + if (heap_bytes >= MAX_MPI_SHM_BYTES) { + std::cout<< " ShmBufferMalloc exceeded shared heap size -- try increasing with --shm flag" < & CartesianCommunicator::ThisProcessorCoor(void) { return _processor_coor; }; +const std::vector & CartesianCommunicator::ProcessorGrid(void) { return _processors; }; +int CartesianCommunicator::ProcessorCount(void) { return _Nprocessors; }; + +//////////////////////////////////////////////////////////////////////////////// +// very VERY rarely (Log, serial RNG) we need world without a grid +//////////////////////////////////////////////////////////////////////////////// + +void CartesianCommunicator::GlobalSum(ComplexF &c) +{ + GlobalSumVector((float *)&c,2); +} +void CartesianCommunicator::GlobalSumVector(ComplexF *c,int N) +{ + GlobalSumVector((float *)c,2*N); +} +void CartesianCommunicator::GlobalSum(ComplexD &c) +{ + GlobalSumVector((double *)&c,2); +} +void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N) +{ + GlobalSumVector((double *)c,2*N); +} + +#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPI3L) + +void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, + void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes) +{ + SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes); +} +void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &waitall) +{ + SendToRecvFromComplete(waitall); +} +void CartesianCommunicator::StencilBarrier(void){}; + +commVector CartesianCommunicator::ShmBufStorageVector; + +void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; } + +void *CartesianCommunicator::ShmBuffer(int rank) { + return NULL; +} +void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { + return NULL; +} +void CartesianCommunicator::ShmInitGeneric(void){ + ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES); + ShmCommBuf=(void *)&ShmBufStorageVector[0]; +} + +#endif + +} + diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h index 94d277e9..94ad1093 100644 --- a/lib/communicator/Communicator_base.h +++ b/lib/communicator/Communicator_base.h @@ -1,3 +1,4 @@ + /************************************************************************************* Grid physics library, www.github.com/paboyle/Grid @@ -34,123 +35,196 @@ Author: Peter Boyle #ifdef GRID_COMMS_MPI #include #endif +#ifdef GRID_COMMS_MPI3 +#include +#endif +#ifdef GRID_COMMS_MPI3L +#include +#endif #ifdef GRID_COMMS_SHMEM #include #endif + namespace Grid { + class CartesianCommunicator { public: + // 65536 ranks per node adequate for now + // 128MB shared memory for comms enought for 48^4 local vol comms + // Give external control (command line override?) of this + + static const int MAXLOG2RANKSPERNODE = 16; + static uint64_t MAX_MPI_SHM_BYTES; + // Communicator should know nothing of the physics grid, only processor grid. + int _Nprocessors; // How many in all + std::vector _processors; // Which dimensions get relayed out over processors lanes. + int _processor; // linear processor rank + std::vector _processor_coor; // linear processor coordinate + unsigned long _ndimension; - int _Nprocessors; // How many in all - std::vector _processors; // Which dimensions get relayed out over processors lanes. - int _processor; // linear processor rank - std::vector _processor_coor; // linear processor coordinate - unsigned long _ndimension; - -#ifdef GRID_COMMS_MPI - MPI_Comm communicator; - typedef MPI_Request CommsRequest_t; +#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPI3L) + static MPI_Comm communicator_world; + MPI_Comm communicator; + typedef MPI_Request CommsRequest_t; #else - typedef int CommsRequest_t; + typedef int CommsRequest_t; #endif - static void Init(int *argc, char ***argv); + //////////////////////////////////////////////////////////////////// + // Helper functionality for SHM Windows common to all other impls + //////////////////////////////////////////////////////////////////// + // Longer term; drop this in favour of a master / slave model with + // cartesian communicator on a subset of ranks, slave ranks controlled + // by group leader with data xfer via shared memory + //////////////////////////////////////////////////////////////////// +#ifdef GRID_COMMS_MPI3 - // Constructor - CartesianCommunicator(const std::vector &pdimensions_in); + static int ShmRank; + static int ShmSize; + static int GroupRank; + static int GroupSize; + static int WorldRank; + static int WorldSize; - // Wraps MPI_Cart routines - void ShiftedRanks(int dim,int shift,int & source, int & dest); - int RankFromProcessorCoor(std::vector &coor); - void ProcessorCoorFromRank(int rank,std::vector &coor); + std::vector WorldDims; + std::vector GroupDims; + std::vector ShmDims; + + std::vector GroupCoor; + std::vector ShmCoor; + std::vector WorldCoor; - ///////////////////////////////// - // Grid information queries - ///////////////////////////////// - int IsBoss(void) { return _processor==0; }; - int BossRank(void) { return 0; }; - int ThisRank(void) { return _processor; }; - const std::vector & ThisProcessorCoor(void) { return _processor_coor; }; - const std::vector & ProcessorGrid(void) { return _processors; }; - int ProcessorCount(void) { return _Nprocessors; }; + static std::vector GroupRanks; + static std::vector MyGroup; + static int ShmSetup; + static MPI_Win ShmWindow; + static MPI_Comm ShmComm; + + std::vector LexicographicToWorldRank; + + static std::vector ShmCommBufs; - //////////////////////////////////////////////////////////// - // Reduction - //////////////////////////////////////////////////////////// - void GlobalSum(RealF &); - void GlobalSumVector(RealF *,int N); +#else + static void ShmInitGeneric(void); + static commVector ShmBufStorageVector; +#endif - void GlobalSum(RealD &); - void GlobalSumVector(RealD *,int N); + ///////////////////////////////// + // Grid information and queries + // Implemented in Communicator_base.C + ///////////////////////////////// + static void * ShmCommBuf; + size_t heap_top; + size_t heap_bytes; - void GlobalSum(uint32_t &); - void GlobalSum(uint64_t &); + void *ShmBufferSelf(void); + void *ShmBuffer(int rank); + void *ShmBufferTranslate(int rank,void * local_p); + void *ShmBufferMalloc(size_t bytes); + void ShmBufferFreeAll(void) ; + + //////////////////////////////////////////////// + // Must call in Grid startup + //////////////////////////////////////////////// + static void Init(int *argc, char ***argv); + + //////////////////////////////////////////////// + // Constructor of any given grid + //////////////////////////////////////////////// + CartesianCommunicator(const std::vector &pdimensions_in); + + //////////////////////////////////////////////////////////////////////////////////////// + // Wraps MPI_Cart routines, or implements equivalent on other impls + //////////////////////////////////////////////////////////////////////////////////////// + void ShiftedRanks(int dim,int shift,int & source, int & dest); + int RankFromProcessorCoor(std::vector &coor); + void ProcessorCoorFromRank(int rank,std::vector &coor); + + int IsBoss(void) ; + int BossRank(void) ; + int ThisRank(void) ; + const std::vector & ThisProcessorCoor(void) ; + const std::vector & ProcessorGrid(void) ; + int ProcessorCount(void) ; - void GlobalSum(ComplexF &c) - { - GlobalSumVector((float *)&c,2); - } - void GlobalSumVector(ComplexF *c,int N) - { - GlobalSumVector((float *)c,2*N); - } + //////////////////////////////////////////////////////////////////////////////// + // very VERY rarely (Log, serial RNG) we need world without a grid + //////////////////////////////////////////////////////////////////////////////// + static int RankWorld(void) ; + static void BroadcastWorld(int root,void* data, int bytes); + + //////////////////////////////////////////////////////////// + // Reduction + //////////////////////////////////////////////////////////// + void GlobalSum(RealF &); + void GlobalSumVector(RealF *,int N); + void GlobalSum(RealD &); + void GlobalSumVector(RealD *,int N); + void GlobalSum(uint32_t &); + void GlobalSum(uint64_t &); + void GlobalSum(ComplexF &c); + void GlobalSumVector(ComplexF *c,int N); + void GlobalSum(ComplexD &c); + void GlobalSumVector(ComplexD *c,int N); + + template void GlobalSum(obj &o){ + typedef typename obj::scalar_type scalar_type; + int words = sizeof(obj)/sizeof(scalar_type); + scalar_type * ptr = (scalar_type *)& o; + GlobalSumVector(ptr,words); + } + + //////////////////////////////////////////////////////////// + // Face exchange, buffer swap in translational invariant way + //////////////////////////////////////////////////////////// + void SendToRecvFrom(void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes); + + void SendRecvPacket(void *xmit, + void *recv, + int xmit_to_rank, + int recv_from_rank, + int bytes); + + void SendToRecvFromBegin(std::vector &list, + void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes); + + void SendToRecvFromComplete(std::vector &waitall); - void GlobalSum(ComplexD &c) - { - GlobalSumVector((double *)&c,2); - } - void GlobalSumVector(ComplexD *c,int N) - { - GlobalSumVector((double *)c,2*N); - } - - template void GlobalSum(obj &o){ - typedef typename obj::scalar_type scalar_type; - int words = sizeof(obj)/sizeof(scalar_type); - scalar_type * ptr = (scalar_type *)& o; - GlobalSumVector(ptr,words); - } - //////////////////////////////////////////////////////////// - // Face exchange, buffer swap in translational invariant way - //////////////////////////////////////////////////////////// - void SendToRecvFrom(void *xmit, - int xmit_to_rank, - void *recv, - int recv_from_rank, - int bytes); + void StencilSendToRecvFromBegin(std::vector &list, + void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes); + + void StencilSendToRecvFromComplete(std::vector &waitall); + void StencilBarrier(void); - void SendRecvPacket(void *xmit, - void *recv, - int xmit_to_rank, - int recv_from_rank, - int bytes); - - void SendToRecvFromBegin(std::vector &list, - void *xmit, - int xmit_to_rank, - void *recv, - int recv_from_rank, - int bytes); - void SendToRecvFromComplete(std::vector &waitall); - - //////////////////////////////////////////////////////////// - // Barrier - //////////////////////////////////////////////////////////// - void Barrier(void); - - //////////////////////////////////////////////////////////// - // Broadcast a buffer and composite larger - //////////////////////////////////////////////////////////// - void Broadcast(int root,void* data, int bytes); - template void Broadcast(int root,obj &data) + //////////////////////////////////////////////////////////// + // Barrier + //////////////////////////////////////////////////////////// + void Barrier(void); + + //////////////////////////////////////////////////////////// + // Broadcast a buffer and composite larger + //////////////////////////////////////////////////////////// + void Broadcast(int root,void* data, int bytes); + + template void Broadcast(int root,obj &data) { Broadcast(root,(void *)&data,sizeof(data)); }; - static void BroadcastWorld(int root,void* data, int bytes); - }; } diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index dff9811a..65ced9c7 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -30,21 +30,23 @@ Author: Peter Boyle namespace Grid { - // Should error check all MPI calls. + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Info that is setup once and indept of cartesian layout +/////////////////////////////////////////////////////////////////////////////////////////////////// +MPI_Comm CartesianCommunicator::communicator_world; + +// Should error check all MPI calls. void CartesianCommunicator::Init(int *argc, char ***argv) { int flag; MPI_Initialized(&flag); // needed to coexist with other libs apparently if ( !flag ) { MPI_Init(argc,argv); } + MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world); + ShmInitGeneric(); } - int Rank(void) { - int pe; - MPI_Comm_rank(MPI_COMM_WORLD,&pe); - return pe; - } - CartesianCommunicator::CartesianCommunicator(const std::vector &processors) { _ndimension = processors.size(); @@ -54,7 +56,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors) _processors = processors; _processor_coor.resize(_ndimension); - MPI_Cart_create(MPI_COMM_WORLD, _ndimension,&_processors[0],&periodic[0],1,&communicator); + MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator); MPI_Comm_rank(communicator,&_processor); MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); @@ -67,7 +69,6 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors) assert(Size==_Nprocessors); } - void CartesianCommunicator::GlobalSum(uint32_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); assert(ierr==0); @@ -168,7 +169,6 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector & int nreq=list.size(); std::vector status(nreq); int ierr = MPI_Waitall(nreq,&list[0],&status[0]); - assert(ierr==0); } @@ -187,14 +187,22 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes) communicator); assert(ierr==0); } - + /////////////////////////////////////////////////////// + // Should only be used prior to Grid Init finished. + // Check for this? + /////////////////////////////////////////////////////// +int CartesianCommunicator::RankWorld(void){ + int r; + MPI_Comm_rank(communicator_world,&r); + return r; +} void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { int ierr= MPI_Bcast(data, bytes, MPI_BYTE, root, - MPI_COMM_WORLD); + communicator_world); assert(ierr==0); } diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc new file mode 100644 index 00000000..c707ec1f --- /dev/null +++ b/lib/communicator/Communicator_mpi3.cc @@ -0,0 +1,580 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./lib/communicator/Communicator_mpi.cc + + Copyright (C) 2015 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include "Grid.h" +#include + +namespace Grid { + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Info that is setup once and indept of cartesian layout +/////////////////////////////////////////////////////////////////////////////////////////////////// +int CartesianCommunicator::ShmSetup = 0; + +int CartesianCommunicator::ShmRank; +int CartesianCommunicator::ShmSize; +int CartesianCommunicator::GroupRank; +int CartesianCommunicator::GroupSize; +int CartesianCommunicator::WorldRank; +int CartesianCommunicator::WorldSize; + +MPI_Comm CartesianCommunicator::communicator_world; +MPI_Comm CartesianCommunicator::ShmComm; +MPI_Win CartesianCommunicator::ShmWindow; + +std::vector CartesianCommunicator::GroupRanks; +std::vector CartesianCommunicator::MyGroup; +std::vector CartesianCommunicator::ShmCommBufs; + +void *CartesianCommunicator::ShmBufferSelf(void) +{ + return ShmCommBufs[ShmRank]; +} +void *CartesianCommunicator::ShmBuffer(int rank) +{ + int gpeer = GroupRanks[rank]; + if (gpeer == MPI_UNDEFINED){ + return NULL; + } else { + return ShmCommBufs[gpeer]; + } +} +void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) +{ + int gpeer = GroupRanks[rank]; + if (gpeer == MPI_UNDEFINED){ + return NULL; + } else { + uint64_t offset = (uint64_t)local_p - (uint64_t)ShmCommBufs[ShmRank]; + uint64_t remote = (uint64_t)ShmCommBufs[gpeer]+offset; + return (void *) remote; + } +} + +void CartesianCommunicator::Init(int *argc, char ***argv) { + int flag; + MPI_Initialized(&flag); // needed to coexist with other libs apparently + if ( !flag ) { + MPI_Init(argc,argv); + } + + MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world); + MPI_Comm_rank(communicator_world,&WorldRank); + MPI_Comm_size(communicator_world,&WorldSize); + + ///////////////////////////////////////////////////////////////////// + // Split into groups that can share memory + ///////////////////////////////////////////////////////////////////// + MPI_Comm_split_type(communicator_world, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm); + MPI_Comm_rank(ShmComm ,&ShmRank); + MPI_Comm_size(ShmComm ,&ShmSize); + GroupSize = WorldSize/ShmSize; + + ///////////////////////////////////////////////////////////////////// + // find world ranks in our SHM group (i.e. which ranks are on our node) + ///////////////////////////////////////////////////////////////////// + MPI_Group WorldGroup, ShmGroup; + MPI_Comm_group (communicator_world, &WorldGroup); + MPI_Comm_group (ShmComm, &ShmGroup); + + std::vector world_ranks(WorldSize); + GroupRanks.resize(WorldSize); + for(int r=0;r()); + int myleader = MyGroup[0]; + + std::vector leaders_1hot(WorldSize,0); + std::vector leaders_group(GroupSize,0); + leaders_1hot [ myleader ] = 1; + + /////////////////////////////////////////////////////////////////// + // global sum leaders over comm world + /////////////////////////////////////////////////////////////////// + int ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,communicator_world); + assert(ierr==0); + + /////////////////////////////////////////////////////////////////// + // find the group leaders world rank + /////////////////////////////////////////////////////////////////// + int group=0; + for(int l=0;l + for(uint64_t page=0;page coor = _processor_coor; + + assert(std::abs(shift) <_processors[dim]); + + coor[dim] = (_processor_coor[dim] + shift + _processors[dim])%_processors[dim]; + Lexicographic::IndexFromCoor(coor,source,_processors); + source = LexicographicToWorldRank[source]; + + coor[dim] = (_processor_coor[dim] - shift + _processors[dim])%_processors[dim]; + Lexicographic::IndexFromCoor(coor,dest,_processors); + dest = LexicographicToWorldRank[dest]; +} +int CartesianCommunicator::RankFromProcessorCoor(std::vector &coor) +{ + int rank; + Lexicographic::IndexFromCoor(coor,rank,_processors); + rank = LexicographicToWorldRank[rank]; + return rank; +} +void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &coor) +{ + Lexicographic::CoorFromIndex(coor,rank,_processors); + rank = LexicographicToWorldRank[rank]; +} + +CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +{ + int ierr; + + communicator=communicator_world; + + _ndimension = processors.size(); + + //////////////////////////////////////////////////////////////// + // Assert power of two shm_size. + //////////////////////////////////////////////////////////////// + int log2size = -1; + for(int i=0;i<=MAXLOG2RANKSPERNODE;i++){ + if ( (0x1< WorldDims = processors; + + ShmDims.resize(_ndimension,1); + GroupDims.resize(_ndimension); + + ShmCoor.resize(_ndimension); + GroupCoor.resize(_ndimension); + WorldCoor.resize(_ndimension); + + for(int l2=0;l2 reqs(0); + SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes); + SendToRecvFromComplete(reqs); +} + +void CartesianCommunicator::SendRecvPacket(void *xmit, + void *recv, + int sender, + int receiver, + int bytes) +{ + MPI_Status stat; + assert(sender != receiver); + int tag = sender; + if ( _processor == sender ) { + MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator); + } + if ( _processor == receiver ) { + MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat); + } +} + +// Basic Halo comms primitive +void CartesianCommunicator::SendToRecvFromBegin(std::vector &list, + void *xmit, + int dest, + void *recv, + int from, + int bytes) +{ +#if 0 + this->StencilBarrier(); + + MPI_Request xrq; + MPI_Request rrq; + + static int sequence; + + int ierr; + int tag; + int check; + + assert(dest != _processor); + assert(from != _processor); + + int gdest = GroupRanks[dest]; + int gfrom = GroupRanks[from]; + int gme = GroupRanks[_processor]; + + sequence++; + + char *from_ptr = (char *)ShmCommBufs[ShmRank]; + + int small = (bytesStencilBarrier(); + + if (small && (gfrom !=MPI_UNDEFINED) ) { + T *ip = (T *)from_ptr; + T *op = (T *)recv; +PARALLEL_FOR_LOOP + for(int w=0;wStencilBarrier(); + +#else + MPI_Request xrq; + MPI_Request rrq; + int rank = _processor; + int ierr; + ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq); + ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq); + + assert(ierr==0); + + list.push_back(xrq); + list.push_back(rrq); +#endif +} + +void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, + void *xmit, + int dest, + void *recv, + int from, + int bytes) +{ + MPI_Request xrq; + MPI_Request rrq; + + int ierr; + + assert(dest != _processor); + assert(from != _processor); + + int gdest = GroupRanks[dest]; + int gfrom = GroupRanks[from]; + int gme = GroupRanks[_processor]; + + assert(gme == ShmRank); + + if ( gdest == MPI_UNDEFINED ) { + ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq); + assert(ierr==0); + list.push_back(xrq); + } + + if ( gfrom ==MPI_UNDEFINED) { + ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq); + assert(ierr==0); + list.push_back(rrq); + } + +} + + +void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &list) +{ + SendToRecvFromComplete(list); +} + +void CartesianCommunicator::StencilBarrier(void) +{ + MPI_Win_sync (ShmWindow); + MPI_Barrier (ShmComm); + MPI_Win_sync (ShmWindow); +} + +void CartesianCommunicator::SendToRecvFromComplete(std::vector &list) +{ + int nreq=list.size(); + std::vector status(nreq); + int ierr = MPI_Waitall(nreq,&list[0],&status[0]); + assert(ierr==0); +} + +void CartesianCommunicator::Barrier(void) +{ + int ierr = MPI_Barrier(communicator); + assert(ierr==0); +} + +void CartesianCommunicator::Broadcast(int root,void* data, int bytes) +{ + int ierr=MPI_Bcast(data, + bytes, + MPI_BYTE, + root, + communicator); + assert(ierr==0); +} + +void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) +{ + int ierr= MPI_Bcast(data, + bytes, + MPI_BYTE, + root, + communicator_world); + assert(ierr==0); +} + +} + diff --git a/lib/communicator/Communicator_mpi3_leader.cc b/lib/communicator/Communicator_mpi3_leader.cc new file mode 100644 index 00000000..71f1a913 --- /dev/null +++ b/lib/communicator/Communicator_mpi3_leader.cc @@ -0,0 +1,874 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./lib/communicator/Communicator_mpi.cc + + Copyright (C) 2015 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include "Grid.h" +#include + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// Workarounds: +/// i) bloody mac os doesn't implement unnamed semaphores since it is "optional" posix. +/// darwin dispatch semaphores don't seem to be multiprocess. +/// +/// ii) openmpi under --mca shmem posix works with two squadrons per node; +/// openmpi under default mca settings (I think --mca shmem mmap) on MacOS makes two squadrons map the SAME +/// memory as each other, despite their living on different communicators. This appears to be a bug in OpenMPI. +/// +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +typedef sem_t *Grid_semaphore; + +#define SEM_INIT(S) S = sem_open(sem_name,0,0600,0); assert ( S != SEM_FAILED ); +#define SEM_INIT_EXCL(S) sem_unlink(sem_name); S = sem_open(sem_name,O_CREAT|O_EXCL,0600,0); assert ( S != SEM_FAILED ); +#define SEM_POST(S) assert ( sem_post(S) == 0 ); +#define SEM_WAIT(S) assert ( sem_wait(S) == 0 ); + +#include + +namespace Grid { + +enum { COMMAND_ISEND, COMMAND_IRECV, COMMAND_WAITALL }; + +struct Descriptor { + uint64_t buf; + size_t bytes; + int rank; + int tag; + int command; + MPI_Request request; +}; + +const int pool = 48; + +class SlaveState { +public: + volatile int head; + volatile int start; + volatile int tail; + volatile Descriptor Descrs[pool]; +}; + +class Slave { +public: + Grid_semaphore sem_head; + Grid_semaphore sem_tail; + SlaveState *state; + MPI_Comm squadron; + uint64_t base; + int universe_rank; + int vertical_rank; + char sem_name [NAME_MAX]; + //////////////////////////////////////////////////////////// + // Descriptor circular pointers + //////////////////////////////////////////////////////////// + Slave() {}; + + void Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank); + + void SemInit(void) { + sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank); + // printf("SEM_NAME: %s \n",sem_name); + SEM_INIT(sem_head); + sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank); + // printf("SEM_NAME: %s \n",sem_name); + SEM_INIT(sem_tail); + } + void SemInitExcl(void) { + sprintf(sem_name,"/Grid_mpi3_sem_head_%d",universe_rank); + // printf("SEM_INIT_EXCL: %s \n",sem_name); + SEM_INIT_EXCL(sem_head); + sprintf(sem_name,"/Grid_mpi3_sem_tail_%d",universe_rank); + // printf("SEM_INIT_EXCL: %s \n",sem_name); + SEM_INIT_EXCL(sem_tail); + } + void WakeUpDMA(void) { + SEM_POST(sem_head); + }; + void WakeUpCompute(void) { + SEM_POST(sem_tail); + }; + void WaitForCommand(void) { + SEM_WAIT(sem_head); + }; + void WaitForComplete(void) { + SEM_WAIT(sem_tail); + }; + void EventLoop (void) { + // std::cout<< " Entering event loop "<tail == state->head ); + } +}; + +//////////////////////////////////////////////////////////////////////// +// One instance of a data mover. +// Master and Slave must agree on location in shared memory +//////////////////////////////////////////////////////////////////////// + +class MPIoffloadEngine { +public: + + static std::vector Slaves; + + static int ShmSetup; + + static int UniverseRank; + static int UniverseSize; + + static MPI_Comm communicator_universe; + static MPI_Comm communicator_cached; + + static MPI_Comm HorizontalComm; + static int HorizontalRank; + static int HorizontalSize; + + static MPI_Comm VerticalComm; + static MPI_Win VerticalWindow; + static int VerticalSize; + static int VerticalRank; + + static std::vector VerticalShmBufs; + static std::vector > UniverseRanks; + static std::vector UserCommunicatorToWorldRanks; + + static MPI_Group WorldGroup, CachedGroup; + + static void CommunicatorInit (MPI_Comm &communicator_world, + MPI_Comm &ShmComm, + void * &ShmCommBuf); + + static void MapCommRankToWorldRank(int &hashtag, int & comm_world_peer,int tag, MPI_Comm comm,int commrank); + + ///////////////////////////////////////////////////////// + // routines for master proc must handle any communicator + ///////////////////////////////////////////////////////// + + static void QueueSend(int slave,void *buf, int bytes, int tag, MPI_Comm comm,int rank) { + // std::cout<< " Queueing send "<< bytes<< " slave "<< slave << " to comm "<= units ) { + mywork = myoff = 0; + } else { + mywork = (nwork+me)/units; + myoff = basework * me; + if ( me > backfill ) + myoff+= (me-backfill); + } + return; + }; + + static void QueueMultiplexedSend(void *buf, int bytes, int tag, MPI_Comm comm,int rank) { + uint8_t * cbuf = (uint8_t *) buf; + int mywork, myoff, procs; + procs = VerticalSize-1; + for(int s=0;s MPIoffloadEngine::Slaves; + +int MPIoffloadEngine::UniverseRank; +int MPIoffloadEngine::UniverseSize; + +MPI_Comm MPIoffloadEngine::communicator_universe; +MPI_Comm MPIoffloadEngine::communicator_cached; +MPI_Group MPIoffloadEngine::WorldGroup; +MPI_Group MPIoffloadEngine::CachedGroup; + +MPI_Comm MPIoffloadEngine::HorizontalComm; +int MPIoffloadEngine::HorizontalRank; +int MPIoffloadEngine::HorizontalSize; + +MPI_Comm MPIoffloadEngine::VerticalComm; +int MPIoffloadEngine::VerticalSize; +int MPIoffloadEngine::VerticalRank; +MPI_Win MPIoffloadEngine::VerticalWindow; +std::vector MPIoffloadEngine::VerticalShmBufs; +std::vector > MPIoffloadEngine::UniverseRanks; +std::vector MPIoffloadEngine::UserCommunicatorToWorldRanks; + +int MPIoffloadEngine::ShmSetup = 0; + +void MPIoffloadEngine::CommunicatorInit (MPI_Comm &communicator_world, + MPI_Comm &ShmComm, + void * &ShmCommBuf) +{ + int flag; + assert(ShmSetup==0); + + ////////////////////////////////////////////////////////////////////// + // Universe is all nodes prior to squadron grouping + ////////////////////////////////////////////////////////////////////// + MPI_Comm_dup (MPI_COMM_WORLD,&communicator_universe); + MPI_Comm_rank(communicator_universe,&UniverseRank); + MPI_Comm_size(communicator_universe,&UniverseSize); + + ///////////////////////////////////////////////////////////////////// + // Split into groups that can share memory (Verticals) + ///////////////////////////////////////////////////////////////////// +#undef MPI_SHARED_MEM_DEBUG +#ifdef MPI_SHARED_MEM_DEBUG + MPI_Comm_split(communicator_universe,(UniverseRank/4),UniverseRank,&VerticalComm); +#else + MPI_Comm_split_type(communicator_universe, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&VerticalComm); +#endif + MPI_Comm_rank(VerticalComm ,&VerticalRank); + MPI_Comm_size(VerticalComm ,&VerticalSize); + + ////////////////////////////////////////////////////////////////////// + // Split into horizontal groups by rank in squadron + ////////////////////////////////////////////////////////////////////// + MPI_Comm_split(communicator_universe,VerticalRank,UniverseRank,&HorizontalComm); + MPI_Comm_rank(HorizontalComm,&HorizontalRank); + MPI_Comm_size(HorizontalComm,&HorizontalSize); + assert(HorizontalSize*VerticalSize==UniverseSize); + + //////////////////////////////////////////////////////////////////////////////// + // What is my place in the world + //////////////////////////////////////////////////////////////////////////////// + int WorldRank=0; + if(VerticalRank==0) WorldRank = HorizontalRank; + int ierr=MPI_Allreduce(MPI_IN_PLACE,&WorldRank,1,MPI_INT,MPI_SUM,VerticalComm); + assert(ierr==0); + + //////////////////////////////////////////////////////////////////////////////// + // Where is the world in the universe? + //////////////////////////////////////////////////////////////////////////////// + UniverseRanks = std::vector >(HorizontalSize,std::vector(VerticalSize,0)); + UniverseRanks[WorldRank][VerticalRank] = UniverseRank; + for(int w=0;w0 ) size = sizeof(SlaveState); + + sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",WorldRank,r); + + shm_unlink(shm_name); + + int fd=shm_open(shm_name,O_RDWR|O_CREAT,0600); + if ( fd < 0 ) { + perror("failed shm_open"); + assert(0); + } + + ftruncate(fd, size); + + VerticalShmBufs[r] = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + if ( VerticalShmBufs[r] == MAP_FAILED ) { + perror("failed mmap"); + assert(0); + } + + uint64_t * check = (uint64_t *) VerticalShmBufs[r]; + check[0] = WorldRank; + check[1] = r; + + // std::cout<<"SHM "<0 ) size = sizeof(SlaveState); + + sprintf(shm_name,"/Grid_mpi3_shm_%d_%d",WorldRank,r); + + int fd=shm_open(shm_name,O_RDWR|O_CREAT,0600); + if ( fd<0 ) { + perror("failed shm_open"); + assert(0); + } + VerticalShmBufs[r] = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + uint64_t * check = (uint64_t *) VerticalShmBufs[r]; + assert(check[0]== WorldRank); + assert(check[1]== r); + std::cerr<<"SHM "<"<"< cached_ranks(size); + + for(int r=0;r"<>0 )&0xFFFF)^((icomm>>16)&0xFFFF) + ^ ((icomm>>32)&0xFFFF)^((icomm>>48)&0xFFFF); + + // hashtag = (comm_hash<<15) | tag; + hashtag = tag; + +}; + +void Slave::Init(SlaveState * _state,MPI_Comm _squadron,int _universe_rank,int _vertical_rank) +{ + squadron=_squadron; + universe_rank=_universe_rank; + vertical_rank=_vertical_rank; + state =_state; + // std::cout << "state "<<_state<<" comm "<<_squadron<<" universe_rank"<head = state->tail = state->start = 0; + base = (uint64_t)MPIoffloadEngine::VerticalShmBufs[0]; + int rank; MPI_Comm_rank(_squadron,&rank); +} +#define PERI_PLUS(A) ( (A+1)%pool ) +int Slave::Event (void) { + + static int tail_last; + static int head_last; + static int start_last; + int ierr; + + //////////////////////////////////////////////////// + // Try to advance the start pointers + //////////////////////////////////////////////////// + int s=state->start; + if ( s != state->head ) { + switch ( state->Descrs[s].command ) { + case COMMAND_ISEND: + /* + std::cout<< " Send "<Descrs[s].buf<< "["<Descrs[s].bytes<<"]" + << " to " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag + << " Comm " << MPIoffloadEngine::communicator_universe<< " me " <Descrs[s].buf+base), + state->Descrs[s].bytes, + MPI_CHAR, + state->Descrs[s].rank, + state->Descrs[s].tag, + MPIoffloadEngine::communicator_universe, + (MPI_Request *)&state->Descrs[s].request); + assert(ierr==0); + state->start = PERI_PLUS(s); + return 1; + break; + + case COMMAND_IRECV: + /* + std::cout<< " Recv "<Descrs[s].buf<< "["<Descrs[s].bytes<<"]" + << " from " << state->Descrs[s].rank<< " tag" << state->Descrs[s].tag + << " Comm " << MPIoffloadEngine::communicator_universe<< " me "<< universe_rank<< std::endl; + */ + ierr=MPI_Irecv((void *)(state->Descrs[s].buf+base), + state->Descrs[s].bytes, + MPI_CHAR, + state->Descrs[s].rank, + state->Descrs[s].tag, + MPIoffloadEngine::communicator_universe, + (MPI_Request *)&state->Descrs[s].request); + + // std::cout<< " Request is "<Descrs[s].request<Descrs[0].request<start = PERI_PLUS(s); + return 1; + break; + + case COMMAND_WAITALL: + + for(int t=state->tail;t!=s; t=PERI_PLUS(t) ){ + MPI_Wait((MPI_Request *)&state->Descrs[t].request,MPI_STATUS_IGNORE); + }; + s=PERI_PLUS(s); + state->start = s; + state->tail = s; + + WakeUpCompute(); + + return 1; + break; + + default: + assert(0); + break; + } + } + return 0; +} + ////////////////////////////////////////////////////////////////////////////// + // External interaction with the queue + ////////////////////////////////////////////////////////////////////////////// + +uint64_t Slave::QueueCommand(int command,void *buf, int bytes, int tag, MPI_Comm comm,int commrank) +{ + ///////////////////////////////////////// + // Spin; if FIFO is full until not full + ///////////////////////////////////////// + int head =state->head; + int next = PERI_PLUS(head); + + // Set up descriptor + int worldrank; + int hashtag; + MPI_Comm communicator; + MPI_Request request; + + MPIoffloadEngine::MapCommRankToWorldRank(hashtag,worldrank,tag,comm,commrank); + + uint64_t relative= (uint64_t)buf - base; + state->Descrs[head].buf = relative; + state->Descrs[head].bytes = bytes; + state->Descrs[head].rank = MPIoffloadEngine::UniverseRanks[worldrank][vertical_rank]; + state->Descrs[head].tag = hashtag; + state->Descrs[head].command= command; + + /* + if ( command == COMMAND_ISEND ) { + std::cout << "QueueSend from "<< universe_rank <<" to commrank " << commrank + << " to worldrank " << worldrank <tail==next ); + + // Msync on weak order architectures + // Advance pointer + state->head = next; + + return 0; +} + + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Info that is setup once and indept of cartesian layout +/////////////////////////////////////////////////////////////////////////////////////////////////// + +MPI_Comm CartesianCommunicator::communicator_world; + +void CartesianCommunicator::Init(int *argc, char ***argv) +{ + int flag; + MPI_Initialized(&flag); // needed to coexist with other libs apparently + if ( !flag ) { + MPI_Init(argc,argv); + } + communicator_world = MPI_COMM_WORLD; + MPI_Comm ShmComm; + MPIoffloadEngine::CommunicatorInit (communicator_world,ShmComm,ShmCommBuf); +} +void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest) +{ + int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest); + assert(ierr==0); +} +int CartesianCommunicator::RankFromProcessorCoor(std::vector &coor) +{ + int rank; + int ierr=MPI_Cart_rank (communicator, &coor[0], &rank); + assert(ierr==0); + return rank; +} +void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &coor) +{ + coor.resize(_ndimension); + int ierr=MPI_Cart_coords (communicator, rank, _ndimension,&coor[0]); + assert(ierr==0); +} + +CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +{ + _ndimension = processors.size(); + std::vector periodic(_ndimension,1); + + _Nprocessors=1; + _processors = processors; + + for(int i=0;i<_ndimension;i++){ + _Nprocessors*=_processors[i]; + } + + int Size; + MPI_Comm_size(communicator_world,&Size); + assert(Size==_Nprocessors); + + _processor_coor.resize(_ndimension); + MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator); + MPI_Comm_rank (communicator,&_processor); + MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); +}; + +void CartesianCommunicator::GlobalSum(uint32_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSum(uint64_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSum(float &f){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSumVector(float *f,int N) +{ + int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSum(double &d) +{ + int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSumVector(double *d,int N) +{ + int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator); + assert(ierr==0); +} + +// Basic Halo comms primitive +void CartesianCommunicator::SendToRecvFrom(void *xmit, + int dest, + void *recv, + int from, + int bytes) +{ + std::vector reqs(0); + SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes); + SendToRecvFromComplete(reqs); +} + +void CartesianCommunicator::SendRecvPacket(void *xmit, + void *recv, + int sender, + int receiver, + int bytes) +{ + MPI_Status stat; + assert(sender != receiver); + int tag = sender; + if ( _processor == sender ) { + MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator); + } + if ( _processor == receiver ) { + MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat); + } +} + +// Basic Halo comms primitive +void CartesianCommunicator::SendToRecvFromBegin(std::vector &list, + void *xmit, + int dest, + void *recv, + int from, + int bytes) +{ + MPI_Request xrq; + MPI_Request rrq; + int rank = _processor; + int ierr; + ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq); + ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq); + + assert(ierr==0); + + list.push_back(xrq); + list.push_back(rrq); +} + +void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, + void *xmit, + int dest, + void *recv, + int from, + int bytes) +{ + uint64_t xmit_i = (uint64_t) xmit; + uint64_t recv_i = (uint64_t) recv; + uint64_t shm = (uint64_t) ShmCommBuf; + // assert xmit and recv lie in shared memory region + assert( (xmit_i >= shm) && (xmit_i+bytes <= shm+MAX_MPI_SHM_BYTES) ); + assert( (recv_i >= shm) && (recv_i+bytes <= shm+MAX_MPI_SHM_BYTES) ); + assert(from!=_processor); + assert(dest!=_processor); + MPIoffloadEngine::QueueMultiplexedSend(xmit,bytes,_processor,communicator,dest); + MPIoffloadEngine::QueueMultiplexedRecv(recv,bytes,from,communicator,from); +} + + +void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &list) +{ + MPIoffloadEngine::WaitAll(); +} + +void CartesianCommunicator::StencilBarrier(void) +{ +} + +void CartesianCommunicator::SendToRecvFromComplete(std::vector &list) +{ + int nreq=list.size(); + std::vector status(nreq); + int ierr = MPI_Waitall(nreq,&list[0],&status[0]); + assert(ierr==0); +} + +void CartesianCommunicator::Barrier(void) +{ + int ierr = MPI_Barrier(communicator); + assert(ierr==0); +} + +void CartesianCommunicator::Broadcast(int root,void* data, int bytes) +{ + int ierr=MPI_Bcast(data, + bytes, + MPI_BYTE, + root, + communicator); + assert(ierr==0); +} + +void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) +{ + int ierr= MPI_Bcast(data, + bytes, + MPI_BYTE, + root, + communicator_world); + assert(ierr==0); +} + +void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; } + +void *CartesianCommunicator::ShmBuffer(int rank) { + return NULL; +} +void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { + return NULL; +} + + +}; + diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc index 8601255a..5e91b305 100644 --- a/lib/communicator/Communicator_none.cc +++ b/lib/communicator/Communicator_none.cc @@ -28,12 +28,15 @@ Author: Peter Boyle #include "Grid.h" namespace Grid { +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Info that is setup once and indept of cartesian layout +/////////////////////////////////////////////////////////////////////////////////////////////////// + void CartesianCommunicator::Init(int *argc, char *** arv) { + ShmInitGeneric(); } -int Rank(void ){ return 0; }; - CartesianCommunicator::CartesianCommunicator(const std::vector &processors) { _processors = processors; @@ -89,30 +92,17 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector & assert(0); } -void CartesianCommunicator::Barrier(void) -{ -} - -void CartesianCommunicator::Broadcast(int root,void* data, int bytes) -{ -} -void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) -{ -} - - +int CartesianCommunicator::RankWorld(void){return 0;} +void CartesianCommunicator::Barrier(void){} +void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {} +void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { } +int CartesianCommunicator::RankFromProcessorCoor(std::vector &coor) { return 0;} +void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &coor){ coor = _processor_coor ;} void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest) { source =0; dest=0; } -int CartesianCommunicator::RankFromProcessorCoor(std::vector &coor) -{ - return 0; -} -void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &coor) -{ -} } diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc index 091e266e..56e03224 100644 --- a/lib/communicator/Communicator_shmem.cc +++ b/lib/communicator/Communicator_shmem.cc @@ -39,14 +39,24 @@ namespace Grid { BACKTRACEFILE(); \ }\ } -int Rank(void) { - return shmem_my_pe(); -} + + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Info that is setup once and indept of cartesian layout +/////////////////////////////////////////////////////////////////////////////////////////////////// + typedef struct HandShake_t { uint64_t seq_local; uint64_t seq_remote; } HandShake; +std::array make_psync_init(void) { + array ret; + ret.fill(SHMEM_SYNC_VALUE); + return ret; +} +static std::array psync_init = make_psync_init(); + static Vector< HandShake > XConnections; static Vector< HandShake > RConnections; @@ -61,7 +71,9 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { RConnections[pe].seq_remote= 0; } shmem_barrier_all(); + ShmInitGeneric(); } + CartesianCommunicator::CartesianCommunicator(const std::vector &processors) { _ndimension = processors.size(); @@ -89,7 +101,7 @@ void CartesianCommunicator::GlobalSum(uint32_t &u){ static long long source ; static long long dest ; static long long llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE]; - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; // int nreduce=1; // int pestart=0; @@ -105,7 +117,7 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){ static long long source ; static long long dest ; static long long llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE]; - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; // int nreduce=1; // int pestart=0; @@ -121,7 +133,7 @@ void CartesianCommunicator::GlobalSum(float &f){ static float source ; static float dest ; static float llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE]; - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; source = f; dest =0.0; @@ -133,7 +145,7 @@ void CartesianCommunicator::GlobalSumVector(float *f,int N) static float source ; static float dest = 0 ; static float llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE]; - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; if ( shmem_addr_accessible(f,_processor) ){ shmem_float_sum_to_all(f,f,N,0,0,_Nprocessors,llwrk,psync); @@ -152,7 +164,7 @@ void CartesianCommunicator::GlobalSum(double &d) static double source; static double dest ; static double llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE]; - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; source = d; dest = 0; @@ -164,7 +176,8 @@ void CartesianCommunicator::GlobalSumVector(double *d,int N) static double source ; static double dest ; static double llwrk[_SHMEM_REDUCE_MIN_WRKDATA_SIZE]; - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; + if ( shmem_addr_accessible(d,_processor) ){ shmem_double_sum_to_all(d,d,N,0,0,_Nprocessors,llwrk,psync); @@ -230,12 +243,9 @@ void CartesianCommunicator::SendRecvPacket(void *xmit, if ( _processor == sender ) { - printf("Sender SHMEM pt2pt %d -> %d\n",sender,receiver); // Check he has posted a receive while(SendSeq->seq_remote == SendSeq->seq_local); - printf("Sender receive %d posted\n",sender,receiver); - // Advance our send count seq = ++(SendSeq->seq_local); @@ -244,26 +254,19 @@ void CartesianCommunicator::SendRecvPacket(void *xmit, shmem_putmem(recv,xmit,bytes,receiver); shmem_fence(); - printf("Sender sent payload %d\n",seq); //Notify him we're done shmem_putmem((void *)&(RecvSeq->seq_remote),&seq,sizeof(seq),receiver); shmem_fence(); - printf("Sender ringing door bell %d\n",seq); } if ( _processor == receiver ) { - printf("Receiver SHMEM pt2pt %d->%d\n",sender,receiver); // Post a receive seq = ++(RecvSeq->seq_local); shmem_putmem((void *)&(SendSeq->seq_remote),&seq,sizeof(seq),sender); - printf("Receiver Opening letter box %d\n",seq); - - // Now wait until he has advanced our reception counter while(RecvSeq->seq_remote != RecvSeq->seq_local); - printf("Receiver Got the mail %d\n",seq); } } @@ -291,7 +294,7 @@ void CartesianCommunicator::Barrier(void) } void CartesianCommunicator::Broadcast(int root,void* data, int bytes) { - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; static uint32_t word; uint32_t *array = (uint32_t *) data; assert( (bytes % 4)==0); @@ -314,7 +317,7 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes) } void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { - static long psync[_SHMEM_REDUCE_SYNC_SIZE]; + static std::array psync = psync_init; static uint32_t word; uint32_t *array = (uint32_t *) data; assert( (bytes % 4)==0); diff --git a/lib/cshift/Cshift_common.h b/lib/cshift/Cshift_common.h index b8e1284a..2b146daa 100644 --- a/lib/cshift/Cshift_common.h +++ b/lib/cshift/Cshift_common.h @@ -1,3 +1,4 @@ + /************************************************************************************* Grid physics library, www.github.com/paboyle/Grid @@ -44,7 +45,7 @@ public: // Gather for when there is no need to SIMD split with compression /////////////////////////////////////////////////////////////////// template void -Gather_plane_simple (const Lattice &rhs,std::vector > &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0) +Gather_plane_simple (const Lattice &rhs,commVector &buffer,int dimension,int plane,int cbmask,compressor &compress, int off=0) { int rd = rhs._grid->_rdimensions[dimension]; @@ -56,6 +57,7 @@ Gather_plane_simple (const Lattice &rhs,std::vector_slice_nblock[dimension]; int e2=rhs._grid->_slice_block[dimension]; + int stride=rhs._grid->_slice_stride[dimension]; if ( cbmask == 0x3 ) { PARALLEL_NESTED_LOOP2 @@ -68,15 +70,20 @@ PARALLEL_NESTED_LOOP2 } } else { int bo=0; + std::vector > table; for(int n=0;nCheckerBoardFromOindex(o+b);// Could easily be a table lookup + int ocb=1<CheckerBoardFromOindexTable(o+b); if ( ocb &cbmask ) { - buffer[off+bo++]=compress(rhs._odata[so+o+b]); + table.push_back(std::pair (bo++,o+b)); } } } +PARALLEL_FOR_LOOP + for(int i=0;i(temp,pointers,offset); } @@ -114,6 +122,7 @@ PARALLEL_NESTED_LOOP2 } else { assert(0); //Fixme think this is buggy + for(int n=0;n_slice_stride[dimension]; @@ -132,7 +141,7 @@ PARALLEL_NESTED_LOOP2 ////////////////////////////////////////////////////// // Gather for when there is no need to SIMD split ////////////////////////////////////////////////////// -template void Gather_plane_simple (const Lattice &rhs,std::vector > &buffer, int dimension,int plane,int cbmask) +template void Gather_plane_simple (const Lattice &rhs,commVector &buffer, int dimension,int plane,int cbmask) { SimpleCompressor dontcompress; Gather_plane_simple (rhs,buffer,dimension,plane,cbmask,dontcompress); @@ -150,7 +159,7 @@ template void Gather_plane_extract(const Lattice &rhs,std::vec ////////////////////////////////////////////////////// // Scatter for when there is no need to SIMD split ////////////////////////////////////////////////////// -template void Scatter_plane_simple (Lattice &rhs,std::vector > &buffer, int dimension,int plane,int cbmask) +template void Scatter_plane_simple (Lattice &rhs,commVector &buffer, int dimension,int plane,int cbmask) { int rd = rhs._grid->_rdimensions[dimension]; diff --git a/lib/cshift/Cshift_mpi.h b/lib/cshift/Cshift_mpi.h index 704fda34..b3c07cd6 100644 --- a/lib/cshift/Cshift_mpi.h +++ b/lib/cshift/Cshift_mpi.h @@ -119,8 +119,8 @@ template void Cshift_comms(Lattice &ret,const Lattice &r assert(shift_slice_nblock[dimension]*rhs._grid->_slice_block[dimension]; - std::vector > send_buf(buffer_size); - std::vector > recv_buf(buffer_size); + commVector send_buf(buffer_size); + commVector recv_buf(buffer_size); int cb= (cbmask==0x2)? Odd : Even; int sshift= rhs._grid->CheckerBoardShiftForCB(rhs.checkerboard,dimension,shift,cb); @@ -191,8 +191,8 @@ template void Cshift_comms_simd(Lattice &ret,const Lattice_slice_nblock[dimension]*grid->_slice_block[dimension]; int words = sizeof(vobj)/sizeof(vector_type); - std::vector > send_buf_extract(Nsimd,Vector(buffer_size) ); - std::vector > recv_buf_extract(Nsimd,Vector(buffer_size) ); + std::vector > send_buf_extract(Nsimd,commVector(buffer_size) ); + std::vector > recv_buf_extract(Nsimd,commVector(buffer_size) ); int bytes = buffer_size*sizeof(scalar_object); diff --git a/lib/fftw/fftw3.h b/lib/fftw/fftw3.h deleted file mode 100644 index 1bf34396..00000000 --- a/lib/fftw/fftw3.h +++ /dev/null @@ -1,412 +0,0 @@ -/* - * Copyright (c) 2003, 2007-14 Matteo Frigo - * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology - * - * The following statement of license applies *only* to this header file, - * and *not* to the other files distributed with FFTW or derived therefrom: - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS - * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE - * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/***************************** NOTE TO USERS ********************************* - * - * THIS IS A HEADER FILE, NOT A MANUAL - * - * If you want to know how to use FFTW, please read the manual, - * online at http://www.fftw.org/doc/ and also included with FFTW. - * For a quick start, see the manual's tutorial section. - * - * (Reading header files to learn how to use a library is a habit - * stemming from code lacking a proper manual. Arguably, it's a - * *bad* habit in most cases, because header files can contain - * interfaces that are not part of the public, stable API.) - * - ****************************************************************************/ - -#ifndef FFTW3_H -#define FFTW3_H - -#include - -#ifdef __cplusplus -extern "C" -{ -#endif /* __cplusplus */ - -/* If is included, use the C99 complex type. Otherwise - define a type bit-compatible with C99 complex */ -#if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I) -# define FFTW_DEFINE_COMPLEX(R, C) typedef R _Complex C -#else -# define FFTW_DEFINE_COMPLEX(R, C) typedef R C[2] -#endif - -#define FFTW_CONCAT(prefix, name) prefix ## name -#define FFTW_MANGLE_DOUBLE(name) FFTW_CONCAT(fftw_, name) -#define FFTW_MANGLE_FLOAT(name) FFTW_CONCAT(fftwf_, name) -#define FFTW_MANGLE_LONG_DOUBLE(name) FFTW_CONCAT(fftwl_, name) -#define FFTW_MANGLE_QUAD(name) FFTW_CONCAT(fftwq_, name) - -/* IMPORTANT: for Windows compilers, you should add a line - #define FFTW_DLL - here and in kernel/ifftw.h if you are compiling/using FFTW as a - DLL, in order to do the proper importing/exporting, or - alternatively compile with -DFFTW_DLL or the equivalent - command-line flag. This is not necessary under MinGW/Cygwin, where - libtool does the imports/exports automatically. */ -#if defined(FFTW_DLL) && (defined(_WIN32) || defined(__WIN32__)) - /* annoying Windows syntax for shared-library declarations */ -# if defined(COMPILING_FFTW) /* defined in api.h when compiling FFTW */ -# define FFTW_EXTERN extern __declspec(dllexport) -# else /* user is calling FFTW; import symbol */ -# define FFTW_EXTERN extern __declspec(dllimport) -# endif -#else -# define FFTW_EXTERN extern -#endif - -enum fftw_r2r_kind_do_not_use_me { - FFTW_R2HC=0, FFTW_HC2R=1, FFTW_DHT=2, - FFTW_REDFT00=3, FFTW_REDFT01=4, FFTW_REDFT10=5, FFTW_REDFT11=6, - FFTW_RODFT00=7, FFTW_RODFT01=8, FFTW_RODFT10=9, FFTW_RODFT11=10 -}; - -struct fftw_iodim_do_not_use_me { - int n; /* dimension size */ - int is; /* input stride */ - int os; /* output stride */ -}; - -#include /* for ptrdiff_t */ -struct fftw_iodim64_do_not_use_me { - ptrdiff_t n; /* dimension size */ - ptrdiff_t is; /* input stride */ - ptrdiff_t os; /* output stride */ -}; - -typedef void (*fftw_write_char_func_do_not_use_me)(char c, void *); -typedef int (*fftw_read_char_func_do_not_use_me)(void *); - -/* - huge second-order macro that defines prototypes for all API - functions. We expand this macro for each supported precision - - X: name-mangling macro - R: real data type - C: complex data type -*/ - -#define FFTW_DEFINE_API(X, R, C) \ - \ -FFTW_DEFINE_COMPLEX(R, C); \ - \ -typedef struct X(plan_s) *X(plan); \ - \ -typedef struct fftw_iodim_do_not_use_me X(iodim); \ -typedef struct fftw_iodim64_do_not_use_me X(iodim64); \ - \ -typedef enum fftw_r2r_kind_do_not_use_me X(r2r_kind); \ - \ -typedef fftw_write_char_func_do_not_use_me X(write_char_func); \ -typedef fftw_read_char_func_do_not_use_me X(read_char_func); \ - \ -FFTW_EXTERN void X(execute)(const X(plan) p); \ - \ -FFTW_EXTERN X(plan) X(plan_dft)(int rank, const int *n, \ - C *in, C *out, int sign, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_dft_1d)(int n, C *in, C *out, int sign, \ - unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_dft_2d)(int n0, int n1, \ - C *in, C *out, int sign, unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_dft_3d)(int n0, int n1, int n2, \ - C *in, C *out, int sign, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_many_dft)(int rank, const int *n, \ - int howmany, \ - C *in, const int *inembed, \ - int istride, int idist, \ - C *out, const int *onembed, \ - int ostride, int odist, \ - int sign, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru_dft)(int rank, const X(iodim) *dims, \ - int howmany_rank, \ - const X(iodim) *howmany_dims, \ - C *in, C *out, \ - int sign, unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_guru_split_dft)(int rank, const X(iodim) *dims, \ - int howmany_rank, \ - const X(iodim) *howmany_dims, \ - R *ri, R *ii, R *ro, R *io, \ - unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru64_dft)(int rank, \ - const X(iodim64) *dims, \ - int howmany_rank, \ - const X(iodim64) *howmany_dims, \ - C *in, C *out, \ - int sign, unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_guru64_split_dft)(int rank, \ - const X(iodim64) *dims, \ - int howmany_rank, \ - const X(iodim64) *howmany_dims, \ - R *ri, R *ii, R *ro, R *io, \ - unsigned flags); \ - \ -FFTW_EXTERN void X(execute_dft)(const X(plan) p, C *in, C *out); \ -FFTW_EXTERN void X(execute_split_dft)(const X(plan) p, R *ri, R *ii, \ - R *ro, R *io); \ - \ -FFTW_EXTERN X(plan) X(plan_many_dft_r2c)(int rank, const int *n, \ - int howmany, \ - R *in, const int *inembed, \ - int istride, int idist, \ - C *out, const int *onembed, \ - int ostride, int odist, \ - unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_dft_r2c)(int rank, const int *n, \ - R *in, C *out, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_dft_r2c_1d)(int n,R *in,C *out,unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_dft_r2c_2d)(int n0, int n1, \ - R *in, C *out, unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_dft_r2c_3d)(int n0, int n1, \ - int n2, \ - R *in, C *out, unsigned flags); \ - \ - \ -FFTW_EXTERN X(plan) X(plan_many_dft_c2r)(int rank, const int *n, \ - int howmany, \ - C *in, const int *inembed, \ - int istride, int idist, \ - R *out, const int *onembed, \ - int ostride, int odist, \ - unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_dft_c2r)(int rank, const int *n, \ - C *in, R *out, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_dft_c2r_1d)(int n,C *in,R *out,unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_dft_c2r_2d)(int n0, int n1, \ - C *in, R *out, unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_dft_c2r_3d)(int n0, int n1, \ - int n2, \ - C *in, R *out, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru_dft_r2c)(int rank, const X(iodim) *dims, \ - int howmany_rank, \ - const X(iodim) *howmany_dims, \ - R *in, C *out, \ - unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_guru_dft_c2r)(int rank, const X(iodim) *dims, \ - int howmany_rank, \ - const X(iodim) *howmany_dims, \ - C *in, R *out, \ - unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru_split_dft_r2c)( \ - int rank, const X(iodim) *dims, \ - int howmany_rank, \ - const X(iodim) *howmany_dims, \ - R *in, R *ro, R *io, \ - unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_guru_split_dft_c2r)( \ - int rank, const X(iodim) *dims, \ - int howmany_rank, \ - const X(iodim) *howmany_dims, \ - R *ri, R *ii, R *out, \ - unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru64_dft_r2c)(int rank, \ - const X(iodim64) *dims, \ - int howmany_rank, \ - const X(iodim64) *howmany_dims, \ - R *in, C *out, \ - unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_guru64_dft_c2r)(int rank, \ - const X(iodim64) *dims, \ - int howmany_rank, \ - const X(iodim64) *howmany_dims, \ - C *in, R *out, \ - unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru64_split_dft_r2c)( \ - int rank, const X(iodim64) *dims, \ - int howmany_rank, \ - const X(iodim64) *howmany_dims, \ - R *in, R *ro, R *io, \ - unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_guru64_split_dft_c2r)( \ - int rank, const X(iodim64) *dims, \ - int howmany_rank, \ - const X(iodim64) *howmany_dims, \ - R *ri, R *ii, R *out, \ - unsigned flags); \ - \ -FFTW_EXTERN void X(execute_dft_r2c)(const X(plan) p, R *in, C *out); \ -FFTW_EXTERN void X(execute_dft_c2r)(const X(plan) p, C *in, R *out); \ - \ -FFTW_EXTERN void X(execute_split_dft_r2c)(const X(plan) p, \ - R *in, R *ro, R *io); \ -FFTW_EXTERN void X(execute_split_dft_c2r)(const X(plan) p, \ - R *ri, R *ii, R *out); \ - \ -FFTW_EXTERN X(plan) X(plan_many_r2r)(int rank, const int *n, \ - int howmany, \ - R *in, const int *inembed, \ - int istride, int idist, \ - R *out, const int *onembed, \ - int ostride, int odist, \ - const X(r2r_kind) *kind, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_r2r)(int rank, const int *n, R *in, R *out, \ - const X(r2r_kind) *kind, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_r2r_1d)(int n, R *in, R *out, \ - X(r2r_kind) kind, unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_r2r_2d)(int n0, int n1, R *in, R *out, \ - X(r2r_kind) kind0, X(r2r_kind) kind1, \ - unsigned flags); \ -FFTW_EXTERN X(plan) X(plan_r2r_3d)(int n0, int n1, int n2, \ - R *in, R *out, X(r2r_kind) kind0, \ - X(r2r_kind) kind1, X(r2r_kind) kind2, \ - unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru_r2r)(int rank, const X(iodim) *dims, \ - int howmany_rank, \ - const X(iodim) *howmany_dims, \ - R *in, R *out, \ - const X(r2r_kind) *kind, unsigned flags); \ - \ -FFTW_EXTERN X(plan) X(plan_guru64_r2r)(int rank, const X(iodim64) *dims, \ - int howmany_rank, \ - const X(iodim64) *howmany_dims, \ - R *in, R *out, \ - const X(r2r_kind) *kind, unsigned flags); \ - \ -FFTW_EXTERN void X(execute_r2r)(const X(plan) p, R *in, R *out); \ - \ -FFTW_EXTERN void X(destroy_plan)(X(plan) p); \ -FFTW_EXTERN void X(forget_wisdom)(void); \ -FFTW_EXTERN void X(cleanup)(void); \ - \ -FFTW_EXTERN void X(set_timelimit)(double t); \ - \ -FFTW_EXTERN void X(plan_with_nthreads)(int nthreads); \ -FFTW_EXTERN int X(init_threads)(void); \ -FFTW_EXTERN void X(cleanup_threads)(void); \ - \ -FFTW_EXTERN int X(export_wisdom_to_filename)(const char *filename); \ -FFTW_EXTERN void X(export_wisdom_to_file)(FILE *output_file); \ -FFTW_EXTERN char *X(export_wisdom_to_string)(void); \ -FFTW_EXTERN void X(export_wisdom)(X(write_char_func) write_char, \ - void *data); \ -FFTW_EXTERN int X(import_system_wisdom)(void); \ -FFTW_EXTERN int X(import_wisdom_from_filename)(const char *filename); \ -FFTW_EXTERN int X(import_wisdom_from_file)(FILE *input_file); \ -FFTW_EXTERN int X(import_wisdom_from_string)(const char *input_string); \ -FFTW_EXTERN int X(import_wisdom)(X(read_char_func) read_char, void *data); \ - \ -FFTW_EXTERN void X(fprint_plan)(const X(plan) p, FILE *output_file); \ -FFTW_EXTERN void X(print_plan)(const X(plan) p); \ -FFTW_EXTERN char *X(sprint_plan)(const X(plan) p); \ - \ -FFTW_EXTERN void *X(malloc)(size_t n); \ -FFTW_EXTERN R *X(alloc_real)(size_t n); \ -FFTW_EXTERN C *X(alloc_complex)(size_t n); \ -FFTW_EXTERN void X(free)(void *p); \ - \ -FFTW_EXTERN void X(flops)(const X(plan) p, \ - double *add, double *mul, double *fmas); \ -FFTW_EXTERN double X(estimate_cost)(const X(plan) p); \ -FFTW_EXTERN double X(cost)(const X(plan) p); \ - \ -FFTW_EXTERN int X(alignment_of)(R *p); \ -FFTW_EXTERN const char X(version)[]; \ -FFTW_EXTERN const char X(cc)[]; \ -FFTW_EXTERN const char X(codelet_optim)[]; - - -/* end of FFTW_DEFINE_API macro */ - -FFTW_DEFINE_API(FFTW_MANGLE_DOUBLE, double, fftw_complex) -FFTW_DEFINE_API(FFTW_MANGLE_FLOAT, float, fftwf_complex) -FFTW_DEFINE_API(FFTW_MANGLE_LONG_DOUBLE, long double, fftwl_complex) - -/* __float128 (quad precision) is a gcc extension on i386, x86_64, and ia64 - for gcc >= 4.6 (compiled in FFTW with --enable-quad-precision) */ -#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) \ - && !(defined(__ICC) || defined(__INTEL_COMPILER)) \ - && (defined(__i386__) || defined(__x86_64__) || defined(__ia64__)) -# if !defined(FFTW_NO_Complex) && defined(_Complex_I) && defined(complex) && defined(I) -/* note: __float128 is a typedef, which is not supported with the _Complex - keyword in gcc, so instead we use this ugly __attribute__ version. - However, we can't simply pass the __attribute__ version to - FFTW_DEFINE_API because the __attribute__ confuses gcc in pointer - types. Hence redefining FFTW_DEFINE_COMPLEX. Ugh. */ -# undef FFTW_DEFINE_COMPLEX -# define FFTW_DEFINE_COMPLEX(R, C) typedef _Complex float __attribute__((mode(TC))) C -# endif -FFTW_DEFINE_API(FFTW_MANGLE_QUAD, __float128, fftwq_complex) -#endif - -#define FFTW_FORWARD (-1) -#define FFTW_BACKWARD (+1) - -#define FFTW_NO_TIMELIMIT (-1.0) - -/* documented flags */ -#define FFTW_MEASURE (0U) -#define FFTW_DESTROY_INPUT (1U << 0) -#define FFTW_UNALIGNED (1U << 1) -#define FFTW_CONSERVE_MEMORY (1U << 2) -#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */ -#define FFTW_PRESERVE_INPUT (1U << 4) /* cancels FFTW_DESTROY_INPUT */ -#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */ -#define FFTW_ESTIMATE (1U << 6) -#define FFTW_WISDOM_ONLY (1U << 21) - -/* undocumented beyond-guru flags */ -#define FFTW_ESTIMATE_PATIENT (1U << 7) -#define FFTW_BELIEVE_PCOST (1U << 8) -#define FFTW_NO_DFT_R2HC (1U << 9) -#define FFTW_NO_NONTHREADED (1U << 10) -#define FFTW_NO_BUFFERING (1U << 11) -#define FFTW_NO_INDIRECT_OP (1U << 12) -#define FFTW_ALLOW_LARGE_GENERIC (1U << 13) /* NO_LARGE_GENERIC is default */ -#define FFTW_NO_RANK_SPLITS (1U << 14) -#define FFTW_NO_VRANK_SPLITS (1U << 15) -#define FFTW_NO_VRECURSE (1U << 16) -#define FFTW_NO_SIMD (1U << 17) -#define FFTW_NO_SLOW (1U << 18) -#define FFTW_NO_FIXED_RADIX_LARGE_N (1U << 19) -#define FFTW_ALLOW_PRUNING (1U << 20) - -#ifdef __cplusplus -} /* extern "C" */ -#endif /* __cplusplus */ - -#endif /* FFTW3_H */ diff --git a/lib/lattice/Lattice_ET.h b/lib/lattice/Lattice_ET.h index 7ebac99d..1bb83901 100644 --- a/lib/lattice/Lattice_ET.h +++ b/lib/lattice/Lattice_ET.h @@ -261,6 +261,7 @@ GridUnopClass(UnaryExp, exp(a)); GridBinOpClass(BinaryAdd, lhs + rhs); GridBinOpClass(BinarySub, lhs - rhs); GridBinOpClass(BinaryMul, lhs *rhs); +GridBinOpClass(BinaryDiv, lhs /rhs); GridBinOpClass(BinaryAnd, lhs &rhs); GridBinOpClass(BinaryOr, lhs | rhs); @@ -385,6 +386,7 @@ GRID_DEF_UNOP(exp, UnaryExp); GRID_DEF_BINOP(operator+, BinaryAdd); GRID_DEF_BINOP(operator-, BinarySub); GRID_DEF_BINOP(operator*, BinaryMul); +GRID_DEF_BINOP(operator/, BinaryDiv); GRID_DEF_BINOP(operator&, BinaryAnd); GRID_DEF_BINOP(operator|, BinaryOr); diff --git a/lib/lattice/Lattice_base.h b/lib/lattice/Lattice_base.h index 3bfa5613..e4dc1ca8 100644 --- a/lib/lattice/Lattice_base.h +++ b/lib/lattice/Lattice_base.h @@ -65,9 +65,6 @@ public: class LatticeExpressionBase {}; -template using Vector = std::vector >; // Aligned allocator?? -template using Matrix = std::vector > >; // Aligned allocator?? - template class LatticeUnaryExpression : public std::pair > , public LatticeExpressionBase { public: @@ -303,17 +300,6 @@ PARALLEL_FOR_LOOP *this = (*this)+r; return *this; } - - strong_inline friend Lattice operator / (const Lattice &lhs,const Lattice &rhs){ - conformable(lhs,rhs); - Lattice ret(lhs._grid); -PARALLEL_FOR_LOOP - for(int ss=0;ssoSites();ss++){ - ret._odata[ss] = lhs._odata[ss]*pow(rhs._odata[ss],-1.0); - } - return ret; - }; - }; // class Lattice template std::ostream& operator<< (std::ostream& stream, const Lattice &o){ diff --git a/lib/lattice/Lattice_peekpoke.h b/lib/lattice/Lattice_peekpoke.h index 9bece943..19d349c4 100644 --- a/lib/lattice/Lattice_peekpoke.h +++ b/lib/lattice/Lattice_peekpoke.h @@ -154,7 +154,7 @@ PARALLEL_FOR_LOOP template void peekLocalSite(sobj &s,const Lattice &l,std::vector &site){ - GridBase *grid=l._grid; + GridBase *grid = l._grid; typedef typename vobj::scalar_type scalar_type; typedef typename vobj::vector_type vector_type; @@ -164,16 +164,18 @@ PARALLEL_FOR_LOOP assert( l.checkerboard== l._grid->CheckerBoard(site)); assert( sizeof(sobj)*Nsimd == sizeof(vobj)); + static const int words=sizeof(vobj)/sizeof(vector_type); int odx,idx; idx= grid->iIndex(site); odx= grid->oIndex(site); - std::vector buf(Nsimd); - - extract(l._odata[odx],buf); + scalar_type * vp = (scalar_type *)&l._odata[odx]; + scalar_type * pt = (scalar_type *)&s; + + for(int w=0;wCheckerBoard(site)); assert( sizeof(sobj)*Nsimd == sizeof(vobj)); + static const int words=sizeof(vobj)/sizeof(vector_type); int odx,idx; idx= grid->iIndex(site); odx= grid->oIndex(site); - std::vector buf(Nsimd); - - // extract-modify-merge cycle is easiest way and this is not perf critical - extract(l._odata[odx],buf); + scalar_type * vp = (scalar_type *)&l._odata[odx]; + scalar_type * pt = (scalar_type *)&s; - buf[idx] = s; - - merge(l._odata[odx],buf); + for(int w=0;wGlobalIndexToGlobalCoor(gidx,gcoor); _grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor); - + int l_idx=generator_idx(o_idx,i_idx); - - std::vector site_seeds(4); - for(int i=0;i<4;i++){ + + const int num_rand_seed=16; + std::vector site_seeds(num_rand_seed); + for(int i=0;i +void CayleyFermion5D::Dminus(const FermionField &psi, FermionField &chi) +{ + int Ls=this->Ls; + FermionField tmp(psi._grid); + + this->DW(psi,tmp,DaggerNo); + + for(int s=0;s +void CayleyFermion5D::DminusDag(const FermionField &psi, FermionField &chi) +{ + int Ls=this->Ls; + FermionField tmp(psi._grid); + + this->DW(psi,tmp,DaggerYes); + + for(int s=0;s void CayleyFermion5D::M5D (const FermionField &psi, FermionField &chi) { diff --git a/lib/qcd/action/fermion/CayleyFermion5D.h b/lib/qcd/action/fermion/CayleyFermion5D.h index 53a93a05..1d8c2b95 100644 --- a/lib/qcd/action/fermion/CayleyFermion5D.h +++ b/lib/qcd/action/fermion/CayleyFermion5D.h @@ -56,6 +56,9 @@ namespace Grid { virtual void M5D (const FermionField &psi, FermionField &chi); virtual void M5Ddag(const FermionField &psi, FermionField &chi); + virtual void Dminus(const FermionField &psi, FermionField &chi); + virtual void DminusDag(const FermionField &psi, FermionField &chi); + ///////////////////////////////////////////////////// // Instantiate different versions depending on Impl ///////////////////////////////////////////////////// @@ -117,6 +120,7 @@ namespace Grid { GridRedBlackCartesian &FourDimRedBlackGrid, RealD _mass,RealD _M5,const ImplParams &p= ImplParams()); + protected: void SetCoefficientsZolotarev(RealD zolohi,Approx::zolotarev_data *zdata,RealD b,RealD c); void SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c); diff --git a/lib/qcd/action/fermion/DomainWallFermion.h b/lib/qcd/action/fermion/DomainWallFermion.h index b72ca8ec..c0b6b6aa 100644 --- a/lib/qcd/action/fermion/DomainWallFermion.h +++ b/lib/qcd/action/fermion/DomainWallFermion.h @@ -42,6 +42,10 @@ namespace Grid { INHERIT_IMPL_TYPES(Impl); public: + void MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { + this->MomentumSpacePropagatorHt(out,in,_m); + }; + virtual void Instantiatable(void) {}; // Constructors DomainWallFermion(GaugeField &_Umu, @@ -51,6 +55,7 @@ namespace Grid { GridRedBlackCartesian &FourDimRedBlackGrid, RealD _mass,RealD _M5,const ImplParams &p= ImplParams()) : + CayleyFermion5D(_Umu, FiveDimGrid, FiveDimRedBlackGrid, diff --git a/lib/qcd/action/fermion/FermionOperator.h b/lib/qcd/action/fermion/FermionOperator.h index ea5583eb..742c6e08 100644 --- a/lib/qcd/action/fermion/FermionOperator.h +++ b/lib/qcd/action/fermion/FermionOperator.h @@ -91,6 +91,20 @@ namespace Grid { virtual void Mdiag (const FermionField &in, FermionField &out) { Mooee(in,out);}; // Same as Mooee applied to both CB's virtual void Mdir (const FermionField &in, FermionField &out,int dir,int disp)=0; // case by case Wilson, Clover, Cayley, ContFrac, PartFrac + + virtual void MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { assert(0);}; + + virtual void FreePropagator(const FermionField &in,FermionField &out,RealD mass) { + FFT theFFT((GridCartesian *) in._grid); + + FermionField in_k(in._grid); + FermionField prop_k(in._grid); + + theFFT.FFT_all_dim(in_k,in,FFT::forward); + this->MomentumSpacePropagator(prop_k,in_k,mass); + theFFT.FFT_all_dim(out,prop_k,FFT::backward); + }; + /////////////////////////////////////////////// // Updates gauge field during HMC /////////////////////////////////////////////// diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h index 910bf488..0800dea6 100644 --- a/lib/qcd/action/fermion/FermionOperatorImpl.h +++ b/lib/qcd/action/fermion/FermionOperatorImpl.h @@ -33,511 +33,500 @@ directory #define GRID_QCD_FERMION_OPERATOR_IMPL_H namespace Grid { - - namespace QCD { +namespace QCD { - ////////////////////////////////////////////// - // Template parameter class constructs to package - // externally control Fermion implementations - // in orthogonal directions - // - // Ultimately need Impl to always define types where XXX is opaque - // - // typedef typename XXX Simd; - // typedef typename XXX GaugeLinkField; - // typedef typename XXX GaugeField; - // typedef typename XXX GaugeActField; - // typedef typename XXX FermionField; - // typedef typename XXX DoubledGaugeField; - // typedef typename XXX SiteSpinor; - // typedef typename XXX SiteHalfSpinor; - // typedef typename XXX Compressor; - // - // and Methods: - // void ImportGauge(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu) - // void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu) - // void multLink(SiteHalfSpinor &phi,const SiteDoubledGaugeField &U,const SiteHalfSpinor &chi,int mu,StencilEntry *SE,StencilImpl &St) - // void InsertForce4D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu) - // void InsertForce5D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu) - // - // - // To acquire the typedefs from "Base" (either a base class or template param) use: - // - // INHERIT_GIMPL_TYPES(Base) - // INHERIT_FIMPL_TYPES(Base) - // INHERIT_IMPL_TYPES(Base) - // - // The Fermion operators will do the following: - // - // struct MyOpParams { - // RealD mass; - // }; - // - // - // template - // class MyOp : public { - // public: - // - // INHERIT_ALL_IMPL_TYPES(Impl); - // - // MyOp(MyOpParams Myparm, ImplParams &ImplParam) : Impl(ImplParam) - // { - // - // }; - // - // } - ////////////////////////////////////////////// - - - //////////////////////////////////////////////////////////////////////// - // Implementation dependent fermion types - //////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////// + // Template parameter class constructs to package + // externally control Fermion implementations + // in orthogonal directions + // + // Ultimately need Impl to always define types where XXX is opaque + // + // typedef typename XXX Simd; + // typedef typename XXX GaugeLinkField; + // typedef typename XXX GaugeField; + // typedef typename XXX GaugeActField; + // typedef typename XXX FermionField; + // typedef typename XXX DoubledGaugeField; + // typedef typename XXX SiteSpinor; + // typedef typename XXX SiteHalfSpinor; + // typedef typename XXX Compressor; + // + // and Methods: + // void ImportGauge(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu) + // void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu) + // void multLink(SiteHalfSpinor &phi,const SiteDoubledGaugeField &U,const SiteHalfSpinor &chi,int mu,StencilEntry *SE,StencilImpl &St) + // void InsertForce4D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu) + // void InsertForce5D(GaugeField &mat,const FermionField &Btilde,const FermionField &A,int mu) + // + // + // To acquire the typedefs from "Base" (either a base class or template param) use: + // + // INHERIT_GIMPL_TYPES(Base) + // INHERIT_FIMPL_TYPES(Base) + // INHERIT_IMPL_TYPES(Base) + // + // The Fermion operators will do the following: + // + // struct MyOpParams { + // RealD mass; + // }; + // + // + // template + // class MyOp : public { + // public: + // + // INHERIT_ALL_IMPL_TYPES(Impl); + // + // MyOp(MyOpParams Myparm, ImplParams &ImplParam) : Impl(ImplParam) + // { + // + // }; + // + // } + ////////////////////////////////////////////// + + //////////////////////////////////////////////////////////////////////// + // Implementation dependent fermion types + //////////////////////////////////////////////////////////////////////// + #define INHERIT_FIMPL_TYPES(Impl)\ - typedef typename Impl::FermionField FermionField; \ - typedef typename Impl::DoubledGaugeField DoubledGaugeField; \ - typedef typename Impl::SiteSpinor SiteSpinor; \ - typedef typename Impl::SiteHalfSpinor SiteHalfSpinor; \ - typedef typename Impl::Compressor Compressor; \ - typedef typename Impl::StencilImpl StencilImpl; \ - typedef typename Impl::ImplParams ImplParams; \ - typedef typename Impl::Coeff_t Coeff_t; - + typedef typename Impl::FermionField FermionField; \ + typedef typename Impl::DoubledGaugeField DoubledGaugeField; \ + typedef typename Impl::SiteSpinor SiteSpinor; \ + typedef typename Impl::SiteHalfSpinor SiteHalfSpinor; \ + typedef typename Impl::Compressor Compressor; \ + typedef typename Impl::StencilImpl StencilImpl; \ + typedef typename Impl::ImplParams ImplParams; \ + typedef typename Impl::Coeff_t Coeff_t; + #define INHERIT_IMPL_TYPES(Base) \ - INHERIT_GIMPL_TYPES(Base) \ - INHERIT_FIMPL_TYPES(Base) + INHERIT_GIMPL_TYPES(Base) \ + INHERIT_FIMPL_TYPES(Base) + + ///////////////////////////////////////////////////////////////////////////// + // Single flavour four spinors with colour index + ///////////////////////////////////////////////////////////////////////////// + template + class WilsonImpl : public PeriodicGaugeImpl > { + + public: + + static const int Dimension = Representation::Dimension; + typedef PeriodicGaugeImpl > Gimpl; + + //Necessary? + constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;} - /////// - // Single flavour four spinors with colour index - /////// - template - class WilsonImpl - : public PeriodicGaugeImpl > { - public: - static const int Dimension = Representation::Dimension; - typedef PeriodicGaugeImpl > Gimpl; - - //Necessary? - constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;} + const bool LsVectorised=false; + typedef _Coeff_t Coeff_t; - const bool LsVectorised=false; - typedef _Coeff_t Coeff_t; - - - INHERIT_GIMPL_TYPES(Gimpl); + INHERIT_GIMPL_TYPES(Gimpl); - template using iImplSpinor = iScalar, Ns> >; - template using iImplHalfSpinor = iScalar, Nhs> >; - template using iImplDoubledGaugeField = iVector >, Nds>; + template using iImplSpinor = iScalar, Ns> >; + template using iImplHalfSpinor = iScalar, Nhs> >; + template using iImplDoubledGaugeField = iVector >, Nds>; + + typedef iImplSpinor SiteSpinor; + typedef iImplHalfSpinor SiteHalfSpinor; + typedef iImplDoubledGaugeField SiteDoubledGaugeField; + + typedef Lattice FermionField; + typedef Lattice DoubledGaugeField; + + typedef WilsonCompressor Compressor; + typedef WilsonImplParams ImplParams; + typedef WilsonStencil StencilImpl; + + ImplParams Params; + + WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){}; - typedef iImplSpinor SiteSpinor; - typedef iImplHalfSpinor SiteHalfSpinor; - typedef iImplDoubledGaugeField SiteDoubledGaugeField; + bool overlapCommsCompute(void) { return Params.overlapCommsCompute; }; - typedef Lattice FermionField; - typedef Lattice DoubledGaugeField; + inline void multLink(SiteHalfSpinor &phi, + const SiteDoubledGaugeField &U, + const SiteHalfSpinor &chi, + int mu, + StencilEntry *SE, + StencilImpl &St) { + mult(&phi(), &U(mu), &chi()); + } - typedef WilsonCompressor Compressor; - typedef WilsonImplParams ImplParams; - typedef WilsonStencil StencilImpl; + template + inline void loadLinkElement(Simd ®, ref &memory) { + reg = memory; + } - ImplParams Params; - - WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){}; - - bool overlapCommsCompute(void) { return Params.overlapCommsCompute; }; - - inline void multLink(SiteHalfSpinor &phi, - const SiteDoubledGaugeField &U, - const SiteHalfSpinor &chi, - int mu, - StencilEntry *SE, - StencilImpl &St) { - mult(&phi(), &U(mu), &chi()); + inline void DoubleStore(GridBase *GaugeGrid, + DoubledGaugeField &Uds, + const GaugeField &Umu) { + conformable(Uds._grid, GaugeGrid); + conformable(Umu._grid, GaugeGrid); + GaugeLinkField U(GaugeGrid); + for (int mu = 0; mu < Nd; mu++) { + U = PeekIndex(Umu, mu); + PokeIndex(Uds, U, mu); + U = adj(Cshift(U, mu, -1)); + PokeIndex(Uds, U, mu + 4); } + } + + inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A,int mu){ + GaugeLinkField link(mat._grid); + link = TraceIndex(outerProduct(Btilde,A)); + PokeIndex(mat,link,mu); + } - template - inline void loadLinkElement(Simd ®, - ref &memory) { - reg = memory; - } + inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu){ - inline void DoubleStore(GridBase *GaugeGrid, - DoubledGaugeField &Uds, - const GaugeField &Umu) { - conformable(Uds._grid, GaugeGrid); - conformable(Umu._grid, GaugeGrid); - GaugeLinkField U(GaugeGrid); - for (int mu = 0; mu < Nd; mu++) { - U = PeekIndex(Umu, mu); - PokeIndex(Uds, U, mu); - U = adj(Cshift(U, mu, -1)); - PokeIndex(Uds, U, mu + 4); + int Ls=Btilde._grid->_fdimensions[0]; + GaugeLinkField tmp(mat._grid); + tmp = zero; + + PARALLEL_FOR_LOOP + for(int sss=0;sssoSites();sss++){ + int sU=sss; + for(int s=0;s(outerProduct(Btilde[sF],Atilde[sF])); // ordering here } } + PokeIndex(mat,tmp,mu); + + } + }; - inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A,int mu){ - GaugeLinkField link(mat._grid); - link = TraceIndex(outerProduct(Btilde,A)); - PokeIndex(mat,link,mu); - } - - inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã,int mu){ - - int Ls=Btilde._grid->_fdimensions[0]; - - GaugeLinkField tmp(mat._grid); - tmp = zero; - PARALLEL_FOR_LOOP - for(int sss=0;sssoSites();sss++){ - int sU=sss; - for(int s=0;s(outerProduct(Btilde[sF],Atilde[sF])); // ordering here - } - } - PokeIndex(mat,tmp,mu); - - } - }; + //////////////////////////////////////////////////////////////////////////////////// + // Single flavour four spinors with colour index, 5d redblack + //////////////////////////////////////////////////////////////////////////////////// - /////// - // Single flavour four spinors with colour index, 5d redblack - /////// - template - class DomainWallVec5dImpl : public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { - public: +template +class DomainWallVec5dImpl : public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepresentation> > { + public: - static const int Dimension = Nrepresentation; - const bool LsVectorised=true; - typedef _Coeff_t Coeff_t; - typedef PeriodicGaugeImpl > Gimpl; + static const int Dimension = Nrepresentation; + const bool LsVectorised=true; + typedef _Coeff_t Coeff_t; + typedef PeriodicGaugeImpl > Gimpl; + + INHERIT_GIMPL_TYPES(Gimpl); + + template using iImplSpinor = iScalar, Ns> >; + template using iImplHalfSpinor = iScalar, Nhs> >; + template using iImplDoubledGaugeField = iVector >, Nds>; + template using iImplGaugeField = iVector >, Nd>; + template using iImplGaugeLink = iScalar > >; + + typedef iImplSpinor SiteSpinor; + typedef iImplHalfSpinor SiteHalfSpinor; + typedef Lattice FermionField; + + // Make the doubled gauge field a *scalar* + typedef iImplDoubledGaugeField SiteDoubledGaugeField; // This is a scalar + typedef iImplGaugeField SiteScalarGaugeField; // scalar + typedef iImplGaugeLink SiteScalarGaugeLink; // scalar + + typedef Lattice DoubledGaugeField; + + typedef WilsonCompressor Compressor; + typedef WilsonImplParams ImplParams; + typedef WilsonStencil StencilImpl; + + ImplParams Params; + + DomainWallVec5dImpl(const ImplParams &p = ImplParams()) : Params(p){}; + + bool overlapCommsCompute(void) { return false; }; + + template + inline void loadLinkElement(Simd ®, ref &memory) { + vsplat(reg, memory); + } - INHERIT_GIMPL_TYPES(Gimpl); - - template using iImplSpinor = iScalar, Ns> >; - template using iImplHalfSpinor = iScalar, Nhs> >; - template using iImplDoubledGaugeField = iVector >, Nds>; - template using iImplGaugeField = iVector >, Nd>; - template using iImplGaugeLink = iScalar > >; - - typedef iImplSpinor SiteSpinor; - typedef iImplHalfSpinor SiteHalfSpinor; - typedef Lattice FermionField; - - // Make the doubled gauge field a *scalar* - typedef iImplDoubledGaugeField - SiteDoubledGaugeField; // This is a scalar - typedef iImplGaugeField - SiteScalarGaugeField; // scalar - typedef iImplGaugeLink - SiteScalarGaugeLink; // scalar - - typedef Lattice DoubledGaugeField; - - typedef WilsonCompressor Compressor; - typedef WilsonImplParams ImplParams; - typedef WilsonStencil StencilImpl; - - ImplParams Params; - - DomainWallVec5dImpl(const ImplParams &p = ImplParams()) : Params(p){}; - - bool overlapCommsCompute(void) { return false; }; - - template - inline void loadLinkElement(Simd ®, ref &memory) { - vsplat(reg, memory); - } - inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U, - const SiteHalfSpinor &chi, int mu, StencilEntry *SE, - StencilImpl &St) { - SiteGaugeLink UU; - for (int i = 0; i < Nrepresentation; i++) { - for (int j = 0; j < Nrepresentation; j++) { - vsplat(UU()()(i, j), U(mu)()(i, j)); - } - } - mult(&phi(), &UU(), &chi()); + inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U, + const SiteHalfSpinor &chi, int mu, StencilEntry *SE, + StencilImpl &St) { + SiteGaugeLink UU; + for (int i = 0; i < Nrepresentation; i++) { + for (int j = 0; j < Nrepresentation; j++) { + vsplat(UU()()(i, j), U(mu)()(i, j)); } + } + mult(&phi(), &UU(), &chi()); + } - inline void DoubleStore(GridBase *GaugeGrid, DoubledGaugeField &Uds, - const GaugeField &Umu) { - SiteScalarGaugeField ScalarUmu; - SiteDoubledGaugeField ScalarUds; - - GaugeLinkField U(Umu._grid); - GaugeField Uadj(Umu._grid); - for (int mu = 0; mu < Nd; mu++) { - U = PeekIndex(Umu, mu); - U = adj(Cshift(U, mu, -1)); - PokeIndex(Uadj, U, mu); - } - - for (int lidx = 0; lidx < GaugeGrid->lSites(); lidx++) { - std::vector lcoor; - GaugeGrid->LocalIndexToLocalCoor(lidx, lcoor); - - peekLocalSite(ScalarUmu, Umu, lcoor); - for (int mu = 0; mu < 4; mu++) ScalarUds(mu) = ScalarUmu(mu); - - peekLocalSite(ScalarUmu, Uadj, lcoor); - for (int mu = 0; mu < 4; mu++) ScalarUds(mu + 4) = ScalarUmu(mu); - - pokeLocalSite(ScalarUds, Uds, lcoor); - } - } + inline void DoubleStore(GridBase *GaugeGrid, DoubledGaugeField &Uds,const GaugeField &Umu) + { + SiteScalarGaugeField ScalarUmu; + SiteDoubledGaugeField ScalarUds; + + GaugeLinkField U(Umu._grid); + GaugeField Uadj(Umu._grid); + for (int mu = 0; mu < Nd; mu++) { + U = PeekIndex(Umu, mu); + U = adj(Cshift(U, mu, -1)); + PokeIndex(Uadj, U, mu); + } + + for (int lidx = 0; lidx < GaugeGrid->lSites(); lidx++) { + std::vector lcoor; + GaugeGrid->LocalIndexToLocalCoor(lidx, lcoor); - inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, - FermionField &A, int mu) { + peekLocalSite(ScalarUmu, Umu, lcoor); + for (int mu = 0; mu < 4; mu++) ScalarUds(mu) = ScalarUmu(mu); + + peekLocalSite(ScalarUmu, Uadj, lcoor); + for (int mu = 0; mu < 4; mu++) ScalarUds(mu + 4) = ScalarUmu(mu); + + pokeLocalSite(ScalarUds, Uds, lcoor); + } + } + + inline void InsertForce4D(GaugeField &mat, FermionField &Btilde,FermionField &A, int mu) + { + assert(0); + } + + inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,FermionField Ã, int mu) + { assert(0); - } - - inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, - FermionField Ã, int mu) { - assert(0); - } - }; + } +}; //////////////////////////////////////////////////////////////////////////////////////// // Flavour doubled spinors; is Gparity the only? what about C*? //////////////////////////////////////////////////////////////////////////////////////// - template - class GparityWilsonImpl - : public ConjugateGaugeImpl > { - public: - static const int Dimension = Nrepresentation; +template +class GparityWilsonImpl : public ConjugateGaugeImpl > { + public: - const bool LsVectorised=false; + static const int Dimension = Nrepresentation; - typedef _Coeff_t Coeff_t; - typedef ConjugateGaugeImpl< GaugeImplTypes > Gimpl; + const bool LsVectorised=false; + + typedef _Coeff_t Coeff_t; + typedef ConjugateGaugeImpl< GaugeImplTypes > Gimpl; + + INHERIT_GIMPL_TYPES(Gimpl); - INHERIT_GIMPL_TYPES(Gimpl); + template using iImplSpinor = iVector, Ns>, Ngp>; + template using iImplHalfSpinor = iVector, Nhs>, Ngp>; + template using iImplDoubledGaugeField = iVector >, Nds>, Ngp>; - template - using iImplSpinor = - iVector, Ns>, Ngp>; - template - using iImplHalfSpinor = - iVector, Nhs>, Ngp>; - template - using iImplDoubledGaugeField = - iVector >, Nds>, Ngp>; + typedef iImplSpinor SiteSpinor; + typedef iImplHalfSpinor SiteHalfSpinor; + typedef iImplDoubledGaugeField SiteDoubledGaugeField; + + typedef Lattice FermionField; + typedef Lattice DoubledGaugeField; + + typedef WilsonCompressor Compressor; + typedef WilsonStencil StencilImpl; + + typedef GparityWilsonImplParams ImplParams; - typedef iImplSpinor SiteSpinor; - typedef iImplHalfSpinor SiteHalfSpinor; - typedef iImplDoubledGaugeField SiteDoubledGaugeField; - - typedef Lattice FermionField; - typedef Lattice DoubledGaugeField; - - typedef WilsonCompressor Compressor; - typedef WilsonStencil StencilImpl; + ImplParams Params; - typedef GparityWilsonImplParams ImplParams; - - ImplParams Params; + GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){}; + bool overlapCommsCompute(void) { return Params.overlapCommsCompute; }; - GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){}; + // provide the multiply by link that is differentiated between Gparity (with + // flavour index) and non-Gparity + inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U, + const SiteHalfSpinor &chi, int mu, StencilEntry *SE, + StencilImpl &St) { - bool overlapCommsCompute(void) { return Params.overlapCommsCompute; }; - - // provide the multiply by link that is differentiated between Gparity (with - // flavour index) and non-Gparity - inline void multLink(SiteHalfSpinor &phi, const SiteDoubledGaugeField &U, - const SiteHalfSpinor &chi, int mu, StencilEntry *SE, - StencilImpl &St) { - typedef SiteHalfSpinor vobj; - typedef typename SiteHalfSpinor::scalar_object sobj; + typedef SiteHalfSpinor vobj; + typedef typename SiteHalfSpinor::scalar_object sobj; - vobj vtmp; - sobj stmp; + vobj vtmp; + sobj stmp; - GridBase *grid = St._grid; + GridBase *grid = St._grid; - const int Nsimd = grid->Nsimd(); + const int Nsimd = grid->Nsimd(); - int direction = St._directions[mu]; - int distance = St._distances[mu]; - int ptype = St._permute_type[mu]; - int sl = St._grid->_simd_layout[direction]; + int direction = St._directions[mu]; + int distance = St._distances[mu]; + int ptype = St._permute_type[mu]; + int sl = St._grid->_simd_layout[direction]; + + // Fixme X.Y.Z.T hardcode in stencil + int mmu = mu % Nd; - // Fixme X.Y.Z.T hardcode in stencil - int mmu = mu % Nd; + // assert our assumptions + assert((distance == 1) || (distance == -1)); // nearest neighbour stencil hard code + assert((sl == 1) || (sl == 2)); + + std::vector icoor; - // assert our assumptions - assert((distance == 1) || (distance == -1)); // nearest neighbour stencil hard code - assert((sl == 1) || (sl == 2)); - - std::vector icoor; - - if ( SE->_around_the_world && Params.twists[mmu] ) { + if ( SE->_around_the_world && Params.twists[mmu] ) { - if ( sl == 2 ) { + if ( sl == 2 ) { + + std::vector vals(Nsimd); - std::vector vals(Nsimd); + extract(chi,vals); + for(int s=0;siCoorFromIindex(icoor,s); + grid->iCoorFromIindex(icoor,s); - assert((icoor[direction]==0)||(icoor[direction]==1)); + assert((icoor[direction]==0)||(icoor[direction]==1)); - int permute_lane; - if ( distance == 1) { - permute_lane = icoor[direction]?1:0; - } else { - permute_lane = icoor[direction]?0:1; + int permute_lane; + if ( distance == 1) { + permute_lane = icoor[direction]?1:0; + } else { + permute_lane = icoor[direction]?0:1; + } + + if ( permute_lane ) { + stmp(0) = vals[s](1); + stmp(1) = vals[s](0); + vals[s] = stmp; } - - if ( permute_lane ) { - stmp(0) = vals[s](1); - stmp(1) = vals[s](0); - vals[s] = stmp; - } - } - merge(vtmp,vals); + } + merge(vtmp,vals); + + } else { + vtmp(0) = chi(1); + vtmp(1) = chi(0); + } + mult(&phi(0),&U(0)(mu),&vtmp(0)); + mult(&phi(1),&U(1)(mu),&vtmp(1)); + + } else { + mult(&phi(0),&U(0)(mu),&chi(0)); + mult(&phi(1),&U(1)(mu),&chi(1)); + } + + } - } else { - vtmp(0) = chi(1); - vtmp(1) = chi(0); - } - mult(&phi(0),&U(0)(mu),&vtmp(0)); - mult(&phi(1),&U(1)(mu),&vtmp(1)); - - } else { - mult(&phi(0),&U(0)(mu),&chi(0)); - mult(&phi(1),&U(1)(mu),&chi(1)); - } + inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu) + { + conformable(Uds._grid,GaugeGrid); + conformable(Umu._grid,GaugeGrid); + + GaugeLinkField Utmp (GaugeGrid); + GaugeLinkField U (GaugeGrid); + GaugeLinkField Uconj(GaugeGrid); + + Lattice > coor(GaugeGrid); - } - - inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu) - { - - conformable(Uds._grid,GaugeGrid); - conformable(Umu._grid,GaugeGrid); - - GaugeLinkField Utmp (GaugeGrid); - GaugeLinkField U (GaugeGrid); - GaugeLinkField Uconj(GaugeGrid); - - Lattice > coor(GaugeGrid); - - - for(int mu=0;mu(Umu,mu); - Uconj = conjugate(U); + U = PeekIndex(Umu,mu); + Uconj = conjugate(U); + + // This phase could come from a simple bc 1,1,-1,1 .. + int neglink = GaugeGrid->GlobalDimensions()[mu]-1; + if ( Params.twists[mu] ) { + Uconj = where(coor==neglink,-Uconj,Uconj); + } - // This phase could come from a simple bc 1,1,-1,1 .. - int neglink = GaugeGrid->GlobalDimensions()[mu]-1; - if ( Params.twists[mu] ) { - Uconj = where(coor==neglink,-Uconj,Uconj); - } +PARALLEL_FOR_LOOP + for(auto ss=U.begin();ss(outerProduct(Btilde, A)); - PARALLEL_FOR_LOOP - for (auto ss = tmp.begin(); ss < tmp.end(); ss++) { - link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1)); - } - PokeIndex(mat, link, mu); - return; - } + inline void InsertForce4D(GaugeField &mat, FermionField &Btilde, FermionField &A, int mu) { + + // DhopDir provides U or Uconj depending on coor/flavour. + GaugeLinkField link(mat._grid); + // use lorentz for flavour as hack. + auto tmp = TraceIndex(outerProduct(Btilde, A)); +PARALLEL_FOR_LOOP + for (auto ss = tmp.begin(); ss < tmp.end(); ss++) { + link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1)); + } + PokeIndex(mat, link, mu); + return; + } - inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, - FermionField Ã, int mu) { - int Ls = Btilde._grid->_fdimensions[0]; + inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField Ã, int mu) { + + int Ls = Btilde._grid->_fdimensions[0]; - GaugeLinkField tmp(mat._grid); - tmp = zero; - PARALLEL_FOR_LOOP - for (int ss = 0; ss < tmp._grid->oSites(); ss++) { - for (int s = 0; s < Ls; s++) { - int sF = s + Ls * ss; - auto ttmp = traceIndex(outerProduct(Btilde[sF], Atilde[sF])); - tmp[ss]() = tmp[ss]() + ttmp(0, 0) + conjugate(ttmp(1, 1)); - } - } - PokeIndex(mat, tmp, mu); - return; - } - }; + GaugeLinkField tmp(mat._grid); + tmp = zero; +PARALLEL_FOR_LOOP + for (int ss = 0; ss < tmp._grid->oSites(); ss++) { + for (int s = 0; s < Ls; s++) { + int sF = s + Ls * ss; + auto ttmp = traceIndex(outerProduct(Btilde[sF], Atilde[sF])); + tmp[ss]() = tmp[ss]() + ttmp(0, 0) + conjugate(ttmp(1, 1)); + } + } + PokeIndex(mat, tmp, mu); + return; + } - typedef WilsonImpl WilsonImplR; // Real.. whichever prec - typedef WilsonImpl WilsonImplF; // Float - typedef WilsonImpl WilsonImplD; // Double +}; + typedef WilsonImpl WilsonImplR; // Real.. whichever prec + typedef WilsonImpl WilsonImplF; // Float + typedef WilsonImpl WilsonImplD; // Double - typedef WilsonImpl ZWilsonImplR; // Real.. whichever prec - typedef WilsonImpl ZWilsonImplF; // Float - typedef WilsonImpl ZWilsonImplD; // Double + typedef WilsonImpl ZWilsonImplR; // Real.. whichever prec + typedef WilsonImpl ZWilsonImplF; // Float + typedef WilsonImpl ZWilsonImplD; // Double + + typedef WilsonImpl WilsonAdjImplR; // Real.. whichever prec + typedef WilsonImpl WilsonAdjImplF; // Float + typedef WilsonImpl WilsonAdjImplD; // Double + + typedef WilsonImpl WilsonTwoIndexSymmetricImplR; // Real.. whichever prec + typedef WilsonImpl WilsonTwoIndexSymmetricImplF; // Float + typedef WilsonImpl WilsonTwoIndexSymmetricImplD; // Double + + typedef DomainWallVec5dImpl DomainWallVec5dImplR; // Real.. whichever prec + typedef DomainWallVec5dImpl DomainWallVec5dImplF; // Float + typedef DomainWallVec5dImpl DomainWallVec5dImplD; // Double + + typedef DomainWallVec5dImpl ZDomainWallVec5dImplR; // Real.. whichever prec + typedef DomainWallVec5dImpl ZDomainWallVec5dImplF; // Float + typedef DomainWallVec5dImpl ZDomainWallVec5dImplD; // Double + + typedef GparityWilsonImpl GparityWilsonImplR; // Real.. whichever prec + typedef GparityWilsonImpl GparityWilsonImplF; // Float + typedef GparityWilsonImpl GparityWilsonImplD; // Double - typedef WilsonImpl WilsonAdjImplR; // Real.. whichever prec - typedef WilsonImpl WilsonAdjImplF; // Float - typedef WilsonImpl WilsonAdjImplD; // Double +}} - typedef WilsonImpl WilsonTwoIndexSymmetricImplR; // Real.. whichever prec - typedef WilsonImpl WilsonTwoIndexSymmetricImplF; // Float - typedef WilsonImpl WilsonTwoIndexSymmetricImplD; // Double - - typedef DomainWallVec5dImpl DomainWallVec5dImplR; // Real.. whichever prec - typedef DomainWallVec5dImpl DomainWallVec5dImplF; // Float - typedef DomainWallVec5dImpl DomainWallVec5dImplD; // Double - - typedef DomainWallVec5dImpl ZDomainWallVec5dImplR; // Real.. whichever prec - typedef DomainWallVec5dImpl ZDomainWallVec5dImplF; // Float - typedef DomainWallVec5dImpl ZDomainWallVec5dImplD; // Double - - typedef GparityWilsonImpl GparityWilsonImplR; // Real.. whichever prec - typedef GparityWilsonImpl GparityWilsonImplF; // Float - typedef GparityWilsonImpl GparityWilsonImplD; // Double -} -} #endif diff --git a/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h b/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h index cd23ddd5..9cab0e22 100644 --- a/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h +++ b/lib/qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h @@ -42,7 +42,11 @@ namespace Grid { INHERIT_IMPL_TYPES(Impl); public: - // Constructors + void MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m) { + this->MomentumSpacePropagatorHw(out,in,_m); + }; + + // Constructors OverlapWilsonCayleyTanhFermion(GaugeField &_Umu, GridCartesian &FiveDimGrid, GridRedBlackCartesian &FiveDimRedBlackGrid, diff --git a/lib/qcd/action/fermion/WilsonFermion.cc b/lib/qcd/action/fermion/WilsonFermion.cc index d887e05b..99baa8a0 100644 --- a/lib/qcd/action/fermion/WilsonFermion.cc +++ b/lib/qcd/action/fermion/WilsonFermion.cc @@ -101,6 +101,7 @@ void WilsonFermion::Meooe(const FermionField &in, FermionField &out) { DhopOE(in, out, DaggerNo); } } + template void WilsonFermion::MeooeDag(const FermionField &in, FermionField &out) { if (in.checkerboard == Odd) { @@ -109,32 +110,87 @@ void WilsonFermion::MeooeDag(const FermionField &in, FermionField &out) { DhopOE(in, out, DaggerYes); } } + + template + void WilsonFermion::Mooee(const FermionField &in, FermionField &out) { + out.checkerboard = in.checkerboard; + typename FermionField::scalar_type scal(4.0 + mass); + out = scal * in; + } -template -void WilsonFermion::Mooee(const FermionField &in, FermionField &out) { - out.checkerboard = in.checkerboard; - typename FermionField::scalar_type scal(4.0 + mass); - out = scal * in; -} + template + void WilsonFermion::MooeeDag(const FermionField &in, FermionField &out) { + out.checkerboard = in.checkerboard; + Mooee(in, out); + } -template -void WilsonFermion::MooeeDag(const FermionField &in, FermionField &out) { - out.checkerboard = in.checkerboard; - Mooee(in, out); -} + template + void WilsonFermion::MooeeInv(const FermionField &in, FermionField &out) { + out.checkerboard = in.checkerboard; + out = (1.0/(4.0+mass))*in; + } + + template + void WilsonFermion::MooeeInvDag(const FermionField &in, FermionField &out) { + out.checkerboard = in.checkerboard; + MooeeInv(in,out); + } -template -void WilsonFermion::MooeeInv(const FermionField &in, FermionField &out) { - out.checkerboard = in.checkerboard; - out = (1.0 / (4.0 + mass)) * in; -} + template + void WilsonFermion::MomentumSpacePropagator(FermionField &out, const FermionField &in,RealD _m) { -template -void WilsonFermion::MooeeInvDag(const FermionField &in, - FermionField &out) { - out.checkerboard = in.checkerboard; - MooeeInv(in, out); -} + // what type LatticeComplex + conformable(_grid,out._grid); + + typedef typename FermionField::vector_type vector_type; + typedef typename FermionField::scalar_type ScalComplex; + + typedef Lattice > LatComplex; + + Gamma::GammaMatrix Gmu [] = { + Gamma::GammaX, + Gamma::GammaY, + Gamma::GammaZ, + Gamma::GammaT + }; + + std::vector latt_size = _grid->_fdimensions; + + FermionField num (_grid); num = zero; + LatComplex wilson(_grid); wilson= zero; + LatComplex one (_grid); one = ScalComplex(1.0,0.0); + + LatComplex denom(_grid); denom= zero; + LatComplex kmu(_grid); + ScalComplex ci(0.0,1.0); + // momphase = n * 2pi / L + for(int mu=0;mu::DerivInternal(StencilImpl &st, DoubledGaugeField &U, //////////////////////// PARALLEL_FOR_LOOP for (int sss = 0; sss < B._grid->oSites(); sss++) { - Kernels::DiracOptDhopDir(st, U, st.comm_buf, sss, sss, B, Btilde, mu, + Kernels::DiracOptDhopDir(st, U, st.CommBuf(), sss, sss, B, Btilde, mu, gamma); } @@ -277,7 +333,7 @@ void WilsonFermion::DhopDirDisp(const FermionField &in, FermionField &out, PARALLEL_FOR_LOOP for (int sss = 0; sss < in._grid->oSites(); sss++) { - Kernels::DiracOptDhopDir(Stencil, Umu, Stencil.comm_buf, sss, sss, in, out, + Kernels::DiracOptDhopDir(Stencil, Umu, Stencil.CommBuf(), sss, sss, in, out, dirdisp, gamma); } }; @@ -295,13 +351,13 @@ void WilsonFermion::DhopInternal(StencilImpl &st, LebesgueOrder &lo, if (dag == DaggerYes) { PARALLEL_FOR_LOOP for (int sss = 0; sss < in._grid->oSites(); sss++) { - Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sss, sss, 1, 1, in, + Kernels::DiracOptDhopSiteDag(st, lo, U, st.CommBuf(), sss, sss, 1, 1, in, out); } } else { PARALLEL_FOR_LOOP for (int sss = 0; sss < in._grid->oSites(); sss++) { - Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sss, sss, 1, 1, in, + Kernels::DiracOptDhopSite(st, lo, U, st.CommBuf(), sss, sss, 1, 1, in, out); } } diff --git a/lib/qcd/action/fermion/WilsonFermion.h b/lib/qcd/action/fermion/WilsonFermion.h index 40616073..40fbd1bf 100644 --- a/lib/qcd/action/fermion/WilsonFermion.h +++ b/lib/qcd/action/fermion/WilsonFermion.h @@ -78,16 +78,15 @@ class WilsonFermion : public WilsonKernels, public WilsonFermionStatic { virtual void MooeeInv(const FermionField &in, FermionField &out); virtual void MooeeInvDag(const FermionField &in, FermionField &out); + virtual void MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _mass) ; + //////////////////////// // Derivative interface //////////////////////// // Interface calls an internal routine - void DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V, - int dag); - void DhopDerivOE(GaugeField &mat, const FermionField &U, - const FermionField &V, int dag); - void DhopDerivEO(GaugeField &mat, const FermionField &U, - const FermionField &V, int dag); + void DhopDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag); + void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag); + void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag); /////////////////////////////////////////////////////////////// // non-hermitian hopping term; half cb or both diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc index 3ced3443..4c2d24bf 100644 --- a/lib/qcd/action/fermion/WilsonFermion5D.cc +++ b/lib/qcd/action/fermion/WilsonFermion5D.cc @@ -184,44 +184,37 @@ void WilsonFermion5D::Report(void) if ( DhopCalls > 0 ) { std::cout << GridLogMessage << "#### Dhop calls report " << std::endl; - std::cout << GridLogMessage << "WilsonFermion5D Number of Dhop Calls : " << DhopCalls << std::endl; - std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " << DhopCommTime - << " us" << std::endl; - std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls : " - << DhopCommTime / DhopCalls << " us" << std::endl; - std::cout << GridLogMessage << "WilsonFermion5D Total Compute time : " - << DhopComputeTime << " us" << std::endl; - std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls : " - << DhopComputeTime / DhopCalls << " us" << std::endl; + std::cout << GridLogMessage << "WilsonFermion5D Number of Dhop Calls : " << DhopCalls << std::endl; + std::cout << GridLogMessage << "WilsonFermion5D Total Communication time : " << DhopCommTime<< " us" << std::endl; + std::cout << GridLogMessage << "WilsonFermion5D CommTime/Calls : " << DhopCommTime / DhopCalls << " us" << std::endl; + std::cout << GridLogMessage << "WilsonFermion5D Total Compute time : " << DhopComputeTime << " us" << std::endl; + std::cout << GridLogMessage << "WilsonFermion5D ComputeTime/Calls : " << DhopComputeTime / DhopCalls << " us" << std::endl; - RealD mflops = 1344*volume*DhopCalls/DhopComputeTime; + RealD mflops = 1344*volume*DhopCalls/DhopComputeTime/2; // 2 for red black counting std::cout << GridLogMessage << "Average mflops/s per call : " << mflops << std::endl; - std::cout << GridLogMessage << "Average mflops/s per call per node : " << mflops/NP << std::endl; + std::cout << GridLogMessage << "Average mflops/s per call per rank : " << mflops/NP << std::endl; } if ( DerivCalls > 0 ) { - std::cout << GridLogMessage << "#### Deriv calls report "<< std::endl; - std::cout << GridLogMessage << "WilsonFermion5D Number of Deriv Calls : " < 0 || DhopCalls > 0){ - std::cout << GridLogMessage << "WilsonFermion5D Stencil"<::DerivInternal(StencilImpl & st, assert(sF < B._grid->oSites()); assert(sU < U._grid->oSites()); - Kernels::DiracOptDhopDir(st, U, st.comm_buf, sF, sU, B, Btilde, mu, - gamma); + Kernels::DiracOptDhopDir(st, U, st.CommBuf(), sF, sU, B, Btilde, mu, gamma); //////////////////////////// // spin trace outer product @@ -342,10 +334,10 @@ void WilsonFermion5D::DerivInternal(StencilImpl & st, } template -void WilsonFermion5D::DhopDeriv( GaugeField &mat, - const FermionField &A, - const FermionField &B, - int dag) +void WilsonFermion5D::DhopDeriv(GaugeField &mat, + const FermionField &A, + const FermionField &B, + int dag) { conformable(A._grid,FermionGrid()); conformable(A._grid,B._grid); @@ -358,9 +350,9 @@ void WilsonFermion5D::DhopDeriv( GaugeField &mat, template void WilsonFermion5D::DhopDerivEO(GaugeField &mat, - const FermionField &A, - const FermionField &B, - int dag) + const FermionField &A, + const FermionField &B, + int dag) { conformable(A._grid,FermionRedBlackGrid()); conformable(GaugeRedBlackGrid(),mat._grid); @@ -376,9 +368,9 @@ void WilsonFermion5D::DhopDerivEO(GaugeField &mat, template void WilsonFermion5D::DhopDerivOE(GaugeField &mat, - const FermionField &A, - const FermionField &B, - int dag) + const FermionField &A, + const FermionField &B, + int dag) { conformable(A._grid,FermionRedBlackGrid()); conformable(GaugeRedBlackGrid(),mat._grid); @@ -393,10 +385,9 @@ void WilsonFermion5D::DhopDerivOE(GaugeField &mat, template void WilsonFermion5D::DhopInternal(StencilImpl & st, LebesgueOrder &lo, - DoubledGaugeField & U, - const FermionField &in, FermionField &out,int dag) + DoubledGaugeField & U, + const FermionField &in, FermionField &out,int dag) { - DhopCalls++; // assert((dag==DaggerNo) ||(dag==DaggerYes)); Compressor compressor(dag); @@ -413,16 +404,35 @@ void WilsonFermion5D::DhopInternal(StencilImpl & st, LebesgueOrder &lo, for (int ss = 0; ss < U._grid->oSites(); ss++) { int sU = ss; int sF = LLs * sU; - Kernels::DiracOptDhopSiteDag(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in, - out); + Kernels::DiracOptDhopSiteDag(st, lo, U, st.CommBuf(), sF, sU, LLs, 1, in, out); } +#ifdef AVX512 + } else if (stat.is_init() ) { + + int nthreads; + stat.start(); +#pragma omp parallel + { +#pragma omp master + nthreads = omp_get_num_threads(); + int mythread = omp_get_thread_num(); + stat.enter(mythread); +#pragma omp for nowait + for(int ss=0;ssoSites();ss++) { + int sU=ss; + int sF=LLs*sU; + Kernels::DiracOptDhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out); + } + stat.exit(mythread); + } + stat.accum(nthreads); +#endif } else { PARALLEL_FOR_LOOP for (int ss = 0; ss < U._grid->oSites(); ss++) { int sU = ss; int sF = LLs * sU; - Kernels::DiracOptDhopSite(st, lo, U, st.comm_buf, sF, sU, LLs, 1, in, - out); + Kernels::DiracOptDhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out); } } DhopComputeTime+=usecond(); @@ -432,6 +442,7 @@ void WilsonFermion5D::DhopInternal(StencilImpl & st, LebesgueOrder &lo, template void WilsonFermion5D::DhopOE(const FermionField &in, FermionField &out,int dag) { + DhopCalls++; conformable(in._grid,FermionRedBlackGrid()); // verifies half grid conformable(in._grid,out._grid); // drops the cb check @@ -443,6 +454,7 @@ void WilsonFermion5D::DhopOE(const FermionField &in, FermionField &out,int template void WilsonFermion5D::DhopEO(const FermionField &in, FermionField &out,int dag) { + DhopCalls++; conformable(in._grid,FermionRedBlackGrid()); // verifies half grid conformable(in._grid,out._grid); // drops the cb check @@ -454,6 +466,7 @@ void WilsonFermion5D::DhopEO(const FermionField &in, FermionField &out,int template void WilsonFermion5D::Dhop(const FermionField &in, FermionField &out,int dag) { + DhopCalls+=2; conformable(in._grid,FermionGrid()); // verifies full grid conformable(in._grid,out._grid); @@ -469,6 +482,148 @@ void WilsonFermion5D::DW(const FermionField &in, FermionField &out,int dag axpy(out,4.0-M5,in,out); } +template +void WilsonFermion5D::MomentumSpacePropagatorHt(FermionField &out,const FermionField &in, RealD mass) +{ + // what type LatticeComplex + GridBase *_grid = _FourDimGrid; + conformable(_grid,out._grid); + + typedef typename FermionField::vector_type vector_type; + typedef typename FermionField::scalar_type ScalComplex; + typedef iSinglet Tcomplex; + typedef Lattice > LatComplex; + + Gamma::GammaMatrix Gmu [] = { + Gamma::GammaX, + Gamma::GammaY, + Gamma::GammaZ, + Gamma::GammaT + }; + + std::vector latt_size = _grid->_fdimensions; + + + FermionField num (_grid); num = zero; + + LatComplex sk(_grid); sk = zero; + LatComplex sk2(_grid); sk2= zero; + LatComplex W(_grid); W= zero; + LatComplex a(_grid); a= zero; + LatComplex one (_grid); one = ScalComplex(1.0,0.0); + LatComplex denom(_grid); denom= zero; + LatComplex cosha(_grid); + LatComplex kmu(_grid); + LatComplex Wea(_grid); + LatComplex Wema(_grid); + + ScalComplex ci(0.0,1.0); + + for(int mu=0;mu alpha + //////////////////////////////////////////// + cosha = (one + W*W + sk) / (W*2.0); + + // FIXME Need a Lattice acosh + for(int idx=0;idx<_grid->lSites();idx++){ + std::vector lcoor(Nd); + Tcomplex cc; + RealD sgn; + _grid->LocalIndexToLocalCoor(idx,lcoor); + peekLocalSite(cc,cosha,lcoor); + assert((double)real(cc)>=1.0); + assert(fabs((double)imag(cc))<=1.0e-15); + cc = ScalComplex(::acosh(real(cc)),0.0); + pokeLocalSite(cc,a,lcoor); + } + + Wea = ( exp( a) * W ); + Wema= ( exp(-a) * W ); + + num = num + ( one - Wema ) * mass * in; + denom= ( Wea - one ) + mass*mass * (one - Wema); + out = num/denom; +} + +template +void WilsonFermion5D::MomentumSpacePropagatorHw(FermionField &out,const FermionField &in,RealD mass) +{ + Gamma::GammaMatrix Gmu [] = { + Gamma::GammaX, + Gamma::GammaY, + Gamma::GammaZ, + Gamma::GammaT + }; + + GridBase *_grid = _FourDimGrid; + conformable(_grid,out._grid); + + typedef typename FermionField::vector_type vector_type; + typedef typename FermionField::scalar_type ScalComplex; + + typedef Lattice > LatComplex; + + + std::vector latt_size = _grid->_fdimensions; + + LatComplex sk(_grid); sk = zero; + LatComplex sk2(_grid); sk2= zero; + + LatComplex w_k(_grid); w_k= zero; + LatComplex b_k(_grid); b_k= zero; + + LatComplex one (_grid); one = ScalComplex(1.0,0.0); + + FermionField num (_grid); num = zero; + LatComplex denom(_grid); denom= zero; + LatComplex kmu(_grid); + ScalComplex ci(0.0,1.0); + + for(int mu=0;mu #ifndef GRID_QCD_WILSON_FERMION_5D_H #define GRID_QCD_WILSON_FERMION_5D_H -namespace Grid { +#include - namespace QCD { +namespace Grid { +namespace QCD { + + //////////////////////////////////////////////////////////////////////////////// + // This is the 4d red black case appropriate to support + // + // parity = (x+y+z+t)|2; + // generalised five dim fermions like mobius, zolotarev etc.. + // + // i.e. even even contains fifth dim hopping term. + // + // [DIFFERS from original CPS red black implementation parity = (x+y+z+t+s)|2 ] + //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// // This is the 4d red black case appropriate to support @@ -60,6 +72,7 @@ namespace Grid { public: INHERIT_IMPL_TYPES(Impl); typedef WilsonKernels Kernels; + PmuStat stat; void Report(void); void ZeroCounters(void); @@ -99,6 +112,9 @@ namespace Grid { virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag); virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag); + void MomentumSpacePropagatorHt(FermionField &out,const FermionField &in,RealD mass) ; + void MomentumSpacePropagatorHw(FermionField &out,const FermionField &in,RealD mass) ; + // Implement hopping term non-hermitian hopping term; half cb or both // Implement s-diagonal DW void DW (const FermionField &in, FermionField &out,int dag); @@ -108,78 +124,78 @@ namespace Grid { // add a DhopComm // -- suboptimal interface will presently trigger multiple comms. - void DhopDir(const FermionField &in, FermionField &out,int dir,int disp); - - /////////////////////////////////////////////////////////////// - // New methods added - /////////////////////////////////////////////////////////////// - void DerivInternal(StencilImpl & st, - DoubledGaugeField & U, - GaugeField &mat, - const FermionField &A, - const FermionField &B, - int dag); - - void DhopInternal(StencilImpl & st, - LebesgueOrder &lo, - DoubledGaugeField &U, - const FermionField &in, - FermionField &out, - int dag); - - // Constructors - WilsonFermion5D(GaugeField &_Umu, - GridCartesian &FiveDimGrid, - GridRedBlackCartesian &FiveDimRedBlackGrid, - GridCartesian &FourDimGrid, - GridRedBlackCartesian &FourDimRedBlackGrid, - double _M5,const ImplParams &p= ImplParams()); - - // Constructors - /* + void DhopDir(const FermionField &in, FermionField &out,int dir,int disp); + + /////////////////////////////////////////////////////////////// + // New methods added + /////////////////////////////////////////////////////////////// + void DerivInternal(StencilImpl & st, + DoubledGaugeField & U, + GaugeField &mat, + const FermionField &A, + const FermionField &B, + int dag); + + void DhopInternal(StencilImpl & st, + LebesgueOrder &lo, + DoubledGaugeField &U, + const FermionField &in, + FermionField &out, + int dag); + + // Constructors + WilsonFermion5D(GaugeField &_Umu, + GridCartesian &FiveDimGrid, + GridRedBlackCartesian &FiveDimRedBlackGrid, + GridCartesian &FourDimGrid, + GridRedBlackCartesian &FourDimRedBlackGrid, + double _M5,const ImplParams &p= ImplParams()); + + // Constructors + /* WilsonFermion5D(int simd, - GaugeField &_Umu, - GridCartesian &FiveDimGrid, - GridRedBlackCartesian &FiveDimRedBlackGrid, - GridCartesian &FourDimGrid, - double _M5,const ImplParams &p= ImplParams()); - */ + GaugeField &_Umu, + GridCartesian &FiveDimGrid, + GridRedBlackCartesian &FiveDimRedBlackGrid, + GridCartesian &FourDimGrid, + double _M5,const ImplParams &p= ImplParams()); + */ + + // DoubleStore + void ImportGauge(const GaugeField &_Umu); + + /////////////////////////////////////////////////////////////// + // Data members require to support the functionality + /////////////////////////////////////////////////////////////// + public: + + // Add these to the support from Wilson + GridBase *_FourDimGrid; + GridBase *_FourDimRedBlackGrid; + GridBase *_FiveDimGrid; + GridBase *_FiveDimRedBlackGrid; + + double M5; + int Ls; + + //Defines the stencils for even and odd + StencilImpl Stencil; + StencilImpl StencilEven; + StencilImpl StencilOdd; + + // Copy of the gauge field , with even and odd subsets + DoubledGaugeField Umu; + DoubledGaugeField UmuEven; + DoubledGaugeField UmuOdd; + + LebesgueOrder Lebesgue; + LebesgueOrder LebesgueEvenOdd; + + // Comms buffer + std::vector > comm_buf; + + }; - // DoubleStore - void ImportGauge(const GaugeField &_Umu); - - /////////////////////////////////////////////////////////////// - // Data members require to support the functionality - /////////////////////////////////////////////////////////////// - public: - - // Add these to the support from Wilson - GridBase *_FourDimGrid; - GridBase *_FourDimRedBlackGrid; - GridBase *_FiveDimGrid; - GridBase *_FiveDimRedBlackGrid; - - double M5; - int Ls; - - //Defines the stencils for even and odd - StencilImpl Stencil; - StencilImpl StencilEven; - StencilImpl StencilOdd; - - // Copy of the gauge field , with even and odd subsets - DoubledGaugeField Umu; - DoubledGaugeField UmuEven; - DoubledGaugeField UmuOdd; - - LebesgueOrder Lebesgue; - LebesgueOrder LebesgueEvenOdd; - - // Comms buffer - std::vector > comm_buf; - - }; - } -} +}} #endif diff --git a/lib/qcd/action/fermion/WilsonKernels.cc b/lib/qcd/action/fermion/WilsonKernels.cc index 49bd98c5..43776c86 100644 --- a/lib/qcd/action/fermion/WilsonKernels.cc +++ b/lib/qcd/action/fermion/WilsonKernels.cc @@ -32,8 +32,7 @@ directory namespace Grid { namespace QCD { -int WilsonKernelsStatic::HandOpt; -int WilsonKernelsStatic::AsmOpt; +int WilsonKernelsStatic::Opt; template WilsonKernels::WilsonKernels(const ImplParams &p) : Base(p){}; @@ -43,10 +42,9 @@ WilsonKernels::WilsonKernels(const ImplParams &p) : Base(p){}; //////////////////////////////////////////// template -void WilsonKernels::DiracOptGenericDhopSiteDag( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, int sF, - int sU, const FermionField &in, FermionField &out) { +void WilsonKernels::DiracOptGenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, + SiteHalfSpinor *buf, int sF, + int sU, const FermionField &in, FermionField &out) { SiteHalfSpinor tmp; SiteHalfSpinor chi; SiteHalfSpinor *chi_p; @@ -220,10 +218,9 @@ void WilsonKernels::DiracOptGenericDhopSiteDag( // Need controls to do interior, exterior, or both template -void WilsonKernels::DiracOptGenericDhopSite( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, int sF, - int sU, const FermionField &in, FermionField &out) { +void WilsonKernels::DiracOptGenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, + SiteHalfSpinor *buf, int sF, + int sU, const FermionField &in, FermionField &out) { SiteHalfSpinor tmp; SiteHalfSpinor chi; SiteHalfSpinor *chi_p; @@ -396,10 +393,9 @@ void WilsonKernels::DiracOptGenericDhopSite( }; template -void WilsonKernels::DiracOptDhopDir( - StencilImpl &st, DoubledGaugeField &U, - std::vector > &buf, int sF, - int sU, const FermionField &in, FermionField &out, int dir, int gamma) { +void WilsonKernels::DiracOptDhopDir( StencilImpl &st, DoubledGaugeField &U,SiteHalfSpinor *buf, int sF, + int sU, const FermionField &in, FermionField &out, int dir, int gamma) { + SiteHalfSpinor tmp; SiteHalfSpinor chi; SiteSpinor result; diff --git a/lib/qcd/action/fermion/WilsonKernels.h b/lib/qcd/action/fermion/WilsonKernels.h index 23c145de..47da2b14 100644 --- a/lib/qcd/action/fermion/WilsonKernels.h +++ b/lib/qcd/action/fermion/WilsonKernels.h @@ -32,175 +32,152 @@ directory #define GRID_QCD_DHOP_H namespace Grid { +namespace QCD { - namespace QCD { - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Helper routines that implement Wilson stencil for a single site. - // Common to both the WilsonFermion and WilsonFermion5D - //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - class WilsonKernelsStatic { - public: - // S-direction is INNERMOST and takes no part in the parity. - static int AsmOpt; // these are a temporary hack - static int HandOpt; // these are a temporary hack - }; - - template class WilsonKernels : public FermionOperator , public WilsonKernelsStatic { - public: - - INHERIT_IMPL_TYPES(Impl); - typedef FermionOperator Base; - - public: - - template - typename std::enable_if::type - DiracOptDhopSite( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, int Ls, int Ns, const FermionField &in, - FermionField &out) { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Helper routines that implement Wilson stencil for a single site. + // Common to both the WilsonFermion and WilsonFermion5D + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class WilsonKernelsStatic { + public: + enum { OptGeneric, OptHandUnroll, OptInlineAsm }; + // S-direction is INNERMOST and takes no part in the parity. + static int Opt; // these are a temporary hack +}; + +template class WilsonKernels : public FermionOperator , public WilsonKernelsStatic { + public: + + INHERIT_IMPL_TYPES(Impl); + typedef FermionOperator Base; + +public: + + template + typename std::enable_if::type + DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) + { + switch(Opt) { #ifdef AVX512 - if (AsmOpt) { - WilsonKernels::DiracOptAsmDhopSite(st, lo, U, buf, sF, sU, Ls, Ns, - in, out); - - } else { -#else - { + case OptInlineAsm: + WilsonKernels::DiracOptAsmDhopSite(st,lo,U,buf,sF,sU,Ls,Ns,in,out); + break; #endif - for (int site = 0; site < Ns; site++) { - for (int s = 0; s < Ls; s++) { - if (HandOpt) - WilsonKernels::DiracOptHandDhopSite(st, lo, U, buf, sF, sU, - in, out); - else - WilsonKernels::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, - in, out); - sF++; - } - sU++; - } - } + case OptHandUnroll: + for (int site = 0; site < Ns; site++) { + for (int s = 0; s < Ls; s++) { + WilsonKernels::DiracOptHandDhopSite(st,lo,U,buf,sF,sU,in,out); + sF++; } - - template - typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type - DiracOptDhopSite( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, int Ls, int Ns, const FermionField &in, - FermionField &out) { - for (int site = 0; site < Ns; site++) { - for (int s = 0; s < Ls; s++) { - WilsonKernels::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in, - out); - sF++; - } - sU++; - } - } - - template - typename std::enable_if::type - DiracOptDhopSiteDag( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, int Ls, int Ns, const FermionField &in, - FermionField &out) { -#ifdef AVX512 - if (AsmOpt) { - WilsonKernels::DiracOptAsmDhopSiteDag(st, lo, U, buf, sF, sU, Ls, - Ns, in, out); - } else { -#else - { -#endif - for (int site = 0; site < Ns; site++) { - for (int s = 0; s < Ls; s++) { - if (HandOpt) - WilsonKernels::DiracOptHandDhopSiteDag(st, lo, U, buf, sF, sU, - in, out); - else - WilsonKernels::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF, - sU, in, out); - sF++; - } - sU++; - } - } - } - - template - typename std::enable_if< - (Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, - void>::type - DiracOptDhopSiteDag( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, int Ls, int Ns, const FermionField &in, - FermionField &out) { - for (int site = 0; site < Ns; site++) { - for (int s = 0; s < Ls; s++) { - WilsonKernels::DiracOptGenericDhopSiteDag(st, lo, U, buf, sF, sU, - in, out); - sF++; - } - sU++; - } - } - - void DiracOptDhopDir( - StencilImpl &st, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, const FermionField &in, FermionField &out, int dirdisp, - int gamma); - - private: - // Specialised variants - void DiracOptGenericDhopSite( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, const FermionField &in, FermionField &out); - - void DiracOptGenericDhopSiteDag( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, const FermionField &in, FermionField &out); - - void DiracOptAsmDhopSite( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, int Ls, int Ns, const FermionField &in, - FermionField &out); - - void DiracOptAsmDhopSiteDag( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, int Ls, int Ns, const FermionField &in, - FermionField &out); - - void DiracOptHandDhopSite( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, const FermionField &in, FermionField &out); - - void DiracOptHandDhopSiteDag( - StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, - std::vector > &buf, - int sF, int sU, const FermionField &in, FermionField &out); - - public: - WilsonKernels(const ImplParams &p = ImplParams()); - }; - + sU++; } + break; + case OptGeneric: + for (int site = 0; site < Ns; site++) { + for (int s = 0; s < Ls; s++) { + WilsonKernels::DiracOptGenericDhopSite(st,lo,U,buf,sF,sU,in,out); + sF++; + } + sU++; + } + break; + default: + assert(0); } + } + + template + typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool, void>::type + DiracOptDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) { + // no kernel choice + for (int site = 0; site < Ns; site++) { + for (int s = 0; s < Ls; s++) { + WilsonKernels::DiracOptGenericDhopSite(st, lo, U, buf, sF, sU, in, out); + sF++; + } + sU++; + } + } + + template + typename std::enable_if::type + DiracOptDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) { + switch(Opt) { +#ifdef AVX512 + case OptInlineAsm: + WilsonKernels::DiracOptAsmDhopSiteDag(st,lo,U,buf,sF,sU,Ls,Ns,in,out); + break; +#endif + case OptHandUnroll: + for (int site = 0; site < Ns; site++) { + for (int s = 0; s < Ls; s++) { + WilsonKernels::DiracOptHandDhopSiteDag(st,lo,U,buf,sF,sU,in,out); + sF++; + } + sU++; + } + break; + case OptGeneric: + for (int site = 0; site < Ns; site++) { + for (int s = 0; s < Ls; s++) { + WilsonKernels::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out); + sF++; + } + sU++; + } + break; + default: + assert(0); + } + } + template + typename std::enable_if<(Impl::Dimension != 3 || (Impl::Dimension == 3 && Nc != 3)) && EnableBool,void>::type + DiracOptDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,SiteHalfSpinor * buf, + int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out) { + for (int site = 0; site < Ns; site++) { + for (int s = 0; s < Ls; s++) { + WilsonKernels::DiracOptGenericDhopSiteDag(st,lo,U,buf,sF,sU,in,out); + sF++; + } + sU++; + } + } + void DiracOptDhopDir(StencilImpl &st, DoubledGaugeField &U,SiteHalfSpinor * buf, + int sF, int sU, const FermionField &in, FermionField &out, int dirdisp, int gamma); + +private: + // Specialised variants + void DiracOptGenericDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, const FermionField &in, FermionField &out); + + void DiracOptGenericDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, const FermionField &in, FermionField &out); + + void DiracOptAsmDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, int Ls, int Ns, const FermionField &in,FermionField &out); + + void DiracOptAsmDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, int Ls, int Ns, const FermionField &in, FermionField &out); + + void DiracOptHandDhopSite(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, const FermionField &in, FermionField &out); + + void DiracOptHandDhopSiteDag(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U, SiteHalfSpinor * buf, + int sF, int sU, const FermionField &in, FermionField &out); + +public: + + WilsonKernels(const ImplParams &p = ImplParams()); + +}; + +}} #endif diff --git a/lib/qcd/action/fermion/WilsonKernelsAsm.cc b/lib/qcd/action/fermion/WilsonKernelsAsm.cc index b09699ef..d7a9edd3 100644 --- a/lib/qcd/action/fermion/WilsonKernelsAsm.cc +++ b/lib/qcd/action/fermion/WilsonKernelsAsm.cc @@ -10,6 +10,7 @@ Author: Peter Boyle Author: paboyle +Author: Guido Cossu This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -33,48 +34,46 @@ Author: paboyle namespace Grid { - namespace QCD { +namespace QCD { - /////////////////////////////////////////////////////////// - // Default to no assembler implementation - /////////////////////////////////////////////////////////// - template - void WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) - { - assert(0); - } - template - void WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) - { - assert(0); - } - +/////////////////////////////////////////////////////////// +// Default to no assembler implementation +/////////////////////////////////////////////////////////// +template void +WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +{ + assert(0); +} +template void +WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +{ + assert(0); +} #if defined(AVX512) - - +#include + /////////////////////////////////////////////////////////// // If we are AVX512 specialise the single precision routine /////////////////////////////////////////////////////////// - -#include + #include - static Vector signs; - - int setupSigns(void ){ - Vector bother(2); - signs = bother; - vrsign(signs[0]); - visign(signs[1]); - return 1; - } - static int signInit = setupSigns(); +static Vector signsF; + + template + int setupSigns(Vector& signs ){ + Vector bother(2); + signs = bother; + vrsign(signs[0]); + visign(signs[1]); + return 1; + } + + static int signInitF = setupSigns(signsF); #define label(A) ilabel(A) #define ilabel(A) ".globl\n" #A ":\n" @@ -82,19 +81,19 @@ namespace Grid { #define MAYBEPERM(A,perm) if (perm) { A ; } #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf) #define FX(A) WILSONASM_ ##A +#define COMPLEX_TYPE vComplexF +#define signs signsF #undef KERNEL_DAG - template<> - void WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) #include #define KERNEL_DAG - template<> - void WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) #include #undef VMOVIDUP @@ -104,36 +103,94 @@ namespace Grid { #undef FX #define FX(A) DWFASM_ ## A #define MAYBEPERM(A,B) -#define VMOVIDUP(A,B,C) VBCASTIDUPf(A,B,C) -#define VMOVRDUP(A,B,C) VBCASTRDUPf(A,B,C) +//#define VMOVIDUP(A,B,C) VBCASTIDUPf(A,B,C) +//#define VMOVRDUP(A,B,C) VBCASTRDUPf(A,B,C) #define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf) #undef KERNEL_DAG - template<> - void WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) #include #define KERNEL_DAG - template<> - void WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +#include +#undef COMPLEX_TYPE +#undef signs +#undef VMOVRDUP +#undef MAYBEPERM +#undef MULT_2SPIN +#undef FX + +/////////////////////////////////////////////////////////// +// If we are AVX512 specialise the double precision routine +/////////////////////////////////////////////////////////// + +#include + +static Vector signsD; +#define signs signsD +static int signInitD = setupSigns(signsD); + +#define MAYBEPERM(A,perm) if (perm) { A ; } +#define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN(ptr,pf) +#define FX(A) WILSONASM_ ##A +#define COMPLEX_TYPE vComplexD + +#undef KERNEL_DAG +template<> void +WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +#include + +#define KERNEL_DAG +template<> void +WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) #include -#endif +#undef VMOVIDUP +#undef VMOVRDUP +#undef MAYBEPERM +#undef MULT_2SPIN +#undef FX +#define FX(A) DWFASM_ ## A +#define MAYBEPERM(A,B) +//#define VMOVIDUP(A,B,C) VBCASTIDUPd(A,B,C) +//#define VMOVRDUP(A,B,C) VBCASTRDUPd(A,B,C) +#define MULT_2SPIN(ptr,pf) MULT_ADDSUB_2SPIN_LS(ptr,pf) + +#undef KERNEL_DAG +template<> void +WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +#include + +#define KERNEL_DAG +template<> void +WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out) +#include + +#undef COMPLEX_TYPE +#undef signs +#undef VMOVRDUP +#undef MAYBEPERM +#undef MULT_2SPIN +#undef FX +#endif //AVX512 #define INSTANTIATE_ASM(A)\ -template void WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\ - std::vector > &buf,\ +template void WilsonKernels::DiracOptAsmDhopSite(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,\ int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\ -template void WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U,\ - std::vector > &buf,\ + \ +template void WilsonKernels::DiracOptAsmDhopSiteDag(StencilImpl &st,LebesgueOrder & lo,DoubledGaugeField &U, SiteHalfSpinor *buf,\ int ss,int ssU,int Ls,int Ns,const FermionField &in, FermionField &out);\ - INSTANTIATE_ASM(WilsonImplF); INSTANTIATE_ASM(WilsonImplD); INSTANTIATE_ASM(ZWilsonImplF); @@ -144,6 +201,6 @@ INSTANTIATE_ASM(DomainWallVec5dImplF); INSTANTIATE_ASM(DomainWallVec5dImplD); INSTANTIATE_ASM(ZDomainWallVec5dImplF); INSTANTIATE_ASM(ZDomainWallVec5dImplD); - } -} + +}} diff --git a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h index d236a774..72e13754 100644 --- a/lib/qcd/action/fermion/WilsonKernelsAsmBody.h +++ b/lib/qcd/action/fermion/WilsonKernelsAsmBody.h @@ -5,7 +5,9 @@ const uint64_t plocal =(uint64_t) & in._odata[0]; // vComplexF isigns[2] = { signs[0], signs[1] }; - vComplexF *isigns = &signs[0]; + //COMPLEX_TYPE is vComplexF of vComplexD depending + //on the chosen precision + COMPLEX_TYPE *isigns = &signs[0]; MASK_REGS; int nmax=U._grid->oSites(); @@ -134,7 +136,9 @@ //////////////////////////////// // Xm //////////////////////////////// +#ifndef STREAM_STORE basep= (uint64_t) &out._odata[ss]; +#endif // basep= st.GetPFInfo(nent,plocal); nent++; if ( local ) { LOAD64(%r10,isigns); // times i => shuffle and xor the real part sign bit @@ -229,7 +233,9 @@ LOAD_CHI(base); } base= (uint64_t) &out._odata[ss]; +#ifndef STREAM_STORE PREFETCH_CHIMU(base); +#endif { MULT_2SPIN_DIR_PFTM(Tm,basep); } diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc index 15c8ab56..f5900832 100644 --- a/lib/qcd/action/fermion/WilsonKernelsHand.cc +++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc @@ -311,10 +311,9 @@ namespace Grid { namespace QCD { - template - void WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int sU,const FermionField &in, FermionField &out) +template void +WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int sU,const FermionField &in, FermionField &out) { typedef typename Simd::scalar_type S; typedef typename Simd::vector_type V; @@ -554,10 +553,9 @@ namespace QCD { } } - template - void WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, - std::vector > &buf, - int ss,int sU,const FermionField &in, FermionField &out) +template +void WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int ss,int sU,const FermionField &in, FermionField &out) { // std::cout << "Hand op Dhop "< -void WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, - std::vector > &buf, - int sF,int sU,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, + SiteHalfSpinor *buf, + int sF,int sU,const FermionField &in, FermionField &out) { assert(0); } -template<> -void WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, - std::vector > &buf, - int sF,int sU,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, + SiteHalfSpinor *buf, + int sF,int sU,const FermionField &in, FermionField &out) { assert(0); } -template<> -void WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, - std::vector > &buf, - int sF,int sU,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int sF,int sU,const FermionField &in, FermionField &out) { assert(0); } -template<> -void WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U, - std::vector > &buf, - int sF,int sU,const FermionField &in, FermionField &out) +template<> void +WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, + int sF,int sU,const FermionField &in, FermionField &out) { assert(0); } @@ -840,12 +835,10 @@ void WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st, // Need Nc=3 though // #define INSTANTIATE_THEM(A) \ -template void WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\ - std::vector > &buf,\ - int ss,int sU,const FermionField &in, FermionField &out);\ -template void WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,\ - std::vector > &buf,\ - int ss,int sU,const FermionField &in, FermionField &out); +template void WilsonKernels::DiracOptHandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\ + int ss,int sU,const FermionField &in, FermionField &out); \ +template void WilsonKernels::DiracOptHandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,\ + int ss,int sU,const FermionField &in, FermionField &out); INSTANTIATE_THEM(WilsonImplF); INSTANTIATE_THEM(WilsonImplD); diff --git a/lib/qcd/hmc/HmcRunner.h b/lib/qcd/hmc/HmcRunner.h index a31ba784..53b127cf 100644 --- a/lib/qcd/hmc/HmcRunner.h +++ b/lib/qcd/hmc/HmcRunner.h @@ -116,7 +116,7 @@ class NerscHmcRunnerTemplate { NoSmearing SmearingPolicy; typedef MinimumNorm2, RepresentationsPolicy > IntegratorType; // change here to change the algorithm - IntegratorParameters MDpar(20, 1.0); + IntegratorParameters MDpar(40, 1.0); IntegratorType MDynamics(UGrid, MDpar, TheAction, SmearingPolicy); // Checkpoint strategy diff --git a/lib/qcd/utils/LinalgUtils.h b/lib/qcd/utils/LinalgUtils.h index c2cb95ee..e7e6a794 100644 --- a/lib/qcd/utils/LinalgUtils.h +++ b/lib/qcd/utils/LinalgUtils.h @@ -39,8 +39,8 @@ namespace QCD{ //on the 5d (rb4d) checkerboarded lattices //////////////////////////////////////////////////////////////////////// -template -void axpibg5x(Lattice &z,const Lattice &x,RealD a,RealD b) +template +void axpibg5x(Lattice &z,const Lattice &x,Coeff a,Coeff b) { z.checkerboard = x.checkerboard; conformable(x,z); @@ -57,8 +57,8 @@ PARALLEL_FOR_LOOP } } -template -void axpby_ssp(Lattice &z, RealD a,const Lattice &x,RealD b,const Lattice &y,int s,int sp) +template +void axpby_ssp(Lattice &z, Coeff a,const Lattice &x,Coeff b,const Lattice &y,int s,int sp) { z.checkerboard = x.checkerboard; conformable(x,y); @@ -72,8 +72,8 @@ PARALLEL_FOR_LOOP } } -template -void ag5xpby_ssp(Lattice &z,RealD a,const Lattice &x,RealD b,const Lattice &y,int s,int sp) +template +void ag5xpby_ssp(Lattice &z,Coeff a,const Lattice &x,Coeff b,const Lattice &y,int s,int sp) { z.checkerboard = x.checkerboard; conformable(x,y); @@ -90,8 +90,8 @@ PARALLEL_FOR_LOOP } } -template -void axpbg5y_ssp(Lattice &z,RealD a,const Lattice &x,RealD b,const Lattice &y,int s,int sp) +template +void axpbg5y_ssp(Lattice &z,Coeff a,const Lattice &x,Coeff b,const Lattice &y,int s,int sp) { z.checkerboard = x.checkerboard; conformable(x,y); @@ -108,8 +108,8 @@ PARALLEL_FOR_LOOP } } -template -void ag5xpbg5y_ssp(Lattice &z,RealD a,const Lattice &x,RealD b,const Lattice &y,int s,int sp) +template +void ag5xpbg5y_ssp(Lattice &z,Coeff a,const Lattice &x,Coeff b,const Lattice &y,int s,int sp) { z.checkerboard = x.checkerboard; conformable(x,y); @@ -127,8 +127,8 @@ PARALLEL_FOR_LOOP } } -template -void axpby_ssp_pminus(Lattice &z,RealD a,const Lattice &x,RealD b,const Lattice &y,int s,int sp) +template +void axpby_ssp_pminus(Lattice &z,Coeff a,const Lattice &x,Coeff b,const Lattice &y,int s,int sp) { z.checkerboard = x.checkerboard; conformable(x,y); @@ -144,8 +144,8 @@ PARALLEL_FOR_LOOP } } -template -void axpby_ssp_pplus(Lattice &z,RealD a,const Lattice &x,RealD b,const Lattice &y,int s,int sp) +template +void axpby_ssp_pplus(Lattice &z,Coeff a,const Lattice &x,Coeff b,const Lattice &y,int s,int sp) { z.checkerboard = x.checkerboard; conformable(x,y); diff --git a/lib/qcd/utils/SUn.h b/lib/qcd/utils/SUn.h index 20cd1889..914c3bad 100644 --- a/lib/qcd/utils/SUn.h +++ b/lib/qcd/utils/SUn.h @@ -674,6 +674,37 @@ class SU { out += la; } } +/* + add GaugeTrans +*/ + +template + static void GaugeTransform( GaugeField &Umu, GaugeMat &g){ + GridBase *grid = Umu._grid; + conformable(grid,g._grid); + + GaugeMat U(grid); + GaugeMat ag(grid); ag = adj(g); + + for(int mu=0;mu(Umu,mu); + U = g*U*Cshift(ag, mu, 1); + PokeIndex(Umu,U,mu); + } + } + template + static void GaugeTransform( std::vector &U, GaugeMat &g){ + GridBase *grid = g._grid; + GaugeMat ag(grid); ag = adj(g); + for(int mu=0;mu + static void RandomGaugeTransform(GridParallelRNG &pRNG, GaugeField &Umu, GaugeMat &g){ + LieRandomize(pRNG,g,1.0); + GaugeTransform(Umu,g); + } // Projects the algebra components a lattice matrix (of dimension ncol*ncol -1 ) // inverse operation: FundamentalLieAlgebraMatrix @@ -702,23 +733,33 @@ class SU { PokeIndex(out, Umu, mu); } } - static void TepidConfiguration(GridParallelRNG &pRNG, - LatticeGaugeField &out) { - LatticeMatrix Umu(out._grid); - for (int mu = 0; mu < Nd; mu++) { - LieRandomize(pRNG, Umu, 0.01); - PokeIndex(out, Umu, mu); + template + static void TepidConfiguration(GridParallelRNG &pRNG,GaugeField &out){ + typedef typename GaugeField::vector_type vector_type; + typedef iSUnMatrix vMatrixType; + typedef Lattice LatticeMatrixType; + + LatticeMatrixType Umu(out._grid); + for(int mu=0;mu(out,Umu,mu); } } - static void ColdConfiguration(GridParallelRNG &pRNG, LatticeGaugeField &out) { - LatticeMatrix Umu(out._grid); - Umu = 1.0; - for (int mu = 0; mu < Nd; mu++) { - PokeIndex(out, Umu, mu); + template + static void ColdConfiguration(GridParallelRNG &pRNG,GaugeField &out){ + typedef typename GaugeField::vector_type vector_type; + typedef iSUnMatrix vMatrixType; + typedef Lattice LatticeMatrixType; + + LatticeMatrixType Umu(out._grid); + Umu=1.0; + for(int mu=0;mu(out,Umu,mu); } } - static void taProj(const LatticeMatrix &in, LatticeMatrix &out) { + template + static void taProj( const LatticeMatrixType &in, LatticeMatrixType &out){ out = Ta(in); } template diff --git a/lib/qcd/utils/WilsonLoops.h b/lib/qcd/utils/WilsonLoops.h index 10022f50..03d45c07 100644 --- a/lib/qcd/utils/WilsonLoops.h +++ b/lib/qcd/utils/WilsonLoops.h @@ -522,4 +522,4 @@ typedef WilsonLoops SU3WilsonLoops; } } -#endif \ No newline at end of file +#endif diff --git a/lib/simd/Grid_avx.h b/lib/simd/Grid_avx.h index 03faabee..f50eae2b 100644 --- a/lib/simd/Grid_avx.h +++ b/lib/simd/Grid_avx.h @@ -365,6 +365,18 @@ namespace Optimization { } }; + struct Div{ + // Real float + inline __m256 operator()(__m256 a, __m256 b){ + return _mm256_div_ps(a,b); + } + // Real double + inline __m256d operator()(__m256d a, __m256d b){ + return _mm256_div_pd(a,b); + } + }; + + struct Conj{ // Complex single inline __m256 operator()(__m256 in){ @@ -437,14 +449,13 @@ namespace Optimization { }; -#if defined (AVX2) || defined (AVXFMA4) -#define _mm256_alignr_epi32(ret,a,b,n) ret=(__m256) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*4)%16) -#define _mm256_alignr_epi64(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16) +#if defined (AVX2) +#define _mm256_alignr_epi32_grid(ret,a,b,n) ret=(__m256) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*4)%16) +#define _mm256_alignr_epi64_grid(ret,a,b,n) ret=(__m256d) _mm256_alignr_epi8((__m256i)a,(__m256i)b,(n*8)%16) #endif -#if defined (AVX1) - -#define _mm256_alignr_epi32(ret,a,b,n) { \ +#if defined (AVX1) || defined (AVXFMA) +#define _mm256_alignr_epi32_grid(ret,a,b,n) { \ __m128 aa, bb; \ \ aa = _mm256_extractf128_ps(a,1); \ @@ -458,7 +469,7 @@ namespace Optimization { ret = _mm256_insertf128_ps(ret,aa,0); \ } -#define _mm256_alignr_epi64(ret,a,b,n) { \ +#define _mm256_alignr_epi64_grid(ret,a,b,n) { \ __m128d aa, bb; \ \ aa = _mm256_extractf128_pd(a,1); \ @@ -474,19 +485,6 @@ namespace Optimization { #endif - inline std::ostream & operator << (std::ostream& stream, const __m256 a) - { - const float *p=(const float *)&a; - stream<< "{"< 3 ) { - _mm256_alignr_epi32(ret,in,tmp,n); + _mm256_alignr_epi32_grid(ret,in,tmp,n); } else { - _mm256_alignr_epi32(ret,tmp,in,n); + _mm256_alignr_epi32_grid(ret,tmp,in,n); } - // std::cout << " align epi32 n=" < "<< ret < 1 ) { - _mm256_alignr_epi64(ret,in,tmp,n); + _mm256_alignr_epi64_grid(ret,in,tmp,n); } else { - _mm256_alignr_epi64(ret,tmp,in,n); + _mm256_alignr_epi64_grid(ret,tmp,in,n); } - // std::cout << " align epi64 n=" < "<< ret < inline Grid::ComplexF Reduce::operator()(__m256 in){ @@ -631,6 +625,7 @@ namespace Optimization { // Arithmetic operations typedef Optimization::Sum SumSIMD; typedef Optimization::Sub SubSIMD; + typedef Optimization::Div DivSIMD; typedef Optimization::Mult MultSIMD; typedef Optimization::MultComplex MultComplexSIMD; typedef Optimization::Conj ConjSIMD; diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h index d13a6b84..38785138 100644 --- a/lib/simd/Grid_avx512.h +++ b/lib/simd/Grid_avx512.h @@ -32,6 +32,16 @@ Author: paboyle namespace Grid{ namespace Optimization { + + union u512f { + __m512 v; + float f[16]; + }; + + union u512d { + __m512d v; + double f[8]; + }; struct Vsplat{ //Complex float @@ -221,6 +231,17 @@ namespace Optimization { } }; + struct Div{ + // Real float + inline __m512 operator()(__m512 a, __m512 b){ + return _mm512_div_ps(a,b); + } + // Real double + inline __m512d operator()(__m512d a, __m512d b){ + return _mm512_div_pd(a,b); + } + }; + struct Conj{ // Complex single @@ -350,7 +371,67 @@ namespace Optimization { ////////////////////////////////////////////// // Some Template specialization + + // Hack for CLANG until mm512_reduce_add_ps etc... are implemented in GCC and Clang releases +#ifndef __INTEL_COMPILER +#warning "Slow reduction due to incomplete reduce intrinsics" + //Complex float Reduce + template<> + inline Grid::ComplexF Reduce::operator()(__m512 in){ + __m512 v1,v2; + v1=Optimization::Permute::Permute0(in); // avx 512; quad complex single + v1= _mm512_add_ps(v1,in); + v2=Optimization::Permute::Permute1(v1); + v1 = _mm512_add_ps(v1,v2); + v2=Optimization::Permute::Permute2(v1); + v1 = _mm512_add_ps(v1,v2); + u512f conv; conv.v = v1; + return Grid::ComplexF(conv.f[0],conv.f[1]); + } + //Real float Reduce + template<> + inline Grid::RealF Reduce::operator()(__m512 in){ + __m512 v1,v2; + v1 = Optimization::Permute::Permute0(in); // avx 512; octo-double + v1 = _mm512_add_ps(v1,in); + v2 = Optimization::Permute::Permute1(v1); + v1 = _mm512_add_ps(v1,v2); + v2 = Optimization::Permute::Permute2(v1); + v1 = _mm512_add_ps(v1,v2); + v2 = Optimization::Permute::Permute3(v1); + v1 = _mm512_add_ps(v1,v2); + u512f conv; conv.v=v1; + return conv.f[0]; + } + + + //Complex double Reduce + template<> + inline Grid::ComplexD Reduce::operator()(__m512d in){ + __m512d v1; + v1 = Optimization::Permute::Permute0(in); // sse 128; paired complex single + v1 = _mm512_add_pd(v1,in); + v1 = Optimization::Permute::Permute1(in); // sse 128; paired complex single + v1 = _mm512_add_pd(v1,in); + u512d conv; conv.v = v1; + return Grid::ComplexD(conv.f[0],conv.f[1]); + } + + //Real double Reduce + template<> + inline Grid::RealD Reduce::operator()(__m512d in){ + __m512d v1,v2; + v1 = Optimization::Permute::Permute0(in); // avx 512; quad double + v1 = _mm512_add_pd(v1,in); + v2 = Optimization::Permute::Permute1(v1); + v1 = _mm512_add_pd(v1,v2); + v2 = Optimization::Permute::Permute2(v1); + v1 = _mm512_add_pd(v1,v2); + u512d conv; conv.v = v1; + return conv.f[0]; + } +#else //Complex float Reduce template<> inline Grid::ComplexF Reduce::operator()(__m512 in){ @@ -362,7 +443,6 @@ namespace Optimization { return _mm512_reduce_add_ps(in); } - //Complex double Reduce template<> inline Grid::ComplexD Reduce::operator()(__m512d in){ @@ -382,6 +462,7 @@ namespace Optimization { printf("Reduce : Missing integer implementation -> FIX\n"); assert(0); } +#endif } @@ -418,6 +499,7 @@ namespace Optimization { typedef Optimization::Sum SumSIMD; typedef Optimization::Sub SubSIMD; typedef Optimization::Mult MultSIMD; + typedef Optimization::Div DivSIMD; typedef Optimization::MultComplex MultComplexSIMD; typedef Optimization::Conj ConjSIMD; typedef Optimization::TimesMinusI TimesMinusISIMD; diff --git a/lib/simd/Grid_generic.h b/lib/simd/Grid_generic.h index ca0029ef..62c78afb 100644 --- a/lib/simd/Grid_generic.h +++ b/lib/simd/Grid_generic.h @@ -6,8 +6,7 @@ Copyright (C) 2015 -Author: Peter Boyle -Author: neo +Author: Antonin Portelli This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -27,133 +26,352 @@ Author: neo *************************************************************************************/ /* END LEGAL */ +static_assert(GEN_SIMD_WIDTH % 16u == 0, "SIMD vector size is not an integer multiple of 16 bytes"); + +//#define VECTOR_LOOPS + +// playing with compiler pragmas +#ifdef VECTOR_LOOPS +#ifdef __clang__ +#define VECTOR_FOR(i, w, inc)\ +_Pragma("clang loop unroll(full) vectorize(enable) interleave(enable) vectorize_width(w)")\ +for (unsigned int i = 0; i < w; i += inc) +#elif defined __INTEL_COMPILER +#define VECTOR_FOR(i, w, inc)\ +_Pragma("simd vectorlength(w*8)")\ +for (unsigned int i = 0; i < w; i += inc) +#else +#define VECTOR_FOR(i, w, inc)\ +for (unsigned int i = 0; i < w; i += inc) +#endif +#else +#define VECTOR_FOR(i, w, inc)\ +for (unsigned int i = 0; i < w; i += inc) +#endif + namespace Grid { namespace Optimization { - template - union uconv { - float f; - vtype v; + // type traits giving the number of elements for each vector type + template struct W; + template <> struct W { + constexpr static unsigned int c = GEN_SIMD_WIDTH/16u; + constexpr static unsigned int r = GEN_SIMD_WIDTH/8u; }; - - union u128f { - float v; - float f[4]; - }; - union u128d { - double v; - double f[2]; + template <> struct W { + constexpr static unsigned int c = GEN_SIMD_WIDTH/8u; + constexpr static unsigned int r = GEN_SIMD_WIDTH/4u; }; + // SIMD vector types + template + struct vec { + alignas(GEN_SIMD_WIDTH) T v[W::r]; + }; + + typedef vec vecf; + typedef vec vecd; + struct Vsplat{ - //Complex float - inline u128f operator()(float a, float b){ - u128f out; - out.f[0] = a; - out.f[1] = b; - out.f[2] = a; - out.f[3] = b; + // Complex + template + inline vec operator()(T a, T b){ + vec out; + + VECTOR_FOR(i, W::r, 2) + { + out.v[i] = a; + out.v[i+1] = b; + } + return out; } - // Real float - inline u128f operator()(float a){ - u128f out; - out.f[0] = a; - out.f[1] = a; - out.f[2] = a; - out.f[3] = a; + + // Real + template + inline vec operator()(T a){ + vec out; + + VECTOR_FOR(i, W::r, 1) + { + out.v[i] = a; + } + return out; } - //Complex double - inline u128d operator()(double a, double b){ - u128d out; - out.f[0] = a; - out.f[1] = b; - return out; - } - //Real double - inline u128d operator()(double a){ - u128d out; - out.f[0] = a; - out.f[1] = a; - return out; - } - //Integer + + // Integer inline int operator()(Integer a){ return a; } }; struct Vstore{ - //Float - inline void operator()(u128f a, float* F){ - memcpy(F,a.f,4*sizeof(float)); - } - //Double - inline void operator()(u128d a, double* D){ - memcpy(D,a.f,2*sizeof(double)); + // Real + template + inline void operator()(vec a, T *D){ + *((vec *)D) = a; } //Integer - inline void operator()(int a, Integer* I){ - I[0] = a; + inline void operator()(int a, Integer *I){ + *I = a; } }; struct Vstream{ - //Float - inline void operator()(float * a, u128f b){ - memcpy(a,b.f,4*sizeof(float)); + // Real + template + inline void operator()(T * a, vec b){ + *((vec *)a) = b; } - //Double - inline void operator()(double * a, u128d b){ - memcpy(a,b.f,2*sizeof(double)); - } - - }; struct Vset{ - // Complex float - inline u128f operator()(Grid::ComplexF *a){ - u128f out; - out.f[0] = a[0].real(); - out.f[1] = a[0].imag(); - out.f[2] = a[1].real(); - out.f[3] = a[1].imag(); + // Complex + template + inline vec operator()(std::complex *a){ + vec out; + + VECTOR_FOR(i, W::c, 1) + { + out.v[2*i] = a[i].real(); + out.v[2*i+1] = a[i].imag(); + } + return out; } - // Complex double - inline u128d operator()(Grid::ComplexD *a){ - u128d out; - out.f[0] = a[0].real(); - out.f[1] = a[0].imag(); - return out; - } - // Real float - inline u128f operator()(float *a){ - u128f out; - out.f[0] = a[0]; - out.f[1] = a[1]; - out.f[2] = a[2]; - out.f[3] = a[3]; - return out; - } - // Real double - inline u128d operator()(double *a){ - u128d out; - out.f[0] = a[0]; - out.f[1] = a[1]; + + // Real + template + inline vec operator()(T *a){ + vec out; + + out = *((vec *)a); + return out; } + // Integer inline int operator()(Integer *a){ - return a[0]; + return *a; } - - }; + ///////////////////////////////////////////////////// + // Arithmetic operations + ///////////////////////////////////////////////////// + struct Sum{ + // Complex/Real + template + inline vec operator()(vec a, vec b){ + vec out; + + VECTOR_FOR(i, W::r, 1) + { + out.v[i] = a.v[i] + b.v[i]; + } + + return out; + } + + //I nteger + inline int operator()(int a, int b){ + return a + b; + } + }; + + struct Sub{ + // Complex/Real + template + inline vec operator()(vec a, vec b){ + vec out; + + VECTOR_FOR(i, W::r, 1) + { + out.v[i] = a.v[i] - b.v[i]; + } + + return out; + } + + //Integer + inline int operator()(int a, int b){ + return a-b; + } + }; + + struct Mult{ + // Real + template + inline vec operator()(vec a, vec b){ + vec out; + + VECTOR_FOR(i, W::r, 1) + { + out.v[i] = a.v[i]*b.v[i]; + } + + return out; + } + + // Integer + inline int operator()(int a, int b){ + return a*b; + } + }; + + #define cmul(a, b, c, i)\ + c[i] = a[i]*b[i] - a[i+1]*b[i+1];\ + c[i+1] = a[i]*b[i+1] + a[i+1]*b[i]; + + struct MultComplex{ + // Complex + template + inline vec operator()(vec a, vec b){ + vec out; + + VECTOR_FOR(i, W::c, 1) + { + cmul(a.v, b.v, out.v, 2*i); + } + + return out; + } + }; + + #undef cmul + + struct Div{ + // Real + template + inline vec operator()(vec a, vec b){ + vec out; + + VECTOR_FOR(i, W::r, 1) + { + out.v[i] = a.v[i]/b.v[i]; + } + + return out; + } + }; + + #define conj(a, b, i)\ + b[i] = a[i];\ + b[i+1] = -a[i+1]; + + struct Conj{ + // Complex + template + inline vec operator()(vec a){ + vec out; + + VECTOR_FOR(i, W::c, 1) + { + conj(a.v, out.v, 2*i); + } + + return out; + } + }; + + #undef conj + + #define timesmi(a, b, i)\ + b[i] = a[i+1];\ + b[i+1] = -a[i]; + + struct TimesMinusI{ + // Complex + template + inline vec operator()(vec a, vec b){ + vec out; + + VECTOR_FOR(i, W::c, 1) + { + timesmi(a.v, out.v, 2*i); + } + + return out; + } + }; + + #undef timesmi + + #define timesi(a, b, i)\ + b[i] = -a[i+1];\ + b[i+1] = a[i]; + + struct TimesI{ + // Complex + template + inline vec operator()(vec a, vec b){ + vec out; + + VECTOR_FOR(i, W::c, 1) + { + timesi(a.v, out.v, 2*i); + } + + return out; + } + }; + + #undef timesi + + ////////////////////////////////////////////// + // Some Template specialization + #define perm(a, b, n, w)\ + unsigned int _mask = w >> (n + 1);\ + VECTOR_FOR(i, w, 1)\ + {\ + b[i] = a[i^_mask];\ + } + + #define DECL_PERMUTE_N(n)\ + template \ + static inline vec Permute##n(vec in) {\ + vec out;\ + perm(in.v, out.v, n, W::r);\ + return out;\ + } + + struct Permute{ + DECL_PERMUTE_N(0); + DECL_PERMUTE_N(1); + DECL_PERMUTE_N(2); + DECL_PERMUTE_N(3); + }; + + #undef perm + #undef DECL_PERMUTE_N + + #define rot(a, b, n, w)\ + VECTOR_FOR(i, w, 1)\ + {\ + b[i] = a[(i + n)%w];\ + } + + struct Rotate{ + template + static inline vec rotate(vec in, int n){ + vec out; + + rot(in.v, out.v, n, W::r); + + return out; + } + }; + + #undef rot + + #define acc(v, a, off, step, n)\ + for (unsigned int i = off; i < n; i += step)\ + {\ + a += v[i];\ + } + template struct Reduce{ //Need templated class to overload output type @@ -164,316 +382,67 @@ namespace Optimization { return 0; } }; - - ///////////////////////////////////////////////////// - // Arithmetic operations - ///////////////////////////////////////////////////// - struct Sum{ - //Complex/Real float - inline u128f operator()(u128f a, u128f b){ - u128f out; - out.f[0] = a.f[0] + b.f[0]; - out.f[1] = a.f[1] + b.f[1]; - out.f[2] = a.f[2] + b.f[2]; - out.f[3] = a.f[3] + b.f[3]; - return out; - } - //Complex/Real double - inline u128d operator()(u128d a, u128d b){ - u128d out; - out.f[0] = a.f[0] + b.f[0]; - out.f[1] = a.f[1] + b.f[1]; - return out; - } - //Integer - inline int operator()(int a, int b){ - return a + b; - } - }; - - struct Sub{ - //Complex/Real float - inline u128f operator()(u128f a, u128f b){ - u128f out; - out.f[0] = a.f[0] - b.f[0]; - out.f[1] = a.f[1] - b.f[1]; - out.f[2] = a.f[2] - b.f[2]; - out.f[3] = a.f[3] - b.f[3]; - return out; - } - //Complex/Real double - inline u128d operator()(u128d a, u128d b){ - u128d out; - out.f[0] = a.f[0] - b.f[0]; - out.f[1] = a.f[1] - b.f[1]; - return out; - } - //Integer - inline int operator()(int a, int b){ - return a-b; - } - }; - - struct MultComplex{ - // Complex float - inline u128f operator()(u128f a, u128f b){ - u128f out; - out.f[0] = a.f[0]*b.f[0] - a.f[1]*b.f[1]; - out.f[1] = a.f[0]*b.f[1] + a.f[1]*b.f[0]; - out.f[2] = a.f[2]*b.f[2] - a.f[3]*b.f[3]; - out.f[3] = a.f[2]*b.f[3] + a.f[3]*b.f[2]; - return out; - } - // Complex double - inline u128d operator()(u128d a, u128d b){ - u128d out; - out.f[0] = a.f[0]*b.f[0] - a.f[1]*b.f[1]; - out.f[1] = a.f[0]*b.f[1] + a.f[1]*b.f[0]; - return out; - } - }; - - struct Mult{ - //CK: Appear unneeded - // inline float mac(float a, float b,double c){ - // return 0; - // } - // inline double mac(double a, double b,double c){ - // return 0; - // } - - // Real float - inline u128f operator()(u128f a, u128f b){ - u128f out; - out.f[0] = a.f[0]*b.f[0]; - out.f[1] = a.f[1]*b.f[1]; - out.f[2] = a.f[2]*b.f[2]; - out.f[3] = a.f[3]*b.f[3]; - return out; - } - // Real double - inline u128d operator()(u128d a, u128d b){ - u128d out; - out.f[0] = a.f[0]*b.f[0]; - out.f[1] = a.f[1]*b.f[1]; - return out; - } - // Integer - inline int operator()(int a, int b){ - return a*b; - } - }; - - struct Conj{ - // Complex single - inline u128f operator()(u128f in){ - u128f out; - out.f[0] = in.f[0]; - out.f[1] = -in.f[1]; - out.f[2] = in.f[2]; - out.f[3] = -in.f[3]; - return out; - } - // Complex double - inline u128d operator()(u128d in){ - u128d out; - out.f[0] = in.f[0]; - out.f[1] = -in.f[1]; - return out; - } - // do not define for integer input - }; - - struct TimesMinusI{ - //Complex single - inline u128f operator()(u128f in, u128f ret){ //note ret is ignored - u128f out; - out.f[0] = in.f[1]; - out.f[1] = -in.f[0]; - out.f[2] = in.f[3]; - out.f[3] = -in.f[2]; - return out; - } - //Complex double - inline u128d operator()(u128d in, u128d ret){ - u128d out; - out.f[0] = in.f[1]; - out.f[1] = -in.f[0]; - return out; - } - }; - - struct TimesI{ - //Complex single - inline u128f operator()(u128f in, u128f ret){ //note ret is ignored - u128f out; - out.f[0] = -in.f[1]; - out.f[1] = in.f[0]; - out.f[2] = -in.f[3]; - out.f[3] = in.f[2]; - return out; - } - //Complex double - inline u128d operator()(u128d in, u128d ret){ - u128d out; - out.f[0] = -in.f[1]; - out.f[1] = in.f[0]; - return out; - } - }; - - ////////////////////////////////////////////// - // Some Template specialization - struct Permute{ - //We just have to mirror the permutes of Grid_sse4.h - static inline u128f Permute0(u128f in){ //AB CD -> CD AB - u128f out; - out.f[0] = in.f[2]; - out.f[1] = in.f[3]; - out.f[2] = in.f[0]; - out.f[3] = in.f[1]; - return out; - }; - static inline u128f Permute1(u128f in){ //AB CD -> BA DC - u128f out; - out.f[0] = in.f[1]; - out.f[1] = in.f[0]; - out.f[2] = in.f[3]; - out.f[3] = in.f[2]; - return out; - }; - static inline u128f Permute2(u128f in){ - return in; - }; - static inline u128f Permute3(u128f in){ - return in; - }; - - static inline u128d Permute0(u128d in){ //AB -> BA - u128d out; - out.f[0] = in.f[1]; - out.f[1] = in.f[0]; - return out; - }; - static inline u128d Permute1(u128d in){ - return in; - }; - static inline u128d Permute2(u128d in){ - return in; - }; - static inline u128d Permute3(u128d in){ - return in; - }; - - }; - template < typename vtype > - void permute(vtype &a, vtype b, int perm) { - }; - - struct Rotate{ - - static inline u128f rotate(u128f in,int n){ - u128f out; - switch(n){ - case 0: - out.f[0] = in.f[0]; - out.f[1] = in.f[1]; - out.f[2] = in.f[2]; - out.f[3] = in.f[3]; - break; - case 1: - out.f[0] = in.f[1]; - out.f[1] = in.f[2]; - out.f[2] = in.f[3]; - out.f[3] = in.f[0]; - break; - case 2: - out.f[0] = in.f[2]; - out.f[1] = in.f[3]; - out.f[2] = in.f[0]; - out.f[3] = in.f[1]; - break; - case 3: - out.f[0] = in.f[3]; - out.f[1] = in.f[0]; - out.f[2] = in.f[1]; - out.f[3] = in.f[2]; - break; - default: assert(0); - } - return out; - } - static inline u128d rotate(u128d in,int n){ - u128d out; - switch(n){ - case 0: - out.f[0] = in.f[0]; - out.f[1] = in.f[1]; - break; - case 1: - out.f[0] = in.f[1]; - out.f[1] = in.f[0]; - break; - default: assert(0); - } - return out; - } - }; - //Complex float Reduce - template<> - inline Grid::ComplexF Reduce::operator()(u128f in){ //2 complex - return Grid::ComplexF(in.f[0] + in.f[2], in.f[1] + in.f[3]); + template <> + inline Grid::ComplexF Reduce::operator()(vecf in){ + float a = 0.f, b = 0.f; + + acc(in.v, a, 0, 2, W::r); + acc(in.v, b, 1, 2, W::r); + + return Grid::ComplexF(a, b); } + //Real float Reduce template<> - inline Grid::RealF Reduce::operator()(u128f in){ //4 floats - return in.f[0] + in.f[1] + in.f[2] + in.f[3]; + inline Grid::RealF Reduce::operator()(vecf in){ + float a = 0.; + + acc(in.v, a, 0, 1, W::r); + + return a; } - //Complex double Reduce template<> - inline Grid::ComplexD Reduce::operator()(u128d in){ //1 complex - return Grid::ComplexD(in.f[0],in.f[1]); + inline Grid::ComplexD Reduce::operator()(vecd in){ + double a = 0., b = 0.; + + acc(in.v, a, 0, 2, W::r); + acc(in.v, b, 1, 2, W::r); + + return Grid::ComplexD(a, b); } //Real double Reduce template<> - inline Grid::RealD Reduce::operator()(u128d in){ //2 doubles - return in.f[0] + in.f[1]; + inline Grid::RealD Reduce::operator()(vecd in){ + double a = 0.f; + + acc(in.v, a, 0, 1, W::r); + + return a; } //Integer Reduce template<> inline Integer Reduce::operator()(int in){ - // FIXME unimplemented - printf("Reduce : Missing integer implementation -> FIX\n"); - assert(0); + return in; } } ////////////////////////////////////////////////////////////////////////////////////// // Here assign types - typedef Optimization::u128f SIMD_Ftype; // Single precision type - typedef Optimization::u128d SIMD_Dtype; // Double precision type + typedef Optimization::vecf SIMD_Ftype; // Single precision type + typedef Optimization::vecd SIMD_Dtype; // Double precision type typedef int SIMD_Itype; // Integer type // prefetch utilities inline void v_prefetch0(int size, const char *ptr){}; inline void prefetch_HINT_T0(const char *ptr){}; - - - // Gpermute function - template < typename VectorSIMD > - inline void Gpermute(VectorSIMD &y,const VectorSIMD &b, int perm ) { - Optimization::permute(y.v,b.v,perm); - } - - // Function name aliases typedef Optimization::Vsplat VsplatSIMD; typedef Optimization::Vstore VstoreSIMD; @@ -481,16 +450,13 @@ namespace Optimization { typedef Optimization::Vstream VstreamSIMD; template using ReduceSIMD = Optimization::Reduce; - - - // Arithmetic operations typedef Optimization::Sum SumSIMD; typedef Optimization::Sub SubSIMD; + typedef Optimization::Div DivSIMD; typedef Optimization::Mult MultSIMD; typedef Optimization::MultComplex MultComplexSIMD; typedef Optimization::Conj ConjSIMD; typedef Optimization::TimesMinusI TimesMinusISIMD; typedef Optimization::TimesI TimesISIMD; - } diff --git a/lib/simd/Grid_imci.h b/lib/simd/Grid_imci.h index b80d1752..173e57d8 100644 --- a/lib/simd/Grid_imci.h +++ b/lib/simd/Grid_imci.h @@ -236,6 +236,17 @@ namespace Optimization { } }; + struct Div{ + // Real float + inline __m512 operator()(__m512 a, __m512 b){ + return _mm512_div_ps(a,b); + } + // Real double + inline __m512d operator()(__m512d a, __m512d b){ + return _mm512_div_pd(a,b); + } + }; + struct Conj{ // Complex single @@ -429,6 +440,7 @@ namespace Optimization { // Arithmetic operations typedef Optimization::Sum SumSIMD; typedef Optimization::Sub SubSIMD; + typedef Optimization::Div DivSIMD; typedef Optimization::Mult MultSIMD; typedef Optimization::MultComplex MultComplexSIMD; typedef Optimization::Conj ConjSIMD; diff --git a/lib/simd/Grid_sse4.h b/lib/simd/Grid_sse4.h index a4002373..560eda11 100644 --- a/lib/simd/Grid_sse4.h +++ b/lib/simd/Grid_sse4.h @@ -224,6 +224,18 @@ namespace Optimization { } }; + struct Div{ + // Real float + inline __m128 operator()(__m128 a, __m128 b){ + return _mm_div_ps(a,b); + } + // Real double + inline __m128d operator()(__m128d a, __m128d b){ + return _mm_div_pd(a,b); + } + }; + + struct Conj{ // Complex single inline __m128 operator()(__m128 in){ @@ -372,6 +384,8 @@ namespace Optimization { } } + + ////////////////////////////////////////////////////////////////////////////////////// // Here assign types @@ -398,6 +412,7 @@ namespace Optimization { // Arithmetic operations typedef Optimization::Sum SumSIMD; typedef Optimization::Sub SubSIMD; + typedef Optimization::Div DivSIMD; typedef Optimization::Mult MultSIMD; typedef Optimization::MultComplex MultComplexSIMD; typedef Optimization::Conj ConjSIMD; diff --git a/lib/simd/Grid_vector_types.h b/lib/simd/Grid_vector_types.h index 30576fb6..184baad9 100644 --- a/lib/simd/Grid_vector_types.h +++ b/lib/simd/Grid_vector_types.h @@ -38,7 +38,7 @@ directory #ifndef GRID_VECTOR_TYPES #define GRID_VECTOR_TYPES -#ifdef GENERIC_VEC +#ifdef GEN #include "Grid_generic.h" #endif #ifdef SSE4 @@ -77,38 +77,24 @@ struct RealPart > { ////////////////////////////////////// // demote a vector to real type ////////////////////////////////////// - // type alias used to simplify the syntax of std::enable_if -template -using Invoke = typename T::type; -template -using EnableIf = Invoke >; -template -using NotEnableIf = Invoke >; +template using Invoke = typename T::type; +template using EnableIf = Invoke >; +template using NotEnableIf = Invoke >; //////////////////////////////////////////////////////// // Check for complexity with type traits -template -struct is_complex : public std::false_type {}; -template <> -struct is_complex > : public std::true_type {}; -template <> -struct is_complex > : public std::true_type {}; +template struct is_complex : public std::false_type {}; +template <> struct is_complex > : public std::true_type {}; +template <> struct is_complex > : public std::true_type {}; -template -using IfReal = Invoke::value, int> >; -template -using IfComplex = Invoke::value, int> >; -template -using IfInteger = Invoke::value, int> >; +template using IfReal = Invoke::value, int> >; +template using IfComplex = Invoke::value, int> >; +template using IfInteger = Invoke::value, int> >; -template -using IfNotReal = - Invoke::value, int> >; -template -using IfNotComplex = Invoke::value, int> >; -template -using IfNotInteger = Invoke::value, int> >; +template using IfNotReal = Invoke::value, int> >; +template using IfNotComplex = Invoke::value, int> >; +template using IfNotInteger = Invoke::value, int> >; //////////////////////////////////////////////////////// // Define the operation templates functors @@ -285,6 +271,20 @@ class Grid_simd { return a * b; } + ////////////////////////////////// + // Divides + ////////////////////////////////// + friend inline Grid_simd operator/(const Scalar_type &a, Grid_simd b) { + Grid_simd va; + vsplat(va, a); + return va / b; + } + friend inline Grid_simd operator/(Grid_simd b, const Scalar_type &a) { + Grid_simd va; + vsplat(va, a); + return b / a; + } + /////////////////////// // Unary negation /////////////////////// @@ -428,7 +428,6 @@ inline void rotate(Grid_simd &ret,Grid_simd b,int nrot) ret.v = Optimization::Rotate::rotate(b.v,2*nrot); } - template inline void vbroadcast(Grid_simd &ret,const Grid_simd &src,int lane){ S* typepun =(S*) &src; @@ -512,7 +511,6 @@ template = 0> inline void vfalse(Grid_simd &ret) { vsplat(ret, 0); } - template inline void zeroit(Grid_simd &z) { vzero(z); @@ -530,7 +528,6 @@ inline void vstream(Grid_simd &out, const Grid_simd &in) { typedef typename S::value_type T; binary((T *)&out.v, in.v, VstreamSIMD()); } - template = 0> inline void vstream(Grid_simd &out, const Grid_simd &in) { out = in; @@ -569,6 +566,34 @@ inline Grid_simd operator*(Grid_simd a, Grid_simd b) { return ret; }; +// Distinguish between complex types and others +template = 0> +inline Grid_simd operator/(Grid_simd a, Grid_simd b) { + typedef Grid_simd simd; + + simd ret; + simd den; + typename simd::conv_t conv; + + ret = a * conjugate(b) ; + den = b * conjugate(b) ; + + + auto real_den = toReal(den); + + ret.v=binary(ret.v, real_den.v, DivSIMD()); + + return ret; +}; + +// Real/Integer types +template = 0> +inline Grid_simd operator/(Grid_simd a, Grid_simd b) { + Grid_simd ret; + ret.v = binary(a.v, b.v, DivSIMD()); + return ret; +}; + /////////////////////// // Conjugate /////////////////////// @@ -582,7 +607,6 @@ template = 0> inline Grid_simd conjugate(const Grid_simd &in) { return in; // for real objects } - // Suppress adj for integer types... // odd; why conjugate above but not adj?? template = 0> inline Grid_simd adj(const Grid_simd &in) { @@ -596,14 +620,12 @@ template = 0> inline void timesMinusI(Grid_simd &ret, const Grid_simd &in) { ret.v = binary(in.v, ret.v, TimesMinusISIMD()); } - template = 0> inline Grid_simd timesMinusI(const Grid_simd &in) { Grid_simd ret; timesMinusI(ret, in); return ret; } - template = 0> inline Grid_simd timesMinusI(const Grid_simd &in) { return in; @@ -616,14 +638,12 @@ template = 0> inline void timesI(Grid_simd &ret, const Grid_simd &in) { ret.v = binary(in.v, ret.v, TimesISIMD()); } - template = 0> inline Grid_simd timesI(const Grid_simd &in) { Grid_simd ret; timesI(ret, in); return ret; } - template = 0> inline Grid_simd timesI(const Grid_simd &in) { return in; diff --git a/lib/simd/Intel512avx.h b/lib/simd/Intel512avx.h index cdd54a33..19157db4 100644 --- a/lib/simd/Intel512avx.h +++ b/lib/simd/Intel512avx.h @@ -53,7 +53,7 @@ Author: paboyle #define ZMULMEM2SPd(O,P,tmp,B,C,Briir,Biirr,Criir,Ciirr)\ VSHUFMEMd(O,P,tmp) \ - VMULMEMd(O,P,B,Biirr) \ + VMULMEMd(O,P,B,Biirr) \ VMULMEMd(O,P,C,Ciirr) \ VMULd(tmp,B,Briir) \ VMULd(tmp,C,Criir) diff --git a/lib/simd/Intel512common.h b/lib/simd/Intel512common.h index dabbf6d8..cfa20c26 100644 --- a/lib/simd/Intel512common.h +++ b/lib/simd/Intel512common.h @@ -37,7 +37,7 @@ Author: paboyle // Opcodes common //////////////////////////////////////////////////////////////////////////////////////////////////// #define MASK_REGS \ - __asm__ ("mov $0xAAAA, %%eax \n"\ + __asm__ ("mov $0xAAAA, %%eax \n"\ "kmovw %%eax, %%k6 \n"\ "mov $0x5555, %%eax \n"\ "kmovw %%eax, %%k7 \n" : : : "%eax"); @@ -138,9 +138,14 @@ Author: paboyle #define ZLOADf(OFF,PTR,ri,ir) VLOADf(OFF,PTR,ir) VSHUFf(ir,ri) #define ZLOADd(OFF,PTR,ri,ir) VLOADd(OFF,PTR,ir) VSHUFd(ir,ri) - +#define STREAM_STORE +#ifdef STREAM_STORE +#define VSTOREf(OFF,PTR,SRC) "vmovntps " #SRC "," #OFF "*64(" #PTR ")" ";\n" +#define VSTOREd(OFF,PTR,SRC) "vmovntpd " #SRC "," #OFF "*64(" #PTR ")" ";\n" +#else #define VSTOREf(OFF,PTR,SRC) "vmovaps " #SRC "," #OFF "*64(" #PTR ")" ";\n" #define VSTOREd(OFF,PTR,SRC) "vmovapd " #SRC "," #OFF "*64(" #PTR ")" ";\n" +#endif // Swaps Re/Im ; could unify this with IMCI #define VSHUFd(A,DEST) "vpshufd $0x4e," #A "," #DEST ";\n" diff --git a/lib/stencil/Lebesgue.cc b/lib/stencil/Lebesgue.cc index c34b5c96..c30c14c7 100644 --- a/lib/stencil/Lebesgue.cc +++ b/lib/stencil/Lebesgue.cc @@ -32,7 +32,7 @@ Author: paboyle namespace Grid { int LebesgueOrder::UseLebesgueOrder; -std::vector LebesgueOrder::Block({2,2,2,2}); +std::vector LebesgueOrder::Block({8,2,2,2}); LebesgueOrder::IndexInteger LebesgueOrder::alignup(IndexInteger n){ n--; // 1000 0011 --> 1000 0010 diff --git a/lib/tensors/Tensor_arith_mul.h b/lib/tensors/Tensor_arith_mul.h index 917f7373..c24853b7 100644 --- a/lib/tensors/Tensor_arith_mul.h +++ b/lib/tensors/Tensor_arith_mul.h @@ -126,6 +126,36 @@ iVector operator * (const iVector& lhs,const iScalar& r mult(&ret,&lhs,&rhs); return ret; } + +////////////////////////////////////////////////////////////////// +// Divide by scalar +////////////////////////////////////////////////////////////////// +template strong_inline +iScalar operator / (const iScalar& lhs,const iScalar& rhs) +{ + iScalar ret; + ret._internal = lhs._internal/rhs._internal; + return ret; +} +template strong_inline +iVector operator / (const iVector& lhs,const iScalar& rhs) +{ + iVector ret; + for(int i=0;i strong_inline +iMatrix operator / (const iMatrix& lhs,const iScalar& rhs) +{ + iMatrix ret; + for(int i=0;i &extracted,int } } -template inline -void merge1(vobj &vec,std::vector &extracted,int offset) +template inline void merge1(vobj &vec,std::vector &extracted,int offset) { typedef typename vobj::scalar_type scalar_type ; typedef typename vobj::vector_type vector_type ; @@ -269,8 +268,7 @@ void merge1(vobj &vec,std::vector &extracted,int }} } -template inline -void merge2(vobj &vec,std::vector &extracted,int offset) +template inline void merge2(vobj &vec,std::vector &extracted,int offset) { typedef typename vobj::scalar_type scalar_type ; typedef typename vobj::vector_type vector_type ; diff --git a/scripts/arm_configure.experimental b/scripts/arm_configure.experimental deleted file mode 100644 index fbad84a5..00000000 --- a/scripts/arm_configure.experimental +++ /dev/null @@ -1 +0,0 @@ -./configure --host=arm-linux-gnueabihf CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target arm-linux-gnueabihf -I/usr/arm-linux-gnueabihf/include/ -I/home/neo/Codes/gmp6.0/gmp-arm/include/ -I/usr/arm-linux-gnueabihf/include/c++/4.8.2/arm-linux-gnueabihf/ -L/home/neo/Codes/gmp6.0/gmp-arm/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-arm/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-arm/lib/ -static -mcpu=cortex-a7' --enable-simd=NEONv7 diff --git a/scripts/arm_configure.experimental_cortex57 b/scripts/arm_configure.experimental_cortex57 deleted file mode 100644 index d229763e..00000000 --- a/scripts/arm_configure.experimental_cortex57 +++ /dev/null @@ -1,3 +0,0 @@ -#./configure --host=arm-linux-gnueabihf CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target arm-linux-gnueabihf -I/usr/arm-linux-gnueabihf/include/ -I/home/neo/Codes/gmp6.0/gmp-arm/include/ -I/usr/lib/llvm-3.5/lib/clang/3.5.0/include/ -L/home/neo/Codes/gmp6.0/gmp-arm/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-arm/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-arm/lib/ -static -mcpu=cortex-a57' --enable-simd=NEONv7 - -./configure --host=aarch64-linux-gnu CXX=clang++-3.5 CXXFLAGS='-std=c++11 -O3 -target aarch64-linux-gnu -static -I/home/neo/Codes/gmp6.0/gmp-armv8/include/ -L/home/neo/Codes/gmp6.0/gmp-armv8/lib/ -I/home/neo/Codes/mpfr3.1.2/mpfr-armv8/include/ -L/home/neo/Codes/mpfr3.1.2/mpfr-armv8/lib/ -I/usr/aarch64-linux-gnu/include/ -I/usr/aarch64-linux-gnu/include/c++/4.8.2/aarch64-linux-gnu/' --enable-simd=NEONv7 diff --git a/scripts/bench_wilson.sh b/scripts/bench_wilson.sh deleted file mode 100755 index af73d591..00000000 --- a/scripts/bench_wilson.sh +++ /dev/null @@ -1,9 +0,0 @@ -for omp in 1 2 4 -do -echo > wilson.t$omp -for vol in 4.4.4.4 4.4.4.8 4.4.8.8 4.8.8.8 8.8.8.8 8.8.8.16 8.8.16.16 8.16.16.16 -do -perf=` ./benchmarks/Grid_wilson --grid $vol --omp $omp | grep mflop | awk '{print $3}'` -echo $vol $perf >> wilson.t$omp -done -done \ No newline at end of file diff --git a/scripts/build-all b/scripts/build-all deleted file mode 100755 index b97dca19..00000000 --- a/scripts/build-all +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -e - -DIRS="clang-avx clang-avx-openmp clang-avx-openmp-mpi clang-avx-mpi clang-avx2 clang-avx2-openmp clang-avx2-openmp-mpi clang-avx2-mpi clang-sse" -EXTRADIRS="g++-avx g++-sse4 icpc-avx icpc-avx2 icpc-avx512" -BLACK="\033[30m" -RED="\033[31m" -GREEN="\033[32m" -YELLOW="\033[33m" -BLUE="\033[34m" -PINK="\033[35m" -CYAN="\033[36m" -WHITE="\033[37m" -NORMAL="\033[0;39m" - -for D in $DIRS -do - -echo -echo -e $RED ============================== -echo -e $GREEN $D -echo -e $RED ============================== -echo -e $BLUE - - cd builds/$D - make clean all -j 8 - cd ../../ -echo -e $NORMAL -done - -if [ "X$1" == "Xextra" ] -then -for D in $EXTRADIRS -do - -echo -echo -e $RED ============================== -echo -e $RED $D -echo -e $RED ============================== -echo -e $BLUE - - cd builds/$D - make clean all -j 8 - cd ../../ -echo -e $NORMAL -done -fi \ No newline at end of file diff --git a/scripts/configure-all b/scripts/configure-all deleted file mode 100755 index ad91034d..00000000 --- a/scripts/configure-all +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DIRS="clang-avx clang-avx-openmp clang-avx-openmp-mpi clang-avx-mpi clang-avx2 clang-avx2-openmp clang-avx2-openmp-mpi clang-avx2-mpi icpc-avx icpc-avx2 icpc-avx512 g++-sse4 g++-avx clang-sse icpc-avx-openmp-mpi icpc-avx-openmp" - -for D in $DIRS -do - mkdir -p builds/$D - cd builds/$D - ../../scripts/configure-commands $D - cd ../.. -done diff --git a/scripts/configure-commands b/scripts/configure-commands deleted file mode 100755 index a3599d1f..00000000 --- a/scripts/configure-commands +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -WD=$1 -BLACK="\033[30m" -RED="\033[31m" -GREEN="\033[32m" -YELLOW="\033[33m" -BLUE="\033[34m" -PINK="\033[35m" -CYAN="\033[36m" -WHITE="\033[37m" -NORMAL="\033[0;39m" -echo -echo -e $RED ============================== -echo -e $GREEN $WD -echo -e $RED ============================== -echo -e $YELLOW - -case $WD in -g++-avx) - CXX=g++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -g++-avx-openmp) - CXX=g++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11" LIBS="-fopenmp -lgmp -lmpfr" --enable-comms=none - ;; -g++5-sse4) - CXX=g++-5 ../../configure --enable-simd=SSE4 CXXFLAGS="-msse4 -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -g++5-avx) - CXX=g++-5 ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -icpc-avx) - CXX=icpc ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -icpc-avx-openmp-mpi) -CXX=icpc ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -fopenmp -lgmp -lmpfr" --enable-comms=mpi -;; -icpc-avx-openmp) -CXX=icpc ../../configure --enable-precision=single --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11" LIBS="-fopenmp -lgmp -lmpfr" --enable-comms=mpi -;; -icpc-avx2) - CXX=icpc ../../configure --enable-simd=AVX2 CXXFLAGS="-march=core-avx2 -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -icpc-avx512) - CXX=icpc ../../configure --enable-simd=AVX512 CXXFLAGS="-xCOMMON-AVX512 -O3 -std=c++11" --host=none LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -icpc-mic) - CXX=icpc ../../configure --host=none --enable-simd=IMCI CXXFLAGS="-mmic -O3 -std=c++11" LDFLAGS=-mmic LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -icpc-mic-avx512) - CXX=icpc ../../configure --host=none --enable-simd=IMCI CXXFLAGS="-xCOMMON_AVX512 -O3 -std=c++11" LDFLAGS=-xCOMMON_AVX512 LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-sse) -CXX=clang++ ../../configure --enable-precision=single --enable-simd=SSE4 CXXFLAGS="-msse4 -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-avx) -CXX=clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-avx2) -CXX=clang++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -O3 -std=c++11" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-avx-openmp) -CXX=clang-omp++ ../../configure --enable-precision=double --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11" LDFLAGS="-fopenmp" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-xc30) -CXX=$HOME/Clang/install/bin/clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -std=c++11 -I/opt/gcc/4.9.2/snos/include/g++/x86_64-suse-linux/ -I/opt/gcc/4.9.2/snos/include/g++/ " LDFLAGS="" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-xc30-openmp) -CXX=$HOME/Clang/install/bin/clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -std=c++11 -I/opt/gcc/4.9.2/snos/include/g++/x86_64-suse-linux/ -I/opt/gcc/4.9.2/snos/include/g++/ " LDFLAGS="-fopenmp" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-avx2-openmp) -CXX=clang-omp++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -fopenmp -O3 -std=c++11" LDFLAGS="-fopenmp" LIBS="-lgmp -lmpfr" --enable-comms=none - ;; -clang-avx-openmp-mpi) -CXX=clang-omp++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -fopenmp -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -fopenmp -lgmp -lmpfr" --enable-comms=mpi -;; -clang-avx2-openmp-mpi) -CXX=clang-omp++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -fopenmp -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -fopenmp -lgmp -lmpfr" --enable-comms=mpi -;; -clang-avx-mpi) -CXX=clang++ ../../configure --enable-simd=AVX CXXFLAGS="-mavx -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -lgmp -lmpfr" --enable-comms=mpi -;; -clang-avx2-mpi) -CXX=clang++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -O3 -I/opt/local/include/openmpi-mp/ -std=c++11" LDFLAGS=-L/opt/local/lib/openmpi-mp/ LIBS="-lmpi -lmpi_cxx -lgmp -lmpfr" --enable-comms=mpi -;; -clang-avx2) -CXX=clang++ ../../configure --enable-simd=AVX2 CXXFLAGS="-mavx2 -mfma -O3 -std=c++11" LDFLAGS="-L/usr/local/lib/" LIBS="-lgmp -lmpfr" --enable-comms=none -;; -esac -echo -e $NORMAL diff --git a/scripts/configure-cray b/scripts/configure-cray deleted file mode 100755 index db581493..00000000 --- a/scripts/configure-cray +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -DIRS="g++-avx-openmp g++-avx clang-xc30 clang-xc30-openmp" - -for D in $DIRS -do - mkdir -p builds/$D - cd builds/$D - ../../scripts/configure-commands $D - cd ../.. -done diff --git a/scripts/configure-mic b/scripts/configure-mic deleted file mode 100755 index 668845fe..00000000 --- a/scripts/configure-mic +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -DIRS="build-icpc-mic" - -for D in $DIRS -do - mkdir -p $D - cd $D - ../configure-commands - cd .. -done diff --git a/scripts/copyright b/scripts/copyright index 92772f16..cc9ed6e5 100755 --- a/scripts/copyright +++ b/scripts/copyright @@ -12,6 +12,7 @@ Grid physics library, www.github.com/paboyle/Grid Source file: $1 Copyright (C) 2015 +Copyright (C) 2016 EOF @@ -38,8 +39,21 @@ See the full license in the file "LICENSE" in the top level distribution directo /* END LEGAL */ EOF + cat message > tmp.fil -cat $1 >> tmp.fil + +NOTICE=`grep -n "END LEGAL" $1 | awk '{ print $1 }' ` + +if [ "X$NOTICE" != "X" ] +then + echo "found notice ending on line $NOTICE" + awk 'BEGIN { P=0 } { if ( P ) print } /END LEGAL/{P=1} ' $1 >> tmp.fil +else + cat $1 >> tmp.fil + +fi + + cp tmp.fil $1 shift diff --git a/scripts/cray-modules b/scripts/cray-modules deleted file mode 100644 index 95de2b0b..00000000 --- a/scripts/cray-modules +++ /dev/null @@ -1,2 +0,0 @@ -module swap PrgEnv-cray PrgEnv-intel -module swap intel/14.0.4.211 intel/15.0.2.164 diff --git a/scripts/filelist b/scripts/filelist index 0802c2a6..dddb605b 100755 --- a/scripts/filelist +++ b/scripts/filelist @@ -20,15 +20,20 @@ for subdir in $dirs; do TESTS=`ls T*.cc` TESTLIST=`echo ${TESTS} | sed s/.cc//g ` PREF=`[ $subdir = '.' ] && echo noinst || echo EXTRA` - echo "tests: ${TESTLIST}" > Make.inc + SUB=`[ $subdir = '.' ] && echo subtests` + echo "tests: ${TESTLIST} ${SUB}" > Make.inc echo ${PREF}_PROGRAMS = ${TESTLIST} >> Make.inc echo >> Make.inc for f in $TESTS; do - BNAME=`basename $f .cc` - echo ${BNAME}_SOURCES=$f >> Make.inc - echo ${BNAME}_LDADD=-lGrid>> Make.inc - echo >> Make.inc + BNAME=`basename $f .cc` + echo ${BNAME}_SOURCES=$f >> Make.inc + echo ${BNAME}_LDADD=-lGrid>> Make.inc + echo >> Make.inc done + if [ $subdir != '.' ]; then + echo CLEANFILES = ${TESTLIST} >> Make.inc + echo >> Make.inc + fi done # benchmarks Make.inc diff --git a/scripts/reconfigure_script b/scripts/reconfigure_script deleted file mode 100755 index d8d7212d..00000000 --- a/scripts/reconfigure_script +++ /dev/null @@ -1,4 +0,0 @@ -aclocal -I m4 -autoheader -f -automake -f --add-missing -autoconf -f diff --git a/scripts/update_fftw.sh b/scripts/update_fftw.sh deleted file mode 100755 index 20e42080..00000000 --- a/scripts/update_fftw.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -if (( $# != 1 )); then - echo "usage: `basename $0` " 1>&2 - exit 1 -fi -ARC=$1 - -INITDIR=`pwd` -rm -rf lib/fftw -mkdir lib/fftw - -ARCDIR=`tar -tf ${ARC} | head -n1 | sed -e 's@/.*@@'` -tar -xf ${ARC} -cp ${ARCDIR}/api/fftw3.h lib/fftw/ - -cd ${INITDIR} -rm -rf ${ARCDIR} diff --git a/scripts/wilson.gnu b/scripts/wilson.gnu deleted file mode 100644 index 69bca5b5..00000000 --- a/scripts/wilson.gnu +++ /dev/null @@ -1,7 +0,0 @@ -plot 'wilson.t1' u 2 w l t "AVX1-OMP=1" -replot 'wilson.t2' u 2 w l t "AVX1-OMP=2" -replot 'wilson.t4' u 2 w l t "AVX1-OMP=4" -set terminal 'pdf' -set output 'wilson_clang.pdf' -replot -quit diff --git a/tests/Makefile.am b/tests/Makefile.am index 2b8cc2d7..2e7c1f0a 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -4,4 +4,9 @@ if BUILD_CHROMA_REGRESSION SUBDIRS+= qdpxx endif +.PHONY: subtests + include Make.inc + +subtests: + for d in $(SUBDIRS); do $(MAKE) -C $${d} tests; done diff --git a/tests/Test_stencil.cc b/tests/Test_stencil.cc index 0e35a414..1b71b8a5 100644 --- a/tests/Test_stencil.cc +++ b/tests/Test_stencil.cc @@ -116,7 +116,7 @@ int main (int argc, char ** argv) else if (SE->_is_local) Check._odata[i] = Foo._odata[SE->_offset]; else - Check._odata[i] = myStencil.comm_buf[SE->_offset]; + Check._odata[i] = myStencil.CommBuf()[SE->_offset]; } Real nrmC = norm2(Check); @@ -207,7 +207,7 @@ int main (int argc, char ** argv) else if (SE->_is_local) OCheck._odata[i] = EFoo._odata[SE->_offset]; else - OCheck._odata[i] = EStencil.comm_buf[SE->_offset]; + OCheck._odata[i] = EStencil.CommBuf()[SE->_offset]; } for(int i=0;ioSites();i++){ int permute_type; @@ -220,7 +220,7 @@ int main (int argc, char ** argv) else if (SE->_is_local) ECheck._odata[i] = OFoo._odata[SE->_offset]; else - ECheck._odata[i] = OStencil.comm_buf[SE->_offset]; + ECheck._odata[i] = OStencil.CommBuf()[SE->_offset]; } setCheckerboard(Check,ECheck); diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc index 2fdaeed2..03c8bc52 100644 --- a/tests/core/Test_fft.cc +++ b/tests/core/Test_fft.cc @@ -1,6 +1,6 @@ /************************************************************************************* - Grid physics library, www.github.com/paboyle/Grid + grid` physics library, www.github.com/paboyle/Grid Source file: ./tests/Test_cshift.cc @@ -46,57 +46,79 @@ int main (int argc, char ** argv) for(int d=0;d p({1,2,3,2}); + std::vector p({1,3,2,3}); one = ComplexD(1.0,0.0); zz = ComplexD(0.0,0.0); ComplexD ci(0.0,1.0); + + std::cout<<"*************************************************"< seeds({1,2,3,4}); + GridSerialRNG sRNG; sRNG.SeedFixedIntegers(seeds); // naughty seeding + GridParallelRNG pRNG(&GRID); + pRNG.SeedFixedIntegers(seeds); + + LatticeGaugeFieldD Umu(&GRID); + + SU3::ColdConfiguration(pRNG,Umu); // Unit gauge + // Umu=zero; + //////////////////////////////////////////////////// + // Wilson test + //////////////////////////////////////////////////// + { + LatticeFermionD src(&GRID); gaussian(pRNG,src); + LatticeFermionD tmp(&GRID); + LatticeFermionD ref(&GRID); + + RealD mass=0.01; + WilsonFermionD Dw(Umu,GRID,RBGRID,mass); + + Dw.M(src,tmp); + + std::cout << "Dw src = " < 1/2 gmu (eip - emip) = i sinp gmu + Kinetic = Kinetic + sin(kmu)*ci*(Gamma(Gmu[mu])*src5_p); + + } + + // NB implicit sum over mu + // + // 1-1/2 Dw = 1 - 1/2 ( eip+emip) + // = - 1/2 (ei - 2 + emi) + // = - 1/4 2 (eih - eimh)(eih - eimh) + // = 2 sink/2 ink/2 = sk2 + + W = one - M5 + sk2; + Kinetic = Kinetic + W * src5_p; + + LatticeCoordinate(scoor,sdir); + + tmp5 = Cshift(src5_p,sdir,+1); + tmp5 = (tmp5 - G5*tmp5)*0.5; + tmp5 = where(scoor==Integer(Ls-1),mass*tmp5,-tmp5); + Kinetic = Kinetic + tmp5; + + tmp5 = Cshift(src5_p,sdir,-1); + tmp5 = (tmp5 + G5*tmp5)*0.5; + tmp5 = where(scoor==Integer(0),mass*tmp5,-tmp5); + Kinetic = Kinetic + tmp5; + + std::cout<<"Momentum space Ddwf "<< norm2(Kinetic)< point(4,0); + src=zero; + SpinColourVectorD ferm; gaussian(sRNG,ferm); + pokeSite(ferm,src,point); + + const int Ls=32; + GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,&GRID); + GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,&GRID); + + RealD mass=0.01; + RealD M5 =0.8; + DomainWallFermionD Ddwf(Umu,*FGrid,*FrbGrid,GRID,RBGRID,mass,M5); + + // Momentum space prop + std::cout << " Solving by FFT and Feynman rules" < HermOp(Ddwf); + ConjugateGradient CG(1.0e-16,10000); + CG(HermOp,src5,result5); + + //////////////////////////////////////////////////////////////////////// + // Domain wall physical field propagator + //////////////////////////////////////////////////////////////////////// + /* + psi = chiralProjectMinus(psi_5[0]); + psi += chiralProjectPlus(psi_5[Ls-1]); + */ + ExtractSlice(tmp,result5,0 ,sdir); result4 = (tmp-G5*tmp)*0.5; + ExtractSlice(tmp,result5,Ls-1,sdir); result4 = result4+(tmp+G5*tmp)*0.5; + + std::cout << " Taking difference" < point(4,0); + src=zero; + SpinColourVectorD ferm; gaussian(sRNG,ferm); + pokeSite(ferm,src,point); + + const int Ls=48; + GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,&GRID); + GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,&GRID); + + RealD mass=0.01; + RealD M5 =0.8; + + OverlapWilsonCayleyTanhFermionD Dov(Umu,*FGrid,*FrbGrid,GRID,RBGRID,mass,M5,1.0); + + // Momentum space prop + std::cout << " Solving by FFT and Feynman rules" < HermOp(Dov); + ConjugateGradient CG(1.0e-16,10000); + CG(HermOp,src5,result5); + + //////////////////////////////////////////////////////////////////////// + // Domain wall physical field propagator + //////////////////////////////////////////////////////////////////////// + /* + psi = chiralProjectMinus(psi_5[0]); + psi += chiralProjectPlus(psi_5[Ls-1]); + */ + ExtractSlice(tmp,result5,0 ,sdir); result4 = (tmp-G5*tmp)*0.5; + ExtractSlice(tmp,result5,Ls-1,sdir); result4 = result4+(tmp+G5*tmp)*0.5; + + std::cout << " Taking difference" < QEDGimplTypesD; + typedef Photon QEDGaction; + + QEDGaction Maxwell(QEDGaction::FEYNMAN_L); + QEDGaction::GaugeField Prop(&GRID);Prop=zero; + QEDGaction::GaugeField Source(&GRID);Source=zero; + + Maxwell.FreePropagator (Source,Prop); + std::cout << " MaxwellFree propagator\n"; + */ + } Grid_finalize(); } diff --git a/tests/core/Test_fft_gfix.cc b/tests/core/Test_fft_gfix.cc new file mode 100644 index 00000000..c6b77a13 --- /dev/null +++ b/tests/core/Test_fft_gfix.cc @@ -0,0 +1,300 @@ + /************************************************************************************* + + grid` physics library, www.github.com/paboyle/Grid + + Source file: ./tests/Test_cshift.cc + + Copyright (C) 2015 + +Author: Azusa Yamaguchi +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +template +class FourierAcceleratedGaugeFixer : public Gimpl { + public: + INHERIT_GIMPL_TYPES(Gimpl); + + typedef typename Gimpl::GaugeLinkField GaugeMat; + typedef typename Gimpl::GaugeField GaugeLorentz; + + static void GaugeLinkToLieAlgebraField(const std::vector &U,std::vector &A) { + for(int mu=0;mu &A,GaugeMat &dmuAmu) { + dmuAmu=zero; + for(int mu=0;mu::avgPlaquette(Umu); + Real org_link_trace=WilsonLoops::linkTrace(Umu); + Real old_trace = org_link_trace; + Real trG; + + std::vector U(Nd,grid); + GaugeMat dmuAmu(grid); + + for(int i=0;i(Umu,mu); + //trG = SteepestDescentStep(U,alpha,dmuAmu); + trG = FourierAccelSteepestDescentStep(U,alpha,dmuAmu); + for(int mu=0;mu(Umu,U[mu],mu); + // Monitor progress and convergence test + // infrequently to minimise cost overhead + if ( i %20 == 0 ) { + Real plaq =WilsonLoops::avgPlaquette(Umu); + Real link_trace=WilsonLoops::linkTrace(Umu); + + std::cout << GridLogMessage << " Iteration "< &U,Real & alpha, GaugeMat & dmuAmu) { + GridBase *grid = U[0]._grid; + + std::vector A(Nd,grid); + GaugeMat g(grid); + + GaugeLinkToLieAlgebraField(U,A); + ExpiAlphaDmuAmu(A,g,alpha,dmuAmu); + + + Real vol = grid->gSites(); + Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc; + + SU::GaugeTransform(U,g); + + return trG; + } + + static Real FourierAccelSteepestDescentStep(std::vector &U,Real & alpha, GaugeMat & dmuAmu) { + + GridBase *grid = U[0]._grid; + + Real vol = grid->gSites(); + + FFT theFFT((GridCartesian *)grid); + + LatticeComplex Fp(grid); + LatticeComplex psq(grid); psq=zero; + LatticeComplex pmu(grid); + LatticeComplex one(grid); one = Complex(1.0,0.0); + + GaugeMat g(grid); + GaugeMat dmuAmu_p(grid); + std::vector A(Nd,grid); + + GaugeLinkToLieAlgebraField(U,A); + + DmuAmu(A,dmuAmu); + + theFFT.FFT_all_dim(dmuAmu_p,dmuAmu,FFT::forward); + + ////////////////////////////////// + // Work out Fp = psq_max/ psq... + ////////////////////////////////// + std::vector latt_size = grid->GlobalDimensions(); + std::vector coor(grid->_ndimension,0); + for(int mu=0;mu::taExp(ciadmam,g); + + Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc; + + SU::GaugeTransform(U,g); + + return trG; + } + + static void ExpiAlphaDmuAmu(const std::vector &A,GaugeMat &g,Real & alpha, GaugeMat &dmuAmu) { + GridBase *grid = g._grid; + Complex cialpha(0.0,-alpha); + GaugeMat ciadmam(grid); + DmuAmu(A,dmuAmu); + ciadmam = dmuAmu*cialpha; + SU::taExp(ciadmam,g); + } +/* + //////////////////////////////////////////////////////////////// + // NB The FT for fields living on links has an extra phase in it + // Could add these to the FFT class as a later task since this code + // might be reused elsewhere ???? + //////////////////////////////////////////////////////////////// + static void InverseFourierTransformAmu(FFT &theFFT,const std::vector &Ap,std::vector &Ax) { + GridBase * grid = theFFT.Grid(); + std::vector latt_size = grid->GlobalDimensions(); + + ComplexField pmu(grid); + ComplexField pha(grid); + GaugeMat Apha(grid); + + Complex ci(0.0,1.0); + + for(int mu=0;mu &Ax,std::vector &Ap) { + GridBase * grid = theFFT.Grid(); + std::vector latt_size = grid->GlobalDimensions(); + + ComplexField pmu(grid); + ComplexField pha(grid); + Complex ci(0.0,1.0); + + // Sign convention for FFTW calls: + // A(x)= Sum_p e^ipx A(p) / V + // A(p)= Sum_p e^-ipx A(x) + + for(int mu=0;mu seeds({1,2,3,4}); + + Grid_init(&argc,&argv); + + int threads = GridThread::GetThreads(); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout( { vComplex::Nsimd(),1,1,1}); + std::vector mpi_layout = GridDefaultMpi(); + + int vol = 1; + for(int d=0;d::avgPlaquette(Umu); + std::cout << " Initial plaquette "<::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-10, 1.0e-10); + + + plaq=WilsonLoops::avgPlaquette(Umu); + std::cout << " Final plaquette "< +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include + +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + Grid_init(&argc,&argv); + + int threads = GridThread::GetThreads(); + std::cout< latt_size ({N,N}); + std::vector simd_layout({vComplexD::Nsimd(),1}); + std::vector mpi_layout ({1,1}); + + int vol = 1; + int nd = latt_size.size(); + for(int d=0;dInteger(N2+W),zz,Charge); + } + + // std::cout << Charge < k(4,&GRID); + LatticeComplexD ksq(&GRID); + + ksq=zero; + for(int mu=0;mu (L/2), k[mu]-L, k[mu]); + + // std::cout << k[mu]< zero_mode(nd,0); + TComplexD Tone = ComplexD(1.0,0.0); + pokeSite(Tone,ksq,zero_mode); + + // std::cout << "Charge\n" << Charge <(mom,mommu,mu); // fourth order exponential approx - parallel_for(auto i=mom.begin();i Nf2(FermOp, CG, CG); // Set smearing (true/false), default: false - Nf2.is_smeared = true; + Nf2.is_smeared = false; // Collect actions ActionLevel Level1(1);