diff --git a/.travis.yml b/.travis.yml index 64dae823..7d8203ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,68 +9,6 @@ matrix: - os: osx osx_image: xcode8.3 compiler: clang - - compiler: gcc - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.9 - - libmpfr-dev - - libgmp-dev - - libmpc-dev - - libopenmpi-dev - - openmpi-bin - - binutils-dev - env: VERSION=-4.9 - - compiler: gcc - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-5 - - libmpfr-dev - - libgmp-dev - - libmpc-dev - - libopenmpi-dev - - openmpi-bin - - binutils-dev - env: VERSION=-5 - - compiler: clang - dist: trusty - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.8 - - libmpfr-dev - - libgmp-dev - - libmpc-dev - - libopenmpi-dev - - openmpi-bin - - binutils-dev - env: CLANG_LINK=http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz - - compiler: clang - dist: trusty - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.8 - - libmpfr-dev - - libgmp-dev - - libmpc-dev - - libopenmpi-dev - - openmpi-bin - - binutils-dev - env: CLANG_LINK=http://llvm.org/releases/3.7.0/clang+llvm-3.7.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz before_install: - export GRIDDIR=`pwd` @@ -106,9 +44,3 @@ script: - make -j4 - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals - make check - - echo make clean - - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto ; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then make -j4; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi - - diff --git a/README.md b/README.md index 9432abe1..13dd6996 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,44 @@ -# Grid - - - - - - - - - -
Last stable release - -
Development branch - -
+# Grid [![Teamcity status](http://ci.cliath.ph.ed.ac.uk/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:Grid)),branch:name:develop)/statusIcon.svg)](http://ci.cliath.ph.ed.ac.uk/project.html?projectId=Grid&tab=projectOverview) [![Travis status](https://travis-ci.org/paboyle/Grid.svg?branch=develop)](https://travis-ci.org/paboyle/Grid) **Data parallel C++ mathematical object library.** License: GPL v2. -Last update Nov 2016. +Last update June 2017. _Please do not send pull requests to the `master` branch which is reserved for releases._ + + +### Description +This library provides data parallel C++ container classes with internal memory layout +that is transformed to map efficiently to SIMD architectures. CSHIFT facilities +are provided, similar to HPF and cmfortran, and user control is given over the mapping of +array indices to both MPI tasks and SIMD processing elements. + +* Identically shaped arrays then be processed with perfect data parallelisation. +* Such identically shaped arrays are called conformable arrays. + +The transformation is based on the observation that Cartesian array processing involves +identical processing to be performed on different regions of the Cartesian array. + +The library will both geometrically decompose into MPI tasks and across SIMD lanes. +Local vector loops are parallelised with OpenMP pragmas. + +Data parallel array operations can then be specified with a SINGLE data parallel paradigm, but +optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a significant simplification +for most programmers. + +The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture. +Presently SSE4, ARM NEON (128 bits) AVX, AVX2, QPX (256 bits), IMCI and AVX512 (512 bits) targets are supported. + +These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. +The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`. + +MPI, OpenMP, and SIMD parallelism are present in the library. +Please see [this paper](https://arxiv.org/abs/1512.03487) for more detail. + + ### Compilers Intel ICPC v16.0.3 and later @@ -56,35 +73,25 @@ When you file an issue, please go though the following checklist: 6. Attach the output of `make V=1`. 7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example. +### Required libraries +Grid requires: +[GMP](https://gmplib.org/), -### Description -This library provides data parallel C++ container classes with internal memory layout -that is transformed to map efficiently to SIMD architectures. CSHIFT facilities -are provided, similar to HPF and cmfortran, and user control is given over the mapping of -array indices to both MPI tasks and SIMD processing elements. +[MPFR](http://www.mpfr.org/) -* Identically shaped arrays then be processed with perfect data parallelisation. -* Such identically shaped arrays are called conformable arrays. +Bootstrapping grid downloads and uses for internal dense matrix (non-QCD operations) the Eigen library. -The transformation is based on the observation that Cartesian array processing involves -identical processing to be performed on different regions of the Cartesian array. +Grid optionally uses: -The library will both geometrically decompose into MPI tasks and across SIMD lanes. -Local vector loops are parallelised with OpenMP pragmas. +[HDF5](https://support.hdfgroup.org/HDF5/) -Data parallel array operations can then be specified with a SINGLE data parallel paradigm, but -optimally use MPI, OpenMP and SIMD parallelism under the hood. This is a significant simplification -for most programmers. +[LIME](http://usqcd-software.github.io/c-lime/) for ILDG and SciDAC file format support. -The layout transformations are parametrised by the SIMD vector length. This adapts according to the architecture. -Presently SSE4 (128 bit) AVX, AVX2, QPX (256 bit), IMCI, and AVX512 (512 bit) targets are supported (ARM NEON on the way). +[FFTW](http://www.fftw.org) either generic version or via the Intel MKL library. -These are presented as `vRealF`, `vRealD`, `vComplexF`, and `vComplexD` internal vector data types. These may be useful in themselves for other programmers. -The corresponding scalar types are named `RealF`, `RealD`, `ComplexF` and `ComplexD`. +LAPACK either generic version or Intel MKL library. -MPI, OpenMP, and SIMD parallelism are present in the library. -Please see https://arxiv.org/abs/1512.03487 for more detail. ### Quick start First, start by cloning the repository: @@ -155,7 +162,6 @@ The following options can be use with the `--enable-comms=` option to target dif | `none` | no communications | | `mpi[-auto]` | MPI communications | | `mpi3[-auto]` | MPI communications using MPI 3 shared memory | -| `mpi3l[-auto]` | MPI communications using MPI 3 shared memory and leader model | | `shmem ` | Cray SHMEM communications | For the MPI interfaces the optional `-auto` suffix instructs the `configure` scripts to determine all the necessary compilation and linking flags. This is done by extracting the informations from the MPI wrapper specified in the environment variable `MPICXX` (if not specified `configure` will scan though a list of default names). The `-auto` suffix is not supported by the Cray environment wrapper scripts. Use the standard versions instead. @@ -173,7 +179,8 @@ The following options can be use with the `--enable-simd=` option to target diff | `AVXFMA4` | AVX (256 bit) + FMA4 | | `AVX2` | AVX 2 (256 bit) | | `AVX512` | AVX 512 bit | -| `QPX` | QPX (256 bit) | +| `NEONv8` | [ARM NEON](http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch07s03.html) (128 bit) | +| `QPX` | IBM QPX (256 bit) | Alternatively, some CPU codenames can be directly used: @@ -195,21 +202,205 @@ The following configuration is recommended for the Intel Knights Landing platfor ``` bash ../configure --enable-precision=double\ --enable-simd=KNL \ - --enable-comms=mpi-auto \ - --with-gmp= \ - --with-mpfr= \ + --enable-comms=mpi-auto \ --enable-mkl \ CXX=icpc MPICXX=mpiicpc ``` +The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library. -where `` is the UNIX prefix where GMP and MPFR are installed. If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use: +If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use: ``` bash ../configure --enable-precision=double\ --enable-simd=KNL \ --enable-comms=mpi \ - --with-gmp= \ - --with-mpfr= \ --enable-mkl \ CXX=CC CC=cc -``` \ No newline at end of file +``` + +If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed: +``` bash + --with-gmp= \ + --with-mpfr= \ +``` +where `` is the UNIX prefix where GMP and MPFR are installed. + +Knight's Landing with Intel Omnipath adapters with two adapters per node +presently performs better with use of more than one rank per node, using shared memory +for interior communication. This is the mpi3 communications implementation. +We recommend four ranks per node for best performance, but optimum is local volume dependent. + +``` bash +../configure --enable-precision=double\ + --enable-simd=KNL \ + --enable-comms=mpi3-auto \ + --enable-mkl \ + CC=icpc MPICXX=mpiicpc +``` + +### Build setup for Intel Haswell Xeon platform + +The following configuration is recommended for the Intel Haswell platform: + +``` bash +../configure --enable-precision=double\ + --enable-simd=AVX2 \ + --enable-comms=mpi3-auto \ + --enable-mkl \ + CXX=icpc MPICXX=mpiicpc +``` +The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library. + +If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed: +``` bash + --with-gmp= \ + --with-mpfr= \ +``` +where `` is the UNIX prefix where GMP and MPFR are installed. + +If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use: + +``` bash +../configure --enable-precision=double\ + --enable-simd=AVX2 \ + --enable-comms=mpi3 \ + --enable-mkl \ + CXX=CC CC=cc +``` +Since Dual socket nodes are commonplace, we recommend MPI-3 as the default with the use of +one rank per socket. If using the Intel MPI library, threads should be pinned to NUMA domains using +``` + export I_MPI_PIN=1 +``` +This is the default. + +### Build setup for Intel Skylake Xeon platform + +The following configuration is recommended for the Intel Skylake platform: + +``` bash +../configure --enable-precision=double\ + --enable-simd=AVX512 \ + --enable-comms=mpi3 \ + --enable-mkl \ + CXX=mpiicpc +``` +The MKL flag enables use of BLAS and FFTW from the Intel Math Kernels Library. + +If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed: +``` bash + --with-gmp= \ + --with-mpfr= \ +``` +where `` is the UNIX prefix where GMP and MPFR are installed. + +If you are working on a Cray machine that does not use the `mpiicpc` wrapper, please use: + +``` bash +../configure --enable-precision=double\ + --enable-simd=AVX512 \ + --enable-comms=mpi3 \ + --enable-mkl \ + CXX=CC CC=cc +``` +Since Dual socket nodes are commonplace, we recommend MPI-3 as the default with the use of +one rank per socket. If using the Intel MPI library, threads should be pinned to NUMA domains using +``` + export I_MPI_PIN=1 +``` +This is the default. + +#### Expected Skylake Gold 6148 dual socket (single prec, single node 20+20 cores) performance using NUMA MPI mapping): + +mpirun -n 2 benchmarks/Benchmark_dwf --grid 16.16.16.16 --mpi 2.1.1.1 --cacheblocking 2.2.2.2 --dslash-asm --shm 1024 --threads 18 + +TBA + + +### Build setup for AMD EPYC / RYZEN + +The AMD EPYC is a multichip module comprising 32 cores spread over four distinct chips each with 8 cores. +So, even with a single socket node there is a quad-chip module. Dual socket nodes with 64 cores total +are common. Each chip within the module exposes a separate NUMA domain. +There are four NUMA domains per socket and we recommend one MPI rank per NUMA domain. +MPI-3 is recommended with the use of four ranks per socket, +and 8 threads per rank. + +The following configuration is recommended for the AMD EPYC platform. + +``` bash +../configure --enable-precision=double\ + --enable-simd=AVX2 \ + --enable-comms=mpi3 \ + CXX=mpicxx +``` + +If gmp and mpfr are NOT in standard places (/usr/) these flags may be needed: +``` bash + --with-gmp= \ + --with-mpfr= \ +``` +where `` is the UNIX prefix where GMP and MPFR are installed. + +Using MPICH and g++ v4.9.2, best performance can be obtained using explicit GOMP_CPU_AFFINITY flags for each MPI rank. +This can be done by invoking MPI on a wrapper script omp_bind.sh to handle this. + +It is recommended to run 8 MPI ranks on a single dual socket AMD EPYC, with 8 threads per rank using MPI3 and +shared memory to communicate within this node: + +mpirun -np 8 ./omp_bind.sh ./Benchmark_dwf --mpi 2.2.2.1 --dslash-unroll --threads 8 --grid 16.16.16.16 --cacheblocking 4.4.4.4 + +Where omp_bind.sh does the following: +``` +#!/bin/bash + +numanode=` expr $PMI_RANK % 8 ` +basecore=`expr $numanode \* 16` +core0=`expr $basecore + 0 ` +core1=`expr $basecore + 2 ` +core2=`expr $basecore + 4 ` +core3=`expr $basecore + 6 ` +core4=`expr $basecore + 8 ` +core5=`expr $basecore + 10 ` +core6=`expr $basecore + 12 ` +core7=`expr $basecore + 14 ` + +export GOMP_CPU_AFFINITY="$core0 $core1 $core2 $core3 $core4 $core5 $core6 $core7" +echo GOMP_CUP_AFFINITY $GOMP_CPU_AFFINITY + +$@ +``` + +Performance: + +#### Expected AMD EPYC 7601 dual socket (single prec, single node 32+32 cores) performance using NUMA MPI mapping): + +mpirun -np 8 ./omp_bind.sh ./Benchmark_dwf --threads 8 --mpi 2.2.2.1 --dslash-unroll --grid 16.16.16.16 --cacheblocking 4.4.4.4 + +TBA + +### Build setup for BlueGene/Q + +To be written... + +### Build setup for ARM Neon + +To be written... + +### Build setup for laptops, other compilers, non-cluster builds + +Many versions of g++ and clang++ work with Grid, and involve merely replacing CXX (and MPICXX), +and omit the enable-mkl flag. + +Single node builds are enabled with +``` + --enable-comms=none +``` + +FFTW support that is not in the default search path may then enabled with +``` + --with-fftw= +``` + +BLAS will not be compiled in by default, and Lanczos will default to Eigen diagonalisation. + diff --git a/TODO b/TODO index 672879cd..c37cbf8b 100644 --- a/TODO +++ b/TODO @@ -1,23 +1,32 @@ TODO: --------------- -Peter's work list: -2)- Precision conversion and sort out localConvert <-- -3)- Remove DenseVector, DenseMatrix; Use Eigen instead. <-- started -4)- Binary I/O speed up & x-strips --- Profile CG, BlockCG, etc... Flop count/rate -- PARTIAL, time but no flop/s yet --- Physical propagator interface --- Conserved currents --- GaugeFix into central location --- Multigrid Wilson and DWF, compare to other Multigrid implementations --- HDCR resume +Large item work list: + +1)- BG/Q port and check +2)- Christoph's local basis expansion Lanczos +3)- Precision conversion and sort out localConvert <-- partial + + - Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet +4)- Physical propagator interface +5)- Conserved currents +6)- Multigrid Wilson and DWF, compare to other Multigrid implementations +7)- HDCR resume Recent DONE + +-- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O. <--- DONE +-- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE +-- GaugeFix into central location <-- DONE +-- Scidac and Ildg metadata handling <-- DONE +-- Binary I/O MPI2 IO <-- DONE +-- Binary I/O speed up & x-strips <-- DONE -- Cut down the exterior overhead <-- DONE -- Interior legs from SHM comms <-- DONE -- Half-precision comms <-- DONE --- Merge high precision reduction into develop --- multiRHS DWF; benchmark on Cori/BNL for comms elimination +-- Merge high precision reduction into develop <-- DONE +-- BlockCG, BCGrQ <-- DONE +-- multiRHS DWF; benchmark on Cori/BNL for comms elimination <-- DONE -- slice* linalg routines for multiRHS, BlockCG ----- diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc new file mode 100644 index 00000000..c0ce451f --- /dev/null +++ b/benchmarks/Benchmark_ITT.cc @@ -0,0 +1,775 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./benchmarks/Benchmark_memory_bandwidth.cc + + Copyright (C) 2015 + +Author: Peter Boyle +Author: paboyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + +typedef WilsonFermion5D WilsonFermion5DR; +typedef WilsonFermion5D WilsonFermion5DF; +typedef WilsonFermion5D WilsonFermion5DD; + + +std::vector L_list; +std::vector Ls_list; +std::vector mflop_list; + +double mflop_ref; +double mflop_ref_err; + +int NN_global; + +struct time_statistics{ + double mean; + double err; + double min; + double max; + + void statistics(std::vector v){ + double sum = std::accumulate(v.begin(), v.end(), 0.0); + mean = sum / v.size(); + + std::vector diff(v.size()); + std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; }); + double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); + err = std::sqrt(sq_sum / (v.size()*(v.size() - 1))); + + auto result = std::minmax_element(v.begin(), v.end()); + min = *result.first; + max = *result.second; +} +}; + +void comms_header(){ + std::cout < simd_layout = GridDefaultSimd(Nd,vComplexD::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + + for(int mu=0;mu1) nmu++; + + std::vector t_time(Nloop); + time_statistics timestat; + + std::cout< latt_size ({lat*mpi_layout[0], + lat*mpi_layout[1], + lat*mpi_layout[2], + lat*mpi_layout[3]}); + + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + RealD Nrank = Grid._Nprocessors; + RealD Nnode = Grid.NodeCount(); + RealD ppn = Nrank/Nnode; + + std::vector xbuf(8); + std::vector rbuf(8); + Grid.ShmBufferFreeAll(); + for(int d=0;d<8;d++){ + xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + } + + int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); + int ncomm; + double dbytes; + std::vector times(Nloop); + for(int i=0;i1 ) { + + int xmit_to_rank; + int recv_from_rank; + if ( dir == mu ) { + int comm_proc=1; + Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); + } else { + int comm_proc = mpi_layout[mu]-1; + Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); + } + tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, + (void *)&rbuf[dir][0], recv_from_rank, + bytes,dir); + +#ifdef GRID_OMP +#pragma omp atomic +#endif + ncomm++; + +#ifdef GRID_OMP +#pragma omp atomic +#endif + dbytes+=tbytes; + } + } + Grid.Barrier(); + double stop=usecond(); + t_time[i] = stop-start; // microseconds + } + + timestat.statistics(t_time); + // for(int i=0;i > LatticeVec; + typedef iVector Vec; + + std::vector simd_layout = GridDefaultSimd(Nd,vReal::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + + std::cout<({45,12,81,9})); + for(int lat=8;lat<=lmax;lat+=4){ + + std::vector latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); + int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + + Vec rn ; random(sRNG,rn); + + LatticeVec z(&Grid); z=rn; + LatticeVec x(&Grid); x=rn; + LatticeVec y(&Grid); y=rn; + double a=2.0; + + uint64_t Nloop=NLOOP; + + double start=usecond(); + for(int i=0;i mflops_all; + + /////////////////////////////////////////////////////// + // Set/Get the layout & grid size + /////////////////////////////////////////////////////// + int threads = GridThread::GetThreads(); + std::vector mpi = GridDefaultMpi(); assert(mpi.size()==4); + std::vector local({L,L,L,L}); + + GridCartesian * TmpGrid = SpaceTimeGrid::makeFourDimGrid(std::vector({64,64,64,64}), + GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi()); + uint64_t NP = TmpGrid->RankCount(); + uint64_t NN = TmpGrid->NodeCount(); + NN_global=NN; + uint64_t SHM=NP/NN; + + std::vector internal; + if ( SHM == 1 ) internal = std::vector({1,1,1,1}); + else if ( SHM == 2 ) internal = std::vector({2,1,1,1}); + else if ( SHM == 4 ) internal = std::vector({2,2,1,1}); + else if ( SHM == 8 ) internal = std::vector({2,2,2,1}); + else assert(0); + + std::vector nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]}); + std::vector latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]}); + + ///////// Welcome message //////////// + std::cout< seeds4({1,2,3,4}); + std::vector seeds5({5,6,7,8}); + GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); + GridParallelRNG RNG5(sFGrid); RNG5.SeedFixedIntegers(seeds5); + std::cout << GridLogMessage << "Initialised RNGs" << std::endl; + + ///////// Source preparation //////////// + LatticeFermion src (sFGrid); random(RNG5,src); + LatticeFermion tmp (sFGrid); + + RealD N2 = 1.0/::sqrt(norm2(src)); + src = src*N2; + + LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(RNG4,Umu); + + WilsonFermion5DR sDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,M5); + LatticeFermion src_e (sFrbGrid); + LatticeFermion src_o (sFrbGrid); + LatticeFermion r_e (sFrbGrid); + LatticeFermion r_o (sFrbGrid); + LatticeFermion r_eo (sFGrid); + LatticeFermion err (sFGrid); + { + + pickCheckerboard(Even,src_e,src); + pickCheckerboard(Odd,src_o,src); + +#if defined(AVX512) + const int num_cases = 6; + std::string fmt("A/S ; A/O ; U/S ; U/O ; G/S ; G/O "); +#else + const int num_cases = 4; + std::string fmt("U/S ; U/O ; G/S ; G/O "); +#endif + controls Cases [] = { +#ifdef AVX512 + { QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential }, + { QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute ,CartesianCommunicator::CommunicatorPolicySequential }, +#endif + { QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential }, + { QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute ,CartesianCommunicator::CommunicatorPolicySequential }, + { QCD::WilsonKernelsStatic::OptGeneric , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential }, + { QCD::WilsonKernelsStatic::OptGeneric , QCD::WilsonKernelsStatic::CommsAndCompute ,CartesianCommunicator::CommunicatorPolicySequential } + }; + + for(int c=0;cBarrier(); + for(int i=0;iBarrier(); + double t1=usecond(); + // uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0); + // if (ncall < 500) ncall = 500; + uint64_t ncall = 500; + + sFGrid->Broadcast(0,&ncall,sizeof(ncall)); + + // std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"< t_time(ncall); + for(uint64_t i=0;iBarrier(); + + double volume=Ls; for(int mu=0;mumflops_best ) mflops_best = mflops; + if ( mflops mflops_all; + + /////////////////////////////////////////////////////// + // Set/Get the layout & grid size + /////////////////////////////////////////////////////// + int threads = GridThread::GetThreads(); + std::vector mpi = GridDefaultMpi(); assert(mpi.size()==4); + std::vector local({L,L,L,L}); + + GridCartesian * TmpGrid = SpaceTimeGrid::makeFourDimGrid(std::vector({64,64,64,64}), + GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi()); + uint64_t NP = TmpGrid->RankCount(); + uint64_t NN = TmpGrid->NodeCount(); + NN_global=NN; + uint64_t SHM=NP/NN; + + std::vector internal; + if ( SHM == 1 ) internal = std::vector({1,1,1,1}); + else if ( SHM == 2 ) internal = std::vector({2,1,1,1}); + else if ( SHM == 4 ) internal = std::vector({2,2,1,1}); + else if ( SHM == 8 ) internal = std::vector({2,2,2,1}); + else assert(0); + + std::vector nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]}); + std::vector latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]}); + + ///////// Welcome message //////////// + std::cout< seeds4({1,2,3,4}); + std::vector seeds5({5,6,7,8}); + GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4); + GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5); + std::cout << GridLogMessage << "Initialised RNGs" << std::endl; + + ///////// Source preparation //////////// + LatticeFermion src (FGrid); random(RNG5,src); + LatticeFermion ref (FGrid); + LatticeFermion tmp (FGrid); + + RealD N2 = 1.0/::sqrt(norm2(src)); + src = src*N2; + + LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(RNG4,Umu); + + DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5); + + //////////////////////////////////// + // Naive wilson implementation + //////////////////////////////////// + { + LatticeGaugeField Umu5d(FGrid); + std::vector U(4,FGrid); + for(int ss=0;ssoSites();ss++){ + for(int s=0;s(Umu5d,mu); + } + for(int mu=0;muBarrier(); + for(int i=0;iBarrier(); + double t1=usecond(); + // uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0); + // if (ncall < 500) ncall = 500; + uint64_t ncall = 1000; + + FGrid->Broadcast(0,&ncall,sizeof(ncall)); + + // std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"< t_time(ncall); + for(uint64_t i=0;iBarrier(); + + double volume=Ls; for(int mu=0;mumflops_best ) mflops_best = mflops; + if ( mflops({8,2,2,2}); +#else + LebesgueOrder::Block = std::vector({2,2,2,2}); +#endif + Benchmark::Decomposition(); + + int do_memory=1; + int do_comms =1; + int do_su3 =0; + int do_wilson=1; + int do_dwf =1; + + if ( do_memory ) { + std::cout< L_list({8,12,16,24}); + std::vector wilson; + std::vector dwf4; + std::vector dwf5; + + if ( do_wilson ) { + int Ls=1; + std::cout<1) nmu++; std::cout << GridLogMessage << "Number of iterations to average: "<< Nloop << std::endl; @@ -80,7 +80,7 @@ int main (int argc, char ** argv) std::cout< latt_size ({lat*mpi_layout[0], lat*mpi_layout[1], @@ -88,12 +88,20 @@ int main (int argc, char ** argv) lat*mpi_layout[3]}); GridCartesian Grid(latt_size,simd_layout,mpi_layout); + RealD Nrank = Grid._Nprocessors; + RealD Nnode = Grid.NodeCount(); + RealD ppn = Nrank/Nnode; - std::vector > xbuf(8,std::vector(lat*lat*lat*Ls)); - std::vector > rbuf(8,std::vector(lat*lat*lat*Ls)); + std::vector > xbuf(8); + std::vector > rbuf(8); int ncomm; int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); + for(int mu=0;mu<8;mu++){ + xbuf[mu].resize(lat*lat*lat*Ls); + rbuf[mu].resize(lat*lat*lat*Ls); + // std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] < latt_size ({lat,lat,lat,lat}); GridCartesian Grid(latt_size,simd_layout,mpi_layout); + RealD Nrank = Grid._Nprocessors; + RealD Nnode = Grid.NodeCount(); + RealD ppn = Nrank/Nnode; - std::vector > xbuf(8,std::vector(lat*lat*lat*Ls)); - std::vector > rbuf(8,std::vector(lat*lat*lat*Ls)); + std::vector > xbuf(8); + std::vector > rbuf(8); + for(int mu=0;mu<8;mu++){ + xbuf[mu].resize(lat*lat*lat*Ls); + rbuf[mu].resize(lat*lat*lat*Ls); + // std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] < latt_size ({lat*mpi_layout[0], lat*mpi_layout[1], @@ -251,6 +266,9 @@ int main (int argc, char ** argv) lat*mpi_layout[3]}); GridCartesian Grid(latt_size,simd_layout,mpi_layout); + RealD Nrank = Grid._Nprocessors; + RealD Nnode = Grid.NodeCount(); + RealD ppn = Nrank/Nnode; std::vector xbuf(8); std::vector rbuf(8); @@ -258,59 +276,66 @@ int main (int argc, char ** argv) for(int d=0;d<8;d++){ xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); } int ncomm; int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); + double dbytes; for(int i=0;i requests; - ncomm=0; for(int mu=0;mu<4;mu++){ + if (mpi_layout[mu]>1 ) { ncomm++; int comm_proc=1; int xmit_to_rank; int recv_from_rank; - Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); - Grid.StencilSendToRecvFromBegin(requests, - (void *)&xbuf[mu][0], - xmit_to_rank, - (void *)&rbuf[mu][0], - recv_from_rank, - bytes); + dbytes+= + Grid.StencilSendToRecvFromBegin(requests, + (void *)&xbuf[mu][0], + xmit_to_rank, + (void *)&rbuf[mu][0], + recv_from_rank, + bytes,mu); comm_proc = mpi_layout[mu]-1; Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); - Grid.StencilSendToRecvFromBegin(requests, - (void *)&xbuf[mu+4][0], - xmit_to_rank, - (void *)&rbuf[mu+4][0], - recv_from_rank, - bytes); + dbytes+= + Grid.StencilSendToRecvFromBegin(requests, + (void *)&xbuf[mu+4][0], + xmit_to_rank, + (void *)&rbuf[mu+4][0], + recv_from_rank, + bytes,mu+4); } } - Grid.StencilSendToRecvFromComplete(requests); + Grid.StencilSendToRecvFromComplete(requests,0); Grid.Barrier(); - double stop=usecond(); - t_time[i] = stop-start; // microseconds - + double stop=usecond(); + t_time[i] = stop-start; // microseconds + } timestat.statistics(t_time); - double dbytes = bytes; - double xbytes = dbytes*2.0*ncomm; - double rbytes = xbytes; - double bidibytes = xbytes+rbytes; + dbytes=dbytes*ppn; + double xbytes = dbytes*0.5; + double rbytes = dbytes*0.5; + double bidibytes = dbytes; std::cout< latt_size ({lat*mpi_layout[0], lat*mpi_layout[1], @@ -338,6 +363,9 @@ int main (int argc, char ** argv) lat*mpi_layout[3]}); GridCartesian Grid(latt_size,simd_layout,mpi_layout); + RealD Nrank = Grid._Nprocessors; + RealD Nnode = Grid.NodeCount(); + RealD ppn = Nrank/Nnode; std::vector xbuf(8); std::vector rbuf(8); @@ -345,16 +373,18 @@ int main (int argc, char ** argv) for(int d=0;d<8;d++){ xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); } int ncomm; int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); - + double dbytes; for(int i=0;i requests; - + dbytes=0; ncomm=0; for(int mu=0;mu<4;mu++){ @@ -366,41 +396,43 @@ int main (int argc, char ** argv) int recv_from_rank; Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); - Grid.StencilSendToRecvFromBegin(requests, - (void *)&xbuf[mu][0], - xmit_to_rank, - (void *)&rbuf[mu][0], - recv_from_rank, - bytes); - Grid.StencilSendToRecvFromComplete(requests); + dbytes+= + Grid.StencilSendToRecvFromBegin(requests, + (void *)&xbuf[mu][0], + xmit_to_rank, + (void *)&rbuf[mu][0], + recv_from_rank, + bytes,mu); + Grid.StencilSendToRecvFromComplete(requests,mu); requests.resize(0); comm_proc = mpi_layout[mu]-1; Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); - Grid.StencilSendToRecvFromBegin(requests, - (void *)&xbuf[mu+4][0], - xmit_to_rank, - (void *)&rbuf[mu+4][0], - recv_from_rank, - bytes); - Grid.StencilSendToRecvFromComplete(requests); + dbytes+= + Grid.StencilSendToRecvFromBegin(requests, + (void *)&xbuf[mu+4][0], + xmit_to_rank, + (void *)&rbuf[mu+4][0], + recv_from_rank, + bytes,mu+4); + Grid.StencilSendToRecvFromComplete(requests,mu+4); requests.resize(0); } } - Grid.Barrier(); - double stop=usecond(); - t_time[i] = stop-start; // microseconds - + Grid.Barrier(); + double stop=usecond(); + t_time[i] = stop-start; // microseconds + } timestat.statistics(t_time); - double dbytes = bytes; - double xbytes = dbytes*2.0*ncomm; - double rbytes = xbytes; - double bidibytes = xbytes+rbytes; + dbytes=dbytes*ppn; + double xbytes = dbytes*0.5; + double rbytes = dbytes*0.5; + double bidibytes = dbytes; std::cout< latt_size ({lat*mpi_layout[0], + lat*mpi_layout[1], + lat*mpi_layout[2], + lat*mpi_layout[3]}); + + GridCartesian Grid(latt_size,simd_layout,mpi_layout); + RealD Nrank = Grid._Nprocessors; + RealD Nnode = Grid.NodeCount(); + RealD ppn = Nrank/Nnode; + + std::vector xbuf(8); + std::vector rbuf(8); + Grid.ShmBufferFreeAll(); + for(int d=0;d<8;d++){ + xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD)); + } + + int ncomm; + int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD); + double dbytes; + for(int i=0;i requests; + dbytes=0; + ncomm=0; + + parallel_for(int dir=0;dir<8;dir++){ + + double tbytes; + int mu =dir % 4; + + if (mpi_layout[mu]>1 ) { + + ncomm++; + int xmit_to_rank; + int recv_from_rank; + if ( dir == mu ) { + int comm_proc=1; + Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); + } else { + int comm_proc = mpi_layout[mu]-1; + Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank); + } + + tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, + (void *)&rbuf[dir][0], recv_from_rank, bytes,dir); + +#pragma omp atomic + dbytes+=tbytes; + } + } + Grid.Barrier(); + double stop=usecond(); + t_time[i] = stop-start; // microseconds + } + + timestat.statistics(t_time); + + dbytes=dbytes*ppn; + double xbytes = dbytes*0.5; + double rbytes = dbytes*0.5; + double bidibytes = dbytes; + + + std::cout<Barrier(); Dw.ZeroCounters(); @@ -302,6 +302,7 @@ int main (int argc, char ** argv) std::cout<< "sD ERR \n " << err < latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; GridCartesian Grid(latt_size,simd_layout,mpi_layout); uint64_t Nloop=NLOOP; - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); + // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); - LatticeVec z(&Grid); //random(pRNG,z); - LatticeVec x(&Grid); //random(pRNG,x); - LatticeVec y(&Grid); //random(pRNG,y); + LatticeVec z(&Grid);// random(pRNG,z); + LatticeVec x(&Grid);// random(pRNG,x); + LatticeVec y(&Grid);// random(pRNG,y); double a=2.0; @@ -83,7 +83,7 @@ int main (int argc, char ** argv) double time = (stop-start)/Nloop*1000; double flops=vol*Nvec*2;// mul,add - double bytes=3*vol*Nvec*sizeof(Real); + double bytes=3.0*vol*Nvec*sizeof(Real); std::cout< latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; GridCartesian Grid(latt_size,simd_layout,mpi_layout); - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); + // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); - LatticeVec z(&Grid); //random(pRNG,z); - LatticeVec x(&Grid); //random(pRNG,x); - LatticeVec y(&Grid); //random(pRNG,y); + LatticeVec z(&Grid);// random(pRNG,z); + LatticeVec x(&Grid);// random(pRNG,x); + LatticeVec y(&Grid);// random(pRNG,y); double a=2.0; uint64_t Nloop=NLOOP; @@ -119,7 +119,7 @@ int main (int argc, char ** argv) double time = (stop-start)/Nloop*1000; double flops=vol*Nvec*2;// mul,add - double bytes=3*vol*Nvec*sizeof(Real); + double bytes=3.0*vol*Nvec*sizeof(Real); std::cout< latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; uint64_t Nloop=NLOOP; GridCartesian Grid(latt_size,simd_layout,mpi_layout); - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); + // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); - LatticeVec z(&Grid); //random(pRNG,z); - LatticeVec x(&Grid); //random(pRNG,x); - LatticeVec y(&Grid); //random(pRNG,y); + LatticeVec z(&Grid);// random(pRNG,z); + LatticeVec x(&Grid);// random(pRNG,x); + LatticeVec y(&Grid);// random(pRNG,y); RealD a=2.0; @@ -154,7 +154,7 @@ int main (int argc, char ** argv) double stop=usecond(); double time = (stop-start)/Nloop*1000; - double bytes=2*vol*Nvec*sizeof(Real); + double bytes=2.0*vol*Nvec*sizeof(Real); double flops=vol*Nvec*1;// mul std::cout< latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; uint64_t Nloop=NLOOP; GridCartesian Grid(latt_size,simd_layout,mpi_layout); - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); - LatticeVec z(&Grid); //random(pRNG,z); - LatticeVec x(&Grid); //random(pRNG,x); - LatticeVec y(&Grid); //random(pRNG,y); + // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); + LatticeVec z(&Grid);// random(pRNG,z); + LatticeVec x(&Grid);// random(pRNG,x); + LatticeVec y(&Grid);// random(pRNG,y); RealD a=2.0; Real nn; double start=usecond(); @@ -187,7 +187,7 @@ int main (int argc, char ** argv) double stop=usecond(); double time = (stop-start)/Nloop*1000; - double bytes=vol*Nvec*sizeof(Real); + double bytes=1.0*vol*Nvec*sizeof(Real); double flops=vol*Nvec*2;// mul,add std::cout< simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); - int threads = GridThread::GetThreads(); + int64_t threads = GridThread::GetThreads(); std::cout< latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; GridCartesian Grid(latt_size,simd_layout,mpi_layout); - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); - LatticeColourMatrix z(&Grid);// random(pRNG,z); - LatticeColourMatrix x(&Grid);// random(pRNG,x); - LatticeColourMatrix y(&Grid);// random(pRNG,y); + LatticeColourMatrix z(&Grid); random(pRNG,z); + LatticeColourMatrix x(&Grid); random(pRNG,x); + LatticeColourMatrix y(&Grid); random(pRNG,y); double start=usecond(); - for(int i=0;i latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; GridCartesian Grid(latt_size,simd_layout,mpi_layout); - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); - LatticeColourMatrix z(&Grid); //random(pRNG,z); - LatticeColourMatrix x(&Grid); //random(pRNG,x); - LatticeColourMatrix y(&Grid); //random(pRNG,y); + LatticeColourMatrix z(&Grid); random(pRNG,z); + LatticeColourMatrix x(&Grid); random(pRNG,x); + LatticeColourMatrix y(&Grid); random(pRNG,y); double start=usecond(); - for(int i=0;i latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; GridCartesian Grid(latt_size,simd_layout,mpi_layout); - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); - LatticeColourMatrix z(&Grid); //random(pRNG,z); - LatticeColourMatrix x(&Grid); //random(pRNG,x); - LatticeColourMatrix y(&Grid); //random(pRNG,y); + LatticeColourMatrix z(&Grid); random(pRNG,z); + LatticeColourMatrix x(&Grid); random(pRNG,x); + LatticeColourMatrix y(&Grid); random(pRNG,y); double start=usecond(); - for(int i=0;i latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); - int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; + int64_t vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; GridCartesian Grid(latt_size,simd_layout,mpi_layout); - // GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9}); + GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(std::vector({45,12,81,9})); - LatticeColourMatrix z(&Grid); //random(pRNG,z); - LatticeColourMatrix x(&Grid); //random(pRNG,x); - LatticeColourMatrix y(&Grid); //random(pRNG,y); + LatticeColourMatrix z(&Grid); random(pRNG,z); + LatticeColourMatrix x(&Grid); random(pRNG,x); + LatticeColourMatrix y(&Grid); random(pRNG,y); double start=usecond(); - for(int i=0;i]]) AC_CHECK_DECLS([be64toh],[], [], [[#include ]]) +############## Standard libraries +AC_CHECK_LIB([m],[cos]) +AC_CHECK_LIB([stdc++],[abort]) + ############### GMP and MPFR AC_ARG_WITH([gmp], [AS_HELP_STRING([--with-gmp=prefix], @@ -184,6 +192,15 @@ AC_SEARCH_LIBS([limeCreateReader], [lime], In order to use ILGG file format please install or provide the correct path to your installation Info at: http://usqcd.jlab.org/usqcd-docs/c-lime/)]) +AC_SEARCH_LIBS([crc32], [z], + [AC_DEFINE([HAVE_ZLIB], [1], [Define to 1 if you have the `LIBZ' library])] + [have_zlib=true] [LIBS="${LIBS} -lz"], + [AC_MSG_ERROR(zlib library was not found in your system.)]) + +AC_SEARCH_LIBS([move_pages], [numa], + [AC_DEFINE([HAVE_LIBNUMA], [1], [Define to 1 if you have the `LIBNUMA' library])] + [have_libnuma=true] [LIBS="${LIBS} -lnuma"], + [AC_MSG_WARN(libnuma library was not found in your system. Some optimisations will not apply)]) AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp], [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])] @@ -237,6 +254,7 @@ case ${ax_cv_cxx_compiler_vendor} in SIMD_FLAGS='';; KNL) AC_DEFINE([AVX512],[1],[AVX512 intrinsics]) + AC_DEFINE([KNL],[1],[Knights landing processor]) SIMD_FLAGS='-march=knl';; GEN) AC_DEFINE([GEN],[1],[generic vector code]) @@ -244,6 +262,9 @@ case ${ax_cv_cxx_compiler_vendor} in [generic SIMD vector width (in bytes)]) SIMD_GEN_WIDTH_MSG=" (width= $ac_gen_simd_width)" SIMD_FLAGS='';; + NEONv8) + AC_DEFINE([NEONV8],[1],[ARMv8 NEON]) + SIMD_FLAGS='-march=armv8-a';; QPX|BGQ) AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q]) SIMD_FLAGS='';; @@ -272,6 +293,7 @@ case ${ax_cv_cxx_compiler_vendor} in SIMD_FLAGS='';; KNL) AC_DEFINE([AVX512],[1],[AVX512 intrinsics for Knights Landing]) + AC_DEFINE([KNL],[1],[Knights landing processor]) SIMD_FLAGS='-xmic-avx512';; GEN) AC_DEFINE([GEN],[1],[generic vector code]) @@ -320,14 +342,14 @@ case ${ac_COMMS} in AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] ) comms_type='none' ;; - mpi3l*) - AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] ) - comms_type='mpi3l' - ;; mpi3*) AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] ) comms_type='mpi3' ;; + mpit) + AC_DEFINE([GRID_COMMS_MPIT],[1],[GRID_COMMS_MPIT] ) + comms_type='mpit' + ;; mpi*) AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] ) comms_type='mpi' @@ -355,7 +377,7 @@ esac AM_CONDITIONAL(BUILD_COMMS_SHMEM, [ test "${comms_type}X" == "shmemX" ]) AM_CONDITIONAL(BUILD_COMMS_MPI, [ test "${comms_type}X" == "mpiX" ]) AM_CONDITIONAL(BUILD_COMMS_MPI3, [ test "${comms_type}X" == "mpi3X" ] ) -AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] ) +AM_CONDITIONAL(BUILD_COMMS_MPIT, [ test "${comms_type}X" == "mpitX" ] ) AM_CONDITIONAL(BUILD_COMMS_NONE, [ test "${comms_type}X" == "noneX" ]) ############### RNG selection diff --git a/extras/Hadrons/Modules.hpp b/extras/Hadrons/Modules.hpp index b36745e3..4e1e62f7 100644 --- a/extras/Hadrons/Modules.hpp +++ b/extras/Hadrons/Modules.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -23,4 +24,3 @@ #include #include #include -#include diff --git a/extras/Hadrons/Modules/Quark.hpp b/extras/Hadrons/Modules/MFermion/GaugeProp.hpp similarity index 65% rename from extras/Hadrons/Modules/Quark.hpp rename to extras/Hadrons/Modules/MFermion/GaugeProp.hpp index cf7d4c28..b4f9edcc 100644 --- a/extras/Hadrons/Modules/Quark.hpp +++ b/extras/Hadrons/Modules/MFermion/GaugeProp.hpp @@ -1,34 +1,5 @@ -/************************************************************************************* - -Grid physics library, www.github.com/paboyle/Grid - -Source file: extras/Hadrons/Modules/Quark.hpp - -Copyright (C) 2015 -Copyright (C) 2016 - -Author: Antonin Portelli - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -See the full license in the file "LICENSE" in the top level distribution directory -*************************************************************************************/ -/* END LEGAL */ - -#ifndef Hadrons_Quark_hpp_ -#define Hadrons_Quark_hpp_ +#ifndef Hadrons_MFermion_GaugeProp_hpp_ +#define Hadrons_MFermion_GaugeProp_hpp_ #include #include @@ -37,27 +8,29 @@ See the full license in the file "LICENSE" in the top level distribution directo BEGIN_HADRONS_NAMESPACE /****************************************************************************** - * TQuark * + * GaugeProp * ******************************************************************************/ -class QuarkPar: Serializable +BEGIN_MODULE_NAMESPACE(MFermion) + +class GaugePropPar: Serializable { public: - GRID_SERIALIZABLE_CLASS_MEMBERS(QuarkPar, + GRID_SERIALIZABLE_CLASS_MEMBERS(GaugePropPar, std::string, source, std::string, solver); }; template -class TQuark: public Module +class TGaugeProp: public Module { public: FGS_TYPE_ALIASES(FImpl,); public: // constructor - TQuark(const std::string name); + TGaugeProp(const std::string name); // destructor - virtual ~TQuark(void) = default; - // dependencies/products + virtual ~TGaugeProp(void) = default; + // dependency relation virtual std::vector getInput(void); virtual std::vector getOutput(void); // setup @@ -69,20 +42,20 @@ private: SolverFn *solver_{nullptr}; }; -MODULE_REGISTER(Quark, TQuark); +MODULE_REGISTER_NS(GaugeProp, TGaugeProp, MFermion); /****************************************************************************** - * TQuark implementation * + * TGaugeProp implementation * ******************************************************************************/ // constructor ///////////////////////////////////////////////////////////////// template -TQuark::TQuark(const std::string name) -: Module(name) +TGaugeProp::TGaugeProp(const std::string name) +: Module(name) {} // dependencies/products /////////////////////////////////////////////////////// template -std::vector TQuark::getInput(void) +std::vector TGaugeProp::getInput(void) { std::vector in = {par().source, par().solver}; @@ -90,7 +63,7 @@ std::vector TQuark::getInput(void) } template -std::vector TQuark::getOutput(void) +std::vector TGaugeProp::getOutput(void) { std::vector out = {getName(), getName() + "_5d"}; @@ -99,7 +72,7 @@ std::vector TQuark::getOutput(void) // setup /////////////////////////////////////////////////////////////////////// template -void TQuark::setup(void) +void TGaugeProp::setup(void) { Ls_ = env().getObjectLs(par().solver); env().template registerLattice(getName()); @@ -111,13 +84,13 @@ void TQuark::setup(void) // execution /////////////////////////////////////////////////////////////////// template -void TQuark::execute(void) +void TGaugeProp::execute(void) { LOG(Message) << "Computing quark propagator '" << getName() << "'" - << std::endl; + << std::endl; FermionField source(env().getGrid(Ls_)), sol(env().getGrid(Ls_)), - tmp(env().getGrid()); + tmp(env().getGrid()); std::string propName = (Ls_ == 1) ? getName() : (getName() + "_5d"); PropagatorField &prop = *env().template createLattice(propName); PropagatorField &fullSrc = *env().template getObject(par().source); @@ -128,12 +101,12 @@ void TQuark::execute(void) } LOG(Message) << "Inverting using solver '" << par().solver - << "' on source '" << par().source << "'" << std::endl; + << "' on source '" << par().source << "'" << std::endl; for (unsigned int s = 0; s < Ns; ++s) for (unsigned int c = 0; c < Nc; ++c) { LOG(Message) << "Inversion for spin= " << s << ", color= " << c - << std::endl; + << std::endl; // source conversion for 4D sources if (!env().isObject5d(par().source)) { @@ -170,7 +143,7 @@ void TQuark::execute(void) if (Ls_ > 1) { PropagatorField &p4d = - *env().template getObject(getName()); + *env().template getObject(getName()); axpby_ssp_pminus(sol, 0., sol, 1., sol, 0, 0); axpby_ssp_pplus(sol, 1., sol, 1., sol, 0, Ls_-1); @@ -180,6 +153,8 @@ void TQuark::execute(void) } } +END_MODULE_NAMESPACE + END_HADRONS_NAMESPACE -#endif // Hadrons_Quark_hpp_ +#endif // Hadrons_MFermion_GaugeProp_hpp_ diff --git a/extras/Hadrons/Modules/MGauge/Load.cc b/extras/Hadrons/Modules/MGauge/Load.cc index e5ee8abb..062e7e98 100644 --- a/extras/Hadrons/Modules/MGauge/Load.cc +++ b/extras/Hadrons/Modules/MGauge/Load.cc @@ -65,7 +65,7 @@ void TLoad::setup(void) // execution /////////////////////////////////////////////////////////////////// void TLoad::execute(void) { - NerscField header; + FieldMetaData header; std::string fileName = par().file + "." + std::to_string(env().getTrajectory()); @@ -74,5 +74,5 @@ void TLoad::execute(void) LatticeGaugeField &U = *env().createLattice(getName()); NerscIO::readConfiguration(U, header, fileName); LOG(Message) << "NERSC header:" << std::endl; - dump_nersc_header(header, LOG(Message)); + dump_meta_data(header, LOG(Message)); } diff --git a/extras/Hadrons/modules.inc b/extras/Hadrons/modules.inc index e324d3ec..29d63f58 100644 --- a/extras/Hadrons/modules.inc +++ b/extras/Hadrons/modules.inc @@ -21,6 +21,7 @@ modules_hpp =\ Modules/MContraction/WeakHamiltonianEye.hpp \ Modules/MContraction/WeakHamiltonianNonEye.hpp \ Modules/MContraction/WeakNeutral4ptDisc.hpp \ + Modules/MFermion/GaugeProp.hpp \ Modules/MGauge/Load.hpp \ Modules/MGauge/Random.hpp \ Modules/MGauge/StochEm.hpp \ @@ -35,6 +36,5 @@ modules_hpp =\ Modules/MSource/Point.hpp \ Modules/MSource/SeqGamma.hpp \ Modules/MSource/Wall.hpp \ - Modules/MSource/Z2.hpp \ - Modules/Quark.hpp + Modules/MSource/Z2.hpp diff --git a/lib/Grid.h b/lib/Grid.h index 543b0330..9dcc207b 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -41,7 +41,9 @@ Author: paboyle #include #include #include +#include #include +#include #include #endif diff --git a/lib/GridStd.h b/lib/GridStd.h index fb5e5b21..097e62ab 100644 --- a/lib/GridStd.h +++ b/lib/GridStd.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include /////////////////// // Grid config diff --git a/lib/Makefile.am b/lib/Makefile.am index fac622ca..6dd7899e 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -10,8 +10,8 @@ if BUILD_COMMS_MPI3 extra_sources+=communicator/Communicator_base.cc endif -if BUILD_COMMS_MPI3L - extra_sources+=communicator/Communicator_mpi3_leader.cc +if BUILD_COMMS_MPIT + extra_sources+=communicator/Communicator_mpit.cc extra_sources+=communicator/Communicator_base.cc endif diff --git a/lib/algorithms/densematrix/DenseMatrix.h b/lib/algorithms/densematrix/DenseMatrix.h deleted file mode 100644 index d86add21..00000000 --- a/lib/algorithms/densematrix/DenseMatrix.h +++ /dev/null @@ -1,137 +0,0 @@ - /************************************************************************************* - - Grid physics library, www.github.com/paboyle/Grid - - Source file: ./lib/algorithms/iterative/DenseMatrix.h - - Copyright (C) 2015 - -Author: Peter Boyle -Author: paboyle - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution directory - *************************************************************************************/ - /* END LEGAL */ -#ifndef GRID_DENSE_MATRIX_H -#define GRID_DENSE_MATRIX_H - -namespace Grid { - ///////////////////////////////////////////////////////////// - // Matrix untils - ///////////////////////////////////////////////////////////// - -template using DenseVector = std::vector; -template using DenseMatrix = DenseVector >; - -template void Size(DenseVector & vec, int &N) -{ - N= vec.size(); -} -template void Size(DenseMatrix & mat, int &N,int &M) -{ - N= mat.size(); - M= mat[0].size(); -} - -template void SizeSquare(DenseMatrix & mat, int &N) -{ - int M; Size(mat,N,M); - assert(N==M); -} - -template void Resize(DenseVector & mat, int N) { - mat.resize(N); -} -template void Resize(DenseMatrix & mat, int N, int M) { - mat.resize(N); - for(int i=0;i void Fill(DenseMatrix & mat, T&val) { - int N,M; - Size(mat,N,M); - for(int i=0;i DenseMatrix Transpose(DenseMatrix & mat){ - int N,M; - Size(mat,N,M); - DenseMatrix C; Resize(C,M,N); - for(int i=0;i void Unity(DenseMatrix &A){ - int N; SizeSquare(A,N); - for(int i=0;i -void PlusUnit(DenseMatrix & A,T c){ - int dim; SizeSquare(A,dim); - for(int i=0;i -DenseMatrix HermitianConj(DenseMatrix &mat){ - - int dim; SizeSquare(mat,dim); - - DenseMatrix C; Resize(C,dim,dim); - - for(int i=0;i -DenseMatrix GetSubMtx(DenseMatrix &A,int row_st, int row_end, int col_st, int col_end) -{ - DenseMatrix H; Resize(H,row_end - row_st,col_end-col_st); - - for(int i = row_st; i - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution directory - *************************************************************************************/ - /* END LEGAL */ -#ifndef FRANCIS_H -#define FRANCIS_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -//#include -//#include -//#include - -namespace Grid { - -template int SymmEigensystem(DenseMatrix &Ain, DenseVector &evals, DenseMatrix &evecs, RealD small); -template int Eigensystem(DenseMatrix &Ain, DenseVector &evals, DenseMatrix &evecs, RealD small); - -/** - Find the eigenvalues of an upper hessenberg matrix using the Francis QR algorithm. -H = - x x x x x x x x x - x x x x x x x x x - 0 x x x x x x x x - 0 0 x x x x x x x - 0 0 0 x x x x x x - 0 0 0 0 x x x x x - 0 0 0 0 0 x x x x - 0 0 0 0 0 0 x x x - 0 0 0 0 0 0 0 x x -Factorization is P T P^H where T is upper triangular (mod cc blocks) and P is orthagonal/unitary. -**/ -template -int QReigensystem(DenseMatrix &Hin, DenseVector &evals, DenseMatrix &evecs, RealD small) -{ - DenseMatrix H = Hin; - - int N ; SizeSquare(H,N); - int M = N; - - Fill(evals,0); - Fill(evecs,0); - - T s,t,x=0,y=0,z=0; - T u,d; - T apd,amd,bc; - DenseVector p(N,0); - T nrm = Norm(H); ///DenseMatrix Norm - int n, m; - int e = 0; - int it = 0; - int tot_it = 0; - int l = 0; - int r = 0; - DenseMatrix P; Resize(P,N,N); Unity(P); - DenseVector trows(N,0); - - /// Check if the matrix is really hessenberg, if not abort - RealD sth = 0; - for(int j=0;j small){ - std::cout << "Non hessenberg H = " << sth << " > " << small << std::endl; - exit(1); - } - } - } - - do{ - std::cout << "Francis QR Step N = " << N << std::endl; - /** Check for convergence - x x x x x - 0 x x x x - 0 0 x x x - 0 0 x x x - 0 0 0 0 x - for this matrix l = 4 - **/ - do{ - l = Chop_subdiag(H,nrm,e,small); - r = 0; ///May have converged on more than one eval - ///Single eval - if(l == N-1){ - evals[e] = H[l][l]; - N--; e++; r++; it = 0; - } - ///RealD eval - if(l == N-2){ - trows[l+1] = 1; ///Needed for UTSolve - apd = H[l][l] + H[l+1][l+1]; - amd = H[l][l] - H[l+1][l+1]; - bc = (T)4.0*H[l+1][l]*H[l][l+1]; - evals[e] = (T)0.5*( apd + sqrt(amd*amd + bc) ); - evals[e+1] = (T)0.5*( apd - sqrt(amd*amd + bc) ); - N-=2; e+=2; r++; it = 0; - } - } while(r>0); - - if(N ==0) break; - - DenseVector ck; Resize(ck,3); - DenseVector v; Resize(v,3); - - for(int m = N-3; m >= l; m--){ - ///Starting vector essentially random shift. - if(it%10 == 0 && N >= 3 && it > 0){ - s = (T)1.618033989*( abs( H[N-1][N-2] ) + abs( H[N-2][N-3] ) ); - t = (T)0.618033989*( abs( H[N-1][N-2] ) + abs( H[N-2][N-3] ) ); - x = H[m][m]*H[m][m] + H[m][m+1]*H[m+1][m] - s*H[m][m] + t; - y = H[m+1][m]*(H[m][m] + H[m+1][m+1] - s); - z = H[m+1][m]*H[m+2][m+1]; - } - ///Starting vector implicit Q theorem - else{ - s = (H[N-2][N-2] + H[N-1][N-1]); - t = (H[N-2][N-2]*H[N-1][N-1] - H[N-2][N-1]*H[N-1][N-2]); - x = H[m][m]*H[m][m] + H[m][m+1]*H[m+1][m] - s*H[m][m] + t; - y = H[m+1][m]*(H[m][m] + H[m+1][m+1] - s); - z = H[m+1][m]*H[m+2][m+1]; - } - ck[0] = x; ck[1] = y; ck[2] = z; - - if(m == l) break; - - /** Some stupid thing from numerical recipies, seems to work**/ - // PAB.. for heaven's sake quote page, purpose, evidence it works. - // what sort of comment is that!?!?!? - u=abs(H[m][m-1])*(abs(y)+abs(z)); - d=abs(x)*(abs(H[m-1][m-1])+abs(H[m][m])+abs(H[m+1][m+1])); - if ((T)abs(u+d) == (T)abs(d) ){ - l = m; break; - } - - //if (u < small){l = m; break;} - } - if(it > 100000){ - std::cout << "QReigensystem: bugger it got stuck after 100000 iterations" << std::endl; - std::cout << "got " << e << " evals " << l << " " << N << std::endl; - exit(1); - } - normalize(ck); ///Normalization cancels in PHP anyway - T beta; - Householder_vector(ck, 0, 2, v, beta); - Householder_mult(H,v,beta,0,l,l+2,0); - Householder_mult(H,v,beta,0,l,l+2,1); - ///Accumulate eigenvector - Householder_mult(P,v,beta,0,l,l+2,1); - int sw = 0; ///Are we on the last row? - for(int k=l;k(ck, 0, 2-sw, v, beta); - Householder_mult(H,v, beta,0,k+1,k+3-sw,0); - Householder_mult(H,v, beta,0,k+1,k+3-sw,1); - ///Accumulate eigenvector - Householder_mult(P,v, beta,0,k+1,k+3-sw,1); - } - it++; - tot_it++; - }while(N > 1); - N = evals.size(); - ///Annoying - UT solves in reverse order; - DenseVector tmp; Resize(tmp,N); - for(int i=0;i -int my_Wilkinson(DenseMatrix &Hin, DenseVector &evals, DenseMatrix &evecs, RealD small) -{ - /** - Find the eigenvalues of an upper Hessenberg matrix using the Wilkinson QR algorithm. - H = - x x 0 0 0 0 - x x x 0 0 0 - 0 x x x 0 0 - 0 0 x x x 0 - 0 0 0 x x x - 0 0 0 0 x x - Factorization is P T P^H where T is upper triangular (mod cc blocks) and P is orthagonal/unitary. **/ - return my_Wilkinson(Hin, evals, evecs, small, small); -} - -template -int my_Wilkinson(DenseMatrix &Hin, DenseVector &evals, DenseMatrix &evecs, RealD small, RealD tol) -{ - int N; SizeSquare(Hin,N); - int M = N; - - ///I don't want to modify the input but matricies must be passed by reference - //Scale a matrix by its "norm" - //RealD Hnorm = abs( Hin.LargestDiag() ); H = H*(1.0/Hnorm); - DenseMatrix H; H = Hin; - - RealD Hnorm = abs(Norm(Hin)); - H = H * (1.0 / Hnorm); - - // TODO use openmp and memset - Fill(evals,0); - Fill(evecs,0); - - T s, t, x = 0, y = 0, z = 0; - T u, d; - T apd, amd, bc; - DenseVector p; Resize(p,N); Fill(p,0); - - T nrm = Norm(H); ///DenseMatrix Norm - int n, m; - int e = 0; - int it = 0; - int tot_it = 0; - int l = 0; - int r = 0; - DenseMatrix P; Resize(P,N,N); - Unity(P); - DenseVector trows(N, 0); - /// Check if the matrix is really symm tridiag - RealD sth = 0; - for(int j = 0; j < N; ++j) - { - for(int i = j + 2; i < N; ++i) - { - if(abs(H[i][j]) > tol || abs(H[j][i]) > tol) - { - std::cout << "Non Tridiagonal H(" << i << ","<< j << ") = |" << Real( real( H[j][i] ) ) << "| > " << tol << std::endl; - std::cout << "Warning tridiagonalize and call again" << std::endl; - // exit(1); // see what is going on - //return; - } - } - } - - do{ - do{ - //Jasper - //Check if the subdiagonal term is small enough ( 0); - //Jasper - //Already converged - //-------------- - if(N == 0) break; - - DenseVector ck,v; Resize(ck,2); Resize(v,2); - - for(int m = N - 3; m >= l; m--) - { - ///Starting vector essentially random shift. - if(it%10 == 0 && N >= 3 && it > 0) - { - t = abs(H[N - 1][N - 2]) + abs(H[N - 2][N - 3]); - x = H[m][m] - t; - z = H[m + 1][m]; - } else { - ///Starting vector implicit Q theorem - d = (H[N - 2][N - 2] - H[N - 1][N - 1]) * (T) 0.5; - t = H[N - 1][N - 1] - H[N - 1][N - 2] * H[N - 1][N - 2] - / (d + sign(d) * sqrt(d * d + H[N - 1][N - 2] * H[N - 1][N - 2])); - x = H[m][m] - t; - z = H[m + 1][m]; - } - //Jasper - //why it is here???? - //----------------------- - if(m == l) - break; - - u = abs(H[m][m - 1]) * (abs(y) + abs(z)); - d = abs(x) * (abs(H[m - 1][m - 1]) + abs(H[m][m]) + abs(H[m + 1][m + 1])); - if ((T)abs(u + d) == (T)abs(d)) - { - l = m; - break; - } - } - //Jasper - if(it > 1000000) - { - std::cout << "Wilkinson: bugger it got stuck after 100000 iterations" << std::endl; - std::cout << "got " << e << " evals " << l << " " << N << std::endl; - exit(1); - } - // - T s, c; - Givens_calc(x, z, c, s); - Givens_mult(H, l, l + 1, c, -s, 0); - Givens_mult(H, l, l + 1, c, s, 1); - Givens_mult(P, l, l + 1, c, s, 1); - // - for(int k = l; k < N - 2; ++k) - { - x = H.A[k + 1][k]; - z = H.A[k + 2][k]; - Givens_calc(x, z, c, s); - Givens_mult(H, k + 1, k + 2, c, -s, 0); - Givens_mult(H, k + 1, k + 2, c, s, 1); - Givens_mult(P, k + 1, k + 2, c, s, 1); - } - it++; - tot_it++; - }while(N > 1); - - N = evals.size(); - ///Annoying - UT solves in reverse order; - DenseVector tmp(N); - for(int i = 0; i < N; ++i) - tmp[i] = evals[N-i-1]; - evals = tmp; - // - UTeigenvectors(H, trows, evals, evecs); - //UTSymmEigenvectors(H, trows, evals, evecs); - for(int i = 0; i < evals.size(); ++i) - { - evecs[i] = P * evecs[i]; - normalize(evecs[i]); - evals[i] = evals[i] * Hnorm; - } - // // FIXME this is to test - // Hin.write("evecs3", evecs); - // Hin.write("evals3", evals); - // // check rsd - // for(int i = 0; i < M; i++) { - // vector Aevec = Hin * evecs[i]; - // RealD norm2(0.); - // for(int j = 0; j < M; j++) { - // norm2 += (Aevec[j] - evals[i] * evecs[i][j]) * (Aevec[j] - evals[i] * evecs[i][j]); - // } - // } - return tot_it; -} - -template -void Hess(DenseMatrix &A, DenseMatrix &Q, int start){ - - /** - turn a matrix A = - x x x x x - x x x x x - x x x x x - x x x x x - x x x x x - into - x x x x x - x x x x x - 0 x x x x - 0 0 x x x - 0 0 0 x x - with householder rotations - Slow. - */ - int N ; SizeSquare(A,N); - DenseVector p; Resize(p,N); Fill(p,0); - - for(int k=start;k ck,v; Resize(ck,N-k-1); Resize(v,N-k-1); - for(int i=k+1;i(ck, 0, ck.size()-1, v, beta); ///Householder vector - Householder_mult(A,v,beta,start,k+1,N-1,0); ///A -> PA - Householder_mult(A,v,beta,start,k+1,N-1,1); ///PA -> PAP^H - ///Accumulate eigenvector - Householder_mult(Q,v,beta,start,k+1,N-1,1); ///Q -> QP^H - } - /*for(int l=0;l -void Tri(DenseMatrix &A, DenseMatrix &Q, int start){ -///Tridiagonalize a matrix - int N; SizeSquare(A,N); - Hess(A,Q,start); - /*for(int l=0;l -void ForceTridiagonal(DenseMatrix &A){ -///Tridiagonalize a matrix - int N ; SizeSquare(A,N); - for(int l=0;l -int my_SymmEigensystem(DenseMatrix &Ain, DenseVector &evals, DenseVector > &evecs, RealD small){ - ///Solve a symmetric eigensystem, not necessarily in tridiagonal form - int N; SizeSquare(Ain,N); - DenseMatrix A; A = Ain; - DenseMatrix Q; Resize(Q,N,N); Unity(Q); - Tri(A,Q,0); - int it = my_Wilkinson(A, evals, evecs, small); - for(int k=0;k -int Wilkinson(DenseMatrix &Ain, DenseVector &evals, DenseVector > &evecs, RealD small){ - return my_Wilkinson(Ain, evals, evecs, small); -} - -template -int SymmEigensystem(DenseMatrix &Ain, DenseVector &evals, DenseVector > &evecs, RealD small){ - return my_SymmEigensystem(Ain, evals, evecs, small); -} - -template -int Eigensystem(DenseMatrix &Ain, DenseVector &evals, DenseVector > &evecs, RealD small){ -///Solve a general eigensystem, not necessarily in tridiagonal form - int N = Ain.dim; - DenseMatrix A(N); A = Ain; - DenseMatrix Q(N);Q.Unity(); - Hess(A,Q,0); - int it = QReigensystem(A, evals, evecs, small); - for(int k=0;k - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution directory - *************************************************************************************/ - /* END LEGAL */ -#ifndef HOUSEHOLDER_H -#define HOUSEHOLDER_H - -#define TIMER(A) std::cout << GridLogMessage << __FUNC__ << " file "<< __FILE__ <<" line " << __LINE__ << std::endl; -#define ENTER() std::cout << GridLogMessage << "ENTRY "<<__FUNC__ << " file "<< __FILE__ <<" line " << __LINE__ << std::endl; -#define LEAVE() std::cout << GridLogMessage << "EXIT "<<__FUNC__ << " file "<< __FILE__ <<" line " << __LINE__ << std::endl; - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace Grid { -/** Comparison function for finding the max element in a vector **/ -template bool cf(T i, T j) { - return abs(i) < abs(j); -} - -/** - Calculate a real Givens angle - **/ -template inline void Givens_calc(T y, T z, T &c, T &s){ - - RealD mz = (RealD)abs(z); - - if(mz==0.0){ - c = 1; s = 0; - } - if(mz >= (RealD)abs(y)){ - T t = -y/z; - s = (T)1.0 / sqrt ((T)1.0 + t * t); - c = s * t; - } else { - T t = -z/y; - c = (T)1.0 / sqrt ((T)1.0 + t * t); - s = c * t; - } -} - -template inline void Givens_mult(DenseMatrix &A, int i, int k, T c, T s, int dir) -{ - int q ; SizeSquare(A,q); - - if(dir == 0){ - for(int j=0;j inline void Householder_vector(DenseVector input, int k, int j, DenseVector &v, T &beta) -{ - int N ; Size(input,N); - T m = *max_element(input.begin() + k, input.begin() + j + 1, cf ); - - if(abs(m) > 0.0){ - T alpha = 0; - - for(int i=k; i 0.0) v[k] = v[k] + (v[k]/abs(v[k]))*alpha; - else v[k] = -alpha; - } else{ - for(int i=k; i inline void Householder_vector(DenseVector input, int k, int j, int dir, DenseVector &v, T &beta) -{ - int N = input.size(); - T m = *max_element(input.begin() + k, input.begin() + j + 1, cf); - - if(abs(m) > 0.0){ - T alpha = 0; - - for(int i=k; i 0.0) v[dir] = v[dir] + (v[dir]/abs(v[dir]))*alpha; - else v[dir] = -alpha; - }else{ - for(int i=k; i inline void Householder_mult(DenseMatrix &A , DenseVector v, T beta, int l, int k, int j, int trans) -{ - int N ; SizeSquare(A,N); - - if(abs(beta) > 0.0){ - for(int p=l; p inline void Householder_mult_tri(DenseMatrix &A , DenseVector v, T beta, int l, int M, int k, int j, int trans) -{ - if(abs(beta) > 0.0){ - - int N ; SizeSquare(A,N); - - DenseMatrix tmp; Resize(tmp,N,N); Fill(tmp,0); - - T s; - for(int p=l; p class BlockConjugateGradient : public OperatorFunction { public: + typedef typename Field::scalar_type scomplex; - const int blockDim = 0; - + int blockDim ; int Nblock; + + BlockCGtype CGtype; bool ErrorOnNoConverge; // throw an assert when the CG fails to converge. // Defaults true. RealD Tolerance; Integer MaxIterations; Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion - BlockConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true) - : Tolerance(tol), - MaxIterations(maxit), - ErrorOnNoConverge(err_on_no_conv){}; + BlockConjugateGradient(BlockCGtype cgtype,int _Orthog,RealD tol, Integer maxit, bool err_on_no_conv = true) + : Tolerance(tol), CGtype(cgtype), blockDim(_Orthog), MaxIterations(maxit), ErrorOnNoConverge(err_on_no_conv) + {}; +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Thin QR factorisation (google it) +//////////////////////////////////////////////////////////////////////////////////////////////////// +void ThinQRfact (Eigen::MatrixXcd &m_rr, + Eigen::MatrixXcd &C, + Eigen::MatrixXcd &Cinv, + Field & Q, + const Field & R) +{ + int Orthog = blockDim; // First dimension is block dim; this is an assumption + //////////////////////////////////////////////////////////////////////////////////////////////////// + //Dimensions + // R_{ferm x Nblock} = Q_{ferm x Nblock} x C_{Nblock x Nblock} -> ferm x Nblock + // + // Rdag R = m_rr = Herm = L L^dag <-- Cholesky decomposition (LLT routine in Eigen) + // + // Q C = R => Q = R C^{-1} + // + // Want Ident = Q^dag Q = C^{-dag} R^dag R C^{-1} = C^{-dag} L L^dag C^{-1} = 1_{Nblock x Nblock} + // + // Set C = L^{dag}, and then Q^dag Q = ident + // + // Checks: + // Cdag C = Rdag R ; passes. + // QdagQ = 1 ; passes + //////////////////////////////////////////////////////////////////////////////////////////////////// + sliceInnerProductMatrix(m_rr,R,R,Orthog); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // Cholesky from Eigen + // There exists a ldlt that is documented as more stable + //////////////////////////////////////////////////////////////////////////////////////////////////// + Eigen::MatrixXcd L = m_rr.llt().matrixL(); + + C = L.adjoint(); + Cinv = C.inverse(); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // Q = R C^{-1} + // + // Q_j = R_i Cinv(i,j) + // + // NB maddMatrix conventions are Right multiplication X[j] a[j,i] already + //////////////////////////////////////////////////////////////////////////////////////////////////// + // FIXME:: make a sliceMulMatrix to avoid zero vector + sliceMulMatrix(Q,Cinv,R,Orthog); +} +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Call one of several implementations +//////////////////////////////////////////////////////////////////////////////////////////////////// void operator()(LinearOperatorBase &Linop, const Field &Src, Field &Psi) { - int Orthog = 0; // First dimension is block dim + if ( CGtype == BlockCGrQ ) { + BlockCGrQsolve(Linop,Src,Psi); + } else if (CGtype == BlockCG ) { + BlockCGsolve(Linop,Src,Psi); + } else if (CGtype == CGmultiRHS ) { + CGmultiRHSsolve(Linop,Src,Psi); + } else { + assert(0); + } +} + +//////////////////////////////////////////////////////////////////////////// +// BlockCGrQ implementation: +//-------------------------- +// X is guess/Solution +// B is RHS +// Solve A X_i = B_i ; i refers to Nblock index +//////////////////////////////////////////////////////////////////////////// +void BlockCGrQsolve(LinearOperatorBase &Linop, const Field &B, Field &X) +{ + int Orthog = blockDim; // First dimension is block dim; this is an assumption + Nblock = B._grid->_fdimensions[Orthog]; + + std::cout< residuals(Nblock); + std::vector ssq(Nblock); + + sliceNorm(ssq,B,Orthog); + RealD sssum=0; + for(int b=0;b Thin QR factorisation (google it) + * for k: + * Z = AD + * M = [D^dag Z]^{-1} + * X = X + D MC + * QS = Q - ZM + * D = Q + D S^dag + * C = S C + */ + /////////////////////////////////////// + // Initial block: initial search dir is guess + /////////////////////////////////////// + std::cout << GridLogMessage<<"BlockCGrQ algorithm initialisation " < Thin QR factorisation (google it) + + Linop.HermOp(X, AD); + tmp = B - AD; + //std::cout << GridLogMessage << " initial tmp " << norm2(tmp)<< std::endl; + ThinQRfact (m_rr, m_C, m_Cinv, Q, tmp); + //std::cout << GridLogMessage << " initial Q " << norm2(Q)<< std::endl; + //std::cout << GridLogMessage << " m_rr " << m_rr< max_resid ) max_resid = rr; + } + + std::cout << GridLogIterative << "\titeration "< &Linop, const Field &Src, Field &Psi) +{ + int Orthog = blockDim; // First dimension is block dim; this is an assumption Nblock = Src._grid->_fdimensions[Orthog]; std::cout< &Linop, const Field &Src, Field &Psi) ********************* */ RealD max_resid=0; + RealD rr; for(int b=0;b max_resid ) max_resid = rr; } @@ -173,13 +431,14 @@ void operator()(LinearOperatorBase &Linop, const Field &Src, Field &Psi) std::cout << GridLogMessage<<"BlockCG converged in "< &Linop, const Field &Src, Field &Psi) if (ErrorOnNoConverge) assert(0); IterationsToComplete = k; } -}; - - ////////////////////////////////////////////////////////////////////////// // multiRHS conjugate gradient. Dimension zero should be the block direction +// Use this for spread out across nodes ////////////////////////////////////////////////////////////////////////// -template -class MultiRHSConjugateGradient : public OperatorFunction { - public: - - typedef typename Field::scalar_type scomplex; - - const int blockDim = 0; - - int Nblock; - bool ErrorOnNoConverge; // throw an assert when the CG fails to converge. - // Defaults true. - RealD Tolerance; - Integer MaxIterations; - Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion - - MultiRHSConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true) - : Tolerance(tol), - MaxIterations(maxit), - ErrorOnNoConverge(err_on_no_conv){}; - -void operator()(LinearOperatorBase &Linop, const Field &Src, Field &Psi) +void CGmultiRHSsolve(LinearOperatorBase &Linop, const Field &Src, Field &Psi) { - int Orthog = 0; // First dimension is block dim + int Orthog = blockDim; // First dimension is block dim Nblock = Src._grid->_fdimensions[Orthog]; std::cout< &Linop, const Field &Src, Field &Psi) MatrixTimer.Stop(); // Alpha - // sliceInnerProductVectorTest(v_pAp_test,P,AP,Orthog); sliceInnerTimer.Start(); sliceInnerProductVector(v_pAp,P,AP,Orthog); sliceInnerTimer.Stop(); for(int b=0;b &Linop, const Field &Src, Field &Psi) std::cout << GridLogMessage<<"MultiRHS solver converged in " < &Linop, const Field &Src, Field &Psi) if (ErrorOnNoConverge) assert(0); IterationsToComplete = k; } + }; - - } #endif diff --git a/lib/algorithms/iterative/EigenSort.h b/lib/algorithms/iterative/EigenSort.h deleted file mode 100644 index 23621544..00000000 --- a/lib/algorithms/iterative/EigenSort.h +++ /dev/null @@ -1,81 +0,0 @@ - /************************************************************************************* - - Grid physics library, www.github.com/paboyle/Grid - - Source file: ./lib/algorithms/iterative/EigenSort.h - - Copyright (C) 2015 - -Author: Peter Boyle - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution directory - *************************************************************************************/ - /* END LEGAL */ -#ifndef GRID_EIGENSORT_H -#define GRID_EIGENSORT_H - - -namespace Grid { - ///////////////////////////////////////////////////////////// - // Eigen sorter to begin with - ///////////////////////////////////////////////////////////// - -template -class SortEigen { - private: - -//hacking for testing for now - private: - static bool less_lmd(RealD left,RealD right){ - return left > right; - } - static bool less_pair(std::pair& left, - std::pair& right){ - return left.first > (right.first); - } - - - public: - - void push(DenseVector& lmd, - DenseVector& evec,int N) { - DenseVector cpy(lmd.size(),evec[0]._grid); - for(int i=0;i > emod(lmd.size()); - for(int i=0;i(lmd[i],&cpy[i]); - - partial_sort(emod.begin(),emod.begin()+N,emod.end(),less_pair); - - typename DenseVector >::iterator it = emod.begin(); - for(int i=0;ifirst; - evec[i]=*(it->second); - ++it; - } - } - void push(DenseVector& lmd,int N) { - std::partial_sort(lmd.begin(),lmd.begin()+N,lmd.end(),less_lmd); - } - bool saturated(RealD lmd, RealD thrs) { - return fabs(lmd) > fabs(thrs); - } -}; - -} -#endif diff --git a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h index 3aa54360..a8723f32 100644 --- a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h +++ b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h @@ -7,7 +7,8 @@ Copyright (C) 2015 Author: Peter Boyle -Author: paboyle +Author: Chulwoo Jung +Author: Guido Cossu This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -31,436 +32,121 @@ Author: paboyle #include //memset -#ifdef USE_LAPACK -void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e, - double *vl, double *vu, int *il, int *iu, double *abstol, - int *m, double *w, double *z, int *ldz, int *isuppz, - double *work, int *lwork, int *iwork, int *liwork, - int *info); -#endif - -#include -#include - namespace Grid { + enum IRLdiagonalisation { + IRLdiagonaliseWithDSTEGR, + IRLdiagonaliseWithQR, + IRLdiagonaliseWithEigen + }; + +//////////////////////////////////////////////////////////////////////////////// +// Helper class for sorting the evalues AND evectors by Field +// Use pointer swizzle on vectors +//////////////////////////////////////////////////////////////////////////////// +template +class SortEigen { + private: + static bool less_lmd(RealD left,RealD right){ + return left > right; + } + static bool less_pair(std::pair& left, + std::pair& right){ + return left.first > (right.first); + } + + public: + void push(std::vector& lmd,std::vector& evec,int N) { + + //////////////////////////////////////////////////////////////////////// + // PAB: FIXME: VERY VERY VERY wasteful: takes a copy of the entire vector set. + // : The vector reorder should be done by pointer swizzle somehow + //////////////////////////////////////////////////////////////////////// + std::vector cpy(lmd.size(),evec[0]._grid); + for(int i=0;i > emod(lmd.size()); + + for(int i=0;i(lmd[i],&cpy[i]); + + partial_sort(emod.begin(),emod.begin()+N,emod.end(),less_pair); + + typename std::vector >::iterator it = emod.begin(); + for(int i=0;ifirst; + evec[i]=*(it->second); + ++it; + } + } + void push(std::vector& lmd,int N) { + std::partial_sort(lmd.begin(),lmd.begin()+N,lmd.end(),less_lmd); + } + bool saturated(RealD lmd, RealD thrs) { + return fabs(lmd) > fabs(thrs); + } +}; + ///////////////////////////////////////////////////////////// // Implicitly restarted lanczos ///////////////////////////////////////////////////////////// - - template - class ImplicitlyRestartedLanczos { +class ImplicitlyRestartedLanczos { - const RealD small = 1.0e-16; +private: + + int MaxIter; // Max iterations + int Nstop; // Number of evecs checked for convergence + int Nk; // Number of converged sought + int Nm; // Nm -- total number of vectors + RealD eresid; + IRLdiagonalisation diagonalisation; + //////////////////////////////////// + // Embedded objects + //////////////////////////////////// + SortEigen _sort; + LinearOperatorBase &_Linop; + OperatorFunction &_poly; + + ///////////////////////// + // Constructor + ///////////////////////// public: - int lock; - int get; - int Niter; - int converged; + ImplicitlyRestartedLanczos(LinearOperatorBase &Linop, // op + OperatorFunction & poly, // polynomial + int _Nstop, // really sought vecs + int _Nk, // sought vecs + int _Nm, // total vecs + RealD _eresid, // resid in lmd deficit + int _MaxIter, // Max iterations + IRLdiagonalisation _diagonalisation= IRLdiagonaliseWithEigen ) : + _Linop(Linop), _poly(poly), + Nstop(_Nstop), Nk(_Nk), Nm(_Nm), + eresid(_eresid), MaxIter(_MaxIter), + diagonalisation(_diagonalisation) + { }; - int Nstop; // Number of evecs checked for convergence - int Nk; // Number of converged sought - int Np; // Np -- Number of spare vecs in kryloc space - int Nm; // Nm -- total number of vectors - - RealD eresid; - - SortEigen _sort; - -// GridCartesian &_fgrid; - - LinearOperatorBase &_Linop; - - OperatorFunction &_poly; - - ///////////////////////// - // Constructor - ///////////////////////// - void init(void){}; - void Abort(int ff, DenseVector &evals, DenseVector > &evecs); - - ImplicitlyRestartedLanczos( - LinearOperatorBase &Linop, // op - OperatorFunction & poly, // polynmial - int _Nstop, // sought vecs - int _Nk, // sought vecs - int _Nm, // spare vecs - RealD _eresid, // resid in lmdue deficit - int _Niter) : // Max iterations - _Linop(Linop), - _poly(poly), - Nstop(_Nstop), - Nk(_Nk), - Nm(_Nm), - eresid(_eresid), - Niter(_Niter) - { - Np = Nm-Nk; assert(Np>0); - }; - - ImplicitlyRestartedLanczos( - LinearOperatorBase &Linop, // op - OperatorFunction & poly, // polynmial - int _Nk, // sought vecs - int _Nm, // spare vecs - RealD _eresid, // resid in lmdue deficit - int _Niter) : // Max iterations - _Linop(Linop), - _poly(poly), - Nstop(_Nk), - Nk(_Nk), - Nm(_Nm), - eresid(_eresid), - Niter(_Niter) - { - Np = Nm-Nk; assert(Np>0); - }; - - ///////////////////////// - // Sanity checked this routine (step) against Saad. - ///////////////////////// - void RitzMatrix(DenseVector& evec,int k){ - - if(1) return; - - GridBase *grid = evec[0]._grid; - Field w(grid); - std::cout << "RitzMatrix "<1 ) { - if (abs(in) >1.0e-9 ) { - std::cout<<"oops"<& lmd, - DenseVector& lme, - DenseVector& evec, - Field& w,int Nm,int k) - { - assert( k< Nm ); - - _poly(_Linop,evec[k],w); // 3. wk:=Avk−βkv_{k−1} - if(k>0){ - w -= lme[k-1] * evec[k-1]; - } - - ComplexD zalph = innerProduct(evec[k],w); // 4. αk:=(wk,vk) - RealD alph = real(zalph); - - w = w - alph * evec[k];// 5. wk:=wk−αkvk - - RealD beta = normalise(w); // 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop - // 7. vk+1 := wk/βk+1 - -// std::cout << "alpha = " << zalph << " beta "<0) { - orthogonalize(w,evec,k); // orthonormalise - } - - if(k < Nm-1) evec[k+1] = w; - } - - void qr_decomp(DenseVector& lmd, - DenseVector& lme, - int Nk, - int Nm, - DenseVector& Qt, - RealD Dsh, - int kmin, - int kmax) - { - int k = kmin-1; - RealD x; - - RealD Fden = 1.0/hypot(lmd[k]-Dsh,lme[k]); - RealD c = ( lmd[k] -Dsh) *Fden; - RealD s = -lme[k] *Fden; - - RealD tmpa1 = lmd[k]; - RealD tmpa2 = lmd[k+1]; - RealD tmpb = lme[k]; - - lmd[k] = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb; - lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb; - lme[k] = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb; - x =-s*lme[k+1]; - lme[k+1] = c*lme[k+1]; - - for(int i=0; i& lmd, - DenseVector& lme, - int N1, - int N2, - DenseVector& Qt, - GridBase *grid){ - const int size = Nm; -// tevals.resize(size); -// tevecs.resize(size); - int NN = N1; - double evals_tmp[NN]; - double evec_tmp[NN][NN]; - memset(evec_tmp[0],0,sizeof(double)*NN*NN); -// double AA[NN][NN]; - double DD[NN]; - double EE[NN]; - for (int i = 0; i< NN; i++) - for (int j = i - 1; j <= i + 1; j++) - if ( j < NN && j >= 0 ) { - if (i==j) DD[i] = lmd[i]; - if (i==j) evals_tmp[i] = lmd[i]; - if (j==(i-1)) EE[j] = lme[j]; - } - int evals_found; - int lwork = ( (18*NN) > (1+4*NN+NN*NN)? (18*NN):(1+4*NN+NN*NN)) ; - int liwork = 3+NN*10 ; - int iwork[liwork]; - double work[lwork]; - int isuppz[2*NN]; - char jobz = 'V'; // calculate evals & evecs - char range = 'I'; // calculate all evals - // char range = 'A'; // calculate all evals - char uplo = 'U'; // refer to upper half of original matrix - char compz = 'I'; // Compute eigenvectors of tridiagonal matrix - int ifail[NN]; - int info; -// int total = QMP_get_number_of_nodes(); -// int node = QMP_get_node_number(); -// GridBase *grid = evec[0]._grid; - int total = grid->_Nprocessors; - int node = grid->_processor; - int interval = (NN/total)+1; - double vl = 0.0, vu = 0.0; - int il = interval*node+1 , iu = interval*(node+1); - if (iu > NN) iu=NN; - double tol = 0.0; - if (1) { - memset(evals_tmp,0,sizeof(double)*NN); - if ( il <= NN){ - printf("total=%d node=%d il=%d iu=%d\n",total,node,il,iu); - LAPACK_dstegr(&jobz, &range, &NN, - (double*)DD, (double*)EE, - &vl, &vu, &il, &iu, // these four are ignored if second parameteris 'A' - &tol, // tolerance - &evals_found, evals_tmp, (double*)evec_tmp, &NN, - isuppz, - work, &lwork, iwork, &liwork, - &info); - for (int i = iu-1; i>= il-1; i--){ - printf("node=%d evals_found=%d evals_tmp[%d] = %g\n",node,evals_found, i - (il-1),evals_tmp[i - (il-1)]); - evals_tmp[i] = evals_tmp[i - (il-1)]; - if (il>1) evals_tmp[i-(il-1)]=0.; - for (int j = 0; j< NN; j++){ - evec_tmp[i][j] = evec_tmp[i - (il-1)][j]; - if (il>1) evec_tmp[i-(il-1)][j]=0.; - } - } - } - { -// QMP_sum_double_array(evals_tmp,NN); -// QMP_sum_double_array((double *)evec_tmp,NN*NN); - grid->GlobalSumVector(evals_tmp,NN); - grid->GlobalSumVector((double*)evec_tmp,NN*NN); - } - } -// cheating a bit. It is better to sort instead of just reversing it, but the document of the routine says evals are sorted in increasing order. qr gives evals in decreasing order. - for(int i=0;i& lmd, - DenseVector& lme, - int N2, - int N1, - DenseVector& Qt, - GridBase *grid) - { - -#ifdef USE_LAPACK - const int check_lapack=0; // just use lapack if 0, check against lapack if 1 - - if(!check_lapack) - return diagonalize_lapack(lmd,lme,N2,N1,Qt,grid); - - DenseVector lmd2(N1); - DenseVector lme2(N1); - DenseVector Qt2(N1*N1); - for(int k=0; k= kmin; --j){ - RealD dds = fabs(lmd[j-1])+fabs(lmd[j]); - if(fabs(lme[j-1])+dds > dds){ - kmax = j+1; - goto continued; - } - } - Niter = iter; -#ifdef USE_LAPACK - if(check_lapack){ - const double SMALL=1e-8; - diagonalize_lapack(lmd2,lme2,N2,N1,Qt2,grid); - DenseVector lmd3(N2); - for(int k=0; kSMALL) std::cout <<"lmd(qr) lmd(lapack) "<< k << ": " << lmd2[k] <<" "<< lmd3[k] <SMALL) std::cout <<"lme(qr)-lme(lapack) "<< k << ": " << lme2[k] - lme[k] <SMALL) std::cout <<"Qt(qr)-Qt(lapack) "<< k << ": " << Qt2[k] - Qt[k] < dds){ - kmin = j+1; - break; - } - } - } - std::cout << "[QL method] Error - Too many iteration: "<& evec, - int k) - { - typedef typename Field::scalar_type MyComplex; - MyComplex ip; - - if ( 0 ) { - for(int j=0; j &Qt) { - for(int i=0; i& evec, int k) + { + typedef typename Field::scalar_type MyComplex; + MyComplex ip; + + for(int j=0; j& eval, - DenseVector& evec, - const Field& src, - int& Nconv) - { - - GridBase *grid = evec[0]._grid; - assert(grid == src._grid); - - std::cout << " -- Nk = " << Nk << " Np = "<< Np << std::endl; - std::cout << " -- Nm = " << Nm << std::endl; - std::cout << " -- size of eval = " << eval.size() << std::endl; - std::cout << " -- size of evec = " << evec.size() << std::endl; - - assert(Nm == evec.size() && Nm == eval.size()); - - DenseVector lme(Nm); - DenseVector lme2(Nm); - DenseVector eval2(Nm); - DenseVector Qt(Nm*Nm); - DenseVector Iconv(Nm); - - DenseVector B(Nm,grid); // waste of space replicating - - Field f(grid); - Field v(grid); - - int k1 = 1; - int k2 = Nk; - - Nconv = 0; - - RealD beta_k; - - // Set initial vector - // (uniform vector) Why not src?? - // evec[0] = 1.0; - evec[0] = src; - std:: cout <<"norm2(src)= " << norm2(src)<& eval, std::vector& evec, const Field& src, int& Nconv) + { - for(int i=0; i<(Nk+1); ++i) B[i] = 0.0; + GridBase *grid = evec[0]._grid; + assert(grid == src._grid); + + std::cout << GridLogMessage <<"**************************************************************************"<< std::endl; + std::cout << GridLogMessage <<" ImplicitlyRestartedLanczos::calc() starting iteration 0 / "<< MaxIter<< std::endl; + std::cout << GridLogMessage <<"**************************************************************************"<< std::endl; + std::cout << GridLogMessage <<" -- seek Nk = " << Nk <<" vectors"<< std::endl; + std::cout << GridLogMessage <<" -- accept Nstop = " << Nstop <<" vectors"<< std::endl; + std::cout << GridLogMessage <<" -- total Nm = " << Nm <<" vectors"<< std::endl; + std::cout << GridLogMessage <<" -- size of eval = " << eval.size() << std::endl; + std::cout << GridLogMessage <<" -- size of evec = " << evec.size() << std::endl; + if ( diagonalisation == IRLdiagonaliseWithDSTEGR ) { + std::cout << GridLogMessage << "Diagonalisation is DSTEGR "< lme(Nm); + std::vector lme2(Nm); + std::vector eval2(Nm); + + Eigen::MatrixXd Qt = Eigen::MatrixXd::Zero(Nm,Nm); + + std::vector Iconv(Nm); + std::vector B(Nm,grid); // waste of space replicating + + Field f(grid); + Field v(grid); + + int k1 = 1; + int k2 = Nk; + + Nconv = 0; + + RealD beta_k; + + // Set initial vector + evec[0] = src; + std::cout << GridLogMessage <<"norm2(src)= " << norm2(src)<=Nstop ){ - goto converged; - } - } // end of iter loop + if((vv=Nstop ){ + goto converged; + } + } // end of iter loop + + std::cout << GridLogMessage <<"**************************************************************************"<< std::endl; + std::cout<< GridLogError <<" ImplicitlyRestartedLanczos::calc() NOT converged."; + std::cout << GridLogMessage <<"**************************************************************************"<< std::endl; + abort(); - converged: - // Sorting - eval.resize(Nconv); - evec.resize(Nconv,grid); - for(int i=0; i & bq, - Field &bf, - DenseMatrix &H){ +private: +/* Saad PP. 195 +1. Choose an initial vector v1 of 2-norm unity. Set β1 ≡ 0, v0 ≡ 0 +2. For k = 1,2,...,m Do: +3. wk:=Avk−βkv_{k−1} +4. αk:=(wk,vk) // +5. wk:=wk−αkvk // wk orthog vk +6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop +7. vk+1 := wk/βk+1 +8. EndDo + */ + void step(std::vector& lmd, + std::vector& lme, + std::vector& evec, + Field& w,int Nm,int k) + { + const RealD tiny = 1.0e-20; + assert( k< Nm ); + + _poly(_Linop,evec[k],w); // 3. wk:=Avk−βkv_{k−1} + + if(k>0) w -= lme[k-1] * evec[k-1]; + + ComplexD zalph = innerProduct(evec[k],w); // 4. αk:=(wk,vk) + RealD alph = real(zalph); + + w = w - alph * evec[k];// 5. wk:=wk−αkvk + + RealD beta = normalise(w); // 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop + // 7. vk+1 := wk/βk+1 + + lmd[k] = alph; + lme[k] = beta; + + if ( k > 0 ) orthogonalize(w,evec,k); // orthonormalise + if ( k < Nm-1) evec[k+1] = w; + + if ( beta < tiny ) std::cout << GridLogMessage << " beta is tiny "<& lmd, std::vector& lme, + int Nk, int Nm, + Eigen::MatrixXd & Qt, // Nm x Nm + GridBase *grid) + { + Eigen::MatrixXd TriDiag = Eigen::MatrixXd::Zero(Nk,Nk); - RealD beta; - RealD sqbt; - RealD alpha; + for(int i=0;i eigensolver(TriDiag); - for(int i=start;i& lmd, // Nm + std::vector& lme, // Nm + int Nk, int Nm, // Nk, Nm + Eigen::MatrixXd& Qt, // Nm x Nm matrix + RealD Dsh, int kmin, int kmax) + { + int k = kmin-1; + RealD x; + + RealD Fden = 1.0/hypot(lmd[k]-Dsh,lme[k]); + RealD c = ( lmd[k] -Dsh) *Fden; + RealD s = -lme[k] *Fden; + + RealD tmpa1 = lmd[k]; + RealD tmpa2 = lmd[k+1]; + RealD tmpb = lme[k]; - // Starting from scratch, bq[0] contains a random vector and |bq[0]| = 1 - int first; - if(start == 0){ + lmd[k] = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb; + lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb; + lme[k] = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb; + x =-s*lme[k+1]; + lme[k+1] = c*lme[k+1]; + + for(int i=0; i 1) std::cout << "orthagonality refined " << re << " times" < evals, - DenseVector evecs){ - int N= evals.size(); - _sort.push(evals,evecs, evals.size(),N); + void diagonalize(std::vector& lmd, std::vector& lme, + int Nk, int Nm, + Eigen::MatrixXd & Qt, + GridBase *grid) + { + Qt = Eigen::MatrixXd::Identity(Nm,Nm); + if ( diagonalisation == IRLdiagonaliseWithDSTEGR ) { + diagonalize_lapack(lmd,lme,Nk,Nm,Qt,grid); + } else if ( diagonalisation == IRLdiagonaliseWithQR ) { + diagonalize_QR(lmd,lme,Nk,Nm,Qt,grid); + } else if ( diagonalisation == IRLdiagonaliseWithEigen ) { + diagonalize_Eigen(lmd,lme,Nk,Nm,Qt,grid); + } else { + assert(0); } + } - void ImplicitRestart(int TM, DenseVector &evals, DenseVector > &evecs, DenseVector &bq, Field &bf, int cont) +#ifdef USE_LAPACK +void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e, + double *vl, double *vu, int *il, int *iu, double *abstol, + int *m, double *w, double *z, int *ldz, int *isuppz, + double *work, int *lwork, int *iwork, int *liwork, + int *info); +#endif + +void diagonalize_lapack(std::vector& lmd, + std::vector& lme, + int Nk, int Nm, + Eigen::MatrixXd& Qt, + GridBase *grid) +{ +#ifdef USE_LAPACK + const int size = Nm; + int NN = Nk; + double evals_tmp[NN]; + double evec_tmp[NN][NN]; + memset(evec_tmp[0],0,sizeof(double)*NN*NN); + double DD[NN]; + double EE[NN]; + for (int i = 0; i< NN; i++) { + for (int j = i - 1; j <= i + 1; j++) { + if ( j < NN && j >= 0 ) { + if (i==j) DD[i] = lmd[i]; + if (i==j) evals_tmp[i] = lmd[i]; + if (j==(i-1)) EE[j] = lme[j]; + } + } + } + int evals_found; + int lwork = ( (18*NN) > (1+4*NN+NN*NN)? (18*NN):(1+4*NN+NN*NN)) ; + int liwork = 3+NN*10 ; + int iwork[liwork]; + double work[lwork]; + int isuppz[2*NN]; + char jobz = 'V'; // calculate evals & evecs + char range = 'I'; // calculate all evals + // char range = 'A'; // calculate all evals + char uplo = 'U'; // refer to upper half of original matrix + char compz = 'I'; // Compute eigenvectors of tridiagonal matrix + int ifail[NN]; + int info; + int total = grid->_Nprocessors; + int node = grid->_processor; + int interval = (NN/total)+1; + double vl = 0.0, vu = 0.0; + int il = interval*node+1 , iu = interval*(node+1); + if (iu > NN) iu=NN; + double tol = 0.0; + if (1) { + memset(evals_tmp,0,sizeof(double)*NN); + if ( il <= NN){ + LAPACK_dstegr(&jobz, &range, &NN, + (double*)DD, (double*)EE, + &vl, &vu, &il, &iu, // these four are ignored if second parameteris 'A' + &tol, // tolerance + &evals_found, evals_tmp, (double*)evec_tmp, &NN, + isuppz, + work, &lwork, iwork, &liwork, + &info); + for (int i = iu-1; i>= il-1; i--){ + evals_tmp[i] = evals_tmp[i - (il-1)]; + if (il>1) evals_tmp[i-(il-1)]=0.; + for (int j = 0; j< NN; j++){ + evec_tmp[i][j] = evec_tmp[i - (il-1)][j]; + if (il>1) evec_tmp[i-(il-1)][j]=0.; + } + } + } { - std::cout << "ImplicitRestart begin. Eigensort starting\n"; + grid->GlobalSumVector(evals_tmp,NN); + grid->GlobalSumVector((double*)evec_tmp,NN*NN); + } + } + // Safer to sort instead of just reversing it, + // but the document of the routine says evals are sorted in increasing order. + // qr gives evals in decreasing order. + for(int i=0;i H; Resize(H,Nm,Nm); + void diagonalize_QR(std::vector& lmd, std::vector& lme, + int Nk, int Nm, + Eigen::MatrixXd & Qt, + GridBase *grid) + { + int Niter = 100*Nm; + int kmin = 1; + int kmax = Nk; - EigenSort(evals, evecs); - - ///Assign shifts - int K=Nk; - int M=Nm; - int P=Np; - int converged=0; - if(K - converged < 4) P = (M - K-1); //one - // DenseVector shifts(P + shift_extra.size()); - DenseVector shifts(P); - for(int k = 0; k < P; ++k) - shifts[k] = evals[k]; - - /// Shift to form a new H and q - DenseMatrix Q; Resize(Q,TM,TM); - Unity(Q); - Shift(Q, shifts); // H is implicitly passed in in Rudy's Shift routine - - int ff = K; - - /// Shifted H defines a new K step Arnoldi factorization - RealD beta = H[ff][ff-1]; - RealD sig = Q[TM - 1][ff - 1]; - std::cout << "beta = " << beta << " sig = " << real(sig) < q Q - times_real(bq, Q, TM); - - std::cout << norm2(bq[0]) << " -- after " << ff < &bq, Field &bf, DenseVector > & evecs,DenseVector &evals) - { - init(); - - int M=Nm; - - DenseMatrix H; Resize(H,Nm,Nm); - Resize(evals,Nm); - Resize(evecs,Nm); - - int ff = Lanczos_Factor(0, M, cont, bq,bf,H); // 0--M to begin with - - if(ff < M) { - std::cout << "Krylov: aborting ff "< " << it << std::endl; - int lock_num = lock ? converged : 0; - DenseVector tevals(M - lock_num ); - DenseMatrix tevecs; Resize(tevecs,M - lock_num,M - lock_num); - - //check residual of polynominal - TestConv(H,M, tevals, tevecs); - - if(converged >= Nk) - break; - - ImplicitRestart(ff, tevals,tevecs,H); - } - Wilkinson(H, evals, evecs, small); - // Check(); - - std::cout << "Done "< & H,DenseMatrix &Q, DenseVector shifts) { - - int P; Size(shifts,P); - int M; SizeSquare(Q,M); - - Unity(Q); - - int lock_num = lock ? converged : 0; - - RealD t_Househoulder_vector(0.0); - RealD t_Househoulder_mult(0.0); - - for(int i=0;i ck(3), v(3); - - x = H[lock_num+0][lock_num+0]-shifts[i]; - y = H[lock_num+1][lock_num+0]; - ck[0] = x; ck[1] = y; ck[2] = 0; - - normalise(ck); ///Normalization cancels in PHP anyway - RealD beta; - - Householder_vector(ck, 0, 2, v, beta); - Householder_mult(H,v,beta,0,lock_num+0,lock_num+2,0); - Householder_mult(H,v,beta,0,lock_num+0,lock_num+2,1); - ///Accumulate eigenvector - Householder_mult(Q,v,beta,0,lock_num+0,lock_num+2,1); - - int sw = 0; - for(int k=lock_num+0;k(ck, 0, 2-sw, v, beta); - Householder_mult(H,v, beta,0,k+1,k+3-sw,0); - Householder_mult(H,v, beta,0,k+1,k+3-sw,1); - ///Accumulate eigenvector - Householder_mult(Q,v, beta,0,k+1,k+3-sw,1); + // determination of 2x2 leading submatrix + RealD dsub = lmd[kmax-1]-lmd[kmax-2]; + RealD dd = sqrt(dsub*dsub + 4.0*lme[kmax-2]*lme[kmax-2]); + RealD Dsh = 0.5*(lmd[kmax-2]+lmd[kmax-1] +dd*(dsub/fabs(dsub))); + // (Dsh: shift) + + // transformation + qr_decomp(lmd,lme,Nk,Nm,Qt,Dsh,kmin,kmax); // Nk, Nm + + // Convergence criterion (redef of kmin and kamx) + for(int j=kmax-1; j>= kmin; --j){ + RealD dds = fabs(lmd[j-1])+fabs(lmd[j]); + if(fabs(lme[j-1])+dds > dds){ + kmax = j+1; + goto continued; } } - } + Niter = iter; + return; - void TestConv(DenseMatrix & H,int SS, - DenseVector &bq, Field &bf, - DenseVector &tevals, DenseVector > &tevecs, - int lock, int converged) - { - std::cout << "Converged " << converged << " so far." << std::endl; - int lock_num = lock ? converged : 0; - int M = Nm; - - ///Active Factorization - DenseMatrix AH; Resize(AH,SS - lock_num,SS - lock_num ); - - AH = GetSubMtx(H,lock_num, SS, lock_num, SS); - - int NN=tevals.size(); - int AHsize=SS-lock_num; - - RealD small=1.0e-16; - Wilkinson(AH, tevals, tevecs, small); - - EigenSort(tevals, tevecs); - - RealD resid_nrm= norm2(bf); - - if(!lock) converged = 0; -#if 0 - for(int i = SS - lock_num - 1; i >= SS - Nk && i >= 0; --i){ - - RealD diff = 0; - diff = abs( tevecs[i][Nm - 1 - lock_num] ) * resid_nrm; - - std::cout << "residual estimate " << SS-1-i << " " << diff << " of (" << tevals[i] << ")" << std::endl; - - if(diff < converged) { - - if(lock) { - - DenseMatrix Q; Resize(Q,M,M); - bool herm = true; - - Lock(H, Q, tevals[i], converged, small, SS, herm); - - times_real(bq, Q, bq.size()); - bf = Q[M - 1][M - 1]* bf; - lock_num++; - } - converged++; - std::cout << " converged on eval " << converged << " of " << Nk << std::endl; - } else { + continued: + for(int j=0; j dds){ + kmin = j+1; break; } } -#endif - std::cout << "Got " << converged << " so far " < &evals, - DenseVector > &evecs) { - - DenseVector goodval(this->get); - - EigenSort(evals,evecs); - - int NM = Nm; - - DenseVector< DenseVector > V; Size(V,NM); - DenseVector QZ(NM*NM); - - for(int i = 0; i < NM; i++){ - for(int j = 0; j < NM; j++){ - // evecs[i][j]; - } - } - } - - -/** - There is some matrix Q such that for any vector y - Q.e_1 = y and Q is unitary. -**/ - template - static T orthQ(DenseMatrix &Q, DenseVector y){ - int N = y.size(); //Matrix Size - Fill(Q,0.0); - T tau; - for(int i=0;i 0.0){ - - T gam = conj( (y[j]/tau)/tau0 ); - for(int k=0;k<=j-1;k++){ - Q[k][j]=-gam*y[k]; - } - Q[j][j]=tau0/tau; - } else { - Q[j-1][j]=1.0; - } - tau0 = tau; - } - return tau; + std::cout << GridLogError << "[QL method] Error - Too many iteration: "< - static T orthU(DenseMatrix &Q, DenseVector y){ - T tau = orthQ(Q,y); - SL(Q); - return tau; - } - - -/** - Wind up with a matrix with the first con rows untouched - -say con = 2 - Q is such that Qdag H Q has {x, x, val, 0, 0, 0, 0, ...} as 1st colum - and the matrix is upper hessenberg - and with f and Q appropriately modidied with Q is the arnoldi factorization - -**/ - -template -static void Lock(DenseMatrix &H, // Hess mtx - DenseMatrix &Q, // Lock Transform - T val, // value to be locked - int con, // number already locked - RealD small, - int dfg, - bool herm) -{ - //ForceTridiagonal(H); - - int M = H.dim; - DenseVector vec; Resize(vec,M-con); - - DenseMatrix AH; Resize(AH,M-con,M-con); - AH = GetSubMtx(H,con, M, con, M); - - DenseMatrix QQ; Resize(QQ,M-con,M-con); - - Unity(Q); Unity(QQ); - - DenseVector evals; Resize(evals,M-con); - DenseMatrix evecs; Resize(evecs,M-con,M-con); - - Wilkinson(AH, evals, evecs, small); - - int k=0; - RealD cold = abs( val - evals[k]); - for(int i=1;icon+2; j--){ - - DenseMatrix U; Resize(U,j-1-con,j-1-con); - DenseVector z; Resize(z,j-1-con); - T nm = norm(z); - for(int k = con+0;k Hb; Resize(Hb,j-1-con,M); - - for(int a = 0;a Qb; Resize(Qb,M,M); - - for(int a = 0;a Hc; Resize(Hc,M,M); - - for(int a = 0;a +#include namespace Grid { @@ -11,7 +9,7 @@ int PointerCache::victim; void *PointerCache::Insert(void *ptr,size_t bytes) { - if (bytes < 4096 ) return NULL; + if (bytes < 4096 ) return ptr; #ifdef GRID_OMP assert(omp_in_parallel()==0); @@ -63,4 +61,37 @@ void *PointerCache::Lookup(size_t bytes) { return NULL; } + +void check_huge_pages(void *Buf,uint64_t BYTES) +{ +#ifdef __linux__ + int fd = open("/proc/self/pagemap", O_RDONLY); + assert(fd >= 0); + const int page_size = 4096; + uint64_t virt_pfn = (uint64_t)Buf / page_size; + off_t offset = sizeof(uint64_t) * virt_pfn; + uint64_t npages = (BYTES + page_size-1) / page_size; + uint64_t pagedata[npages]; + uint64_t ret = lseek(fd, offset, SEEK_SET); + assert(ret == offset); + ret = ::read(fd, pagedata, sizeof(uint64_t)*npages); + assert(ret == sizeof(uint64_t) * npages); + int nhugepages = npages / 512; + int n4ktotal, nnothuge; + n4ktotal = 0; + nnothuge = 0; + for (int i = 0; i < nhugepages; ++i) { + uint64_t baseaddr = (pagedata[i*512] & 0x7fffffffffffffULL) * page_size; + for (int j = 0; j < 512; ++j) { + uint64_t pageaddr = (pagedata[i*512+j] & 0x7fffffffffffffULL) * page_size; + ++n4ktotal; + if (pageaddr != baseaddr + j * page_size) + ++nnothuge; + } + } + int rank = CartesianCommunicator::RankWorld(); + printf("rank %d Allocated %d 4k pages, %d not in huge pages\n", rank, n4ktotal, nnothuge); +#endif +} + } diff --git a/lib/allocator/AlignedAllocator.h b/lib/allocator/AlignedAllocator.h index 6e85ab27..e64a5949 100644 --- a/lib/allocator/AlignedAllocator.h +++ b/lib/allocator/AlignedAllocator.h @@ -64,6 +64,8 @@ namespace Grid { }; + void check_huge_pages(void *Buf,uint64_t BYTES); + //////////////////////////////////////////////////////////////////// // A lattice of something, but assume the something is SIMDized. //////////////////////////////////////////////////////////////////// @@ -92,18 +94,34 @@ public: size_type bytes = __n*sizeof(_Tp); _Tp *ptr = (_Tp *) PointerCache::Lookup(bytes); - -#ifdef HAVE_MM_MALLOC_H - if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) _mm_malloc(bytes,128); -#else - if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) memalign(128,bytes); -#endif + // if ( ptr != NULL ) + // std::cout << "alignedAllocator "<<__n << " cache hit "<< std::hex << ptr < & processor_grid) : CartesianCommunicator(processor_grid) {}; - // Physics Grid information. std::vector _simd_layout;// Which dimensions get relayed out over simd lanes. std::vector _fdimensions;// (full) Global dimensions of array prior to cb removal @@ -63,13 +62,12 @@ public: int _isites; int _fsites; // _isites*_osites = product(dimensions). int _gsites; - std::vector _slice_block; // subslice information + std::vector _slice_block;// subslice information std::vector _slice_stride; std::vector _slice_nblock; - // Might need these at some point - // std::vector _lstart; // local start of array in gcoors. _processor_coor[d]*_ldimensions[d] - // std::vector _lend; // local end of array in gcoors _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1 + std::vector _lstart; // local start of array in gcoors _processor_coor[d]*_ldimensions[d] + std::vector _lend ; // local end of array in gcoors _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1 public: @@ -176,6 +174,7 @@ public: inline int gSites(void) const { return _isites*_osites*_Nprocessors; }; inline int Nd (void) const { return _ndimension;}; + inline const std::vector LocalStarts(void) { return _lstart; }; inline const std::vector &FullDimensions(void) { return _fdimensions;}; inline const std::vector &GlobalDimensions(void) { return _gdimensions;}; inline const std::vector &LocalDimensions(void) { return _ldimensions;}; @@ -186,17 +185,18 @@ public: //////////////////////////////////////////////////////////////// void show_decomposition(){ - std::cout << GridLogMessage << "Full Dimensions : " << _fdimensions << std::endl; - std::cout << GridLogMessage << "Global Dimensions : " << _gdimensions << std::endl; - std::cout << GridLogMessage << "Local Dimensions : " << _ldimensions << std::endl; - std::cout << GridLogMessage << "Reduced Dimensions : " << _rdimensions << std::endl; - std::cout << GridLogMessage << "Outer strides : " << _ostride << std::endl; - std::cout << GridLogMessage << "Inner strides : " << _istride << std::endl; - std::cout << GridLogMessage << "iSites : " << _isites << std::endl; - std::cout << GridLogMessage << "oSites : " << _osites << std::endl; - std::cout << GridLogMessage << "lSites : " << lSites() << std::endl; - std::cout << GridLogMessage << "gSites : " << gSites() << std::endl; - std::cout << GridLogMessage << "Nd : " << _ndimension << std::endl; + std::cout << GridLogMessage << "\tFull Dimensions : " << _fdimensions << std::endl; + std::cout << GridLogMessage << "\tSIMD layout : " << _simd_layout << std::endl; + std::cout << GridLogMessage << "\tGlobal Dimensions : " << _gdimensions << std::endl; + std::cout << GridLogMessage << "\tLocal Dimensions : " << _ldimensions << std::endl; + std::cout << GridLogMessage << "\tReduced Dimensions : " << _rdimensions << std::endl; + std::cout << GridLogMessage << "\tOuter strides : " << _ostride << std::endl; + std::cout << GridLogMessage << "\tInner strides : " << _istride << std::endl; + std::cout << GridLogMessage << "\tiSites : " << _isites << std::endl; + std::cout << GridLogMessage << "\toSites : " << _osites << std::endl; + std::cout << GridLogMessage << "\tlSites : " << lSites() << std::endl; + std::cout << GridLogMessage << "\tgSites : " << gSites() << std::endl; + std::cout << GridLogMessage << "\tNd : " << _ndimension << std::endl; } //////////////////////////////////////////////////////////////// diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h index 7e29d311..815e3b22 100644 --- a/lib/cartesian/Cartesian_full.h +++ b/lib/cartesian/Cartesian_full.h @@ -62,73 +62,81 @@ public: return shift; } GridCartesian(const std::vector &dimensions, - const std::vector &simd_layout, - const std::vector &processor_grid - ) : GridBase(processor_grid) + const std::vector &simd_layout, + const std::vector &processor_grid) : GridBase(processor_grid) { - /////////////////////// - // Grid information - /////////////////////// - _ndimension = dimensions.size(); - - _fdimensions.resize(_ndimension); - _gdimensions.resize(_ndimension); - _ldimensions.resize(_ndimension); - _rdimensions.resize(_ndimension); - _simd_layout.resize(_ndimension); - - _ostride.resize(_ndimension); - _istride.resize(_ndimension); - - _fsites = _gsites = _osites = _isites = 1; + /////////////////////// + // Grid information + /////////////////////// + _ndimension = dimensions.size(); - for(int d=0;d<_ndimension;d++){ - _fdimensions[d] = dimensions[d]; // Global dimensions - _gdimensions[d] = _fdimensions[d]; // Global dimensions - _simd_layout[d] = simd_layout[d]; - _fsites = _fsites * _fdimensions[d]; - _gsites = _gsites * _gdimensions[d]; + _fdimensions.resize(_ndimension); + _gdimensions.resize(_ndimension); + _ldimensions.resize(_ndimension); + _rdimensions.resize(_ndimension); + _simd_layout.resize(_ndimension); + _lstart.resize(_ndimension); + _lend.resize(_ndimension); - //FIXME check for exact division + _ostride.resize(_ndimension); + _istride.resize(_ndimension); - // Use a reduced simd grid - _ldimensions[d]= _gdimensions[d]/_processors[d]; //local dimensions - _rdimensions[d]= _ldimensions[d]/_simd_layout[d]; //overdecomposition - _osites *= _rdimensions[d]; - _isites *= _simd_layout[d]; - - // Addressing support - if ( d==0 ) { - _ostride[d] = 1; - _istride[d] = 1; - } else { - _ostride[d] = _ostride[d-1]*_rdimensions[d-1]; - _istride[d] = _istride[d-1]*_simd_layout[d-1]; - } + _fsites = _gsites = _osites = _isites = 1; + + for (int d = 0; d < _ndimension; d++) + { + _fdimensions[d] = dimensions[d]; // Global dimensions + _gdimensions[d] = _fdimensions[d]; // Global dimensions + _simd_layout[d] = simd_layout[d]; + _fsites = _fsites * _fdimensions[d]; + _gsites = _gsites * _gdimensions[d]; + + // Use a reduced simd grid + _ldimensions[d] = _gdimensions[d] / _processors[d]; //local dimensions + assert(_ldimensions[d] * _processors[d] == _gdimensions[d]); + + _rdimensions[d] = _ldimensions[d] / _simd_layout[d]; //overdecomposition + assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]); + + _lstart[d] = _processor_coor[d] * _ldimensions[d]; + _lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1; + _osites *= _rdimensions[d]; + _isites *= _simd_layout[d]; + + // Addressing support + if (d == 0) + { + _ostride[d] = 1; + _istride[d] = 1; } - - /////////////////////// - // subplane information - /////////////////////// - _slice_block.resize(_ndimension); - _slice_stride.resize(_ndimension); - _slice_nblock.resize(_ndimension); - - int block =1; - int nblock=1; - for(int d=0;d<_ndimension;d++) nblock*=_rdimensions[d]; - - for(int d=0;d<_ndimension;d++){ - nblock/=_rdimensions[d]; - _slice_block[d] =block; - _slice_stride[d]=_ostride[d]*_rdimensions[d]; - _slice_nblock[d]=nblock; - block = block*_rdimensions[d]; + else + { + _ostride[d] = _ostride[d - 1] * _rdimensions[d - 1]; + _istride[d] = _istride[d - 1] * _simd_layout[d - 1]; } + } + /////////////////////// + // subplane information + /////////////////////// + _slice_block.resize(_ndimension); + _slice_stride.resize(_ndimension); + _slice_nblock.resize(_ndimension); + + int block = 1; + int nblock = 1; + for (int d = 0; d < _ndimension; d++) + nblock *= _rdimensions[d]; + + for (int d = 0; d < _ndimension; d++) + { + nblock /= _rdimensions[d]; + _slice_block[d] = block; + _slice_stride[d] = _ostride[d] * _rdimensions[d]; + _slice_nblock[d] = nblock; + block = block * _rdimensions[d]; + } }; }; - - } #endif diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h index 2f132c19..b1a5b9ef 100644 --- a/lib/cartesian/Cartesian_red_black.h +++ b/lib/cartesian/Cartesian_red_black.h @@ -131,132 +131,155 @@ public: Init(dimensions,simd_layout,processor_grid,checker_dim_mask,0); } void Init(const std::vector &dimensions, - const std::vector &simd_layout, - const std::vector &processor_grid, - const std::vector &checker_dim_mask, - int checker_dim) + const std::vector &simd_layout, + const std::vector &processor_grid, + const std::vector &checker_dim_mask, + int checker_dim) { - /////////////////////// - // Grid information - /////////////////////// + /////////////////////// + // Grid information + /////////////////////// _checker_dim = checker_dim; - assert(checker_dim_mask[checker_dim]==1); + assert(checker_dim_mask[checker_dim] == 1); _ndimension = dimensions.size(); - assert(checker_dim_mask.size()==_ndimension); - assert(processor_grid.size()==_ndimension); - assert(simd_layout.size()==_ndimension); - + assert(checker_dim_mask.size() == _ndimension); + assert(processor_grid.size() == _ndimension); + assert(simd_layout.size() == _ndimension); + _fdimensions.resize(_ndimension); _gdimensions.resize(_ndimension); _ldimensions.resize(_ndimension); _rdimensions.resize(_ndimension); _simd_layout.resize(_ndimension); - + _lstart.resize(_ndimension); + _lend.resize(_ndimension); + _ostride.resize(_ndimension); _istride.resize(_ndimension); - + _fsites = _gsites = _osites = _isites = 1; - - _checker_dim_mask=checker_dim_mask; - for(int d=0;d<_ndimension;d++){ - _fdimensions[d] = dimensions[d]; - _gdimensions[d] = _fdimensions[d]; - _fsites = _fsites * _fdimensions[d]; - _gsites = _gsites * _gdimensions[d]; - - if (d==_checker_dim) { - _gdimensions[d] = _gdimensions[d]/2; // Remove a checkerboard - } - _ldimensions[d] = _gdimensions[d]/_processors[d]; + _checker_dim_mask = checker_dim_mask; - // Use a reduced simd grid - _simd_layout[d] = simd_layout[d]; - _rdimensions[d]= _ldimensions[d]/_simd_layout[d]; - assert(_rdimensions[d]>0); + for (int d = 0; d < _ndimension; d++) + { + _fdimensions[d] = dimensions[d]; + _gdimensions[d] = _fdimensions[d]; + _fsites = _fsites * _fdimensions[d]; + _gsites = _gsites * _gdimensions[d]; - // all elements of a simd vector must have same checkerboard. - // If Ls vectorised, this must still be the case; e.g. dwf rb5d - if ( _simd_layout[d]>1 ) { - if ( checker_dim_mask[d] ) { - assert( (_rdimensions[d]&0x1) == 0 ); - } - } + if (d == _checker_dim) + { + assert((_gdimensions[d] & 0x1) == 0); + _gdimensions[d] = _gdimensions[d] / 2; // Remove a checkerboard + } + _ldimensions[d] = _gdimensions[d] / _processors[d]; + assert(_ldimensions[d] * _processors[d] == _gdimensions[d]); + _lstart[d] = _processor_coor[d] * _ldimensions[d]; + _lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1; - _osites *= _rdimensions[d]; - _isites *= _simd_layout[d]; - - // Addressing support - if ( d==0 ) { - _ostride[d] = 1; - _istride[d] = 1; - } else { - _ostride[d] = _ostride[d-1]*_rdimensions[d-1]; - _istride[d] = _istride[d-1]*_simd_layout[d-1]; - } + // Use a reduced simd grid + _simd_layout[d] = simd_layout[d]; + _rdimensions[d] = _ldimensions[d] / _simd_layout[d]; // this is not checking if this is integer + assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]); + assert(_rdimensions[d] > 0); + // all elements of a simd vector must have same checkerboard. + // If Ls vectorised, this must still be the case; e.g. dwf rb5d + if (_simd_layout[d] > 1) + { + if (checker_dim_mask[d]) + { + assert((_rdimensions[d] & 0x1) == 0); + } + } + _osites *= _rdimensions[d]; + _isites *= _simd_layout[d]; + + // Addressing support + if (d == 0) + { + _ostride[d] = 1; + _istride[d] = 1; + } + else + { + _ostride[d] = _ostride[d - 1] * _rdimensions[d - 1]; + _istride[d] = _istride[d - 1] * _simd_layout[d - 1]; + } } - + //////////////////////////////////////////////////////////////////////////////////////////// // subplane information //////////////////////////////////////////////////////////////////////////////////////////// _slice_block.resize(_ndimension); _slice_stride.resize(_ndimension); _slice_nblock.resize(_ndimension); - - int block =1; - int nblock=1; - for(int d=0;d<_ndimension;d++) nblock*=_rdimensions[d]; - - for(int d=0;d<_ndimension;d++){ - nblock/=_rdimensions[d]; - _slice_block[d] =block; - _slice_stride[d]=_ostride[d]*_rdimensions[d]; - _slice_nblock[d]=nblock; - block = block*_rdimensions[d]; + + int block = 1; + int nblock = 1; + for (int d = 0; d < _ndimension; d++) + nblock *= _rdimensions[d]; + + for (int d = 0; d < _ndimension; d++) + { + nblock /= _rdimensions[d]; + _slice_block[d] = block; + _slice_stride[d] = _ostride[d] * _rdimensions[d]; + _slice_nblock[d] = nblock; + block = block * _rdimensions[d]; } //////////////////////////////////////////////// // Create a checkerboard lookup table //////////////////////////////////////////////// int rvol = 1; - for(int d=0;d<_ndimension;d++){ - rvol=rvol * _rdimensions[d]; + for (int d = 0; d < _ndimension; d++) + { + rvol = rvol * _rdimensions[d]; } _checker_board.resize(rvol); - for(int osite=0;osite<_osites;osite++){ - _checker_board[osite] = CheckerBoardFromOindex (osite); + for (int osite = 0; osite < _osites; osite++) + { + _checker_board[osite] = CheckerBoardFromOindex(osite); } - }; -protected: + + protected: virtual int oIndex(std::vector &coor) { - int idx=0; - for(int d=0;d<_ndimension;d++) { - if( d==_checker_dim ) { - idx+=_ostride[d]*((coor[d]/2)%_rdimensions[d]); - } else { - idx+=_ostride[d]*(coor[d]%_rdimensions[d]); - } + int idx = 0; + for (int d = 0; d < _ndimension; d++) + { + if (d == _checker_dim) + { + idx += _ostride[d] * ((coor[d] / 2) % _rdimensions[d]); + } + else + { + idx += _ostride[d] * (coor[d] % _rdimensions[d]); + } } return idx; }; - + virtual int iIndex(std::vector &lcoor) { - int idx=0; - for(int d=0;d<_ndimension;d++) { - if( d==_checker_dim ) { - idx+=_istride[d]*(lcoor[d]/(2*_rdimensions[d])); - } else { - idx+=_istride[d]*(lcoor[d]/_rdimensions[d]); - } - } - return idx; + int idx = 0; + for (int d = 0; d < _ndimension; d++) + { + if (d == _checker_dim) + { + idx += _istride[d] * (lcoor[d] / (2 * _rdimensions[d])); + } + else + { + idx += _istride[d] * (lcoor[d] / _rdimensions[d]); + } + } + return idx; } }; - } #endif diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc index 98d2abf4..956de0d2 100644 --- a/lib/communicator/Communicator_base.cc +++ b/lib/communicator/Communicator_base.cc @@ -26,6 +26,10 @@ Author: Peter Boyle *************************************************************************************/ /* END LEGAL */ #include +#include +#include +#include +#include namespace Grid { @@ -34,7 +38,10 @@ namespace Grid { /////////////////////////////////////////////////////////////// void * CartesianCommunicator::ShmCommBuf; uint64_t CartesianCommunicator::MAX_MPI_SHM_BYTES = 128*1024*1024; -CartesianCommunicator::CommunicatorPolicy_t CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent; +CartesianCommunicator::CommunicatorPolicy_t +CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent; +int CartesianCommunicator::nCommThreads = -1; +int CartesianCommunicator::Hugepages = 0; ///////////////////////////////// // Alloc, free shmem region @@ -60,6 +67,7 @@ void CartesianCommunicator::ShmBufferFreeAll(void) { ///////////////////////////////// // Grid information queries ///////////////////////////////// +int CartesianCommunicator::Dimensions(void) { return _ndimension; }; int CartesianCommunicator::IsBoss(void) { return _processor==0; }; int CartesianCommunicator::BossRank(void) { return 0; }; int CartesianCommunicator::ThisRank(void) { return _processor; }; @@ -88,24 +96,43 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N) GlobalSumVector((double *)c,2*N); } -#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPI3L) +#if !defined( GRID_COMMS_MPI3) int CartesianCommunicator::NodeCount(void) { return ProcessorCount();}; - -double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, - void *xmit, - int xmit_to_rank, - void *recv, - int recv_from_rank, - int bytes) +int CartesianCommunicator::RankCount(void) { return ProcessorCount();}; +#endif +#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPIT) +double CartesianCommunicator::StencilSendToRecvFrom( void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes, int dir) { + std::vector list; + // Discard the "dir" + SendToRecvFromBegin (list,xmit,xmit_to_rank,recv,recv_from_rank,bytes); + SendToRecvFromComplete(list); + return 2.0*bytes; +} +double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, + void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes, int dir) +{ + // Discard the "dir" SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes); return 2.0*bytes; } -void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &waitall) +void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &waitall,int dir) { SendToRecvFromComplete(waitall); } +#endif + +#if !defined( GRID_COMMS_MPI3) + void CartesianCommunicator::StencilBarrier(void){}; commVector CartesianCommunicator::ShmBufStorageVector; @@ -119,8 +146,22 @@ void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { return NULL; } void CartesianCommunicator::ShmInitGeneric(void){ +#if 1 + + int mmap_flag = MAP_SHARED | MAP_ANONYMOUS; +#ifdef MAP_HUGETLB + if ( Hugepages ) mmap_flag |= MAP_HUGETLB; +#endif + ShmCommBuf =(void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE, mmap_flag, -1, 0); + if (ShmCommBuf == (void *)MAP_FAILED) { + perror("mmap failed "); + exit(EXIT_FAILURE); + } +#else ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES); ShmCommBuf=(void *)&ShmBufStorageVector[0]; +#endif + bzero(ShmCommBuf,MAX_MPI_SHM_BYTES); } #endif diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h index e0b9f2c3..ac866ced 100644 --- a/lib/communicator/Communicator_base.h +++ b/lib/communicator/Communicator_base.h @@ -38,7 +38,7 @@ Author: Peter Boyle #ifdef GRID_COMMS_MPI3 #include #endif -#ifdef GRID_COMMS_MPI3L +#ifdef GRID_COMMS_MPIT #include #endif #ifdef GRID_COMMS_SHMEM @@ -50,12 +50,24 @@ namespace Grid { class CartesianCommunicator { public: - // 65536 ranks per node adequate for now + + //////////////////////////////////////////// + // Isend/Irecv/Wait, or Sendrecv blocking + //////////////////////////////////////////// + enum CommunicatorPolicy_t { CommunicatorPolicyConcurrent, CommunicatorPolicySequential }; + static CommunicatorPolicy_t CommunicatorPolicy; + static void SetCommunicatorPolicy(CommunicatorPolicy_t policy ) { CommunicatorPolicy = policy; } + + /////////////////////////////////////////// + // Up to 65536 ranks per node adequate for now // 128MB shared memory for comms enought for 48^4 local vol comms // Give external control (command line override?) of this - - static const int MAXLOG2RANKSPERNODE = 16; - static uint64_t MAX_MPI_SHM_BYTES; + /////////////////////////////////////////// + static const int MAXLOG2RANKSPERNODE = 16; + static uint64_t MAX_MPI_SHM_BYTES; + static int nCommThreads; + // use explicit huge pages + static int Hugepages; // Communicator should know nothing of the physics grid, only processor grid. int _Nprocessors; // How many in all @@ -64,14 +76,18 @@ class CartesianCommunicator { std::vector _processor_coor; // linear processor coordinate unsigned long _ndimension; -#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPI3L) +#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPIT) static MPI_Comm communicator_world; - MPI_Comm communicator; + + MPI_Comm communicator; + std::vector communicator_halo; + typedef MPI_Request CommsRequest_t; #else typedef int CommsRequest_t; #endif + //////////////////////////////////////////////////////////////////// // Helper functionality for SHM Windows common to all other impls //////////////////////////////////////////////////////////////////// @@ -117,11 +133,7 @@ class CartesianCommunicator { ///////////////////////////////// static void * ShmCommBuf; - // Isend/Irecv/Wait, or Sendrecv blocking - enum CommunicatorPolicy_t { CommunicatorPolicyConcurrent, CommunicatorPolicySequential }; - static CommunicatorPolicy_t CommunicatorPolicy; - static void SetCommunicatorPolicy(CommunicatorPolicy_t policy ) { CommunicatorPolicy = policy; } - + size_t heap_top; size_t heap_bytes; @@ -148,6 +160,7 @@ class CartesianCommunicator { int RankFromProcessorCoor(std::vector &coor); void ProcessorCoorFromRank(int rank,std::vector &coor); + int Dimensions(void) ; int IsBoss(void) ; int BossRank(void) ; int ThisRank(void) ; @@ -155,6 +168,7 @@ class CartesianCommunicator { const std::vector & ProcessorGrid(void) ; int ProcessorCount(void) ; int NodeCount(void) ; + int RankCount(void) ; //////////////////////////////////////////////////////////////////////////////// // very VERY rarely (Log, serial RNG) we need world without a grid @@ -175,6 +189,8 @@ class CartesianCommunicator { void GlobalSumVector(ComplexF *c,int N); void GlobalSum(ComplexD &c); void GlobalSumVector(ComplexD *c,int N); + void GlobalXOR(uint32_t &); + void GlobalXOR(uint64_t &); template void GlobalSum(obj &o){ typedef typename obj::scalar_type scalar_type; @@ -207,14 +223,21 @@ class CartesianCommunicator { void SendToRecvFromComplete(std::vector &waitall); + double StencilSendToRecvFrom(void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes,int dir); + double StencilSendToRecvFromBegin(std::vector &list, - void *xmit, - int xmit_to_rank, - void *recv, - int recv_from_rank, - int bytes); + void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes,int dir); - void StencilSendToRecvFromComplete(std::vector &waitall); + + void StencilSendToRecvFromComplete(std::vector &waitall,int i); void StencilBarrier(void); //////////////////////////////////////////////////////////// diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index 470a06c7..bd2a62fb 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -83,6 +83,14 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator); assert(ierr==0); } +void CartesianCommunicator::GlobalXOR(uint32_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalXOR(uint64_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator); + assert(ierr==0); +} void CartesianCommunicator::GlobalSum(float &f){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator); assert(ierr==0); diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc index a8bffc14..cb7fa390 100644 --- a/lib/communicator/Communicator_mpi3.cc +++ b/lib/communicator/Communicator_mpi3.cc @@ -37,11 +37,12 @@ Author: Peter Boyle #include #include #include -//#include -#ifndef SHM_HUGETLB -#define SHM_HUGETLB 04000 +#include +#ifdef HAVE_NUMAIF_H +#include #endif + namespace Grid { /////////////////////////////////////////////////////////////////////////////////////////////////// @@ -65,6 +66,7 @@ std::vector CartesianCommunicator::MyGroup; std::vector CartesianCommunicator::ShmCommBufs; int CartesianCommunicator::NodeCount(void) { return GroupSize;}; +int CartesianCommunicator::RankCount(void) { return WorldSize;}; #undef FORCE_COMMS @@ -209,10 +211,35 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { int fd=shm_open(shm_name,O_RDWR|O_CREAT,0666); if ( fd < 0 ) { perror("failed shm_open"); assert(0); } ftruncate(fd, size); + + int mmap_flag = MAP_SHARED; +#ifdef MAP_HUGETLB + if (Hugepages) mmap_flag |= MAP_HUGETLB; +#endif + void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, mmap_flag, fd, 0); - void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if ( ptr == MAP_FAILED ) { perror("failed mmap"); assert(0); } assert(((uint64_t)ptr&0x3F)==0); + +// Experiments; Experiments; Try to force numa domain on the shm segment if we have numaif.h +#if 0 +//#ifdef HAVE_NUMAIF_H + int status; + int flags=MPOL_MF_MOVE; +#ifdef KNL + int nodes=1; // numa domain == MCDRAM + // Find out if in SNC2,SNC4 mode ? +#else + int nodes=r; // numa domain == MPI ID +#endif + unsigned long count=1; + for(uint64_t page=0;page &processors) { int ierr; communicator=communicator_world; + _ndimension = processors.size(); + communicator_halo.resize (2*_ndimension); + for(int i=0;i<_ndimension*2;i++){ + MPI_Comm_dup(communicator,&communicator_halo[i]); + } + //////////////////////////////////////////////////////////////// // Assert power of two shm_size. //////////////////////////////////////////////////////////////// @@ -509,6 +546,14 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator); assert(ierr==0); } +void CartesianCommunicator::GlobalXOR(uint32_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalXOR(uint64_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator); + assert(ierr==0); +} void CartesianCommunicator::GlobalSum(float &f){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator); assert(ierr==0); @@ -590,13 +635,27 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector &lis } } -double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, - void *xmit, - int dest, - void *recv, - int from, - int bytes) +double CartesianCommunicator::StencilSendToRecvFrom( void *xmit, + int dest, + void *recv, + int from, + int bytes,int dir) { + std::vector list; + double offbytes = StencilSendToRecvFromBegin(list,xmit,dest,recv,from,bytes,dir); + StencilSendToRecvFromComplete(list,dir); + return offbytes; +} + +double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, + void *xmit, + int dest, + void *recv, + int from, + int bytes,int dir) +{ + assert(dir < communicator_halo.size()); + MPI_Request xrq; MPI_Request rrq; @@ -615,26 +674,26 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vectorStencilSendToRecvFromComplete(list); + this->StencilSendToRecvFromComplete(list,dir); } return off_node_bytes; } -void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &waitall) +void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &waitall,int dir) { SendToRecvFromComplete(waitall); } diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc new file mode 100644 index 00000000..eb6ef87d --- /dev/null +++ b/lib/communicator/Communicator_mpit.cc @@ -0,0 +1,286 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./lib/communicator/Communicator_mpi.cc + + Copyright (C) 2015 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include +#include +#include +#include + +namespace Grid { + + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Info that is setup once and indept of cartesian layout +/////////////////////////////////////////////////////////////////////////////////////////////////// +MPI_Comm CartesianCommunicator::communicator_world; + +// Should error check all MPI calls. +void CartesianCommunicator::Init(int *argc, char ***argv) { + int flag; + int provided; + MPI_Initialized(&flag); // needed to coexist with other libs apparently + if ( !flag ) { + MPI_Init_thread(argc,argv,MPI_THREAD_MULTIPLE,&provided); + if ( provided != MPI_THREAD_MULTIPLE ) { + QCD::WilsonKernelsStatic::Comms = QCD::WilsonKernelsStatic::CommsThenCompute; + } + } + MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world); + ShmInitGeneric(); +} + +CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +{ + _ndimension = processors.size(); + std::vector periodic(_ndimension,1); + + _Nprocessors=1; + _processors = processors; + _processor_coor.resize(_ndimension); + + MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator); + MPI_Comm_rank(communicator,&_processor); + MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); + + for(int i=0;i<_ndimension;i++){ + _Nprocessors*=_processors[i]; + } + + communicator_halo.resize (2*_ndimension); + for(int i=0;i<_ndimension*2;i++){ + MPI_Comm_dup(communicator,&communicator_halo[i]); + } + + int Size; + MPI_Comm_size(communicator,&Size); + + assert(Size==_Nprocessors); +} +void CartesianCommunicator::GlobalSum(uint32_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSum(uint64_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalXOR(uint32_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalXOR(uint64_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSum(float &f){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSumVector(float *f,int N) +{ + int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSum(double &d) +{ + int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalSumVector(double *d,int N) +{ + int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator); + assert(ierr==0); +} +void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest) +{ + int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest); + assert(ierr==0); +} +int CartesianCommunicator::RankFromProcessorCoor(std::vector &coor) +{ + int rank; + int ierr=MPI_Cart_rank (communicator, &coor[0], &rank); + assert(ierr==0); + return rank; +} +void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &coor) +{ + coor.resize(_ndimension); + int ierr=MPI_Cart_coords (communicator, rank, _ndimension,&coor[0]); + assert(ierr==0); +} + +// Basic Halo comms primitive +void CartesianCommunicator::SendToRecvFrom(void *xmit, + int dest, + void *recv, + int from, + int bytes) +{ + std::vector reqs(0); + SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes); + SendToRecvFromComplete(reqs); +} + +void CartesianCommunicator::SendRecvPacket(void *xmit, + void *recv, + int sender, + int receiver, + int bytes) +{ + MPI_Status stat; + assert(sender != receiver); + int tag = sender; + if ( _processor == sender ) { + MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator); + } + if ( _processor == receiver ) { + MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat); + } +} + +// Basic Halo comms primitive +void CartesianCommunicator::SendToRecvFromBegin(std::vector &list, + void *xmit, + int dest, + void *recv, + int from, + int bytes) +{ + int myrank = _processor; + int ierr; + if ( CommunicatorPolicy == CommunicatorPolicyConcurrent ) { + MPI_Request xrq; + MPI_Request rrq; + + ierr =MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq); + ierr|=MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq); + + assert(ierr==0); + list.push_back(xrq); + list.push_back(rrq); + } else { + // Give the CPU to MPI immediately; can use threads to overlap optionally + ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,dest,myrank, + recv,bytes,MPI_CHAR,from, from, + communicator,MPI_STATUS_IGNORE); + assert(ierr==0); + } +} +void CartesianCommunicator::SendToRecvFromComplete(std::vector &list) +{ + if ( CommunicatorPolicy == CommunicatorPolicyConcurrent ) { + int nreq=list.size(); + std::vector status(nreq); + int ierr = MPI_Waitall(nreq,&list[0],&status[0]); + assert(ierr==0); + } +} + +void CartesianCommunicator::Barrier(void) +{ + int ierr = MPI_Barrier(communicator); + assert(ierr==0); +} + +void CartesianCommunicator::Broadcast(int root,void* data, int bytes) +{ + int ierr=MPI_Bcast(data, + bytes, + MPI_BYTE, + root, + communicator); + assert(ierr==0); +} + /////////////////////////////////////////////////////// + // Should only be used prior to Grid Init finished. + // Check for this? + /////////////////////////////////////////////////////// +int CartesianCommunicator::RankWorld(void){ + int r; + MPI_Comm_rank(communicator_world,&r); + return r; +} +void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) +{ + int ierr= MPI_Bcast(data, + bytes, + MPI_BYTE, + root, + communicator_world); + assert(ierr==0); +} + +double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, + void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes,int dir) +{ + int myrank = _processor; + int ierr; + assert(dir < communicator_halo.size()); + + // std::cout << " sending on communicator "< &waitall,int dir) +{ + int nreq=waitall.size(); + MPI_Waitall(nreq, &waitall[0], MPI_STATUSES_IGNORE); +}; +double CartesianCommunicator::StencilSendToRecvFrom(void *xmit, + int xmit_to_rank, + void *recv, + int recv_from_rank, + int bytes,int dir) +{ + int myrank = _processor; + int ierr; + assert(dir < communicator_halo.size()); + + // std::cout << " sending on communicator "< #include #endif -#ifdef GRID_COMMS_MPI3L +#ifdef GRID_COMMS_MPIT #include #endif diff --git a/lib/lattice/Lattice_reduction.h b/lib/lattice/Lattice_reduction.h index 14234fe0..db012c8c 100644 --- a/lib/lattice/Lattice_reduction.h +++ b/lib/lattice/Lattice_reduction.h @@ -1,4 +1,4 @@ - /************************************************************************************* +/************************************************************************************* Grid physics library, www.github.com/paboyle/Grid Source file: ./lib/lattice/Lattice_reduction.h Copyright (C) 2015 @@ -369,71 +369,7 @@ static void sliceMaddVector(Lattice &R,std::vector &a,const Lattice } }; - /* -template -static void sliceMaddVectorSlow (Lattice &R,std::vector &a,const Lattice &X,const Lattice &Y, - int Orthog,RealD scale=1.0) -{ - // FIXME: Implementation is slow - // Best base the linear combination by constructing a - // set of vectors of size grid->_rdimensions[Orthog]. - typedef typename vobj::scalar_object sobj; - typedef typename vobj::scalar_type scalar_type; - typedef typename vobj::vector_type vector_type; - - int Nblock = X._grid->GlobalDimensions()[Orthog]; - - GridBase *FullGrid = X._grid; - GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog); - - Lattice Xslice(SliceGrid); - Lattice Rslice(SliceGrid); - // If we based this on Cshift it would work for spread out - // but it would be even slower - for(int i=0;i -static void sliceInnerProductVectorSlow( std::vector & vec, const Lattice &lhs,const Lattice &rhs,int Orthog) - { - // FIXME: Implementation is slow - // Look at localInnerProduct implementation, - // and do inside a site loop with block strided iterators - typedef typename vobj::scalar_object sobj; - typedef typename vobj::scalar_type scalar_type; - typedef typename vobj::vector_type vector_type; - typedef typename vobj::tensor_reduced scalar; - typedef typename scalar::scalar_object scomplex; - - int Nblock = lhs._grid->GlobalDimensions()[Orthog]; - vec.resize(Nblock); - std::vector sip(Nblock); - Lattice IP(lhs._grid); - IP=localInnerProduct(lhs,rhs); - sliceSum(IP,sip,Orthog); - - for(int ss=0;ss_rdimensions[Orthog]. -////////////////////////////////////////////////////////////////////////////////////////// - inline GridBase *makeSubSliceGrid(const GridBase *BlockSolverGrid,int Orthog) { int NN = BlockSolverGrid->_ndimension; @@ -452,7 +388,7 @@ inline GridBase *makeSubSliceGrid(const GridBase *BlockSolverGrid,int Or } return (GridBase *)new GridCartesian(latt_phys,simd_phys,mpi_phys); } - +*/ template static void sliceMaddMatrix (Lattice &R,Eigen::MatrixXcd &aa,const Lattice &X,const Lattice &Y,int Orthog,RealD scale=1.0) @@ -462,57 +398,170 @@ static void sliceMaddMatrix (Lattice &R,Eigen::MatrixXcd &aa,const Lattice typedef typename vobj::vector_type vector_type; int Nblock = X._grid->GlobalDimensions()[Orthog]; - + GridBase *FullGrid = X._grid; - GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog); - - Lattice Xslice(SliceGrid); - Lattice Rslice(SliceGrid); - - for(int i=0;i Xslice(SliceGrid); + // Lattice Rslice(SliceGrid); + + assert( FullGrid->_simd_layout[Orthog]==1); + int nh = FullGrid->_ndimension; + // int nl = SliceGrid->_ndimension; + int nl = nh-1; + + //FIXME package in a convenient iterator + //Should loop over a plane orthogonal to direction "Orthog" + int stride=FullGrid->_slice_stride[Orthog]; + int block =FullGrid->_slice_block [Orthog]; + int nblock=FullGrid->_slice_nblock[Orthog]; + int ostride=FullGrid->_ostride[Orthog]; +#pragma omp parallel + { + std::vector s_x(Nblock); + +#pragma omp for collapse(2) + for(int n=0;n +static void sliceMulMatrix (Lattice &R,Eigen::MatrixXcd &aa,const Lattice &X,int Orthog,RealD scale=1.0) +{ + typedef typename vobj::scalar_object sobj; + typedef typename vobj::scalar_type scalar_type; + typedef typename vobj::vector_type vector_type; + + int Nblock = X._grid->GlobalDimensions()[Orthog]; + + GridBase *FullGrid = X._grid; + // GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog); + // Lattice Xslice(SliceGrid); + // Lattice Rslice(SliceGrid); + + assert( FullGrid->_simd_layout[Orthog]==1); + int nh = FullGrid->_ndimension; + // int nl = SliceGrid->_ndimension; + int nl=1; + + //FIXME package in a convenient iterator + //Should loop over a plane orthogonal to direction "Orthog" + int stride=FullGrid->_slice_stride[Orthog]; + int block =FullGrid->_slice_block [Orthog]; + int nblock=FullGrid->_slice_nblock[Orthog]; + int ostride=FullGrid->_ostride[Orthog]; +#pragma omp parallel + { + std::vector s_x(Nblock); + +#pragma omp for collapse(2) + for(int n=0;n static void sliceInnerProductMatrix( Eigen::MatrixXcd &mat, const Lattice &lhs,const Lattice &rhs,int Orthog) { - // FIXME: Implementation is slow - // Not sure of best solution.. think about it typedef typename vobj::scalar_object sobj; typedef typename vobj::scalar_type scalar_type; typedef typename vobj::vector_type vector_type; GridBase *FullGrid = lhs._grid; - GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog); + // GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog); int Nblock = FullGrid->GlobalDimensions()[Orthog]; - Lattice Lslice(SliceGrid); - Lattice Rslice(SliceGrid); + // Lattice Lslice(SliceGrid); + // Lattice Rslice(SliceGrid); mat = Eigen::MatrixXcd::Zero(Nblock,Nblock); - - for(int i=0;i_simd_layout[Orthog]==1); + int nh = FullGrid->_ndimension; + // int nl = SliceGrid->_ndimension; + int nl = nh-1; + + //FIXME package in a convenient iterator + //Should loop over a plane orthogonal to direction "Orthog" + int stride=FullGrid->_slice_stride[Orthog]; + int block =FullGrid->_slice_block [Orthog]; + int nblock=FullGrid->_slice_nblock[Orthog]; + int ostride=FullGrid->_ostride[Orthog]; + + typedef typename vobj::vector_typeD vector_typeD; + +#pragma omp parallel + { + std::vector Left(Nblock); + std::vector Right(Nblock); + Eigen::MatrixXcd mat_thread = Eigen::MatrixXcd::Zero(Nblock,Nblock); + +#pragma omp for collapse(2) + for(int n=0;nGlobalSum(sum); + mat(i,j)=sum; + }} + return; } diff --git a/lib/lattice/Lattice_transfer.h b/lib/lattice/Lattice_transfer.h index 68de52d0..cbf31f86 100644 --- a/lib/lattice/Lattice_transfer.h +++ b/lib/lattice/Lattice_transfer.h @@ -551,7 +551,10 @@ void Replicate(Lattice &coarse,Lattice & fine) //Copy SIMD-vectorized lattice to array of scalar objects in lexicographic order template -typename std::enable_if::value && !isSIMDvectorized::value, void>::type unvectorizeToLexOrdArray(std::vector &out, const Lattice &in){ +typename std::enable_if::value && !isSIMDvectorized::value, void>::type +unvectorizeToLexOrdArray(std::vector &out, const Lattice &in) +{ + typedef typename vobj::vector_type vtype; GridBase* in_grid = in._grid; @@ -590,6 +593,54 @@ typename std::enable_if::value && !isSIMDvectorized extract1(in_vobj, out_ptrs, 0); } } +//Copy SIMD-vectorized lattice to array of scalar objects in lexicographic order +template +typename std::enable_if::value + && !isSIMDvectorized::value, void>::type +vectorizeFromLexOrdArray( std::vector &in, Lattice &out) +{ + + typedef typename vobj::vector_type vtype; + + GridBase* grid = out._grid; + assert(in.size()==grid->lSites()); + + int ndim = grid->Nd(); + int nsimd = vtype::Nsimd(); + + std::vector > icoor(nsimd); + + for(int lane=0; lane < nsimd; lane++){ + icoor[lane].resize(ndim); + grid->iCoorFromIindex(icoor[lane],lane); + } + + parallel_for(uint64_t oidx = 0; oidx < grid->oSites(); oidx++){ //loop over outer index + //Assemble vector of pointers to output elements + std::vector ptrs(nsimd); + + std::vector ocoor(ndim); + grid->oCoorFromOindex(ocoor, oidx); + + std::vector lcoor(grid->Nd()); + + for(int lane=0; lane < nsimd; lane++){ + + for(int mu=0;mu_rdimensions[mu]*icoor[lane][mu]; + } + + int lex; + Lexicographic::IndexFromCoor(lcoor, lex, grid->_ldimensions); + ptrs[lane] = &in[lex]; + } + + //pack from those ptrs + vobj vecobj; + merge1(vecobj, ptrs, 0); + out._odata[oidx] = vecobj; + } +} //Convert a Lattice from one precision to another template @@ -615,7 +666,7 @@ void precisionChange(Lattice &out, const Lattice &in){ std::vector in_slex_conv(in_grid->lSites()); unvectorizeToLexOrdArray(in_slex_conv, in); - parallel_for(int out_oidx=0;out_oidxoSites();out_oidx++){ + parallel_for(uint64_t out_oidx=0;out_oidxoSites();out_oidx++){ std::vector out_ocoor(ndim); out_grid->oCoorFromOindex(out_ocoor, out_oidx); diff --git a/lib/log/Log.cc b/lib/log/Log.cc index 69a9a0a8..65dc2812 100644 --- a/lib/log/Log.cc +++ b/lib/log/Log.cc @@ -95,7 +95,7 @@ void GridLogConfigure(std::vector &logstreams) { //////////////////////////////////////////////////////////// void Grid_quiesce_nodes(void) { int me = 0; -#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPI3L) +#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPIT) MPI_Comm_rank(MPI_COMM_WORLD, &me); #endif #ifdef GRID_COMMS_SHMEM diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h index afa7eb2e..d14f3fe2 100644 --- a/lib/parallelIO/BinaryIO.h +++ b/lib/parallelIO/BinaryIO.h @@ -29,16 +29,25 @@ #ifndef GRID_BINARY_IO_H #define GRID_BINARY_IO_H - -#include "IldgIOtypes.h" +#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPIT) +#define USE_MPI_IO +#else +#undef USE_MPI_IO +#endif #ifdef HAVE_ENDIAN_H #include #endif + #include #include +namespace Grid { + +///////////////////////////////////////////////////////////////////////////////// +// Byte reversal garbage +///////////////////////////////////////////////////////////////////////////////// inline uint32_t byte_reverse32(uint32_t f) { f = ((f&0xFF)<<24) | ((f&0xFF00)<<8) | ((f&0xFF0000)>>8) | ((f&0xFF000000UL)>>24) ; return f; @@ -60,18 +69,125 @@ inline uint64_t Grid_ntohll(uint64_t A) { } #endif -namespace Grid { - - // A little helper - inline void removeWhitespace(std::string &key) - { - key.erase(std::remove_if(key.begin(), key.end(), ::isspace),key.end()); - } +// A little helper +inline void removeWhitespace(std::string &key) +{ + key.erase(std::remove_if(key.begin(), key.end(), ::isspace),key.end()); +} +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Static class holding the parallel IO code +// Could just use a namespace +/////////////////////////////////////////////////////////////////////////////////////////////////// class BinaryIO { - public: + ///////////////////////////////////////////////////////////////////////////// + // more byte manipulation helpers + ///////////////////////////////////////////////////////////////////////////// + + template static inline void Uint32Checksum(Lattice &lat,uint32_t &nersc_csum) + { + typedef typename vobj::scalar_object sobj; + + GridBase *grid = lat._grid; + int lsites = grid->lSites(); + + std::vector scalardata(lsites); + unvectorizeToLexOrdArray(scalardata,lat); + + NerscChecksum(grid,scalardata,nersc_csum); + } + + template + static inline void NerscChecksum(GridBase *grid, std::vector &fbuf, uint32_t &nersc_csum) + { + const uint64_t size32 = sizeof(fobj) / sizeof(uint32_t); + + uint64_t lsites = grid->lSites(); + if (fbuf.size() == 1) + { + lsites = 1; + } + + #pragma omp parallel + { + uint32_t nersc_csum_thr = 0; + + #pragma omp for + for (uint64_t local_site = 0; local_site < lsites; local_site++) + { + uint32_t *site_buf = (uint32_t *)&fbuf[local_site]; + for (uint64_t j = 0; j < size32; j++) + { + nersc_csum_thr = nersc_csum_thr + site_buf[j]; + } + } + + #pragma omp critical + { + nersc_csum += nersc_csum_thr; + } + } + } + + template static inline void ScidacChecksum(GridBase *grid,std::vector &fbuf,uint32_t &scidac_csuma,uint32_t &scidac_csumb) + { + const uint64_t size32 = sizeof(fobj)/sizeof(uint32_t); + + + int nd = grid->_ndimension; + + uint64_t lsites =grid->lSites(); + if (fbuf.size()==1) { + lsites=1; + } + std::vector local_vol =grid->LocalDimensions(); + std::vector local_start =grid->LocalStarts(); + std::vector global_vol =grid->FullDimensions(); + +#pragma omp parallel + { + std::vector coor(nd); + uint32_t scidac_csuma_thr=0; + uint32_t scidac_csumb_thr=0; + uint32_t site_crc=0; + +#pragma omp for + for(uint64_t local_site=0;local_site>(32-gsite29); + scidac_csumb_thr ^= site_crc<>(32-gsite31); + } + +#pragma omp critical + { + scidac_csuma^= scidac_csuma_thr; + scidac_csumb^= scidac_csumb_thr; + } + } + } // Network is big endian static inline void htobe32_v(void *file_object,uint32_t bytes){ be32toh_v(file_object,bytes);} @@ -79,21 +195,22 @@ class BinaryIO { static inline void htole32_v(void *file_object,uint32_t bytes){ le32toh_v(file_object,bytes);} static inline void htole64_v(void *file_object,uint32_t bytes){ le64toh_v(file_object,bytes);} - static inline void be32toh_v(void *file_object,uint32_t bytes) + static inline void be32toh_v(void *file_object,uint64_t bytes) { uint32_t * f = (uint32_t *)file_object; - for(int i=0;i*sizeof(uint32_t)>8) | ((f&0xFF000000UL)>>24) ; @@ -102,21 +219,23 @@ class BinaryIO { } // BE is same as network - static inline void be64toh_v(void *file_object,uint32_t bytes) + static inline void be64toh_v(void *file_object,uint64_t bytes) { uint64_t * f = (uint64_t *)file_object; - for(int i=0;i*sizeof(uint64_t)>8) | ((f&0xFF000000UL)>>24) ; @@ -126,674 +245,465 @@ class BinaryIO { fp[i] = Grid_ntohll(g); } } + ///////////////////////////////////////////////////////////////////////////// + // Real action: + // Read or Write distributed lexico array of ANY object to a specific location in file + ////////////////////////////////////////////////////////////////////////////////////// - template static inline void Uint32Checksum(Lattice &lat,munger munge,uint32_t &csum) + static const int BINARYIO_MASTER_APPEND = 0x10; + static const int BINARYIO_UNORDERED = 0x08; + static const int BINARYIO_LEXICOGRAPHIC = 0x04; + static const int BINARYIO_READ = 0x02; + static const int BINARYIO_WRITE = 0x01; + + template + static inline void IOobject(word w, + GridBase *grid, + std::vector &iodata, + std::string file, + int offset, + const std::string &format, int control, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) { - typedef typename vobj::scalar_object sobj; - GridBase *grid = lat._grid ; - std::cout < lcoor; - for(int l=0;llSites();l++){ - Lexicographic::CoorFromIndex(lcoor,l,grid->_ldimensions); - peekLocalSite(siteObj,lat,lcoor); - munge(siteObj,fileObj,csum); - } - grid->GlobalSum(csum); - } + grid->Barrier(); + GridStopWatch timer; + GridStopWatch bstimer; - static inline void Uint32Checksum(uint32_t *buf,uint32_t buf_size_bytes,uint32_t &csum) - { - for(int i=0;i*sizeof(uint32_t)Dimensions(); + int nrank = grid->ProcessorCount(); + int myrank = grid->ThisRank(); + + std::vector psizes = grid->ProcessorGrid(); + std::vector pcoor = grid->ThisProcessorCoor(); + std::vector gLattice= grid->GlobalDimensions(); + std::vector lLattice= grid->LocalDimensions(); + + std::vector lStart(ndim); + std::vector gStart(ndim); + + // Flatten the file + uint64_t lsites = grid->lSites(); + if ( control & BINARYIO_MASTER_APPEND ) { + assert(iodata.size()==1); + } else { + assert(lsites==iodata.size()); } - } - - // Simple classes for precision conversion - template - struct BinarySimpleUnmunger { - typedef typename getPrecision::real_scalar_type fobj_stype; - typedef typename getPrecision::real_scalar_type sobj_stype; - - void operator()(sobj &in, fobj &out, uint32_t &csum) { - // take word by word and transform accoding to the status - fobj_stype *out_buffer = (fobj_stype *)&out; - sobj_stype *in_buffer = (sobj_stype *)∈ - size_t fobj_words = sizeof(out) / sizeof(fobj_stype); - size_t sobj_words = sizeof(in) / sizeof(sobj_stype); - assert(fobj_words == sobj_words); - - for (unsigned int word = 0; word < sobj_words; word++) - out_buffer[word] = in_buffer[word]; // type conversion on the fly - - BinaryIO::Uint32Checksum((uint32_t *)&out, sizeof(out), csum); + for(int d=0;d - struct BinarySimpleMunger { - typedef typename getPrecision::real_scalar_type fobj_stype; - typedef typename getPrecision::real_scalar_type sobj_stype; +#ifdef USE_MPI_IO + std::vector distribs(ndim,MPI_DISTRIBUTE_BLOCK); + std::vector dargs (ndim,MPI_DISTRIBUTE_DFLT_DARG); + MPI_Datatype mpiObject; + MPI_Datatype fileArray; + MPI_Datatype localArray; + MPI_Datatype mpiword; + MPI_Offset disp = offset; + MPI_File fh ; + MPI_Status status; + int numword; - void operator()(fobj &in, sobj &out, uint32_t &csum) { - // take word by word and transform accoding to the status - fobj_stype *in_buffer = (fobj_stype *)∈ - sobj_stype *out_buffer = (sobj_stype *)&out; - size_t fobj_words = sizeof(in) / sizeof(fobj_stype); - size_t sobj_words = sizeof(out) / sizeof(sobj_stype); - assert(fobj_words == sobj_words); - - for (unsigned int word = 0; word < sobj_words; word++) - out_buffer[word] = in_buffer[word]; // type conversion on the fly - - BinaryIO::Uint32Checksum((uint32_t *)&in, sizeof(in), csum); + if ( sizeof( word ) == sizeof(float ) ) { + numword = sizeof(fobj)/sizeof(float); + mpiword = MPI_FLOAT; + } else { + numword = sizeof(fobj)/sizeof(double); + mpiword = MPI_DOUBLE; } - }; - template - static inline uint32_t readObjectSerial(Lattice &Umu,std::string file,munger munge,int offset,const std::string &format) - { - typedef typename vobj::scalar_object sobj; + ////////////////////////////////////////////////////////////////////////////// + // Sobj in MPI phrasing + ////////////////////////////////////////////////////////////////////////////// + int ierr; + ierr = MPI_Type_contiguous(numword,mpiword,&mpiObject); assert(ierr==0); + ierr = MPI_Type_commit(&mpiObject); - GridBase *grid = Umu._grid; + ////////////////////////////////////////////////////////////////////////////// + // File global array data type + ////////////////////////////////////////////////////////////////////////////// + ierr=MPI_Type_create_subarray(ndim,&gLattice[0],&lLattice[0],&gStart[0],MPI_ORDER_FORTRAN, mpiObject,&fileArray); assert(ierr==0); + ierr=MPI_Type_commit(&fileArray); assert(ierr==0); - std::cout<< GridLogMessage<< "Serial read I/O "<< file<< std::endl; - GridStopWatch timer; timer.Start(); + ////////////////////////////////////////////////////////////////////////////// + // local lattice array + ////////////////////////////////////////////////////////////////////////////// + ierr=MPI_Type_create_subarray(ndim,&lLattice[0],&lLattice[0],&lStart[0],MPI_ORDER_FORTRAN, mpiObject,&localArray); assert(ierr==0); + ierr=MPI_Type_commit(&localArray); assert(ierr==0); +#endif + ////////////////////////////////////////////////////////////////////////////// + // Byte order + ////////////////////////////////////////////////////////////////////////////// int ieee32big = (format == std::string("IEEE32BIG")); int ieee32 = (format == std::string("IEEE32")); int ieee64big = (format == std::string("IEEE64BIG")); int ieee64 = (format == std::string("IEEE64")); - // Find the location of each site and send to primary node - // Take loop order from Chroma; defines loop order now that NERSC doc no longer - // available (how short sighted is that?) - std::ifstream fin(file,std::ios::binary|std::ios::in); - fin.seekg(offset); + ////////////////////////////////////////////////////////////////////////////// + // Do the I/O + ////////////////////////////////////////////////////////////////////////////// + if ( control & BINARYIO_READ ) { - Umu = zero; - uint32_t csum=0; - uint64_t bytes=0; - fobj file_object; - sobj munged; - - for(int t=0;t_fdimensions[3];t++){ - for(int z=0;z_fdimensions[2];z++){ - for(int y=0;y_fdimensions[1];y++){ - for(int x=0;x_fdimensions[0];x++){ + timer.Start(); - std::vector site({x,y,z,t}); - - if (grid->IsBoss()) { - fin.read((char *)&file_object, sizeof(file_object));assert( fin.fail()==0); - bytes += sizeof(file_object); - if (ieee32big) be32toh_v((void *)&file_object, sizeof(file_object)); - if (ieee32) le32toh_v((void *)&file_object, sizeof(file_object)); - if (ieee64big) be64toh_v((void *)&file_object, sizeof(file_object)); - if (ieee64) le64toh_v((void *)&file_object, sizeof(file_object)); - - munge(file_object, munged, csum); - } - // The boss who read the file has their value poked - pokeSite(munged,Umu,site); - }}}} - timer.Stop(); - std::cout<Broadcast(0,(void *)&csum,sizeof(csum)); - return csum; - } - - template - static inline uint32_t writeObjectSerial(Lattice &Umu,std::string file,munger munge,int offset, - const std::string & format) - { - typedef typename vobj::scalar_object sobj; - - GridBase *grid = Umu._grid; - - int ieee32big = (format == std::string("IEEE32BIG")); - int ieee32 = (format == std::string("IEEE32")); - int ieee64big = (format == std::string("IEEE64BIG")); - int ieee64 = (format == std::string("IEEE64")); - - ////////////////////////////////////////////////// - // Serialise through node zero - ////////////////////////////////////////////////// - std::cout<< GridLogMessage<< "Serial write I/O "<< file<IsBoss() ) { - fout.open(file,std::ios::binary|std::ios::out|std::ios::in); - fout.seekp(offset); - } - uint64_t bytes=0; - uint32_t csum=0; - fobj file_object; - sobj unmunged; - for(int t=0;t_fdimensions[3];t++){ - for(int z=0;z_fdimensions[2];z++){ - for(int y=0;y_fdimensions[1];y++){ - for(int x=0;x_fdimensions[0];x++){ - - std::vector site({x,y,z,t}); - // peek & write - peekSite(unmunged,Umu,site); - - munge(unmunged,file_object,csum); - - - if ( grid->IsBoss() ) { - if(ieee32big) htobe32_v((void *)&file_object,sizeof(file_object)); - if(ieee32) htole32_v((void *)&file_object,sizeof(file_object)); - if(ieee64big) htobe64_v((void *)&file_object,sizeof(file_object)); - if(ieee64) htole64_v((void *)&file_object,sizeof(file_object)); - - // NB could gather an xstrip as an optimisation. - fout.write((char *)&file_object,sizeof(file_object));assert( fout.fail()==0); - bytes+=sizeof(file_object); - } - }}}} - timer.Stop(); - std::cout<Broadcast(0,(void *)&csum,sizeof(csum)); - return csum; - } - - static inline uint32_t writeRNGSerial(GridSerialRNG &serial,GridParallelRNG ¶llel,std::string file,int offset) - { - typedef typename GridSerialRNG::RngStateType RngStateType; - const int RngStateCount = GridSerialRNG::RngStateCount; - - GridBase *grid = parallel._grid; - int gsites = grid->_gsites; - - GridStopWatch timer; timer.Start(); - ////////////////////////////////////////////////// - // Serialise through node zero - ////////////////////////////////////////////////// - std::ofstream fout; - if (grid->IsBoss()) { - fout.open(file, std::ios::binary | std::ios::out); - if (!fout.is_open()) { - std::cout << GridLogMessage << "writeRNGSerial: Error opening file " << file << std::endl; - exit(0);// write better error handling - } - fout.seekp(offset); - } - - std::cout << GridLogMessage << "Serial RNG write I/O on file " << file << std::endl; - uint32_t csum = 0; - std::vector saved(RngStateCount); - int bytes = sizeof(RngStateType) * saved.size(); - std::cout << GridLogDebug << "RngStateCount: " << RngStateCount << std::endl; - std::cout << GridLogDebug << "Type has " << bytes << " bytes" << std::endl; - std::vector gcoor; - - for(int gidx=0;gidxGlobalIndexToGlobalCoor(gidx,gcoor); - grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor); - int l_idx=parallel.generator_idx(o_idx,i_idx); - - if( rank == grid->ThisRank() ){ - // std::cout << "rank" << rank<<" Getting state for index "<Broadcast(rank, (void *)&saved[0], bytes); - - if ( grid->IsBoss() ) { - Uint32Checksum((uint32_t *)&saved[0],bytes,csum); - fout.write((char *)&saved[0],bytes);assert( fout.fail()==0); - } - - } - - if ( grid->IsBoss() ) { - serial.GetState(saved,0); - Uint32Checksum((uint32_t *)&saved[0],bytes,csum); - fout.write((char *)&saved[0],bytes);assert( fout.fail()==0); - } - - grid->Broadcast(0, (void *)&csum, sizeof(csum)); - - if (grid->IsBoss()) - fout.close(); - - timer.Stop(); - - std::cout << GridLogMessage << "RNG file checksum " << std::hex << csum << std::dec << std::endl; - std::cout << GridLogMessage << "RNG state saved in " << timer.Elapsed() << std::endl; - return csum; - } - - - static inline uint32_t readRNGSerial(GridSerialRNG &serial,GridParallelRNG ¶llel,std::string file,int offset) - { - typedef typename GridSerialRNG::RngStateType RngStateType; - const int RngStateCount = GridSerialRNG::RngStateCount; - - GridBase *grid = parallel._grid; - int gsites = grid->_gsites; - - ////////////////////////////////////////////////// - // Serialise through node zero - ////////////////////////////////////////////////// - std::cout<< GridLogMessage<< "Serial RNG read I/O of file "<IsBoss()) { - fin.open(file, std::ios::binary | std::ios::in); - if (!fin.is_open()) { - std::cout << GridLogMessage << "readRNGSerial: Error opening file " << file << std::endl; - exit(0);// write better error handling - } - fin.seekg(offset); - } - - - uint32_t csum=0; - std::vector saved(RngStateCount); - int bytes = sizeof(RngStateType)*saved.size(); - std::cout << GridLogDebug << "RngStateCount: " << RngStateCount << std::endl; - std::cout << GridLogDebug << "Type has " << bytes << " bytes" << std::endl; - std::vector gcoor; - - std::cout << GridLogDebug << "gsites: " << gsites << " loop" << std::endl; - for(int gidx=0;gidxGlobalIndexToGlobalCoor(gidx,gcoor); - grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor); - int l_idx=parallel.generator_idx(o_idx,i_idx); - //std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx - // << " i_idx " << i_idx << " rank " << rank << std::endl; - - if ( grid->IsBoss() ) { - fin.read((char *)&saved[0],bytes);assert( fin.fail()==0); - Uint32Checksum((uint32_t *)&saved[0],bytes,csum); - } - - grid->Broadcast(0,(void *)&saved[0],bytes); - - if( rank == grid->ThisRank() ){ - parallel.SetState(saved,l_idx); - } - } - - if ( grid->IsBoss() ) { - fin.read((char *)&saved[0],bytes);assert( fin.fail()==0); - serial.SetState(saved,0); - Uint32Checksum((uint32_t *)&saved[0],bytes,csum); - } - - std::cout << GridLogMessage << "RNG file checksum " << std::hex << csum << std::dec << std::endl; - - grid->Broadcast(0,(void *)&csum,sizeof(csum)); - - return csum; - } - - - template - static inline uint32_t readObjectParallel(Lattice &Umu, - std::string file, - munger munge, - int offset, - const std::string &format, - ILDGtype ILDG = ILDGtype()) { - typedef typename vobj::scalar_object sobj; - - GridBase *grid = Umu._grid; - - int ieee32big = (format == std::string("IEEE32BIG")); - int ieee32 = (format == std::string("IEEE32")); - int ieee64big = (format == std::string("IEEE64BIG")); - int ieee64 = (format == std::string("IEEE64")); - - - // Take into account block size of parallel file systems want about - // 4-16MB chunks. - // Ideally one reader/writer per xy plane and read these contiguously - // with comms from nominated I/O nodes. - std::ifstream fin; - - int nd = grid->_ndimension; - std::vector parallel(nd,1); - std::vector ioproc (nd); - std::vector start(nd); - std::vector range(nd); - - for(int d=0;dCheckerBoarded(d) == 0); - } - - uint64_t slice_vol = 1; - - int IOnode = 1; - for(int d=0;d_ndimension;d++) { - - if ( d == 0 ) parallel[d] = 0; - if (parallel[d]) { - range[d] = grid->_ldimensions[d]; - start[d] = grid->_processor_coor[d]*range[d]; - ioproc[d]= grid->_processor_coor[d]; + if ( (control & BINARYIO_LEXICOGRAPHIC) && (nrank > 1) ) { +#ifdef USE_MPI_IO + std::cout<< GridLogMessage<< "MPI read I/O "<< file<< std::endl; + ierr=MPI_File_open(grid->communicator,(char *) file.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &fh); assert(ierr==0); + ierr=MPI_File_set_view(fh, disp, mpiObject, fileArray, "native", MPI_INFO_NULL); assert(ierr==0); + ierr=MPI_File_read_all(fh, &iodata[0], 1, localArray, &status); assert(ierr==0); + MPI_File_close(&fh); + MPI_Type_free(&fileArray); + MPI_Type_free(&localArray); +#else + assert(0); +#endif } else { - range[d] = grid->_gdimensions[d]; - start[d] = 0; - ioproc[d]= 0; - - if ( grid->_processor_coor[d] != 0 ) IOnode = 0; - } - slice_vol = slice_vol * range[d]; - } - - { - uint32_t tmp = IOnode; - grid->GlobalSum(tmp); - std::cout<< std::dec ; - std::cout<< GridLogMessage<< "Parallel read I/O to "<< file << " with " <_ndimension;d++){ - std::cout<< range[d]; - if( d< grid->_ndimension-1 ) - std::cout<< " x "; - } - std::cout << std::endl; - } - - GridStopWatch timer; timer.Start(); - uint64_t bytes=0; - - int myrank = grid->ThisRank(); - int iorank = grid->RankFromProcessorCoor(ioproc); - - if (!ILDG.is_ILDG) - if ( IOnode ) { - fin.open(file,std::ios::binary|std::ios::in); - } - - ////////////////////////////////////////////////////////// - // Find the location of each site and send to primary node - // Take loop order from Chroma; defines loop order now that NERSC doc no longer - // available (how short sighted is that?) - ////////////////////////////////////////////////////////// - Umu = zero; - static uint32_t csum; csum=0;//static for SHMEM - - fobj fileObj; - static sobj siteObj; // Static to place in symmetric region for SHMEM - - // need to implement these loops in Nd independent way with a lexico conversion - for(int tlex=0;tlex tsite(nd); // temporary mixed up site - std::vector gsite(nd); - std::vector lsite(nd); - std::vector iosite(nd); - - Lexicographic::CoorFromIndex(tsite,tlex,range); - - for(int d=0;d_ldimensions[d]; // local site - gsite[d] = tsite[d]+start[d]; // global site - } - - - ///////////////////////// - // Get the rank of owner of data - ///////////////////////// - int rank, o_idx,i_idx, g_idx; - grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gsite); - grid->GlobalCoorToGlobalIndex(gsite,g_idx); - - //////////////////////////////// - // iorank reads from the seek - //////////////////////////////// - if (myrank == iorank) { - - - if (ILDG.is_ILDG){ - // use C-LIME to populate the record - #ifdef HAVE_LIME - uint64_t sizeFO = sizeof(fileObj); - limeReaderSeek(ILDG.LR, g_idx*sizeFO, SEEK_SET); - int status = limeReaderReadData((void *)&fileObj, &sizeFO, ILDG.LR); - #endif - } else{ - fin.seekg(offset+g_idx*sizeof(fileObj)); - fin.read((char *)&fileObj,sizeof(fileObj)); + std::cout << GridLogMessage << "C++ read I/O " << file << " : " + << iodata.size() * sizeof(fobj) << " bytes" << std::endl; + std::ifstream fin; + fin.open(file, std::ios::binary | std::ios::in); + if (control & BINARYIO_MASTER_APPEND) + { + fin.seekg(-sizeof(fobj), fin.end); } - bytes+=sizeof(fileObj); - - if(ieee32big) be32toh_v((void *)&fileObj,sizeof(fileObj)); - if(ieee32) le32toh_v((void *)&fileObj,sizeof(fileObj)); - if(ieee64big) be64toh_v((void *)&fileObj,sizeof(fileObj)); - if(ieee64) le64toh_v((void *)&fileObj,sizeof(fileObj)); - - munge(fileObj,siteObj,csum); - - } - - // Possibly do transport through pt2pt - if ( rank != iorank ) { - if ( (myrank == rank) || (myrank==iorank) ) { - grid->SendRecvPacket((void *)&siteObj,(void *)&siteObj,iorank,rank,sizeof(siteObj)); - } + else + { + fin.seekg(offset + myrank * lsites * sizeof(fobj)); + } + fin.read((char *)&iodata[0], iodata.size() * sizeof(fobj)); + assert(fin.fail() == 0); + fin.close(); } - // Poke at destination - if ( myrank == rank ) { - pokeLocalSite(siteObj,Umu,lsite); - } - grid->Barrier(); // necessary? + timer.Stop(); + + grid->Barrier(); + + bstimer.Start(); + ScidacChecksum(grid,iodata,scidac_csuma,scidac_csumb); + if (ieee32big) be32toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee32) le32toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64big) be64toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64) le64toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + NerscChecksum(grid,iodata,nersc_csum); + bstimer.Stop(); } + + if ( control & BINARYIO_WRITE ) { - grid->GlobalSum(csum); - grid->GlobalSum(bytes); + bstimer.Start(); + NerscChecksum(grid,iodata,nersc_csum); + if (ieee32big) htobe32_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee32) htole32_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64big) htobe64_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64) htole64_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + ScidacChecksum(grid,iodata,scidac_csuma,scidac_csumb); + bstimer.Stop(); + + grid->Barrier(); + + timer.Start(); + if ( (control & BINARYIO_LEXICOGRAPHIC) && (nrank > 1) ) { +#ifdef USE_MPI_IO + std::cout << GridLogMessage << "MPI write I/O " << file << std::endl; + ierr = MPI_File_open(grid->communicator, (char *)file.c_str(), MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fh); + std::cout << GridLogMessage << "Checking for errors" << std::endl; + if (ierr != MPI_SUCCESS) + { + char error_string[BUFSIZ]; + int length_of_error_string, error_class; + + MPI_Error_class(ierr, &error_class); + MPI_Error_string(error_class, error_string, &length_of_error_string); + fprintf(stderr, "%3d: %s\n", myrank, error_string); + MPI_Error_string(ierr, error_string, &length_of_error_string); + fprintf(stderr, "%3d: %s\n", myrank, error_string); + MPI_Abort(MPI_COMM_WORLD, 1); //assert(ierr == 0); + } + + std::cout << GridLogDebug << "MPI read I/O set view " << file << std::endl; + ierr = MPI_File_set_view(fh, disp, mpiObject, fileArray, "native", MPI_INFO_NULL); + assert(ierr == 0); + + std::cout << GridLogDebug << "MPI read I/O write all " << file << std::endl; + ierr = MPI_File_write_all(fh, &iodata[0], 1, localArray, &status); + assert(ierr == 0); + + MPI_File_close(&fh); + MPI_Type_free(&fileArray); + MPI_Type_free(&localArray); +#else + assert(0); +#endif + } else { + + std::ofstream fout; + fout.exceptions ( std::fstream::failbit | std::fstream::badbit ); + try { + fout.open(file,std::ios::binary|std::ios::out|std::ios::in); + } catch (const std::fstream::failure& exc) { + std::cout << GridLogError << "Error in opening the file " << file << " for output" <Barrier(); + grid->GlobalSum(nersc_csum); + grid->GlobalXOR(scidac_csuma); + grid->GlobalXOR(scidac_csumb); + grid->Barrier(); + } + } + + ///////////////////////////////////////////////////////////////////////////// + // Read a Lattice of object + ////////////////////////////////////////////////////////////////////////////////////// + template + static inline void readLatticeObject(Lattice &Umu, + std::string file, + munger munge, + int offset, + const std::string &format, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) + { + typedef typename vobj::scalar_object sobj; + typedef typename vobj::Realified::scalar_type word; word w=0; + + GridBase *grid = Umu._grid; + int lsites = grid->lSites(); + + std::vector scalardata(lsites); + std::vector iodata(lsites); // Munge, checksum, byte order in here + + IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); + + GridStopWatch timer; + timer.Start(); + + parallel_for(int x=0;xBarrier(); timer.Stop(); - std::cout< - static inline uint32_t writeObjectParallel(Lattice &Umu, - std::string file, munger munge, - int offset, - const std::string &format, - ILDGtype ILDG = ILDGtype()) { + ///////////////////////////////////////////////////////////////////////////// + // Write a Lattice of object + ////////////////////////////////////////////////////////////////////////////////////// + template + static inline void writeLatticeObject(Lattice &Umu, + std::string file, + munger munge, + int offset, + const std::string &format, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) + { typedef typename vobj::scalar_object sobj; + typedef typename vobj::Realified::scalar_type word; word w=0; GridBase *grid = Umu._grid; + int lsites = grid->lSites(); - int ieee32big = (format == std::string("IEEE32BIG")); - int ieee32 = (format == std::string("IEEE32")); - int ieee64big = (format == std::string("IEEE64BIG")); - int ieee64 = (format == std::string("IEEE64")); + std::vector scalardata(lsites); + std::vector iodata(lsites); // Munge, checksum, byte order in here - if (!(ieee32big || ieee32 || ieee64big || ieee64)) { - std::cout << GridLogError << "Unrecognized file format " << format - << std::endl; - std::cout << GridLogError - << "Allowed: IEEE32BIG | IEEE32 | IEEE64BIG | IEEE64" - << std::endl; - exit(0); + ////////////////////////////////////////////////////////////////////////////// + // Munge [ .e.g 3rd row recon ] + ////////////////////////////////////////////////////////////////////////////// + GridStopWatch timer; timer.Start(); + unvectorizeToLexOrdArray(scalardata,Umu); + + parallel_for(int x=0;xBarrier(); + timer.Stop(); + + IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); + + std::cout< RNGstate; + typedef RngStateType word; word w=0; + + std::string format = "IEEE32BIG"; + + GridBase *grid = parallel._grid; + int gsites = grid->gSites(); + int lsites = grid->lSites(); + + uint32_t nersc_csum_tmp = 0; + uint32_t scidac_csuma_tmp = 0; + uint32_t scidac_csumb_tmp = 0; + + GridStopWatch timer; + + std::cout << GridLogMessage << "RNG read I/O on file " << file << std::endl; + + std::vector iodata(lsites); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); + + timer.Start(); + parallel_for(int lidx=0;lidx tmp(RngStateCount); + std::copy(iodata[lidx].begin(),iodata[lidx].end(),tmp.begin()); + parallel.SetState(tmp,lidx); } + timer.Stop(); - int nd = grid->_ndimension; - for (int d = 0; d < nd; d++) { - assert(grid->CheckerBoarded(d) == 0); - } - - std::vector parallel(nd, 1); - std::vector ioproc(nd); - std::vector start(nd); - std::vector range(nd); - - uint64_t slice_vol = 1; - - int IOnode = 1; - - for (int d = 0; d < grid->_ndimension; d++) { - if (d != grid->_ndimension - 1) parallel[d] = 0; - - if (parallel[d]) { - range[d] = grid->_ldimensions[d]; - start[d] = grid->_processor_coor[d]*range[d]; - ioproc[d]= grid->_processor_coor[d]; - } else { - range[d] = grid->_gdimensions[d]; - start[d] = 0; - ioproc[d]= 0; - - if ( grid->_processor_coor[d] != 0 ) IOnode = 0; - } - - slice_vol = slice_vol * range[d]; - } + iodata.resize(1); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_MASTER_APPEND, + nersc_csum_tmp,scidac_csuma_tmp,scidac_csumb_tmp); { - uint32_t tmp = IOnode; - grid->GlobalSum(tmp); - std::cout<< GridLogMessage<< "Parallel write I/O from "<< file - << " with " <_ndimension;d++){ - std::cout<< range[d]; - if( d< grid->_ndimension-1 ) - std::cout<< " x "; - } - std::cout << std::endl; + std::vector tmp(RngStateCount); + std::copy(iodata[0].begin(),iodata[0].end(),tmp.begin()); + serial.SetState(tmp,0); } - + + nersc_csum = nersc_csum + nersc_csum_tmp; + scidac_csuma = scidac_csuma ^ scidac_csuma_tmp; + scidac_csumb = scidac_csumb ^ scidac_csumb_tmp; + + std::cout << GridLogMessage << "RNG file nersc_checksum " << std::hex << nersc_csum << std::dec << std::endl; + std::cout << GridLogMessage << "RNG file scidac_checksuma " << std::hex << scidac_csuma << std::dec << std::endl; + std::cout << GridLogMessage << "RNG file scidac_checksumb " << std::hex << scidac_csumb << std::dec << std::endl; + + std::cout << GridLogMessage << "RNG state overhead " << timer.Elapsed() << std::endl; + } + ///////////////////////////////////////////////////////////////////////////// + // Write a RNG; lexico map to an array of state and use IOobject + ////////////////////////////////////////////////////////////////////////////////////// + static inline void writeRNG(GridSerialRNG &serial, + GridParallelRNG ¶llel, + std::string file, + int offset, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) + { + typedef typename GridSerialRNG::RngStateType RngStateType; + typedef RngStateType word; word w=0; + const int RngStateCount = GridSerialRNG::RngStateCount; + typedef std::array RNGstate; + + GridBase *grid = parallel._grid; + int gsites = grid->gSites(); + int lsites = grid->lSites(); + + uint32_t nersc_csum_tmp; + uint32_t scidac_csuma_tmp; + uint32_t scidac_csumb_tmp; + GridStopWatch timer; + std::string format = "IEEE32BIG"; + + std::cout << GridLogMessage << "RNG write I/O on file " << file << std::endl; + timer.Start(); - uint64_t bytes=0; - - int myrank = grid->ThisRank(); - int iorank = grid->RankFromProcessorCoor(ioproc); - - // Take into account block size of parallel file systems want about - // 4-16MB chunks. - // Ideally one reader/writer per xy plane and read these contiguously - // with comms from nominated I/O nodes. - std::ofstream fout; - if (!ILDG.is_ILDG) - if (IOnode){ - fout.open(file, std::ios::binary | std::ios::in | std::ios::out); - if (!fout.is_open()) { - std::cout << GridLogMessage << "writeObjectParallel: Error opening file " << file - << std::endl; - exit(0); - } - } - - - ////////////////////////////////////////////////////////// - // Find the location of each site and send to primary node - // Take loop order from Chroma; defines loop order now that NERSC doc no - // longer - // available (how short sighted is that?) - ////////////////////////////////////////////////////////// - - uint32_t csum = 0; - fobj fileObj; - static sobj siteObj; // static for SHMEM target; otherwise dynamic allocate - // with AlignedAllocator - - // should aggregate a whole chunk and then write. - // need to implement these loops in Nd independent way with a lexico - // conversion - for (int tlex = 0; tlex < slice_vol; tlex++) { - std::vector tsite(nd); // temporary mixed up site - std::vector gsite(nd); - std::vector lsite(nd); - std::vector iosite(nd); - - Lexicographic::CoorFromIndex(tsite, tlex, range); - - for(int d = 0;d < nd; d++){ - lsite[d] = tsite[d] % grid->_ldimensions[d]; // local site - gsite[d] = tsite[d] + start[d]; // global site - } - - ///////////////////////// - // Get the rank of owner of data - ///////////////////////// - int rank, o_idx, i_idx, g_idx; - grid->GlobalCoorToRankIndex(rank, o_idx, i_idx, gsite); - grid->GlobalCoorToGlobalIndex(gsite, g_idx); - - //////////////////////////////// - // iorank writes from the seek - //////////////////////////////// - - // Owner of data peeks it - peekLocalSite(siteObj, Umu, lsite); - - // Pair of nodes may need to do pt2pt send - if ( rank != iorank ) { // comms is necessary - if ( (myrank == rank) || (myrank==iorank) ) { // and we have to do it - // Send to IOrank - grid->SendRecvPacket((void *)&siteObj,(void *)&siteObj,rank,iorank,sizeof(siteObj)); - } - } - - grid->Barrier(); // necessary? - - if (myrank == iorank) { - munge(siteObj, fileObj, csum); - - if (ieee32big) htobe32_v((void *)&fileObj, sizeof(fileObj)); - if (ieee32) htole32_v((void *)&fileObj, sizeof(fileObj)); - if (ieee64big) htobe64_v((void *)&fileObj, sizeof(fileObj)); - if (ieee64) htole64_v((void *)&fileObj, sizeof(fileObj)); - - - if (ILDG.is_ILDG) { - #ifdef HAVE_LIME - uint64_t sizeFO = sizeof(fileObj); - limeWriterSeek(ILDG.LW, g_idx*sizeFO, SEEK_SET); - int status = limeWriteRecordData((void *)&fileObj, &sizeFO, ILDG.LW); - #endif - } - - else { - fout.seekp(offset + g_idx * sizeof(fileObj)); - fout.write((char *)&fileObj, sizeof(fileObj));assert( fout.fail()==0); - } - bytes += sizeof(fileObj); - } + std::vector iodata(lsites); + parallel_for(int lidx=0;lidx tmp(RngStateCount); + parallel.GetState(tmp,lidx); + std::copy(tmp.begin(),tmp.end(),iodata[lidx].begin()); } - - grid->GlobalSum(csum); - grid->GlobalSum(bytes); - timer.Stop(); - std::cout << GridLogPerformance << "writeObjectParallel: wrote " << bytes - << " bytes in " << timer.Elapsed() << " " - << (double)bytes / timer.useconds() << " MB/s " << std::endl; + IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); + iodata.resize(1); + { + std::vector tmp(RngStateCount); + serial.GetState(tmp,0); + std::copy(tmp.begin(),tmp.end(),iodata[0].begin()); + } + IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_MASTER_APPEND, + nersc_csum_tmp,scidac_csuma_tmp,scidac_csumb_tmp); - grid->Barrier(); // necessary? - if (IOnode) - fout.close(); - - - return csum; + nersc_csum = nersc_csum + nersc_csum_tmp; + scidac_csuma = scidac_csuma ^ scidac_csuma_tmp; + scidac_csumb = scidac_csumb ^ scidac_csumb_tmp; + + std::cout << GridLogMessage << "RNG file checksum " << std::hex << nersc_csum << std::dec << std::endl; + std::cout << GridLogMessage << "RNG file checksuma " << std::hex << scidac_csuma << std::dec << std::endl; + std::cout << GridLogMessage << "RNG file checksumb " << std::hex << scidac_csumb << std::dec << std::endl; + std::cout << GridLogMessage << "RNG state overhead " << timer.Elapsed() << std::endl; } }; } - #endif diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h index 0912e2f6..17ce4a06 100644 --- a/lib/parallelIO/IldgIO.h +++ b/lib/parallelIO/IldgIO.h @@ -27,6 +27,7 @@ directory #ifndef GRID_ILDG_IO_H #define GRID_ILDG_IO_H +#ifdef HAVE_LIME #include #include #include @@ -37,213 +38,677 @@ directory #include #include -#ifdef HAVE_LIME - -extern "C" { // for linkage +//C-Lime is a must have for this functionality +extern "C" { #include "lime.h" } namespace Grid { namespace QCD { -inline void ILDGGrid(GridBase *grid, ILDGField &header) { - assert(grid->_ndimension == 4); // emit error if not - header.dimension.resize(4); - header.boundary.resize(4); - for (int d = 0; d < 4; d++) { - header.dimension[d] = grid->_fdimensions[d]; - // Read boundary conditions from ... ? - header.boundary[d] = std::string("periodic"); - } -} + ///////////////////////////////// + // Encode word types as strings + ///////////////////////////////// + template inline std::string ScidacWordMnemonic(void){ return std::string("unknown"); } + template<> inline std::string ScidacWordMnemonic (void){ return std::string("D"); } + template<> inline std::string ScidacWordMnemonic (void){ return std::string("F"); } + template<> inline std::string ScidacWordMnemonic< int32_t>(void){ return std::string("I32_t"); } + template<> inline std::string ScidacWordMnemonic(void){ return std::string("U32_t"); } + template<> inline std::string ScidacWordMnemonic< int64_t>(void){ return std::string("I64_t"); } + template<> inline std::string ScidacWordMnemonic(void){ return std::string("U64_t"); } -inline void ILDGChecksum(uint32_t *buf, uint32_t buf_size_bytes, - uint32_t &csum) { - BinaryIO::Uint32Checksum(buf, buf_size_bytes, csum); -} + ///////////////////////////////////////// + // Encode a generic tensor as a string + ///////////////////////////////////////// + template std::string ScidacRecordTypeString(int &colors, int &spins, int & typesize,int &datacount) { -////////////////////////////////////////////////////////////////////// -// Utilities ; these are QCD aware -////////////////////////////////////////////////////////////////////// -template -inline void ILDGStatistics(GaugeField &data, ILDGField &header) { - // How to convert data precision etc... - header.link_trace = Grid::QCD::WilsonLoops::linkTrace(data); - header.plaquette = Grid::QCD::WilsonLoops::avgPlaquette(data); - // header.polyakov = -} + typedef typename getPrecision::real_scalar_type stype; -// Forcing QCD here -template -struct ILDGMunger { - void operator()(fobj &in, sobj &out, uint32_t &csum) { - for (int mu = 0; mu < 4; mu++) { - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - } - } - } - ILDGChecksum((uint32_t *)&in, sizeof(in), csum); - }; -}; + int _ColourN = indexRank(); + int _ColourScalar = isScalar(); + int _ColourVector = isVector(); + int _ColourMatrix = isMatrix(); -template -struct ILDGUnmunger { - void operator()(sobj &in, fobj &out, uint32_t &csum) { - for (int mu = 0; mu < 4; mu++) { - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - } - } - } - ILDGChecksum((uint32_t *)&out, sizeof(out), csum); - }; -}; + int _SpinN = indexRank(); + int _SpinScalar = isScalar(); + int _SpinVector = isVector(); + int _SpinMatrix = isMatrix(); -//////////////////////////////////////////////////////////////////////////////// -// Write and read from fstream; compute header offset for payload -//////////////////////////////////////////////////////////////////////////////// -enum ILDGstate {ILDGread, ILDGwrite}; + int _LorentzN = indexRank(); + int _LorentzScalar = isScalar(); + int _LorentzVector = isVector(); + int _LorentzMatrix = isMatrix(); -class ILDGIO : public BinaryIO { - FILE *File; - LimeWriter *LimeW; - LimeRecordHeader *LimeHeader; - LimeReader *LimeR; - std::string filename; + std::stringstream stream; + + stream << "GRID_"; + stream << ScidacWordMnemonic(); + + // std::cout << " Lorentz N/S/V/M : " << _LorentzN<<" "<<_LorentzScalar<<"/"<<_LorentzVector<<"/"<<_LorentzMatrix< std::string ScidacRecordTypeString(Lattice & lat,int &colors, int &spins, int & typesize,int &datacount) { + return ScidacRecordTypeString(colors,spins,typesize,datacount); + }; + + + //////////////////////////////////////////////////////////// + // Helper to fill out metadata + //////////////////////////////////////////////////////////// + template void ScidacMetaData(Lattice & field, + FieldMetaData &header, + scidacRecord & _scidacRecord, + scidacFile & _scidacFile) + { + typedef typename getPrecision::real_scalar_type stype; + + ///////////////////////////////////// + // Pull Grid's metadata + ///////////////////////////////////// + PrepareMetaData(field,header); + + ///////////////////////////////////// + // Scidac Private File structure + ///////////////////////////////////// + _scidacFile = scidacFile(field._grid); + + ///////////////////////////////////// + // Scidac Private Record structure + ///////////////////////////////////// + scidacRecord sr; + sr.datatype = ScidacRecordTypeString(field,sr.colors,sr.spins,sr.typesize,sr.datacount); + sr.date = header.creation_date; + sr.precision = ScidacWordMnemonic(); + sr.recordtype = GRID_IO_FIELD; + + _scidacRecord = sr; + + std::cout << GridLogMessage << "Build SciDAC datatype " < + void readLimeLatticeBinaryObject(Lattice &field,std::string record_name) + { + typedef typename vobj::scalar_object sobj; + scidacChecksum scidacChecksum_; + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + + std::string format = getFormatString(); + + while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { + + std::cout << GridLogMessage << limeReaderType(LimeR) < munge; + BinaryIO::readLatticeObject< sobj, sobj >(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); + + ///////////////////////////////////////////// + // Insist checksum is next record + ///////////////////////////////////////////// + readLimeObject(scidacChecksum_,std::string("scidacChecksum"),record_name); + + ///////////////////////////////////////////// + // Verify checksums + ///////////////////////////////////////////// + scidacChecksumVerify(scidacChecksum_,scidac_csuma,scidac_csumb); + return; + } } } + //////////////////////////////////////////// + // Read a generic serialisable object + //////////////////////////////////////////// + template + void readLimeObject(serialisable_object &object,std::string object_name,std::string record_name) + { + std::string xmlstring; + // should this be a do while; can we miss a first record?? + while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { - ~ILDGIO() { fclose(File); } + uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration) - int createHeader(std::string message, int MB, int ME, size_t PayloadSize, LimeWriter* L){ + if ( strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) ) ) { + std::vector xmlc(nbytes+1,'\0'); + limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR); + XmlReader RD(&xmlc[0],""); + read(RD,object_name,object); + return; + } + + } + assert(0); + } +}; + +class GridLimeWriter : public BinaryIO { + public: + /////////////////////////////////////////////////// + // FIXME: format for RNG? Now just binary out instead + /////////////////////////////////////////////////// + + FILE *File; + LimeWriter *LimeW; + std::string filename; + + void open(std::string &_filename) { + filename= _filename; + File = fopen(filename.c_str(), "w"); + LimeW = limeCreateWriter(File); assert(LimeW != NULL ); + } + ///////////////////////////////////////////// + // Close the file + ///////////////////////////////////////////// + void close(void) { + fclose(File); + // limeDestroyWriter(LimeW); + } + /////////////////////////////////////////////////////// + // Lime utility functions + /////////////////////////////////////////////////////// + int createLimeRecordHeader(std::string message, int MB, int ME, size_t PayloadSize) + { LimeRecordHeader *h; h = limeCreateHeader(MB, ME, const_cast(message.c_str()), PayloadSize); - int status = limeWriteRecordHeader(h, L); - if (status < 0) { - std::cerr << "ILDG Header error\n"; - return status; - } + assert(limeWriteRecordHeader(h, LimeW) >= 0); limeDestroyHeader(h); return LIME_SUCCESS; } + //////////////////////////////////////////// + // Write a generic serialisable object + //////////////////////////////////////////// + template + void writeLimeObject(int MB,int ME,serialisable_object &object,std::string object_name,std::string record_name) + { + std::string xmlstring; + { + XmlWriter WR("",""); + write(WR,object_name,object); + xmlstring = WR.XmlString(); + } + uint64_t nbytes = xmlstring.size(); + int err; + LimeRecordHeader *h = limeCreateHeader(MB, ME,(char *)record_name.c_str(), nbytes); assert(h!= NULL); - unsigned int writeHeader(ILDGField &header) { - // write header in LIME - n_uint64_t nbytes; - int MB_flag = 1, ME_flag = 0; + err=limeWriteRecordHeader(h, LimeW); assert(err>=0); + err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0); + err=limeWriterCloseRecord(LimeW); assert(err>=0); + limeDestroyHeader(h); + } + //////////////////////////////////////////// + // Write a generic lattice field and csum + //////////////////////////////////////////// + template + void writeLimeLatticeBinaryObject(Lattice &field,std::string record_name) + { + //////////////////////////////////////////// + // Create record header + //////////////////////////////////////////// + typedef typename vobj::scalar_object sobj; + int err; + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + uint64_t PayloadSize = sizeof(sobj) * field._grid->_gsites; + createLimeRecordHeader(record_name, 0, 0, PayloadSize); - char message[] = "ildg-format"; - nbytes = strlen(message); - LimeHeader = limeCreateHeader(MB_flag, ME_flag, message, nbytes); - limeWriteRecordHeader(LimeHeader, LimeW); - limeDestroyHeader(LimeHeader); - // save the xml header here - // use the xml_writer to c++ streams in pugixml - // and convert to char message - limeWriteRecordData(message, &nbytes, LimeW); - limeWriterCloseRecord(LimeW); + //////////////////////////////////////////////////////////////////// + // NB: FILE and iostream are jointly writing disjoint sequences in the + // the same file through different file handles (integer units). + // + // These are both buffered, so why I think this code is right is as follows. + // + // i) write record header to FILE *File, telegraphing the size. + // ii) ftell reads the offset from FILE *File . + // iii) iostream / MPI Open independently seek this offset. Write sequence direct to disk. + // Closes iostream and flushes. + // iv) fseek on FILE * to end of this disjoint section. + // v) Continue writing scidac record. + //////////////////////////////////////////////////////////////////// + off_t offset = ftell(File); + std::string format = getFormatString(); + BinarySimpleMunger munge; + BinaryIO::writeLatticeObject(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); + err=limeWriterCloseRecord(LimeW); assert(err>=0); + //////////////////////////////////////// + // Write checksum element, propagaing forward from the BinaryIO + // Always pair a checksum with a binary object, and close message + //////////////////////////////////////// + scidacChecksum checksum; + std::stringstream streama; streama << std::hex << scidac_csuma; + std::stringstream streamb; streamb << std::hex << scidac_csumb; + checksum.suma= streama.str(); + checksum.sumb= streamb.str(); + std::cout << GridLogMessage<<" writing scidac checksums "< + void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile) + { + scidacFile _scidacFile(grid); + writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML)); + writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML)); + } + //////////////////////////////////////////////// + // Write generic lattice field in scidac format + //////////////////////////////////////////////// + template + void writeScidacFieldRecord(Lattice &field,userRecord _userRecord) + { + typedef typename vobj::scalar_object sobj; + uint64_t nbytes; + GridBase * grid = field._grid; + + //////////////////////////////////////// + // fill the Grid header + //////////////////////////////////////// + FieldMetaData header; + scidacRecord _scidacRecord; + scidacFile _scidacFile; + + ScidacMetaData(field,header,_scidacRecord,_scidacFile); + + ////////////////////////////////////////////// + // Fill the Lime file record by record + ////////////////////////////////////////////// + writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message + writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML)); + writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML)); + writeLimeLatticeBinaryObject(field,std::string(ILDG_BINARY_DATA)); // Closes message with checksum + } +}; + +class IldgWriter : public ScidacWriter { + public: + + /////////////////////////////////// + // A little helper + /////////////////////////////////// + void writeLimeIldgLFN(std::string &LFN) + { + uint64_t PayloadSize = LFN.size(); + int err; + createLimeRecordHeader(ILDG_DATA_LFN, 0 , 0, PayloadSize); + err=limeWriteRecordData(const_cast(LFN.c_str()), &PayloadSize,LimeW); assert(err>=0); + err=limeWriterCloseRecord(LimeW); assert(err>=0); } + //////////////////////////////////////////////////////////////// + // Special ILDG operations ; gauge configs only. + // Don't require scidac records EXCEPT checksum + // Use Grid MetaData object if present. + //////////////////////////////////////////////////////////////// template - uint32_t readConfiguration(Lattice > &Umu) { - typedef Lattice > GaugeField; - typedef LorentzColourMatrixD sobjd; - typedef LorentzColourMatrixF sobjf; - typedef iLorentzColourMatrix itype; - typedef LorentzColourMatrix sobj; - GridBase *grid = Umu._grid; - - ILDGField header; - readHeader(header); - - // now just the conf, ignore the header - std::string format = std::string("IEEE64BIG"); - do {limeReaderNextRecord(LimeR);} - while (strncmp(limeReaderType(LimeR), "ildg-binary-data",16)); - - n_uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration) - - - ILDGtype ILDGt(true, LimeR); - // this is special for double prec data, just for the moment - uint32_t csum = BinaryIO::readObjectParallel< itype, sobjd >( - Umu, filename, ILDGMunger(), 0, format, ILDGt); - - // Check configuration - // todo - - return csum; - } - - template - uint32_t writeConfiguration(Lattice > &Umu, std::string format) { + void writeConfiguration(Lattice > &Umu,int sequence,std::string LFN,std::string description) + { + GridBase * grid = Umu._grid; typedef Lattice > GaugeField; typedef iLorentzColourMatrix vobj; typedef typename vobj::scalar_object sobj; - typedef LorentzColourMatrixD fobj; - ILDGField header; - // fill the header - header.floating_point = format; + uint64_t nbytes; - ILDGUnmunger munge; - unsigned int offset = writeHeader(header); + //////////////////////////////////////// + // fill the Grid header + //////////////////////////////////////// + FieldMetaData header; + scidacRecord _scidacRecord; + scidacFile _scidacFile; - BinaryIO::Uint32Checksum(Umu, munge, header.checksum); + ScidacMetaData(Umu,header,_scidacRecord,_scidacFile); - // Write data record header - n_uint64_t PayloadSize = sizeof(fobj) * Umu._grid->_gsites; - createHeader("ildg-binary-data", 0, 1, PayloadSize, LimeW); + std::string format = header.floating_point; + header.ensemble_id = description; + header.ensemble_label = description; + header.sequence_number = sequence; + header.ildg_lfn = LFN; - ILDGtype ILDGt(true, LimeW); - uint32_t csum = BinaryIO::writeObjectParallel( - Umu, filename, munge, 0, header.floating_point, ILDGt); + assert ( (format == std::string("IEEE32BIG")) + ||(format == std::string("IEEE64BIG")) ); - limeWriterCloseRecord(LimeW); + ////////////////////////////////////////////////////// + // Fill ILDG header data struct + ////////////////////////////////////////////////////// + ildgFormat ildgfmt ; + ildgfmt.field = std::string("su3gauge"); - // Last record - // the logical file name LNF - // look into documentation on how to generate this string - std::string LNF = "empty"; + if ( format == std::string("IEEE32BIG") ) { + ildgfmt.precision = 32; + } else { + ildgfmt.precision = 64; + } + ildgfmt.version = 1.0; + ildgfmt.lx = header.dimension[0]; + ildgfmt.ly = header.dimension[1]; + ildgfmt.lz = header.dimension[2]; + ildgfmt.lt = header.dimension[3]; + assert(header.nd==4); + assert(header.nd==header.dimension.size()); + ////////////////////////////////////////////////////////////////////////////// + // Fill the USQCD info field + ////////////////////////////////////////////////////////////////////////////// + usqcdInfo info; + info.version=1.0; + info.plaq = header.plaquette; + info.linktr = header.link_trace; - PayloadSize = sizeof(LNF); - createHeader("ildg-binary-lfn", 1 , 1, PayloadSize, LimeW); - limeWriteRecordData(const_cast(LNF.c_str()), &PayloadSize, LimeW); - - limeWriterCloseRecord(LimeW); - - return csum; + std::cout << GridLogMessage << " Writing config; IldgIO "< + void readConfiguration(Lattice > &Umu, FieldMetaData &FieldMetaData_) { + + typedef Lattice > GaugeField; + typedef typename GaugeField::vector_object vobj; + typedef typename vobj::scalar_object sobj; + + typedef LorentzColourMatrixF fobj; + typedef LorentzColourMatrixD dobj; + + GridBase *grid = Umu._grid; + + std::vector dims = Umu._grid->FullDimensions(); + + assert(dims.size()==4); + + // Metadata holders + ildgFormat ildgFormat_ ; + std::string ildgLFN_ ; + scidacChecksum scidacChecksum_; + usqcdInfo usqcdInfo_ ; + + // track what we read from file + int found_ildgFormat =0; + int found_ildgLFN =0; + int found_scidacChecksum=0; + int found_usqcdInfo =0; + int found_ildgBinary =0; + int found_FieldMetaData =0; + + uint32_t nersc_csum; + uint32_t scidac_csuma; + uint32_t scidac_csumb; + + // Binary format + std::string format; + + ////////////////////////////////////////////////////////////////////////// + // Loop over all records + // -- Order is poorly guaranteed except ILDG header preceeds binary section. + // -- Run like an event loop. + // -- Impose trust hierarchy. Grid takes precedence & look for ILDG, and failing + // that Scidac. + // -- Insist on Scidac checksum record. + ////////////////////////////////////////////////////////////////////////// + + while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { + + uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration) + + ////////////////////////////////////////////////////////////////// + // If not BINARY_DATA read a string and parse + ////////////////////////////////////////////////////////////////// + if ( strncmp(limeReaderType(LimeR), ILDG_BINARY_DATA,strlen(ILDG_BINARY_DATA) ) ) { + + // Copy out the string + std::vector xmlc(nbytes+1,'\0'); + limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR); + std::cout << GridLogMessage<< "Non binary record :" < munge; + BinaryIO::readLatticeObject< vobj, dobj >(Umu, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); + } else { + GaugeSimpleMunger munge; + BinaryIO::readLatticeObject< vobj, fobj >(Umu, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); + } + + found_ildgBinary = 1; + } + + } + + ////////////////////////////////////////////////////// + // Minimally must find binary segment and checksum + // Since this is an ILDG reader require ILDG format + ////////////////////////////////////////////////////// + assert(found_ildgBinary); + assert(found_ildgFormat); + assert(found_scidacChecksum); + + // Must find something with the lattice dimensions + assert(found_FieldMetaData||found_ildgFormat); + + if ( found_FieldMetaData ) { + + std::cout << GridLogMessage<<"Grid MetaData was record found: configuration was probably written by Grid ! Yay ! "<1.1416 16 16 32 0 +//////////////////////// +struct scidacFile : Serializable { public: - // header strings (not in order) - std::vector dimension; - std::vector boundary; - int data_start; - std::string hdr_version; - std::string storage_format; - // Checks on data - double link_trace; - double plaquette; - uint32_t checksum; - unsigned int sequence_number; - std::string data_type; - std::string ensemble_id; - std::string ensemble_label; - std::string creator; - std::string creator_hardware; - std::string creation_date; - std::string archive_date; - std::string floating_point; -}; -} -#else -namespace Grid { + GRID_SERIALIZABLE_CLASS_MEMBERS(scidacFile, + double, version, + int, spacetime, + std::string, dims, // must convert to int + int, volfmt); -struct ILDGtype { - bool is_ILDG; - ILDGtype() : is_ILDG(false) {} -}; -} + std::vector getDimensions(void) { + std::stringstream stream(dims); + std::vector dimensions; + int n; + while(stream >> n){ + dimensions.push_back(n); + } + return dimensions; + } + void setDimensions(std::vector dimensions) { + char delimiter = ' '; + std::stringstream stream; + for(int i=0;i_ndimension; + setDimensions(grid->FullDimensions()); + volfmt = GRID_IO_SINGLEFILE; + } + +}; + +/////////////////////////////////////////////////////////////////////// +// scidac-private-record-xml : example +// +// 1.1Tue Jul 26 21:14:44 2011 UTC0 +// QDP_D3_ColorMatrixD34 +// 1444 +// +/////////////////////////////////////////////////////////////////////// + +struct scidacRecord : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(scidacRecord, + double, version, + std::string, date, + int, recordtype, + std::string, datatype, + std::string, precision, + int, colors, + int, spins, + int, typesize, + int, datacount); + + scidacRecord() { version =1.0; } + +}; + +//////////////////////// +// ILDG format +//////////////////////// +struct ildgFormat : Serializable { +public: + GRID_SERIALIZABLE_CLASS_MEMBERS(ildgFormat, + double, version, + std::string, field, + int, precision, + int, lx, + int, ly, + int, lz, + int, lt); + ildgFormat() { version=1.0; }; +}; +//////////////////////// +// USQCD info +//////////////////////// +struct usqcdInfo : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdInfo, + double, version, + double, plaq, + double, linktr, + std::string, info); + usqcdInfo() { + version=1.0; + }; +}; +//////////////////////// +// Scidac Checksum +//////////////////////// +struct scidacChecksum : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(scidacChecksum, + double, version, + std::string, suma, + std::string, sumb); + scidacChecksum() { + version=1.0; + }; +}; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Type: scidac-file-xml MILC ILDG archival gauge configuration +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Type: +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +//////////////////////// +// Scidac private file xml +// 1.1416 16 16 32 0 +//////////////////////// + +#if 0 +//////////////////////////////////////////////////////////////////////////////////////// +// From http://www.physics.utah.edu/~detar/scidac/qio_2p3.pdf +//////////////////////////////////////////////////////////////////////////////////////// +struct usqcdPropFile : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdPropFile, + double, version, + std::string, type, + std::string, info); + usqcdPropFile() { + version=1.0; + }; +}; +struct usqcdSourceInfo : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdSourceInfo, + double, version, + std::string, info); + usqcdSourceInfo() { + version=1.0; + }; +}; +struct usqcdPropInfo : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdPropInfo, + double, version, + int, spin, + int, color, + std::string, info); + usqcdPropInfo() { + version=1.0; + }; +}; +#endif + +} #endif #endif diff --git a/lib/parallelIO/MetaData.h b/lib/parallelIO/MetaData.h new file mode 100644 index 00000000..6d45d0a5 --- /dev/null +++ b/lib/parallelIO/MetaData.h @@ -0,0 +1,325 @@ +/************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./lib/parallelIO/NerscIO.h + + Copyright (C) 2015 + + + Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory +*************************************************************************************/ +/* END LEGAL */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Grid { + + /////////////////////////////////////////////////////// + // Precision mapping + /////////////////////////////////////////////////////// + template static std::string getFormatString (void) + { + std::string format; + typedef typename getPrecision::real_scalar_type stype; + if ( sizeof(stype) == sizeof(float) ) { + format = std::string("IEEE32BIG"); + } + if ( sizeof(stype) == sizeof(double) ) { + format = std::string("IEEE64BIG"); + } + return format; + } + //////////////////////////////////////////////////////////////////////////////// + // header specification/interpretation + //////////////////////////////////////////////////////////////////////////////// + class FieldMetaData : Serializable { + public: + + GRID_SERIALIZABLE_CLASS_MEMBERS(FieldMetaData, + int, nd, + std::vector, dimension, + std::vector, boundary, + int, data_start, + std::string, hdr_version, + std::string, storage_format, + double, link_trace, + double, plaquette, + uint32_t, checksum, + uint32_t, scidac_checksuma, + uint32_t, scidac_checksumb, + unsigned int, sequence_number, + std::string, data_type, + std::string, ensemble_id, + std::string, ensemble_label, + std::string, ildg_lfn, + std::string, creator, + std::string, creator_hardware, + std::string, creation_date, + std::string, archive_date, + std::string, floating_point); + FieldMetaData(void) { + nd=4; + dimension.resize(4); + boundary.resize(4); + } + }; + + + + namespace QCD { + + using namespace Grid; + + + ////////////////////////////////////////////////////////////////////// + // Bit and Physical Checksumming and QA of data + ////////////////////////////////////////////////////////////////////// + inline void GridMetaData(GridBase *grid,FieldMetaData &header) + { + int nd = grid->_ndimension; + header.nd = nd; + header.dimension.resize(nd); + header.boundary.resize(nd); + for(int d=0;d_fdimensions[d]; + } + for(int d=0;dpw_name); + + // When + std::time_t t = std::time(nullptr); + std::tm tm_ = *std::localtime(&t); + std::ostringstream oss; + // oss << std::put_time(&tm_, "%c %Z"); + header.creation_date = oss.str(); + header.archive_date = header.creation_date; + + // What + struct utsname name; uname(&name); + header.creator_hardware = std::string(name.nodename)+"-"; + header.creator_hardware+= std::string(name.machine)+"-"; + header.creator_hardware+= std::string(name.sysname)+"-"; + header.creator_hardware+= std::string(name.release); + } + +#define dump_meta_data(field, s) \ + s << "BEGIN_HEADER" << std::endl; \ + s << "HDR_VERSION = " << field.hdr_version << std::endl; \ + s << "DATATYPE = " << field.data_type << std::endl; \ + s << "STORAGE_FORMAT = " << field.storage_format << std::endl; \ + for(int i=0;i<4;i++){ \ + s << "DIMENSION_" << i+1 << " = " << field.dimension[i] << std::endl ; \ + } \ + s << "LINK_TRACE = " << std::setprecision(10) << field.link_trace << std::endl; \ + s << "PLAQUETTE = " << std::setprecision(10) << field.plaquette << std::endl; \ + for(int i=0;i<4;i++){ \ + s << "BOUNDARY_"< inline void PrepareMetaData(Lattice & field, FieldMetaData &header) +{ + GridBase *grid = field._grid; + std::string format = getFormatString(); + header.floating_point = format; + header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac + GridMetaData(grid,header); + MachineCharacteristics(header); + } + inline void GaugeStatistics(Lattice & data,FieldMetaData &header) + { + // How to convert data precision etc... + header.link_trace=Grid::QCD::WilsonLoops::linkTrace(data); + header.plaquette =Grid::QCD::WilsonLoops::avgPlaquette(data); + } + inline void GaugeStatistics(Lattice & data,FieldMetaData &header) + { + // How to convert data precision etc... + header.link_trace=Grid::QCD::WilsonLoops::linkTrace(data); + header.plaquette =Grid::QCD::WilsonLoops::avgPlaquette(data); + } + template<> inline void PrepareMetaData(Lattice & field, FieldMetaData &header) + { + + GridBase *grid = field._grid; + std::string format = getFormatString(); + header.floating_point = format; + header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac + GridMetaData(grid,header); + GaugeStatistics(field,header); + MachineCharacteristics(header); + } + template<> inline void PrepareMetaData(Lattice & field, FieldMetaData &header) + { + GridBase *grid = field._grid; + std::string format = getFormatString(); + header.floating_point = format; + header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac + GridMetaData(grid,header); + GaugeStatistics(field,header); + MachineCharacteristics(header); + } + + ////////////////////////////////////////////////////////////////////// + // Utilities ; these are QCD aware + ////////////////////////////////////////////////////////////////////// + inline void reconstruct3(LorentzColourMatrix & cm) + { + const int x=0; + const int y=1; + const int z=2; + for(int mu=0;mu using iLorentzColour2x3 = iVector, 2>, Nd >; + + typedef iLorentzColour2x3 LorentzColour2x3; + typedef iLorentzColour2x3 LorentzColour2x3F; + typedef iLorentzColour2x3 LorentzColour2x3D; + +///////////////////////////////////////////////////////////////////////////////// +// Simple classes for precision conversion +///////////////////////////////////////////////////////////////////////////////// +template +struct BinarySimpleUnmunger { + typedef typename getPrecision::real_scalar_type fobj_stype; + typedef typename getPrecision::real_scalar_type sobj_stype; + + void operator()(sobj &in, fobj &out) { + // take word by word and transform accoding to the status + fobj_stype *out_buffer = (fobj_stype *)&out; + sobj_stype *in_buffer = (sobj_stype *)∈ + size_t fobj_words = sizeof(out) / sizeof(fobj_stype); + size_t sobj_words = sizeof(in) / sizeof(sobj_stype); + assert(fobj_words == sobj_words); + + for (unsigned int word = 0; word < sobj_words; word++) + out_buffer[word] = in_buffer[word]; // type conversion on the fly + + } +}; + +template +struct BinarySimpleMunger { + typedef typename getPrecision::real_scalar_type fobj_stype; + typedef typename getPrecision::real_scalar_type sobj_stype; + + void operator()(fobj &in, sobj &out) { + // take word by word and transform accoding to the status + fobj_stype *in_buffer = (fobj_stype *)∈ + sobj_stype *out_buffer = (sobj_stype *)&out; + size_t fobj_words = sizeof(in) / sizeof(fobj_stype); + size_t sobj_words = sizeof(out) / sizeof(sobj_stype); + assert(fobj_words == sobj_words); + + for (unsigned int word = 0; word < sobj_words; word++) + out_buffer[word] = in_buffer[word]; // type conversion on the fly + + } +}; + + + template + struct GaugeSimpleMunger{ + void operator()(fobj &in, sobj &out) { + for (int mu = 0; mu < Nd; mu++) { + for (int i = 0; i < Nc; i++) { + for (int j = 0; j < Nc; j++) { + out(mu)()(i, j) = in(mu)()(i, j); + }} + } + }; + }; + + template + struct GaugeSimpleUnmunger { + + void operator()(sobj &in, fobj &out) { + for (int mu = 0; mu < Nd; mu++) { + for (int i = 0; i < Nc; i++) { + for (int j = 0; j < Nc; j++) { + out(mu)()(i, j) = in(mu)()(i, j); + }} + } + }; + }; + + template + struct Gauge3x2munger{ + void operator() (fobj &in,sobj &out){ + for(int mu=0;mu + struct Gauge3x2unmunger{ + void operator() (sobj &in,fobj &out){ + for(int mu=0;mu -#include -#include -#include -#include - -#include -#include -#include - namespace Grid { namespace QCD { using namespace Grid; - //////////////////////////////////////////////////////////////////////////////// - // Some data types for intermediate storage - //////////////////////////////////////////////////////////////////////////////// - template using iLorentzColour2x3 = iVector, 2>, 4 >; - - typedef iLorentzColour2x3 LorentzColour2x3; - typedef iLorentzColour2x3 LorentzColour2x3F; - typedef iLorentzColour2x3 LorentzColour2x3D; - - //////////////////////////////////////////////////////////////////////////////// - // header specification/interpretation - //////////////////////////////////////////////////////////////////////////////// - class NerscField { - public: - // header strings (not in order) - int dimension[4]; - std::string boundary[4]; - int data_start; - std::string hdr_version; - std::string storage_format; - // Checks on data - double link_trace; - double plaquette; - uint32_t checksum; - unsigned int sequence_number; - std::string data_type; - std::string ensemble_id ; - std::string ensemble_label ; - std::string creator ; - std::string creator_hardware ; - std::string creation_date ; - std::string archive_date ; - std::string floating_point; - }; - - ////////////////////////////////////////////////////////////////////// - // Bit and Physical Checksumming and QA of data - ////////////////////////////////////////////////////////////////////// - - inline void NerscGrid(GridBase *grid,NerscField &header) - { - assert(grid->_ndimension==4); - for(int d=0;d<4;d++) { - header.dimension[d] = grid->_fdimensions[d]; - } - for(int d=0;d<4;d++) { - header.boundary[d] = std::string("PERIODIC"); - } - } - template - inline void NerscStatistics(GaugeField & data,NerscField &header) - { - // How to convert data precision etc... - header.link_trace=Grid::QCD::WilsonLoops::linkTrace(data); - header.plaquette =Grid::QCD::WilsonLoops::avgPlaquette(data); - } - - inline void NerscMachineCharacteristics(NerscField &header) - { - // Who - struct passwd *pw = getpwuid (getuid()); - if (pw) header.creator = std::string(pw->pw_name); - - // When - std::time_t t = std::time(nullptr); - std::tm tm = *std::localtime(&t); - std::ostringstream oss; - // oss << std::put_time(&tm, "%c %Z"); - header.creation_date = oss.str(); - header.archive_date = header.creation_date; - - // What - struct utsname name; uname(&name); - header.creator_hardware = std::string(name.nodename)+"-"; - header.creator_hardware+= std::string(name.machine)+"-"; - header.creator_hardware+= std::string(name.sysname)+"-"; - header.creator_hardware+= std::string(name.release); - - } - ////////////////////////////////////////////////////////////////////// - // Utilities ; these are QCD aware - ////////////////////////////////////////////////////////////////////// - inline void NerscChecksum(uint32_t *buf,uint32_t buf_size_bytes,uint32_t &csum) - { - BinaryIO::Uint32Checksum(buf,buf_size_bytes,csum); - } - inline void reconstruct3(LorentzColourMatrix & cm) - { - const int x=0; - const int y=1; - const int z=2; - for(int mu=0;mu<4;mu++){ - cm(mu)()(2,x) = adj(cm(mu)()(0,y)*cm(mu)()(1,z)-cm(mu)()(0,z)*cm(mu)()(1,y)); //x= yz-zy - cm(mu)()(2,y) = adj(cm(mu)()(0,z)*cm(mu)()(1,x)-cm(mu)()(0,x)*cm(mu)()(1,z)); //y= zx-xz - cm(mu)()(2,z) = adj(cm(mu)()(0,x)*cm(mu)()(1,y)-cm(mu)()(0,y)*cm(mu)()(1,x)); //z= xy-yx - } - } - - template - struct NerscSimpleMunger{ - void operator()(fobj &in, sobj &out, uint32_t &csum) { - for (int mu = 0; mu < Nd; mu++) { - for (int i = 0; i < Nc; i++) { - for (int j = 0; j < Nc; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - } - } - } - NerscChecksum((uint32_t *)&in, sizeof(in), csum); - }; - }; - - template - struct NerscSimpleUnmunger { - void operator()(sobj &in, fobj &out, uint32_t &csum) { - for (int mu = 0; mu < Nd; mu++) { - for (int i = 0; i < Nc; i++) { - for (int j = 0; j < Nc; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - } - } - } - NerscChecksum((uint32_t *)&out, sizeof(out), csum); - }; - }; - - template - struct Nersc3x2munger{ - void operator() (fobj &in,sobj &out,uint32_t &csum){ - - NerscChecksum((uint32_t *)&in,sizeof(in),csum); - - for(int mu=0;mu<4;mu++){ - for(int i=0;i<2;i++){ - for(int j=0;j<3;j++){ - out(mu)()(i,j) = in(mu)(i)(j); - }} - } - reconstruct3(out); - } - }; - - template - struct Nersc3x2unmunger{ - - void operator() (sobj &in,fobj &out,uint32_t &csum){ - - - for(int mu=0;mu<4;mu++){ - for(int i=0;i<2;i++){ - for(int j=0;j<3;j++){ - out(mu)(i)(j) = in(mu)()(i,j); - }} - } - - NerscChecksum((uint32_t *)&out,sizeof(out),csum); - - } - }; - - //////////////////////////////////////////////////////////////////////////////// // Write and read from fstream; comput header offset for payload //////////////////////////////////////////////////////////////////////////////// @@ -216,42 +45,17 @@ namespace Grid { std::ofstream fout(file,std::ios::out); } -#define dump_nersc_header(field, s) \ - s << "BEGIN_HEADER" << std::endl; \ - s << "HDR_VERSION = " << field.hdr_version << std::endl; \ - s << "DATATYPE = " << field.data_type << std::endl; \ - s << "STORAGE_FORMAT = " << field.storage_format << std::endl; \ - for(int i=0;i<4;i++){ \ - s << "DIMENSION_" << i+1 << " = " << field.dimension[i] << std::endl ; \ - } \ - s << "LINK_TRACE = " << std::setprecision(10) << field.link_trace << std::endl; \ - s << "PLAQUETTE = " << std::setprecision(10) << field.plaquette << std::endl; \ - for(int i=0;i<4;i++){ \ - s << "BOUNDARY_"< header; @@ -323,21 +127,21 @@ namespace Grid { return field.data_start; } - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Now the meat: the object readers - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#define PARALLEL_READ -#define PARALLEL_WRITE + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Now the meat: the object readers + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - template - static inline void readConfiguration(Lattice > &Umu,NerscField& header,std::string file) - { + template + static inline void readConfiguration(Lattice > &Umu, + FieldMetaData& header, + std::string file) + { typedef Lattice > GaugeField; GridBase *grid = Umu._grid; int offset = readHeader(file,Umu._grid,header); - NerscField clone(header); + FieldMetaData clone(header); std::string format(header.floating_point); @@ -346,76 +150,78 @@ namespace Grid { int ieee64big = (format == std::string("IEEE64BIG")); int ieee64 = (format == std::string("IEEE64")); - uint32_t csum; + uint32_t nersc_csum,scidac_csuma,scidac_csumb; // depending on datatype, set up munger; // munger is a function of if ( header.data_type == std::string("4D_SU3_GAUGE") ) { - if ( ieee32 || ieee32big ) { -#ifdef PARALLEL_READ - csum=BinaryIO::readObjectParallel, LorentzColour2x3F> - (Umu,file,Nersc3x2munger(), offset,format); -#else - csum=BinaryIO::readObjectSerial, LorentzColour2x3F> - (Umu,file,Nersc3x2munger(), offset,format); -#endif - } - if ( ieee64 || ieee64big ) { -#ifdef PARALLEL_READ - csum=BinaryIO::readObjectParallel, LorentzColour2x3D> - (Umu,file,Nersc3x2munger(),offset,format); -#else - csum=BinaryIO::readObjectSerial, LorentzColour2x3D> - (Umu,file,Nersc3x2munger(),offset,format); -#endif - } - } else if ( header.data_type == std::string("4D_SU3_GAUGE_3x3") ) { if ( ieee32 || ieee32big ) { -#ifdef PARALLEL_READ - csum=BinaryIO::readObjectParallel,LorentzColourMatrixF> - (Umu,file,NerscSimpleMunger(),offset,format); -#else - csum=BinaryIO::readObjectSerial,LorentzColourMatrixF> - (Umu,file,NerscSimpleMunger(),offset,format); -#endif + BinaryIO::readLatticeObject, LorentzColour2x3F> + (Umu,file,Gauge3x2munger(), offset,format, + nersc_csum,scidac_csuma,scidac_csumb); } if ( ieee64 || ieee64big ) { -#ifdef PARALLEL_READ - csum=BinaryIO::readObjectParallel,LorentzColourMatrixD> - (Umu,file,NerscSimpleMunger(),offset,format); -#else - csum=BinaryIO::readObjectSerial,LorentzColourMatrixD> - (Umu,file,NerscSimpleMunger(),offset,format); -#endif + BinaryIO::readLatticeObject, LorentzColour2x3D> + (Umu,file,Gauge3x2munger(),offset,format, + nersc_csum,scidac_csuma,scidac_csumb); + } + } else if ( header.data_type == std::string("4D_SU3_GAUGE_3x3") ) { + if ( ieee32 || ieee32big ) { + BinaryIO::readLatticeObject,LorentzColourMatrixF> + (Umu,file,GaugeSimpleMunger(),offset,format, + nersc_csum,scidac_csuma,scidac_csumb); + } + if ( ieee64 || ieee64big ) { + BinaryIO::readLatticeObject,LorentzColourMatrixD> + (Umu,file,GaugeSimpleMunger(),offset,format, + nersc_csum,scidac_csuma,scidac_csumb); } } else { assert(0); } - NerscStatistics(Umu,clone); + GaugeStatistics(Umu,clone); - std::cout<= 1.0e-5 ) { + std::cout << " Plaquette mismatch "< - static inline void writeConfiguration(Lattice > &Umu,std::string file, int two_row,int bits32) + static inline void writeConfiguration(Lattice > &Umu, + std::string file, + int two_row, + int bits32) { typedef Lattice > GaugeField; typedef iLorentzColourMatrix vobj; typedef typename vobj::scalar_object sobj; + FieldMetaData header; + /////////////////////////////////////////// // Following should become arguments - NerscField header; + /////////////////////////////////////////// header.sequence_number = 1; header.ensemble_id = "UKQCD"; header.ensemble_label = "DWF"; @@ -425,45 +231,32 @@ namespace Grid { GridBase *grid = Umu._grid; - NerscGrid(grid,header); - NerscStatistics(Umu,header); - NerscMachineCharacteristics(header); + GridMetaData(grid,header); + assert(header.nd==4); + GaugeStatistics(Umu,header); + MachineCharacteristics(header); - uint32_t csum; int offset; truncate(file); - if ( two_row ) { + // Sod it -- always write 3x3 double + header.floating_point = std::string("IEEE64BIG"); + header.data_type = std::string("4D_SU3_GAUGE_3x3"); + GaugeSimpleUnmunger munge; + offset = writeHeader(header,file); - header.floating_point = std::string("IEEE64BIG"); - header.data_type = std::string("4D_SU3_GAUGE"); - Nersc3x2unmunger munge; - BinaryIO::Uint32Checksum(Umu, munge,header.checksum); - offset = writeHeader(header,file); -#ifdef PARALLEL_WRITE - csum=BinaryIO::writeObjectParallel(Umu,file,munge,offset,header.floating_point); -#else - csum=BinaryIO::writeObjectSerial(Umu,file,munge,offset,header.floating_point); -#endif - } else { - header.floating_point = std::string("IEEE64BIG"); - header.data_type = std::string("4D_SU3_GAUGE_3x3"); - NerscSimpleUnmunger munge; - BinaryIO::Uint32Checksum(Umu, munge,header.checksum); - offset = writeHeader(header,file); -#ifdef PARALLEL_WRITE - csum=BinaryIO::writeObjectParallel(Umu,file,munge,offset,header.floating_point); -#else - csum=BinaryIO::writeObjectSerial(Umu,file,munge,offset,header.floating_point); -#endif - } + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::writeLatticeObject(Umu,file,munge,offset,header.floating_point, + nersc_csum,scidac_csuma,scidac_csumb); + header.checksum = nersc_csum; + writeHeader(header,file); - std::cout< - uint32_t csum=BinaryIO::readRNGSerial(serial,parallel,file,offset); + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::readRNG(serial,parallel,file,offset,nersc_csum,scidac_csuma,scidac_csumb); - assert(csum == header.checksum ); + if ( nersc_csum != header.checksum ) { + std::cerr << "checksum mismatch "<::SetCoefficientsInternal(RealD zolo_hi,std::vector0.0); + assert(omega[i]!=Coeff_t(0.0)); bs[i] = 0.5*(bpc/omega[i] + bmc); cs[i] = 0.5*(bpc/omega[i] - bmc); } @@ -429,7 +429,7 @@ void CayleyFermion5D::SetCoefficientsInternal(RealD zolo_hi,std::vectorM5) +1.0); - // assert(fabs(bee[i])>0.0); + assert(bee[i]!=Coeff_t(0.0)); cee[i]=as[i]*(1.0-cs[i]*(4.0-this->M5)); beo[i]=as[i]*bs[i]; ceo[i]=-as[i]*cs[i]; @@ -455,11 +455,17 @@ void CayleyFermion5D::SetCoefficientsInternal(RealD zolo_hi,std::vector::SetCoefficientsInternal(RealD zolo_hi,std::vector0.0); + assert(bee[j] != Coeff_t(0.0)); delta_d *= cee[j]/bee[j]; } dee[Ls-1] += delta_d; diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h index 20458b6d..524179f5 100644 --- a/lib/qcd/action/fermion/FermionOperatorImpl.h +++ b/lib/qcd/action/fermion/FermionOperatorImpl.h @@ -644,19 +644,16 @@ class StaggeredImpl : public PeriodicGaugeImpl using iImplScalar = iScalar > >; template using iImplSpinor = iScalar > >; template using iImplHalfSpinor = iScalar > >; template using iImplDoubledGaugeField = iVector >, Nds>; template using iImplPropagator = iScalar > >; - typedef iImplScalar SiteComplex; typedef iImplSpinor SiteSpinor; typedef iImplHalfSpinor SiteHalfSpinor; typedef iImplDoubledGaugeField SiteDoubledGaugeField; typedef iImplPropagator SitePropagator; - typedef Lattice ComplexField; typedef Lattice FermionField; typedef Lattice DoubledGaugeField; typedef Lattice PropagatorField; @@ -775,7 +772,6 @@ class StaggeredImpl : public PeriodicGaugeImpl using iImplScalar = iScalar > >; template using iImplSpinor = iScalar > >; template using iImplHalfSpinor = iScalar > >; template using iImplDoubledGaugeField = iVector >, Nds>; @@ -792,12 +788,10 @@ class StaggeredImpl : public PeriodicGaugeImpl DoubledGaugeField; typedef Lattice PropagatorField; - typedef iImplScalar SiteComplex; typedef iImplSpinor SiteSpinor; typedef iImplHalfSpinor SiteHalfSpinor; - typedef Lattice ComplexField; typedef Lattice FermionField; typedef SimpleCompressor Compressor; diff --git a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc index 61a3c559..7d988d89 100644 --- a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc +++ b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc @@ -230,8 +230,15 @@ void ImprovedStaggeredFermion5D::DhopInternal(StencilImpl & st, LebesgueOr { Compressor compressor; int LLs = in._grid->_rdimensions[0]; + + + + DhopTotalTime -= usecond(); + DhopCommTime -= usecond(); st.HaloExchange(in,compressor); + DhopCommTime += usecond(); + DhopComputeTime -= usecond(); // Dhop takes the 4d grid from U, and makes a 5d index for fermion if (dag == DaggerYes) { parallel_for (int ss = 0; ss < U._grid->oSites(); ss++) { @@ -244,12 +251,15 @@ void ImprovedStaggeredFermion5D::DhopInternal(StencilImpl & st, LebesgueOr Kernels::DhopSite(st,lo,U,UUU,st.CommBuf(),LLs,sU,in,out); } } + DhopComputeTime += usecond(); + DhopTotalTime += usecond(); } template void ImprovedStaggeredFermion5D::DhopOE(const FermionField &in, FermionField &out,int dag) { + DhopCalls+=1; conformable(in._grid,FermionRedBlackGrid()); // verifies half grid conformable(in._grid,out._grid); // drops the cb check @@ -261,6 +271,7 @@ void ImprovedStaggeredFermion5D::DhopOE(const FermionField &in, FermionFie template void ImprovedStaggeredFermion5D::DhopEO(const FermionField &in, FermionField &out,int dag) { + DhopCalls+=1; conformable(in._grid,FermionRedBlackGrid()); // verifies half grid conformable(in._grid,out._grid); // drops the cb check @@ -272,6 +283,7 @@ void ImprovedStaggeredFermion5D::DhopEO(const FermionField &in, FermionFie template void ImprovedStaggeredFermion5D::Dhop(const FermionField &in, FermionField &out,int dag) { + DhopCalls+=2; conformable(in._grid,FermionGrid()); // verifies full grid conformable(in._grid,out._grid); @@ -280,6 +292,54 @@ void ImprovedStaggeredFermion5D::Dhop(const FermionField &in, FermionField DhopInternal(Stencil,Lebesgue,Umu,UUUmu,in,out,dag); } +template +void ImprovedStaggeredFermion5D::Report(void) +{ + std::vector latt = GridDefaultLatt(); + RealD volume = Ls; for(int mu=0;mu_Nprocessors; + RealD NN = _FourDimGrid->NodeCount(); + + std::cout << GridLogMessage << "#### Dhop calls report " << std::endl; + + std::cout << GridLogMessage << "ImprovedStaggeredFermion5D Number of DhopEO Calls : " + << DhopCalls << std::endl; + std::cout << GridLogMessage << "ImprovedStaggeredFermion5D TotalTime /Calls : " + << DhopTotalTime / DhopCalls << " us" << std::endl; + std::cout << GridLogMessage << "ImprovedStaggeredFermion5D CommTime /Calls : " + << DhopCommTime / DhopCalls << " us" << std::endl; + std::cout << GridLogMessage << "ImprovedStaggeredFermion5D ComputeTime/Calls : " + << DhopComputeTime / DhopCalls << " us" << std::endl; + + // Average the compute time + _FourDimGrid->GlobalSum(DhopComputeTime); + DhopComputeTime/=NP; + + RealD mflops = 1154*volume*DhopCalls/DhopComputeTime/2; // 2 for red black counting + std::cout << GridLogMessage << "Average mflops/s per call : " << mflops << std::endl; + std::cout << GridLogMessage << "Average mflops/s per call per rank : " << mflops/NP << std::endl; + std::cout << GridLogMessage << "Average mflops/s per call per node : " << mflops/NN << std::endl; + + RealD Fullmflops = 1154*volume*DhopCalls/(DhopTotalTime)/2; // 2 for red black counting + std::cout << GridLogMessage << "Average mflops/s per call (full) : " << Fullmflops << std::endl; + std::cout << GridLogMessage << "Average mflops/s per call per rank (full): " << Fullmflops/NP << std::endl; + std::cout << GridLogMessage << "Average mflops/s per call per node (full): " << Fullmflops/NN << std::endl; + + std::cout << GridLogMessage << "ImprovedStaggeredFermion5D Stencil" < +void ImprovedStaggeredFermion5D::ZeroCounters(void) +{ + DhopCalls = 0; + DhopTotalTime = 0; + DhopCommTime = 0; + DhopComputeTime = 0; + Stencil.ZeroCounters(); + StencilEven.ZeroCounters(); + StencilOdd.ZeroCounters(); +} ///////////////////////////////////////////////////////////////////////// // Implement the general interface. Here we use SAME mass on all slices diff --git a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h index 4961da49..ca1a955a 100644 --- a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h +++ b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h @@ -55,6 +55,16 @@ namespace QCD { FermionField _tmp; FermionField &tmp(void) { return _tmp; } + //////////////////////////////////////// + // Performance monitoring + //////////////////////////////////////// + void Report(void); + void ZeroCounters(void); + double DhopTotalTime; + double DhopCalls; + double DhopCommTime; + double DhopComputeTime; + /////////////////////////////////////////////////////////////// // Implement the abstract base /////////////////////////////////////////////////////////////// diff --git a/lib/qcd/action/fermion/WilsonCompressor.h b/lib/qcd/action/fermion/WilsonCompressor.h index 96cbe1ec..cc5c3c63 100644 --- a/lib/qcd/action/fermion/WilsonCompressor.h +++ b/lib/qcd/action/fermion/WilsonCompressor.h @@ -238,7 +238,33 @@ template using WilsonCompressor = WilsonCom template class WilsonStencil : public CartesianStencil { public: - + double timer0; + double timer1; + double timer2; + double timer3; + double timer4; + double timer5; + double timer6; + uint64_t callsi; + void ZeroCountersi(void) + { + timer0=0; + timer1=0; + timer2=0; + timer3=0; + timer4=0; + timer5=0; + timer6=0; + callsi=0; + } + void Reporti(int calls) + { + if ( timer0 ) std::cout << GridLogMessage << " timer0 (HaloGatherOpt) " < same_node; @@ -252,6 +278,7 @@ public: : CartesianStencil (grid,npoints,checkerboard,directions,distances) , same_node(npoints) { + ZeroCountersi(); surface_list.resize(0); }; @@ -261,7 +288,6 @@ public: // Here we know the distance is 1 for WilsonStencil for(int point=0;point_npoints;point++){ same_node[point] = this->SameNode(point); - // std::cout << " dir " < > reqs; this->HaloExchangeOptGather(source,compress); - this->CommunicateBegin(reqs); - this->CommunicateComplete(reqs); + double t1=usecond(); + // Asynchronous MPI calls multidirectional, Isend etc... + // this->CommunicateBegin(reqs); + // this->CommunicateComplete(reqs); + // Non-overlapped directions within a thread. Asynchronous calls except MPI3, threaded up to comm threads ways. + this->Communicate(); + double t2=usecond(); timer1 += t2-t1; this->CommsMerge(compress); + double t3=usecond(); timer2 += t3-t2; this->CommsMergeSHM(compress); + double t4=usecond(); timer3 += t4-t3; } template void HaloExchangeOptGather(const Lattice &source,compressor &compress) { this->Prepare(); + double t0=usecond(); this->HaloGatherOpt(source,compress); + double t1=usecond(); + timer0 += t1-t0; + callsi++; } template @@ -304,7 +341,9 @@ public: typedef typename compressor::SiteHalfSpinor SiteHalfSpinor; typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor; + this->mpi3synctime_g-=usecond(); this->_grid->StencilBarrier(); + this->mpi3synctime_g+=usecond(); assert(source._grid==this->_grid); this->halogtime-=usecond(); @@ -323,7 +362,6 @@ public: int dag = compress.dag; int face_idx=0; if ( dag ) { - // std::cout << " Optimised Dagger compress " <HaloGatherDir(source,XpCompress,Xp,face_idx)); assert(same_node[Yp]==this->HaloGatherDir(source,YpCompress,Yp,face_idx)); assert(same_node[Zp]==this->HaloGatherDir(source,ZpCompress,Zp,face_idx)); diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc index 27319fb0..1da58ddb 100644 --- a/lib/qcd/action/fermion/WilsonFermion5D.cc +++ b/lib/qcd/action/fermion/WilsonFermion5D.cc @@ -123,22 +123,24 @@ WilsonFermion5D::WilsonFermion5D(GaugeField &_Umu, int vol4; vol4=FourDimGrid.oSites(); Stencil.BuildSurfaceList(LLs,vol4); + vol4=FourDimRedBlackGrid.oSites(); StencilEven.BuildSurfaceList(LLs,vol4); StencilOdd.BuildSurfaceList(LLs,vol4); - std::cout << GridLogMessage << " SurfaceLists "<< Stencil.surface_list.size() - <<" " << StencilEven.surface_list.size()< void WilsonFermion5D::Report(void) { - std::vector latt = GridDefaultLatt(); - RealD volume = Ls; for(int mu=0;mu_Nprocessors; - RealD NN = _FourDimGrid->NodeCount(); + RealD NP = _FourDimGrid->_Nprocessors; + RealD NN = _FourDimGrid->NodeCount(); + RealD volume = Ls; + std::vector latt = _FourDimGrid->GlobalDimensions(); + for(int mu=0;mu 0 ) { std::cout << GridLogMessage << "#### Dhop calls report " << std::endl; @@ -184,6 +186,11 @@ void WilsonFermion5D::Report(void) std::cout << GridLogMessage << "WilsonFermion5D StencilEven"< 0){ + std::cout << GridLogMessage << "WilsonFermion5D Stencil Reporti()" < @@ -203,6 +210,9 @@ void WilsonFermion5D::ZeroCounters(void) { Stencil.ZeroCounters(); StencilEven.ZeroCounters(); StencilOdd.ZeroCounters(); + Stencil.ZeroCountersi(); + StencilEven.ZeroCountersi(); + StencilOdd.ZeroCountersi(); } @@ -379,7 +389,6 @@ void WilsonFermion5D::DhopInternalOverlappedComms(StencilImpl & st, Lebesg { #ifdef GRID_OMP // assert((dag==DaggerNo) ||(dag==DaggerYes)); - typedef CartesianCommunicator::CommsRequest_t CommsRequest_t; Compressor compressor(dag); @@ -388,46 +397,70 @@ void WilsonFermion5D::DhopInternalOverlappedComms(StencilImpl & st, Lebesg DhopFaceTime-=usecond(); st.HaloExchangeOptGather(in,compressor); - DhopFaceTime+=usecond(); - std::vector > reqs; - - // Rely on async comms; start comms before merge of local data - DhopCommTime-=usecond(); - st.CommunicateBegin(reqs); - - DhopFaceTime-=usecond(); - st.CommsMergeSHM(compressor); + st.CommsMergeSHM(compressor);// Could do this inside parallel region overlapped with comms DhopFaceTime+=usecond(); - // Perhaps use omp task and region -#pragma omp parallel + double ctime=0; + double ptime=0; + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // Ugly explicit thread mapping introduced for OPA reasons. + ////////////////////////////////////////////////////////////////////////////////////////////////////// +#pragma omp parallel reduction(max:ctime) reduction(max:ptime) { + int tid = omp_get_thread_num(); int nthreads = omp_get_num_threads(); - int me = omp_get_thread_num(); - int myoff, mywork; - - GridThread::GetWork(len,me-1,mywork,myoff,nthreads-1); - int sF = LLs * myoff; - - if ( me == 0 ) { - st.CommunicateComplete(reqs); - DhopCommTime+=usecond(); - } else { - // Interior links in stencil - if ( me==1 ) DhopComputeTime-=usecond(); - if (dag == DaggerYes) Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,myoff,LLs,mywork,in,out,1,0); - else Kernels::DhopSite(st,lo,U,st.CommBuf(),sF,myoff,LLs,mywork,in,out,1,0); - if ( me==1 ) DhopComputeTime+=usecond(); + int ncomms = CartesianCommunicator::nCommThreads; + if (ncomms == -1) ncomms = 1; + assert(nthreads > ncomms); + if (tid >= ncomms) { + double start = usecond(); + nthreads -= ncomms; + int ttid = tid - ncomms; + int n = U._grid->oSites(); + int chunk = n / nthreads; + int rem = n % nthreads; + int myblock, myn; + if (ttid < rem) { + myblock = ttid * chunk + ttid; + myn = chunk+1; + } else { + myblock = ttid*chunk + rem; + myn = chunk; + } + + // do the compute + if (dag == DaggerYes) { + for (int ss = myblock; ss < myblock+myn; ++ss) { + int sU = ss; + int sF = LLs * sU; + Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0); + } + } else { + for (int ss = myblock; ss < myblock+myn; ++ss) { + int sU = ss; + int sF = LLs * sU; + Kernels::DhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0); + } + } + ptime = usecond() - start; + } + { + double start = usecond(); + st.CommunicateThreaded(); + ctime = usecond() - start; } } + DhopCommTime += ctime; + DhopComputeTime+=ptime; + + // First to enter, last to leave timing + st.CollateThreads(); DhopFaceTime-=usecond(); st.CommsMerge(compressor); DhopFaceTime+=usecond(); - // Load imbalance alert. Should use dynamic schedule OMP for loop - // Perhaps create a list of only those sites with face work, and - // load balance process the list. DhopComputeTime2-=usecond(); if (dag == DaggerYes) { int sz=st.surface_list.size(); @@ -448,11 +481,9 @@ void WilsonFermion5D::DhopInternalOverlappedComms(StencilImpl & st, Lebesg #else assert(0); #endif - } - template void WilsonFermion5D::DhopInternalSerialComms(StencilImpl & st, LebesgueOrder &lo, DoubledGaugeField & U, diff --git a/lib/qcd/action/gauge/GaugeImplTypes.h b/lib/qcd/action/gauge/GaugeImplTypes.h index 29e79581..9e3e0d68 100644 --- a/lib/qcd/action/gauge/GaugeImplTypes.h +++ b/lib/qcd/action/gauge/GaugeImplTypes.h @@ -40,12 +40,15 @@ namespace QCD { typedef typename GImpl::Simd Simd; \ typedef typename GImpl::LinkField GaugeLinkField; \ typedef typename GImpl::Field GaugeField; \ + typedef typename GImpl::ComplexField ComplexField;\ typedef typename GImpl::SiteField SiteGaugeField; \ + typedef typename GImpl::SiteComplex SiteComplex; \ typedef typename GImpl::SiteLink SiteGaugeLink; -#define INHERIT_FIELD_TYPES(Impl) \ - typedef typename Impl::Simd Simd; \ - typedef typename Impl::SiteField SiteField; \ +#define INHERIT_FIELD_TYPES(Impl) \ + typedef typename Impl::Simd Simd; \ + typedef typename Impl::ComplexField ComplexField; \ + typedef typename Impl::SiteField SiteField; \ typedef typename Impl::Field Field; // hardcodes the exponential approximation in the template @@ -53,14 +56,17 @@ template class GaugeImplType public: typedef S Simd; - template using iImplGaugeLink = iScalar>>; - template using iImplGaugeField = iVector>, Nd>; + template using iImplScalar = iScalar > >; + template using iImplGaugeLink = iScalar > >; + template using iImplGaugeField = iVector >, Nd>; + typedef iImplScalar SiteComplex; typedef iImplGaugeLink SiteLink; typedef iImplGaugeField SiteField; - typedef Lattice LinkField; - typedef Lattice Field; + typedef Lattice ComplexField; + typedef Lattice LinkField; + typedef Lattice Field; // Guido: we can probably separate the types from the HMC functions // this will create 2 kind of implementations diff --git a/lib/qcd/action/gauge/Photon.h b/lib/qcd/action/gauge/Photon.h index 1512d4e3..7e21a1de 100644 --- a/lib/qcd/action/gauge/Photon.h +++ b/lib/qcd/action/gauge/Photon.h @@ -41,11 +41,13 @@ namespace QCD{ template using iImplGaugeField = iVector>, Nd>; - typedef iImplGaugeLink SiteLink; + typedef iImplGaugeLink SiteLink; typedef iImplGaugeField SiteField; + typedef SiteField SiteComplex; - typedef Lattice LinkField; + typedef Lattice LinkField; typedef Lattice Field; + typedef Field ComplexField; }; typedef QedGimpl QedGimplR; diff --git a/lib/qcd/action/scalar/Scalar.h b/lib/qcd/action/scalar/Scalar.h index e5bea275..485a6765 100644 --- a/lib/qcd/action/scalar/Scalar.h +++ b/lib/qcd/action/scalar/Scalar.h @@ -31,6 +31,7 @@ directory #include #include +#include namespace Grid { namespace QCD { @@ -39,6 +40,10 @@ namespace QCD { typedef ScalarAction ScalarActionF; typedef ScalarAction ScalarActionD; + template using ScalarAdjActionR = ScalarInteractionAction, Dimensions>; + template using ScalarAdjActionF = ScalarInteractionAction, Dimensions>; + template using ScalarAdjActionD = ScalarInteractionAction, Dimensions>; + } } diff --git a/lib/qcd/action/scalar/ScalarAction.h b/lib/qcd/action/scalar/ScalarAction.h index f10ec9a6..2c82d2e3 100644 --- a/lib/qcd/action/scalar/ScalarAction.h +++ b/lib/qcd/action/scalar/ScalarAction.h @@ -6,10 +6,10 @@ Copyright (C) 2015 -Author: Azusa Yamaguchi -Author: Peter Boyle -Author: neo -Author: paboyle + Author: Azusa Yamaguchi + Author: Peter Boyle + Author: neo + Author: paboyle This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -35,50 +35,49 @@ directory namespace Grid { // FIXME drop the QCD namespace everywhere here - - template - class ScalarAction : public QCD::Action { - public: + +template +class ScalarAction : public QCD::Action { + public: INHERIT_FIELD_TYPES(Impl); - - private: + + private: RealD mass_square; RealD lambda; - - public: - ScalarAction(RealD ms, RealD l) : mass_square(ms), lambda(l){}; - virtual std::string LogParameters(){ + public: + ScalarAction(RealD ms, RealD l) : mass_square(ms), lambda(l) {} + + virtual std::string LogParameters() { std::stringstream sstream; sstream << GridLogMessage << "[ScalarAction] lambda : " << lambda << std::endl; sstream << GridLogMessage << "[ScalarAction] mass_square : " << mass_square << std::endl; return sstream.str(); - } - - virtual std::string action_name(){return "ScalarAction";} - - virtual void refresh(const Field &U, - GridParallelRNG &pRNG){}; // noop as no pseudoferms - + virtual std::string action_name() {return "ScalarAction";} + + virtual void refresh(const Field &U, GridParallelRNG &pRNG) {} // noop as no pseudoferms + virtual RealD S(const Field &p) { return (mass_square * 0.5 + QCD::Nd) * ScalarObs::sumphisquared(p) + - (lambda / 24.) * ScalarObs::sumphifourth(p) + - ScalarObs::sumphider(p); + (lambda / 24.) * ScalarObs::sumphifourth(p) + + ScalarObs::sumphider(p); }; - + virtual void deriv(const Field &p, - Field &force) { + Field &force) { Field tmp(p._grid); Field p2(p._grid); ScalarObs::phisquared(p2, p); tmp = -(Cshift(p, 0, -1) + Cshift(p, 0, 1)); for (int mu = 1; mu < QCD::Nd; mu++) tmp -= Cshift(p, mu, -1) + Cshift(p, mu, 1); - - force=+(mass_square + 2. * QCD::Nd) * p + (lambda / 6.) * p2 * p + tmp; - }; - }; - -} // Grid + + force =+(mass_square + 2. * QCD::Nd) * p + (lambda / 6.) * p2 * p + tmp; + } +}; + + + +} // namespace Grid #endif // SCALAR_ACTION_H diff --git a/lib/qcd/action/scalar/ScalarImpl.h b/lib/qcd/action/scalar/ScalarImpl.h index 1b5fefea..f85ab840 100644 --- a/lib/qcd/action/scalar/ScalarImpl.h +++ b/lib/qcd/action/scalar/ScalarImpl.h @@ -5,43 +5,45 @@ namespace Grid { //namespace QCD { - template - class ScalarImplTypes { - public: +template +class ScalarImplTypes { + public: typedef S Simd; - + template using iImplField = iScalar > >; - + typedef iImplField SiteField; typedef SiteField SitePropagator; + typedef SiteField SiteComplex; typedef Lattice Field; + typedef Field ComplexField; typedef Field FermionField; typedef Field PropagatorField; static inline void generate_momenta(Field& P, GridParallelRNG& pRNG){ gaussian(pRNG, P); } - + static inline Field projectForce(Field& P){return P;} - - static inline void update_field(Field& P, Field& U, double ep){ + + static inline void update_field(Field& P, Field& U, double ep) { U += P*ep; } - - static inline RealD FieldSquareNorm(Field& U){ + + static inline RealD FieldSquareNorm(Field& U) { return (- sum(trace(U*U))/2.0); } - + static inline void HotConfiguration(GridParallelRNG &pRNG, Field &U) { gaussian(pRNG, U); } - + static inline void TepidConfiguration(GridParallelRNG &pRNG, Field &U) { gaussian(pRNG, U); } - + static inline void ColdConfiguration(GridParallelRNG &pRNG, Field &U) { U = 1.0; } @@ -88,57 +90,73 @@ namespace Grid { }; template - class ScalarMatrixImplTypes { + class ScalarAdjMatrixImplTypes { public: typedef S Simd; + typedef QCD::SU Group; template - using iImplField = iScalar > >; + using iImplField = iScalar>>; + template + using iImplComplex = iScalar>>; + + typedef iImplField SiteField; + typedef SiteField SitePropagator; + typedef iImplComplex SiteComplex; - typedef iImplField SiteField; - - - typedef Lattice Field; - - static inline void generate_momenta(Field& P, GridParallelRNG& pRNG){ - gaussian(pRNG, P); + typedef Lattice Field; + typedef Lattice ComplexField; + typedef Field FermionField; + typedef Field PropagatorField; + + static inline void generate_momenta(Field& P, GridParallelRNG& pRNG) { + Group::GaussianFundamentalLieAlgebraMatrix(pRNG, P); } - - static inline Field projectForce(Field& P){return P;} - - static inline void update_field(Field& P, Field& U, double ep){ + + static inline Field projectForce(Field& P) {return P;} + + static inline void update_field(Field& P, Field& U, double ep) { U += P*ep; } - - static inline RealD FieldSquareNorm(Field& U){ - return (TensorRemove(- sum(trace(U*U))*0.5).real()); + + static inline RealD FieldSquareNorm(Field& U) { + return (TensorRemove(sum(trace(U*U))).real()); } - + static inline void HotConfiguration(GridParallelRNG &pRNG, Field &U) { - gaussian(pRNG, U); + Group::GaussianFundamentalLieAlgebraMatrix(pRNG, U); } - + static inline void TepidConfiguration(GridParallelRNG &pRNG, Field &U) { - gaussian(pRNG, U); + Group::GaussianFundamentalLieAlgebraMatrix(pRNG, U, 0.01); } - + static inline void ColdConfiguration(GridParallelRNG &pRNG, Field &U) { - U = 1.0; + U = zero; } - + }; - - + + typedef ScalarImplTypes ScalarImplR; typedef ScalarImplTypes ScalarImplF; typedef ScalarImplTypes ScalarImplD; typedef ScalarImplTypes ScalarImplCR; typedef ScalarImplTypes ScalarImplCF; typedef ScalarImplTypes ScalarImplCD; + + // Hardcoding here the size of the matrices + typedef ScalarAdjMatrixImplTypes ScalarAdjImplR; + typedef ScalarAdjMatrixImplTypes ScalarAdjImplF; + typedef ScalarAdjMatrixImplTypes ScalarAdjImplD; + + template using ScalarNxNAdjImplR = ScalarAdjMatrixImplTypes; + template using ScalarNxNAdjImplF = ScalarAdjMatrixImplTypes; + template using ScalarNxNAdjImplD = ScalarAdjMatrixImplTypes; - //} -} + //} +} #endif diff --git a/lib/qcd/action/scalar/ScalarInteractionAction.h b/lib/qcd/action/scalar/ScalarInteractionAction.h index bd54a010..4d189352 100644 --- a/lib/qcd/action/scalar/ScalarInteractionAction.h +++ b/lib/qcd/action/scalar/ScalarInteractionAction.h @@ -6,10 +6,7 @@ Copyright (C) 2015 -Author: Azusa Yamaguchi -Author: Peter Boyle -Author: neo -Author: paboyle + Author: Guido Cossu This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -30,55 +27,122 @@ directory *************************************************************************************/ /* END LEGAL */ -#ifndef SCALAR_ACTION_H -#define SCALAR_ACTION_H +#ifndef SCALAR_INT_ACTION_H +#define SCALAR_INT_ACTION_H + + +// Note: this action can completely absorb the ScalarAction for real float fields +// use the scalarObjs to generalise the structure namespace Grid { // FIXME drop the QCD namespace everywhere here - - template + + template class ScalarInteractionAction : public QCD::Action { public: INHERIT_FIELD_TYPES(Impl); - private: RealD mass_square; RealD lambda; - - public: - ScalarAction(RealD ms, RealD l) : mass_square(ms), lambda(l){}; - virtual std::string LogParameters(){ + + typedef typename Field::vector_object vobj; + typedef CartesianStencil Stencil; + + SimpleCompressor compressor; + int npoint = 2*Ndim; + std::vector directions;// = {0,1,2,3,0,1,2,3}; // forcing 4 dimensions + std::vector displacements;// = {1,1,1,1, -1,-1,-1,-1}; + + + public: + + ScalarInteractionAction(RealD ms, RealD l) : mass_square(ms), lambda(l), displacements(2*Ndim,0), directions(2*Ndim,0){ + for (int mu = 0 ; mu < Ndim; mu++){ + directions[mu] = mu; directions[mu+Ndim] = mu; + displacements[mu] = 1; displacements[mu+Ndim] = -1; + } + } + + virtual std::string LogParameters() { std::stringstream sstream; sstream << GridLogMessage << "[ScalarAction] lambda : " << lambda << std::endl; sstream << GridLogMessage << "[ScalarAction] mass_square : " << mass_square << std::endl; return sstream.str(); - } - - virtual std::string action_name(){return "ScalarAction";} - - virtual void refresh(const Field &U, - GridParallelRNG &pRNG){}; // noop as no pseudoferms - + + virtual std::string action_name() {return "ScalarAction";} + + virtual void refresh(const Field &U, GridParallelRNG &pRNG) {} + virtual RealD S(const Field &p) { - return (mass_square * 0.5 + QCD::Nd) * ScalarObs::sumphisquared(p) + - (lambda / 24.) * ScalarObs::sumphifourth(p) + - ScalarObs::sumphider(p); + assert(p._grid->Nd() == Ndim); + static Stencil phiStencil(p._grid, npoint, 0, directions, displacements); + phiStencil.HaloExchange(p, compressor); + Field action(p._grid), pshift(p._grid), phisquared(p._grid); + phisquared = p*p; + action = (2.0*Ndim + mass_square)*phisquared - lambda/24.*phisquared*phisquared; + for (int mu = 0; mu < Ndim; mu++) { + // pshift = Cshift(p, mu, +1); // not efficient, implement with stencils + parallel_for (int i = 0; i < p._grid->oSites(); i++) { + int permute_type; + StencilEntry *SE; + vobj temp2; + const vobj *temp, *t_p; + + SE = phiStencil.GetEntry(permute_type, mu, i); + t_p = &p._odata[i]; + if ( SE->_is_local ) { + temp = &p._odata[SE->_offset]; + if ( SE->_permute ) { + permute(temp2, *temp, permute_type); + action._odata[i] -= temp2*(*t_p) + (*t_p)*temp2; + } else { + action._odata[i] -= (*temp)*(*t_p) + (*t_p)*(*temp); + } + } else { + action._odata[i] -= phiStencil.CommBuf()[SE->_offset]*(*t_p) + (*t_p)*phiStencil.CommBuf()[SE->_offset]; + } + } + // action -= pshift*p + p*pshift; + } + // NB the trace in the algebra is normalised to 1/2 + // minus sign coming from the antihermitian fields + return -(TensorRemove(sum(trace(action)))).real(); }; - - virtual void deriv(const Field &p, - Field &force) { - Field tmp(p._grid); - Field p2(p._grid); - ScalarObs::phisquared(p2, p); - tmp = -(Cshift(p, 0, -1) + Cshift(p, 0, 1)); - for (int mu = 1; mu < QCD::Nd; mu++) tmp -= Cshift(p, mu, -1) + Cshift(p, mu, 1); + + virtual void deriv(const Field &p, Field &force) { + assert(p._grid->Nd() == Ndim); + force = (2.0*Ndim + mass_square)*p - lambda/12.*p*p*p; + // move this outside + static Stencil phiStencil(p._grid, npoint, 0, directions, displacements); + phiStencil.HaloExchange(p, compressor); - force=+(mass_square + 2. * QCD::Nd) * p + (lambda / 6.) * p2 * p + tmp; - }; + //for (int mu = 0; mu < QCD::Nd; mu++) force -= Cshift(p, mu, -1) + Cshift(p, mu, 1); + for (int point = 0; point < npoint; point++) { + parallel_for (int i = 0; i < p._grid->oSites(); i++) { + const vobj *temp; + vobj temp2; + int permute_type; + StencilEntry *SE; + SE = phiStencil.GetEntry(permute_type, point, i); + + if ( SE->_is_local ) { + temp = &p._odata[SE->_offset]; + if ( SE->_permute ) { + permute(temp2, *temp, permute_type); + force._odata[i] -= temp2; + } else { + force._odata[i] -= *temp; + } + } else { + force._odata[i] -= phiStencil.CommBuf()[SE->_offset]; + } + } + } + } }; -} // Grid +} // namespace Grid -#endif // SCALAR_ACTION_H +#endif // SCALAR_INT_ACTION_H diff --git a/lib/qcd/hmc/GenericHMCrunner.h b/lib/qcd/hmc/GenericHMCrunner.h index c0c5079e..4f6c1af0 100644 --- a/lib/qcd/hmc/GenericHMCrunner.h +++ b/lib/qcd/hmc/GenericHMCrunner.h @@ -207,6 +207,12 @@ using GenericHMCRunnerTemplate = HMCWrapperTemplate ScalarGenericHMCRunner; +typedef HMCWrapperTemplate + ScalarAdjGenericHMCRunner; + +template +using ScalarNxNAdjGenericHMCRunner = HMCWrapperTemplate < ScalarNxNAdjImplR, MinimumNorm2, ScalarNxNMatrixFields >; + } // namespace QCD } // namespace Grid diff --git a/lib/qcd/hmc/HMC.h b/lib/qcd/hmc/HMC.h index ac690b60..5688bb24 100644 --- a/lib/qcd/hmc/HMC.h +++ b/lib/qcd/hmc/HMC.h @@ -76,7 +76,7 @@ struct HMCparameters: Serializable { template < class ReaderClass > void initialize(Reader &TheReader){ - std::cout << "Reading HMC\n"; + std::cout << GridLogMessage << "Reading HMC\n"; read(TheReader, "HMC", *this); } diff --git a/lib/qcd/hmc/HMCResourceManager.h b/lib/qcd/hmc/HMCResourceManager.h index 9f4c99a9..3e20a8c1 100644 --- a/lib/qcd/hmc/HMCResourceManager.h +++ b/lib/qcd/hmc/HMCResourceManager.h @@ -165,7 +165,7 @@ class HMCResourceManager { // Grids ////////////////////////////////////////////////////////////// - void AddGrid(std::string s, GridModule& M) { + void AddGrid(const std::string s, GridModule& M) { // Check for name clashes auto search = Grids.find(s); if (search != Grids.end()) { @@ -174,14 +174,24 @@ class HMCResourceManager { exit(1); } Grids[s] = std::move(M); + std::cout << GridLogMessage << "::::::::::::::::::::::::::::::::::::::::" < Mod; AddGrid(s, Mod); } + // Add a named grid set, 4d shortcut + tweak simd lanes + void AddFourDimGrid(const std::string s, const std::vector simd_decomposition) { + GridFourDimModule Mod(simd_decomposition); + AddGrid(s, Mod); + } GridCartesian* GetCartesian(std::string s = "") { @@ -253,6 +263,7 @@ class HMCResourceManager { template void AddObservable(Types&&... Args){ ObservablesList.push_back(std::unique_ptr(new T(std::forward(Args)...))); + ObservablesList.back()->print_parameters(); } std::vector* > GetObservables(){ @@ -297,4 +308,4 @@ private: } } -#endif // HMC_RESOURCE_MANAGER_H \ No newline at end of file +#endif // HMC_RESOURCE_MANAGER_H diff --git a/lib/qcd/hmc/HMC_GridModules.h b/lib/qcd/hmc/HMC_GridModules.h index 8331c02b..0f34e9a7 100644 --- a/lib/qcd/hmc/HMC_GridModules.h +++ b/lib/qcd/hmc/HMC_GridModules.h @@ -33,28 +33,29 @@ directory namespace Grid { // Resources -// Modules for grids +// Modules for grids // Introduce another namespace HMCModules? -class GridModuleParameters: Serializable{ +class GridModuleParameters: Serializable{ public: GRID_SERIALIZABLE_CLASS_MEMBERS(GridModuleParameters, std::string, lattice, std::string, mpi); - std::vector getLattice(){return strToVec(lattice);} - std::vector getMpi() {return strToVec(mpi);} + std::vector getLattice() const {return strToVec(lattice);} + std::vector getMpi() const {return strToVec(mpi);} - void check(){ - if (getLattice().size() != getMpi().size()) { - std::cout << GridLogError + + void check() const { + if (getLattice().size() != getMpi().size() ) { + std::cout << GridLogError << "Error in GridModuleParameters: lattice and mpi dimensions " "do not match" << std::endl; exit(1); } - } + } template GridModuleParameters(Reader& Reader, std::string n = "LatticeGrid"):name(n) { @@ -75,51 +76,94 @@ private: // Lower level class class GridModule { public: - GridCartesian* get_full() { + GridCartesian* get_full() { std::cout << GridLogDebug << "Getting cartesian in module"<< std::endl; return grid_.get(); } - GridRedBlackCartesian* get_rb() { + GridRedBlackCartesian* get_rb() { std::cout << GridLogDebug << "Getting rb-cartesian in module"<< std::endl; return rbgrid_.get(); } void set_full(GridCartesian* grid) { grid_.reset(grid); } void set_rb(GridRedBlackCartesian* rbgrid) { rbgrid_.reset(rbgrid); } + void show_full_decomposition(){ grid_->show_decomposition(); } + void show_rb_decomposition(){ rbgrid_->show_decomposition(); } protected: std::unique_ptr grid_; std::unique_ptr rbgrid_; - + }; //////////////////////////////////// // Classes for the user //////////////////////////////////// // Note: the space time grid should be out of the QCD namespace -template< class vector_type> -class GridFourDimModule : public GridModule { - public: - GridFourDimModule() { +template +class GridFourDimModule : public GridModule +{ +public: + GridFourDimModule() + { using namespace QCD; set_full(SpaceTimeGrid::makeFourDimGrid( - GridDefaultLatt(), GridDefaultSimd(4, vector_type::Nsimd()), + GridDefaultLatt(), + GridDefaultSimd(4, vector_type::Nsimd()), GridDefaultMpi())); set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get())); } - GridFourDimModule(GridModuleParameters Params) { + GridFourDimModule(const std::vector tweak_simd) + { + using namespace QCD; + if (tweak_simd.size() != 4) + { + std::cout << GridLogError + << "Error in GridFourDimModule: SIMD size different from 4" + << std::endl; + exit(1); + } + + // Checks that the product agrees with the expectation + int simd_sum = 1; + for (auto &n : tweak_simd) + simd_sum *= n; + std::cout << GridLogDebug << "TweakSIMD: " << tweak_simd << " Sum: " << simd_sum << std::endl; + + if (simd_sum == vector_type::Nsimd()) + { + set_full(SpaceTimeGrid::makeFourDimGrid( + GridDefaultLatt(), + tweak_simd, + GridDefaultMpi())); + set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get())); + } + else + { + std::cout << GridLogError + << "Error in GridFourDimModule: SIMD lanes must sum to " + << vector_type::Nsimd() + << std::endl; + } + } + + GridFourDimModule(const GridModuleParameters Params) + { using namespace QCD; - Params.check(); std::vector lattice_v = Params.getLattice(); std::vector mpi_v = Params.getMpi(); - if (lattice_v.size() == 4) { + if (lattice_v.size() == 4) + { set_full(SpaceTimeGrid::makeFourDimGrid( - lattice_v, GridDefaultSimd(4, vector_type::Nsimd()), + lattice_v, + GridDefaultSimd(4, vector_type::Nsimd()), mpi_v)); set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get())); - } else { - std::cout << GridLogError - << "Error in GridFourDimModule: lattice dimension different from 4" - << std::endl; + } + else + { + std::cout << GridLogError + << "Error in GridFourDimModule: lattice dimension different from 4" + << std::endl; exit(1); } } diff --git a/lib/qcd/hmc/checkpointers/BinaryCheckpointer.h b/lib/qcd/hmc/checkpointers/BinaryCheckpointer.h index 251ed042..59d655ad 100644 --- a/lib/qcd/hmc/checkpointers/BinaryCheckpointer.h +++ b/lib/qcd/hmc/checkpointers/BinaryCheckpointer.h @@ -62,36 +62,50 @@ class BinaryHmcCheckpointer : public BaseHmcCheckpointer { fout.close(); } - void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, - GridParallelRNG &pRNG) { + void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { + if ((traj % Params.saveInterval) == 0) { std::string config, rng; this->build_filenames(traj, Params, config, rng); - BinaryIO::BinarySimpleUnmunger munge; + uint32_t nersc_csum; + uint32_t scidac_csuma; + uint32_t scidac_csumb; + + BinarySimpleUnmunger munge; truncate(rng); - BinaryIO::writeRNGSerial(sRNG, pRNG, rng, 0); + BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); truncate(config); - uint32_t csum = BinaryIO::writeObjectParallel( - U, config, munge, 0, Params.format); + + BinaryIO::writeLatticeObject(U, config, munge, 0, Params.format, + nersc_csum,scidac_csuma,scidac_csumb); std::cout << GridLogMessage << "Written Binary Configuration " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksum " << std::hex + << nersc_csum <<"/" + << scidac_csuma <<"/" + << scidac_csumb + << std::dec << std::endl; } + }; - void CheckpointRestore(int traj, Field &U, GridSerialRNG &sRNG, - GridParallelRNG &pRNG) { + void CheckpointRestore(int traj, Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { std::string config, rng; this->build_filenames(traj, Params, config, rng); - BinaryIO::BinarySimpleMunger munge; - BinaryIO::readRNGSerial(sRNG, pRNG, rng, 0); - uint32_t csum = BinaryIO::readObjectParallel( - U, config, munge, 0, Params.format); + BinarySimpleMunger munge; + uint32_t nersc_csum; + uint32_t scidac_csuma; + uint32_t scidac_csumb; + BinaryIO::readRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); + BinaryIO::readLatticeObject(U, config, munge, 0, Params.format, + nersc_csum,scidac_csuma,scidac_csumb); + std::cout << GridLogMessage << "Read Binary Configuration " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksums " << std::hex << nersc_csum<<"/"< { // check here that the format is valid int ieee32big = (Params.format == std::string("IEEE32BIG")); - int ieee32 = (Params.format == std::string("IEEE32")); + int ieee32 = (Params.format == std::string("IEEE32")); int ieee64big = (Params.format == std::string("IEEE64BIG")); - int ieee64 = (Params.format == std::string("IEEE64")); + int ieee64 = (Params.format == std::string("IEEE64")); if (!(ieee64big || ieee32 || ieee32big || ieee64)) { std::cout << GridLogError << "Unrecognized file format " << Params.format @@ -74,13 +74,20 @@ class ILDGHmcCheckpointer : public BaseHmcCheckpointer { if ((traj % Params.saveInterval) == 0) { std::string config, rng; this->build_filenames(traj, Params, config, rng); - - ILDGIO IO(config, ILDGwrite); - BinaryIO::writeRNGSerial(sRNG, pRNG, rng, 0); - uint32_t csum = IO.writeConfiguration(U, Params.format); + + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); + IldgWriter _IldgWriter; + _IldgWriter.open(config); + _IldgWriter.writeConfiguration(U, traj, config, config); + _IldgWriter.close(); std::cout << GridLogMessage << "Written ILDG Configuration on " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksum " << std::hex + << nersc_csum<<"/" + << scidac_csuma<<"/" + << scidac_csumb + << std::dec << std::endl; } }; @@ -89,12 +96,21 @@ class ILDGHmcCheckpointer : public BaseHmcCheckpointer { std::string config, rng; this->build_filenames(traj, Params, config, rng); - ILDGIO IO(config, ILDGread); - BinaryIO::readRNGSerial(sRNG, pRNG, rng, 0); - uint32_t csum = IO.readConfiguration(U); // format from the header + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::readRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); + + FieldMetaData header; + IldgReader _IldgReader; + _IldgReader.open(config); + _IldgReader.readConfiguration(U,header); // format from the header + _IldgReader.close(); std::cout << GridLogMessage << "Read ILDG Configuration from " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksum " << std::hex + << nersc_csum<<"/" + << scidac_csuma<<"/" + << scidac_csumb + << std::dec << std::endl; }; }; } diff --git a/lib/qcd/hmc/checkpointers/NerscCheckpointer.h b/lib/qcd/hmc/checkpointers/NerscCheckpointer.h index 395369a0..a4b1b480 100644 --- a/lib/qcd/hmc/checkpointers/NerscCheckpointer.h +++ b/lib/qcd/hmc/checkpointers/NerscCheckpointer.h @@ -70,7 +70,7 @@ class NerscHmcCheckpointer : public BaseHmcCheckpointer { std::string config, rng; this->build_filenames(traj, Params, config, rng); - NerscField header; + FieldMetaData header; NerscIO::readRNGState(sRNG, pRNG, header, rng); NerscIO::readConfiguration(U, header, config); }; diff --git a/lib/qcd/modules/ObservableModules.h b/lib/qcd/modules/ObservableModules.h index 579fc1ec..24511617 100644 --- a/lib/qcd/modules/ObservableModules.h +++ b/lib/qcd/modules/ObservableModules.h @@ -84,8 +84,6 @@ class PlaquetteMod: public ObservableModule, NoParameters> typedef ObservableModule, NoParameters> ObsBase; using ObsBase::ObsBase; // for constructors - - // acquire resource virtual void initialize(){ this->ObservablePtr.reset(new PlaquetteLogger()); @@ -94,23 +92,22 @@ class PlaquetteMod: public ObservableModule, NoParameters> PlaquetteMod(): ObsBase(NoParameters()){} }; + template < class Impl > -class TopologicalChargeMod: public ObservableModule, NoParameters>{ - typedef ObservableModule, NoParameters> ObsBase; +class TopologicalChargeMod: public ObservableModule, TopologyObsParameters>{ + typedef ObservableModule, TopologyObsParameters> ObsBase; using ObsBase::ObsBase; // for constructors - - // acquire resource virtual void initialize(){ - this->ObservablePtr.reset(new TopologicalCharge()); + this->ObservablePtr.reset(new TopologicalCharge(this->Par_)); } public: - TopologicalChargeMod(): ObsBase(NoParameters()){} + TopologicalChargeMod(TopologyObsParameters Par): ObsBase(Par){} + TopologicalChargeMod(): ObsBase(){} }; - }// QCD temporarily here diff --git a/lib/qcd/observables/topological_charge.h b/lib/qcd/observables/topological_charge.h index 5d09c420..5af8d77b 100644 --- a/lib/qcd/observables/topological_charge.h +++ b/lib/qcd/observables/topological_charge.h @@ -33,9 +33,45 @@ directory namespace Grid { namespace QCD { +struct TopologySmearingParameters : Serializable { + GRID_SERIALIZABLE_CLASS_MEMBERS(TopologySmearingParameters, + int, steps, + float, step_size, + int, meas_interval, + float, maxTau); + + TopologySmearingParameters(int s = 0, float ss = 0.0f, int mi = 0, float mT = 0.0f): + steps(s), step_size(ss), meas_interval(mi), maxTau(mT){} + + template < class ReaderClass > + TopologySmearingParameters(Reader& Reader){ + read(Reader, "Smearing", *this); + } +}; + + + +struct TopologyObsParameters : Serializable { + GRID_SERIALIZABLE_CLASS_MEMBERS(TopologyObsParameters, + int, interval, + bool, do_smearing, + TopologySmearingParameters, Smearing); + + TopologyObsParameters(int interval = 1, bool smearing = false): + interval(interval), Smearing(smearing){} + + template + TopologyObsParameters(Reader& Reader){ + read(Reader, "TopologyMeasurement", *this); + } +}; + + // this is only defined for a gauge theory template class TopologicalCharge : public HmcObservable { + TopologyObsParameters Pars; + public: // here forces the Impl to be of gauge fields // if not the compiler will complain @@ -44,20 +80,39 @@ class TopologicalCharge : public HmcObservable { // necessary for HmcObservable compatibility typedef typename Impl::Field Field; + TopologicalCharge(int interval = 1, bool do_smearing = false): + Pars(interval, do_smearing){} + + TopologicalCharge(TopologyObsParameters P):Pars(P){ + std::cout << GridLogDebug << "Creating TopologicalCharge " << std::endl; + } + void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { - Real q = WilsonLoops::TopologicalCharge(U); + if (traj%Pars.interval == 0){ + // Smearing + Field Usmear = U; + int def_prec = std::cout.precision(); + + if (Pars.do_smearing){ + // using wilson flow by default here + WilsonFlow WF(Pars.Smearing.steps, Pars.Smearing.step_size, Pars.Smearing.meas_interval); + WF.smear_adaptive(Usmear, U, Pars.Smearing.maxTau); + Real T0 = WF.energyDensityPlaquette(Usmear); + std::cout << GridLogMessage << std::setprecision(std::numeric_limits::digits10 + 1) + << "T0 : [ " << traj << " ] "<< T0 << std::endl; + } - int def_prec = std::cout.precision(); + Real q = WilsonLoops::TopologicalCharge(Usmear); + std::cout << GridLogMessage + << std::setprecision(std::numeric_limits::digits10 + 1) + << "Topological Charge: [ " << traj << " ] "<< q << std::endl; - std::cout << GridLogMessage - << std::setprecision(std::numeric_limits::digits10 + 1) - << "Topological Charge: [ " << traj << " ] "<< q << std::endl; - - std::cout.precision(def_prec); + std::cout.precision(def_prec); + } } }; diff --git a/lib/qcd/representations/hmc_types.h b/lib/qcd/representations/hmc_types.h index 3701c9b2..3fee377e 100644 --- a/lib/qcd/representations/hmc_types.h +++ b/lib/qcd/representations/hmc_types.h @@ -62,7 +62,10 @@ class Representations { typedef Representations NoHirep; typedef Representations > ScalarFields; - //typedef Representations > ScalarMatrixFields; +typedef Representations > ScalarMatrixFields; + +template < int Colours> +using ScalarNxNMatrixFields = Representations::Field> >; // Helper classes to access the elements // Strips the first N parameters from the tuple diff --git a/lib/qcd/smearing/WilsonFlow.h b/lib/qcd/smearing/WilsonFlow.h index 5e9f2d95..4f5c0d43 100644 --- a/lib/qcd/smearing/WilsonFlow.h +++ b/lib/qcd/smearing/WilsonFlow.h @@ -108,7 +108,7 @@ void WilsonFlow::evolve_step_adaptive(typename Gimpl::GaugeField &U, Real if (maxTau - taus < epsilon){ epsilon = maxTau-taus; } - std::cout << GridLogMessage << "Integration epsilon : " << epsilon << std::endl; + //std::cout << GridLogMessage << "Integration epsilon : " << epsilon << std::endl; GaugeField Z(U._grid); GaugeField Zprime(U._grid); GaugeField tmp(U._grid), Uprime(U._grid); @@ -138,10 +138,10 @@ void WilsonFlow::evolve_step_adaptive(typename Gimpl::GaugeField &U, Real // adjust integration step taus += epsilon; - std::cout << GridLogMessage << "Adjusting integration step with distance: " << diff << std::endl; + //std::cout << GridLogMessage << "Adjusting integration step with distance: " << diff << std::endl; epsilon = epsilon*0.95*std::pow(1e-4/diff,1./3.); - std::cout << GridLogMessage << "New epsilon : " << epsilon << std::endl; + //std::cout << GridLogMessage << "New epsilon : " << epsilon << std::endl; } @@ -166,7 +166,6 @@ void WilsonFlow::smear(GaugeField& out, const GaugeField& in) const { out = in; for (unsigned int step = 1; step <= Nstep; step++) { auto start = std::chrono::high_resolution_clock::now(); - std::cout << GridLogMessage << "Evolution time :"<< tau(step) << std::endl; evolve_step(out); auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration diff = end - start; @@ -191,7 +190,7 @@ void WilsonFlow::smear_adaptive(GaugeField& out, const GaugeField& in, Re unsigned int step = 0; do{ step++; - std::cout << GridLogMessage << "Evolution time :"<< taus << std::endl; + //std::cout << GridLogMessage << "Evolution time :"<< taus << std::endl; evolve_step_adaptive(out, maxTau); std::cout << GridLogMessage << "[WilsonFlow] Energy density (plaq) : " << step << " " diff --git a/lib/qcd/utils/GaugeFix.h b/lib/qcd/utils/GaugeFix.h new file mode 100644 index 00000000..c4ea31aa --- /dev/null +++ b/lib/qcd/utils/GaugeFix.h @@ -0,0 +1,193 @@ + /************************************************************************************* + + grid` physics library, www.github.com/paboyle/Grid + + Copyright (C) 2015 + +Author: Azusa Yamaguchi +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +//#include + +#ifndef GRID_QCD_GAUGE_FIX_H +#define GRID_QCD_GAUGE_FIX_H +namespace Grid { +namespace QCD { + +template +class FourierAcceleratedGaugeFixer : public Gimpl { + public: + INHERIT_GIMPL_TYPES(Gimpl); + + typedef typename Gimpl::GaugeLinkField GaugeMat; + typedef typename Gimpl::GaugeField GaugeLorentz; + + static void GaugeLinkToLieAlgebraField(const std::vector &U,std::vector &A) { + for(int mu=0;mu &A,GaugeMat &dmuAmu) { + dmuAmu=zero; + for(int mu=0;mu::avgPlaquette(Umu); + Real org_link_trace=WilsonLoops::linkTrace(Umu); + Real old_trace = org_link_trace; + Real trG; + + std::vector U(Nd,grid); + GaugeMat dmuAmu(grid); + + for(int i=0;i(Umu,mu); + if ( Fourier==false ) { + trG = SteepestDescentStep(U,alpha,dmuAmu); + } else { + trG = FourierAccelSteepestDescentStep(U,alpha,dmuAmu); + } + for(int mu=0;mu(Umu,U[mu],mu); + // Monitor progress and convergence test + // infrequently to minimise cost overhead + if ( i %20 == 0 ) { + Real plaq =WilsonLoops::avgPlaquette(Umu); + Real link_trace=WilsonLoops::linkTrace(Umu); + + if (Fourier) + std::cout << GridLogMessage << "Fourier Iteration "< &U,Real & alpha, GaugeMat & dmuAmu) { + GridBase *grid = U[0]._grid; + + std::vector A(Nd,grid); + GaugeMat g(grid); + + GaugeLinkToLieAlgebraField(U,A); + ExpiAlphaDmuAmu(A,g,alpha,dmuAmu); + + + Real vol = grid->gSites(); + Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc; + + SU::GaugeTransform(U,g); + + return trG; + } + + static Real FourierAccelSteepestDescentStep(std::vector &U,Real & alpha, GaugeMat & dmuAmu) { + + GridBase *grid = U[0]._grid; + + Real vol = grid->gSites(); + + FFT theFFT((GridCartesian *)grid); + + LatticeComplex Fp(grid); + LatticeComplex psq(grid); psq=zero; + LatticeComplex pmu(grid); + LatticeComplex one(grid); one = Complex(1.0,0.0); + + GaugeMat g(grid); + GaugeMat dmuAmu_p(grid); + std::vector A(Nd,grid); + + GaugeLinkToLieAlgebraField(U,A); + + DmuAmu(A,dmuAmu); + + theFFT.FFT_all_dim(dmuAmu_p,dmuAmu,FFT::forward); + + ////////////////////////////////// + // Work out Fp = psq_max/ psq... + ////////////////////////////////// + std::vector latt_size = grid->GlobalDimensions(); + std::vector coor(grid->_ndimension,0); + for(int mu=0;mu::taExp(ciadmam,g); + + Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc; + + SU::GaugeTransform(U,g); + + return trG; + } + + static void ExpiAlphaDmuAmu(const std::vector &A,GaugeMat &g,Real & alpha, GaugeMat &dmuAmu) { + GridBase *grid = g._grid; + Complex cialpha(0.0,-alpha); + GaugeMat ciadmam(grid); + DmuAmu(A,dmuAmu); + ciadmam = dmuAmu*cialpha; + SU::taExp(ciadmam,g); + } +}; + +} +} +#endif diff --git a/lib/qcd/utils/SUn.h b/lib/qcd/utils/SUn.h index 99a620bc..8f0c0a7b 100644 --- a/lib/qcd/utils/SUn.h +++ b/lib/qcd/utils/SUn.h @@ -716,8 +716,7 @@ template for (int a = 0; a < AdjointDimension; a++) { generator(a, Ta); - auto tmp = - 2.0 * (trace(timesI(Ta) * in)) * scale;// 2.0 for the normalization of the trace in the fundamental rep - pokeColour(h_out, tmp, a); + pokeColour(h_out, - 2.0 * (trace(timesI(Ta) * in)) * scale, a); } } diff --git a/lib/qcd/utils/Utils.h b/lib/qcd/utils/Utils.h index 61c81cb5..1786db54 100644 --- a/lib/qcd/utils/Utils.h +++ b/lib/qcd/utils/Utils.h @@ -12,7 +12,4 @@ #include #include - - - #endif diff --git a/lib/qcd/utils/WilsonLoops.h b/lib/qcd/utils/WilsonLoops.h index 3c0e9ee4..90d905d3 100644 --- a/lib/qcd/utils/WilsonLoops.h +++ b/lib/qcd/utils/WilsonLoops.h @@ -73,7 +73,7 @@ public: ////////////////////////////////////////////////// // trace of directed plaquette oriented in mu,nu plane ////////////////////////////////////////////////// - static void traceDirPlaquette(LatticeComplex &plaq, + static void traceDirPlaquette(ComplexField &plaq, const std::vector &U, const int mu, const int nu) { GaugeMat sp(U[0]._grid); @@ -83,9 +83,9 @@ public: ////////////////////////////////////////////////// // sum over all planes of plaquette ////////////////////////////////////////////////// - static void sitePlaquette(LatticeComplex &Plaq, + static void sitePlaquette(ComplexField &Plaq, const std::vector &U) { - LatticeComplex sitePlaq(U[0]._grid); + ComplexField sitePlaq(U[0]._grid); Plaq = zero; for (int mu = 1; mu < Nd; mu++) { for (int nu = 0; nu < mu; nu++) { @@ -104,11 +104,11 @@ public: U[mu] = PeekIndex(Umu, mu); } - LatticeComplex Plaq(Umu._grid); + ComplexField Plaq(Umu._grid); sitePlaquette(Plaq, U); - TComplex Tp = sum(Plaq); - Complex p = TensorRemove(Tp); + auto Tp = sum(Plaq); + auto p = TensorRemove(Tp); return p.real(); } @@ -129,15 +129,15 @@ public: static RealD linkTrace(const GaugeLorentz &Umu) { std::vector U(Nd, Umu._grid); - LatticeComplex Tr(Umu._grid); + ComplexField Tr(Umu._grid); Tr = zero; for (int mu = 0; mu < Nd; mu++) { U[mu] = PeekIndex(Umu, mu); Tr = Tr + trace(U[mu]); } - TComplex Tp = sum(Tr); - Complex p = TensorRemove(Tp); + auto Tp = sum(Tr); + auto p = TensorRemove(Tp); double vol = Umu._grid->gSites(); @@ -355,8 +355,8 @@ static void StapleMult(GaugeMat &staple, const GaugeLorentz &Umu, int mu) { double coeff = 8.0/(32.0*M_PI*M_PI); - LatticeComplex qfield = coeff*trace(Bx*Ex + By*Ey + Bz*Ez); - TComplex Tq = sum(qfield); + ComplexField qfield = coeff*trace(Bx*Ex + By*Ey + Bz*Ez); + auto Tq = sum(qfield); return TensorRemove(Tq).real(); } @@ -375,16 +375,16 @@ static void StapleMult(GaugeMat &staple, const GaugeLorentz &Umu, int mu) { adj(Gimpl::CovShiftForward( U[nu], nu, Gimpl::CovShiftForward(U[nu], nu, U[mu]))); } - static void traceDirRectangle(LatticeComplex &rect, + static void traceDirRectangle(ComplexField &rect, const std::vector &U, const int mu, const int nu) { GaugeMat sp(U[0]._grid); dirRectangle(sp, U, mu, nu); rect = trace(sp); } - static void siteRectangle(LatticeComplex &Rect, + static void siteRectangle(ComplexField &Rect, const std::vector &U) { - LatticeComplex siteRect(U[0]._grid); + ComplexField siteRect(U[0]._grid); Rect = zero; for (int mu = 1; mu < Nd; mu++) { for (int nu = 0; nu < mu; nu++) { @@ -404,12 +404,12 @@ static void StapleMult(GaugeMat &staple, const GaugeLorentz &Umu, int mu) { U[mu] = PeekIndex(Umu, mu); } - LatticeComplex Rect(Umu._grid); + ComplexField Rect(Umu._grid); siteRectangle(Rect, U); - TComplex Tp = sum(Rect); - Complex p = TensorRemove(Tp); + auto Tp = sum(Rect); + auto p = TensorRemove(Tp); return p.real(); } ////////////////////////////////////////////////// diff --git a/lib/serialisation/Hdf5IO.cc b/lib/serialisation/Hdf5IO.cc index b9bb0b87..1fb7be0c 100644 --- a/lib/serialisation/Hdf5IO.cc +++ b/lib/serialisation/Hdf5IO.cc @@ -65,10 +65,12 @@ Hdf5Reader::Hdf5Reader(const std::string &fileName) Hdf5Type::type()); } -void Hdf5Reader::push(const std::string &s) +bool Hdf5Reader::push(const std::string &s) { group_ = group_.openGroup(s); path_.push_back(s); + + return true; } void Hdf5Reader::pop(void) diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h index 2f891cd4..94ad9736 100644 --- a/lib/serialisation/Hdf5IO.h +++ b/lib/serialisation/Hdf5IO.h @@ -54,7 +54,7 @@ namespace Grid public: Hdf5Reader(const std::string &fileName); virtual ~Hdf5Reader(void) = default; - void push(const std::string &s); + bool push(const std::string &s); void pop(void); template void readDefault(const std::string &s, U &output); diff --git a/lib/serialisation/MacroMagic.h b/lib/serialisation/MacroMagic.h index a864989c..774c947f 100644 --- a/lib/serialisation/MacroMagic.h +++ b/lib/serialisation/MacroMagic.h @@ -110,11 +110,12 @@ THE SOFTWARE. #define GRID_MACRO_MEMBER(A,B) A B; #define GRID_MACRO_COMP_MEMBER(A,B) result = (result and (lhs. B == rhs. B)); -#define GRID_MACRO_OS_WRITE_MEMBER(A,B) os<< #A <<" "#B <<" = "<< obj. B <<" ; " <\ static inline void write(Writer &WR,const std::string &s, const cname &obj){ \ diff --git a/lib/serialisation/XmlIO.cc b/lib/serialisation/XmlIO.cc index b04263c9..a132a2f0 100644 --- a/lib/serialisation/XmlIO.cc +++ b/lib/serialisation/XmlIO.cc @@ -32,16 +32,21 @@ using namespace Grid; using namespace std; // Writer implementation /////////////////////////////////////////////////////// -XmlWriter::XmlWriter(const string &fileName) -: fileName_(fileName) +XmlWriter::XmlWriter(const string &fileName, string toplev) : fileName_(fileName) { - node_ = doc_.append_child(); - node_.set_name("grid"); + if ( toplev == std::string("") ) { + node_=doc_; + } else { + node_=doc_.append_child(); + node_.set_name(toplev.c_str()); + } } XmlWriter::~XmlWriter(void) { - doc_.save_file(fileName_.c_str(), " "); + if ( fileName_ != std::string("") ) { + doc_.save_file(fileName_.c_str(), " "); + } } void XmlWriter::push(const string &s) @@ -53,21 +58,44 @@ void XmlWriter::pop(void) { node_ = node_.parent(); } - -// Reader implementation /////////////////////////////////////////////////////// -XmlReader::XmlReader(const string &fileName) -: fileName_(fileName) +std::string XmlWriter::XmlString(void) { - pugi::xml_parse_result result = doc_.load_file(fileName_.c_str()); - - if ( !result ) - { + std::ostringstream oss; + doc_.save(oss); + return oss.str(); +} + +XmlReader::XmlReader(const char *xmlstring,string toplev) : fileName_("") +{ + pugi::xml_parse_result result; + result = doc_.load_string(xmlstring); + if ( !result ) { cerr << "XML error description: " << result.description() << "\n"; cerr << "XML error offset : " << result.offset << "\n"; abort(); } - - node_ = doc_.child("grid"); + if ( toplev == std::string("") ) { + node_ = doc_; + } else { + node_ = doc_.child(toplev.c_str()); + } +} + +// Reader implementation /////////////////////////////////////////////////////// +XmlReader::XmlReader(const string &fileName,string toplev) : fileName_(fileName) +{ + pugi::xml_parse_result result; + result = doc_.load_file(fileName_.c_str()); + if ( !result ) { + cerr << "XML error description: " << result.description() << "\n"; + cerr << "XML error offset : " << result.offset << "\n"; + abort(); + } + if ( toplev == std::string("") ) { + node_ = doc_; + } else { + node_ = doc_.child(toplev.c_str()); + } } bool XmlReader::push(const string &s) diff --git a/lib/serialisation/XmlIO.h b/lib/serialisation/XmlIO.h index f333b9aa..fcdbf1e4 100644 --- a/lib/serialisation/XmlIO.h +++ b/lib/serialisation/XmlIO.h @@ -44,10 +44,9 @@ namespace Grid { class XmlWriter: public Writer - { - + { public: - XmlWriter(const std::string &fileName); + XmlWriter(const std::string &fileName,std::string toplev = std::string("grid") ); virtual ~XmlWriter(void); void push(const std::string &s); void pop(void); @@ -55,6 +54,7 @@ namespace Grid void writeDefault(const std::string &s, const U &x); template void writeDefault(const std::string &s, const std::vector &x); + std::string XmlString(void); private: pugi::xml_document doc_; pugi::xml_node node_; @@ -64,7 +64,8 @@ namespace Grid class XmlReader: public Reader { public: - XmlReader(const std::string &fileName); + XmlReader(const char *xmlstring,std::string toplev = std::string("grid") ); + XmlReader(const std::string &fileName,std::string toplev = std::string("grid") ); virtual ~XmlReader(void) = default; bool push(const std::string &s); void pop(void); @@ -118,7 +119,7 @@ namespace Grid std::string buf; readDefault(s, buf); - std::cout << s << " " << buf << std::endl; + // std::cout << s << " " << buf << std::endl; fromString(output, buf); } diff --git a/lib/simd/Grid_avx.h b/lib/simd/Grid_avx.h index 52be9c05..f4634432 100644 --- a/lib/simd/Grid_avx.h +++ b/lib/simd/Grid_avx.h @@ -701,9 +701,28 @@ namespace Optimization { //Integer Reduce template<> inline Integer Reduce::operator()(__m256i in){ - // FIXME unimplemented - printf("Reduce : Missing integer implementation -> FIX\n"); - assert(0); + __m128i ret; +#if defined (AVX2) + // AVX2 horizontal adds within upper and lower halves of register; use + // SSE to add upper and lower halves for result. + __m256i v1, v2; + __m128i u1, u2; + v1 = _mm256_hadd_epi32(in, in); + v2 = _mm256_hadd_epi32(v1, v1); + u1 = _mm256_castsi256_si128(v2); // upper half + u2 = _mm256_extracti128_si256(v2, 1); // lower half + ret = _mm_add_epi32(u1, u2); +#else + // No AVX horizontal add; extract upper and lower halves of register & use + // SSE intrinsics. + __m128i u1, u2, u3; + u1 = _mm256_extractf128_si256(in, 0); // upper half + u2 = _mm256_extractf128_si256(in, 1); // lower half + u3 = _mm_add_epi32(u1, u2); + u1 = _mm_hadd_epi32(u3, u3); + ret = _mm_hadd_epi32(u1, u1); +#endif + return _mm_cvtsi128_si32(ret); } } diff --git a/lib/simd/Grid_avx512.h b/lib/simd/Grid_avx512.h index ba054665..85d27421 100644 --- a/lib/simd/Grid_avx512.h +++ b/lib/simd/Grid_avx512.h @@ -543,6 +543,24 @@ namespace Optimization { u512d conv; conv.v = v1; return conv.f[0]; } + + //Integer Reduce + template<> + inline Integer Reduce::operator()(__m512i in){ + // No full vector reduce, use AVX to add upper and lower halves of register + // and perform AVX reduction. + __m256i v1, v2, v3; + __m128i u1, u2, ret; + v1 = _mm512_castsi512_si256(in); // upper half + v2 = _mm512_extracti32x8_epi32(in, 1); // lower half + v3 = _mm256_add_epi32(v1, v2); + v1 = _mm256_hadd_epi32(v3, v3); + v2 = _mm256_hadd_epi32(v1, v1); + u1 = _mm256_castsi256_si128(v2) // upper half + u2 = _mm256_extracti128_si256(v2, 1); // lower half + ret = _mm_add_epi32(u1, u2); + return _mm_cvtsi128_si32(ret); + } #else //Complex float Reduce template<> @@ -570,9 +588,7 @@ namespace Optimization { //Integer Reduce template<> inline Integer Reduce::operator()(__m512i in){ - // FIXME unimplemented - printf("Reduce : Missing integer implementation -> FIX\n"); - assert(0); + return _mm512_reduce_add_epi32(in); } #endif diff --git a/lib/simd/Grid_imci.h b/lib/simd/Grid_imci.h index 173e57d8..a1dae565 100644 --- a/lib/simd/Grid_imci.h +++ b/lib/simd/Grid_imci.h @@ -401,9 +401,7 @@ namespace Optimization { //Integer Reduce template<> inline Integer Reduce::operator()(__m512i in){ - // FIXME unimplemented - printf("Reduce : Missing integer implementation -> FIX\n"); - assert(0); + return _mm512_reduce_add_epi32(in); } diff --git a/lib/simd/Grid_neon.h b/lib/simd/Grid_neon.h index 7c1ad443..d6eb9c5a 100644 --- a/lib/simd/Grid_neon.h +++ b/lib/simd/Grid_neon.h @@ -1,13 +1,14 @@ - /************************************************************************************* +/************************************************************************************* - Grid physics library, www.github.com/paboyle/Grid + Grid physics library, www.github.com/paboyle/Grid Source file: ./lib/simd/Grid_neon.h Copyright (C) 2015 -Author: Peter Boyle -Author: neo + Author: Nils Meyer + Author: Peter Boyle + Author: neo This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -26,19 +27,25 @@ Author: neo See the full license in the file "LICENSE" in the top level distribution directory *************************************************************************************/ /* END LEGAL */ -//---------------------------------------------------------------------- -/*! @file Grid_sse4.h - @brief Optimization libraries for NEON (ARM) instructions set ARMv8 - Experimental - Using intrinsics - DEVELOPING! +/* + + ARMv8 NEON intrinsics layer by + + Nils Meyer , + University of Regensburg, Germany + SFB/TRR55 + */ -// Time-stamp: <2015-07-10 17:45:09 neo> -//---------------------------------------------------------------------- +#ifndef GEN_SIMD_WIDTH +#define GEN_SIMD_WIDTH 16u +#endif + +#include "Grid_generic_types.h" #include -// ARMv8 supports double precision - +namespace Grid { namespace Optimization { template @@ -46,16 +53,20 @@ namespace Optimization { float32x4_t f; vtype v; }; - union u128f { float32x4_t v; float f[4]; }; union u128d { float64x2_t v; - double f[4]; + double f[2]; }; - + // half precision + union u128h { + float16x8_t v; + uint16_t f[8]; + }; + struct Vsplat{ //Complex float inline float32x4_t operator()(float a, float b){ @@ -64,31 +75,31 @@ namespace Optimization { } // Real float inline float32x4_t operator()(float a){ - return vld1q_dup_f32(&a); + return vdupq_n_f32(a); } //Complex double - inline float32x4_t operator()(double a, double b){ - float tmp[4]={(float)a,(float)b,(float)a,(float)b}; - return vld1q_f32(tmp); + inline float64x2_t operator()(double a, double b){ + double tmp[2]={a,b}; + return vld1q_f64(tmp); } - //Real double - inline float32x4_t operator()(double a){ - return vld1q_dup_f32(&a); + //Real double // N:tbc + inline float64x2_t operator()(double a){ + return vdupq_n_f64(a); } - //Integer + //Integer // N:tbc inline uint32x4_t operator()(Integer a){ - return vld1q_dup_u32(&a); + return vdupq_n_u32(a); } }; struct Vstore{ - //Float + //Float inline void operator()(float32x4_t a, float* F){ vst1q_f32(F, a); } //Double - inline void operator()(float32x4_t a, double* D){ - vst1q_f32((float*)D, a); + inline void operator()(float64x2_t a, double* D){ + vst1q_f64(D, a); } //Integer inline void operator()(uint32x4_t a, Integer* I){ @@ -97,54 +108,54 @@ namespace Optimization { }; - struct Vstream{ - //Float + struct Vstream{ // N:equivalents to _mm_stream_p* in NEON? + //Float // N:generic inline void operator()(float * a, float32x4_t b){ - + memcpy(a,&b,4*sizeof(float)); } - //Double - inline void operator()(double * a, float32x4_t b){ - + //Double // N:generic + inline void operator()(double * a, float64x2_t b){ + memcpy(a,&b,2*sizeof(double)); } }; + // Nils: Vset untested; not used currently in Grid at all; + // git commit 4a8c4ccfba1d05159348d21a9698028ea847e77b struct Vset{ - // Complex float + // Complex float // N:ok inline float32x4_t operator()(Grid::ComplexF *a){ - float32x4_t foo; - return foo; + float tmp[4]={a[1].imag(),a[1].real(),a[0].imag(),a[0].real()}; + return vld1q_f32(tmp); } - // Complex double - inline float32x4_t operator()(Grid::ComplexD *a){ - float32x4_t foo; - return foo; + // Complex double // N:ok + inline float64x2_t operator()(Grid::ComplexD *a){ + double tmp[2]={a[0].imag(),a[0].real()}; + return vld1q_f64(tmp); } - // Real float + // Real float // N:ok inline float32x4_t operator()(float *a){ - float32x4_t foo; - return foo; + float tmp[4]={a[3],a[2],a[1],a[0]}; + return vld1q_f32(tmp); } - // Real double - inline float32x4_t operator()(double *a){ - float32x4_t foo; - return foo; + // Real double // N:ok + inline float64x2_t operator()(double *a){ + double tmp[2]={a[1],a[0]}; + return vld1q_f64(tmp); } - // Integer + // Integer // N:ok inline uint32x4_t operator()(Integer *a){ - uint32x4_t foo; - return foo; + return vld1q_dup_u32(a); } - - }; + // N:leaving as is template struct Reduce{ //Need templated class to overload output type //General form must generate error if compiled - inline Out_type operator()(In_type in){ + inline Out_type operator()(In_type in){ printf("Error, using wrong Reduce function\n"); exit(1); return 0; @@ -184,26 +195,98 @@ namespace Optimization { } }; + struct MultRealPart{ + inline float32x4_t operator()(float32x4_t a, float32x4_t b){ + float32x4_t re = vtrn1q_f32(a, a); + return vmulq_f32(re, b); + } + inline float64x2_t operator()(float64x2_t a, float64x2_t b){ + float64x2_t re = vzip1q_f64(a, a); + return vmulq_f64(re, b); + } + }; + + struct MaddRealPart{ + inline float32x4_t operator()(float32x4_t a, float32x4_t b, float32x4_t c){ + float32x4_t re = vtrn1q_f32(a, a); + return vfmaq_f32(c, re, b); + } + inline float64x2_t operator()(float64x2_t a, float64x2_t b, float64x2_t c){ + float64x2_t re = vzip1q_f64(a, a); + return vfmaq_f64(c, re, b); + } + }; + + struct Div{ + // Real float + inline float32x4_t operator()(float32x4_t a, float32x4_t b){ + return vdivq_f32(a, b); + } + // Real double + inline float64x2_t operator()(float64x2_t a, float64x2_t b){ + return vdivq_f64(a, b); + } + }; + struct MultComplex{ // Complex float inline float32x4_t operator()(float32x4_t a, float32x4_t b){ - float32x4_t foo; - return foo; + + float32x4_t r0, r1, r2, r3, r4; + + // a = ar ai Ar Ai + // b = br bi Br Bi + // collect real/imag part, negate bi and Bi + r0 = vtrn1q_f32(b, b); // br br Br Br + r1 = vnegq_f32(b); // -br -bi -Br -Bi + r2 = vtrn2q_f32(b, r1); // bi -bi Bi -Bi + + // the fun part + r3 = vmulq_f32(r2, a); // bi*ar -bi*ai ... + r4 = vrev64q_f32(r3); // -bi*ai bi*ar ... + + // fma(a,b,c) = a+b*c + return vfmaq_f32(r4, r0, a); // ar*br-ai*bi ai*br+ar*bi ... + + // no fma, use mul and add + //float32x4_t r5; + //r5 = vmulq_f32(r0, a); + //return vaddq_f32(r4, r5); } // Complex double inline float64x2_t operator()(float64x2_t a, float64x2_t b){ - float32x4_t foo; - return foo; + + float64x2_t r0, r1, r2, r3, r4; + + // b = br bi + // collect real/imag part, negate bi + r0 = vtrn1q_f64(b, b); // br br + r1 = vnegq_f64(b); // -br -bi + r2 = vtrn2q_f64(b, r1); // bi -bi + + // the fun part + r3 = vmulq_f64(r2, a); // bi*ar -bi*ai + r4 = vextq_f64(r3,r3,1); // -bi*ai bi*ar + + // fma(a,b,c) = a+b*c + return vfmaq_f64(r4, r0, a); // ar*br-ai*bi ai*br+ar*bi + + // no fma, use mul and add + //float64x2_t r5; + //r5 = vmulq_f64(r0, a); + //return vaddq_f64(r4, r5); } }; struct Mult{ // Real float inline float32x4_t mac(float32x4_t a, float32x4_t b, float32x4_t c){ - return vaddq_f32(vmulq_f32(b,c),a); + //return vaddq_f32(vmulq_f32(b,c),a); + return vfmaq_f32(a, b, c); } inline float64x2_t mac(float64x2_t a, float64x2_t b, float64x2_t c){ - return vaddq_f64(vmulq_f64(b,c),a); + //return vaddq_f64(vmulq_f64(b,c),a); + return vfmaq_f64(a, b, c); } inline float32x4_t operator()(float32x4_t a, float32x4_t b){ return vmulq_f32(a,b); @@ -221,89 +304,275 @@ namespace Optimization { struct Conj{ // Complex single inline float32x4_t operator()(float32x4_t in){ - return in; + // ar ai br bi -> ar -ai br -bi + float32x4_t r0, r1; + r0 = vnegq_f32(in); // -ar -ai -br -bi + r1 = vrev64q_f32(r0); // -ai -ar -bi -br + return vtrn1q_f32(in, r1); // ar -ai br -bi } // Complex double - //inline float32x4_t operator()(float32x4_t in){ - // return 0; - //} + inline float64x2_t operator()(float64x2_t in){ + + float64x2_t r0, r1; + r0 = vextq_f64(in, in, 1); // ai ar + r1 = vnegq_f64(r0); // -ai -ar + return vextq_f64(r0, r1, 1); // ar -ai + } // do not define for integer input }; struct TimesMinusI{ //Complex single inline float32x4_t operator()(float32x4_t in, float32x4_t ret){ - return in; + // ar ai br bi -> ai -ar ai -br + float32x4_t r0, r1; + r0 = vnegq_f32(in); // -ar -ai -br -bi + r1 = vrev64q_f32(in); // ai ar bi br + return vtrn1q_f32(r1, r0); // ar -ai br -bi } //Complex double - //inline float32x4_t operator()(float32x4_t in, float32x4_t ret){ - // return in; - //} - - + inline float64x2_t operator()(float64x2_t in, float64x2_t ret){ + // a ib -> b -ia + float64x2_t tmp; + tmp = vnegq_f64(in); + return vextq_f64(in, tmp, 1); + } }; struct TimesI{ //Complex single inline float32x4_t operator()(float32x4_t in, float32x4_t ret){ - //need shuffle - return in; + // ar ai br bi -> -ai ar -bi br + float32x4_t r0, r1; + r0 = vnegq_f32(in); // -ar -ai -br -bi + r1 = vrev64q_f32(r0); // -ai -ar -bi -br + return vtrn1q_f32(r1, in); // -ai ar -bi br } //Complex double - //inline float32x4_t operator()(float32x4_t in, float32x4_t ret){ - // return 0; - //} + inline float64x2_t operator()(float64x2_t in, float64x2_t ret){ + // a ib -> -b ia + float64x2_t tmp; + tmp = vnegq_f64(in); + return vextq_f64(tmp, in, 1); + } + }; + + struct Permute{ + + static inline float32x4_t Permute0(float32x4_t in){ // N:ok + // AB CD -> CD AB + return vextq_f32(in, in, 2); + }; + static inline float32x4_t Permute1(float32x4_t in){ // N:ok + // AB CD -> BA DC + return vrev64q_f32(in); + }; + static inline float32x4_t Permute2(float32x4_t in){ // N:not used by Boyle + return in; + }; + static inline float32x4_t Permute3(float32x4_t in){ // N:not used by Boyle + return in; + }; + + static inline float64x2_t Permute0(float64x2_t in){ // N:ok + // AB -> BA + return vextq_f64(in, in, 1); + }; + static inline float64x2_t Permute1(float64x2_t in){ // N:not used by Boyle + return in; + }; + static inline float64x2_t Permute2(float64x2_t in){ // N:not used by Boyle + return in; + }; + static inline float64x2_t Permute3(float64x2_t in){ // N:not used by Boyle + return in; + }; + + }; + + struct Rotate{ + + static inline float32x4_t rotate(float32x4_t in,int n){ // N:ok + switch(n){ + case 0: // AB CD -> AB CD + return tRotate<0>(in); + break; + case 1: // AB CD -> BC DA + return tRotate<1>(in); + break; + case 2: // AB CD -> CD AB + return tRotate<2>(in); + break; + case 3: // AB CD -> DA BC + return tRotate<3>(in); + break; + default: assert(0); + } + } + static inline float64x2_t rotate(float64x2_t in,int n){ // N:ok + switch(n){ + case 0: // AB -> AB + return tRotate<0>(in); + break; + case 1: // AB -> BA + return tRotate<1>(in); + break; + default: assert(0); + } + } + +// working, but no restriction on n +// template static inline float32x4_t tRotate(float32x4_t in){ return vextq_f32(in,in,n); }; +// template static inline float64x2_t tRotate(float64x2_t in){ return vextq_f64(in,in,n); }; + +// restriction on n + template static inline float32x4_t tRotate(float32x4_t in){ return vextq_f32(in,in,n%4); }; + template static inline float64x2_t tRotate(float64x2_t in){ return vextq_f64(in,in,n%2); }; + + }; + + struct PrecisionChange { + + static inline float16x8_t StoH (const float32x4_t &a,const float32x4_t &b) { + float16x4_t h = vcvt_f16_f32(a); + return vcvt_high_f16_f32(h, b); + } + static inline void HtoS (float16x8_t h,float32x4_t &sa,float32x4_t &sb) { + sb = vcvt_high_f32_f16(h); + // there is no direct conversion from lower float32x4_t to float64x2_t + // vextq_f16 not supported by clang 3.8 / 4.0 / arm clang + //float16x8_t h1 = vextq_f16(h, h, 4); // correct, but not supported by clang + // workaround for clang + uint32x4_t h1u = reinterpret_cast(h); + float16x8_t h1 = reinterpret_cast(vextq_u32(h1u, h1u, 2)); + sa = vcvt_high_f32_f16(h1); + } + static inline float32x4_t DtoS (float64x2_t a,float64x2_t b) { + float32x2_t s = vcvt_f32_f64(a); + return vcvt_high_f32_f64(s, b); + + } + static inline void StoD (float32x4_t s,float64x2_t &a,float64x2_t &b) { + b = vcvt_high_f64_f32(s); + // there is no direct conversion from lower float32x4_t to float64x2_t + float32x4_t s1 = vextq_f32(s, s, 2); + a = vcvt_high_f64_f32(s1); + + } + static inline float16x8_t DtoH (float64x2_t a,float64x2_t b,float64x2_t c,float64x2_t d) { + float32x4_t s1 = DtoS(a, b); + float32x4_t s2 = DtoS(c, d); + return StoH(s1, s2); + } + static inline void HtoD (float16x8_t h,float64x2_t &a,float64x2_t &b,float64x2_t &c,float64x2_t &d) { + float32x4_t s1, s2; + HtoS(h, s1, s2); + StoD(s1, a, b); + StoD(s2, c, d); + } + }; + + ////////////////////////////////////////////// + // Exchange support + + struct Exchange{ + static inline void Exchange0(float32x4_t &out1,float32x4_t &out2,float32x4_t in1,float32x4_t in2){ + // in1: ABCD -> out1: ABEF + // in2: EFGH -> out2: CDGH + + // z: CDAB + float32x4_t z = vextq_f32(in1, in1, 2); + // out1: ABEF + out1 = vextq_f32(z, in2, 2); + + // z: GHEF + z = vextq_f32(in2, in2, 2); + // out2: CDGH + out2 = vextq_f32(in1, z, 2); + }; + + static inline void Exchange1(float32x4_t &out1,float32x4_t &out2,float32x4_t in1,float32x4_t in2){ + // in1: ABCD -> out1: AECG + // in2: EFGH -> out2: BFDH + out1 = vtrn1q_f32(in1, in2); + out2 = vtrn2q_f32(in1, in2); + }; + static inline void Exchange2(float32x4_t &out1,float32x4_t &out2,float32x4_t in1,float32x4_t in2){ + assert(0); + return; + }; + static inline void Exchange3(float32x4_t &out1,float32x4_t &out2,float32x4_t in1,float32x4_t in2){ + assert(0); + return; + }; + // double precision + static inline void Exchange0(float64x2_t &out1,float64x2_t &out2,float64x2_t in1,float64x2_t in2){ + // in1: AB -> out1: AC + // in2: CD -> out2: BD + out1 = vzip1q_f64(in1, in2); + out2 = vzip2q_f64(in1, in2); + }; + static inline void Exchange1(float64x2_t &out1,float64x2_t &out2,float64x2_t in1,float64x2_t in2){ + assert(0); + return; + }; + static inline void Exchange2(float64x2_t &out1,float64x2_t &out2,float64x2_t in1,float64x2_t in2){ + assert(0); + return; + }; + static inline void Exchange3(float64x2_t &out1,float64x2_t &out2,float64x2_t in1,float64x2_t in2){ + assert(0); + return; + }; }; ////////////////////////////////////////////// // Some Template specialization - template < typename vtype > - void permute(vtype &a, vtype b, int perm) { - }; //Complex float Reduce template<> inline Grid::ComplexF Reduce::operator()(float32x4_t in){ - return 0; + float32x4_t v1; // two complex + v1 = Optimization::Permute::Permute0(in); + v1 = vaddq_f32(v1,in); + u128f conv; conv.v=v1; + return Grid::ComplexF(conv.f[0],conv.f[1]); } //Real float Reduce template<> inline Grid::RealF Reduce::operator()(float32x4_t in){ - float32x2_t high = vget_high_f32(in); - float32x2_t low = vget_low_f32(in); - float32x2_t tmp = vadd_f32(low, high); - float32x2_t sum = vpadd_f32(tmp, tmp); - return vget_lane_f32(sum,0); + return vaddvq_f32(in); } - - + + //Complex double Reduce - template<> + template<> // N:by Boyle inline Grid::ComplexD Reduce::operator()(float64x2_t in){ - return 0; + u128d conv; conv.v = in; + return Grid::ComplexD(conv.f[0],conv.f[1]); } - + //Real double Reduce template<> inline Grid::RealD Reduce::operator()(float64x2_t in){ - float64x2_t sum = vpaddq_f64(in, in); - return vgetq_lane_f64(sum,0); + return vaddvq_f64(in); } //Integer Reduce template<> inline Integer Reduce::operator()(uint32x4_t in){ // FIXME unimplemented - printf("Reduce : Missing integer implementation -> FIX\n"); + printf("Reduce : Missing integer implementation -> FIX\n"); assert(0); } } ////////////////////////////////////////////////////////////////////////////////////// -// Here assign types -namespace Grid { +// Here assign types +// typedef Optimization::vech SIMD_Htype; // Reduced precision type + typedef float16x8_t SIMD_Htype; // Half precision type typedef float32x4_t SIMD_Ftype; // Single precision type typedef float64x2_t SIMD_Dtype; // Double precision type typedef uint32x4_t SIMD_Itype; // Integer type @@ -312,13 +581,6 @@ namespace Grid { inline void prefetch_HINT_T0(const char *ptr){}; - // Gpermute function - template < typename VectorSIMD > - inline void Gpermute(VectorSIMD &y,const VectorSIMD &b, int perm ) { - Optimization::permute(y.v,b.v,perm); - } - - // Function name aliases typedef Optimization::Vsplat VsplatSIMD; typedef Optimization::Vstore VstoreSIMD; @@ -326,16 +588,19 @@ namespace Grid { typedef Optimization::Vstream VstreamSIMD; template using ReduceSIMD = Optimization::Reduce; - + // Arithmetic operations typedef Optimization::Sum SumSIMD; typedef Optimization::Sub SubSIMD; + typedef Optimization::Div DivSIMD; typedef Optimization::Mult MultSIMD; typedef Optimization::MultComplex MultComplexSIMD; + typedef Optimization::MultRealPart MultRealPartSIMD; + typedef Optimization::MaddRealPart MaddRealPartSIMD; typedef Optimization::Conj ConjSIMD; typedef Optimization::TimesMinusI TimesMinusISIMD; typedef Optimization::TimesI TimesISIMD; -} +} \ No newline at end of file diff --git a/lib/simd/Grid_qpx.h b/lib/simd/Grid_qpx.h index cbca9118..8de7bde8 100644 --- a/lib/simd/Grid_qpx.h +++ b/lib/simd/Grid_qpx.h @@ -374,6 +374,84 @@ namespace Optimization { // Complex float FLOAT_WRAP_2(operator(), inline) }; +#define USE_FP16 + struct PrecisionChange { + static inline vech StoH (const vector4float &a, const vector4float &b) { + vech ret; + std::cout << GridLogError << "QPX single to half precision conversion not yet supported." << std::endl; + assert(0); + return ret; + } + static inline void HtoS (vech h, vector4float &sa, vector4float &sb) { + std::cout << GridLogError << "QPX half to single precision conversion not yet supported." << std::endl; + assert(0); + } + static inline vector4float DtoS (vector4double a, vector4double b) { + vector4float ret; + std::cout << GridLogError << "QPX double to single precision conversion not yet supported." << std::endl; + assert(0); + return ret; + } + static inline void StoD (vector4float s, vector4double &a, vector4double &b) { + std::cout << GridLogError << "QPX single to double precision conversion not yet supported." << std::endl; + assert(0); + } + static inline vech DtoH (vector4double a, vector4double b, + vector4double c, vector4double d) { + vech ret; + std::cout << GridLogError << "QPX double to half precision conversion not yet supported." << std::endl; + assert(0); + return ret; + } + static inline void HtoD (vech h, vector4double &a, vector4double &b, + vector4double &c, vector4double &d) { + std::cout << GridLogError << "QPX half to double precision conversion not yet supported." << std::endl; + assert(0); + } + }; + + ////////////////////////////////////////////// + // Exchange support +#define FLOAT_WRAP_EXCHANGE(fn) \ + static inline void fn(vector4float &out1, vector4float &out2, \ + vector4float in1, vector4float in2) \ + { \ + vector4double out1d, out2d, in1d, in2d; \ + in1d = Vset()(in1); \ + in2d = Vset()(in2); \ + fn(out1d, out2d, in1d, in2d); \ + Vstore()(out1d, out1); \ + Vstore()(out2d, out2); \ + } + + struct Exchange{ + + // double precision + static inline void Exchange0(vector4double &out1, vector4double &out2, + vector4double in1, vector4double in2) { + out1 = vec_perm(in1, in2, vec_gpci(0145)); + out2 = vec_perm(in1, in2, vec_gpci(02367)); + } + static inline void Exchange1(vector4double &out1, vector4double &out2, + vector4double in1, vector4double in2) { + out1 = vec_perm(in1, in2, vec_gpci(0426)); + out2 = vec_perm(in1, in2, vec_gpci(01537)); + } + static inline void Exchange2(vector4double &out1, vector4double &out2, + vector4double in1, vector4double in2) { + assert(0); + } + static inline void Exchange3(vector4double &out1, vector4double &out2, + vector4double in1, vector4double in2) { + assert(0); + } + + // single precision + FLOAT_WRAP_EXCHANGE(Exchange0); + FLOAT_WRAP_EXCHANGE(Exchange1); + FLOAT_WRAP_EXCHANGE(Exchange2); + FLOAT_WRAP_EXCHANGE(Exchange3); + }; struct Permute{ //Complex double @@ -497,15 +575,19 @@ namespace Optimization { //Integer Reduce template<> - inline Integer Reduce::operator()(int in){ - // FIXME unimplemented - printf("Reduce : Missing integer implementation -> FIX\n"); - assert(0); + inline Integer Reduce::operator()(veci in){ + Integer a = 0; + for (unsigned int i = 0; i < W::r; ++i) + { + a += in.v[i]; + } + return a; } } //////////////////////////////////////////////////////////////////////////////// // Here assign types +typedef Optimization::vech SIMD_Htype; // Half precision type typedef Optimization::vector4float SIMD_Ftype; // Single precision type typedef vector4double SIMD_Dtype; // Double precision type typedef Optimization::veci SIMD_Itype; // Integer type diff --git a/lib/simd/Grid_sse4.h b/lib/simd/Grid_sse4.h index 2fb2df76..0b1f9ffb 100644 --- a/lib/simd/Grid_sse4.h +++ b/lib/simd/Grid_sse4.h @@ -570,9 +570,9 @@ namespace Optimization { //Integer Reduce template<> inline Integer Reduce::operator()(__m128i in){ - // FIXME unimplemented - printf("Reduce : Missing integer implementation -> FIX\n"); - assert(0); + __m128i v1 = _mm_hadd_epi32(in, in); + __m128i v2 = _mm_hadd_epi32(v1, v1); + return _mm_cvtsi128_si32(v2); } } diff --git a/lib/simd/Grid_vector_types.h b/lib/simd/Grid_vector_types.h index 0048382f..27585547 100644 --- a/lib/simd/Grid_vector_types.h +++ b/lib/simd/Grid_vector_types.h @@ -53,7 +53,7 @@ directory #if defined IMCI #include "Grid_imci.h" #endif -#ifdef NEONv8 +#ifdef NEONV8 #include "Grid_neon.h" #endif #if defined QPX @@ -327,10 +327,6 @@ class Grid_simd { // provides support /////////////////////////////////////// - //#if (__GNUC__ == 5 ) || ( ( __GNUC__ == 6 ) && __GNUC_MINOR__ < 3 ) - //#pragma GCC push_options - //#pragma GCC optimize ("O0") - //#endif template friend inline Grid_simd SimdApply(const functor &func, const Grid_simd &v) { Grid_simd ret; @@ -364,9 +360,6 @@ class Grid_simd { ret.v = cx.v; return ret; } - //#if (__GNUC__ == 5 ) || ( ( __GNUC__ == 6 ) && __GNUC_MINOR__ < 3 ) - //#pragma GCC pop_options - //#endif /////////////////////// // Exchange // Al Ah , Bl Bh -> Al Bl Ah,Bh @@ -428,7 +421,6 @@ class Grid_simd { }; // end of Grid_simd class definition - inline void permute(ComplexD &y,ComplexD b, int perm) { y=b; } inline void permute(ComplexF &y,ComplexF b, int perm) { y=b; } inline void permute(RealD &y,RealD b, int perm) { y=b; } @@ -759,8 +751,8 @@ inline Grid_simd, V> toComplex(const Grid_simd &in) { conv.v = in.v; for (int i = 0; i < Rsimd::Nsimd(); i += 2) { - assert(conv.s[i + 1] == - conv.s[i]); // trap any cases where real was not duplicated + assert(conv.s[i + 1] == conv.s[i]); + // trap any cases where real was not duplicated // indicating the SIMD grids of real and imag assignment did not correctly // match conv.s[i + 1] = 0.0; // zero imaginary parts @@ -838,8 +830,6 @@ inline void precisionChange(vComplexD *out,vComplexF *in,int nvec){ precisionCha inline void precisionChange(vComplexD *out,vComplexH *in,int nvec){ precisionChange((vRealD *)out,(vRealH *)in,nvec);} inline void precisionChange(vComplexF *out,vComplexH *in,int nvec){ precisionChange((vRealF *)out,(vRealH *)in,nvec);} - - // Check our vector types are of an appropriate size. #if defined QPX static_assert(2*sizeof(SIMD_Ftype) == sizeof(SIMD_Dtype), "SIMD vector lengths incorrect"); @@ -854,21 +844,14 @@ static_assert(sizeof(SIMD_Ftype) == sizeof(SIMD_Itype), "SIMD vector lengths inc ///////////////////////////////////////// template struct is_simd : public std::false_type {}; -template <> -struct is_simd : public std::true_type {}; -template <> -struct is_simd : public std::true_type {}; -template <> -struct is_simd : public std::true_type {}; -template <> -struct is_simd : public std::true_type {}; -template <> -struct is_simd : public std::true_type {}; +template <> struct is_simd : public std::true_type {}; +template <> struct is_simd : public std::true_type {}; +template <> struct is_simd : public std::true_type {}; +template <> struct is_simd : public std::true_type {}; +template <> struct is_simd : public std::true_type {}; -template -using IfSimd = Invoke::value, int> >; -template -using IfNotSimd = Invoke::value, unsigned> >; +template using IfSimd = Invoke::value, int> >; +template using IfNotSimd = Invoke::value, unsigned> >; } #endif diff --git a/lib/simd/Grid_vector_unops.h b/lib/simd/Grid_vector_unops.h index 2afac190..2244566f 100644 --- a/lib/simd/Grid_vector_unops.h +++ b/lib/simd/Grid_vector_unops.h @@ -179,13 +179,6 @@ inline Grid_simd div(const Grid_simd &r, Integer y) { //////////////////////////////////////////////////////////////////////////// // Allows us to assign into **conformable** real vectors from complex //////////////////////////////////////////////////////////////////////////// -// template < class S, class V > -// inline auto ComplexRemove(const Grid_simd &c) -> -// Grid_simd::Real,V> { -// Grid_simd::Real,V> ret; -// ret.v = c.v; -// return ret; -// } template struct AndFunctor { scalar operator()(const scalar &x, const scalar &y) const { return x & y; } diff --git a/lib/stencil/Lebesgue.cc b/lib/stencil/Lebesgue.cc index 4551878c..2880e4b6 100644 --- a/lib/stencil/Lebesgue.cc +++ b/lib/stencil/Lebesgue.cc @@ -32,8 +32,11 @@ Author: paboyle namespace Grid { int LebesgueOrder::UseLebesgueOrder; +#ifdef KNL std::vector LebesgueOrder::Block({8,2,2,2}); - +#else +std::vector LebesgueOrder::Block({2,2,2,2}); +#endif LebesgueOrder::IndexInteger LebesgueOrder::alignup(IndexInteger n){ n--; // 1000 0011 --> 1000 0010 n |= n >> 1; // 1000 0010 | 0100 0001 = 1100 0011 @@ -51,8 +54,31 @@ LebesgueOrder::LebesgueOrder(GridBase *_grid) if ( Block[0]==0) ZGraph(); else if ( Block[1]==0) NoBlocking(); else CartesianBlocking(); -} + if (0) { + std::cout << "Thread Interleaving"< reorder = _LebesgueReorder; + std::vector throrder; + int vol = _LebesgueReorder.size(); + int threads = GridThread::GetThreads(); + int blockbits=3; + int blocklen = 8; + int msk = 0x7; + + for(int t=0;t> blockbits) % threads == t ) { + throrder.push_back(reorder[ss]); + } + } + } + _LebesgueReorder = throrder; +} void LebesgueOrder::NoBlocking(void) { std::cout< & xi, std::vector &dims); + void ThreadInterleave(void); + private: std::vector _LebesgueReorder; diff --git a/lib/stencil/Stencil.h b/lib/stencil/Stencil.h index 05a531fe..cd0792d5 100644 --- a/lib/stencil/Stencil.h +++ b/lib/stencil/Stencil.h @@ -176,6 +176,9 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal // Timing info; ugly; possibly temporary ///////////////////////////////////////// double commtime; + double mpi3synctime; + double mpi3synctime_g; + double shmmergetime; double gathertime; double gathermtime; double halogtime; @@ -185,6 +188,10 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal double splicetime; double nosplicetime; double calls; + std::vector comm_bytes_thr; + std::vector comm_time_thr; + std::vector comm_enter_thr; + std::vector comm_leave_thr; //////////////////////////////////////// // Stencil query @@ -248,35 +255,120 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal ////////////////////////////////////////// // Comms packet queue for asynch thread ////////////////////////////////////////// + void CommunicateThreaded() + { +#ifdef GRID_OMP + // must be called in parallel region + int mythread = omp_get_thread_num(); + int nthreads = CartesianCommunicator::nCommThreads; +#else + int mythread = 0; + int nthreads = 1; +#endif + if (nthreads == -1) nthreads = 1; + if (mythread < nthreads) { + comm_enter_thr[mythread] = usecond(); + for (int i = mythread; i < Packets.size(); i += nthreads) { + uint64_t bytes = _grid->StencilSendToRecvFrom(Packets[i].send_buf, + Packets[i].to_rank, + Packets[i].recv_buf, + Packets[i].from_rank, + Packets[i].bytes,i); + comm_bytes_thr[mythread] += bytes; + } + comm_leave_thr[mythread]= usecond(); + comm_time_thr[mythread] += comm_leave_thr[mythread] - comm_enter_thr[mythread]; + } + } + + void CollateThreads(void) + { + int nthreads = CartesianCommunicator::nCommThreads; + double first=0.0; + double last =0.0; + + for(int t=0;t 0.0) && ( t0 < first ) ) first = t0; // min time seen + + if ( t1 > last ) last = t1; // max time seen + + } + commtime+= last-first; + } void CommunicateBegin(std::vector > &reqs) { reqs.resize(Packets.size()); commtime-=usecond(); for(int i=0;iStencilSendToRecvFromBegin(reqs[i], - Packets[i].send_buf, - Packets[i].to_rank, - Packets[i].recv_buf, - Packets[i].from_rank, - Packets[i].bytes); + Packets[i].send_buf, + Packets[i].to_rank, + Packets[i].recv_buf, + Packets[i].from_rank, + Packets[i].bytes,i); } } void CommunicateComplete(std::vector > &reqs) { for(int i=0;iStencilSendToRecvFromComplete(reqs[i]); + _grid->StencilSendToRecvFromComplete(reqs[i],i); } commtime+=usecond(); } + void Communicate(void) + { +#ifdef GRID_OMP +#pragma omp parallel + { + // must be called in parallel region + int mythread = omp_get_thread_num(); + int maxthreads= omp_get_max_threads(); + int nthreads = CartesianCommunicator::nCommThreads; + assert(nthreads <= maxthreads); + + if (nthreads == -1) nthreads = 1; +#else + int mythread = 0; + int nthreads = 1; +#endif + if (mythread < nthreads) { + for (int i = mythread; i < Packets.size(); i += nthreads) { + double start = usecond(); + comm_bytes_thr[mythread] += _grid->StencilSendToRecvFrom(Packets[i].send_buf, + Packets[i].to_rank, + Packets[i].recv_buf, + Packets[i].from_rank, + Packets[i].bytes,i); + comm_time_thr[mythread] += usecond() - start; + } + } +#ifdef GRID_OMP + } +#endif + } template void HaloExchange(const Lattice &source,compressor &compress) { std::vector > reqs; Prepare(); HaloGather(source,compress); - CommunicateBegin(reqs); - CommunicateComplete(reqs); + // Concurrent + //CommunicateBegin(reqs); + //CommunicateComplete(reqs); + // Sequential, possibly threaded + Communicate(); CommsMergeSHM(compress); CommsMerge(compress); } @@ -285,7 +377,7 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal { int dimension = _directions[point]; int displacement = _distances[point]; - + int fd = _grid->_fdimensions[dimension]; int rd = _grid->_rdimensions[dimension]; @@ -337,7 +429,9 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal template void HaloGather(const Lattice &source,compressor &compress) { + mpi3synctime_g-=usecond(); _grid->StencilBarrier();// Synch shared memory on a single nodes + mpi3synctime_g+=usecond(); // conformable(source._grid,_grid); assert(source._grid==_grid); @@ -397,8 +491,12 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal CommsMerge(decompress,Mergers,Decompressions); } template void CommsMergeSHM(decompressor decompress) { + mpi3synctime-=usecond(); _grid->StencilBarrier();// Synch shared memory on a single nodes + mpi3synctime+=usecond(); + shmmergetime-=usecond(); CommsMerge(decompress,MergersSHM,DecompressionsSHM); + shmmergetime+=usecond(); } template @@ -442,7 +540,12 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal int checkerboard, const std::vector &directions, const std::vector &distances) - : _permute_type(npoints), _comm_buf_size(npoints) + : _permute_type(npoints), + _comm_buf_size(npoints), + comm_bytes_thr(npoints), + comm_enter_thr(npoints), + comm_leave_thr(npoints), + comm_time_thr(npoints) { face_table_computed=0; _npoints = npoints; @@ -996,6 +1099,15 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal void ZeroCounters(void) { gathertime = 0.; commtime = 0.; + mpi3synctime=0.; + mpi3synctime_g=0.; + shmmergetime=0.; + for(int i=0;i<_npoints;i++){ + comm_time_thr[i]=0; + comm_bytes_thr[i]=0; + comm_enter_thr[i]=0; + comm_leave_thr[i]=0; + } halogtime = 0.; mergetime = 0.; decompresstime = 0.; @@ -1011,6 +1123,18 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal #define PRINTIT(A) AVERAGE(A); std::cout << GridLogMessage << " Stencil " << #A << " "<< A/calls<_Nprocessors; RealD NN = _grid->NodeCount(); + double t = 0; + // if comm_time_thr is set they were all done in parallel so take the max + // but add up the bytes + int threaded = 0 ; + for (int i = 0; i < 8; ++i) { + if ( comm_time_thr[i]>0.0 ) { + threaded = 1; + comms_bytes += comm_bytes_thr[i]; + if (t < comm_time_thr[i]) t = comm_time_thr[i]; + } + } + if (threaded) commtime += t; _grid->GlobalSum(commtime); commtime/=NP; if ( calls > 0. ) { @@ -1026,6 +1150,9 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal std::cout << GridLogMessage << " Stencil " << comms_bytes/commtime/1000. << " GB/s per rank"< strong_inline void mult(iVector * __restrict__ ret, const iVector * __restrict__ rhs, const iScalar * __restrict__ lhs){ - mult(ret,lhs,rhs); + for(int c1=0;c1_internal[c1],&rhs->_internal[c1],&lhs->_internal); + } } diff --git a/lib/tensors/Tensor_class.h b/lib/tensors/Tensor_class.h index cb90da6c..c7f868db 100644 --- a/lib/tensors/Tensor_class.h +++ b/lib/tensors/Tensor_class.h @@ -156,11 +156,18 @@ class iScalar { // convert from a something to a scalar via constructor of something arg template ::value, T>::type * = nullptr> - strong_inline iScalar operator=(T arg) { + strong_inline iScalar operator=(T arg) { _internal = arg; return *this; } + // Convert elements + template + strong_inline iScalar operator=(iScalar &&arg) { + _internal = arg._internal; + return *this; + } + friend std::ostream &operator<<(std::ostream &stream,const iScalar &o) { stream << "S {" << o._internal << "}"; return stream; diff --git a/lib/tensors/Tensor_exp.h b/lib/tensors/Tensor_exp.h index e18fed70..f7eee8f0 100644 --- a/lib/tensors/Tensor_exp.h +++ b/lib/tensors/Tensor_exp.h @@ -80,8 +80,11 @@ template inline iVector Exponentiate(const iVector mat iQ2 = arg*arg*alpha*alpha; mat iQ3 = arg*iQ2*alpha; // sign in c0 from the conventions on the Ta - c0 = -imag( trace(iQ3) ) * one_over_three; - c1 = -real( trace(iQ2) ) * one_over_two; + scalar imQ3, reQ2; + imQ3 = imag( trace(iQ3) ); + reQ2 = real( trace(iQ2) ); + c0 = -imQ3 * one_over_three; + c1 = -reQ2 * one_over_two; // Cayley Hamilton checks to machine precision, tested tmp = c1 * one_over_three; diff --git a/lib/tensors/Tensor_index.h b/lib/tensors/Tensor_index.h index 7f34f3ac..f114baf8 100644 --- a/lib/tensors/Tensor_index.h +++ b/lib/tensors/Tensor_index.h @@ -47,6 +47,28 @@ template class TensorIndexRecursion { public: + + //////////////////////////////////////////////////// + // Type Queries + //////////////////////////////////////////////////// + template static inline int indexRank(const iScalar tmp) { return TensorIndexRecursion::indexRank(tmp._internal); } + template static inline int indexRank(const iVector tmp){ return TensorIndexRecursion::indexRank(tmp._internal[0]); } + template static inline int indexRank(const iMatrix tmp){ return TensorIndexRecursion::indexRank(tmp._internal[0][0]); } + + template static inline int isScalar(const iScalar tmp) { return TensorIndexRecursion::isScalar(tmp._internal); } + template static inline int isScalar(const iVector tmp){ return TensorIndexRecursion::isScalar(tmp._internal[0]); } + template static inline int isScalar(const iMatrix tmp){ return TensorIndexRecursion::isScalar(tmp._internal[0][0]); } + + template static inline int isVector(const iScalar tmp) { return TensorIndexRecursion::isVector(tmp._internal); } + template static inline int isVector(const iVector tmp){ return TensorIndexRecursion::isVector(tmp._internal[0]); } + template static inline int isVector(const iMatrix tmp){ return TensorIndexRecursion::isVector(tmp._internal[0][0]); } + + template static inline int isMatrix(const iScalar tmp) { return TensorIndexRecursion::isMatrix(tmp._internal); } + template static inline int isMatrix(const iVector tmp){ return TensorIndexRecursion::isMatrix(tmp._internal[0]); } + template static inline int isMatrix(const iMatrix tmp){ return TensorIndexRecursion::isMatrix(tmp._internal[0][0]); } + //////////////////////////////////////////////////// + // Trace + //////////////////////////////////////////////////// template static auto traceIndex(const iScalar arg) -> iScalar::traceIndex(arg._internal))> { @@ -215,6 +237,24 @@ class TensorIndexRecursion { template<> class TensorIndexRecursion<0> { public: + //////////////////////////////////////////////////// + // Type Queries + //////////////////////////////////////////////////// + template static inline int indexRank(const iScalar tmp) { return 1; } + template static inline int indexRank(const iVector tmp){ return N; } + template static inline int indexRank(const iMatrix tmp){ return N; } + + template static inline int isScalar(const iScalar tmp) { return true;} + template static inline int isScalar(const iVector tmp){ return false;} + template static inline int isScalar(const iMatrix tmp){ return false;} + + template static inline int isVector(const iScalar tmp) { return false;} + template static inline int isVector(const iVector tmp){ return true;} + template static inline int isVector(const iMatrix tmp){ return false;} + + template static inline int isMatrix(const iScalar tmp) { return false;} + template static inline int isMatrix(const iVector tmp){ return false;} + template static inline int isMatrix(const iMatrix tmp){ return true;} ///////////////////////////////////////// // Ends recursion for trace (scalar/vector/matrix) @@ -302,6 +342,26 @@ class TensorIndexRecursion<0> { //////////////////////////////////////////////////////////////////////////////////////////////////////// // External wrappers //////////////////////////////////////////////////////////////////////////////////////////////////////// +template inline int indexRank(void) +{ + vtype tmp; + return TensorIndexRecursion::indexRank(tmp); +} +template inline int isScalar(void) +{ + vtype tmp; + return TensorIndexRecursion::isScalar(tmp); +} +template inline int isVector(void) +{ + vtype tmp; + return TensorIndexRecursion::isVector(tmp); +} +template inline int isMatrix(void) +{ + vtype tmp; + return TensorIndexRecursion::isMatrix(tmp); +} template inline auto traceIndex (const vtype &arg) -> RemoveCRV(TensorIndexRecursion::traceIndex(arg)) { diff --git a/lib/tensors/Tensor_traits.h b/lib/tensors/Tensor_traits.h index ab20b807..c1ef397a 100644 --- a/lib/tensors/Tensor_traits.h +++ b/lib/tensors/Tensor_traits.h @@ -281,8 +281,8 @@ namespace Grid { template class getPrecision{ public: - typedef typename getVectorType::type vector_obj; //get the vector_obj (i.e. a grid Tensor) if its a Lattice, do nothing otherwise (i.e. if fundamental or grid Tensor) - + //get the vector_obj (i.e. a grid Tensor) if its a Lattice, do nothing otherwise (i.e. if fundamental or grid Tensor) + typedef typename getVectorType::type vector_obj; typedef typename GridTypeMapper::scalar_type scalar_type; //get the associated scalar type. Works on fundamental and tensor types typedef typename GridTypeMapper::Realified real_scalar_type; //remove any std::complex wrapper, should get us to the fundamental type diff --git a/lib/util/Init.cc b/lib/util/Init.cc index fe3b1734..3fd8b4cd 100644 --- a/lib/util/Init.cc +++ b/lib/util/Init.cc @@ -222,6 +222,11 @@ void Grid_init(int *argc,char ***argv) CartesianCommunicator::MAX_MPI_SHM_BYTES = MB*1024*1024; } + if( GridCmdOptionExists(*argv,*argv+*argc,"--shm-hugepages") ){ + CartesianCommunicator::Hugepages = 1; + } + + if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-signals") ){ Grid_debug_handler_init(); } @@ -304,6 +309,7 @@ void Grid_init(int *argc,char ***argv) std::cout< +Author: Peter Boyle +Author: paboyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + + +int main (int argc, char ** argv) +{ +#ifdef HAVE_LIME + Grid_init(&argc,&argv); + + std::cout < simd_layout = GridDefaultSimd(4,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + //std::vector latt_size ({48,48,48,96}); + //std::vector latt_size ({32,32,32,32}); + std::vector latt_size ({16,16,16,32}); + std::vector clatt_size ({4,4,4,8}); + int orthodir=3; + int orthosz =latt_size[orthodir]; + + GridCartesian Fine(latt_size,simd_layout,mpi_layout); + GridCartesian Coarse(clatt_size,simd_layout,mpi_layout); + + + GridParallelRNG pRNGa(&Fine); + GridParallelRNG pRNGb(&Fine); + GridSerialRNG sRNGa; + GridSerialRNG sRNGb; + + std::cout <({45,12,81,9})); + sRNGa.SeedFixedIntegers(std::vector({45,12,81,9})); + std::cout < U(4,&Fine); + + SU3::HotConfiguration(pRNGa,Umu); + + + FieldMetaData header; + + std::cout < + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + + +int main (int argc, char ** argv) +{ +#ifdef HAVE_LIME + Grid_init(&argc,&argv); + + + std::vector simd_layout = GridDefaultSimd(4,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + std::vector latt_size = GridDefaultLatt(); + int orthodir=3; + int orthosz =latt_size[orthodir]; + + GridCartesian Fine(latt_size,simd_layout,mpi_layout); + + LatticeGaugeField Umu(&Fine); + std::vector U(4,&Fine); + + FieldMetaData header; + std::string file("./ildg.file"); + IldgReader IR; + IR.open(file); + IR.readConfiguration(Umu,header); + IR.close(); + + for(int mu=0;mu(Umu,mu); + } + + // Painful ; fix syntactical niceness + LatticeComplex LinkTrace(&Fine); + LinkTrace=zero; + for(int mu=0;mu Plaq_T(orthosz); + sliceSum(Plaq,Plaq_T,Nd-1); + int Nt = Plaq_T.size(); + + TComplex Plaq_T_sum; + Plaq_T_sum=zero; + for(int t=0;t simd_layout = GridDefaultSimd(4,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); - std::vector latt_size ({16,16,16,16}); + //std::vector latt_size ({48,48,48,96}); + //std::vector latt_size ({32,32,32,32}); + std::vector latt_size ({16,16,16,32}); std::vector clatt_size ({4,4,4,8}); int orthodir=3; int orthosz =latt_size[orthodir]; @@ -49,30 +52,32 @@ int main (int argc, char ** argv) GridCartesian Fine(latt_size,simd_layout,mpi_layout); GridCartesian Coarse(clatt_size,simd_layout,mpi_layout); + GridParallelRNG pRNGa(&Fine); GridParallelRNG pRNGb(&Fine); GridSerialRNG sRNGa; GridSerialRNG sRNGb; + std::cout <({45,12,81,9})); sRNGa.SeedFixedIntegers(std::vector({45,12,81,9})); - + std::cout < U(4,&Fine); - SU3::ColdConfiguration(pRNGa,Umu); + SU3::HotConfiguration(pRNGa,Umu); - NerscField header; + FieldMetaData header; std::string file("./ckpoint_lat.4000"); int precision32 = 0; int tworow = 0; NerscIO::writeConfiguration(Umu,file,tworow,precision32); + Umu_saved = Umu; NerscIO::readConfiguration(Umu,header,file); + Umu_diff = Umu - Umu_saved; + //std::cout << "Umu_save "< Plaq_T(orthosz); sliceSum(Plaq,Plaq_T,Nd-1); @@ -139,7 +148,6 @@ int main (int argc, char ** argv) Complex p = TensorRemove(Tp); std::cout< U(4,&Fine); - NerscField header; + FieldMetaData header; std::string file("./ckpoint_lat"); NerscIO::readConfiguration(Umu,header,file); diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index 7d911dfd..d5b52044 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -31,6 +31,7 @@ Author: Peter Boyle using namespace Grid; +using namespace Grid::QCD; GRID_SERIALIZABLE_ENUM(myenum, undef, red, 1, blue, 2, green, 3); @@ -44,8 +45,8 @@ public: double, y, bool , b, std::vector, array, - std::vector>, twodimarray, - std::vector>>, cmplx3darray + std::vector >, twodimarray, + std::vector > >, cmplx3darray ); myclass() {} myclass(int i) @@ -237,7 +238,7 @@ int main(int argc,char **argv) std::cout << "Loaded (JSON) -----------------" << std::endl; std::cout << jcopy1 << std::endl << jveccopy1 << std::endl; } - + /* // This is still work in progress { diff --git a/tests/Test_simd.cc b/tests/Test_simd.cc index c0bbef1d..b2e8d68e 100644 --- a/tests/Test_simd.cc +++ b/tests/Test_simd.cc @@ -183,8 +183,6 @@ void IntTester(const functor &func) { typedef Integer scal; typedef vInteger vec; - GridSerialRNG sRNG; - sRNG.SeedFixedIntegers(std::vector({45,12,81,9})); int Nsimd = vec::Nsimd(); @@ -287,6 +285,50 @@ void ReductionTester(const functor &func) } +template +void IntReductionTester(const functor &func) +{ + int Nsimd = vec::Nsimd(); + + std::vector input1(Nsimd); + std::vector input2(Nsimd); + reduced result(0); + reduced reference(0); + reduced tmp; + + std::vector > buf(3); + vec & v_input1 = buf[0]; + vec & v_input2 = buf[1]; + + for(int i=0;i(v_input1,input1); + merge(v_input2,input2); + + func.template vfunc(result,v_input1,v_input2); + + for(int i=0;i(tmp,input1[i],input2[i]); + reference+=tmp; + } + + std::cout<(funcReduce()); std::cout< mpi_layout = GridDefaultMpi(); double volume = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; - + GridCartesian Fine(latt_size,simd_layout,mpi_layout); GridRedBlackCartesian rbFine(latt_size,simd_layout,mpi_layout); GridParallelRNG fRNG(&Fine); @@ -55,14 +54,14 @@ int main (int argc, char ** argv) // fRNG.SeedFixedIntegers(std::vector({45,12,81,9}); std::vector seeds({1,2,3,4}); fRNG.SeedFixedIntegers(seeds); - + Field Foo(&Fine); Field Bar(&Fine); Field Check(&Fine); Field Diff(&Fine); LatticeComplex lex(&Fine); - lex = zero; + lex = zero; random(fRNG,Foo); gaussian(fRNG,Bar); @@ -98,7 +97,7 @@ int main (int argc, char ** argv) Fine.oCoorFromOindex(ocoor,o); ocoor[dir]=(ocoor[dir]+disp)%Fine._rdimensions[dir]; } - + SimpleCompressor compress; myStencil.HaloExchange(Foo,compress); @@ -147,7 +146,7 @@ int main (int argc, char ** argv) <<") " < 1.0e-4) { @@ -187,16 +186,15 @@ int main (int argc, char ** argv) Fine.oCoorFromOindex(ocoor,o); ocoor[dir]=(ocoor[dir]+disp)%Fine._rdimensions[dir]; } - + SimpleCompressor compress; - Bar = Cshift(Foo,dir,disp); if ( disp & 0x1 ) { ECheck.checkerboard = Even; OCheck.checkerboard = Odd; - } else { + } else { ECheck.checkerboard = Odd; OCheck.checkerboard = Even; } @@ -213,7 +211,7 @@ int main (int argc, char ** argv) permute(OCheck._odata[i],EFoo._odata[SE->_offset],permute_type); else if (SE->_is_local) OCheck._odata[i] = EFoo._odata[SE->_offset]; - else + else OCheck._odata[i] = EStencil.CommBuf()[SE->_offset]; } OStencil.HaloExchange(OFoo,compress); @@ -222,18 +220,18 @@ int main (int argc, char ** argv) StencilEntry *SE; SE = OStencil.GetEntry(permute_type,0,i); // std::cout << "ODD source "<< i<<" -> " <_offset << " "<< SE->_is_local<_is_local && SE->_permute ) permute(ECheck._odata[i],OFoo._odata[SE->_offset],permute_type); else if (SE->_is_local) ECheck._odata[i] = OFoo._odata[SE->_offset]; - else + else ECheck._odata[i] = OStencil.CommBuf()[SE->_offset]; } - + setCheckerboard(Check,ECheck); setCheckerboard(Check,OCheck); - + Real nrmC = norm2(Check); Real nrmB = norm2(Bar); Diff = Check-Bar; @@ -256,10 +254,10 @@ int main (int argc, char ** argv) diff =norm2(ddiff); if ( diff > 0){ std::cout <<"Coor (" << coor[0]<<","< 1.0e-4) exit(-1); diff --git a/tests/core/Test_GaugeAction.cc b/tests/core/Test_GaugeAction.cc index 2f0535f1..572f19fb 100644 --- a/tests/core/Test_GaugeAction.cc +++ b/tests/core/Test_GaugeAction.cc @@ -73,7 +73,7 @@ int main (int argc, char ** argv) std::vector U(4,&Fine); - NerscField header; + FieldMetaData header; std::string file("./ckpoint_lat.4000"); NerscIO::readConfiguration(Umu,header,file); diff --git a/tests/core/Test_RectPlaq.cc b/tests/core/Test_RectPlaq.cc index 9154f879..2e9cc832 100644 --- a/tests/core/Test_RectPlaq.cc +++ b/tests/core/Test_RectPlaq.cc @@ -90,7 +90,7 @@ int main (int argc, char ** argv) std::vector U(4,&Fine); - NerscField header; + FieldMetaData header; std::string file("./ckpoint_lat.4000"); NerscIO::readConfiguration(Umu,header,file); diff --git a/tests/core/Test_fft_gfix.cc b/tests/core/Test_fft_gfix.cc index 7938241e..916c4b0b 100644 --- a/tests/core/Test_fft_gfix.cc +++ b/tests/core/Test_fft_gfix.cc @@ -31,209 +31,6 @@ Author: Peter Boyle using namespace Grid; using namespace Grid::QCD; -template -class FourierAcceleratedGaugeFixer : public Gimpl { - public: - INHERIT_GIMPL_TYPES(Gimpl); - - typedef typename Gimpl::GaugeLinkField GaugeMat; - typedef typename Gimpl::GaugeField GaugeLorentz; - - static void GaugeLinkToLieAlgebraField(const std::vector &U,std::vector &A) { - for(int mu=0;mu &A,GaugeMat &dmuAmu) { - dmuAmu=zero; - for(int mu=0;mu::avgPlaquette(Umu); - Real org_link_trace=WilsonLoops::linkTrace(Umu); - Real old_trace = org_link_trace; - Real trG; - - std::vector U(Nd,grid); - GaugeMat dmuAmu(grid); - - for(int i=0;i(Umu,mu); - //trG = SteepestDescentStep(U,alpha,dmuAmu); - trG = FourierAccelSteepestDescentStep(U,alpha,dmuAmu); - for(int mu=0;mu(Umu,U[mu],mu); - // Monitor progress and convergence test - // infrequently to minimise cost overhead - if ( i %20 == 0 ) { - Real plaq =WilsonLoops::avgPlaquette(Umu); - Real link_trace=WilsonLoops::linkTrace(Umu); - - std::cout << GridLogMessage << " Iteration "< &U,Real & alpha, GaugeMat & dmuAmu) { - GridBase *grid = U[0]._grid; - - std::vector A(Nd,grid); - GaugeMat g(grid); - - GaugeLinkToLieAlgebraField(U,A); - ExpiAlphaDmuAmu(A,g,alpha,dmuAmu); - - - Real vol = grid->gSites(); - Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc; - - SU::GaugeTransform(U,g); - - return trG; - } - - static Real FourierAccelSteepestDescentStep(std::vector &U,Real & alpha, GaugeMat & dmuAmu) { - - GridBase *grid = U[0]._grid; - - Real vol = grid->gSites(); - - FFT theFFT((GridCartesian *)grid); - - LatticeComplex Fp(grid); - LatticeComplex psq(grid); psq=zero; - LatticeComplex pmu(grid); - LatticeComplex one(grid); one = Complex(1.0,0.0); - - GaugeMat g(grid); - GaugeMat dmuAmu_p(grid); - std::vector A(Nd,grid); - - GaugeLinkToLieAlgebraField(U,A); - - DmuAmu(A,dmuAmu); - - theFFT.FFT_all_dim(dmuAmu_p,dmuAmu,FFT::forward); - - ////////////////////////////////// - // Work out Fp = psq_max/ psq... - ////////////////////////////////// - std::vector latt_size = grid->GlobalDimensions(); - std::vector coor(grid->_ndimension,0); - for(int mu=0;mu::taExp(ciadmam,g); - - Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc; - - SU::GaugeTransform(U,g); - - return trG; - } - - static void ExpiAlphaDmuAmu(const std::vector &A,GaugeMat &g,Real & alpha, GaugeMat &dmuAmu) { - GridBase *grid = g._grid; - Complex cialpha(0.0,-alpha); - GaugeMat ciadmam(grid); - DmuAmu(A,dmuAmu); - ciadmam = dmuAmu*cialpha; - SU::taExp(ciadmam,g); - } -/* - //////////////////////////////////////////////////////////////// - // NB The FT for fields living on links has an extra phase in it - // Could add these to the FFT class as a later task since this code - // might be reused elsewhere ???? - //////////////////////////////////////////////////////////////// - static void InverseFourierTransformAmu(FFT &theFFT,const std::vector &Ap,std::vector &Ax) { - GridBase * grid = theFFT.Grid(); - std::vector latt_size = grid->GlobalDimensions(); - - ComplexField pmu(grid); - ComplexField pha(grid); - GaugeMat Apha(grid); - - Complex ci(0.0,1.0); - - for(int mu=0;mu &Ax,std::vector &Ap) { - GridBase * grid = theFFT.Grid(); - std::vector latt_size = grid->GlobalDimensions(); - - ComplexField pmu(grid); - ComplexField pha(grid); - Complex ci(0.0,1.0); - - // Sign convention for FFTW calls: - // A(x)= Sum_p e^ipx A(p) / V - // A(p)= Sum_p e^-ipx A(x) - - for(int mu=0;mu seeds({1,2,3,4}); @@ -264,38 +61,55 @@ int main (int argc, char ** argv) std::cout<< "*****************************************************************" <::avgPlaquette(Umu); std::cout << " Initial plaquette "<::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-10, 1.0e-10); + Umu = Urnd; + FourierAcceleratedGaugeFixer::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-12, 1.0e-12,false); plaq=WilsonLoops::avgPlaquette(Umu); std::cout << " Final plaquette "<::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-12, 1.0e-12,true); - // std::cout<< "*****************************************************************" <::avgPlaquette(Umu); + std::cout << " Final plaquette "<::avgPlaquette(Umu); + std::cout << " Initial plaquette "<::SteepestDescentGaugeFix(Umu,alpha,10000,1.0e-12, 1.0e-12,true); + + plaq=WilsonLoops::avgPlaquette(Umu); + std::cout << " Final plaquette "< IRL(HermOp,X,Nk,Nm,eresid,Nit); - ImplicitlyRestartedLanczos ChebyIRL(HermOp,Cheby,Nk,Nm,eresid,Nit); + ImplicitlyRestartedLanczos IRL(HermOp,X,Nk,Nk,Nm,eresid,Nit); + ImplicitlyRestartedLanczos ChebyIRL(HermOp,Cheby,Nk,Nk,Nm,eresid,Nit); LatticeComplex src(grid); gaussian(RNG,src); { diff --git a/tests/forces/Test_contfrac_force.cc b/tests/forces/Test_contfrac_force.cc index 227ad5a0..2afb4dde 100644 --- a/tests/forces/Test_contfrac_force.cc +++ b/tests/forces/Test_contfrac_force.cc @@ -139,7 +139,7 @@ int main (int argc, char ** argv) } - Complex dSpred = sum(dS); + ComplexD dSpred = sum(dS); std::cout << GridLogMessage << " S "< - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution - directory. - *******************************************************************************/ - -#include - -using namespace Grid; -using namespace Hadrons; - -/******************************************************************************* - * Macros to reduce code duplication. - ******************************************************************************/ -// Useful definitions -#define ZERO_MOM "0. 0. 0. 0." -#define INIT_INDEX(s, n) (std::string(s) + "_" + std::to_string(n)) -#define ADD_INDEX(s, n) (s + "_" + std::to_string(n)) -#define LABEL_3PT(s, t1, t2) ADD_INDEX(INIT_INDEX(s, t1), t2) -#define LABEL_4PT(s, t1, t2, t3) ADD_INDEX(ADD_INDEX(INIT_INDEX(s, t1), t2), t3) -#define LABEL_4PT_NOISE(s, t1, t2, t3, nn) ADD_INDEX(ADD_INDEX(ADD_INDEX(INIT_INDEX(s, t1), t2), t3), nn) - -// Wall source/sink macros -#define NAME_3MOM_WALL_SOURCE(t, mom) ("wall_" + std::to_string(t) + "_" + mom) -#define NAME_WALL_SOURCE(t) NAME_3MOM_WALL_SOURCE(t, ZERO_MOM) -#define NAME_POINT_SOURCE(pos) ("point_" + pos) - -#define MAKE_3MOM_WALL_PROP(tW, mom, propName, solver)\ -{\ - std::string srcName = NAME_3MOM_WALL_SOURCE(tW, mom);\ - makeWallSource(application, srcName, tW, mom);\ - makePropagator(application, propName, srcName, solver);\ -} - -#define MAKE_WALL_PROP(tW, propName, solver)\ - MAKE_3MOM_WALL_PROP(tW, ZERO_MOM, propName, solver) - -// Sequential source macros -#define MAKE_SEQUENTIAL_PROP(tS, qSrc, mom, propName, solver)\ -{\ - std::string srcName = ADD_INDEX(qSrc + "_seq", tS);\ - makeSequentialSource(application, srcName, qSrc, tS, mom);\ - makePropagator(application, propName, srcName, solver);\ -} - -// Point source macros -#define MAKE_POINT_PROP(pos, propName, solver)\ -{\ - std::string srcName = NAME_POINT_SOURCE(pos);\ - makePointSource(application, srcName, pos);\ - makePropagator(application, propName, srcName, solver);\ -} - -/******************************************************************************* - * Functions for propagator construction. - ******************************************************************************/ - -/******************************************************************************* - * Name: makePointSource - * Purpose: Construct point source and add to application module. - * Parameters: application - main application that stores modules. - * srcName - name of source module to create. - * pos - Position of point source. - * Returns: None. - ******************************************************************************/ -inline void makePointSource(Application &application, std::string srcName, - std::string pos) -{ - // If the source already exists, don't make the module again. - if (!(Environment::getInstance().hasModule(srcName))) - { - MSource::Point::Par pointPar; - pointPar.position = pos; - application.createModule(srcName, pointPar); - } -} - -/******************************************************************************* - * Name: makeSequentialSource - * Purpose: Construct sequential source and add to application module. - * Parameters: application - main application that stores modules. - * srcName - name of source module to create. - * qSrc - Input quark for sequential inversion. - * tS - sequential source timeslice. - * mom - momentum insertion (default is zero). - * Returns: None. - ******************************************************************************/ -inline void makeSequentialSource(Application &application, std::string srcName, - std::string qSrc, unsigned int tS, - std::string mom = ZERO_MOM) -{ - // If the source already exists, don't make the module again. - if (!(Environment::getInstance().hasModule(srcName))) - { - MSource::SeqGamma::Par seqPar; - seqPar.q = qSrc; - seqPar.tA = tS; - seqPar.tB = tS; - seqPar.mom = mom; - seqPar.gamma = Gamma::Algebra::GammaT; - application.createModule(srcName, seqPar); - } -} - -/******************************************************************************* - * Name: makeWallSource - * Purpose: Construct wall source and add to application module. - * Parameters: application - main application that stores modules. - * srcName - name of source module to create. - * tW - wall source timeslice. - * mom - momentum insertion (default is zero). - * Returns: None. - ******************************************************************************/ -inline void makeWallSource(Application &application, std::string srcName, - unsigned int tW, std::string mom = ZERO_MOM) -{ - // If the source already exists, don't make the module again. - if (!(Environment::getInstance().hasModule(srcName))) - { - MSource::Wall::Par wallPar; - wallPar.tW = tW; - wallPar.mom = mom; - application.createModule(srcName, wallPar); - } -} - -/******************************************************************************* - * Name: makeWallSink - * Purpose: Wall sink smearing of a propagator. - * Parameters: application - main application that stores modules. - * propName - name of input propagator. - * wallName - name of smeared propagator. - * mom - momentum insertion (default is zero). - * Returns: None. - ******************************************************************************/ -inline void makeWallSink(Application &application, std::string propName, - std::string wallName, std::string mom = ZERO_MOM) -{ - // If the propagator has already been smeared, don't smear it again. - // Temporarily removed, strategy for sink smearing likely to change. - /*if (!(Environment::getInstance().hasModule(wallName))) - { - MSink::Wall::Par wallPar; - wallPar.q = propName; - wallPar.mom = mom; - application.createModule(wallName, wallPar); - }*/ -} - -/******************************************************************************* - * Name: makePropagator - * Purpose: Construct source and propagator then add to application module. - * Parameters: application - main application that stores modules. - * propName - name of propagator module to create. - * srcName - name of source module to use. - * solver - solver to use (default is CG). - * Returns: None. - ******************************************************************************/ -inline void makePropagator(Application &application, std::string &propName, - std::string &srcName, std::string &solver) -{ - // If the propagator already exists, don't make the module again. - if (!(Environment::getInstance().hasModule(propName))) - { - Quark::Par quarkPar; - quarkPar.source = srcName; - quarkPar.solver = solver; - application.createModule(propName, quarkPar); - } -} - -/******************************************************************************* - * Name: makeLoop - * Purpose: Use noise source and inversion result to make loop propagator, then - * add to application module. - * Parameters: application - main application that stores modules. - * propName - name of propagator module to create. - * srcName - name of noise source module to use. - * resName - name of inversion result on given noise source. - * Returns: None. - ******************************************************************************/ -inline void makeLoop(Application &application, std::string &propName, - std::string &srcName, std::string &resName) -{ - // If the loop propagator already exists, don't make the module again. - if (!(Environment::getInstance().hasModule(propName))) - { - MLoop::NoiseLoop::Par loopPar; - loopPar.q = resName; - loopPar.eta = srcName; - application.createModule(propName, loopPar); - } -} - -/******************************************************************************* - * Contraction module creation. - ******************************************************************************/ - -/******************************************************************************* - * Name: mesonContraction - * Purpose: Create meson contraction module and add to application module. - * Parameters: application - main application that stores modules. - * npt - specify n-point correlator (for labelling). - * q1 - quark propagator 1. - * q2 - quark propagator 2. - * label - unique label to construct module name. - * mom - momentum to project (default is zero) - * gammas - gamma insertions at source and sink. - * Returns: None. - ******************************************************************************/ -inline void mesonContraction(Application &application, unsigned int npt, - std::string &q1, std::string &q2, - std::string &label, - std::string mom = ZERO_MOM, - std::string gammas = "") -{ - std::string modName = std::to_string(npt) + "pt_" + label; - if (!(Environment::getInstance().hasModule(modName))) - { - MContraction::Meson::Par mesPar; - mesPar.output = std::to_string(npt) + "pt/" + label; - mesPar.q1 = q1; - mesPar.q2 = q2; - mesPar.mom = mom; - mesPar.gammas = gammas; - application.createModule(modName, mesPar); - } - } - -/******************************************************************************* - * Name: gamma3ptContraction - * Purpose: Create gamma3pt contraction module and add to application module. - * Parameters: application - main application that stores modules. - * npt - specify n-point correlator (for labelling). - * q1 - quark propagator 1. - * q2 - quark propagator 2. - * q3 - quark propagator 3. - * label - unique label to construct module name. - * gamma - gamma insertions between q2 and q3. - * Returns: None. - ******************************************************************************/ -inline void gamma3ptContraction(Application &application, unsigned int npt, - std::string &q1, std::string &q2, - std::string &q3, std::string &label, - Gamma::Algebra gamma = Gamma::Algebra::Identity) -{ - std::string modName = std::to_string(npt) + "pt_" + label; - if (!(Environment::getInstance().hasModule(modName))) - { - MContraction::Gamma3pt::Par gamma3ptPar; - gamma3ptPar.output = std::to_string(npt) + "pt/" + label; - gamma3ptPar.q1 = q1; - gamma3ptPar.q2 = q2; - gamma3ptPar.q3 = q3; - gamma3ptPar.gamma = gamma; - application.createModule(modName, gamma3ptPar); - } - } - -/******************************************************************************* - * Name: weakContraction[Eye,NonEye] - * Purpose: Create Weak Hamiltonian contraction module for Eye/NonEye topology - * and add to application module. - * Parameters: application - main application that stores modules. - * npt - specify n-point correlator (for labelling). - * q1 - quark propagator 1. - * q2 - quark propagator 2. - * q3 - quark propagator 3. - * q4 - quark propagator 4. - * label - unique label to construct module name. - * Returns: None. - ******************************************************************************/ -#define HW_CONTRACTION(top) \ -inline void weakContraction##top(Application &application, unsigned int npt,\ - std::string &q1, std::string &q2, \ - std::string &q3, std::string &q4, \ - std::string &label)\ -{\ - std::string modName = std::to_string(npt) + "pt_" + label;\ - if (!(Environment::getInstance().hasModule(modName)))\ - {\ - MContraction::WeakHamiltonian##top::Par weakPar;\ - weakPar.output = std::to_string(npt) + "pt/" + label;\ - weakPar.q1 = q1;\ - weakPar.q2 = q2;\ - weakPar.q3 = q3;\ - weakPar.q4 = q4;\ - application.createModule(modName, weakPar);\ - }\ -} -HW_CONTRACTION(Eye) // weakContractionEye -HW_CONTRACTION(NonEye) // weakContractionNonEye - -/******************************************************************************* - * Name: disc0Contraction - * Purpose: Create contraction module for 4pt Weak Hamiltonian + current - * disconnected topology for neutral mesons and add to application - * module. - * Parameters: application - main application that stores modules. - * q1 - quark propagator 1. - * q2 - quark propagator 2. - * q3 - quark propagator 3. - * q4 - quark propagator 4. - * label - unique label to construct module name. - * Returns: None. - ******************************************************************************/ -inline void disc0Contraction(Application &application, - std::string &q1, std::string &q2, - std::string &q3, std::string &q4, - std::string &label) -{ - std::string modName = "4pt_" + label; - if (!(Environment::getInstance().hasModule(modName))) - { - MContraction::WeakNeutral4ptDisc::Par disc0Par; - disc0Par.output = "4pt/" + label; - disc0Par.q1 = q1; - disc0Par.q2 = q2; - disc0Par.q3 = q3; - disc0Par.q4 = q4; - application.createModule(modName, disc0Par); - } - } - -/******************************************************************************* - * Name: discLoopContraction - * Purpose: Create contraction module for disconnected loop and add to - * application module. - * Parameters: application - main application that stores modules. - * q_loop - loop quark propagator. - * modName - unique module name. - * gamma - gamma matrix to use in contraction. - * Returns: None. - ******************************************************************************/ -inline void discLoopContraction(Application &application, - std::string &q_loop, std::string &modName, - Gamma::Algebra gamma = Gamma::Algebra::Identity) -{ - if (!(Environment::getInstance().hasModule(modName))) - { - MContraction::DiscLoop::Par discPar; - discPar.output = "disc/" + modName; - discPar.q_loop = q_loop; - discPar.gamma = gamma; - application.createModule(modName, discPar); - } - } diff --git a/tests/hadrons/Test_hadrons_meson_3pt.cc b/tests/hadrons/Test_hadrons_meson_3pt.cc index 7e487153..382c39d4 100644 --- a/tests/hadrons/Test_hadrons_meson_3pt.cc +++ b/tests/hadrons/Test_hadrons_meson_3pt.cc @@ -65,6 +65,10 @@ int main(int argc, char *argv[]) // set fermion boundary conditions to be periodic space, antiperiodic time. std::string boundary = "1 1 1 -1"; + // sink + MSink::Point::Par sinkPar; + sinkPar.mom = "0 0 0"; + application.createModule("sink", sinkPar); for (unsigned int i = 0; i < flavour.size(); ++i) { // actions @@ -115,15 +119,15 @@ int main(int argc, char *argv[]) } // propagators - Quark::Par quarkPar; + MFermion::GaugeProp::Par quarkPar; quarkPar.solver = "CG_" + flavour[i]; quarkPar.source = srcName; - application.createModule(qName[i], quarkPar); + application.createModule(qName[i], quarkPar); for (unsigned int mu = 0; mu < Nd; ++mu) { quarkPar.source = seqName[i][mu]; seqName[i][mu] = "Q_" + flavour[i] + "-" + seqName[i][mu]; - application.createModule(seqName[i][mu], quarkPar); + application.createModule(seqName[i][mu], quarkPar); } } @@ -136,7 +140,7 @@ int main(int argc, char *argv[]) mesPar.q1 = qName[i]; mesPar.q2 = qName[j]; mesPar.gammas = "all"; - mesPar.mom = "0. 0. 0. 0."; + mesPar.sink = "sink"; application.createModule("meson_Z2_" + std::to_string(t) + "_" @@ -155,7 +159,7 @@ int main(int argc, char *argv[]) mesPar.q1 = qName[i]; mesPar.q2 = seqName[j][mu]; mesPar.gammas = "all"; - mesPar.mom = "0. 0. 0. 0."; + mesPar.sink = "sink"; application.createModule("3pt_Z2_" + std::to_string(t) + "_" diff --git a/tests/hadrons/Test_hadrons_rarekaon.cc b/tests/hadrons/Test_hadrons_rarekaon.cc deleted file mode 100644 index ab4d3ef1..00000000 --- a/tests/hadrons/Test_hadrons_rarekaon.cc +++ /dev/null @@ -1,342 +0,0 @@ -/******************************************************************************* - Grid physics library, www.github.com/paboyle/Grid - - Source file: tests/hadrons/Test_hadrons_rarekaon.cc - - Copyright (C) 2017 - - Author: Andrew Lawson - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - See the full license in the file "LICENSE" in the top level distribution - directory. - *******************************************************************************/ - -#include "Test_hadrons.hpp" - -using namespace Grid; -using namespace Hadrons; - -enum quarks -{ - light = 0, - strange = 1, - charm = 2 -}; - -int main(int argc, char *argv[]) -{ - // parse command line ////////////////////////////////////////////////////// - std::string configStem; - - if (argc < 2) - { - std::cerr << "usage: " << argv[0] << " [Grid options]"; - std::cerr << std::endl; - std::exit(EXIT_FAILURE); - } - configStem = argv[1]; - - // initialization ////////////////////////////////////////////////////////// - Grid_init(&argc, &argv); - HadronsLogError.Active(GridLogError.isActive()); - HadronsLogWarning.Active(GridLogWarning.isActive()); - HadronsLogMessage.Active(GridLogMessage.isActive()); - HadronsLogIterative.Active(GridLogIterative.isActive()); - HadronsLogDebug.Active(GridLogDebug.isActive()); - LOG(Message) << "Grid initialized" << std::endl; - - // run setup /////////////////////////////////////////////////////////////// - Application application; - std::vector mass = {.01, .04, .2}; - std::vector flavour = {"l", "s", "c"}; - std::vector solvers = {"CG_l", "CG_s", "CG_c"}; - std::string kmom = "0. 0. 0. 0."; - std::string pmom = "1. 0. 0. 0."; - std::string qmom = "-1. 0. 0. 0."; - std::string mqmom = "1. 0. 0. 0."; - std::vector tKs = {0}; - unsigned int dt_pi = 16; - std::vector tJs = {8}; - unsigned int n_noise = 1; - unsigned int nt = 32; - bool do_disconnected(false); - - // Global parameters. - Application::GlobalPar globalPar; - globalPar.trajCounter.start = 1500; - globalPar.trajCounter.end = 1520; - globalPar.trajCounter.step = 20; - globalPar.seed = "1 2 3 4"; - globalPar.genetic.maxGen = 1000; - globalPar.genetic.maxCstGen = 200; - globalPar.genetic.popSize = 20; - globalPar.genetic.mutationRate = .1; - application.setPar(globalPar); - - // gauge field - if (configStem == "None") - { - application.createModule("gauge"); - } - else - { - MGauge::Load::Par gaugePar; - gaugePar.file = configStem; - application.createModule("gauge", gaugePar); - } - - // set fermion boundary conditions to be periodic space, antiperiodic time. - std::string boundary = "1 1 1 -1"; - - for (unsigned int i = 0; i < flavour.size(); ++i) - { - // actions - MAction::DWF::Par actionPar; - actionPar.gauge = "gauge"; - actionPar.Ls = 16; - actionPar.M5 = 1.8; - actionPar.mass = mass[i]; - actionPar.boundary = boundary; - application.createModule("DWF_" + flavour[i], actionPar); - - // solvers - // RBPrecCG -> CG - MSolver::RBPrecCG::Par solverPar; - solverPar.action = "DWF_" + flavour[i]; - solverPar.residual = 1.0e-8; - application.createModule(solvers[i], - solverPar); - } - - // Create noise propagators for loops. - std::vector noiseSrcs; - std::vector> noiseRes; - std::vector> noiseProps; - if (n_noise > 0) - { - MSource::Z2::Par noisePar; - noisePar.tA = 0; - noisePar.tB = nt - 1; - std::string loop_stem = "loop_"; - - noiseRes.resize(flavour.size()); - noiseProps.resize(flavour.size()); - for (unsigned int nn = 0; nn < n_noise; ++nn) - { - std::string eta = INIT_INDEX("noise", nn); - application.createModule(eta, noisePar); - noiseSrcs.push_back(eta); - - for (unsigned int f = 0; f < flavour.size(); ++f) - { - std::string loop_prop = INIT_INDEX(loop_stem + flavour[f], nn); - std::string loop_res = loop_prop + "_res"; - makePropagator(application, loop_res, eta, solvers[f]); - makeLoop(application, loop_prop, eta, loop_res); - noiseRes[f].push_back(loop_res); - noiseProps[f].push_back(loop_prop); - } - } - } - - // Translate rare kaon decay across specified timeslices. - for (unsigned int i = 0; i < tKs.size(); ++i) - { - // Zero-momentum wall source propagators for kaon and pion. - unsigned int tK = tKs[i]; - unsigned int tpi = (tK + dt_pi) % nt; - std::string q_Kl_0 = INIT_INDEX("Q_l_0", tK); - std::string q_pil_0 = INIT_INDEX("Q_l_0", tpi); - MAKE_WALL_PROP(tK, q_Kl_0, solvers[light]); - MAKE_WALL_PROP(tpi, q_pil_0, solvers[light]); - - // Wall sources for kaon and pion with momentum insertion. If either - // p or k are zero, or p = k, re-use the existing name to avoid - // duplicating a propagator. - std::string q_Ks_k = INIT_INDEX("Q_Ks_k", tK); - std::string q_Ks_p = INIT_INDEX((kmom == pmom) ? "Q_Ks_k" : "Q_Ks_p", tK); - std::string q_pil_k = INIT_INDEX((kmom == ZERO_MOM) ? "Q_l_0" : "Q_l_k", tpi); - std::string q_pil_p = INIT_INDEX((pmom == kmom) ? q_pil_k : ((pmom == ZERO_MOM) ? "Q_l_0" : "Q_l_p"), tpi); - MAKE_3MOM_WALL_PROP(tK, kmom, q_Ks_k, solvers[strange]); - MAKE_3MOM_WALL_PROP(tK, pmom, q_Ks_p, solvers[strange]); - MAKE_3MOM_WALL_PROP(tpi, kmom, q_pil_k, solvers[light]); - MAKE_3MOM_WALL_PROP(tpi, pmom, q_pil_p, solvers[light]); - - /*********************************************************************** - * CONTRACTIONS: pi and K 2pt contractions with mom = p, k. - **********************************************************************/ - // Wall-Point - std::string PW_K_k = INIT_INDEX("PW_K_k", tK); - std::string PW_K_p = INIT_INDEX("PW_K_p", tK); - std::string PW_pi_k = INIT_INDEX("PW_pi_k", tpi); - std::string PW_pi_p = INIT_INDEX("PW_pi_p", tpi); - mesonContraction(application, 2, q_Kl_0, q_Ks_k, PW_K_k, kmom); - mesonContraction(application, 2, q_Kl_0, q_Ks_p, PW_K_p, pmom); - mesonContraction(application, 2, q_pil_k, q_pil_0, PW_pi_k, kmom); - mesonContraction(application, 2, q_pil_p, q_pil_0, PW_pi_p, pmom); - // Wall-Wall, to be done - requires modification of meson module. - - /*********************************************************************** - * CONTRACTIONS: 3pt Weak Hamiltonian, C & W (non-Eye type) classes. - **********************************************************************/ - std::string HW_CW_k = LABEL_3PT("HW_CW_k", tK, tpi); - std::string HW_CW_p = LABEL_3PT("HW_CW_p", tK, tpi); - weakContractionNonEye(application, 3, q_Kl_0, q_Ks_k, q_pil_k, q_pil_0, HW_CW_k); - weakContractionNonEye(application, 3, q_Kl_0, q_Ks_p, q_pil_p, q_pil_0, HW_CW_p); - - /*********************************************************************** - * CONTRACTIONS: 3pt sd insertion. - **********************************************************************/ - // Note: eventually will use wall sink smeared q_Kl_0 instead. - std::string sd_k = LABEL_3PT("sd_k", tK, tpi); - std::string sd_p = LABEL_3PT("sd_p", tK, tpi); - gamma3ptContraction(application, 3, q_Kl_0, q_Ks_k, q_pil_k, sd_k); - gamma3ptContraction(application, 3, q_Kl_0, q_Ks_p, q_pil_p, sd_p); - - for (unsigned int nn = 0; nn < n_noise; ++nn) - { - /******************************************************************* - * CONTRACTIONS: 3pt Weak Hamiltonian, S and E (Eye type) classes. - ******************************************************************/ - // Note: eventually will use wall sink smeared q_Kl_0 instead. - for (unsigned int f = 0; f < flavour.size(); ++f) - { - if ((f != strange) || do_disconnected) - { - std::string HW_SE_k = LABEL_3PT("HW_SE_k_" + flavour[f], tK, tpi); - std::string HW_SE_p = LABEL_3PT("HW_SE_p_" + flavour[f], tK, tpi); - std::string loop_q = noiseProps[f][nn]; - weakContractionEye(application, 3, q_Kl_0, q_Ks_k, q_pil_k, loop_q, HW_CW_k); - weakContractionEye(application, 3, q_Kl_0, q_Ks_p, q_pil_p, loop_q, HW_CW_p); - } - } - } - - // Perform separate contractions for each t_J position. - for (unsigned int j = 0; j < tJs.size(); ++j) - { - // Sequential sources for current insertions. Local for now, - // gamma_0 only. - unsigned int tJ = (tJs[j] + tK) % nt; - MSource::SeqGamma::Par seqPar; - std::string q_KlCl_q = LABEL_3PT("Q_KlCl_q", tK, tJ); - std::string q_KsCs_mq = LABEL_3PT("Q_KsCs_mq", tK, tJ); - std::string q_pilCl_q = LABEL_3PT("Q_pilCl_q", tpi, tJ); - std::string q_pilCl_mq = LABEL_3PT("Q_pilCl_mq", tpi, tJ); - MAKE_SEQUENTIAL_PROP(tJ, q_Kl_0, qmom, q_KlCl_q, solvers[light]); - MAKE_SEQUENTIAL_PROP(tJ, q_Ks_k, mqmom, q_KsCs_mq, solvers[strange]); - MAKE_SEQUENTIAL_PROP(tJ, q_pil_p, qmom, q_pilCl_q, solvers[light]); - MAKE_SEQUENTIAL_PROP(tJ, q_pil_0, mqmom, q_pilCl_mq, solvers[light]); - - /******************************************************************* - * CONTRACTIONS: pi and K 3pt contractions with current insertion. - ******************************************************************/ - // Wall-Point - std::string C_PW_Kl = LABEL_3PT("C_PW_Kl", tK, tJ); - std::string C_PW_Ksb = LABEL_3PT("C_PW_Ksb", tK, tJ); - std::string C_PW_pilb = LABEL_3PT("C_PW_pilb", tK, tJ); - std::string C_PW_pil = LABEL_3PT("C_PW_pil", tK, tJ); - mesonContraction(application, 3, q_KlCl_q, q_Ks_k, C_PW_Kl, pmom); - mesonContraction(application, 3, q_Kl_0, q_KsCs_mq, C_PW_Ksb, pmom); - mesonContraction(application, 3, q_pil_0, q_pilCl_q, C_PW_pilb, kmom); - mesonContraction(application, 3, q_pilCl_mq, q_pil_p, C_PW_pil, kmom); - // Wall-Wall, to be done. - - /******************************************************************* - * CONTRACTIONS: 4pt contractions, C & W classes. - ******************************************************************/ - std::string CW_Kl = LABEL_4PT("CW_Kl", tK, tJ, tpi); - std::string CW_Ksb = LABEL_4PT("CW_Ksb", tK, tJ, tpi); - std::string CW_pilb = LABEL_4PT("CW_pilb", tK, tJ, tpi); - std::string CW_pil = LABEL_4PT("CW_pil", tK, tJ, tpi); - weakContractionNonEye(application, 4, q_KlCl_q, q_Ks_k, q_pil_p, q_pil_0, CW_Kl); - weakContractionNonEye(application, 4, q_Kl_0, q_KsCs_mq, q_pil_p, q_pil_0, CW_Ksb); - weakContractionNonEye(application, 4, q_Kl_0, q_Ks_k, q_pilCl_q, q_pil_0, CW_pilb); - weakContractionNonEye(application, 4, q_Kl_0, q_Ks_k, q_pil_p, q_pilCl_mq, CW_pil); - - /******************************************************************* - * CONTRACTIONS: 4pt contractions, sd insertions. - ******************************************************************/ - // Note: eventually will use wall sink smeared q_Kl_0/q_KlCl_q instead. - std::string sd_Kl = LABEL_4PT("sd_Kl", tK, tJ, tpi); - std::string sd_Ksb = LABEL_4PT("sd_Ksb", tK, tJ, tpi); - std::string sd_pilb = LABEL_4PT("sd_pilb", tK, tJ, tpi); - gamma3ptContraction(application, 4, q_KlCl_q, q_Ks_k, q_pil_p, sd_Kl); - gamma3ptContraction(application, 4, q_Kl_0, q_KsCs_mq, q_pil_p, sd_Ksb); - gamma3ptContraction(application, 4, q_Kl_0, q_Ks_k, q_pilCl_q, sd_pilb); - - // Sequential sources for each noise propagator. - for (unsigned int nn = 0; nn < n_noise; ++nn) - { - std::string loop_stem = "loop_"; - - // Contraction required for each quark flavour - alternatively - // drop the strange loop if not performing disconnected - // contractions or neglecting H_W operators Q_3 -> Q_10. - for (unsigned int f = 0; f < flavour.size(); ++f) - { - if ((f != strange) || do_disconnected) - { - std::string eta = noiseSrcs[nn]; - std::string loop_q = noiseProps[f][nn]; - std::string loop_qCq = LABEL_3PT(loop_stem + flavour[f], tJ, nn); - std::string loop_qCq_res = loop_qCq + "_res"; - MAKE_SEQUENTIAL_PROP(tJ, noiseRes[f][nn], qmom, - loop_qCq_res, solvers[f]); - makeLoop(application, loop_qCq, eta, loop_qCq_res); - - /******************************************************* - * CONTRACTIONS: 4pt contractions, S & E classes. - ******************************************************/ - // Note: eventually will use wall sink smeared q_Kl_0/q_KlCl_q instead. - std::string SE_Kl = LABEL_4PT_NOISE("SE_Kl", tK, tJ, tpi, nn); - std::string SE_Ksb = LABEL_4PT_NOISE("SE_Ksb", tK, tJ, tpi, nn); - std::string SE_pilb = LABEL_4PT_NOISE("SE_pilb", tK, tJ, tpi, nn); - std::string SE_loop = LABEL_4PT_NOISE("SE_loop", tK, tJ, tpi, nn); - weakContractionEye(application, 4, q_KlCl_q, q_Ks_k, q_pil_p, loop_q, SE_Kl); - weakContractionEye(application, 4, q_Kl_0, q_KsCs_mq, q_pil_p, loop_q, SE_Ksb); - weakContractionEye(application, 4, q_Kl_0, q_Ks_k, q_pilCl_q, loop_q, SE_pilb); - weakContractionEye(application, 4, q_Kl_0, q_Ks_k, q_pil_p, loop_qCq, SE_loop); - - /******************************************************* - * CONTRACTIONS: 4pt contractions, pi0 disconnected - * loop. - ******************************************************/ - std::string disc0 = LABEL_4PT_NOISE("disc0", tK, tJ, tpi, nn); - disc0Contraction(application, q_Kl_0, q_Ks_k, q_pilCl_q, loop_q, disc0); - - /******************************************************* - * CONTRACTIONS: Disconnected loop. - ******************************************************/ - std::string discLoop = "disc_" + loop_qCq; - discLoopContraction(application, loop_qCq, discLoop); - } - } - } - } - } - // execution - std::string par_file_name = "rarekaon_000_100_tK0_tpi16_tJ8_noloop_mc0.2.xml"; - application.saveParameterFile(par_file_name); - application.run(); - - // epilogue - LOG(Message) << "Grid is finalizing now" << std::endl; - Grid_finalize(); - - return EXIT_SUCCESS; -} diff --git a/tests/hadrons/Test_hadrons_spectrum.cc b/tests/hadrons/Test_hadrons_spectrum.cc index 8f7b30c8..801674f7 100644 --- a/tests/hadrons/Test_hadrons_spectrum.cc +++ b/tests/hadrons/Test_hadrons_spectrum.cc @@ -90,12 +90,12 @@ int main(int argc, char *argv[]) solverPar); // propagators - Quark::Par quarkPar; + MFermion::GaugeProp::Par quarkPar; quarkPar.solver = "CG_" + flavour[i]; quarkPar.source = "pt"; - application.createModule("Qpt_" + flavour[i], quarkPar); + application.createModule("Qpt_" + flavour[i], quarkPar); quarkPar.source = "z2"; - application.createModule("QZ2_" + flavour[i], quarkPar); + application.createModule("QZ2_" + flavour[i], quarkPar); } for (unsigned int i = 0; i < flavour.size(); ++i) for (unsigned int j = i; j < flavour.size(); ++j) diff --git a/tests/hmc/Test_hmc_EOMobiusRatio.cc b/tests/hmc/Test_hmc_EOMobiusRatio.cc index 4b5da356..d6ca96db 100644 --- a/tests/hmc/Test_hmc_EOMobiusRatio.cc +++ b/tests/hmc/Test_hmc_EOMobiusRatio.cc @@ -40,12 +40,6 @@ namespace Grid{ double, StoppingCondition, int, MaxCGIterations, bool, ApplySmearing); - - //template - //FermionParameters(Reader& Reader){ - // read(Reader, "Mobius", *this); - //} - }; @@ -113,9 +107,17 @@ int main(int argc, char **argv) { bool ApplySmearing = MyParams.Mobius.ApplySmearing; + // Use this if you want to tweak the default decomposition + // commented out as very architecture speficic + + //std::vector simd_lanes({2,2,1,1}); - // Grid from the command line - TheHMC.Resources.AddFourDimGrid("gauge"); + // Grid from the command line arguments --grid and --mpi + // drop the simd_lanes argument to fall back to the default decomposition for the SIMD lanes + + //TheHMC.Resources.AddFourDimGrid("gauge", simd_lanes); // tweak the SIMD lanes + TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition + // Possibile to create the module by hand // hardcoding parameters or using a Reader diff --git a/tests/hmc/Test_hmc_ScalarActionNxN.cc b/tests/hmc/Test_hmc_ScalarActionNxN.cc new file mode 100644 index 00000000..a4dad1a3 --- /dev/null +++ b/tests/hmc/Test_hmc_ScalarActionNxN.cc @@ -0,0 +1,193 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: ./tests/Test_hmc_WilsonFermionGauge.cc + +Copyright (C) 2016 + +Author: Guido Cossu + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution directory +*************************************************************************************/ +/* END LEGAL */ +#include +namespace Grid { +class ScalarActionParameters : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(ScalarActionParameters, + double, mass_squared, + double, lambda); + + template + ScalarActionParameters(Reader& Reader){ + read(Reader, "ScalarAction", *this); + } + +}; +} + +using namespace Grid; +using namespace Grid::QCD; + +template +class MagMeas : public HmcObservable { +public: + typedef typename Impl::Field Field; + typedef typename Impl::Simd::scalar_type Trace; + + void TrajectoryComplete(int traj, + Field &U, + GridSerialRNG &sRNG, + GridParallelRNG &pRNG) { + + int def_prec = std::cout.precision(); + + std::cout << std::setprecision(std::numeric_limits::digits10 + 1); + std::cout << GridLogMessage + << "m= " << TensorRemove(trace(sum(U))) << std::endl; + std::cout << GridLogMessage + << "m^2= " << TensorRemove(trace(sum(U)*sum(U))) << std::endl; + std::cout << GridLogMessage + << "phi^2= " << TensorRemove(sum(trace(U*U))) << std::endl; + std::cout.precision(def_prec); + + } +private: + +}; + +template +class MagMod: public ObservableModule, NoParameters>{ + typedef ObservableModule, NoParameters> ObsBase; + using ObsBase::ObsBase; // for constructors + + // acquire resource + virtual void initialize(){ + this->ObservablePtr.reset(new MagMeas()); + } +public: + MagMod(): ObsBase(NoParameters()){} +}; + +int main(int argc, char **argv) { + typedef Grid::JSONReader Serialiser; + + Grid_init(&argc, &argv); + int threads = GridThread::GetThreads(); + // here make a routine to print all the relevant information on the run + std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl; + + // Typedefs to simplify notation + constexpr int Ncolours = 2; + constexpr int Ndimensions = 3; + typedef ScalarNxNAdjGenericHMCRunner HMCWrapper; // Uses the default minimum norm, real scalar fields + typedef ScalarAdjActionR ScalarAction; + //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + HMCWrapper TheHMC; + TheHMC.ReadCommandLine(argc, argv); + + if (TheHMC.ParameterFile.empty()){ + std::cout << "Input file not specified." + << "Use --ParameterFile option in the command line.\nAborting" + << std::endl; + exit(1); + } + Serialiser Reader(TheHMC.ParameterFile); + + // Grid from the command line + GridModule ScalarGrid; + if (GridDefaultLatt().size() != Ndimensions){ + std::cout << "Incorrect dimension of the grid\n. Expected dim="<< Ndimensions << std::endl; + exit(1); + } + if (GridDefaultMpi().size() != Ndimensions){ + std::cout << "Incorrect dimension of the mpi grid\n. Expected dim="<< Ndimensions << std::endl; + exit(1); + } + ScalarGrid.set_full(new GridCartesian(GridDefaultLatt(),GridDefaultSimd(Ndimensions, vComplex::Nsimd()),GridDefaultMpi())); + ScalarGrid.set_rb(new GridRedBlackCartesian(ScalarGrid.get_full())); + TheHMC.Resources.AddGrid("scalar", ScalarGrid); + std::cout << "Lattice size : " << GridDefaultLatt() << std::endl; + + // Checkpointer definition + CheckpointerParameters CPparams(Reader); + TheHMC.Resources.LoadBinaryCheckpointer(CPparams); + + RNGModuleParameters RNGpar(Reader); + TheHMC.Resources.SetRNGSeeds(RNGpar); + + // Construct observables + typedef MagMod MagObs; + TheHMC.Resources.AddObservable(); + + ///////////////////////////////////////////////////////////// + // Collect actions, here use more encapsulation + + // Scalar action in adjoint representation + ScalarActionParameters SPar(Reader); + ScalarAction Saction(SPar.mass_squared, SPar.lambda); + + // Collect actions + ActionLevel> Level1(1); + Level1.push_back(&Saction); + TheHMC.TheAction.push_back(Level1); + ///////////////////////////////////////////////////////////// + TheHMC.Parameters.initialize(Reader); + + TheHMC.Run(); + + Grid_finalize(); +} // main + +/* Examples for input files + +JSON + +{ + "Checkpointer": { + "config_prefix": "ckpoint_scalar_lat", + "rng_prefix": "ckpoint_scalar_rng", + "saveInterval": 1, + "format": "IEEE64BIG" + }, + "RandomNumberGenerator": { + "serial_seeds": "1 2 3 4 6", + "parallel_seeds": "6 7 8 9 11" + }, + "ScalarAction":{ + "mass_squared": 0.5, + "lambda": 0.1 + }, + "HMC":{ + "StartTrajectory": 0, + "Trajectories": 100, + "MetropolisTest": true, + "NoMetropolisUntil": 10, + "StartingType": "HotStart", + "MD":{ + "name": "MinimumNorm2", + "MDsteps": 15, + "trajL": 2.0 + } + } +} + + +XML example not provided yet + +*/ diff --git a/tests/hmc/Test_hmc_WilsonGauge.cc b/tests/hmc/Test_hmc_WilsonGauge.cc index b2d5fb02..05bf81a2 100644 --- a/tests/hmc/Test_hmc_WilsonGauge.cc +++ b/tests/hmc/Test_hmc_WilsonGauge.cc @@ -66,7 +66,14 @@ int main(int argc, char **argv) { typedef PlaquetteMod PlaqObs; typedef TopologicalChargeMod QObs; TheHMC.Resources.AddObservable(); - TheHMC.Resources.AddObservable(); + TopologyObsParameters TopParams; + TopParams.interval = 5; + TopParams.do_smearing = true; + TopParams.Smearing.steps = 200; + TopParams.Smearing.step_size = 0.01; + TopParams.Smearing.meas_interval = 50; + TopParams.Smearing.maxTau = 2.0; + TheHMC.Resources.AddObservable(TopParams); ////////////////////////////////////////////// ///////////////////////////////////////////////////////////// diff --git a/tests/solver/Test_dwf_hdcr.cc b/tests/solver/Test_dwf_hdcr.cc index 64ca0b33..c553ba0a 100644 --- a/tests/solver/Test_dwf_hdcr.cc +++ b/tests/solver/Test_dwf_hdcr.cc @@ -516,7 +516,7 @@ int main (int argc, char ** argv) LatticeColourMatrix U(UGrid); LatticeColourMatrix zz(UGrid); - NerscField header; + FieldMetaData header; std::string file("./ckpoint_lat.4000"); NerscIO::readConfiguration(Umu,header,file); diff --git a/tests/solver/Test_dwf_lanczos.cc b/tests/solver/Test_dwf_lanczos.cc index bb978186..1dd5dae3 100644 --- a/tests/solver/Test_dwf_lanczos.cc +++ b/tests/solver/Test_dwf_lanczos.cc @@ -54,7 +54,7 @@ int main (int argc, char ** argv) GridParallelRNG RNG5rb(FrbGrid); RNG5.SeedFixedIntegers(seeds5); LatticeGaugeField Umu(UGrid); - SU3::TepidConfiguration(RNG4, Umu); + SU3::HotConfiguration(RNG4, Umu); std::vector U(4,UGrid); for(int mu=0;mu eval(Nm); - FermionField src(FrbGrid); gaussian(RNG5rb,src); + FermionField src(FrbGrid); + gaussian(RNG5rb,src); std::vector evec(Nm,FrbGrid); for(int i=0;i<1;i++){ - std::cout << i<<" / "<< Nm<< " grid pointer "< HermOp(Ds); ConjugateGradient CG(1.0e-8,10000); - BlockConjugateGradient BCG(1.0e-8,10000); - MultiRHSConjugateGradient mCG(1.0e-8,10000); + int blockDim = 0; + BlockConjugateGradient BCGrQ(BlockCGrQ,blockDim,1.0e-8,10000); + BlockConjugateGradient BCG (BlockCG,blockDim,1.0e-8,10000); + BlockConjugateGradient mCG (CGmultiRHS,blockDim,1.0e-8,10000); - std::cout << GridLogMessage << "************************************************************************ "< HermOp4d(Ds4d); FermionField src4d(UGrid); random(pRNG,src4d); @@ -97,21 +99,27 @@ int main (int argc, char ** argv) std::cout << GridLogMessage << " Calling 5d CG for "<