Merge 32e6d58356 into 89c0519f83

Grid/algorithms/FFT.h

@@ -29,7 +29,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #define _GRID_FFT_H_
 
 #ifdef HAVE_FFTW
-#if defined(USE_MKL) || defined(GRID_SYCL)
+#ifdef USE_MKL
 #include <fftw/fftw3.h>
 #else
 #include <fftw3.h>

Grid/lattice/Lattice_reduction.h

@@ -285,13 +285,10 @@ inline ComplexD rankInnerProduct(const Lattice<vobj> &left,const Lattice<vobj> &
 template<class vobj>
 inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right) {
   GridBase *grid = left.Grid();
-  uint32_t csum=0;
-  //  Uint32Checksum(left,csum);
   ComplexD nrm = rankInnerProduct(left,right);
-  RealD local = real(nrm);
-  GridNormLog(real(nrm),csum); // Could log before and after global sum to distinguish local and MPI
+  //  GridNormLog(real(nrm)); // Could log before and after global sum to distinguish local and MPI
   grid->GlobalSum(nrm);
-  GridMPINormLog(local,real(nrm)); 
+  //  GridNormLog(real(nrm)); 
   return nrm;
 }
 

Grid/util/Init.cc

@@ -96,10 +96,7 @@ static Coordinate Grid_default_mpi;
 ///////////////////////////////////////////////////////
 int GridNormLoggingMode;
 int32_t GridNormLoggingCounter;
-int32_t GridMPINormLoggingCounter;
 std::vector<double> GridNormLogVector;
-std::vector<double> GridMPINormLogVector;
-std::vector<uint32_t> GridCsumLogVector;
 
 void SetGridNormLoggingMode(GridNormLoggingMode_t mode)
 {
@@ -116,10 +113,7 @@ void SetGridNormLoggingMode(GridNormLoggingMode_t mode)
   case GridNormLoggingModeNone:
     GridNormLoggingMode = mode;
     GridNormLoggingCounter=0;
-    GridMPINormLoggingCounter=0;
     GridNormLogVector.resize(0);
-    GridCsumLogVector.resize(0);
-    GridMPINormLogVector.resize(0);
     break;
   default:
     assert(0);
@@ -128,90 +122,40 @@ void SetGridNormLoggingMode(GridNormLoggingMode_t mode)
 
 void SetGridNormLoggingModePrint(void)
 {
-  std::cout << " GridNormLogging Reproducibility logging set to print output " <<std::endl;
   GridNormLoggingCounter = 0;
-  GridMPINormLoggingCounter=0;
   GridNormLogVector.resize(0);
-  GridCsumLogVector.resize(0);
-  GridMPINormLogVector.resize(0);
   GridNormLoggingMode = GridNormLoggingModePrint;
 }
 void SetGridNormLoggingModeRecord(void)
 {
-  std::cout << " GridNormLogging Reproducibility logging set to RECORD " <<std::endl;
   GridNormLoggingCounter = 0;
-  GridMPINormLoggingCounter=0;
   GridNormLogVector.resize(0);
-  GridCsumLogVector.resize(0);
-  GridMPINormLogVector.resize(0);
   GridNormLoggingMode = GridNormLoggingModeRecord;
 }
 void SetGridNormLoggingModeVerify(void)
 {
-  std::cout << " GridNormLogging Reproducibility logging set to VERIFY " << GridNormLogVector.size()<< " log entries "<<std::endl;
   GridNormLoggingCounter = 0;
-  GridMPINormLoggingCounter=0;
   GridNormLoggingMode = GridNormLoggingModeVerify;
 }
-void GridNormLog(double value,uint32_t csum)
+void GridNormLog(double value)
 {
   if(GridNormLoggingMode == GridNormLoggingModePrint) {
-    std::cerr<<"GridNormLog : "<< GridNormLoggingCounter <<" " << std::hexfloat << value << " csum " <<std::hex<<csum<<std::dec <<std::endl;
+    std::cerr<<"GridNormLog : "<< GridNormLoggingCounter <<" " << std::hexfloat << value <<std::endl;
     GridNormLoggingCounter++;
   }
   if(GridNormLoggingMode == GridNormLoggingModeRecord) {
     GridNormLogVector.push_back(value);
-    GridCsumLogVector.push_back(csum);
     GridNormLoggingCounter++;
   }
   if(GridNormLoggingMode == GridNormLoggingModeVerify) {
     assert(GridNormLoggingCounter < GridNormLogVector.size());
-    if ( (value != GridNormLogVector[GridNormLoggingCounter])
-	 || (csum!=GridCsumLogVector[GridNormLoggingCounter]) ) {
-      std::cerr << " Oops got norm "<< std::hexfloat<<value<<" expect "<<GridNormLogVector[GridNormLoggingCounter] <<std::endl;
-      std::cerr << " Oops got csum "<< std::hex<<csum<<" expect "<<GridCsumLogVector[GridNormLoggingCounter] <<std::endl;
-      fprintf(stderr,"%s:%d Oops, I did it again! Reproduce failure for norm %d/%zu %.16e %.16e %x %x\n",
-	      GridHostname(),
-	      GlobalSharedMemory::WorldShmRank,
-	      GridNormLoggingCounter,GridNormLogVector.size(),
-	      value, GridNormLogVector[GridNormLoggingCounter],
-	      csum, GridCsumLogVector[GridNormLoggingCounter]); fflush(stderr);
-      assert(0); // Force takedown of job
-    }
-    if ( GridNormLogVector.size()==GridNormLoggingCounter ) {
-      std::cout << " GridNormLogging : Verified entire sequence of "<<GridNormLoggingCounter<<" norms "<<std::endl;
+    if ( value != GridNormLogVector[GridNormLoggingCounter] ) {
+      fprintf(stderr,"%s Oops, I did it again! Reproduce failure for norm %d/%zu %.16e %.16e\n",GridHostname(),GridNormLoggingCounter,GridNormLogVector.size(),
+	      value, GridNormLogVector[GridNormLoggingCounter]); fflush(stderr);
     }
     GridNormLoggingCounter++;
   }
 }
-void GridMPINormLog(double local,double result)
-{
-  if(GridNormLoggingMode == GridNormLoggingModePrint) {
-    std::cerr<<"GridMPINormLog : "<< GridMPINormLoggingCounter <<" " << std::hexfloat << local << " -> " <<result <<std::endl;
-    GridMPINormLoggingCounter++;
-  }
-  if(GridNormLoggingMode == GridNormLoggingModeRecord) {
-    std::cerr<<"GridMPINormLog RECORDING : "<< GridMPINormLoggingCounter <<" " << std::hexfloat << local << "-> "<< result <<std::endl;
-    GridMPINormLogVector.push_back(result);
-    GridMPINormLoggingCounter++;
-  }
-  if(GridNormLoggingMode == GridNormLoggingModeVerify) {
-    std::cerr<<"GridMPINormLog : "<< GridMPINormLoggingCounter <<" " << std::hexfloat << local << "-> "<< result <<std::endl;
-    assert(GridMPINormLoggingCounter < GridMPINormLogVector.size());
-    if ( result != GridMPINormLogVector[GridMPINormLoggingCounter] ) {
-      fprintf(stderr,"%s:%d MPI_Allreduce did it again! Reproduce failure for norm %d/%zu glb %.16e lcl %.16e hist %.16e\n",
-	      GridHostname(),
-	      GlobalSharedMemory::WorldShmRank,
-	      GridMPINormLoggingCounter,GridMPINormLogVector.size(),
-	      result, local, GridMPINormLogVector[GridMPINormLoggingCounter]); fflush(stderr);
-      assert(0); // Force takedown of job
-    }
-    if ( GridMPINormLogVector.size()==GridMPINormLoggingCounter ) {
-      std::cout << " GridMPINormLogging : Verified entire sequence of "<<GridMPINormLoggingCounter<<" norms "<<std::endl;
-    }
-    GridMPINormLoggingCounter++;
-  }
-}
 
 int GridThread::_threads =1;
 int GridThread::_hyperthreads=1;

Grid/util/Init.h

@@ -76,15 +76,14 @@ enum GridNormLoggingMode_t {
   GridNormLoggingModeRecord,
   GridNormLoggingModeVerify
 };
-//extern int GridNormLoggingMode;
-//extern int32_t GridNormLoggingCounter;
-//extern std::vector<double> GridNormLogVector;
+extern int GridNormLoggingMode;
+extern int32_t GridNormLoggingCounter;
+extern std::vector<double> GridNormLogVector;
 void SetGridNormLoggingModePrint(void);
 void SetGridNormLoggingModeRecord(void);
 void SetGridNormLoggingModeVerify(void);
 void SetGridNormLoggingMode(GridNormLoggingMode_t mode);
-void GridNormLog(double value,uint32_t csum);
-void GridMPINormLog(double lcl, double glbl);
+void GridNormLog(double value);
 
 NAMESPACE_END(Grid);
 

systems/Aurora/tests/repro16.pbs

@@ -2,39 +2,26 @@
 
 ## qsub -q EarlyAppAccess -A Aurora_Deployment -I -l select=1 -l walltime=60:00
 
-#PBS -l select=16:system=sunspot,place=scatter
+#PBS -q EarlyAppAccess
+#PBS -l select=16
+#PBS -l walltime=02:00:00
 #PBS -A LatticeQCD_aesp_CNDA
-#PBS -l walltime=01:00:00
-#PBS -N dwf
-#PBS -k doe
 
 #export OMP_PROC_BIND=spread
 #unset OMP_PLACES
 
 cd $PBS_O_WORKDIR
 
-#source ../sourceme.sh
+source ../sourceme.sh
 
 cat $PBS_NODEFILE
 
-#export MPICH_COLL_SYNC=1
-#export MPICH_ENV_DISPLAY=1
-export MPICH_
 export OMP_NUM_THREADS=3
 export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
-module load oneapi/eng-compiler/2023.05.15.003
-module load mpich/51.2/icc-all-deterministic-pmix-gpu
-#export LD_LIBRARY_PATH=/soft/restricted/CNDA/updates/2023.05.15.001/oneapi/compiler/eng-20230512/compiler/linux/lib/:$LD_LIBRARY_PATH
 
 #unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE
 #unset MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE
 #unset MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST
-export MPIR_CVAR_ALLREDUCE_DEVICE_COLLECTIVE=0
-export MPIR_CVAR_REDUCE_DEVICE_COLLECTIVE=0
-export MPIR_CVAR_ALLREDUCE_INTRA_ALGORITHM=recursive_doubling
-unset MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE
-unset MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE
-unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE
 
 export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
 export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
@@ -45,17 +32,10 @@ export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
 export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
 export MPICH_OFI_NIC_POLICY=GPU
 
-DIR=repro.$PBS_JOBID
-mkdir $DIR
-cd $DIR
-
+# 12 ppn, 16 nodes, 192 ranks
 CMD="mpiexec -np 192 -ppn 12  -envall \
-	     ../gpu_tile_compact.sh \
-	     ../Test_dwf_mixedcg_prec --mpi 2.4.4.6 --grid 64.128.128.192 \
-		--shm-mpi 1 --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 3000 --debug-stdout --log Message,Iterative"
+	     ./gpu_tile_compact.sh \
+	     ./Test_dwf_mixedcg_prec --mpi 2.4.4.6 --grid 64.128.128.192 \
+		--shm-mpi 1 --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 6000 "
 #--comms-overlap
 $CMD 
-
-grep Oops Grid.stderr.* > failures.$PBS_JOBID
-rm core.*
-

systems/Aurora/tests/repro1gpu.pbs

@@ -1,81 +0,0 @@
-#!/bin/bash
-
-#PBS -l select=16:system=sunspot,place=scatter
-#PBS -A LatticeQCD_aesp_CNDA
-#PBS -l walltime=02:00:00
-#PBS -N repro1gpu
-#PBS -k doe
-
-#export OMP_PROC_BIND=spread
-#unset OMP_PLACES
-
-module load oneapi/eng-compiler/2023.05.15.003
-module load mpich/51.2/icc-all-deterministic-pmix-gpu
-
-# 56 cores / 6 threads ~9
-export OMP_NUM_THREADS=6
-export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
-export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
-export MPICH_OFI_NIC_POLICY=GPU
-
-export MPIR_CVAR_ALLREDUCE_DEVICE_COLLECTIVE=0
-export MPIR_CVAR_REDUCE_DEVICE_COLLECTIVE=0
-export MPIR_CVAR_ALLREDUCE_INTRA_ALGORITHM=recursive_doubling
-unset MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE
-unset MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE
-unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE
-
-cd $PBS_O_WORKDIR
-
-NN=`cat $PBS_NODEFILE | wc -l`
-echo $PBS_NODEFILE
-cat $PBS_NODEFILE
-
-echo $NN nodes in node file
-for n in `eval echo {1..$NN}`
-do
-
-THIS_NODE=`head -n$n $PBS_NODEFILE | tail -n1 `
-echo Node $n is $THIS_NODE
-
-
-for g in {0..11}
-do
-export NUMA_MAP=(0 0 0 1 1 1 0 0 0 1 1 1 )
-export TILE_MAP=(0 0 0 0 0 0 1 1 1 1 1 1 )
-export  GPU_MAP=(0 1 2 3 4 5 0 1 2 3 4 5 )
-
-export numa=${NUMA_MAP[$g]}
-export gpu_id=${GPU_MAP[$g]}
-export tile_id=${TILE_MAP[$g]}
-export gpu=$gpu_id.$tile_id
-
-cd $PBS_O_WORKDIR
-
-DIR=repro.1gpu.$PBS_JOBID/node-$n-$THIS_NODE-GPU-$gpu
-mkdir -p $DIR
-cd $DIR
-
-echo $THIS_NODE > nodefile
-echo $gpu > gpu
-
-export ZE_AFFINITY_MASK=$gpu
-export ONEAPI_DEVICE_FILTER=gpu,level_zero
-
-CMD="mpiexec -np 1 -ppn 1  -envall --hostfile nodefile \
-	     numactl -N $numa -m $numa ../../Test_dwf_mixedcg_prec --mpi 1.1.1.1 --grid 16.16.32.32 \
-		--shm-mpi 0 --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 6000 --debug-stdout --log Message"
-echo $CMD
-$CMD &
-
-done
-done
-
-wait
-

systems/Aurora/tests/reproN.pbs

@@ -1,78 +0,0 @@
-#!/bin/bash
-
-#PBS -l select=16:system=sunspot,place=scatter
-#PBS -A LatticeQCD_aesp_CNDA
-#PBS -l walltime=02:00:00
-#PBS -N reproN
-#PBS -k doe
-
-#export OMP_PROC_BIND=spread
-#unset OMP_PLACES
-
-module load oneapi/eng-compiler/2023.05.15.003
-module load mpich/51.2/icc-all-deterministic-pmix-gpu
-
-# 56 cores / 6 threads ~9
-export OMP_NUM_THREADS=6
-export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0
-export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=1048576
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16
-export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16
-export MPICH_OFI_NIC_POLICY=GPU
-
-export MPIR_CVAR_ALLREDUCE_DEVICE_COLLECTIVE=0
-export MPIR_CVAR_REDUCE_DEVICE_COLLECTIVE=0
-export MPIR_CVAR_ALLREDUCE_INTRA_ALGORITHM=recursive_doubling
-unset MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE
-unset MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE
-unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE
-
-cd $PBS_O_WORKDIR
-
-NN=`cat $PBS_NODEFILE | wc -l`
-echo $PBS_NODEFILE
-cat $PBS_NODEFILE
-
-echo $NN nodes in node file
-for n in `eval echo {1..$NN}`
-do
-
-cd $PBS_O_WORKDIR
-
-THIS_NODE=`head -n$n $PBS_NODEFILE | tail -n1 `
-echo Node $n is $THIS_NODE
-
-DIR=repro.$PBS_JOBID/node-$n-$THIS_NODE
-
-mkdir -p $DIR
-cd $DIR
-
-echo $THIS_NODE > nodefile
-
-CMD="mpiexec -np 12 -ppn 12  -envall --hostfile nodefile \
-	     ../../gpu_tile_compact.sh \
-	     ../../Test_dwf_mixedcg_prec --mpi 1.2.2.3 --grid 32.64.64.96 \
-		--shm-mpi 0 --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 6000 --debug-stdout --log Message --comms-overlap"
-
-$CMD &
-
-done
-
-wait
-
-for n in ` eval echo {1..$NN} `
-do
-
-THIS_NODE=`head -n$n $PBS_NODEFILE | tail -n1 `
-DIR=repro.$PBS_JOBID/node-$n-$THIS_NODE
-
-cd $DIR
-
-grep Oops Grid.stderr.* > failures.$PBS_JOBID
-rm core.*
-
-done

tests/Test_dwf_mixedcg_prec.cc

@@ -34,45 +34,6 @@ using namespace Grid;
 #define HOST_NAME_MAX _POSIX_HOST_NAME_MAX
 #endif
 
-NAMESPACE_BEGIN(Grid);
-template<class Matrix,class Field>
-  class SchurDiagMooeeOperatorParanoid :  public SchurOperatorBase<Field> {
- public:
-    Matrix &_Mat;
-    SchurDiagMooeeOperatorParanoid (Matrix &Mat): _Mat(Mat){};
-    virtual  void Mpc      (const Field &in, Field &out) {
-      Field tmp(in.Grid());
-      tmp.Checkerboard() = !in.Checkerboard();
-      //      std::cout <<" Mpc starting"<<std::endl;
-
-      RealD nn = norm2(in); // std::cout <<" Mpc Prior to dslash norm is "<<nn<<std::endl;
-      _Mat.Meooe(in,tmp);
-      nn = norm2(tmp); //std::cout <<" Mpc Prior to Mooeinv "<<nn<<std::endl;
-      _Mat.MooeeInv(tmp,out);
-      nn = norm2(out); //std::cout <<" Mpc Prior to dslash norm is "<<nn<<std::endl;
-      _Mat.Meooe(out,tmp);
-      nn = norm2(tmp); //std::cout <<" Mpc Prior to Mooee "<<nn<<std::endl;
-      _Mat.Mooee(in,out);
-      nn = norm2(out); //std::cout <<" Mpc Prior to axpy "<<nn<<std::endl;
-      axpy(out,-1.0,tmp,out);
-    }
-    virtual void MpcDag   (const Field &in, Field &out){
-      Field tmp(in.Grid());
-      //      std::cout <<" MpcDag starting"<<std::endl;
-      RealD nn = norm2(in);// std::cout <<" MpcDag Prior to dslash norm is "<<nn<<std::endl;
-      _Mat.MeooeDag(in,tmp);
-      _Mat.MooeeInvDag(tmp,out);
-      nn = norm2(out);// std::cout <<" MpcDag Prior to dslash norm is "<<nn<<std::endl;
-      _Mat.MeooeDag(out,tmp);
-      nn = norm2(tmp);// std::cout <<" MpcDag Prior to Mooee "<<nn<<std::endl;
-      _Mat.MooeeDag(in,out);
-      nn = norm2(out);// std::cout <<" MpcDag Prior to axpy "<<nn<<std::endl;
-      axpy(out,-1.0,tmp,out);
-    }
-};
-
-NAMESPACE_END(Grid);
-
 int main (int argc, char ** argv)
 {
   char hostname[HOST_NAME_MAX+1];
@@ -121,8 +82,8 @@ int main (int argc, char ** argv)
   result_o_2.Checkerboard() = Odd;
   result_o_2 = Zero();
 
-  SchurDiagMooeeOperatorParanoid<DomainWallFermionD,LatticeFermionD> HermOpEO(Ddwf);
-  SchurDiagMooeeOperatorParanoid<DomainWallFermionF,LatticeFermionF> HermOpEO_f(Ddwf_f);
+  SchurDiagMooeeOperator<DomainWallFermionD,LatticeFermionD> HermOpEO(Ddwf);
+  SchurDiagMooeeOperator<DomainWallFermionF,LatticeFermionF> HermOpEO_f(Ddwf_f);
 
   int nsecs=600;
   if( GridCmdOptionExists(argv,argv+argc,"--seconds") ){
@@ -183,7 +144,7 @@ int main (int argc, char ** argv)
   csumref=0;
   int i=0;
   do { 
-    if ( i == 0 ) {
+    if ( iter == 0 ) {
       SetGridNormLoggingMode(GridNormLoggingModeRecord);
     } else {
       SetGridNormLoggingMode(GridNormLoggingModeVerify);