diff --git a/Grid/algorithms/approx/Chebyshev.h b/Grid/algorithms/approx/Chebyshev.h index 3c679685..5863e948 100644 --- a/Grid/algorithms/approx/Chebyshev.h +++ b/Grid/algorithms/approx/Chebyshev.h @@ -269,7 +269,9 @@ public: RealD xscale = 2.0/(hi-lo); RealD mscale = -(hi+lo)/(hi-lo); Linop.HermOp(T0,y); + grid->Barrier(); axpby(T1,xscale,mscale,y,in); + grid->Barrier(); // sum = .5 c[0] T0 + c[1] T1 // out = ()*T0 + Coeffs[1]*T1; diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc index de7d81f8..8f3bf9af 100644 --- a/Grid/communicator/Communicator_mpi3.cc +++ b/Grid/communicator/Communicator_mpi3.cc @@ -260,32 +260,39 @@ CartesianCommunicator::~CartesianCommunicator() } #ifdef USE_GRID_REDUCTION void CartesianCommunicator::GlobalSum(float &f){ + FlightRecorder::StepLog("GlobalSumP2P"); CartesianCommunicator::GlobalSumP2P(f); } void CartesianCommunicator::GlobalSum(double &d) { + FlightRecorder::StepLog("GlobalSumP2P"); CartesianCommunicator::GlobalSumP2P(d); } #else void CartesianCommunicator::GlobalSum(float &f){ + FlightRecorder::StepLog("AllReduce"); int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator); assert(ierr==0); } void CartesianCommunicator::GlobalSum(double &d) { + FlightRecorder::StepLog("AllReduce"); int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator); assert(ierr==0); } #endif void CartesianCommunicator::GlobalSum(uint32_t &u){ + FlightRecorder::StepLog("AllReduce"); int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); assert(ierr==0); } void CartesianCommunicator::GlobalSum(uint64_t &u){ + FlightRecorder::StepLog("AllReduce"); int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator); assert(ierr==0); } void CartesianCommunicator::GlobalSumVector(uint64_t* u,int N){ + FlightRecorder::StepLog("AllReduceVector"); int ierr=MPI_Allreduce(MPI_IN_PLACE,u,N,MPI_UINT64_T,MPI_SUM,communicator); assert(ierr==0); } @@ -794,6 +801,7 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &list) @@ -801,11 +809,13 @@ void CartesianCommunicator::StencilBarrier(void) //} void CartesianCommunicator::Barrier(void) { + FlightRecorder::StepLog("GridBarrier"); int ierr = MPI_Barrier(communicator); assert(ierr==0); } void CartesianCommunicator::Broadcast(int root,void* data, int bytes) { + FlightRecorder::StepLog("Broadcast"); int ierr=MPI_Bcast(data, bytes, MPI_BYTE, @@ -824,6 +834,7 @@ void CartesianCommunicator::BarrierWorld(void){ } void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { + FlightRecorder::StepLog("BroadcastWorld"); int ierr= MPI_Bcast(data, bytes, MPI_BYTE, @@ -846,6 +857,7 @@ void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words, } void CartesianCommunicator::AllToAll(void *in,void *out,uint64_t words,uint64_t bytes) { + FlightRecorder::StepLog("AllToAll"); // MPI is a pain and uses "int" arguments // 64*64*64*128*16 == 500Million elements of data. // When 24*4 bytes multiples get 50x 10^9 >>> 2x10^9 Y2K bug. diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index 7d500af9..b5fd4197 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -990,7 +990,7 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm) } #endif - SharedMemoryTest(); + // SharedMemoryTest(); } ////////////////////////////////////////////////////////////////// // On node barrier diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h index 0f59f293..20237732 100644 --- a/Grid/stencil/Stencil.h +++ b/Grid/stencil/Stencil.h @@ -396,6 +396,7 @@ public: Packets[i].from_rank,Packets[i].do_recv, Packets[i].xbytes,Packets[i].rbytes,i); } + FlightRecorder::StepLog("Communicate begin has finished"); // Get comms started then run checksums // Having this PRIOR to the dslash seems to make Sunspot work... (!) for(int i=0;i void writeFile(T& in, std::string const fname){ Grid::emptyUserRecord record; Grid::ScidacWriter WR(in.Grid()->IsBoss()); WR.open(fname); - WR.writeScidacFieldRecord(in,record,0); + WR.writeScidacFieldRecord(in,record,0); // Lexico WR.close(); #endif // What is the appropriate way to throw error? @@ -190,8 +190,8 @@ int main(int argc, char **argv) { std::string ufile = file_pre + "U_" + std::to_string(tau) + "_" + file_post; { - PeriodicGimplR::GaugeField Ucopy = U; - NerscIO::writeConfiguration(Ucopy,ufile); + // PeriodicGimplR::GaugeField Ucopy = U; + // NerscIO::writeConfiguration(Ucopy,ufile); } RealD E = real(sum(R))/ RealD(U.Grid()->gSites()); @@ -206,7 +206,7 @@ int main(int argc, char **argv) { int t=WFPar.maxTau; WF.smear(Uflow, Umu); - NerscIO::writeConfiguration(Uflow,filesmr); + // NerscIO::writeConfiguration(Uflow,filesmr); RealD WFlow_plaq = WilsonLoops::avgPlaquette(Uflow); diff --git a/visualisation/CMakeLists.txt b/visualisation/CMakeLists.txt index 5790f425..4e9cf7b6 100644 --- a/visualisation/CMakeLists.txt +++ b/visualisation/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.12 FATAL_ERROR) project(GridViewer) -list(APPEND CMAKE_PREFIX_PATH "/Users/peterboyle/QCD/vtk/VTK-9.4.2-install/") +list(APPEND CMAKE_PREFIX_PATH "/home/paboyle/Visualisation/install/") find_package(VTK COMPONENTS CommonColor diff --git a/visualisation/cmake-command b/visualisation/cmake-command index 40c2d4f6..4f18c110 100644 --- a/visualisation/cmake-command +++ b/visualisation/cmake-command @@ -1,10 +1,17 @@ -libs=`grid-config --libs` -ldflags=`grid-config --ldflags` -cxxflags=`grid-config --cxxflags` -cxx=`grid-config --cxx` -cc=clang +export grid_config=/home/paboyle/GPT/install/bin/grid-config +libs=`$grid_config --libs` +ldflags=`$grid_config --ldflags` +cxxflags=`$grid_config --cxxflags` +cxx=`$grid_config --cxx` +cc=icx mkdir build cd build -LDFLAGS="$ldflags $libs " cmake .. -DCMAKE_C_COMPILER=$cc -DCMAKE_CXX_COMPILER=$cxx -DCMAKE_CXX_FLAGS=$cxxflags +echo CC $cc +echo CXX $cxx +echo CXXFLAGS $cxxflags +echo LDFLAGS $ldflags +echo LIBS $libs + +LDFLAGS="$ldflags $libs " cmake .. -DCMAKE_C_COMPILER=$cc -DCMAKE_CXX_COMPILER="$cxx" -DCMAKE_CXX_FLAGS="$cxxflags "