diff --git a/Grid/communicator/Communicator_base.cc b/Grid/communicator/Communicator_base.cc index f9a4c442d..90bdf4fe4 100644 --- a/Grid/communicator/Communicator_base.cc +++ b/Grid/communicator/Communicator_base.cc @@ -34,6 +34,7 @@ Author: Peter Boyle NAMESPACE_BEGIN(Grid); bool Stencil_force_mpi = true; +bool Stencil_force_barrier = false; /////////////////////////////////////////////////////////////// // Info that is setup once and indept of cartesian layout diff --git a/Grid/communicator/Communicator_base.h b/Grid/communicator/Communicator_base.h index d519304a5..e98e26ec5 100644 --- a/Grid/communicator/Communicator_base.h +++ b/Grid/communicator/Communicator_base.h @@ -38,6 +38,7 @@ Author: Peter Boyle NAMESPACE_BEGIN(Grid); extern bool Stencil_force_mpi ; +extern bool Stencil_force_barrier ; class CartesianCommunicator : public SharedMemory { diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc index 1f8ff6f70..ef9a2c965 100644 --- a/Grid/communicator/Communicator_mpi3.cc +++ b/Grid/communicator/Communicator_mpi3.cc @@ -872,6 +872,8 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &list) diff --git a/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc b/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc index d40a8eff1..a131b9be3 100644 --- a/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc +++ b/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc @@ -58,6 +58,7 @@ Author: Peter Boyle // #include +#include // std::sort, for the runtime-environment dump in ParseEnvironment #include #include #include @@ -202,13 +203,46 @@ void ParseEnvironment(void) P("PowerIterations",PowerIterations); P("PolyRecordIters",PolyRecordIters); P("PolyRecordStart",PolyRecordStart); P("PolyRecordSelect",PolyRecordSelect); P("PolyRefresh",PolyRefresh); P("PolyVerbose",PolyVerbose); P("SmootherCoeffLog",SmootherCoeffLog); - // library-side and runtime knobs, read straight from the environment as the library will + // library-side knobs, read straight from the environment as the library will const char *envs[] = {"BLOCK","BLOCK2","SUBSPACE_FILE","CONFIG","HOT_START","MRHS_COARSEN","V1_CHECK", "DENSE_CC","DENSE_SCHUR","DENSE_SCHUR2D","DENSE_DEVICE_SUM","DENSE_SPLITK","DENSE_NB", "SCHUR2D_LEAF_SPAN","SCHUR2D_LEAF_LU","SCHUR2D_PROBE","SUMMA_HANDSHAKE","DENSE_APPLY_PROFILE","SLAB_FILE", - "SOLVE_SRHS","GRID_ALLOC_NCACHE_LARGE","OMP_NUM_THREADS", - "MPICH_GPU_SUPPORT_ENABLED","FI_MR_CACHE_MONITOR","FI_MR_CACHE_MAX_COUNT","AMD_SERIALIZE_KERNEL","AMD_LOG_LEVEL"}; + "SOLVE_SRHS"}; for(auto e : envs) P(e, getenv(e) ? std::string(getenv(e)) : std::string("(unset)")); + + //////////////////////////////////////////////////////////////////////////////////////// + // The COMPLETE fabric/runtime environment, scanned from environ rather than a curated + // list. 2026-08-30: we could not tell from a failing log whether FI_CXI_ATS was set, + // because it was not in the list -- and a module or the submitting shell can set + // anything. A log must describe its own run without reference to the job script. + //////////////////////////////////////////////////////////////////////////////////////// + { + extern char **environ; + const char *prefixes[] = {"FI_","OFI_","MPICH_","MPIR_","PMI_","CXI_","HSA_","HIP_","ROCR_", + "ROCM_","AMD_","GPU_","NUMA_","OMP_","GOMP_","GRID_","CRAY_","LIBFABRIC"}; + std::vector hits; + for(char **e = environ; e && *e; e++){ + std::string s(*e); + for(auto p : prefixes){ + if( s.compare(0,strlen(p),p)==0 ){ + if(s.size()>200) s = s.substr(0,197)+"..."; // paths can be enormous + hits.push_back(s); + break; + } + } + } + std::sort(hits.begin(),hits.end()); + std::cout << GridLogMessage << "PARAM: ---- runtime environment: "<