portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2024-11-09 23:45:36 +00:00
Code Releases Activity

Updated flight logging for Britney test

Browse Source
This commit is contained in:
Peter Boyle 2024-03-12 20:10:04 +00:00
parent 95f3d69cf9
commit 62e7bf024a
2 changed files with 46 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
Grid/util/Init.cc
View File

@ -96,7 +96,9 @@ static Coordinate Grid_default_mpi;
///////////////////////////////////////////////////////
int GridNormLoggingMode;
int32_t GridNormLoggingCounter;
int32_t GridMPINormLoggingCounter;
std::vector<double> GridNormLogVector;
std::vector<double> GridMPINormLogVector;
void SetGridNormLoggingMode(GridNormLoggingMode_t mode)
{
@ -113,6 +115,7 @@ void SetGridNormLoggingMode(GridNormLoggingMode_t mode)
case GridNormLoggingModeNone:
GridNormLoggingMode = mode;
GridNormLoggingCounter=0;
GridMPINormLoggingCounter=0;
GridNormLogVector.resize(0);
break;
default:
@ -122,19 +125,25 @@ void SetGridNormLoggingMode(GridNormLoggingMode_t mode)
void SetGridNormLoggingModePrint(void)
{
std::cout << " GridNormLogging Reproducibility logging set to print output " <<std::endl;
GridNormLoggingCounter = 0;
GridMPINormLoggingCounter=0;
GridNormLogVector.resize(0);
GridNormLoggingMode = GridNormLoggingModePrint;
}
void SetGridNormLoggingModeRecord(void)
{
std::cout << " GridNormLogging Reproducibility logging set to RECORD " <<std::endl;
GridNormLoggingCounter = 0;
GridMPINormLoggingCounter=0;
GridNormLogVector.resize(0);
GridNormLoggingMode = GridNormLoggingModeRecord;
}
void SetGridNormLoggingModeVerify(void)
{
std::cout << " GridNormLogging Reproducibility logging set to VERIFY " << GridNormLogVector.size()<< " log entries "<<std::endl;
GridNormLoggingCounter = 0;
GridMPINormLoggingCounter=0;
GridNormLoggingMode = GridNormLoggingModeVerify;
}
void GridNormLog(double value)
@ -150,12 +159,47 @@ void GridNormLog(double value)
if(GridNormLoggingMode == GridNormLoggingModeVerify) {
assert(GridNormLoggingCounter < GridNormLogVector.size());
if ( value != GridNormLogVector[GridNormLoggingCounter] ) {
fprintf(stderr,"%s Oops, I did it again! Reproduce failure for norm %d/%zu %.16e %.16e\n",GridHostname(),GridNormLoggingCounter,GridNormLogVector.size(),
fprintf(stderr,"%s:%d Oops, I did it again! Reproduce failure for norm %d/%zu %.16e %.16e\n",
GridHostname(),
GlobalSharedMemory::WorldShmRank,
GridNormLoggingCounter,GridNormLogVector.size(),
value, GridNormLogVector[GridNormLoggingCounter]); fflush(stderr);
assert(0); // Force takedown of job
}
if ( GridNormLogVector.size()==GridNormLoggingCounter ) {
std::cout << " GridNormLogging : Verified entire sequence of "<<GridNormLoggingCounter<<" norms "<<std::endl;
}
GridNormLoggingCounter++;
}
}
void GridMPINormLog(double local,double result)
{
if(GridNormLoggingMode == GridNormLoggingModePrint) {
std::cerr<<"GridMPINormLog : "<< GridMPINormLoggingCounter <<" " << std::hexfloat << local << " -> " <<result <<std::endl;
GridMPINormLoggingCounter++;
}
if(GridNormLoggingMode == GridNormLoggingModeRecord) {
std::cerr<<"GridMPINormLog RECORDING : "<< GridMPINormLoggingCounter <<" " << std::hexfloat << local << "-> "<< result <<std::endl;
GridMPINormLogVector.push_back(result);
GridMPINormLoggingCounter++;
}
if(GridNormLoggingMode == GridNormLoggingModeVerify) {
std::cerr<<"GridMPINormLog : "<< GridMPINormLoggingCounter <<" " << std::hexfloat << local << "-> "<< result <<std::endl;
assert(GridMPINormLoggingCounter < GridMPINormLogVector.size());
if ( result != GridMPINormLogVector[GridMPINormLoggingCounter] ) {
fprintf(stderr,"%s:%d MPI_Allreduce did it again! Reproduce failure for norm %d/%zu glb %.16e lcl %.16e hist %.16e\n",
GridHostname(),
GlobalSharedMemory::WorldShmRank,
GridMPINormLoggingCounter,GridMPINormLogVector.size(),
result, local, GridMPINormLogVector[GridMPINormLoggingCounter]); fflush(stderr);
assert(0); // Force takedown of job
}
if ( GridMPINormLogVector.size()==GridMPINormLoggingCounter ) {
std::cout << " GridMPINormLogging : Verified entire sequence of "<<GridMPINormLoggingCounter<<" norms "<<std::endl;
}
GridMPINormLoggingCounter++;
}
}
int GridThread::_threads =1;
int GridThread::_hyperthreads=1;

1
Grid/util/Init.h
View File

@ -84,6 +84,7 @@ void SetGridNormLoggingModeRecord(void);
void SetGridNormLoggingModeVerify(void);
void SetGridNormLoggingMode(GridNormLoggingMode_t mode);
void GridNormLog(double value);
void GridMPINormLog(double lcl, double glbl);
NAMESPACE_END(Grid);