From 5ce9996f9fd5183e66929c0591bb4b80a41a6554 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 27 Oct 2026 04:08:03 -0500 Subject: [PATCH 1/4] Let critical messages through on tasks that do not log Tasks that do not log turned spdlog off entirely, so a critical message raised only on such a task, such as a NaN found by state validation on the task that owns the cell, was lost along with the reason for the abort that followed. These tasks now keep critical messages only, on stderr and labeled with the task. Co-Authored-By: Claude Opus 5.5 (1M context) --- components/omega/src/infra/Logging.cpp | 23 +++++++++++++++++++-- components/omega/test/infra/LoggingTest.cpp | 18 ++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/components/omega/src/infra/Logging.cpp b/components/omega/src/infra/Logging.cpp index b05aca60a734..0d0c30b0554f 100644 --- a/components/omega/src/infra/Logging.cpp +++ b/components/omega/src/infra/Logging.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -228,6 +229,24 @@ static std::vector resolveLogTasks(const OMEGA::MachEnv *DefEnv, return Tasks; } +//------------------------------------------------------------------------------ +// Set up logging on a task that does not log. Only critical messages get +// through, to stderr and labeled with the task. They come just before an +// abort, often from a task other than the ones that log (e.g. the task that +// found a NaN), and would otherwise be lost. +static void initCriticalOnlyLogging(OMEGA::I4 TaskId) { + + const std::string Name = "critical_only"; + auto Logger = spdlog::get(Name); + if (!Logger) + Logger = spdlog::stderr_logger_mt(Name); + spdlog::set_default_logger(Logger); + + spdlog::set_pattern("[%l] task " + std::to_string(TaskId) + ": %v"); + spdlog::set_level(spdlog::level::critical); + spdlog::flush_on(spdlog::level::critical); +} + //------------------------------------------------------------------------------ // Initialize logging for case of a pre-defined custom logger // Return code: 1->enabled, 0->disabled, negative values->errors @@ -261,7 +280,7 @@ int initLogging( RetVal = 1; // log enabled } else { - spdlog::set_level(spdlog::level::off); + initCriticalOnlyLogging(TaskId); RetVal = 0; // log disabled } @@ -330,7 +349,7 @@ int initLogging( } } else { - spdlog::set_level(spdlog::level::off); + initCriticalOnlyLogging(TaskId); RetVal = 0; // log disabled } diff --git a/components/omega/test/infra/LoggingTest.cpp b/components/omega/test/infra/LoggingTest.cpp index 1409959ffb4b..5467a401cf02 100644 --- a/components/omega/test/infra/LoggingTest.cpp +++ b/components/omega/test/infra/LoggingTest.cpp @@ -238,6 +238,23 @@ int testKokkosDataTypes(bool LogEnabled) { return RetVal; } +// Tasks that do not log still let critical messages through, and only those +int testOtherTasksKeepCritical(bool LogEnabled) { + + if (LogEnabled) + return 0; + + auto Logger = spdlog::default_logger(); + bool Pass = Logger->name() == "critical_only" && + Logger->should_log(spdlog::level::critical) && + !Logger->should_log(spdlog::level::err); + + std::cout << "Critical only on other tasks: " << (Pass ? "PASS" : "FAIL") + << std::endl; + + return Pass ? 0 : 1; +} + int main(int argc, char **argv) { int RetVal = 0; @@ -270,6 +287,7 @@ int main(int argc, char **argv) { RetVal += testDefaultLogLevel(LogEnabled); RetVal += testKokkosDataTypes(LogEnabled); + RetVal += testOtherTasksKeepCritical(LogEnabled); } catch (const std::exception &Ex) { std::cout << Ex.what() << ": FAIL" << std::endl; From 3797bce282e0fe1634600bc17a4f5b192d3d9da6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 27 Oct 2026 04:12:22 -0500 Subject: [PATCH 2/4] Report where state validation finds the first bad value State validation gave only the number of NaN or out-of-range entries in each field, which left no way to find them in a large mesh without rerunning. Each critical message now also gives the global cell ID, latitude, longitude and layer of the first such entry, and the value when it is out of range. checkOceanState can return the same issues to the caller, which the unit test uses to check the reported location. The first entry is searched for only when a count is nonzero, so a valid state costs no more than before. Co-Authored-By: Claude Opus 5.5 (1M context) --- components/omega/src/ocn/StateValidation.cpp | 332 +++++++++++------- components/omega/src/ocn/StateValidation.h | 28 +- .../omega/test/ocn/StateValidationTest.cpp | 88 +++++ 3 files changed, 310 insertions(+), 138 deletions(-) diff --git a/components/omega/src/ocn/StateValidation.cpp b/components/omega/src/ocn/StateValidation.cpp index 80bef3ea65c1..abf1815402e5 100644 --- a/components/omega/src/ocn/StateValidation.cpp +++ b/components/omega/src/ocn/StateValidation.cpp @@ -14,8 +14,11 @@ #include "AuxiliaryState.h" #include "Config.h" #include "DataTypes.h" +#include "Decomp.h" #include "Eos.h" #include "Error.h" +#include "GlobalConstants.h" +#include "HorzMesh.h" #include "Logging.h" #include "MachEnv.h" #include "OceanState.h" @@ -25,10 +28,13 @@ #include "VertCoord.h" #include "mpi.h" +#include #include #include +#include #include #include +#include namespace OMEGA { @@ -71,90 +77,182 @@ static void abortWithMessage(const std::string &Msg) { } //------------------------------------------------------------------------------ -// Helper: count NaN entries and out-of-range entries in a 2-D Real device -// array over the first NCells/NEdges rows and NVert columns, restricted to -// active cells (CellMask > 0). -// Returns {NaNCount, OutOfRangeCount}. -static std::pair checkArray2D(const Array2DReal &Arr, I4 NRows, - I4 NCols, Real MinVal, Real MaxVal, - bool CheckMin, - const Array2DReal &CellMask) { - I4 NaNCount = 0; - I4 OutOfRangeCount = 0; - +// Helper: count the entries over the first NRows rows and NCols columns for +// which IsBad(Row, Col) is true +template +static I4 countEntries(const std::string &Label, I4 NRows, I4 NCols, + const F &IsBad) { + I4 Count = 0; parallelReduce( - "CheckNaN", {NRows, NCols}, + Label, {NRows, NCols}, KOKKOS_LAMBDA(int Row, int Col, int &Accum) { - if (CellMask(Row, Col) == 0) { - return; - } - Real Val = Arr(Row, Col); - if (Kokkos::isnan(Val)) { + if (IsBad(Row, Col)) { ++Accum; } }, - NaNCount); + Count); + return Count; +} +//------------------------------------------------------------------------------ +// Helper: the smallest flat index Row * NCols + Col for which IsBad(Row, Col) +// is true, or -1 if there is none +template +static I4 firstEntry(const std::string &Label, I4 NRows, I4 NCols, + const F &IsBad) { + I4 First = INT_MAX; parallelReduce( - "CheckBounds", {NRows, NCols}, - KOKKOS_LAMBDA(int Row, int Col, int &Accum) { - if (CellMask(Row, Col) == 0) { - return; - } - Real Val = Arr(Row, Col); - if (!Kokkos::isnan(Val)) { - if (Val > MaxVal) { - ++Accum; - } else if (CheckMin && Val < MinVal) { - ++Accum; - } + Label, {NRows, NCols}, + KOKKOS_LAMBDA(int Row, int Col, I4 &Accum) { + if (IsBad(Row, Col)) { + Accum = Kokkos::min(Accum, Row * NCols + Col); } }, - OutOfRangeCount); + Kokkos::Min(First)); + return First == INT_MAX ? -1 : First; +} + +//------------------------------------------------------------------------------ +// NaN and out-of-range counts for one field, with the flat index of the first +// of each (-1 if there is none) +struct FieldCheck { + I4 NaNs = 0; + I4 OOBs = 0; + I4 FirstNaN = -1; + I4 FirstOOB = -1; +}; + +//------------------------------------------------------------------------------ +// Helper: count the NaN and out-of-range entries of a field and, only when +// there are some, find the first of each, so a valid field costs only the +// two counts +template +static FieldCheck checkEntries(I4 NRows, I4 NCols, const FNaN &IsNaN, + const FOOB &IsOOB) { + FieldCheck Check; + Check.NaNs = countEntries("CheckNaN", NRows, NCols, IsNaN); + Check.OOBs = countEntries("CheckBounds", NRows, NCols, IsOOB); + if (Check.NaNs > 0) { + Check.FirstNaN = firstEntry("FirstNaN", NRows, NCols, IsNaN); + } + if (Check.OOBs > 0) { + Check.FirstOOB = firstEntry("FirstOOB", NRows, NCols, IsOOB); + } + return Check; +} - return {NaNCount, OutOfRangeCount}; +//------------------------------------------------------------------------------ +// Helper: check a 2-D Real device array over the first NCells/NEdges rows and +// NVert columns for NaN and out-of-range entries, restricted to active cells +// (CellMask > 0). +static FieldCheck checkArray2D(const Array2DReal &Arr, I4 NRows, I4 NCols, + Real MinVal, Real MaxVal, bool CheckMin, + const Array2DReal &CellMask) { + auto IsNaN = KOKKOS_LAMBDA(int Row, int Col) { + return CellMask(Row, Col) != 0 && Kokkos::isnan(Arr(Row, Col)); + }; + // NaNs are excluded explicitly, since comparisons with NaN are false + auto IsOOB = KOKKOS_LAMBDA(int Row, int Col) { + if (CellMask(Row, Col) == 0) { + return false; + } + Real Val = Arr(Row, Col); + return !Kokkos::isnan(Val) && + (Val > MaxVal || (CheckMin && Val < MinVal)); + }; + return checkEntries(NRows, NCols, IsNaN, IsOOB); } //------------------------------------------------------------------------------ -// Helper: count NaN and out-of-range entries for a single tracer (row = cell, -// col = vert) extracted from the 3-D tracer array at the given tracer index, +// Helper: check a single tracer (row = cell, col = vert) of the 3-D tracer +// array at the given tracer index for NaN and out-of-range entries, // restricted to active cells (CellMask > 0). -static std::pair checkTracerArray(const Array3DReal &Tracers3D, - I4 TracerIdx, I4 NCells, I4 NVert, - Real MinVal, Real MaxVal, - const Array2DReal &CellMask) { - I4 NaNCount = 0; - I4 OutOfRangeCount = 0; +static FieldCheck checkTracerArray(const Array3DReal &Tracers3D, I4 TracerIdx, + I4 NCells, I4 NVert, Real MinVal, + Real MaxVal, const Array2DReal &CellMask) { + auto IsNaN = KOKKOS_LAMBDA(int Cell, int K) { + return CellMask(Cell, K) != 0 && + Kokkos::isnan(Tracers3D(TracerIdx, Cell, K)); + }; + auto IsOOB = KOKKOS_LAMBDA(int Cell, int K) { + if (CellMask(Cell, K) == 0) { + return false; + } + Real Val = Tracers3D(TracerIdx, Cell, K); + return !Kokkos::isnan(Val) && (Val < MinVal || Val > MaxVal); + }; + return checkEntries(NCells, NVert, IsNaN, IsOOB); +} - parallelReduce( - "CheckTracerNaN", {NCells, NVert}, - KOKKOS_LAMBDA(int Cell, int K, int &Accum) { - if (CellMask(Cell, K) == 0) { - return; - } - Real Val = Tracers3D(TracerIdx, Cell, K); - if (Kokkos::isnan(Val)) { - ++Accum; - } - }, - NaNCount); +//------------------------------------------------------------------------------ +// Helpers: copy a single entry of a device array to the host +static Real getEntry(const Array2DReal &Arr, I4 Row, I4 Col) { + Real Val; + Kokkos::deep_copy(Val, Kokkos::subview(Arr, Row, Col)); + return Val; +} - parallelReduce( - "CheckTracerBounds", {NCells, NVert}, - KOKKOS_LAMBDA(int Cell, int K, int &Accum) { - if (CellMask(Cell, K) == 0) { - return; - } - Real Val = Tracers3D(TracerIdx, Cell, K); - if (!Kokkos::isnan(Val)) { - if (Val < MinVal || Val > MaxVal) { - ++Accum; - } - } - }, - OutOfRangeCount); +static Real getEntry(const Array3DReal &Arr, I4 Idx, I4 Row, I4 Col) { + Real Val; + Kokkos::deep_copy(Val, Kokkos::subview(Arr, Idx, Row, Col)); + return Val; +} + +//------------------------------------------------------------------------------ +// Helper: describe where a cell-layer entry is, for critical messages +static std::string describeLocation(I4 Cell, I4 K) { + std::string Where = "local cell " + std::to_string(Cell); + + Decomp *DefDecomp = Decomp::getDefault(); + if (DefDecomp) { + Where = "cell ID " + std::to_string(DefDecomp->CellIDH(Cell)); + } + + HorzMesh *DefMesh = HorzMesh::getDefault(); + if (DefMesh) { + const Real RadToDeg = 180.0 / Pi; + Where += fmt::format(" (lat {:.4f}, lon {:.4f})", + DefMesh->LatCellH(Cell) * RadToDeg, + DefMesh->LonCellH(Cell) * RadToDeg); + } - return {NaNCount, OutOfRangeCount}; + return Where + ", layer " + std::to_string(K); +} + +//------------------------------------------------------------------------------ +// Helper: log a critical message for each problem found in one field, giving +// where the first one is, record them in Issues if it is given, and add them +// to the totals +static void reportField(const std::string &Field, const std::string &Range, + const FieldCheck &Check, I4 NVert, + const std::function &GetValue, + I4 &TotalNaNs, I4 &TotalOOBs, + std::vector *Issues) { + + if (Check.NaNs > 0) { + I4 Cell = Check.FirstNaN / NVert; + I4 K = Check.FirstNaN % NVert; + LOG_CRITICAL("StateValidation: {} contains {} NaN value(s); the first " + "is at {}", + Field, Check.NaNs, describeLocation(Cell, K)); + if (Issues) { + Issues->push_back( + {Field, true, Check.NaNs, Cell, K, GetValue(Cell, K)}); + } + TotalNaNs += Check.NaNs; + } + if (Check.OOBs > 0) { + I4 Cell = Check.FirstOOB / NVert; + I4 K = Check.FirstOOB % NVert; + Real Val = GetValue(Cell, K); + LOG_CRITICAL("StateValidation: {} has {} value(s) outside valid range " + "{}; the first is {} at {}", + Field, Check.OOBs, Range, Val, describeLocation(Cell, K)); + if (Issues) { + Issues->push_back({Field, false, Check.OOBs, Cell, K, Val}); + } + TotalOOBs += Check.OOBs; + } } //------------------------------------------------------------------------------ @@ -163,35 +261,28 @@ static std::pair checkTracerArray(const Array3DReal &Tracers3D, /// Returns the total count of errors found; does not abort. std::pair checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel) { + const VertCoord *VCoord, I4 TimeLevel, + std::vector *Issues) { I4 TotalNaNs = 0; I4 TotalOOBs = 0; const Array2DReal &CellMask = VCoord->CellMask; + const I4 NCells = State->NCellsOwned; + const I4 NVert = State->NVertLayers; // ------------------------------------------------------------------------- // PseudoThickness: valid range [1e-10, 1000] // ------------------------------------------------------------------------- { Array2DReal PseudoThick = State->getPseudoThickness(TimeLevel); - auto [NaNs, OOB] = - checkArray2D(PseudoThick, State->NCellsOwned, State->NVertLayers, - static_cast(1e-10), static_cast(1000.0), - /*CheckMin=*/true, CellMask); - - if (NaNs > 0) { - LOG_CRITICAL( - "StateValidation: PseudoThickness contains {} NaN value(s)", NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL( - "StateValidation: PseudoThickness has {} value(s) outside " - "valid range [1e-10, 1000]", - OOB); - TotalOOBs += OOB; - } + FieldCheck Check = + checkArray2D(PseudoThick, NCells, NVert, static_cast(1e-10), + static_cast(1000.0), /*CheckMin=*/true, CellMask); + reportField( + "PseudoThickness", "[1e-10, 1000]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(PseudoThick, Cell, K); }, + TotalNaNs, TotalOOBs, Issues); } // ------------------------------------------------------------------------- @@ -199,24 +290,13 @@ std::pair checkOceanState(const OceanState *State, // ------------------------------------------------------------------------- { const Array2DReal &KE = AuxState->KineticAux.KineticEnergyCell; - auto [NaNs, OOB] = - checkArray2D(KE, State->NCellsOwned, State->NVertLayers, - static_cast(0.0), static_cast(10.0), - /*CheckMin=*/true, CellMask); - - if (NaNs > 0) { - LOG_CRITICAL( - "StateValidation: KineticEnergyCell contains {} NaN value(s)", - NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL( - "StateValidation: KineticEnergyCell has {} value(s) outside " - "valid range [0, 10]", - OOB); - TotalOOBs += OOB; - } + FieldCheck Check = + checkArray2D(KE, NCells, NVert, static_cast(0.0), + static_cast(10.0), /*CheckMin=*/true, CellMask); + reportField( + "KineticEnergyCell", "[0, 10]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(KE, Cell, K); }, TotalNaNs, + TotalOOBs, Issues); } // ------------------------------------------------------------------------- @@ -224,21 +304,14 @@ std::pair checkOceanState(const OceanState *State, // ------------------------------------------------------------------------- if (Tracers::IndxTemp != -1) { Array3DReal AllTracers = Tracers::getAll(TimeLevel); - auto [NaNs, OOB] = checkTracerArray( - AllTracers, Tracers::IndxTemp, State->NCellsOwned, State->NVertLayers, - static_cast(-10.0), static_cast(50.0), CellMask); - - if (NaNs > 0) { - LOG_CRITICAL("StateValidation: Temperature contains {} NaN value(s)", - NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL("StateValidation: Temperature has {} value(s) outside " - "valid range [-10, 50]", - OOB); - TotalOOBs += OOB; - } + const I4 Indx = Tracers::IndxTemp; + FieldCheck Check = checkTracerArray(AllTracers, Indx, NCells, NVert, + static_cast(-10.0), + static_cast(50.0), CellMask); + reportField( + "Temperature", "[-10, 50]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(AllTracers, Indx, Cell, K); }, + TotalNaNs, TotalOOBs, Issues); } // ------------------------------------------------------------------------- @@ -246,21 +319,14 @@ std::pair checkOceanState(const OceanState *State, // ------------------------------------------------------------------------- if (Tracers::IndxSalt != -1) { Array3DReal AllTracers = Tracers::getAll(TimeLevel); - auto [NaNs, OOB] = checkTracerArray( - AllTracers, Tracers::IndxSalt, State->NCellsOwned, State->NVertLayers, - static_cast(-2.0), static_cast(60.0), CellMask); - - if (NaNs > 0) { - LOG_CRITICAL("StateValidation: Salinity contains {} NaN value(s)", - NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL("StateValidation: Salinity has {} value(s) outside " - "valid range [-2, 60]", - OOB); - TotalOOBs += OOB; - } + const I4 Indx = Tracers::IndxSalt; + FieldCheck Check = checkTracerArray(AllTracers, Indx, NCells, NVert, + static_cast(-2.0), + static_cast(60.0), CellMask); + reportField( + "Salinity", "[-2, 60]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(AllTracers, Indx, Cell, K); }, + TotalNaNs, TotalOOBs, Issues); } return {TotalNaNs, TotalOOBs}; diff --git a/components/omega/src/ocn/StateValidation.h b/components/omega/src/ocn/StateValidation.h index 9e22f8447166..80f8387218e3 100644 --- a/components/omega/src/ocn/StateValidation.h +++ b/components/omega/src/ocn/StateValidation.h @@ -21,13 +21,29 @@ #include "Tracers.h" #include "VertCoord.h" +#include +#include + namespace OMEGA { +/// One kind of problem (NaN or out of range) found in one field on this task, +/// with the local cell, layer and value of the first entry that has it +struct ValidationIssue { + std::string Field; ///< Name of the field, e.g. Temperature + bool IsNaN; ///< NaN (true) or out of the valid range (false) + I4 Count; ///< Number of entries on this task with the problem + I4 Cell; ///< Local index of the cell of the first entry + I4 Layer; ///< Vertical layer of the first entry + Real Value; ///< Value of the first entry +}; + /// Check ocean state fields for NaN values and out-of-bounds conditions /// without aborting, returning counts of each error type found. /// -/// Only active ocean cells (where CellMask > 0) are checked. Critical log -/// messages are emitted for each type of error found. +/// Only active ocean cells (where CellMask > 0) are checked. A critical log +/// message is emitted for each type of error found, giving the number of +/// entries and the global cell ID, latitude, longitude and layer of the +/// first. If Issues is given, the same problems are appended to it. /// /// The following fields are validated: /// - PseudoThickness : [1e-10, 1000] (from OceanState) @@ -40,10 +56,12 @@ namespace OMEGA { /// \param[in] AuxState Auxiliary state containing KineticEnergyCell /// \param[in] VCoord Vertical coordinate containing the CellMask /// \param[in] TimeLevel Time level index to validate (typically 0 = current) +/// \param[out] Issues Optional list to which the problems found are added /// \return std::pair {NaN count, out-of-bounds count}; both 0 if valid -std::pair checkOceanState(const OceanState *State, - const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel); +std::pair +checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, + const VertCoord *VCoord, I4 TimeLevel, + std::vector *Issues = nullptr); /// Check ocean state fields for NaN values and out-of-bounds conditions. /// diff --git a/components/omega/test/ocn/StateValidationTest.cpp b/components/omega/test/ocn/StateValidationTest.cpp index 47dccbdecd43..00f9da182a77 100644 --- a/components/omega/test/ocn/StateValidationTest.cpp +++ b/components/omega/test/ocn/StateValidationTest.cpp @@ -8,6 +8,8 @@ /// (NaN and out-of-bounds values in each checked field are detected) of /// validateOceanState / checkOceanState. Checked fields are: /// - PseudoThickness, KineticEnergyCell, Temperature, Salinity. +/// Also checks that a single bad value is reported at the cell and layer +/// where it was put. // //===-----------------------------------------------------------------------===/ @@ -37,6 +39,8 @@ #include #include +#include +#include using namespace OMEGA; @@ -308,6 +312,86 @@ static int testOOBSalinity(OceanState *State, AuxiliaryState *AuxState, return expectErrors("OOB in Salinity", State, AuxState, VCoord); } +// --- Location of the first bad entry --- + +// Find the last owned cell that is active in its second layer (or its only +// one), so that a bad value put there is away from the first index checked +static std::pair pickActiveEntry(OceanState *State, VertCoord *VCoord) { + auto CellMaskH = createHostMirrorCopy(VCoord->CellMask); + const I4 K = State->NVertLayers > 1 ? 1 : 0; + for (I4 Cell = State->NCellsOwned - 1; Cell >= 0; --Cell) { + if (CellMaskH(Cell, K) != 0) { + return {Cell, K}; + } + } + return {-1, -1}; +} + +// Check that a single bad value is reported as the only issue, at the cell +// and layer where it was put +static int expectIssueAt(const char *TestName, OceanState *State, + AuxiliaryState *AuxState, VertCoord *VCoord, + const std::string &Field, bool IsNaN, I4 Cell, I4 K) { + std::vector Issues; + checkOceanState(State, AuxState, VCoord, 0, &Issues); + + bool Pass = Issues.size() == 1 && Issues[0].Field == Field && + Issues[0].IsNaN == IsNaN && Issues[0].Count == 1 && + Issues[0].Cell == Cell && Issues[0].Layer == K; + if (!Pass) { + LOG_ERROR("StateValidationTest: {} - expected one {} issue in {} at " + "cell {}, layer {}; got {} issue(s)", + TestName, IsNaN ? "NaN" : "OOB", Field, Cell, K, Issues.size()); + for (const auto &Issue : Issues) { + LOG_ERROR("StateValidationTest: {} {} x{} at cell {}, layer {}", + Issue.Field, Issue.IsNaN ? "NaN" : "OOB", Issue.Count, + Issue.Cell, Issue.Layer); + } + return 1; + } + LOG_INFO("StateValidationTest: {} PASS", TestName); + return 0; +} + +static int testLocationOfNaN(OceanState *State, AuxiliaryState *AuxState, + VertCoord *VCoord) { + if (Tracers::IndxTemp == -1) + return 0; // tracer not active; skip + restoreValidState(State, AuxState); + auto [Cell, K] = pickActiveEntry(State, VCoord); + if (Cell < 0) + return 0; // no active owned cell on this task; nothing to check + const Real NaN = std::numeric_limits::quiet_NaN(); + Array2DReal TempArr = Tracers::getByIndex(0, Tracers::IndxTemp); + Kokkos::deep_copy(Kokkos::subview(TempArr, Cell, K), NaN); + return expectIssueAt("Location of NaN in Temperature", State, AuxState, + VCoord, "Temperature", /*IsNaN=*/true, Cell, K); +} + +static int testLocationOfOOB(OceanState *State, AuxiliaryState *AuxState, + VertCoord *VCoord) { + if (Tracers::IndxSalt == -1) + return 0; // tracer not active; skip + restoreValidState(State, AuxState); + auto [Cell, K] = pickActiveEntry(State, VCoord); + if (Cell < 0) + return 0; // no active owned cell on this task; nothing to check + Array2DReal SaltArr = Tracers::getByIndex(0, Tracers::IndxSalt); + // 99 g/kg is above the valid max of 60 g/kg + Kokkos::deep_copy(Kokkos::subview(SaltArr, Cell, K), 99.0); + + std::vector Issues; + checkOceanState(State, AuxState, VCoord, 0, &Issues); + if (Issues.size() == 1 && Issues[0].Value != 99.0) { + LOG_ERROR("StateValidationTest: OOB in Salinity reported value {}, " + "expected 99", + Issues[0].Value); + return 1; + } + return expectIssueAt("Location of OOB in Salinity", State, AuxState, VCoord, + "Salinity", /*IsNaN=*/false, Cell, K); +} + //------------------------------------------------------------------------------ // Run all state validation tests @@ -363,6 +447,10 @@ int testStateValidation() { Err += testNaNSalinity(DefState, DefAuxState, VCoord); Err += testOOBSalinity(DefState, DefAuxState, VCoord); + // Location of the first bad entry + Err += testLocationOfNaN(DefState, DefAuxState, VCoord); + Err += testLocationOfOOB(DefState, DefAuxState, VCoord); + AuxiliaryState::clear(); return Err; } From ad011c4687984134bcb066ac53f5182153f16f59 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 27 Oct 2026 04:16:31 -0500 Subject: [PATCH 3/4] Write the state on every task before a validation abort Each task decided on its own whether to abort, so only the tasks that found a NaN called MPI_Abort and no collective work was possible first. validateOceanState now sums the NaN and out-of-range counts over all tasks, in the reduction it already did for negative salinity, so every task makes the same decision. Before aborting, every task writes the stream named by the new option State::Validation::WriteStreamOnAbort (default History), giving a record of the bad state to inspect. The time steppers pass their clock for the time stamp, which is the start of the failing step. The master task now logs the totals over all tasks, both before an abort and when the run continues. Co-Authored-By: Claude Opus 5.5 (1M context) --- components/omega/configs/Default.yml | 1 + components/omega/src/infra/IOStream.cpp | 7 ++ components/omega/src/infra/IOStream.h | 5 + components/omega/src/ocn/StateValidation.cpp | 93 ++++++++++++++----- components/omega/src/ocn/StateValidation.h | 25 +++-- .../timeStepping/ForwardBackwardStepper.cpp | 3 +- .../src/timeStepping/RungeKutta2Stepper.cpp | 3 +- .../src/timeStepping/RungeKutta4Stepper.cpp | 3 +- .../timeStepping/SplitExplicitRK2Stepper.cpp | 3 +- 9 files changed, 109 insertions(+), 34 deletions(-) diff --git a/components/omega/configs/Default.yml b/components/omega/configs/Default.yml index 635a155a20a5..2457a95d7b5c 100644 --- a/components/omega/configs/Default.yml +++ b/components/omega/configs/Default.yml @@ -34,6 +34,7 @@ Omega: Validation: AbortOnOOB: false AbortOnNan: true + WriteStreamOnAbort: History Advection: Coef3rdOrder: 0.25 FluxThicknessType: Center diff --git a/components/omega/src/infra/IOStream.cpp b/components/omega/src/infra/IOStream.cpp index 8afb4d3b4523..4041cd1192c4 100644 --- a/components/omega/src/infra/IOStream.cpp +++ b/components/omega/src/infra/IOStream.cpp @@ -274,6 +274,13 @@ bool IOStream::validateAll() { } // End validateAll +//------------------------------------------------------------------------------ +// Returns true if a stream with the given name has been defined. +bool IOStream::isDefined(const std::string &StreamName // [in] stream name +) { + return AllStreams.find(StreamName) != AllStreams.end(); +} + //------------------------------------------------------------------------------ // Reads a single stream if it is time. Error IOStream::read( diff --git a/components/omega/src/infra/IOStream.h b/components/omega/src/infra/IOStream.h index a02e18e210f9..c21bcc81bf55 100644 --- a/components/omega/src/infra/IOStream.h +++ b/components/omega/src/infra/IOStream.h @@ -365,6 +365,11 @@ class IOStream { /// Returns true if all streams are valid. static bool validateAll(); + //--------------------------------------------------------------------------- + /// Returns true if a stream with the given name has been defined. + static bool isDefined(const std::string &StreamName ///< [in] stream name + ); + //--------------------------------------------------------------------------- /// Reads a stream if it is time. static Error read(const std::string &StreamName, ///< [in] Name of stream diff --git a/components/omega/src/ocn/StateValidation.cpp b/components/omega/src/ocn/StateValidation.cpp index abf1815402e5..0d21b645d8e6 100644 --- a/components/omega/src/ocn/StateValidation.cpp +++ b/components/omega/src/ocn/StateValidation.cpp @@ -3,9 +3,11 @@ // Validates ocean state fields by checking for NaN values and // out-of-bounds conditions. Abort behaviour is controlled at runtime via the // YAML options Omega::State::Validation::AbortOnNan (default true) and -// Omega::State::Validation::AbortOnOOB (default false). When an abort flag is -// set a critical error log with backtrace is emitted and MPI_Abort is called; -// otherwise a CRITICAL log message is written and execution continues. +// Omega::State::Validation::AbortOnOOB (default false). The decision is made +// from the counts on all tasks. When an abort flag is set, every task writes +// the stream named by Omega::State::Validation::WriteStreamOnAbort (default +// History), a critical error log with backtrace is emitted and MPI_Abort is +// called; otherwise a CRITICAL log message is written and execution continues. // //===----------------------------------------------------------------------===// @@ -19,6 +21,7 @@ #include "Error.h" #include "GlobalConstants.h" #include "HorzMesh.h" +#include "IOStream.h" #include "Logging.h" #include "MachEnv.h" #include "OceanState.h" @@ -39,8 +42,9 @@ namespace OMEGA { struct ValidationAbortConfig { - bool AbortOnOOB = false; - bool AbortOnNan = true; + bool AbortOnOOB = false; + bool AbortOnNan = true; + std::string WriteStreamOnAbort = "History"; }; static ValidationAbortConfig getValidationAbortConfig() { @@ -63,16 +67,50 @@ static ValidationAbortConfig getValidationAbortConfig() { ValidationConfig.get("AbortOnOOB", AbortConfig.AbortOnOOB); ValidationConfig.get("AbortOnNan", AbortConfig.AbortOnNan); + ValidationConfig.get("WriteStreamOnAbort", AbortConfig.WriteStreamOnAbort); return AbortConfig; } //------------------------------------------------------------------------------ -// Helper: abort on the local Omega communicator with a message and backtrace -static void abortWithMessage(const std::string &Msg) { - LOG_CRITICAL("{}", Msg); - cpptrace::generate_trace().print(); - MPI_Comm Comm = MachEnv::getDefault()->getComm(); +// Helper: abort the run, first writing the given stream if there is one and a +// clock to stamp it with. It must be called on every task, since writing a +// stream is collective. The tasks that found problems have already logged +// them; the master task adds the totals and a backtrace. +static void abortRun(I8 NaNs, I8 OOBs, const std::string &StreamName, + const Clock *ModelClock) { + + MachEnv *DefEnv = MachEnv::getDefault(); + MPI_Comm Comm = DefEnv->getComm(); + bool IsMaster = DefEnv->isMasterTask(); + + if (IsMaster) { + LOG_CRITICAL("StateValidation: Ocean state validation aborting with {} " + "NaNs and {} OOBs on all tasks. The tasks that found them " + "give the field and location in critical messages, on " + "stderr for tasks that do not log.", + NaNs, OOBs); + } + + if (!StreamName.empty() && ModelClock) { + if (IOStream::isDefined(StreamName)) { + if (IsMaster) { + LOG_CRITICAL("StateValidation: Writing stream {} before aborting, " + "with the time at the start of the failing step", + StreamName); + } + IOStream::write(StreamName, ModelClock, /*ForceWrite=*/true); + } else if (IsMaster) { + LOG_CRITICAL("StateValidation: Not writing stream {} before " + "aborting because it is not defined", + StreamName); + } + } + + if (IsMaster) { + cpptrace::generate_trace().print(); + } + MPI_Barrier(Comm); MPI_Abort(Comm, static_cast(ErrorCode::Critical)); } @@ -337,11 +375,27 @@ std::pair checkOceanState(const OceanState *State, /// Only active cells (where CellMask > 0) are checked. /// Aborts via MPI_Abort on failure. void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel) { + const VertCoord *VCoord, I4 TimeLevel, + const Clock *ModelClock) { - auto [NaNs, OOBs] = checkOceanState(State, AuxState, VCoord, TimeLevel); + auto [LocalNaNs, LocalOOBs] = + checkOceanState(State, AuxState, VCoord, TimeLevel); static const auto AbortConfig = getValidationAbortConfig(); - bool abort = false; + + MachEnv *DefEnv = MachEnv::getDefault(); + Eos *EosInstance = Eos::getInstance(); + + // One reduction gathers everything validation needs from all tasks, so + // that every task makes the same decision about aborting + I8 LocalNNegSalinity = EosInstance ? EosInstance->NNegSalinity : 0; + std::vector Totals = + globalSum(std::vector{LocalNaNs, LocalOOBs, LocalNNegSalinity}, + DefEnv->getComm()); + const I8 NaNs = Totals[0]; + const I8 OOBs = Totals[1]; + const I8 NNegSalinity = Totals[2]; + + bool abort = false; if (NaNs > 0 && AbortConfig.AbortOnNan) { abort = true; @@ -350,11 +404,8 @@ void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, abort = true; } if (abort) { - abortWithMessage( - "StateValidation: Ocean state validation aborting with " + - std::to_string(NaNs) + " NaNs and " + std::to_string(OOBs) + - " OOBs." + "See critical messages above for details."); - } else if ((NaNs + OOBs) > 0) { + abortRun(NaNs, OOBs, AbortConfig.WriteStreamOnAbort, ModelClock); + } else if ((NaNs + OOBs) > 0 && DefEnv->isMasterTask()) { // Only report the flags that are actually suppressing an abort, i.e. // those corresponding to the issues that were detected std::string Flags; @@ -367,14 +418,12 @@ void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, Flags += "AbortOnOOB is false"; } LOG_CRITICAL("StateValidation: Ocean state validation failed with {} " - "NaNs and {} OOBs. {}; continuing after logging.", + "NaNs and {} OOBs on all tasks. {}; continuing after " + "logging.", NaNs, OOBs, Flags); } - Eos *EosInstance = Eos::getInstance(); if (EosInstance) { - const I8 NNegSalinity = globalSum(EosInstance->NNegSalinity, - MachEnv::getDefault()->getComm()); if (NNegSalinity > 0) { LOG_WARN("StateValidation: {} negative salinity value(s) seen by " "the EOS during this time step (summed over all stages " diff --git a/components/omega/src/ocn/StateValidation.h b/components/omega/src/ocn/StateValidation.h index 80f8387218e3..4cb7de27f6bb 100644 --- a/components/omega/src/ocn/StateValidation.h +++ b/components/omega/src/ocn/StateValidation.h @@ -11,13 +11,15 @@ /// aborting. Suitable for testing. /// - validateOceanState: calls checkOceanState and conditionally aborts /// via MPI_Abort based on runtime YAML options AbortOnNan and AbortOnOOB -/// (under Omega::State::Validation). When both flags are false a -/// CRITICAL log message is emitted and the run continues. +/// (under Omega::State::Validation), first writing the stream named by +/// WriteStreamOnAbort. When both flags are false a CRITICAL log message +/// is emitted and the run continues. // //===----------------------------------------------------------------------===// #include "AuxiliaryState.h" #include "OceanState.h" +#include "TimeMgr.h" #include "Tracers.h" #include "VertCoord.h" @@ -67,8 +69,9 @@ checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, /// /// Calls checkOceanState and conditionally aborts via MPI_Abort based on the /// runtime YAML options AbortOnNan and AbortOnOOB (under -/// Omega::State::Validation). Only active ocean cells (where CellMask > 0) -/// are checked. +/// Omega::State::Validation) and the counts summed over all tasks. It must +/// therefore be called on every task. Only active ocean cells (where +/// CellMask > 0) are checked. /// /// The following fields are validated: /// - PseudoThickness : [1e-10, 1000] (from OceanState) @@ -78,17 +81,23 @@ checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, /// - Salinity tracer : [-2, 60] (from Tracers) /// /// If AbortOnNan is true and NaN values are detected, or AbortOnOOB is true -/// and out-of-bounds values are detected, a critical error is logged with a -/// stack backtrace and the run is aborted via MPI_Abort. When both flags are -/// false but errors are found, a CRITICAL log message is emitted and execution +/// and out-of-bounds values are detected, the stream named by the YAML option +/// WriteStreamOnAbort (default History) is written on every task if it is +/// defined and ModelClock is given, a critical error is logged with a stack +/// backtrace and the run is aborted via MPI_Abort. When both flags are false +/// but errors are found, a CRITICAL log message is emitted and execution /// continues. /// /// \param[in] State Ocean state to validate /// \param[in] AuxState Auxiliary state containing KineticEnergyCell /// \param[in] VCoord Vertical coordinate containing the CellMask /// \param[in] TimeLevel Time level index to validate (typically 0 = current) +/// \param[in] ModelClock Optional clock for the time stamp of the stream +/// written before an abort; without it, no stream is +/// written void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel); + const VertCoord *VCoord, I4 TimeLevel, + const Clock *ModelClock = nullptr); } // namespace OMEGA diff --git a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp index 7667019dea97..6af4ecf3cc71 100644 --- a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp +++ b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp @@ -105,7 +105,8 @@ void ForwardBackwardStepper::doStep( applyImplicitVerticalMixing(State, TracerCurLevel, ThickCurLevel, VelCurLevel, "ForwardBackward"); - validateOceanState(State, AuxState, VertCoord::getDefault(), 0); + validateOceanState(State, AuxState, VertCoord::getDefault(), 0, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); diff --git a/components/omega/src/timeStepping/RungeKutta2Stepper.cpp b/components/omega/src/timeStepping/RungeKutta2Stepper.cpp index 99410e07ea71..0304112849a1 100644 --- a/components/omega/src/timeStepping/RungeKutta2Stepper.cpp +++ b/components/omega/src/timeStepping/RungeKutta2Stepper.cpp @@ -83,7 +83,8 @@ void RungeKutta2Stepper::doStep(OceanState *State, // model state applyImplicitVerticalMixing(State, CurLevel, CurLevel, CurLevel, "RK2"); - validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel); + validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); diff --git a/components/omega/src/timeStepping/RungeKutta4Stepper.cpp b/components/omega/src/timeStepping/RungeKutta4Stepper.cpp index d72e49b60065..d6bdb4b22cc9 100644 --- a/components/omega/src/timeStepping/RungeKutta4Stepper.cpp +++ b/components/omega/src/timeStepping/RungeKutta4Stepper.cpp @@ -146,7 +146,8 @@ void RungeKutta4Stepper::doStep(OceanState *State, // model state applyImplicitVerticalMixing(State, CurLevel, CurLevel, CurLevel, "RK4"); - validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel); + validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); diff --git a/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp b/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp index 892295b7e469..90cebc82fe89 100644 --- a/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp +++ b/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp @@ -640,7 +640,8 @@ void SplitExplicitRK2Stepper::doStep(OceanState *State, Pacer::stop("SE-RK2:vMixHaloExch", 3); } - validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel); + validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); From cf8dd3a8852994eb7b7b060cb4068d779ead14b0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 27 Oct 2026 04:24:29 -0500 Subject: [PATCH 4/4] Document validation reports and critical messages on all tasks The user's guide gains a section on state validation, its options and what it reports, and says where tasks that do not log write critical messages. The design doc covers the location of the first bad value, the collective abort and the stream written before it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../omega/doc/design/StateValidation.md | 140 +++++++++++++----- components/omega/doc/userGuide/Logging.md | 8 + components/omega/doc/userGuide/OceanState.md | 26 ++++ 3 files changed, 135 insertions(+), 39 deletions(-) diff --git a/components/omega/doc/design/StateValidation.md b/components/omega/doc/design/StateValidation.md index 7c98acfb177c..a1a732dca7d6 100644 --- a/components/omega/doc/design/StateValidation.md +++ b/components/omega/doc/design/StateValidation.md @@ -22,6 +22,8 @@ configuration options under `Omega::State::Validation`: (with a full stack backtrace printed first). - **`AbortOnOOB`** (default: `false`) — when `true`, out-of-bounds values trigger the same hard abort. +- **`WriteStreamOnAbort`** (default: `History`) — the stream that every task + writes before an abort, so the state that failed can be inspected. When both flags are `false` and errors are detected, a critical-level log message is emitted and the run continues rather than being terminated. @@ -81,7 +83,9 @@ parallel reductions. On detection of any failure the module must log a critical-level message that identifies the field name, the nature of the problem (NaN or out-of-bounds), -and the number of offending elements. +the number of offending elements, and the global cell ID, latitude, longitude, +layer and value of the first of them. The message must reach the user from +the task that found the problem, whether or not that task writes a log. ### 2.9 Requirement: Graceful handling of absent tracers @@ -103,7 +107,11 @@ out-of-bounds errors cause the run to abort. Two boolean YAML options - When both are `false`, errors are logged at critical severity and the run continues. -The defaults are `AbortOnNan: true` and `AbortOnOOB: false`. +The defaults are `AbortOnNan: true` and `AbortOnOOB: false`. The decision is +made from the counts summed over all tasks, so that every task takes part in +the abort. Before aborting, every task writes the stream named by +`WriteStreamOnAbort` (default `History`, `''` for none), so that the state that +failed can be inspected afterwards. ### 2.11 Desired: Configurable valid ranges @@ -129,7 +137,10 @@ skipped): $[\text{MinVal},\, \text{MaxVal}]$. Separating the two passes avoids potentially undefined behaviour when -comparing NaN values with `<` or `>`. +comparing NaN values with `<` or `>`. Only when a count is nonzero, a further +`parallelReduce` with a `Kokkos::Min` reducer finds the smallest flat index +$i N_\text{vert} + k$ of an offending element, whose location and value are +then reported. A valid state therefore costs only the two counting passes. For a 2-D field $f_{i,k}$ with $i \in [0,\, N_\text{cells})$ and $k \in [0,\, N_\text{vert})$ the two counts are: @@ -178,18 +189,41 @@ under `Omega::State::Validation`: |---------------|------|---------|----------------------------------------------| | `AbortOnNan` | bool | `true` | Abort via `MPI_Abort` when NaNs are detected | | `AbortOnOOB` | bool | `false` | Abort via `MPI_Abort` when OOBs are detected | +| `WriteStreamOnAbort` | string | `History` | Stream written on every task before an abort (`''` for none) | When both flags are `false` and errors are found, a CRITICAL log message is emitted and the run continues. #### 4.1.2 Class/structs/data types -The module introduces one file-local plain struct: +The module introduces one public struct, which `checkOceanState` can fill +for the caller: + +```c++ +struct ValidationIssue { + std::string Field; // e.g. Temperature + bool IsNaN; // NaN (true) or out of the valid range (false) + I4 Count; // number of entries on this task with the problem + I4 Cell; // local cell index of the first entry + I4 Layer; // vertical layer of the first entry + Real Value; // value of the first entry +}; +``` + +and two file-local plain structs: ```c++ struct ValidationAbortConfig { - bool AbortOnOOB = false; - bool AbortOnNan = true; + bool AbortOnOOB = false; + bool AbortOnNan = true; + std::string WriteStreamOnAbort = "History"; +}; + +struct FieldCheck { + I4 NaNs = 0; + I4 OOBs = 0; + I4 FirstNaN = -1; // flat index of the first NaN, -1 if none + I4 FirstOOB = -1; // flat index of the first OOB, -1 if none }; ``` @@ -208,13 +242,15 @@ Performs all field checks and returns separate NaN and out-of-bounds counts: std::pair checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, const VertCoord *VCoord, - I4 TimeLevel); + I4 TimeLevel, + std::vector *Issues = nullptr); ``` -Checks all fields described in Section 2, skipping inactive cells -(`CellMask == 0`). Logs critical messages for each type of error. Returns -`{TotalNaNs, TotalOOBs}` where both values are 0 if all checks pass. Does -**not** abort. Suitable for calling from tests. +Checks all fields described in Section 2 on the local task, skipping inactive +cells (`CellMask == 0`). Logs a critical message for each type of error, with +the location and value of the first, and appends the same to `Issues` if it +is given. Returns `{TotalNaNs, TotalOOBs}` where both values are 0 if all +checks pass. Does **not** abort. Suitable for calling from tests. #### 4.2.2 `validateOceanState` (public) @@ -224,48 +260,53 @@ Production entry-point that applies the configured abort policy on failure: void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, const VertCoord *VCoord, - I4 TimeLevel); + I4 TimeLevel, + const Clock *ModelClock = nullptr); ``` -Calls `checkOceanState` to obtain `{NaNs, OOBs}`, then reads -`ValidationAbortConfig` via `getValidationAbortConfig`. Aborts via -`abortWithMessage` (which logs at CRITICAL severity, prints a stack -backtrace, and calls `MPI_Abort`) if: +Calls `checkOceanState` to obtain the local `{NaNs, OOBs}`, sums them over all +tasks (in one `globalSum` together with the EOS negative-salinity count), then +reads `ValidationAbortConfig` via `getValidationAbortConfig`. It must +therefore be called on every task. Aborts via `abortRun` if: - `NaNs > 0` **and** `AbortOnNan` is `true`, **or** - `OOBs > 0` **and** `AbortOnOOB` is `true`. -If neither abort condition is met but errors were detected, a CRITICAL log -message is emitted and the function returns normally. +If neither abort condition is met but errors were detected, the master task +logs a CRITICAL message with the totals and the function returns normally. +The time steppers pass their clock as `ModelClock`, which time-stamps the +stream written before an abort. #### 4.2.3 `checkArray2D` (file-local helper) ```c++ -static std::pair checkArray2D(const Array2DReal &Arr, - I4 NRows, I4 NCols, - Real MinVal, Real MaxVal, - bool CheckMin, - const Array2DReal &CellMask); +static FieldCheck checkArray2D(const Array2DReal &Arr, + I4 NRows, I4 NCols, + Real MinVal, Real MaxVal, + bool CheckMin, + const Array2DReal &CellMask); ``` Performs the NaN and bounds counts for a 2-D device array over the first `NRows` rows and `NCols` columns, restricted to active cells (`CellMask(Row, Col) > 0`). When `CheckMin` is `false` only the upper bound is enforced (not needed for any current field but useful for future -extension). Returns `{NaNCount, OutOfRangeCount}`. +extension). Returns the counts and the flat index of the first NaN and +out-of-range element. #### 4.2.4 `checkTracerArray` (file-local helper) ```c++ -static std::pair checkTracerArray(const Array3DReal &Tracers3D, - I4 TracerIdx, - I4 NCells, I4 NVert, - Real MinVal, Real MaxVal, - const Array2DReal &CellMask); +static FieldCheck checkTracerArray(const Array3DReal &Tracers3D, + I4 TracerIdx, + I4 NCells, I4 NVert, + Real MinVal, Real MaxVal, + const Array2DReal &CellMask); ``` Performs the NaN and bounds counts for a single tracer slice (identified by `TracerIdx`) of the 3-D tracer array, restricted to active cells -(`CellMask(Cell, K) > 0`). Returns `{NaNCount, OutOfRangeCount}`. +(`CellMask(Cell, K) > 0`). Returns the counts and the flat index of the +first NaN and out-of-range element. #### 4.2.5 `getValidationAbortConfig` (file-local helper) @@ -273,21 +314,32 @@ Performs the NaN and bounds counts for a single tracer slice (identified by static ValidationAbortConfig getValidationAbortConfig(); ``` -Reads the two boolean options `AbortOnNan` and `AbortOnOOB` from the OMEGA -config under `Omega::State::Validation` and returns a populated +Reads the options `AbortOnNan`, `AbortOnOOB` and `WriteStreamOnAbort` from +the OMEGA config under `Omega::State::Validation` and returns a populated `ValidationAbortConfig`. If the config node is absent the struct's -default-initialised values (`AbortOnNan = true`, `AbortOnOOB = false`) are -returned unchanged. +default-initialised values (`AbortOnNan = true`, `AbortOnOOB = false`, +`WriteStreamOnAbort = "History"`) are returned unchanged. -#### 4.2.6 `abortWithMessage` (file-local helper) +#### 4.2.6 `abortRun` (file-local helper) ```c++ -static void abortWithMessage(const std::string &Msg); +static void abortRun(I8 NaNs, I8 OOBs, const std::string &StreamName, + const Clock *ModelClock); ``` -Logs `Msg` at critical severity, prints a stack backtrace using `cpptrace`, -then calls `MPI_Abort` on the default OMEGA communicator with error code -`ErrorCode::Critical`. +Called on every task. The master task logs the totals at critical severity. +If `StreamName` is not empty, the stream is defined (`IOStream::isDefined`) +and `ModelClock` is given, every task force-writes the stream with +`IOStream::write`. The master task then prints a stack backtrace using +`cpptrace`, and after a barrier every task calls `MPI_Abort` on the default +OMEGA communicator with error code `ErrorCode::Critical`. + +#### 4.2.7 Critical messages on tasks that do not log + +The critical messages of `checkOceanState` come from the task that owns the +offending cells, which is usually not a task that writes a log. Logging +therefore keeps critical messages on the other tasks, writing them to stderr +labeled with the task, while all lower levels stay off there. ## 5 Verification and Testing @@ -336,3 +388,13 @@ The following sub-tests are implemented: | `testOOBSalinity` | 9999 g kg⁻¹ (> max 60) | Salinity | Tests requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.10. + +### 5.3 Location of the first bad value + +On each task, a single NaN is put in `Temperature`, and then a single value of +99 g kg⁻¹ in `Salinity`, at the last owned cell that is active in the second +layer. `checkOceanState` must return exactly one `ValidationIssue`, for the +right field and kind, with a count of 1, that cell and layer and, for the +out-of-range value, a value of 99. + +Tests requirement: 2.8. diff --git a/components/omega/doc/userGuide/Logging.md b/components/omega/doc/userGuide/Logging.md index a05504efb42a..e134840e9772 100644 --- a/components/omega/doc/userGuide/Logging.md +++ b/components/omega/doc/userGuide/Logging.md @@ -89,6 +89,14 @@ shared log for the whole test suite. In Standalone mode, Omega redirects `std::cout` and `std::cerr` to the log file on the ranks that log. +The ranks that do not log still write critical messages, and only those, to +`std::cerr`, each labeled with the rank, e.g. `[critical] task 1475: ...`. +Critical messages come before an abort, and are often raised only on the rank +that found the problem, such as a NaN found by +[state validation](omega-user-state-validation) in a cell that rank owns. +When a run aborts, look for them in the job's standard error as well as in the +log. + ## E3SM Component Build The E3SM driver sets the log file name, so Omega's messages go to `ocn.log.*` diff --git a/components/omega/doc/userGuide/OceanState.md b/components/omega/doc/userGuide/OceanState.md index 53bc70e6d404..30efb9f5c68c 100644 --- a/components/omega/doc/userGuide/OceanState.md +++ b/components/omega/doc/userGuide/OceanState.md @@ -15,3 +15,29 @@ The relative pressure at the top of the ocean column, `SurfacePressure`, is owne boundary condition of the pressure field. It is still registered in the `State` and `Restart` field groups, so it continues to be written to restart files and read from the initial-condition and restart files when present; if it is absent, it defaults to zero. + +(omega-user-state-validation)= + +### State validation + +After each time step, Omega checks `PseudoThickness`, `KineticEnergyCell`, `Temperature` and +`Salinity` for NaNs and for values outside a plausible range. For each problem it finds, the rank +that owns the cells writes a critical message giving the field, the number of entries and the +global cell ID, latitude, longitude and layer of the first one, with its value if it is out of +range. Ranks that do not log write these messages to standard error (see +[logging](omega-user-logging)). What happens next is set under `State: Validation`: + +```yaml +Omega: + State: + Validation: + AbortOnOOB: false + AbortOnNan: true + WriteStreamOnAbort: History +``` + +`AbortOnNan` and `AbortOnOOB` decide whether NaNs and out-of-range values abort the run; the +decision is based on the counts summed over all ranks. Before aborting, every rank writes the +stream named by `WriteStreamOnAbort`, so the bad state can be inspected. The record has the time +at the start of the failing step. Set it to `''` to write nothing; a stream that is not defined +is skipped. When the run continues instead, the master rank logs the total counts.