diff --git a/components/omega/configs/Default.yml b/components/omega/configs/Default.yml index 635a155a20a5..2457a95d7b5c 100644 --- a/components/omega/configs/Default.yml +++ b/components/omega/configs/Default.yml @@ -34,6 +34,7 @@ Omega: Validation: AbortOnOOB: false AbortOnNan: true + WriteStreamOnAbort: History Advection: Coef3rdOrder: 0.25 FluxThicknessType: Center diff --git a/components/omega/doc/design/StateValidation.md b/components/omega/doc/design/StateValidation.md index 7c98acfb177c..a1a732dca7d6 100644 --- a/components/omega/doc/design/StateValidation.md +++ b/components/omega/doc/design/StateValidation.md @@ -22,6 +22,8 @@ configuration options under `Omega::State::Validation`: (with a full stack backtrace printed first). - **`AbortOnOOB`** (default: `false`) — when `true`, out-of-bounds values trigger the same hard abort. +- **`WriteStreamOnAbort`** (default: `History`) — the stream that every task + writes before an abort, so the state that failed can be inspected. When both flags are `false` and errors are detected, a critical-level log message is emitted and the run continues rather than being terminated. @@ -81,7 +83,9 @@ parallel reductions. On detection of any failure the module must log a critical-level message that identifies the field name, the nature of the problem (NaN or out-of-bounds), -and the number of offending elements. +the number of offending elements, and the global cell ID, latitude, longitude, +layer and value of the first of them. The message must reach the user from +the task that found the problem, whether or not that task writes a log. ### 2.9 Requirement: Graceful handling of absent tracers @@ -103,7 +107,11 @@ out-of-bounds errors cause the run to abort. Two boolean YAML options - When both are `false`, errors are logged at critical severity and the run continues. -The defaults are `AbortOnNan: true` and `AbortOnOOB: false`. +The defaults are `AbortOnNan: true` and `AbortOnOOB: false`. The decision is +made from the counts summed over all tasks, so that every task takes part in +the abort. Before aborting, every task writes the stream named by +`WriteStreamOnAbort` (default `History`, `''` for none), so that the state that +failed can be inspected afterwards. ### 2.11 Desired: Configurable valid ranges @@ -129,7 +137,10 @@ skipped): $[\text{MinVal},\, \text{MaxVal}]$. Separating the two passes avoids potentially undefined behaviour when -comparing NaN values with `<` or `>`. +comparing NaN values with `<` or `>`. Only when a count is nonzero, a further +`parallelReduce` with a `Kokkos::Min` reducer finds the smallest flat index +$i N_\text{vert} + k$ of an offending element, whose location and value are +then reported. A valid state therefore costs only the two counting passes. For a 2-D field $f_{i,k}$ with $i \in [0,\, N_\text{cells})$ and $k \in [0,\, N_\text{vert})$ the two counts are: @@ -178,18 +189,41 @@ under `Omega::State::Validation`: |---------------|------|---------|----------------------------------------------| | `AbortOnNan` | bool | `true` | Abort via `MPI_Abort` when NaNs are detected | | `AbortOnOOB` | bool | `false` | Abort via `MPI_Abort` when OOBs are detected | +| `WriteStreamOnAbort` | string | `History` | Stream written on every task before an abort (`''` for none) | When both flags are `false` and errors are found, a CRITICAL log message is emitted and the run continues. #### 4.1.2 Class/structs/data types -The module introduces one file-local plain struct: +The module introduces one public struct, which `checkOceanState` can fill +for the caller: + +```c++ +struct ValidationIssue { + std::string Field; // e.g. Temperature + bool IsNaN; // NaN (true) or out of the valid range (false) + I4 Count; // number of entries on this task with the problem + I4 Cell; // local cell index of the first entry + I4 Layer; // vertical layer of the first entry + Real Value; // value of the first entry +}; +``` + +and two file-local plain structs: ```c++ struct ValidationAbortConfig { - bool AbortOnOOB = false; - bool AbortOnNan = true; + bool AbortOnOOB = false; + bool AbortOnNan = true; + std::string WriteStreamOnAbort = "History"; +}; + +struct FieldCheck { + I4 NaNs = 0; + I4 OOBs = 0; + I4 FirstNaN = -1; // flat index of the first NaN, -1 if none + I4 FirstOOB = -1; // flat index of the first OOB, -1 if none }; ``` @@ -208,13 +242,15 @@ Performs all field checks and returns separate NaN and out-of-bounds counts: std::pair checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, const VertCoord *VCoord, - I4 TimeLevel); + I4 TimeLevel, + std::vector *Issues = nullptr); ``` -Checks all fields described in Section 2, skipping inactive cells -(`CellMask == 0`). Logs critical messages for each type of error. Returns -`{TotalNaNs, TotalOOBs}` where both values are 0 if all checks pass. Does -**not** abort. Suitable for calling from tests. +Checks all fields described in Section 2 on the local task, skipping inactive +cells (`CellMask == 0`). Logs a critical message for each type of error, with +the location and value of the first, and appends the same to `Issues` if it +is given. Returns `{TotalNaNs, TotalOOBs}` where both values are 0 if all +checks pass. Does **not** abort. Suitable for calling from tests. #### 4.2.2 `validateOceanState` (public) @@ -224,48 +260,53 @@ Production entry-point that applies the configured abort policy on failure: void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, const VertCoord *VCoord, - I4 TimeLevel); + I4 TimeLevel, + const Clock *ModelClock = nullptr); ``` -Calls `checkOceanState` to obtain `{NaNs, OOBs}`, then reads -`ValidationAbortConfig` via `getValidationAbortConfig`. Aborts via -`abortWithMessage` (which logs at CRITICAL severity, prints a stack -backtrace, and calls `MPI_Abort`) if: +Calls `checkOceanState` to obtain the local `{NaNs, OOBs}`, sums them over all +tasks (in one `globalSum` together with the EOS negative-salinity count), then +reads `ValidationAbortConfig` via `getValidationAbortConfig`. It must +therefore be called on every task. Aborts via `abortRun` if: - `NaNs > 0` **and** `AbortOnNan` is `true`, **or** - `OOBs > 0` **and** `AbortOnOOB` is `true`. -If neither abort condition is met but errors were detected, a CRITICAL log -message is emitted and the function returns normally. +If neither abort condition is met but errors were detected, the master task +logs a CRITICAL message with the totals and the function returns normally. +The time steppers pass their clock as `ModelClock`, which time-stamps the +stream written before an abort. #### 4.2.3 `checkArray2D` (file-local helper) ```c++ -static std::pair checkArray2D(const Array2DReal &Arr, - I4 NRows, I4 NCols, - Real MinVal, Real MaxVal, - bool CheckMin, - const Array2DReal &CellMask); +static FieldCheck checkArray2D(const Array2DReal &Arr, + I4 NRows, I4 NCols, + Real MinVal, Real MaxVal, + bool CheckMin, + const Array2DReal &CellMask); ``` Performs the NaN and bounds counts for a 2-D device array over the first `NRows` rows and `NCols` columns, restricted to active cells (`CellMask(Row, Col) > 0`). When `CheckMin` is `false` only the upper bound is enforced (not needed for any current field but useful for future -extension). Returns `{NaNCount, OutOfRangeCount}`. +extension). Returns the counts and the flat index of the first NaN and +out-of-range element. #### 4.2.4 `checkTracerArray` (file-local helper) ```c++ -static std::pair checkTracerArray(const Array3DReal &Tracers3D, - I4 TracerIdx, - I4 NCells, I4 NVert, - Real MinVal, Real MaxVal, - const Array2DReal &CellMask); +static FieldCheck checkTracerArray(const Array3DReal &Tracers3D, + I4 TracerIdx, + I4 NCells, I4 NVert, + Real MinVal, Real MaxVal, + const Array2DReal &CellMask); ``` Performs the NaN and bounds counts for a single tracer slice (identified by `TracerIdx`) of the 3-D tracer array, restricted to active cells -(`CellMask(Cell, K) > 0`). Returns `{NaNCount, OutOfRangeCount}`. +(`CellMask(Cell, K) > 0`). Returns the counts and the flat index of the +first NaN and out-of-range element. #### 4.2.5 `getValidationAbortConfig` (file-local helper) @@ -273,21 +314,32 @@ Performs the NaN and bounds counts for a single tracer slice (identified by static ValidationAbortConfig getValidationAbortConfig(); ``` -Reads the two boolean options `AbortOnNan` and `AbortOnOOB` from the OMEGA -config under `Omega::State::Validation` and returns a populated +Reads the options `AbortOnNan`, `AbortOnOOB` and `WriteStreamOnAbort` from +the OMEGA config under `Omega::State::Validation` and returns a populated `ValidationAbortConfig`. If the config node is absent the struct's -default-initialised values (`AbortOnNan = true`, `AbortOnOOB = false`) are -returned unchanged. +default-initialised values (`AbortOnNan = true`, `AbortOnOOB = false`, +`WriteStreamOnAbort = "History"`) are returned unchanged. -#### 4.2.6 `abortWithMessage` (file-local helper) +#### 4.2.6 `abortRun` (file-local helper) ```c++ -static void abortWithMessage(const std::string &Msg); +static void abortRun(I8 NaNs, I8 OOBs, const std::string &StreamName, + const Clock *ModelClock); ``` -Logs `Msg` at critical severity, prints a stack backtrace using `cpptrace`, -then calls `MPI_Abort` on the default OMEGA communicator with error code -`ErrorCode::Critical`. +Called on every task. The master task logs the totals at critical severity. +If `StreamName` is not empty, the stream is defined (`IOStream::isDefined`) +and `ModelClock` is given, every task force-writes the stream with +`IOStream::write`. The master task then prints a stack backtrace using +`cpptrace`, and after a barrier every task calls `MPI_Abort` on the default +OMEGA communicator with error code `ErrorCode::Critical`. + +#### 4.2.7 Critical messages on tasks that do not log + +The critical messages of `checkOceanState` come from the task that owns the +offending cells, which is usually not a task that writes a log. Logging +therefore keeps critical messages on the other tasks, writing them to stderr +labeled with the task, while all lower levels stay off there. ## 5 Verification and Testing @@ -336,3 +388,13 @@ The following sub-tests are implemented: | `testOOBSalinity` | 9999 g kg⁻¹ (> max 60) | Salinity | Tests requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.10. + +### 5.3 Location of the first bad value + +On each task, a single NaN is put in `Temperature`, and then a single value of +99 g kg⁻¹ in `Salinity`, at the last owned cell that is active in the second +layer. `checkOceanState` must return exactly one `ValidationIssue`, for the +right field and kind, with a count of 1, that cell and layer and, for the +out-of-range value, a value of 99. + +Tests requirement: 2.8. diff --git a/components/omega/doc/userGuide/Logging.md b/components/omega/doc/userGuide/Logging.md index a05504efb42a..e134840e9772 100644 --- a/components/omega/doc/userGuide/Logging.md +++ b/components/omega/doc/userGuide/Logging.md @@ -89,6 +89,14 @@ shared log for the whole test suite. In Standalone mode, Omega redirects `std::cout` and `std::cerr` to the log file on the ranks that log. +The ranks that do not log still write critical messages, and only those, to +`std::cerr`, each labeled with the rank, e.g. `[critical] task 1475: ...`. +Critical messages come before an abort, and are often raised only on the rank +that found the problem, such as a NaN found by +[state validation](omega-user-state-validation) in a cell that rank owns. +When a run aborts, look for them in the job's standard error as well as in the +log. + ## E3SM Component Build The E3SM driver sets the log file name, so Omega's messages go to `ocn.log.*` diff --git a/components/omega/doc/userGuide/OceanState.md b/components/omega/doc/userGuide/OceanState.md index 53bc70e6d404..30efb9f5c68c 100644 --- a/components/omega/doc/userGuide/OceanState.md +++ b/components/omega/doc/userGuide/OceanState.md @@ -15,3 +15,29 @@ The relative pressure at the top of the ocean column, `SurfacePressure`, is owne boundary condition of the pressure field. It is still registered in the `State` and `Restart` field groups, so it continues to be written to restart files and read from the initial-condition and restart files when present; if it is absent, it defaults to zero. + +(omega-user-state-validation)= + +### State validation + +After each time step, Omega checks `PseudoThickness`, `KineticEnergyCell`, `Temperature` and +`Salinity` for NaNs and for values outside a plausible range. For each problem it finds, the rank +that owns the cells writes a critical message giving the field, the number of entries and the +global cell ID, latitude, longitude and layer of the first one, with its value if it is out of +range. Ranks that do not log write these messages to standard error (see +[logging](omega-user-logging)). What happens next is set under `State: Validation`: + +```yaml +Omega: + State: + Validation: + AbortOnOOB: false + AbortOnNan: true + WriteStreamOnAbort: History +``` + +`AbortOnNan` and `AbortOnOOB` decide whether NaNs and out-of-range values abort the run; the +decision is based on the counts summed over all ranks. Before aborting, every rank writes the +stream named by `WriteStreamOnAbort`, so the bad state can be inspected. The record has the time +at the start of the failing step. Set it to `''` to write nothing; a stream that is not defined +is skipped. When the run continues instead, the master rank logs the total counts. diff --git a/components/omega/src/infra/IOStream.cpp b/components/omega/src/infra/IOStream.cpp index 8afb4d3b4523..4041cd1192c4 100644 --- a/components/omega/src/infra/IOStream.cpp +++ b/components/omega/src/infra/IOStream.cpp @@ -274,6 +274,13 @@ bool IOStream::validateAll() { } // End validateAll +//------------------------------------------------------------------------------ +// Returns true if a stream with the given name has been defined. +bool IOStream::isDefined(const std::string &StreamName // [in] stream name +) { + return AllStreams.find(StreamName) != AllStreams.end(); +} + //------------------------------------------------------------------------------ // Reads a single stream if it is time. Error IOStream::read( diff --git a/components/omega/src/infra/IOStream.h b/components/omega/src/infra/IOStream.h index a02e18e210f9..c21bcc81bf55 100644 --- a/components/omega/src/infra/IOStream.h +++ b/components/omega/src/infra/IOStream.h @@ -365,6 +365,11 @@ class IOStream { /// Returns true if all streams are valid. static bool validateAll(); + //--------------------------------------------------------------------------- + /// Returns true if a stream with the given name has been defined. + static bool isDefined(const std::string &StreamName ///< [in] stream name + ); + //--------------------------------------------------------------------------- /// Reads a stream if it is time. static Error read(const std::string &StreamName, ///< [in] Name of stream diff --git a/components/omega/src/infra/Logging.cpp b/components/omega/src/infra/Logging.cpp index b05aca60a734..0d0c30b0554f 100644 --- a/components/omega/src/infra/Logging.cpp +++ b/components/omega/src/infra/Logging.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -228,6 +229,24 @@ static std::vector resolveLogTasks(const OMEGA::MachEnv *DefEnv, return Tasks; } +//------------------------------------------------------------------------------ +// Set up logging on a task that does not log. Only critical messages get +// through, to stderr and labeled with the task. They come just before an +// abort, often from a task other than the ones that log (e.g. the task that +// found a NaN), and would otherwise be lost. +static void initCriticalOnlyLogging(OMEGA::I4 TaskId) { + + const std::string Name = "critical_only"; + auto Logger = spdlog::get(Name); + if (!Logger) + Logger = spdlog::stderr_logger_mt(Name); + spdlog::set_default_logger(Logger); + + spdlog::set_pattern("[%l] task " + std::to_string(TaskId) + ": %v"); + spdlog::set_level(spdlog::level::critical); + spdlog::flush_on(spdlog::level::critical); +} + //------------------------------------------------------------------------------ // Initialize logging for case of a pre-defined custom logger // Return code: 1->enabled, 0->disabled, negative values->errors @@ -261,7 +280,7 @@ int initLogging( RetVal = 1; // log enabled } else { - spdlog::set_level(spdlog::level::off); + initCriticalOnlyLogging(TaskId); RetVal = 0; // log disabled } @@ -330,7 +349,7 @@ int initLogging( } } else { - spdlog::set_level(spdlog::level::off); + initCriticalOnlyLogging(TaskId); RetVal = 0; // log disabled } diff --git a/components/omega/src/ocn/StateValidation.cpp b/components/omega/src/ocn/StateValidation.cpp index 80bef3ea65c1..0d21b645d8e6 100644 --- a/components/omega/src/ocn/StateValidation.cpp +++ b/components/omega/src/ocn/StateValidation.cpp @@ -3,9 +3,11 @@ // Validates ocean state fields by checking for NaN values and // out-of-bounds conditions. Abort behaviour is controlled at runtime via the // YAML options Omega::State::Validation::AbortOnNan (default true) and -// Omega::State::Validation::AbortOnOOB (default false). When an abort flag is -// set a critical error log with backtrace is emitted and MPI_Abort is called; -// otherwise a CRITICAL log message is written and execution continues. +// Omega::State::Validation::AbortOnOOB (default false). The decision is made +// from the counts on all tasks. When an abort flag is set, every task writes +// the stream named by Omega::State::Validation::WriteStreamOnAbort (default +// History), a critical error log with backtrace is emitted and MPI_Abort is +// called; otherwise a CRITICAL log message is written and execution continues. // //===----------------------------------------------------------------------===// @@ -14,8 +16,12 @@ #include "AuxiliaryState.h" #include "Config.h" #include "DataTypes.h" +#include "Decomp.h" #include "Eos.h" #include "Error.h" +#include "GlobalConstants.h" +#include "HorzMesh.h" +#include "IOStream.h" #include "Logging.h" #include "MachEnv.h" #include "OceanState.h" @@ -25,16 +31,20 @@ #include "VertCoord.h" #include "mpi.h" +#include #include #include +#include #include #include +#include namespace OMEGA { struct ValidationAbortConfig { - bool AbortOnOOB = false; - bool AbortOnNan = true; + bool AbortOnOOB = false; + bool AbortOnNan = true; + std::string WriteStreamOnAbort = "History"; }; static ValidationAbortConfig getValidationAbortConfig() { @@ -57,104 +67,230 @@ static ValidationAbortConfig getValidationAbortConfig() { ValidationConfig.get("AbortOnOOB", AbortConfig.AbortOnOOB); ValidationConfig.get("AbortOnNan", AbortConfig.AbortOnNan); + ValidationConfig.get("WriteStreamOnAbort", AbortConfig.WriteStreamOnAbort); return AbortConfig; } //------------------------------------------------------------------------------ -// Helper: abort on the local Omega communicator with a message and backtrace -static void abortWithMessage(const std::string &Msg) { - LOG_CRITICAL("{}", Msg); - cpptrace::generate_trace().print(); - MPI_Comm Comm = MachEnv::getDefault()->getComm(); +// Helper: abort the run, first writing the given stream if there is one and a +// clock to stamp it with. It must be called on every task, since writing a +// stream is collective. The tasks that found problems have already logged +// them; the master task adds the totals and a backtrace. +static void abortRun(I8 NaNs, I8 OOBs, const std::string &StreamName, + const Clock *ModelClock) { + + MachEnv *DefEnv = MachEnv::getDefault(); + MPI_Comm Comm = DefEnv->getComm(); + bool IsMaster = DefEnv->isMasterTask(); + + if (IsMaster) { + LOG_CRITICAL("StateValidation: Ocean state validation aborting with {} " + "NaNs and {} OOBs on all tasks. The tasks that found them " + "give the field and location in critical messages, on " + "stderr for tasks that do not log.", + NaNs, OOBs); + } + + if (!StreamName.empty() && ModelClock) { + if (IOStream::isDefined(StreamName)) { + if (IsMaster) { + LOG_CRITICAL("StateValidation: Writing stream {} before aborting, " + "with the time at the start of the failing step", + StreamName); + } + IOStream::write(StreamName, ModelClock, /*ForceWrite=*/true); + } else if (IsMaster) { + LOG_CRITICAL("StateValidation: Not writing stream {} before " + "aborting because it is not defined", + StreamName); + } + } + + if (IsMaster) { + cpptrace::generate_trace().print(); + } + MPI_Barrier(Comm); MPI_Abort(Comm, static_cast(ErrorCode::Critical)); } //------------------------------------------------------------------------------ -// Helper: count NaN entries and out-of-range entries in a 2-D Real device -// array over the first NCells/NEdges rows and NVert columns, restricted to -// active cells (CellMask > 0). -// Returns {NaNCount, OutOfRangeCount}. -static std::pair checkArray2D(const Array2DReal &Arr, I4 NRows, - I4 NCols, Real MinVal, Real MaxVal, - bool CheckMin, - const Array2DReal &CellMask) { - I4 NaNCount = 0; - I4 OutOfRangeCount = 0; - +// Helper: count the entries over the first NRows rows and NCols columns for +// which IsBad(Row, Col) is true +template +static I4 countEntries(const std::string &Label, I4 NRows, I4 NCols, + const F &IsBad) { + I4 Count = 0; parallelReduce( - "CheckNaN", {NRows, NCols}, + Label, {NRows, NCols}, KOKKOS_LAMBDA(int Row, int Col, int &Accum) { - if (CellMask(Row, Col) == 0) { - return; - } - Real Val = Arr(Row, Col); - if (Kokkos::isnan(Val)) { + if (IsBad(Row, Col)) { ++Accum; } }, - NaNCount); + Count); + return Count; +} +//------------------------------------------------------------------------------ +// Helper: the smallest flat index Row * NCols + Col for which IsBad(Row, Col) +// is true, or -1 if there is none +template +static I4 firstEntry(const std::string &Label, I4 NRows, I4 NCols, + const F &IsBad) { + I4 First = INT_MAX; parallelReduce( - "CheckBounds", {NRows, NCols}, - KOKKOS_LAMBDA(int Row, int Col, int &Accum) { - if (CellMask(Row, Col) == 0) { - return; - } - Real Val = Arr(Row, Col); - if (!Kokkos::isnan(Val)) { - if (Val > MaxVal) { - ++Accum; - } else if (CheckMin && Val < MinVal) { - ++Accum; - } + Label, {NRows, NCols}, + KOKKOS_LAMBDA(int Row, int Col, I4 &Accum) { + if (IsBad(Row, Col)) { + Accum = Kokkos::min(Accum, Row * NCols + Col); } }, - OutOfRangeCount); + Kokkos::Min(First)); + return First == INT_MAX ? -1 : First; +} + +//------------------------------------------------------------------------------ +// NaN and out-of-range counts for one field, with the flat index of the first +// of each (-1 if there is none) +struct FieldCheck { + I4 NaNs = 0; + I4 OOBs = 0; + I4 FirstNaN = -1; + I4 FirstOOB = -1; +}; + +//------------------------------------------------------------------------------ +// Helper: count the NaN and out-of-range entries of a field and, only when +// there are some, find the first of each, so a valid field costs only the +// two counts +template +static FieldCheck checkEntries(I4 NRows, I4 NCols, const FNaN &IsNaN, + const FOOB &IsOOB) { + FieldCheck Check; + Check.NaNs = countEntries("CheckNaN", NRows, NCols, IsNaN); + Check.OOBs = countEntries("CheckBounds", NRows, NCols, IsOOB); + if (Check.NaNs > 0) { + Check.FirstNaN = firstEntry("FirstNaN", NRows, NCols, IsNaN); + } + if (Check.OOBs > 0) { + Check.FirstOOB = firstEntry("FirstOOB", NRows, NCols, IsOOB); + } + return Check; +} - return {NaNCount, OutOfRangeCount}; +//------------------------------------------------------------------------------ +// Helper: check a 2-D Real device array over the first NCells/NEdges rows and +// NVert columns for NaN and out-of-range entries, restricted to active cells +// (CellMask > 0). +static FieldCheck checkArray2D(const Array2DReal &Arr, I4 NRows, I4 NCols, + Real MinVal, Real MaxVal, bool CheckMin, + const Array2DReal &CellMask) { + auto IsNaN = KOKKOS_LAMBDA(int Row, int Col) { + return CellMask(Row, Col) != 0 && Kokkos::isnan(Arr(Row, Col)); + }; + // NaNs are excluded explicitly, since comparisons with NaN are false + auto IsOOB = KOKKOS_LAMBDA(int Row, int Col) { + if (CellMask(Row, Col) == 0) { + return false; + } + Real Val = Arr(Row, Col); + return !Kokkos::isnan(Val) && + (Val > MaxVal || (CheckMin && Val < MinVal)); + }; + return checkEntries(NRows, NCols, IsNaN, IsOOB); } //------------------------------------------------------------------------------ -// Helper: count NaN and out-of-range entries for a single tracer (row = cell, -// col = vert) extracted from the 3-D tracer array at the given tracer index, +// Helper: check a single tracer (row = cell, col = vert) of the 3-D tracer +// array at the given tracer index for NaN and out-of-range entries, // restricted to active cells (CellMask > 0). -static std::pair checkTracerArray(const Array3DReal &Tracers3D, - I4 TracerIdx, I4 NCells, I4 NVert, - Real MinVal, Real MaxVal, - const Array2DReal &CellMask) { - I4 NaNCount = 0; - I4 OutOfRangeCount = 0; +static FieldCheck checkTracerArray(const Array3DReal &Tracers3D, I4 TracerIdx, + I4 NCells, I4 NVert, Real MinVal, + Real MaxVal, const Array2DReal &CellMask) { + auto IsNaN = KOKKOS_LAMBDA(int Cell, int K) { + return CellMask(Cell, K) != 0 && + Kokkos::isnan(Tracers3D(TracerIdx, Cell, K)); + }; + auto IsOOB = KOKKOS_LAMBDA(int Cell, int K) { + if (CellMask(Cell, K) == 0) { + return false; + } + Real Val = Tracers3D(TracerIdx, Cell, K); + return !Kokkos::isnan(Val) && (Val < MinVal || Val > MaxVal); + }; + return checkEntries(NCells, NVert, IsNaN, IsOOB); +} - parallelReduce( - "CheckTracerNaN", {NCells, NVert}, - KOKKOS_LAMBDA(int Cell, int K, int &Accum) { - if (CellMask(Cell, K) == 0) { - return; - } - Real Val = Tracers3D(TracerIdx, Cell, K); - if (Kokkos::isnan(Val)) { - ++Accum; - } - }, - NaNCount); +//------------------------------------------------------------------------------ +// Helpers: copy a single entry of a device array to the host +static Real getEntry(const Array2DReal &Arr, I4 Row, I4 Col) { + Real Val; + Kokkos::deep_copy(Val, Kokkos::subview(Arr, Row, Col)); + return Val; +} - parallelReduce( - "CheckTracerBounds", {NCells, NVert}, - KOKKOS_LAMBDA(int Cell, int K, int &Accum) { - if (CellMask(Cell, K) == 0) { - return; - } - Real Val = Tracers3D(TracerIdx, Cell, K); - if (!Kokkos::isnan(Val)) { - if (Val < MinVal || Val > MaxVal) { - ++Accum; - } - } - }, - OutOfRangeCount); +static Real getEntry(const Array3DReal &Arr, I4 Idx, I4 Row, I4 Col) { + Real Val; + Kokkos::deep_copy(Val, Kokkos::subview(Arr, Idx, Row, Col)); + return Val; +} + +//------------------------------------------------------------------------------ +// Helper: describe where a cell-layer entry is, for critical messages +static std::string describeLocation(I4 Cell, I4 K) { + std::string Where = "local cell " + std::to_string(Cell); + + Decomp *DefDecomp = Decomp::getDefault(); + if (DefDecomp) { + Where = "cell ID " + std::to_string(DefDecomp->CellIDH(Cell)); + } + + HorzMesh *DefMesh = HorzMesh::getDefault(); + if (DefMesh) { + const Real RadToDeg = 180.0 / Pi; + Where += fmt::format(" (lat {:.4f}, lon {:.4f})", + DefMesh->LatCellH(Cell) * RadToDeg, + DefMesh->LonCellH(Cell) * RadToDeg); + } + + return Where + ", layer " + std::to_string(K); +} - return {NaNCount, OutOfRangeCount}; +//------------------------------------------------------------------------------ +// Helper: log a critical message for each problem found in one field, giving +// where the first one is, record them in Issues if it is given, and add them +// to the totals +static void reportField(const std::string &Field, const std::string &Range, + const FieldCheck &Check, I4 NVert, + const std::function &GetValue, + I4 &TotalNaNs, I4 &TotalOOBs, + std::vector *Issues) { + + if (Check.NaNs > 0) { + I4 Cell = Check.FirstNaN / NVert; + I4 K = Check.FirstNaN % NVert; + LOG_CRITICAL("StateValidation: {} contains {} NaN value(s); the first " + "is at {}", + Field, Check.NaNs, describeLocation(Cell, K)); + if (Issues) { + Issues->push_back( + {Field, true, Check.NaNs, Cell, K, GetValue(Cell, K)}); + } + TotalNaNs += Check.NaNs; + } + if (Check.OOBs > 0) { + I4 Cell = Check.FirstOOB / NVert; + I4 K = Check.FirstOOB % NVert; + Real Val = GetValue(Cell, K); + LOG_CRITICAL("StateValidation: {} has {} value(s) outside valid range " + "{}; the first is {} at {}", + Field, Check.OOBs, Range, Val, describeLocation(Cell, K)); + if (Issues) { + Issues->push_back({Field, false, Check.OOBs, Cell, K, Val}); + } + TotalOOBs += Check.OOBs; + } } //------------------------------------------------------------------------------ @@ -163,35 +299,28 @@ static std::pair checkTracerArray(const Array3DReal &Tracers3D, /// Returns the total count of errors found; does not abort. std::pair checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel) { + const VertCoord *VCoord, I4 TimeLevel, + std::vector *Issues) { I4 TotalNaNs = 0; I4 TotalOOBs = 0; const Array2DReal &CellMask = VCoord->CellMask; + const I4 NCells = State->NCellsOwned; + const I4 NVert = State->NVertLayers; // ------------------------------------------------------------------------- // PseudoThickness: valid range [1e-10, 1000] // ------------------------------------------------------------------------- { Array2DReal PseudoThick = State->getPseudoThickness(TimeLevel); - auto [NaNs, OOB] = - checkArray2D(PseudoThick, State->NCellsOwned, State->NVertLayers, - static_cast(1e-10), static_cast(1000.0), - /*CheckMin=*/true, CellMask); - - if (NaNs > 0) { - LOG_CRITICAL( - "StateValidation: PseudoThickness contains {} NaN value(s)", NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL( - "StateValidation: PseudoThickness has {} value(s) outside " - "valid range [1e-10, 1000]", - OOB); - TotalOOBs += OOB; - } + FieldCheck Check = + checkArray2D(PseudoThick, NCells, NVert, static_cast(1e-10), + static_cast(1000.0), /*CheckMin=*/true, CellMask); + reportField( + "PseudoThickness", "[1e-10, 1000]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(PseudoThick, Cell, K); }, + TotalNaNs, TotalOOBs, Issues); } // ------------------------------------------------------------------------- @@ -199,24 +328,13 @@ std::pair checkOceanState(const OceanState *State, // ------------------------------------------------------------------------- { const Array2DReal &KE = AuxState->KineticAux.KineticEnergyCell; - auto [NaNs, OOB] = - checkArray2D(KE, State->NCellsOwned, State->NVertLayers, - static_cast(0.0), static_cast(10.0), - /*CheckMin=*/true, CellMask); - - if (NaNs > 0) { - LOG_CRITICAL( - "StateValidation: KineticEnergyCell contains {} NaN value(s)", - NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL( - "StateValidation: KineticEnergyCell has {} value(s) outside " - "valid range [0, 10]", - OOB); - TotalOOBs += OOB; - } + FieldCheck Check = + checkArray2D(KE, NCells, NVert, static_cast(0.0), + static_cast(10.0), /*CheckMin=*/true, CellMask); + reportField( + "KineticEnergyCell", "[0, 10]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(KE, Cell, K); }, TotalNaNs, + TotalOOBs, Issues); } // ------------------------------------------------------------------------- @@ -224,21 +342,14 @@ std::pair checkOceanState(const OceanState *State, // ------------------------------------------------------------------------- if (Tracers::IndxTemp != -1) { Array3DReal AllTracers = Tracers::getAll(TimeLevel); - auto [NaNs, OOB] = checkTracerArray( - AllTracers, Tracers::IndxTemp, State->NCellsOwned, State->NVertLayers, - static_cast(-10.0), static_cast(50.0), CellMask); - - if (NaNs > 0) { - LOG_CRITICAL("StateValidation: Temperature contains {} NaN value(s)", - NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL("StateValidation: Temperature has {} value(s) outside " - "valid range [-10, 50]", - OOB); - TotalOOBs += OOB; - } + const I4 Indx = Tracers::IndxTemp; + FieldCheck Check = checkTracerArray(AllTracers, Indx, NCells, NVert, + static_cast(-10.0), + static_cast(50.0), CellMask); + reportField( + "Temperature", "[-10, 50]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(AllTracers, Indx, Cell, K); }, + TotalNaNs, TotalOOBs, Issues); } // ------------------------------------------------------------------------- @@ -246,21 +357,14 @@ std::pair checkOceanState(const OceanState *State, // ------------------------------------------------------------------------- if (Tracers::IndxSalt != -1) { Array3DReal AllTracers = Tracers::getAll(TimeLevel); - auto [NaNs, OOB] = checkTracerArray( - AllTracers, Tracers::IndxSalt, State->NCellsOwned, State->NVertLayers, - static_cast(-2.0), static_cast(60.0), CellMask); - - if (NaNs > 0) { - LOG_CRITICAL("StateValidation: Salinity contains {} NaN value(s)", - NaNs); - TotalNaNs += NaNs; - } - if (OOB > 0) { - LOG_CRITICAL("StateValidation: Salinity has {} value(s) outside " - "valid range [-2, 60]", - OOB); - TotalOOBs += OOB; - } + const I4 Indx = Tracers::IndxSalt; + FieldCheck Check = checkTracerArray(AllTracers, Indx, NCells, NVert, + static_cast(-2.0), + static_cast(60.0), CellMask); + reportField( + "Salinity", "[-2, 60]", Check, NVert, + [&](I4 Cell, I4 K) { return getEntry(AllTracers, Indx, Cell, K); }, + TotalNaNs, TotalOOBs, Issues); } return {TotalNaNs, TotalOOBs}; @@ -271,11 +375,27 @@ std::pair checkOceanState(const OceanState *State, /// Only active cells (where CellMask > 0) are checked. /// Aborts via MPI_Abort on failure. void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel) { + const VertCoord *VCoord, I4 TimeLevel, + const Clock *ModelClock) { - auto [NaNs, OOBs] = checkOceanState(State, AuxState, VCoord, TimeLevel); + auto [LocalNaNs, LocalOOBs] = + checkOceanState(State, AuxState, VCoord, TimeLevel); static const auto AbortConfig = getValidationAbortConfig(); - bool abort = false; + + MachEnv *DefEnv = MachEnv::getDefault(); + Eos *EosInstance = Eos::getInstance(); + + // One reduction gathers everything validation needs from all tasks, so + // that every task makes the same decision about aborting + I8 LocalNNegSalinity = EosInstance ? EosInstance->NNegSalinity : 0; + std::vector Totals = + globalSum(std::vector{LocalNaNs, LocalOOBs, LocalNNegSalinity}, + DefEnv->getComm()); + const I8 NaNs = Totals[0]; + const I8 OOBs = Totals[1]; + const I8 NNegSalinity = Totals[2]; + + bool abort = false; if (NaNs > 0 && AbortConfig.AbortOnNan) { abort = true; @@ -284,11 +404,8 @@ void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, abort = true; } if (abort) { - abortWithMessage( - "StateValidation: Ocean state validation aborting with " + - std::to_string(NaNs) + " NaNs and " + std::to_string(OOBs) + - " OOBs." + "See critical messages above for details."); - } else if ((NaNs + OOBs) > 0) { + abortRun(NaNs, OOBs, AbortConfig.WriteStreamOnAbort, ModelClock); + } else if ((NaNs + OOBs) > 0 && DefEnv->isMasterTask()) { // Only report the flags that are actually suppressing an abort, i.e. // those corresponding to the issues that were detected std::string Flags; @@ -301,14 +418,12 @@ void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, Flags += "AbortOnOOB is false"; } LOG_CRITICAL("StateValidation: Ocean state validation failed with {} " - "NaNs and {} OOBs. {}; continuing after logging.", + "NaNs and {} OOBs on all tasks. {}; continuing after " + "logging.", NaNs, OOBs, Flags); } - Eos *EosInstance = Eos::getInstance(); if (EosInstance) { - const I8 NNegSalinity = globalSum(EosInstance->NNegSalinity, - MachEnv::getDefault()->getComm()); if (NNegSalinity > 0) { LOG_WARN("StateValidation: {} negative salinity value(s) seen by " "the EOS during this time step (summed over all stages " diff --git a/components/omega/src/ocn/StateValidation.h b/components/omega/src/ocn/StateValidation.h index 9e22f8447166..4cb7de27f6bb 100644 --- a/components/omega/src/ocn/StateValidation.h +++ b/components/omega/src/ocn/StateValidation.h @@ -11,23 +11,41 @@ /// aborting. Suitable for testing. /// - validateOceanState: calls checkOceanState and conditionally aborts /// via MPI_Abort based on runtime YAML options AbortOnNan and AbortOnOOB -/// (under Omega::State::Validation). When both flags are false a -/// CRITICAL log message is emitted and the run continues. +/// (under Omega::State::Validation), first writing the stream named by +/// WriteStreamOnAbort. When both flags are false a CRITICAL log message +/// is emitted and the run continues. // //===----------------------------------------------------------------------===// #include "AuxiliaryState.h" #include "OceanState.h" +#include "TimeMgr.h" #include "Tracers.h" #include "VertCoord.h" +#include +#include + namespace OMEGA { +/// One kind of problem (NaN or out of range) found in one field on this task, +/// with the local cell, layer and value of the first entry that has it +struct ValidationIssue { + std::string Field; ///< Name of the field, e.g. Temperature + bool IsNaN; ///< NaN (true) or out of the valid range (false) + I4 Count; ///< Number of entries on this task with the problem + I4 Cell; ///< Local index of the cell of the first entry + I4 Layer; ///< Vertical layer of the first entry + Real Value; ///< Value of the first entry +}; + /// Check ocean state fields for NaN values and out-of-bounds conditions /// without aborting, returning counts of each error type found. /// -/// Only active ocean cells (where CellMask > 0) are checked. Critical log -/// messages are emitted for each type of error found. +/// Only active ocean cells (where CellMask > 0) are checked. A critical log +/// message is emitted for each type of error found, giving the number of +/// entries and the global cell ID, latitude, longitude and layer of the +/// first. If Issues is given, the same problems are appended to it. /// /// The following fields are validated: /// - PseudoThickness : [1e-10, 1000] (from OceanState) @@ -40,17 +58,20 @@ namespace OMEGA { /// \param[in] AuxState Auxiliary state containing KineticEnergyCell /// \param[in] VCoord Vertical coordinate containing the CellMask /// \param[in] TimeLevel Time level index to validate (typically 0 = current) +/// \param[out] Issues Optional list to which the problems found are added /// \return std::pair {NaN count, out-of-bounds count}; both 0 if valid -std::pair checkOceanState(const OceanState *State, - const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel); +std::pair +checkOceanState(const OceanState *State, const AuxiliaryState *AuxState, + const VertCoord *VCoord, I4 TimeLevel, + std::vector *Issues = nullptr); /// Check ocean state fields for NaN values and out-of-bounds conditions. /// /// Calls checkOceanState and conditionally aborts via MPI_Abort based on the /// runtime YAML options AbortOnNan and AbortOnOOB (under -/// Omega::State::Validation). Only active ocean cells (where CellMask > 0) -/// are checked. +/// Omega::State::Validation) and the counts summed over all tasks. It must +/// therefore be called on every task. Only active ocean cells (where +/// CellMask > 0) are checked. /// /// The following fields are validated: /// - PseudoThickness : [1e-10, 1000] (from OceanState) @@ -60,17 +81,23 @@ std::pair checkOceanState(const OceanState *State, /// - Salinity tracer : [-2, 60] (from Tracers) /// /// If AbortOnNan is true and NaN values are detected, or AbortOnOOB is true -/// and out-of-bounds values are detected, a critical error is logged with a -/// stack backtrace and the run is aborted via MPI_Abort. When both flags are -/// false but errors are found, a CRITICAL log message is emitted and execution +/// and out-of-bounds values are detected, the stream named by the YAML option +/// WriteStreamOnAbort (default History) is written on every task if it is +/// defined and ModelClock is given, a critical error is logged with a stack +/// backtrace and the run is aborted via MPI_Abort. When both flags are false +/// but errors are found, a CRITICAL log message is emitted and execution /// continues. /// /// \param[in] State Ocean state to validate /// \param[in] AuxState Auxiliary state containing KineticEnergyCell /// \param[in] VCoord Vertical coordinate containing the CellMask /// \param[in] TimeLevel Time level index to validate (typically 0 = current) +/// \param[in] ModelClock Optional clock for the time stamp of the stream +/// written before an abort; without it, no stream is +/// written void validateOceanState(const OceanState *State, const AuxiliaryState *AuxState, - const VertCoord *VCoord, I4 TimeLevel); + const VertCoord *VCoord, I4 TimeLevel, + const Clock *ModelClock = nullptr); } // namespace OMEGA diff --git a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp index 7667019dea97..6af4ecf3cc71 100644 --- a/components/omega/src/timeStepping/ForwardBackwardStepper.cpp +++ b/components/omega/src/timeStepping/ForwardBackwardStepper.cpp @@ -105,7 +105,8 @@ void ForwardBackwardStepper::doStep( applyImplicitVerticalMixing(State, TracerCurLevel, ThickCurLevel, VelCurLevel, "ForwardBackward"); - validateOceanState(State, AuxState, VertCoord::getDefault(), 0); + validateOceanState(State, AuxState, VertCoord::getDefault(), 0, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); diff --git a/components/omega/src/timeStepping/RungeKutta2Stepper.cpp b/components/omega/src/timeStepping/RungeKutta2Stepper.cpp index 99410e07ea71..0304112849a1 100644 --- a/components/omega/src/timeStepping/RungeKutta2Stepper.cpp +++ b/components/omega/src/timeStepping/RungeKutta2Stepper.cpp @@ -83,7 +83,8 @@ void RungeKutta2Stepper::doStep(OceanState *State, // model state applyImplicitVerticalMixing(State, CurLevel, CurLevel, CurLevel, "RK2"); - validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel); + validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); diff --git a/components/omega/src/timeStepping/RungeKutta4Stepper.cpp b/components/omega/src/timeStepping/RungeKutta4Stepper.cpp index d72e49b60065..d6bdb4b22cc9 100644 --- a/components/omega/src/timeStepping/RungeKutta4Stepper.cpp +++ b/components/omega/src/timeStepping/RungeKutta4Stepper.cpp @@ -146,7 +146,8 @@ void RungeKutta4Stepper::doStep(OceanState *State, // model state applyImplicitVerticalMixing(State, CurLevel, CurLevel, CurLevel, "RK4"); - validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel); + validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); diff --git a/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp b/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp index 892295b7e469..90cebc82fe89 100644 --- a/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp +++ b/components/omega/src/timeStepping/SplitExplicitRK2Stepper.cpp @@ -640,7 +640,8 @@ void SplitExplicitRK2Stepper::doStep(OceanState *State, Pacer::stop("SE-RK2:vMixHaloExch", 3); } - validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel); + validateOceanState(State, AuxState, VertCoord::getDefault(), CurLevel, + StepClock.get()); // Advance the clock and update the simulation time StepClock->advance(); diff --git a/components/omega/test/infra/LoggingTest.cpp b/components/omega/test/infra/LoggingTest.cpp index 1409959ffb4b..5467a401cf02 100644 --- a/components/omega/test/infra/LoggingTest.cpp +++ b/components/omega/test/infra/LoggingTest.cpp @@ -238,6 +238,23 @@ int testKokkosDataTypes(bool LogEnabled) { return RetVal; } +// Tasks that do not log still let critical messages through, and only those +int testOtherTasksKeepCritical(bool LogEnabled) { + + if (LogEnabled) + return 0; + + auto Logger = spdlog::default_logger(); + bool Pass = Logger->name() == "critical_only" && + Logger->should_log(spdlog::level::critical) && + !Logger->should_log(spdlog::level::err); + + std::cout << "Critical only on other tasks: " << (Pass ? "PASS" : "FAIL") + << std::endl; + + return Pass ? 0 : 1; +} + int main(int argc, char **argv) { int RetVal = 0; @@ -270,6 +287,7 @@ int main(int argc, char **argv) { RetVal += testDefaultLogLevel(LogEnabled); RetVal += testKokkosDataTypes(LogEnabled); + RetVal += testOtherTasksKeepCritical(LogEnabled); } catch (const std::exception &Ex) { std::cout << Ex.what() << ": FAIL" << std::endl; diff --git a/components/omega/test/ocn/StateValidationTest.cpp b/components/omega/test/ocn/StateValidationTest.cpp index 47dccbdecd43..00f9da182a77 100644 --- a/components/omega/test/ocn/StateValidationTest.cpp +++ b/components/omega/test/ocn/StateValidationTest.cpp @@ -8,6 +8,8 @@ /// (NaN and out-of-bounds values in each checked field are detected) of /// validateOceanState / checkOceanState. Checked fields are: /// - PseudoThickness, KineticEnergyCell, Temperature, Salinity. +/// Also checks that a single bad value is reported at the cell and layer +/// where it was put. // //===-----------------------------------------------------------------------===/ @@ -37,6 +39,8 @@ #include #include +#include +#include using namespace OMEGA; @@ -308,6 +312,86 @@ static int testOOBSalinity(OceanState *State, AuxiliaryState *AuxState, return expectErrors("OOB in Salinity", State, AuxState, VCoord); } +// --- Location of the first bad entry --- + +// Find the last owned cell that is active in its second layer (or its only +// one), so that a bad value put there is away from the first index checked +static std::pair pickActiveEntry(OceanState *State, VertCoord *VCoord) { + auto CellMaskH = createHostMirrorCopy(VCoord->CellMask); + const I4 K = State->NVertLayers > 1 ? 1 : 0; + for (I4 Cell = State->NCellsOwned - 1; Cell >= 0; --Cell) { + if (CellMaskH(Cell, K) != 0) { + return {Cell, K}; + } + } + return {-1, -1}; +} + +// Check that a single bad value is reported as the only issue, at the cell +// and layer where it was put +static int expectIssueAt(const char *TestName, OceanState *State, + AuxiliaryState *AuxState, VertCoord *VCoord, + const std::string &Field, bool IsNaN, I4 Cell, I4 K) { + std::vector Issues; + checkOceanState(State, AuxState, VCoord, 0, &Issues); + + bool Pass = Issues.size() == 1 && Issues[0].Field == Field && + Issues[0].IsNaN == IsNaN && Issues[0].Count == 1 && + Issues[0].Cell == Cell && Issues[0].Layer == K; + if (!Pass) { + LOG_ERROR("StateValidationTest: {} - expected one {} issue in {} at " + "cell {}, layer {}; got {} issue(s)", + TestName, IsNaN ? "NaN" : "OOB", Field, Cell, K, Issues.size()); + for (const auto &Issue : Issues) { + LOG_ERROR("StateValidationTest: {} {} x{} at cell {}, layer {}", + Issue.Field, Issue.IsNaN ? "NaN" : "OOB", Issue.Count, + Issue.Cell, Issue.Layer); + } + return 1; + } + LOG_INFO("StateValidationTest: {} PASS", TestName); + return 0; +} + +static int testLocationOfNaN(OceanState *State, AuxiliaryState *AuxState, + VertCoord *VCoord) { + if (Tracers::IndxTemp == -1) + return 0; // tracer not active; skip + restoreValidState(State, AuxState); + auto [Cell, K] = pickActiveEntry(State, VCoord); + if (Cell < 0) + return 0; // no active owned cell on this task; nothing to check + const Real NaN = std::numeric_limits::quiet_NaN(); + Array2DReal TempArr = Tracers::getByIndex(0, Tracers::IndxTemp); + Kokkos::deep_copy(Kokkos::subview(TempArr, Cell, K), NaN); + return expectIssueAt("Location of NaN in Temperature", State, AuxState, + VCoord, "Temperature", /*IsNaN=*/true, Cell, K); +} + +static int testLocationOfOOB(OceanState *State, AuxiliaryState *AuxState, + VertCoord *VCoord) { + if (Tracers::IndxSalt == -1) + return 0; // tracer not active; skip + restoreValidState(State, AuxState); + auto [Cell, K] = pickActiveEntry(State, VCoord); + if (Cell < 0) + return 0; // no active owned cell on this task; nothing to check + Array2DReal SaltArr = Tracers::getByIndex(0, Tracers::IndxSalt); + // 99 g/kg is above the valid max of 60 g/kg + Kokkos::deep_copy(Kokkos::subview(SaltArr, Cell, K), 99.0); + + std::vector Issues; + checkOceanState(State, AuxState, VCoord, 0, &Issues); + if (Issues.size() == 1 && Issues[0].Value != 99.0) { + LOG_ERROR("StateValidationTest: OOB in Salinity reported value {}, " + "expected 99", + Issues[0].Value); + return 1; + } + return expectIssueAt("Location of OOB in Salinity", State, AuxState, VCoord, + "Salinity", /*IsNaN=*/false, Cell, K); +} + //------------------------------------------------------------------------------ // Run all state validation tests @@ -363,6 +447,10 @@ int testStateValidation() { Err += testNaNSalinity(DefState, DefAuxState, VCoord); Err += testOOBSalinity(DefState, DefAuxState, VCoord); + // Location of the first bad entry + Err += testLocationOfNaN(DefState, DefAuxState, VCoord); + Err += testLocationOfOOB(DefState, DefAuxState, VCoord); + AuxiliaryState::clear(); return Err; }