diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 0000000..087ffc5 --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "/Users/yohann/.local/bin/graphify hook-check" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 06c96e8..f294880 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ test-install .vscode/ *.err *.out +/graphify-out +/.idea diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..74d8c8e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,62 @@ +# Repository Guidelines + +Any changes should be also reflected in CLAUDE.md + +## Project Structure & Module Organization + +This is the MATLAB high-level interface to IMAS Access Layer. MATLAB entry points live in `matlab/`; their MEX implementations and shared C helpers are in `src/`. XSLT generators at the repository root and in `common/` generate IDS-specific sources from the IMAS Data Dictionary—edit generators rather than generated output where applicable. `tests/` contains MATLAB unit, integration, and performance tests; `examples/` contains runnable MATLAB examples. Sphinx documentation is in `doc/`, and CI and cluster scripts are in `ci/` and `.github/workflows/`. + +## Build, Test, and Development Commands + +MATLAB and a C/C++ toolchain are required. CMake fetches IMAS Core and the Data Dictionary unless configured for a local development layout. + +```bash +cmake -B build --preset=https -DAL_BACKEND_HDF5=ON -DAL_TESTS=ON -DAL_EXAMPLES=ON +cmake --build build --parallel +ctest --test-dir build --output-on-failure +cmake --install build +``` + +The first command configures an HTTPS-based dependency checkout and enables the HDF5 test backend. The test command runs the CTest registrations, which invoke MATLAB's `runtests('imas_unit_tests')`. For documentation, use `ci/build_docs.sh` or configure with `-DAL_HLI_DOCS=ON -DAL_DOCS_ONLY=ON` and build the resulting tree. Do not commit build directories or generated local artifacts. + +For multiversion DD conversion, install IMAS-Multiversion-DD-Loader and configure with `-DAL_USE_MULTIVERSION_SHIM=ON` plus its prefix in `CMAKE_PREFIX_PATH` (or `imas-mvdd-loader_DIR`). The MATLAB MEX targets then link the shim while retaining IMAS-Core for headers and runtime loading. CTest sets `IMAS_MVDD_HLI_DD_VERSION` and `IMAS_CORE_LIBRARY`; set them explicitly when launching MATLAB outside CTest. `AL_CORE_RUNTIME_LIBRARY` overrides CTest's Core path. See `doc/doc_common/building_installing.rst` and `docs/SHIM_INTEGRATION_CONTRACT.md`. + +## Coding Style & Naming Conventions + +Follow the surrounding file's formatting: MATLAB uses two-space indentation, `function` blocks, and lower-case underscore-separated API names such as `ids_get_slice`; C uses four-space indentation and lower-case underscore-separated filenames such as `imas_mex_utils.c`. Keep MATLAB help comments immediately above public functions and use established `IMAS::` error identifiers in MEX code. Prefer focused changes; preserve the generator/source relationship when changing IDS behavior. + +## Testing Guidelines + +Add coverage in the relevant `matlab.unittest.TestCase` class, using descriptive `test...` method names. Exercise both HDF5 and MDSplus only when the change is backend-specific; HDF5 is sufficient for the standard local path. Run the narrow MATLAB suite while iterating when the built libraries are on the path, then run CTest before opening a PR. Update examples or docs when public MATLAB behavior changes. + +`imas_utils_unit_tests` is registered as its own backend-independent CTest suite; it runs even when neither HDF5 nor MDSplus is configured. + +## Skipped-path policy + +The shared MEX utility library owns the process-global skipped-path record and +the refusal-band (`-1000..-1099`) tolerance chokepoint. Keep the record +observable only through `imas_get_skipped_paths` and +`imas_get_skipped_path_count`. Generated traversal routes only leaf data seams +and array-of-structures opens through the chokepoint: a tolerated read leaf +receives its default value, a tolerated read array open becomes empty, a +tolerated write leaf is omitted, a tolerated write array open skips its +subtree, and a tolerated delete leaf is omitted. Root read, write, and delete +generators clear the record on entry; an `ids_put` clears once before its +internal delete phase so both delete and write skips remain in one record. + +## Commit & Pull Request Guidelines + +Recent history uses brief, imperative summaries (for example, `Fix warnings in windows`) and conventional prefixes for automation such as `ci:`. Use one focused change per commit; include the affected component when helpful. Start work from the latest `develop` branch, as required by `CONTRIBUTING.md`. PRs should describe the behavior change, link the agreed issue, list validation performed, and include MATLAB output or screenshots when they clarify a user-facing change. + +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a279971 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,62 @@ +# Repository Guidelines + +Any changes should be also reflected in AGENTS.md + +## Project Structure & Module Organization + +This is the MATLAB high-level interface to IMAS Access Layer. MATLAB entry points live in `matlab/`; their MEX implementations and shared C helpers are in `src/`. XSLT generators at the repository root and in `common/` generate IDS-specific sources from the IMAS Data Dictionary—edit generators rather than generated output where applicable. `tests/` contains MATLAB unit, integration, and performance tests; `examples/` contains runnable MATLAB examples. Sphinx documentation is in `doc/`, and CI and cluster scripts are in `ci/` and `.github/workflows/`. + +## Build, Test, and Development Commands + +MATLAB and a C/C++ toolchain are required. CMake fetches IMAS Core and the Data Dictionary unless configured for a local development layout. + +```bash +cmake -B build --preset=https -DAL_BACKEND_HDF5=ON -DAL_TESTS=ON -DAL_EXAMPLES=ON +cmake --build build --parallel +ctest --test-dir build --output-on-failure +cmake --install build +``` + +The first command configures an HTTPS-based dependency checkout and enables the HDF5 test backend. The test command runs the CTest registrations, which invoke MATLAB's `runtests('imas_unit_tests')`. For documentation, use `ci/build_docs.sh` or configure with `-DAL_HLI_DOCS=ON -DAL_DOCS_ONLY=ON` and build the resulting tree. Do not commit build directories or generated local artifacts. + +For multiversion DD conversion, install IMAS-Multiversion-DD-Loader and configure with `-DAL_USE_MULTIVERSION_SHIM=ON` plus its prefix in `CMAKE_PREFIX_PATH` (or `imas-mvdd-loader_DIR`). The MATLAB MEX targets then link the shim while retaining IMAS-Core for headers and runtime loading. CTest sets `IMAS_MVDD_HLI_DD_VERSION` and `IMAS_CORE_LIBRARY`; set them explicitly when launching MATLAB outside CTest. `AL_CORE_RUNTIME_LIBRARY` overrides CTest's Core path. See `doc/doc_common/building_installing.rst` and `docs/SHIM_INTEGRATION_CONTRACT.md`. + +## Coding Style & Naming Conventions + +Follow the surrounding file's formatting: MATLAB uses two-space indentation, `function` blocks, and lower-case underscore-separated API names such as `ids_get_slice`; C uses four-space indentation and lower-case underscore-separated filenames such as `imas_mex_utils.c`. Keep MATLAB help comments immediately above public functions and use established `IMAS::` error identifiers in MEX code. Prefer focused changes; preserve the generator/source relationship when changing IDS behavior. + +## Testing Guidelines + +Add coverage in the relevant `matlab.unittest.TestCase` class, using descriptive `test...` method names. Exercise both HDF5 and MDSplus only when the change is backend-specific; HDF5 is sufficient for the standard local path. Run the narrow MATLAB suite while iterating when the built libraries are on the path, then run CTest before opening a PR. Update examples or docs when public MATLAB behavior changes. + +`imas_utils_unit_tests` is registered as its own backend-independent CTest suite; it runs even when neither HDF5 nor MDSplus is configured. + +## Skipped-path policy + +The shared MEX utility library owns the process-global skipped-path record and +the refusal-band (`-1000..-1099`) tolerance chokepoint. Keep the record +observable only through `imas_get_skipped_paths` and +`imas_get_skipped_path_count`. Generated traversal routes only leaf data seams +and array-of-structures opens through the chokepoint: a tolerated read leaf +receives its default value, a tolerated read array open becomes empty, a +tolerated write leaf is omitted, a tolerated write array open skips its +subtree, and a tolerated delete leaf is omitted. Root read, write, and delete +generators clear the record on entry; an `ids_put` clears once before its +internal delete phase so both delete and write skips remain in one record. + +## Commit & Pull Request Guidelines + +Recent history uses brief, imperative summaries (for example, `Fix warnings in windows`) and conventional prefixes for automation such as `ci:`. Use one focused change per commit; include the affected component when helpful. Start work from the latest `develop` branch, as required by `CONTRIBUTING.md`. PRs should describe the behavior change, link the agreed issue, list validation performed, and include MATLAB output or screenshots when they clarify a user-facing change. + +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/CMakeLists.txt b/CMakeLists.txt index 2eecac0..d6340e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,7 @@ endif() # Use full path since CMAKE_MODULE_PATH hasn't been configured yet include(${CMAKE_CURRENT_SOURCE_DIR}/common/cmake/ALLocalPaths.cmake) option( AL_DOWNLOAD_DEPENDENCIES "Automatically download assets from the AL git repository" ON ) +option( AL_USE_MULTIVERSION_SHIM "Route MATLAB Access Layer calls through IMAS-Multiversion-DD-Loader" OFF ) set( AL_CORE_GIT_REPOSITORY "https://github.com/iterorganization/IMAS-Core.git" CACHE STRING "Git repository of AL-core" ) set( AL_CORE_VERSION "main" CACHE STRING "Git commit/tag/branch of AL-core" ) @@ -110,6 +111,31 @@ include( ALBuildDataDictionary ) # AL core, plugins and documentation include( ALCore ) +# Keep Core available for its headers and as the library the shim opens at run +# time, but do not put it on a MEX link line: its mirrored C symbols would +# bypass conversion. Both al-mex and each MEX target use this one selection. +set( AL_MATLAB_CORE_TARGET al ) +set( AL_MATLAB_SHIM_TEST_ENVIRONMENT "" ) +if( AL_USE_MULTIVERSION_SHIM AND NOT AL_DOCS_ONLY ) + find_package( imas-mvdd-loader REQUIRED CONFIG ) + set( AL_MATLAB_CORE_TARGET imas-mvdd-loader::imas-mvdd-loader ) + set( AL_MATLAB_SHIM_TEST_ENVIRONMENT "IMAS_MVDD_HLI_DD_VERSION=${DD_VERSION}" ) + + set( AL_CORE_RUNTIME_LIBRARY "" CACHE FILEPATH + "AL core shared library opened by the shim at run time; empty uses the Core built here" ) + if( AL_CORE_RUNTIME_LIBRARY ) + if( NOT EXISTS "${AL_CORE_RUNTIME_LIBRARY}" ) + message( FATAL_ERROR "AL_CORE_RUNTIME_LIBRARY does not exist: ${AL_CORE_RUNTIME_LIBRARY}" ) + endif() + list( APPEND AL_MATLAB_SHIM_TEST_ENVIRONMENT + "IMAS_CORE_LIBRARY=${AL_CORE_RUNTIME_LIBRARY}" ) + elseif( AL_DOWNLOAD_DEPENDENCIES OR AL_DEVELOPMENT_LAYOUT ) + list( APPEND AL_MATLAB_SHIM_TEST_ENVIRONMENT + "IMAS_CORE_LIBRARY=$" ) + endif() + message( STATUS "MATLAB Access Layer calls use the multiversion shim (${imas-mvdd-loader_DIR})" ) +endif() + # Stop processing when only building documentation if( AL_DOCS_ONLY ) return() @@ -136,7 +162,7 @@ if(WIN32) # This should be built as a shared library (not a MEX file) with name libal-mex.so/.dll # It's a utility library that MEX files link to, so use add_library instead of matlab_add_mex add_library( al-mex SHARED ${MEX_UTIL_SOURCES} ) - target_link_libraries( al-mex PUBLIC al Matlab::mex Matlab::mx ) + target_link_libraries( al-mex PUBLIC ${AL_MATLAB_CORE_TARGET} Matlab::mex Matlab::mx ) target_include_directories( al-mex PRIVATE src ) target_include_directories( al-mex PUBLIC ${Matlab_INCLUDE_DIRS} ) @@ -154,9 +180,18 @@ if(WIN32) set_target_properties(al-mex PROPERTIES LINKER_LANGUAGE C) else() # This should be built as a shared library with name libal-mex.so - matlab_add_mex( NAME al-mex SRC ${MEX_UTIL_SOURCES} SHARED LINK_TO al ) + matlab_add_mex( NAME al-mex SRC ${MEX_UTIL_SOURCES} SHARED LINK_TO ${AL_MATLAB_CORE_TARGET} ) target_include_directories( al-mex PRIVATE src ) endif() +if( AL_USE_MULTIVERSION_SHIM ) + # The shim publishes its own header, not the IMAS-Core C ABI headers used by + # these sources. Export Core's include paths without its link interface. + target_include_directories( al-mex PUBLIC + "$>" ) + if( AL_DOWNLOAD_DEPENDENCIES OR AL_DEVELOPMENT_LAYOUT ) + add_dependencies( al-mex al ) + endif() +endif() set_target_properties( al-mex PROPERTIES PREFIX "lib" @@ -235,7 +270,7 @@ if(WIN32) macro( ADD_IMAS_MEX NAME SOURCES ) # Let matlab_add_mex handle the suffix automatically # Or explicitly set .mexw64 if needed: - matlab_add_mex( NAME "mex-${NAME}" SRC ${SOURCES} LINK_TO al al-mex OUTPUT_NAME "${NAME}" ) + matlab_add_mex( NAME "mex-${NAME}" SRC ${SOURCES} LINK_TO ${AL_MATLAB_CORE_TARGET} al-mex OUTPUT_NAME "${NAME}" ) if(MATLAB_VERSION VERSION_GREATER_EQUAL "9.4") # R2018a+ set_target_properties("mex-${NAME}" PROPERTIES SUFFIX ".mexw64") endif() @@ -253,7 +288,7 @@ if(WIN32) else() # Macro to add a mex target macro( ADD_IMAS_MEX NAME SOURCES ) - matlab_add_mex( NAME "mex-${NAME}" SRC ${SOURCES} LINK_TO al al-mex OUTPUT_NAME "${NAME}" ) + matlab_add_mex( NAME "mex-${NAME}" SRC ${SOURCES} LINK_TO ${AL_MATLAB_CORE_TARGET} al-mex OUTPUT_NAME "${NAME}" ) target_include_directories( "mex-${NAME}" PRIVATE src ) # Ensure all source-generation targets finish before this MEX target compiles. # Without this, parallel builds (cmake --build -j N) can race: the MEX target @@ -328,11 +363,14 @@ set( TARGETS imas_deserialize imas_get_backendID imas_get_mex_params + imas_get_skipped_path_count + imas_get_skipped_paths imas_open imas_open_env imas_open_env_backend imas_serialize imas_set_mex_params + imas_test_inject_skipped_path imas_build_uri_from_legacy_parameters imas_list_all_occurrences imas_partial_get diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..93ad600 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,38 @@ +# IMAS-MATLAB + +The MATLAB High Level Interface of the IMAS Access Layer. Most IDS traversal +sources are generated at build time by XSLT stylesheets, so update the +stylesheets rather than generated sources. + +Shim-side vocabulary (shim, seam, occurrence, DD-version stamp, stored DD +version, loss log, rule, fidelity verdict) is owned by +`IMAS-Multiversion-DD-Loader/CONTEXT.md` and is used here unchanged. The terms +below are the ones this repository owns. + +## Language + +**Refusal band**: +The status codes `-1000..-1099`, reserved for a shim that declines to serve a +path. Disjoint from IMAS-Core's own `-1..-4`. +_Avoid_: error range, MVDD codes + +**Tolerated refusal**: +A refusal that the generated traversal absorbs at a single field and carries +on past, instead of ending the operation. Only a leaf data seam and an +array-of-structures open may tolerate one. +_Avoid_: ignored error, swallowed refusal, soft failure + +**Skipped path**: +One field the traversal left unset because of a tolerated refusal, recorded +with the path, the status code and the refusal message. +_Avoid_: missing field, failed path, skipped node + +**Partial read**: +The outcome of a read that completed after at least one tolerated refusal. A +positive status, because the operation succeeded and the IDS is usable. +_Avoid_: failed read, incomplete read, degraded read + +**Partial put**: +The write-side counterpart of a partial read. Covers both a refused write and +a refused delete, which reach the caller through the same outcome. +_Avoid_: failed put, partial write diff --git a/common/cmake/ALExampleUtilities.cmake b/common/cmake/ALExampleUtilities.cmake index b465397..d7d62d1 100644 --- a/common/cmake/ALExampleUtilities.cmake +++ b/common/cmake/ALExampleUtilities.cmake @@ -29,7 +29,8 @@ function( set_al_example_properties TEST DISABLED USE_PLUGINS EXTRA_ENVIRONMENT else() set( P_ENV ${EXAMPLE_ENVIRONMENT_WITHOUT_PLUGINS} ) endif() - set_tests_properties( ${TEST} PROPERTIES ENVIRONMENT "${P_ENV};${EXTRA_ENVIRONMENT}" ) + set_tests_properties( ${TEST} PROPERTIES + ENVIRONMENT "${P_ENV};${EXTRA_ENVIRONMENT};${AL_MATLAB_SHIM_TEST_ENVIRONMENT}" ) # Set fixtures: put/put_slice must run before get/get_slice string( TOLOWER ${TEST} TEST_LOWER ) @@ -52,4 +53,3 @@ function( error_on_missing_tests SOURCE_EXTENSION TESTS ) endif() endforeach() endfunction() - diff --git a/delete.xsl b/delete.xsl index d4e6e08..d79b56f 100644 --- a/delete.xsl +++ b/delete.xsl @@ -32,6 +32,7 @@ fieldPath = ""; status = al_delete_data(ctx, fieldPath); + if (status.code < 0) tolerateRefusal(&status, IMAS_MEX_DELETE_OPERATION, fieldPath); /* Error handling */ if (status.code < 0) { addIdsPathInfoToErrMsg("\n ... in field ",0); diff --git a/doc/api_ids.rst b/doc/api_ids.rst index 545d8a7..b1c7a18 100644 --- a/doc/api_ids.rst +++ b/doc/api_ids.rst @@ -25,3 +25,7 @@ IDS API .. mat:autofunction:: ids_isdefined() .. mat:autofunction:: ids_validate() + +.. mat:autofunction:: imas_get_skipped_paths() + +.. mat:autofunction:: imas_get_skipped_path_count() diff --git a/doc/doc_common/building_installing.rst b/doc/doc_common/building_installing.rst index b46099a..0b3114b 100644 --- a/doc/doc_common/building_installing.rst +++ b/doc/doc_common/building_installing.rst @@ -217,6 +217,31 @@ MATLAB-specific configuration options The following options are specific to the MATLAB High Level Interface: +- ``AL_USE_MULTIVERSION_SHIM``: Link the MATLAB MEX libraries to the + IMAS-Multiversion-DD-Loader C ABI instead of linking IMAS-Core directly + (default: ``OFF``). Install the shim first and provide its install prefix + with ``CMAKE_PREFIX_PATH`` or ``imas-mvdd-loader_DIR``. Core is still built + for headers and as the library the shim opens at run time. + + .. code-block:: bash + + cmake -B build --preset=https -DAL_USE_MULTIVERSION_SHIM=ON \ + -DCMAKE_PREFIX_PATH=/path/to/IMAS-Multiversion-DD-Loader/install \ + -DAL_BACKEND_HDF5=ON + cmake --build build --parallel + ctest --test-dir build --output-on-failure + + The CTest registrations set ``IMAS_MVDD_HLI_DD_VERSION`` to the DD version + used to generate this HLI and ``IMAS_CORE_LIBRARY`` to the Core library + built in the same tree. When using an installed Core, or running MATLAB + outside CTest, set ``IMAS_CORE_LIBRARY`` to its shared-library path and + ``IMAS_MVDD_HLI_DD_VERSION`` to this HLI's generated DD version before + launching MATLAB, and make the shim's installed library discoverable by + the platform dynamic loader. ``AL_CORE_RUNTIME_LIBRARY`` can name a + different Core shared library for CTest. Conversion is currently available only for the + equilibrium 3.39.0/4.1.1 pair; see + ``docs/SHIM_INTEGRATION_CONTRACT.md`` for the exact behavior. + - ``AL_CREATE_TOOLBOX``: Automatically create MATLAB toolbox package (``.mltbx``) during installation - **Default:** ``OFF`` @@ -323,4 +348,3 @@ Troubleshooting **Problem:** ``Target Boost::log already has an imported location`` This problem is known to occur with the ``2020b`` toolchain on SDCC. Add the CMake configuration option ``-D Boost_NO_BOOST_CMAKE=ON`` to work around the problem. - diff --git a/docs/SHIM_INTEGRATION_CONTRACT.md b/docs/SHIM_INTEGRATION_CONTRACT.md new file mode 100644 index 0000000..65b5673 --- /dev/null +++ b/docs/SHIM_INTEGRATION_CONTRACT.md @@ -0,0 +1,555 @@ +# What an HLI should expect from the shim: read, write, delete under a DD version mismatch + +Audience: someone writing HLI-side integration tests against this shim (including +round-trip tests) who needs to know precisely what varies by scenario, what is +guaranteed, and what a round trip cannot prove. This is a synthesis of +`docs/adr/0002`, `0005`, `0007`, `0008`, `0009`, `0012`, `0016`–`0021`, `0023`, +`0024` and the +current `src/conversion/seam_policy.rs` / `src/lib.rs`. Where this document and +an ADR disagree, the ADR (or the code) is authoritative — this file only +collects and orders what they already say, with one exception: **§8 is a +frozen surface, not a summary.** The refusal reason strings listed there are +what downstream tests are invited to assert on, so a change to one of those +strings in `src/` is a change to this document's contract and must land in +the same commit. + +## 1. Vocabulary you need before reading the matrices + +- **HLI DD version**: the version latched process-wide via + `imas_mvdd_set_hli_dd_version` (first-use-wins) or `IMAS_MVDD_HLI_DD_VERSION`. + One process gets exactly one value for its whole life; a later conflicting + report is refused, an identical repeat is accepted (ADR 0005). If neither is + ever set, the shim does zero version discovery on any seam and is pure + passthrough — this is the "unset" row in every matrix below. +- **Stamp**: `ids_properties/version_put/data_dictionary` on an IDS occurrence. + Read once, at occurrence-open time, to discover the *stored* DD version. +- **Artifact**: one hand-authored conversion map. Exactly one exists today — + equilibrium, 3.39.0 ⇄ 4.1.1 (`docs/3.39.0--4.1.1.xml`). Its rule mix: 4 + `identical`, 23 `left_only`, 13 `right_only`, 5 `renamed`, 3 `moved`, 13 + `merged`, 1 `split`, 1 `retyped`. +- **A registered conversion context** ("root record") exists only when the + stamp names a stored version that (a) differs from the HLI version and (b) + has an embedded artifact for that (IDS, stored, HLI) triple. This is the + only condition under which reads/writes/deletes on that occurrence are + translated or logged at all. +- **Fidelity**: `Exact`, `PotentiallyLossy`, `Lossy`, `Unmappable` (ADR 0008). + Never surfaced through `al_status_t` on a successful call — only through the + loss log (§7). +- **Candidate plan**: where one HLI-side path can mean several stored paths + (a `merged` or `split` rule), each stored path is a candidate with a + declared precedence. Read tries them in order; write touches only + precedence 1; delete fans out over all of them. This asymmetry is + deliberate (ADR 0017) — see §6. + +## 2. Preconditions your test harness must set up deliberately + +1. **Pick the HLI DD version once per process**, before any open. You cannot + test "what happens on a version conflict" and "what happens on a clean + mismatch" in the same process — the second `imas_mvdd_set_hli_dd_version` + call with a *different* value is refused, not applied (ADR 0005). Two + scenarios need two processes. +2. **To exercise real conversion, use IDS `equilibrium` with stored/HLI + versions `3.39.0` and `4.1.1` (either direction).** Any other IDS, or any + other version pair, has no embedded artifact: the stamp is read, a + mismatch is *detected*, but `known_artifacts::lookup` returns `None`, no + root context is registered, and every subsequent read/write/delete on that + occurrence forwards completely unconverted — indistinguishable at the ABI + from a matching-version occurrence. **A version mismatch alone does not + imply conversion.** Don't write a test that asserts conversion happened + just because the stamps differ; assert it only for the one covered pair. +3. **The stamp is read once, at occurrence-open time** (`al_begin_global_action`, + `al_begin_slice_action`, `al_begin_timerange_action`), not per-field. A test + that wants a mismatched-occurrence scenario must have written that stamp + before the open your test measures — which in practice means either a + fixture file with the stamp pre-set, or a prior process run under the + *other* DD version that performed an ordinary `put`. +4. **`al_begin_global_action`'s `datapath` translates only on the occurrence's + second-or-later open in the same process.** The first open of any given + occurrence forwards `datapath` unchanged, because discovery (which needs + the open to have already happened) hasn't cached anything for it yet + (ADR 0002). If your test cares about `datapath` translation specifically, + open, close, and reopen the same occurrence. +5. **A write/delete-mode open still discovers the stamp correctly** as of + ADR 0020: for any `rwmode != READ_OP`, the shim opens its own throwaway + `READ_OP` probe context first to read the stamp, then proceeds with your + open. You do not need to open `READ_OP` yourself to get correct + registration on a `WRITE_OP` open — but know this costs one extra + open/read/close per occurrence-open, invisible to you except as latency, + and it requires opening a second context on the same pulse (proven on + HDF5; unproven on the other five backends). + +## 3. Occurrence-open outcomes, by stamp state + +This happens before any `al_read_data`/`al_write_data`/`al_delete_data` call — +get this table wrong in your fixture and every downstream expectation is +wrong too. + +| Stamp as read at open | Registration | What happens to the open call itself | +|---|---|---| +| Absent (no `version_put/data_dictionary` at all) | Nothing registered; occurrence presumed to match HLI (ADR 0007) | Open succeeds, forwarded exactly as issued | +| Present, valid, equal to HLI version | Nothing registered | Open succeeds, forwarded exactly as issued | +| Present, valid, differs from HLI version, **no embedded artifact** for that (IDS, stored, HLI) triple | Nothing registered, but the occurrence-cache remembers the mismatch (affects only a later `datapath` translation, §2.4) | Open succeeds; every later data seam on this occurrence forwards unconverted | +| Present, valid, differs from HLI version, **artifact exists** | Root conversion context registered | Open succeeds; later data seams on this occurrence convert | +| Present but **malformed** (fails the grammar in ADR 0009: not a bare `MAJOR.MINOR.PATCH` from the known chain, and not exactly `MAJOR.MINOR.PATCH-N-gHASH`) | Nothing registered | **The open itself refuses**, and the context IMAS-Core just opened is closed again by the shim before returning to you. You get a refusal from the *open* seam, not from a later read/write. Test this by opening, not by reading. | + +A malformed stamp is not treated as absent — absence means "no stamp field"; +a present-but-invalid value is treated as unsafe metadata and refused loudly +(ADR 0009). Don't conflate these two in a fixture. + +## 4. Read (`al_read_data` / `al_plugin_read_data`) + +Applies only when a root context is registered (§3, last two rows). Otherwise +every read is a plain forward: `field`/`timebase` unchanged, no value +transformation, `code == 0` and null data mean not-found exactly as IMAS-Core +reports it, and nothing is logged. + +When a root *is* registered: + +- `field` and `timebase` are each resolved independently against the + artifact. Either one resolving to a refusal (e.g. the one `retyped` rule — + always refused, unconditionally, even where the artifact marks it `exact`, + because the shim cannot reshape an int array into an identifier struct) or + to "no source" (a `left_only`/`right_only` case with nothing on the other + side) ends the call immediately: the *other* argument is reported at + `Fidelity::Exact` regardless of its own resolution, because the loop never + gets far enough to evaluate it. +- Otherwise, every combination of a field candidate × a timebase candidate is + tried in declared precedence order until one field candidate returns data. + A `renamed`/`moved`/identity resolution is a one-candidate list; a `merged` + or `split` resolution is an ordered multi-candidate list. **The first + candidate that returns data wins**, and that candidate's own declared + fidelity (which may be `PotentiallyLossy`, e.g. for a `merged` field) is + what gets logged — not automatically `Lossy` just because it wasn't the + first candidate tried. +- A value transformation (today: only a COCOS sign flip, only on + `DOUBLE_DATA`, rank `0..=MAXDIM`) is validated against the buffer's declared + shape *before* any candidate is tried, and applied *in place* on the buffer + IMAS-Core wrote, only after data is actually found. An unsupported shape + (wrong datatype, or rank outside `0..=MAXDIM`) refuses before the reader is + ever called, at `Fidelity::Unmappable`. +- The three-way outcome IMAS-Core actually reports — failure (`code != 0`), + not-found (`code == 0`, null data), success — is what drives whether the + loop tries the next candidate (only on not-found) or returns immediately + (on failure or on data). A backend failure on one candidate is *not* + swallowed to try the next candidate; only "not found" continues the loop. +- If every candidate reports not-found, the overall result is not-found + (`code == 0`, no data), and the reported fidelity is the resolution's own + translated fidelity even though nothing was read. + +**What to assert in a read test:** the returned value and `al_status_t.code`; +the fidelity and DD path recorded in the loss log for that read (§7) — never +assume anything about fidelity from `al_status_t` alone, since a lossy read +still returns `code == 0`. + +## 5. Write (`al_write_data` / `al_plugin_write_data`) + +Same gate: only a registered root context converts anything; otherwise plain +forward. When registered, `field` and `timebase` resolve **independently**, +and either one failing to produce a safe stored spelling refuses the *entire* +write before IMAS-Core is ever called (`field`'s refusal is reported ahead of +`timebase`'s, if both would refuse). The refusal is always reported before +any data reaches Core — the shim's own rule is "never return `code == 0` for +data it did not store", with exactly one exception (the sentinel case +below). + +| Situation | Outcome | +|---|---| +| `field` resolves through a non-primary source (`precedence != 1` in a `merged`/`split` rule) | **Refused**, before Core is called. Only the precedence-1 HLI-side spelling may write. | +| `field` resolves to a stored slot that doesn't exist (a `right_only` rule — in this artifact, 13 paths all under `time_slice`) | **Refused**, before Core is called. This is the one case reachable through ordinary `put_slice` use, not just a full `put` — see the torn-write note below. | +| `field` resolves via `merged`/`split` to several stored candidates, one at precedence 1 | Only the precedence-1 candidate is written. Every other candidate's **complete stored-DD path** (not the caller's own path) is appended to the loss log as `PotentiallyLossy`, and **only after** Core accepts the precedence-1 write. `Lossy` never appears as a write-path verdict in this artifact — it is asserted unreachable, not merely unused. | +| The value transformation for `field` cannot be inverted (would need `ValueTransformation::inverse()` to succeed; today every COCOS flip in this artifact does invert) | **Refused**, before Core is called. | +| The buffer's declared shape can't carry the transformation (wrong datatype, or rank outside `0..=MAXDIM`) | **Refused**, before Core is called, before the source buffer is even copied. | +| The source is a **rank-0 scalar exactly equal to IMAS-Core's own sentinel** (`EMPTY_DOUBLE = -9.0E40` for `DOUBLE_DATA`) | Forwarded **unchanged**, no transformation applied, **no loss log entry**, `code == 0`. This is not a refusal and not a lossy write — IMAS-Core itself would have skipped storing it regardless (`data_has_non_zero_shape` gate), so nothing was lost. A sentinel-valued *element inside a non-scalar array* is **not** caught by this — it is transformed like any other value, which is a known, accepted gap (not guarded, for cost reasons). | +| A write to the DD-version stamp itself (`ids_properties/version_put/data_dictionary`), under a mismatch | **Refused** always. The shim never rewrites the stamp. In practice this only fires on a full `put` into an already-mismatched, already-stamped occurrence (a "migration" write) — `put_slice` never touches this field at all (its generated code has an empty body for it), so the ordinary append workflow never hits this refusal. | +| A delete that would remove the stamp while leaving data behind | **Refused** (see §6) — this is a delete rule, listed here only because it protects the same invariant. | +| Everything else | Written, exact, no loss entry. | + +**Torn-write hazard — required reading before writing a `put_slice` +integration test.** IMAS-Fortran's generated `put`/`put_slice` routines have +no rollback. A refusal partway through a slice (most likely: one of the 13 +`right_only` fields under `time_slice` that a DD4 caller fills and a DD3 +occurrence has no slot for) leaves **everything already written earlier in +that same call on disk**, and the `time_slice` container one element longer +regardless (the caller's own `al_begin_arraystruct_action` widened it before +any leaf write ran, and Core commits that shape at end-action time no matter +what happens after). Against an unmodified upstream IMAS-Fortran, **do not +expect a clean all-or-nothing failure from a refused `put_slice`** — expect a +torn slice plus a refusal. Only a patched IMAS-Fortran (tracked upstream as +`yohannmarguier/IMAS-Fortran#61`, not yet merged as of this writing) tolerates +the refusal field-by-field the way the read path already does via +`al_get_policy`. This is a documented limitation of the shim, not a defect to +chase — see README.md's "Scope and limitations". + +The generated MATLAB `put`/`put_slice` routines do tolerate a refusal +field-by-field, at a leaf write and at an array-of-structures open: they warn, +record each skipped HLI-DD path for `imas_get_skipped_paths`, and continue the +traversal. That changes which fields survive, not the torn-write hazard — there +is no rollback in MATLAB either, so a caller must treat a successful return +with skipped paths as a partial put rather than an all-or-nothing write. + +**A refusal here can also crash the process, not just fail the call**, in one +specific structural case unrelated to your own writes: IMAS-Core's own +internal plugin machinery (`AccessLayerPluginManager::write_field` and +friends) calls back into these same seams and `assert()`s `code == 0` on the +result. For the shipped equilibrium artifact this is unreachable (nothing +claims a rule under `ids_properties/plugins/**`), but it is a structural +exposure the first artifact touching that subtree must reopen. + +## 6. Delete (`al_delete_data`) + +Same registration gate as read/write. When registered: + +| Situation | Outcome | +|---|---| +| `path` resolves through a non-primary source | **Refused**, same rule as write. | +| `path` resolves to one stored path (identity, `renamed`, `moved`) | One `al_delete_data` call to Core with that stored spelling. | +| `path` resolves to several candidates (`merged`/`split`) | **Every candidate is deleted**, unconditionally, with **no presence probe** beforehand. This is the opposite answer from write's precedence-1-only rule, and it is deliberate (ADR 0017): a write asserts a value it must not fabricate into an assumed-equivalent slot, but a delete asserts an absence, and leaving a stale candidate behind would let the read path's own fallback serve it as live data after a delete the caller was told succeeded. | +| One or more candidates fail | **All candidates are still attempted** (no early exit); the **first** nonzero status is what's returned to the caller, after every candidate has been tried. An absent candidate is indistinguishable from a genuine backend failure at the ABI (`al_delete_data` has no not-found outcome), so a missing candidate can *look like* a failure even when the delete "worked" as well as it could. | +| A delete refuses | One `UNMAPPABLE` `DELETE` loss records the caller's complete HLI-DD path before the refusal reaches the HLI. The generated MATLAB traversal warns, records the skipped path, and continues at this leaf seam. | +| A candidate plan completes | One `POTENTIALLY_LOSSY` `DELETE` loss records each stored candidate after all candidates have been attempted, including when the first failure is returned. | +| `path` names a structure (not a leaf) whose subtree contains an **escaping rule** — a rule at or under `path` with at least one stored-side target outside the resolved stored subtree | **Refused**, before any candidate is touched. A leaf delete is always trivial (never refuses on this basis). On the shipped artifact this refuses `time_slice/boundary_separatrix` from a DD3 HLI and `time_slice/boundary` from a DD4 HLI, but allows `time_slice`, `time_slice/constraints`, and every leaf. | +| `path` is empty | Forwards **unchanged**, unconditionally — this is IMAS-Core's own "delete the whole DATAOBJECT" contract. It is the *only* legitimate way to migrate a mismatched occurrence: afterwards the occurrence is unstamped, so ADR 0007 makes the next open treat it as matching the HLI, and it can be written fresh. It is also the sole exception to "any delete touching the stamp refuses" — because it removes the data too, nothing is left to misread. | + +**The delete seam records destructive fidelity explicitly** (ADR 0024). A +refusal names the caller's HLI-DD path as `UNMAPPABLE`; a fan-out names every +visited stored candidate as `POTENTIALLY_LOSSY`, in both the root-context log +and its append-only file. A one-path delete remains exact and records nothing. + +**Real-backend caveat you must not paper over in a round trip.** Real +IMAS-Core's HDF5 `deleteData` ignores its `path` argument completely and +deletes the whole IDS pulse file plus its master-file link. So on the only +backend that actually implements delete, a candidate-plan fan-out's first +candidate destroys the entire occurrence and every later candidate in the +same fan-out simply finds nothing left. **A test cannot observe *per-path* +deletion against real HDF5** — it can only observe that the occurrence is +gone. This is tracked as a known, accepted gap (issue #139; stated in +README.md's "Scope and limitations"), not something a passing test should +paper over by asserting per-candidate effects that the backend doesn't +actually provide. Test the fan-out call sequence (e.g., via the recording +stub, which does let you observe each candidate individually) separately +from the on-disk consequence (which real HDF5 collapses to one). + +## 7. Cross-cutting: how the caller learns any of this + +- **`al_status_t.code == 0` always means success, full stop** — including a + lossy or partially-served read, and including the unwritten-candidate case + on write. Never infer fidelity from `code`. +- **A shim-originated refusal returns `code == IMAS_MVDD_CONVERSION_ERROR` + (`-1000`)**, distinct from any IMAS-Core code (upstream only uses `-1` + through `-4`). The message is `"IMAS-MVDD: {reason}; DD path: {path}; HLI DD + version: {v}; stored DD version: {v}"`, built by one formatter and + truncated (versions dropped first, then the path cut **from the left** with + a leading `...` so the leaf name — the identifying part — survives) to fit + the ABI's fixed 256-byte message buffer. A refusal raised before any + context exists (bad HLI version, malformed stamp) has no path or version + pair and uses a shorter `"IMAS-MVDD: {reason}"` form instead. **Every + `{reason}` the shipped shim can produce is listed verbatim in §8** — that + is the only part of the message worth asserting on, since `code` alone is + `-1000` for all of them and the rest of the message can be truncated away. +- **Loss/fidelity never travels through `al_status_t`.** It travels through a + per-root-context log, drained via three shim-owned exports: + `imas_mvdd_context_loss_count(ctx, *count)`, + `imas_mvdd_context_loss_at(ctx, index, path_buf, buf_len, *verdict)` (verdict + is one of `IMAS_MVDD_FIDELITY_POTENTIALLY_LOSSY` / `_LOSSY` / `_UNMAPPABLE`; + exact-fidelity entries are never logged at all), and + `imas_mvdd_context_loss_operation_at(ctx, index, *operation)` (` + IMAS_MVDD_LOSS_OPERATION_READ`, `_WRITE`, or `_DELETE`). Querying **any** context under + one IDS occurrence (a root or one of its arraystruct children) returns the + **whole root's** log — there is no per-child scoping. An untracked + context — including any occurrence with no registered root — reports a + count of `0`, not a refusal. +- **A refused read, write or delete is logged too, at `Unmappable`**, in addition to + being returned through `al_status_t`. This is deliberate redundancy, not a + bug: it means `Unmappable` in the log conflates "this was refused" with + "this candidate genuinely doesn't exist and came back not-found" — a test + reading the log should not assume every `Unmappable` entry corresponds to a + visible failure at the call site. +- **The in-memory log dies with its root context at `al_end_action`.** A + patched HLI must drain it through the exports before closing, or lose that + view of it — but the exports are no longer the *only* channel. ADR 0023 + deliberately reverses ADR 0012 decision 7 on this point: no shipped HLI is + patched to call the exports, so a second, file-based channel now carries + the same entries without requiring any HLI change at all (below). An + unmodified HLI — one that never calls the exports and only ever sees + refusals through `al_status_t` — still gets this second channel for free. +- **The loss log file** (ADR 0023) is process-local, append-only, and + tab-separated: `uri`, `ids`, `stored-dd`, `hli-dd`, `operation`, `fidelity`, + `path`, preceded by a `#`-comment preamble (format version, write + timestamp, PID, HLI DD version). The shim creates it lazily, only on the + first loss any seam produces, named + `imas-mvdd-loss--.txt` (a `-N` suffix disambiguates a + same-second collision) in the current working directory by default. Set + `IMAS_MVDD_LOSS_LOG_DIR` to an existing directory to redirect it, or to an + empty value to disable the file entirely. Every non-exact read, write and + delete loss reaches it — not delete alone, though §6 mentions it there too + — carrying the same three fidelity verdicts and the same READ/WRITE/DELETE + operation tags as the exports. +- **The file survives exactly the failure the in-memory log cannot.** If a + root context ends while an operation is still in flight, its in-memory log + is already gone and drops the entry — but the file's own process-wide + written-key set has no context lifetime, so the entry is retained there + regardless. That disagreement between the two channels is deliberate, not + a bug to reconcile in a test. Conversely, the file is exact-once, not + cumulative: the key is the complete rendered line, so the identical loss + encountered twice within one process is written once, while two + occurrences of the same IDS — or two separate processes — each still get + their own line. A filesystem write failure is reported once to stderr and + otherwise silently disables the file for the rest of the process; it never + changes `al_status_t`, and it never touches the in-memory log. +- **The `uri` column is the caller-supplied URI verbatim, unredacted.** A + site whose URIs can carry credentials should set `IMAS_MVDD_LOSS_LOG_DIR` + to an empty value, or point it at an appropriately access-controlled + directory, before running against real data. +- Write's loss entries name the **stored**-DD spelling of each unwritten + candidate (where else a stale value might be found); read's (and any + refusal's) entries name the **HLI**-DD spelling of the argument in + question (what was actually asked for). Don't expect the same kind of path + string in both cases. + +## 8. Refusal reason strings: a frozen, named surface + +Everything above tells you *whether* a call refuses. This section tells you +*what it says*, verbatim, so an HLI-side test can assert on the reason rather +than on `code == -1000` alone — which every refusal in the shim shares and +which therefore proves almost nothing about *which* rule fired. + +**This list is the contract.** The strings below are the ones a test may +depend on. They are literals in `src/`, but treating them as an +implementation detail is exactly the failure mode this section exists to +prevent: a reworded string turns a precise assertion into a vacuous one +without failing a single test in this repository. Anyone changing one of +these strings must update this table in the same commit, and should treat the +change as breaking for downstream HLI suites. New reasons may be *added* as +new artifacts land; existing ones do not get reworded silently. + +### 8.1 The envelope around every reason + +Two shapes, one formatter (`src/lib.rs`, reached through +`src/interpose/refusal.rs`): + +| Raised | Message | +|---|---| +| By a seam holding a live conversion record (every read, write, delete and arraystruct refusal) | `IMAS-MVDD: {reason}; DD path: {path}; HLI DD version: {hli}; stored DD version: {stored}` | +| Before any context exists (bad HLI DD version, malformed stamp, loss-export argument errors) | `IMAS-MVDD: {reason}` | + +`{path}` is the DD path *in the caller's own spelling*, anchor-joined — not +the stored spelling. Where a seam has a record but no resolved path to name +(the two arraystruct arguments), it falls back to the context's own resolved +path, and to the literal `(no path argument)` when there is no path at either +place. + +**Do not assert on the whole message.** It is truncated to fit +`MAX_ERR_MSG_LEN` (256) in a fixed order: the two versions are dropped first, +then the path is cut **from the left** and marked with a leading `...`. A +deep DD path plus a long reason can therefore legitimately produce a message +with no version pair in it. Assert `code == IMAS_MVDD_CONVERSION_ERROR` plus +a **substring** match on the reason; assert the full string only where you +control the path length. (This repository's own C suite asserts exact strings +via `CHECK_REFUSAL_MESSAGE` precisely because it controls both.) + +`{...}` below marks runtime substitution; everything outside braces is +literal, including punctuation. Note the em-dash (`—`, U+2014) in the +version-conflict message. + +### 8.2 Path-resolution reasons + +Raised by the artifact's own rules, shared across seams. These are the ones a +conversion test most wants to name. + +| Reason string | Raised by | +|---|---| +| `this path's container changed shape and cannot be served` | any seam, on the `retyped` rule — unconditional, even where the rule declares itself `exact` | +| `this path's unit was redefined and cannot be converted` | any seam, on a unit-redefinition rule | +| `this path has no safe conversion between DD versions` | any seam, on a declared-`unmappable` rule. **Unreachable from the shipped artifact** (ADR 0011) and asserted to be so — a test that hits it means a new artifact made it reachable | +| `this path is unclaimed by the conversion map` | write, delete | +| `this path has no stored source` | write, delete | +| `this path is a non-primary source and cannot write a shared stored slot` | write | +| `this path is a non-primary source and cannot delete a shared stored slot` | delete | +| `this candidate plan has no precedence-1 source for a write` | write, where a `merged`/`split` plan has no precedence-1 slot at all | +| `the DD-version stamp is immutable under a version mismatch` | write, on `ids_properties/version_put/data_dictionary` (§5) | +| `this delete would remove the DD-version stamp while stored data remains` | delete, on the stamp or any ancestor of it | +| `this timebase needs a value transformation, which al_write_data cannot apply` | write, `timebase` argument only | +| `this path needs a value transformation that cannot be inverted for a write` | write, `field` argument | +| `this subtree delete would leave data at a stored path outside the requested subtree` | delete, on an escaping-rule subtree (§6) | + +### 8.3 Context-open and arraystruct reasons + +Raised by `al_begin_arraystruct_action` / its plugin twin, and by the shared +anchor resolution beneath every relative path argument. + +| Reason string | Raised when | +|---|---| +| `this path needs a value transformation, which only a data read can apply` | a context open resolves to a rule carrying a value transformation — an open has no buffer to transform | +| `this path is served by several stored candidates, and only a data read can try them in turn` | a context open resolves to a `merged`/`split` candidate plan | +| `arraystruct path has no stored source` | the AOS `path` argument resolves to nothing on the stored side | +| `arraystruct timebase has no stored source` | same, for `timebase` | +| `arraystruct path is unclaimed by the conversion map` | the AOS `path` argument is claimed by no rule | +| `arraystruct timebase is unclaimed by the conversion map` | same, for `timebase` | +| `translated path does not lie beneath this context's stored anchor` | a relative argument translated to a path outside its own context | +| `translated field contains an interior NUL byte` | the translated spelling cannot be formed as a C string | +| `context anchor has no stored-DD conversion rule` | the enclosing context's own anchor is unclaimed | +| `context anchor has no stored source` | the enclosing context's anchor has nothing on the stored side | + +The two `arraystruct ...` families are built from a `{label}` substitution +over `path` and `timebase`; those four are the only spellings the shipped +seams produce. + +### 8.4 Value-transform execution reasons + +Raised when a buffer's declared shape cannot carry the transformation the +rule asks for. Read and write share the first three; the last two are +write-only. + +| Reason string | Raised when | +|---|---| +| `value-transform execution requires DOUBLE_DATA and a rank no greater than MAXDIM` | the datatype is not `DOUBLE_DATA`, or `dim` is outside `0..=7` | +| `value-transform execution needs array dimensions` | `dim > 0` with a null `size` | +| `value-transform execution received an invalid array shape` | a negative extent, or extents whose product overflows | +| `value-transform execution needs a data buffer` | write: a non-scalar write with a null `data` | +| `this value transformation was not inverted for the write direction` | write: a defensive assertion that a read-direction transformation never reaches the write path. Not reachable through the shipped resolver; a test hitting it has found a real bug | + +### 8.5 Version-latch and stamp reasons (pre-context, short envelope) + +| Reason string | Raised by | +|---|---| +| `HLI DD version must not be null` | `imas_mvdd_set_hli_dd_version(NULL)` | +| `HLI DD version must be valid UTF-8` | a non-UTF-8 version string | +| `conflicting HLI DD version: this process already latched to '{existing}' and cannot also serve '{parsed}' — one process cannot host two HLIs built against different DD versions` | a second setter call with a different version (§2.1) | +| `cannot set HLI DD version to '{parsed}': this process already latched to unset, after an earlier open found no setter call and no valid IMAS_MVDD_HLI_DD_VERSION` | a setter call after an open already latched the process to "no conversion" | +| `cannot set HLI DD version to '{parsed}': this process already latched to an invalid IMAS_MVDD_HLI_DD_VERSION value at an earlier open ({reason})` | a setter call after an open latched an invalid environment value; `{reason}` is one of §8.6 | +| `malformed DD-version stamp at 'ids_properties/version_put/data_dictionary'` | the occurrence-open refusal of §3's last row | + +### 8.6 DD version grammar reasons + +Produced by version parsing (ADR 0009) and delivered through the short +envelope, either from `imas_mvdd_set_hli_dd_version` or nested inside the +last message of §8.5. `{input}`/`{raw}`/`{whole}` is the offending string. + +| Reason string | +|---| +| `DD version '{input}' must not contain whitespace` | +| `'{input}' is not a known DD release` | +| `'{input}' is not MAJOR.MINOR.PATCH` | +| `'{input}' has extra '.'-separated components` | +| `'{whole}' has a non-canonical version component '{component}'` | +| `'{whole}' has an out-of-range version component '{component}'` | +| `'{raw}' has an unknown base release '{base}'` | +| `'{raw}' is missing the '-N-gHASH' development suffix` | +| `'{raw}' has a non-canonical development commit distance` | +| `'{raw}' has a zero commit distance, which is not a development build` | +| `'{raw}' is missing the 'g' hash prefix` | +| `'{raw}' hash must be 7 to 64 characters, got {n}` | +| `'{raw}' hash must be lowercase hexadecimal` | + +### 8.7 Loss-export argument reasons + +Argument errors from the three `imas_mvdd_context_loss_*` exports (§7). These +are programming errors in the caller, not conversion outcomes — an untracked +context is *not* one of them (it reports a count of `0`). + +| Reason string | +|---| +| `imas_mvdd_context_loss_count requires a non-null count output` | +| `imas_mvdd_context_loss_at requires a non-null verdict output` | +| `imas_mvdd_context_loss_at requires a non-null path buffer` | +| `imas_mvdd_context_loss_at index must not be negative` | +| `imas_mvdd_context_loss_at buffer length must not be negative` | +| `imas_mvdd_context_loss_at index is out of range for this context` | +| `imas_mvdd_context_loss_at buffer is too small for this path` | +| `imas_mvdd_context_loss_operation_at requires a non-null operation output` | +| `imas_mvdd_context_loss_operation_at index must not be negative` | +| `imas_mvdd_context_loss_operation_at index is out of range for this context` | + +### 8.8 One status the shim emits that is *not* a refusal + +If IMAS-Core itself cannot be resolved at runtime — `libal` not found, or an +ABI major-version mismatch — every mirrored seam returns `code == -1` (not +`-1000`) with a message shaped `override with $IMAS_CORE_LIBRARY if this is +wrong; {detail}`, where `{detail}` is the platform's own `dlerror()` text or +the version comparison. It carries **no** `IMAS-MVDD:` prefix and predates +the reserved `-1000..=-1099` block. A test that sees `-1` has a broken +environment, not a conversion outcome; don't fold it into refusal handling. + +## 9. What a round trip can prove, and what it structurally cannot + +A write-then-read round trip through the shim is a **consistency check**, not +a correctness proof, for exactly one class of case: any value transformation +(today, COCOS sign flips). Write flips HLI→stored and read flips +stored→HLI, so **the caller's own value comes back whether or not the shim's +sign convention is actually right, and whether or not the value on disk is +even in the stored convention at all**. A round trip that only asserts "I +read back what I wrote" gives zero evidence about: + +- whether the value was actually stored under the *stored* DD path (as + opposed to the caller's own path, forwarded by accident), +- whether the sign on disk matches the *stored* COCOS convention rather than + the HLI's, +- whether the DD-version stamp still reads the *stored* version after a + `put_slice` (nothing round-trippable touches this), +- whether a precedence-2 candidate was correctly left alone rather than also + written. + +Proving any of those requires reading the on-disk file **natively** (outside +the shim) and checking the stored path, the raw sign, and the stamp directly +— what this project calls its "native/on-disk oracle" (see +`tests/real_core/write_delete_oracle_test.c`). Keep native-oracle assertions +and shim-round-trip assertions in separate tests, and don't delete the native +ones as "redundant" with a passing round trip — they are proving different +things. + +Other things a round trip (or any black-box HLI test) cannot observe, listed +so you don't chase a false negative: + +- **Per-candidate delete effects against real HDF5** (§6) — the backend + collapses a fan-out to one whole-occurrence deletion regardless of how many + candidates the shim submits. +- **A clean failure from a refused `put_slice`** against an unmodified HLI + (§5) — expect a torn slice, not an atomic rollback. +- **Any conversion effect on an occurrence whose (IDS, stored, HLI) triple has + no embedded artifact** — a genuine version mismatch with no artifact is + byte-for-byte indistinguishable from no mismatch at all, at every seam. +- **`datapath` translation on a fresh occurrence's first open** (§2.4) — it + only ever fires from the second open onward. +- **Anything about `timebase` conversion beyond identity** — in the shipped + artifact `time` is untouched by any rule, so `timebase` resolution is + exercised only at `Fidelity::Exact`, identity-forward, in every scenario + above. The write path explicitly documents this as unproven territory for a + future artifact, not a guarantee that a non-identity `timebase` write is + safe. +- **Merged-rule loss beyond what's logged.** The shim never performs an + auxiliary read to check whether a `merged` field's untried candidates + *actually* held different data — `PotentiallyLossy` is a statement about + ambiguity in the rule, never a verified fact about the specific occurrence. + +## 10. A minimal scenario checklist + +For each operation (read, write, delete), a reasonably complete integration +suite exercises, at minimum: + +1. HLI version unset → pure passthrough, no seam does version discovery at all. +2. Occurrence stamp absent → forwards, presumed match, nothing logged. +3. Occurrence stamp present and equal to HLI version → forwards, nothing logged. +4. Occurrence stamp present, differs from HLI version, **no artifact for that + pair** → forwards unconverted, nothing logged (must not be conflated with + case 3, even though the observable behavior is identical). +5. Occurrence stamp present, differs, **artifact present** (equilibrium + 3.39.0⇄4.1.1) → per §4/§5/§6 above, per rule kind actually exercised + (`identical`, `renamed`, `moved`, `merged`, `split`, `retyped`, + `left_only`, `right_only`). +6. Occurrence stamp present but malformed → the **open** call refuses; no + data seam is ever reached. +7. For write specifically: a non-primary-source write, a no-stored-slot + write, a `merged`-rule write with an unwritten candidate (check the loss + log's stored-path entry), an unset-sentinel scalar write (check *no* loss + entry and unchanged forwarding), and a stamp-write attempt (must refuse). +8. For delete specifically: a single-candidate delete, a multi-candidate + delete with an injected mid-fan-out failure (check every candidate was + still attempted), a trivial subtree delete, an escaping-rule subtree + delete (must refuse), and the empty-path whole-DATAOBJECT delete. +9. Loss-log lifecycle: query counts/entries before `al_end_action`, confirm + the log is unreachable (reports `0`) once queried through a context ID + that no longer exists. +10. Loss log file: trigger a non-exact operation, confirm a + `imas-mvdd-loss-*.txt` file appears under `IMAS_MVDD_LOSS_LOG_DIR` (or the + working directory) with a matching line, and confirm an empty + `IMAS_MVDD_LOSS_LOG_DIR` suppresses it. diff --git a/get_single.xsl b/get_single.xsl index 9324e07..7989093 100644 --- a/get_single.xsl +++ b/get_single.xsl @@ -53,6 +53,9 @@ aosArraySize = 0; } + if (status.code < 0 && tolerateRefusalWithConsequence(&status, IMAS_MEX_READ_OPERATION, field.fieldPath, "array of structures was set to empty")) { + aosArraySize = 0; + } if (status.code >= 0) status = begin_dataTree_array_read("", aosArraySize); for (int i=0; i<aosArraySize; i++) { if (status.code >= 0) status = iterate_dataTree_array(i); @@ -110,6 +113,8 @@ status = mxArray_default_value(field.datatype, field.dim, &data); } + if (status.code < 0 && tolerateRefusal(&status, IMAS_MEX_READ_OPERATION, field.fieldPath)) + status = mxArray_default_value(field.datatype, field.dim, &data); if (status.code >= 0) put_data_in_dataTree("", data); /* Error handling */ if (status.code < 0) { diff --git a/ids_delete.xsl b/ids_delete.xsl index 1ac47d3..63540ae 100644 --- a/ids_delete.xsl +++ b/ids_delete.xsl @@ -122,8 +122,9 @@ void mexFunction(int nlhs, mxArray *plhs[], mexErrMsgIdAndTxt("IMAS:ids_delete:unknown_ids", "Unknown IDS name: %s", name); - /* Clean-up previous errors */ + /* Clean-up previous errors and the previous operation's skipped paths */ resetErrMsgIdAndTxt(); + resetSkippedPaths(); /* Call function */ al_status_t err = ids_delete(idx, IDSpath); plhs[0] = mxCreateNumericMatrix(1, 1, mxINT32_CLASS, mxREAL); diff --git a/ids_get.xsl b/ids_get.xsl index 43288ea..73239ad 100644 --- a/ids_get.xsl +++ b/ids_get.xsl @@ -126,8 +126,9 @@ void mexFunction(int nlhs, mxArray *plhs[], /* free now as name uses the same memory */ free(IDSpathcopy); - /* Clean-up previous errors */ + /* Clean-up previous errors and the previous operation's skipped paths */ resetErrMsgIdAndTxt(); + resetSkippedPaths(); /* Call function */ al_status_t err = ids_get(idx, IDSpath, &plhs[0]); if (err.code < 0 ) diff --git a/ids_get_sample.xsl b/ids_get_sample.xsl index d24a250..43cde4c 100644 --- a/ids_get_sample.xsl +++ b/ids_get_sample.xsl @@ -185,8 +185,9 @@ void mexFunction(int nlhs, mxArray *plhs[], /* free now as name uses the same memory */ free(IDSpathcopy); - /* Clean-up previous errors */ + /* Clean-up previous errors and the previous operation's skipped paths */ resetErrMsgIdAndTxt(); + resetSkippedPaths(); /* Call function */ al_status_t err = ids_get_sample(idx, IDSpath, &plhs[0], tmin, tmax, dtime, csize, interpmode); if (err.code < 0 ) diff --git a/ids_get_slice.xsl b/ids_get_slice.xsl index 1f0beaf..cc098cf 100644 --- a/ids_get_slice.xsl +++ b/ids_get_slice.xsl @@ -147,8 +147,9 @@ void mexFunction(int nlhs, mxArray *plhs[], /* free now as name uses the same memory */ free(IDSpathcopy); - /* Clean-up previous errors */ + /* Clean-up previous errors and the previous operation's skipped paths */ resetErrMsgIdAndTxt(); + resetSkippedPaths(); /* Call function */ al_status_t err = ids_get_slice(idx, IDSpath, inTime, interpolMode, &plhs[0]); if (err.code < 0) diff --git a/ids_put.xsl b/ids_put.xsl index a02e500..b65ba02 100644 --- a/ids_put.xsl +++ b/ids_put.xsl @@ -136,8 +136,9 @@ void mexFunction(int nlhs, mxArray *plhs[], /* free now as name uses the same memory */ free(IDSpathcopy); - /* Clean-up previous errors */ + /* Clean-up previous errors and the previous operation's skipped paths */ resetErrMsgIdAndTxt(); + resetSkippedPaths(); /* Call function */ al_status_t err = ids_put(idx, IDSpath, prhs[nrhs-1]); if (err.code < 0) diff --git a/ids_put_slice.xsl b/ids_put_slice.xsl index 82d6f8f..8dcfadd 100644 --- a/ids_put_slice.xsl +++ b/ids_put_slice.xsl @@ -136,8 +136,9 @@ void mexFunction(int nlhs, mxArray *plhs[], /* free now as name uses the same memory */ free(IDSpathcopy); - /* Clean-up previous errors */ + /* Clean-up previous errors and the previous operation's skipped paths */ resetErrMsgIdAndTxt(); + resetSkippedPaths(); /* Call function */ al_status_t err = ids_put_slice(idx, IDSpath, prhs[nrhs-1]); if (err.code < 0) diff --git a/matlab/ids_delete.m b/matlab/ids_delete.m index 60bae06..129c1bd 100644 --- a/matlab/ids_delete.m +++ b/matlab/ids_delete.m @@ -2,6 +2,11 @@ % % Delete the IDS from the open database. % +% When a multiversion Data Dictionary shim refuses an individual field, the +% delete can complete partially. MATLAB warns for every refused field; inspect +% imas_get_skipped_paths or imas_get_skipped_path_count immediately afterwards. +% There is no rollback: data deleted before a refusal remains deleted. +% % expIdx : index to database, returned by imas_open/imas_create. % IDSpath : the IDS/occurrence to delete. % occurence: diff --git a/matlab/ids_get.m b/matlab/ids_get.m index 0a6dbad..524079b 100644 --- a/matlab/ids_get.m +++ b/matlab/ids_get.m @@ -7,6 +7,10 @@ % % Empty fields within the IDS in the Data Entry are returned with the % default values indicated in :ref:`Default values`. +% A read can be partial when the multiversion shim refuses an individual +% field: the returned IDS retains its normal shape and uses that field's +% default value. Inspect imas_get_skipped_paths or +% imas_get_skipped_path_count after the call for the refused paths. % % Args: % expIdx: Data entry context created with diff --git a/matlab/ids_get_sample.m b/matlab/ids_get_sample.m index caf3936..8f79dab 100644 --- a/matlab/ids_get_sample.m +++ b/matlab/ids_get_sample.m @@ -22,6 +22,10 @@ % 'ids_properties.homogeneous_time = 1'. % % Empty fields within the IDS in the Data Entry are returned with default values. +% A read can be partial when the multiversion shim refuses an individual +% field: the returned IDS retains its normal shape and uses that field's +% default value. Inspect imas_get_skipped_paths or +% imas_get_skipped_path_count after the call for the refused paths. % % Args: % expIdx: Data entry context created with diff --git a/matlab/ids_get_slice.m b/matlab/ids_get_slice.m index bf0e673..0dd5475 100644 --- a/matlab/ids_get_slice.m +++ b/matlab/ids_get_slice.m @@ -5,6 +5,10 @@ % This method fetches the IDS object with all constant/static data filled. % The dynamic data is interpolated on the requested time slice. This means % that the size of the time dimension in the returned data is 1. +% A read can be partial when the multiversion shim refuses an individual +% field: the returned IDS retains its normal shape and uses that field's +% default value. Inspect imas_get_skipped_paths or +% imas_get_skipped_path_count after the call for the refused paths. % % Args: % expIdx: Data entry context created with diff --git a/matlab/ids_put.m b/matlab/ids_put.m index 9e433d4..925de95 100644 --- a/matlab/ids_put.m +++ b/matlab/ids_put.m @@ -12,6 +12,11 @@ % The put method deletes any previously existing data within the target IDS % occurrence in the Database Entry. % +% When a multiversion Data Dictionary shim refuses an individual field, the +% put can complete partially. MATLAB warns for every refused field; inspect +% imas_get_skipped_paths or imas_get_skipped_path_count immediately afterwards. +% There is no rollback: data deleted or written before a refusal remains changed. +% % Args: % expIdx: Data entry context created with % imas_open_uri, imas_open_env, imas_open_env_backend, diff --git a/matlab/ids_put_slice.m b/matlab/ids_put_slice.m index 4c5a923..8063773 100644 --- a/matlab/ids_put_slice.m +++ b/matlab/ids_put_slice.m @@ -20,6 +20,11 @@ % in one put_slice() call, however the user must ensure that the size of the % time dimension of the node remains consistent with the size of its timebase. % +% When a multiversion Data Dictionary shim refuses an individual field, the +% put can complete partially. MATLAB warns for every refused field; inspect +% imas_get_skipped_paths or imas_get_skipped_path_count immediately afterwards. +% There is no rollback: data written before a refusal remains changed. +% % Args: % expIdx: Data entry context created with % imas_open_uri, imas_open_env, imas_open_env_backend, diff --git a/matlab/imas_get_skipped_path_count.m b/matlab/imas_get_skipped_path_count.m new file mode 100644 index 0000000..34f8425 --- /dev/null +++ b/matlab/imas_get_skipped_path_count.m @@ -0,0 +1,3 @@ +% count = imas_get_skipped_path_count +% Return the number of paths the multiversion shim refused during the last root +% operation. diff --git a/matlab/imas_get_skipped_paths.m b/matlab/imas_get_skipped_paths.m new file mode 100644 index 0000000..743bcf5 --- /dev/null +++ b/matlab/imas_get_skipped_paths.m @@ -0,0 +1,6 @@ +% skipped_paths = imas_get_skipped_paths +% Return paths the multiversion shim refused during the last root operation. +% +% skipped_paths is an N-by-1 struct array with fields operation, path, message, +% and code. It is a 0-by-0 struct carrying the same field names when no path +% was skipped. diff --git a/matlab/imas_partial_get.m b/matlab/imas_partial_get.m index 702abf4..8ec47ec 100644 --- a/matlab/imas_partial_get.m +++ b/matlab/imas_partial_get.m @@ -11,6 +11,10 @@ % % Empty fields within the IDS in the Data Entry are returned with the % default values indicated in :ref:`Default values`. +% A read can be partial when the multiversion shim refuses an individual +% field: the returned IDS retains its normal shape and uses that field's +% default value. Inspect imas_get_skipped_paths or +% imas_get_skipped_path_count after the call for the refused paths. % % Args: % expIdx: Data entry context created with diff --git a/put_single.xsl b/put_single.xsl index 7f1f522..b503358 100644 --- a/put_single.xsl +++ b/put_single.xsl @@ -57,6 +57,10 @@ if (status.code >= 0) { status = al_begin_arraystruct_action(ctx, field.fieldPath, field.timebasePath, &aosArraySize, &aosCtx); + if (status.code < 0 && tolerateRefusalWithConsequence(&status, IMAS_MEX_WRITE_OPERATION, field.fieldPath, "array of structures subtree was not written")) { + aosArraySize = 0; + } + if(aosCtx>0 && aosArraySize>0 && hliAosArraySize == 0) status = begin_dataTree_array_write("", &aosArraySize); } @@ -133,6 +137,7 @@ field.datatype = ; field.dim = ; status = my_al_write_data(&action, &field, data, idsFullName, ""); + if (status.code < 0) tolerateRefusal(&status, IMAS_MEX_WRITE_OPERATION, field.fieldPath); } mxDestroyArray((mxArray *) data); diff --git a/src/imas_get_skipped_path_count.c b/src/imas_get_skipped_path_count.c new file mode 100644 index 0000000..93ec95f --- /dev/null +++ b/src/imas_get_skipped_path_count.c @@ -0,0 +1,14 @@ +#include "imas_mex_utils.h" + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +{ + if (nrhs != 0) + mexErrMsgIdAndTxt("IMAS:imas_get_skipped_path_count:nargin", + "No inputs required."); + if (nlhs > 1) + mexErrMsgIdAndTxt("IMAS:imas_get_skipped_path_count:nargout", + "One output maximum required."); + + if (nlhs == 1) + plhs[0] = mxCreateDoubleScalar((double) getSkippedPathCount()); +} diff --git a/src/imas_get_skipped_paths.c b/src/imas_get_skipped_paths.c new file mode 100644 index 0000000..9dbfb5e --- /dev/null +++ b/src/imas_get_skipped_paths.c @@ -0,0 +1,19 @@ +#include "imas_mex_utils.h" + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +{ + mxArray * paths; + + if (nrhs != 0) + mexErrMsgIdAndTxt("IMAS:imas_get_skipped_paths:nargin", + "No inputs required."); + if (nlhs > 1) + mexErrMsgIdAndTxt("IMAS:imas_get_skipped_paths:nargout", + "One output maximum required."); + + paths = getSkippedPaths(); + if (nlhs == 1) + plhs[0] = paths; + else + mxDestroyArray(paths); +} diff --git a/src/imas_mex_utils.c b/src/imas_mex_utils.c index ceaecc9..be3c133 100644 --- a/src/imas_mex_utils.c +++ b/src/imas_mex_utils.c @@ -26,6 +26,17 @@ char mex_errmsgtxt[MAXERRMSGTXTSIZE]; /*!< Error message */ int msglen = 0; /*!< Length of the mex_errmsgtxt string */ int msg_haspathinfo = 0; +struct imas_mex_skipped_path { + char * operation; + char * path; + char * message; + int code; +}; + +static struct imas_mex_skipped_path * skippedPaths = NULL; +static int skippedPathCount = 0; +static int skippedPathCapacity = 0; + #ifdef _WIN32 #include // For _getpid() @@ -223,6 +234,173 @@ void resetErrMsgIdAndTxt(void) msg_haspathinfo = 0; } +/* + Copies a string into memory the record owns. + + The record outlives the MEX function that wrote it: one entry point records a + refusal, and a later, separate MEX call reads it back. MATLAB frees mxMalloc + memory as soon as the allocating MEX function returns, so the record holds + plain heap memory instead. + */ +static char * duplicateString(const char * string) +{ + char * duplicate; + size_t length = strlen(string) + 1; + + duplicate = malloc(length); + if (duplicate == NULL) + mexErrMsgIdAndTxt("IMAS:imas_mex_utils:allocation_failed", + "Unable to record a refused path."); + memcpy(duplicate, string, length); + return duplicate; +} + +void resetSkippedPaths(void) +{ + int index; + + for (index = 0; index < skippedPathCount; index++) { + free(skippedPaths[index].operation); + free(skippedPaths[index].path); + free(skippedPaths[index].message); + } + free(skippedPaths); + skippedPaths = NULL; + skippedPathCount = 0; + skippedPathCapacity = 0; +} + +/* + Everything the record and the warning need to say about one operation, in one + place: the tag stored in the record, the REFUSED spelling shared with + IMAS-Fortran and IMAS-Cpp, and the warning identifier callers filter on. + */ +struct imas_mex_operation_description { + enum imas_mex_operation operation; + const char * name; + const char * label; + const char * warningId; +}; + +static const struct imas_mex_operation_description operationDescriptions[] = { + {IMAS_MEX_READ_OPERATION, "read", "REFUSED READ", "IMAS:read:refused"}, + {IMAS_MEX_WRITE_OPERATION, "write", "REFUSED WRITE", "IMAS:write:refused"}, + {IMAS_MEX_DELETE_OPERATION, "delete", "REFUSED DELETE", "IMAS:delete:refused"} +}; + +static const size_t operationCount = + sizeof(operationDescriptions) / sizeof(operationDescriptions[0]); + +static const struct imas_mex_operation_description * describeOperation( + enum imas_mex_operation operation) +{ + size_t index; + + for (index = 0; index < operationCount; index++) + if (operationDescriptions[index].operation == operation) + return &operationDescriptions[index]; + return &operationDescriptions[0]; +} + +int operationFromName(const char * name, enum imas_mex_operation * operation) +{ + size_t index; + + for (index = 0; index < operationCount; index++) + if (strcmp(operationDescriptions[index].name, name) == 0) { + *operation = operationDescriptions[index].operation; + return 1; + } + return 0; +} + +static void addSkippedPath(al_status_t status, enum imas_mex_operation operation, + const char * path) +{ + struct imas_mex_skipped_path * resizedPaths; + struct imas_mex_skipped_path * skippedPath; + + if (skippedPathCount == skippedPathCapacity) { + int newCapacity = skippedPathCapacity == 0 ? 8 : skippedPathCapacity * 2; + resizedPaths = realloc(skippedPaths, + newCapacity * sizeof(struct imas_mex_skipped_path)); + if (resizedPaths == NULL) + mexErrMsgIdAndTxt("IMAS:imas_mex_utils:allocation_failed", + "Unable to record a refused path."); + skippedPaths = resizedPaths; + skippedPathCapacity = newCapacity; + } + + skippedPath = &skippedPaths[skippedPathCount]; + skippedPath->operation = duplicateString(describeOperation(operation)->name); + skippedPath->path = duplicateString(path); + skippedPath->message = duplicateString(status.message); + skippedPath->code = status.code; + skippedPathCount++; +} + +int tolerateRefusal(al_status_t * status, enum imas_mex_operation operation, + const char * path) +{ + return tolerateRefusalWithConsequence(status, operation, path, NULL); +} + +int tolerateRefusalWithConsequence(al_status_t * status, + enum imas_mex_operation operation, + const char * path, + const char * consequence) +{ + const struct imas_mex_operation_description * description; + + if (status->code < IMAS_MEX_REFUSAL_BAND_MIN || + status->code > IMAS_MEX_REFUSAL_BAND_MAX) + return 0; + + addSkippedPath(*status, operation, path); + description = describeOperation(operation); + + if (consequence == NULL) + mexWarnMsgIdAndTxt(description->warningId, "%s: %s (status %d): %s", + description->label, path, status->code, + status->message); + else + mexWarnMsgIdAndTxt(description->warningId, "%s: %s (status %d): %s; %s", + description->label, path, status->code, + status->message, consequence); + + /* The traversal carries on from a clean status; the accumulating + error-message buffer is deliberately left untouched. */ + status->code = 0; + status->message[0] = '\0'; + return 1; +} + +int getSkippedPathCount(void) +{ + return skippedPathCount; +} + +mxArray * getSkippedPaths(void) +{ + const char * fieldNames[] = {"operation", "path", "message", "code"}; + mxArray * paths; + int index; + + if (skippedPathCount == 0) + paths = mxCreateStructMatrix(0, 0, 4, fieldNames); + else + paths = mxCreateStructMatrix(skippedPathCount, 1, 4, fieldNames); + + for (index = 0; index < skippedPathCount; index++) { + mxSetField(paths, index, "operation", mxCreateString(skippedPaths[index].operation)); + mxSetField(paths, index, "path", mxCreateString(skippedPaths[index].path)); + mxSetField(paths, index, "message", mxCreateString(skippedPaths[index].message)); + mxSetField(paths, index, "code", mxCreateDoubleScalar((double) skippedPaths[index].code)); + } + + return paths; +} + /** Assembles text and identifier for an error message then throws it. When the global mex_errmsgid variable is not an empty string, this function assembles the error message identifier from the prefix given in input and the content of the mex_errmsgid global variable. The text of the error message is then taken from the mex_errmsgtxt global variable. If mex_errmsgid was an empty string, the identifier and text are the default ones and contain the error code provided by the status parameter. diff --git a/src/imas_mex_utils.h b/src/imas_mex_utils.h index 9212d12..9616443 100644 --- a/src/imas_mex_utils.h +++ b/src/imas_mex_utils.h @@ -152,6 +152,49 @@ char * getFilenameFromPath(char *); void resetErrMsgIdAndTxt(void); +enum imas_mex_operation { + IMAS_MEX_READ_OPERATION, + IMAS_MEX_WRITE_OPERATION, + IMAS_MEX_DELETE_OPERATION +}; + +/* + The refusal band: the status codes a multiversion shim uses to decline a + path it cannot convert. Disjoint from IMAS-Core's own -1..-4. See CONTEXT.md. + */ +#define IMAS_MEX_REFUSAL_BAND_MIN (-1099) +#define IMAS_MEX_REFUSAL_BAND_MAX (-1000) + +/* + Process-global record of paths a multiversion shim refused during the root + read, write, or delete operation that just ran. See CONTEXT.md. + */ +void resetSkippedPaths(void); + +/* + Resolves the MATLAB spelling of an operation tag to its enum value. Returns 1 + on a match, 0 otherwise. + */ +int operationFromName(const char * name, enum imas_mex_operation * operation); + +/* + Decides whether a non-zero status at one field is fatal or tolerable. A status + in the refusal band is recorded, warned about, and cleared from *status so the + traversal can carry on; 1 is returned. Any other status is left untouched and + 0 is returned, so the caller's existing fatal handling runs. + */ +int tolerateRefusal(al_status_t * status, enum imas_mex_operation operation, + const char * path); + +int tolerateRefusalWithConsequence(al_status_t * status, + enum imas_mex_operation operation, + const char * path, + const char * consequence); + +int getSkippedPathCount(void); + +mxArray * getSkippedPaths(void); + void my_mexErrMsgIdAndTxt(al_status_t status, const char * prefix); void my_validation_mexErrMsgIdAndTxt(al_validation_status_t status, const char * prefix); diff --git a/src/imas_test_inject_skipped_path.c b/src/imas_test_inject_skipped_path.c new file mode 100644 index 0000000..d316c50 --- /dev/null +++ b/src/imas_test_inject_skipped_path.c @@ -0,0 +1,49 @@ +#include "imas_mex_utils.h" + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +{ + al_status_t status = {0, ""}; + enum imas_mex_operation operationType; + char * operation; + char * path; + char * message; + + if (nrhs != 4) + mexErrMsgIdAndTxt("IMAS:imas_test_inject_skipped_path:nargin", + "Status, operation, path, and message inputs required."); + if (nlhs != 0) + mexErrMsgIdAndTxt("IMAS:imas_test_inject_skipped_path:nargout", + "No outputs required."); + if (!mxIsNumeric(prhs[0]) || !mxIsScalar(prhs[0])) + mexErrMsgIdAndTxt("IMAS:imas_test_inject_skipped_path:notScalar", + "Status must be a numeric scalar."); + if (!mxIsChar(prhs[1]) || !mxIsChar(prhs[2]) || !mxIsChar(prhs[3])) + mexErrMsgIdAndTxt("IMAS:imas_test_inject_skipped_path:notChar", + "Operation, path, and message must be character arrays."); + + operation = mxArrayToString(prhs[1]); + path = mxArrayToString(prhs[2]); + message = mxArrayToString(prhs[3]); + if (!operationFromName(operation, &operationType)) + mexErrMsgIdAndTxt("IMAS:imas_test_inject_skipped_path:invalid_operation", + "Operation must be read, write, or delete."); + + resetErrMsgIdAndTxt(); + resetSkippedPaths(); + status.code = (int) mxGetScalar(prhs[0]); + strncpy(status.message, message, MAX_ERR_MSG_LEN - 1); + status.message[MAX_ERR_MSG_LEN - 1] = '\0'; + + if (tolerateRefusal(&status, operationType, path)) { + mxFree(operation); + mxFree(path); + mxFree(message); + return; + } + + mxFree(operation); + mxFree(path); + mxFree(message); + if (status.code != 0) + my_mexErrMsgIdAndTxt(status, "IMAS:imas_test_inject_skipped_path:"); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7b2e150..36939aa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,8 +22,16 @@ add_test( NAME al-mex-test -batch "addpath ${LIB_FOLDER} ${M_FOLDER}; res = runtests('imas_unit_tests');exit(~all([res.Passed]))" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) +add_test( NAME al-utils-unit-test + COMMAND + ${Matlab_MAIN_PROGRAM} + -nodisplay + -batch "addpath ${LIB_FOLDER} ${M_FOLDER}; res = runtests('imas_utils_unit_tests');exit(~all([res.Passed]))" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) # Note: added ids_path for backward compatibility with IMAS-Core<5.6 set( MATLAB_ENV "IMAS_AL_DISABLE_VALIDATE=1;IMAS_AL_DISABLE_OBSOLESCENT_WARNING=1;ids_path=${MDSPLUS_MODEL_DIR};MDSPLUS_MODELS_PATH=${MDSPLUS_MODEL_DIR}" ) +list( APPEND MATLAB_ENV ${AL_MATLAB_SHIM_TEST_ENVIRONMENT} ) if( AL_BACKEND_HDF5 ) # The HDF5 library built-in with matlab may conflict with the one that the AL # was built against. Preload the AL-linked library: @@ -33,11 +41,36 @@ if( AL_BACKEND_HDF5 ) list( APPEND MATLAB_ENV "LD_PRELOAD=${PRELOAD_LIBS}" ) endif() -set_tests_properties( al-mex-test PROPERTIES +set_tests_properties( al-mex-test al-utils-unit-test PROPERTIES TIMEOUT 3600 ENVIRONMENT "${MATLAB_ENV}" ) +if( AL_USE_MULTIVERSION_SHIM ) + if( APPLE ) + find_program( AL_SHIM_INSPECT_TOOL otool ) + set( AL_SHIM_INSPECT_ARGS -L ) + else() + find_program( AL_SHIM_INSPECT_TOOL readelf ) + set( AL_SHIM_INSPECT_ARGS -d ) + if( NOT AL_SHIM_INSPECT_TOOL AND CMAKE_OBJDUMP ) + set( AL_SHIM_INSPECT_TOOL ${CMAKE_OBJDUMP} ) + set( AL_SHIM_INSPECT_ARGS -p ) + endif() + endif() + if( NOT AL_SHIM_INSPECT_TOOL ) + message( FATAL_ERROR "A shared-library inspector (otool, readelf or objdump) is required for shim linkage tests" ) + endif() + foreach( _target IN ITEMS al-mex mex-imas_open ) + add_test( NAME ${_target}-shim-linkage + COMMAND ${CMAKE_COMMAND} + -DLIBRARY=$ + -DINSPECT_TOOL=${AL_SHIM_INSPECT_TOOL} + -DINSPECT_ARGS=${AL_SHIM_INSPECT_ARGS} + -P ${CMAKE_CURRENT_SOURCE_DIR}/shim/check_shim_linkage.cmake ) + endforeach() +endif() + if( AL_PLUGINS ) if( TARGET al-plugins ) add_test( NAME al-partial-get-test @@ -77,4 +110,3 @@ else() message(STATUS "Tests will run with both HDF5 and MDSplus backends") endif() endif() - diff --git a/tests/imas_unit_tests.m b/tests/imas_unit_tests.m index 8300b2c..a0e6337 100644 --- a/tests/imas_unit_tests.m +++ b/tests/imas_unit_tests.m @@ -59,6 +59,18 @@ function closeIMASDb(testCase,backend,useCache,homogeneousTime) IDSname = IDS_list.'; end + methods (Access = private) + % Leave a skipped path behind from a notional earlier operation. An entry + % point that clears the record on its own entry wipes it; one that does not + % reports a previous operation's skips as its own. + function injectPreviousRefusal(testCase, operation) + testCase.verifyWarning(@() imas_test_inject_skipped_path( ... + -1000, operation, 'previous/path', 'previous refusal'), ... + ['IMAS:' operation ':refused']); + testCase.verifyEqual(imas_get_skipped_path_count, 1); + end + end + %% Test Method Block methods (Test) @@ -82,7 +94,70 @@ function testPut(testCase, IDSname) comparator(sdi,sdi_slice,IDSname); end end + + function testReadEntryPointsClearSkippedPaths(testCase) + idx = testCase.TestData.idx; + ids = testCase.TestData.IDS.equilibrium; + ids_put(idx, 'equilibrium', ids); + + testCase.injectPreviousRefusal('read'); + ids_get(idx, 'equilibrium'); + testCase.verifyEqual(imas_get_skipped_path_count, 0); + + testCase.injectPreviousRefusal('read'); + ids_get_slice(idx, 'equilibrium', ids.time(2), 1); + testCase.verifyEqual(imas_get_skipped_path_count, 0); + + testCase.injectPreviousRefusal('read'); + ids_get_sample(idx, 'equilibrium', ids.time(1), ids.time(end), [], 0); + testCase.verifyEqual(imas_get_skipped_path_count, 0); + end + + function testReadIsUsableDownstreamWhileSkippedPathsAreRecorded(testCase) + % Manufacturing a genuinely partial read needs a multiversion shim and an + % occurrence stored under another DD version, neither of which this suite + % can build; that acceptance belongs to the conformance suite. What is + % observable here is that a non-empty record does not get in the way of + % the calls a caller reaches for next, and that only a root operation + % clears it. + idx = testCase.TestData.idx; + ids = testCase.TestData.IDS.equilibrium; + ids_put(idx, 'equilibrium', ids); + fullIds = ids_get(idx, 'equilibrium'); + + testCase.injectPreviousRefusal('read'); + + testCase.verifyTrue(ids_isdefined(fullIds)); + if ~ispc + ids_validate('equilibrium', fullIds); + end + testCase.verifyEqual(imas_get_skipped_path_count, 1); + + ids_put(idx, 'equilibrium', fullIds); + testCase.verifyEqual(imas_get_skipped_path_count, 0); + end + function testWriteEntryPointsClearSkippedPaths(testCase) + % Whether ids_put's delete and write phases share one record, and whether + % each phase's skips carry the right operation tag, needs a shim that + % actually refuses; that is the conformance suite's job. What is + % observable here is the clear-on-entry half of the contract. + idx = testCase.TestData.idx; + ids_slice = testCase.TestData.IDS_slice.equilibrium; + + testCase.injectPreviousRefusal('write'); + ids_put(idx, 'equilibrium', ids_slice{1}); + testCase.verifyEqual(imas_get_skipped_path_count, 0); + + testCase.injectPreviousRefusal('write'); + ids_put_slice(idx, 'equilibrium', ids_slice{2}); + testCase.verifyEqual(imas_get_skipped_path_count, 0); + + testCase.injectPreviousRefusal('delete'); + ids_delete(idx, 'equilibrium'); + testCase.verifyEqual(imas_get_skipped_path_count, 0); + end + function testPutSlice(testCase, IDSname) idx = testCase.TestData.idx; ids = testCase.TestData.IDS.(IDSname); diff --git a/tests/imas_utils_unit_tests.m b/tests/imas_utils_unit_tests.m index 2112ff2..02ac02a 100644 --- a/tests/imas_utils_unit_tests.m +++ b/tests/imas_utils_unit_tests.m @@ -7,6 +7,14 @@ %% Class-level setup methods (TestClassSetup) + function verifySkippedPathsAreInitiallyEmpty(TestCase) + skippedPaths = imas_get_skipped_paths; + + TestCase.verifySize(skippedPaths, [0 0]); + TestCase.verifyEqual(fieldnames(skippedPaths), ... + {'operation'; 'path'; 'message'; 'code'}); + TestCase.verifyEqual(imas_get_skipped_path_count, 0); + end end %% Test Method Parameters @@ -18,6 +26,63 @@ %% Test Method Block methods (Test) + function testZeroStatusDoesNotRecordASkippedPath(TestCase) + imas_test_inject_skipped_path(0, 'read', '', ''); + + skippedPaths = imas_get_skipped_paths; + + TestCase.verifySize(skippedPaths, [0 0]); + TestCase.verifyEqual(imas_get_skipped_path_count, 0); + end + + function testRefusalPolicyTruthTable(TestCase) + import matlab.unittest.constraints.IssuesWarnings + + operations = {'read', 'write', 'delete'}; + statuses = [0, -1, -2, -3, -4, -1000, -1050, -1099, -999, -1100]; + + for operationIndex = 1:numel(operations) + operation = operations{operationIndex}; + for status = statuses + call = @() imas_test_inject_skipped_path(status, operation, 'a/b', 'shim refusal'); + isRefusal = status >= -1099 && status <= -1000; + + if (isRefusal) + % One warning, that identifier, and nothing else: a refusal that + % warned twice would announce one skipped path as two. + TestCase.verifyThat(call, IssuesWarnings({['IMAS:' operation ':refused']}, ... + 'RespectingSet', true, 'RespectingCount', true)); + elseif (status < 0) + TestCase.verifyError(call, 'IMAS:imas_test_inject_skipped_path:internal_error'); + else + call(); + end + + skippedPaths = imas_get_skipped_paths; + TestCase.verifyEqual(imas_get_skipped_path_count, numel(skippedPaths)); + TestCase.verifyEqual(numel(skippedPaths), double(isRefusal)); + end + end + end + + function testSkippedPathRecordRoundTripsAndResets(TestCase) + imas_test_inject_skipped_path(-1050, 'write', 'equilibrium/time_slice', 'cannot convert this field'); + skippedPaths = imas_get_skipped_paths; + + TestCase.verifyEqual(imas_get_skipped_path_count, 1); + TestCase.verifyEqual(skippedPaths.operation, 'write'); + TestCase.verifyEqual(skippedPaths.path, 'equilibrium/time_slice'); + TestCase.verifyEqual(skippedPaths.message, 'cannot convert this field'); + TestCase.verifyEqual(skippedPaths.code, -1050); + + imas_test_inject_skipped_path(-1000, 'delete', 'equilibrium', 'cannot delete this field'); + skippedPaths = imas_get_skipped_paths; + + TestCase.verifyEqual(imas_get_skipped_path_count, 1); + TestCase.verifyEqual(skippedPaths.operation, 'delete'); + TestCase.verifyEqual(skippedPaths.path, 'equilibrium'); + end + function rand(TestCase, IDSname, ntime) ids1 = ids_rand(IDSname, ntime, 0); ids2 = ids_rand(IDSname, ntime, 2); diff --git a/tests/shim/check_shim_linkage.cmake b/tests/shim/check_shim_linkage.cmake new file mode 100644 index 0000000..20acd66 --- /dev/null +++ b/tests/shim/check_shim_linkage.cmake @@ -0,0 +1,16 @@ +# Check the actual shared-library dependencies, since a MEX link line that also +# contains Core can resolve the mirrored C ABI symbols without conversion. +execute_process( + COMMAND "${INSPECT_TOOL}" ${INSPECT_ARGS} "${LIBRARY}" + RESULT_VARIABLE inspect_result + OUTPUT_VARIABLE dependencies + ERROR_VARIABLE inspect_error ) +if( NOT inspect_result EQUAL 0 ) + message( FATAL_ERROR "Could not inspect ${LIBRARY}: ${inspect_error}" ) +endif() +if( NOT dependencies MATCHES "libimas_mvdd_loader" ) + message( FATAL_ERROR "${LIBRARY} does not depend on the multiversion shim:\n${dependencies}" ) +endif() +if( dependencies MATCHES "libal[.]" ) + message( FATAL_ERROR "${LIBRARY} links IMAS-Core directly, bypassing the shim:\n${dependencies}" ) +endif()