[ZFDB] FDB-721 Gribjump extractor

.github/workflows/z3fdb.yml

@@ -24,8 +24,14 @@ on:
 
 jobs:
   prepare-deps:
-    runs-on: ubuntu-latest
     if: ${{ !github.event.pull_request.draft && (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
+    runs-on: [self-hosted, Linux, platform-builder-docker-xl]
+    container:
+      image: eccr.ecmwf.int/platform-builder/platform-builder:ubuntu-24.04
+      credentials:
+        username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
+        password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
+      options: '--user root' # see https://github.com/ecmwf/reusable-workflows/blob/main/.github/workflows/cd-system-package.yml
     steps:
       - name: Get ecbuild
         uses: actions/checkout@v5
@@ -108,11 +114,17 @@ jobs:
           retention-days: 1
   build-wheels:
     needs: prepare-deps
-    runs-on: ubuntu-latest
     if: ${{ !github.event.pull_request.draft && (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
+    runs-on: [self-hosted, Linux, platform-builder-docker-xl]
+    container:
+      image: eccr.ecmwf.int/platform-builder/platform-builder:ubuntu-24.04
+      credentials:
+        username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
+        password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
+      options: '--user root' # see https://github.com/ecmwf/reusable-workflows/blob/main/.github/workflows/cd-system-package.yml
     strategy:
       matrix:
-        python-version: ['3.11', '3.12', '3.13'] # eccodes does not yet support 3.14
+        python-version: ['3.11', '3.12', '3.13', '3.14'] # eccodes does not yet support 3.14
       fail-fast: false  # Continue running other versions if one fails
 
     steps:

CMakeLists.txt

@@ -139,6 +139,15 @@ ecbuild_add_option( FEATURE PYTHON_ZARR_INTERFACE
                     DEFAULT OFF
                     REQUIRED_PACKAGES "NAME pybind11 VERSION 3.0.1" )
 
+### Enables the GribJump-backed extractor for the Zarr interface.
+### NOTE: gribjump itself depends on fdb5, so this is only satisfiable in a bundle build where
+### gribjump is a sibling project. It can never be satisfied by a standalone fdb build against
+### an installed gribjump, which is why it defaults to OFF.
+ecbuild_add_option( FEATURE ZARR_GRIBJUMP_EXTRACTOR
+                    DESCRIPTION "Build the GribJump-backed extractor (bundle builds only)"
+                    DEFAULT OFF
+                    REQUIRED_PACKAGES "NAME gribjump" )
+
 # We need PyFDB in case z3fdb is enabled
 if(HAVE_PYTHON_ZARR_INTERFACE AND NOT HAVE_PYTHON_FDB_INTERFACE)
     message( WARN "FDB Zarr interface requires python FDB interface. I enable ENABLE_PYTHON_FDB_INTERFACE to build the PyFDB interface" )

cmake/z3fdb_setup.py.in

@@ -8,6 +8,16 @@ from wheel.bdist_wheel import bdist_wheel
 import sys
 
 
+def _read_requirements(name):
+    """Requirement lines from a pip requirements file staged next to this setup.py.
+
+    Comments and blank lines are dropped; everything else is passed through verbatim.
+    """
+    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), name)
+    with open(path) as requirements_file:
+        return [line.strip() for line in requirements_file if line.strip() and not line.lstrip().startswith("#")]
+
+
 # NOTE this we need to correctly link with fdb5lib version on cd. For local builds feel free to ignore
 version_suffix = os.environ.get("VERSION_SUFFIX", "")
 if version_suffix:
@@ -45,9 +55,17 @@ setup(
         "z3fdb._internal",
         "pychunked_data_view",
         "chunked_data_view_bindings",
+        # PEP 561 stub-only package. The hyphen is mandated by the PEP but is not a valid
+        # Python identifier, so setuptools cannot derive its directory from the name,
+        # package_dir below states it explicitly.
+        "chunked_data_view_bindings-stubs",
     ],
+    package_dir={
+        "chunked_data_view_bindings-stubs": "chunked_data_view_bindings-stubs",
+    },
     package_data={
-        "chunked_data_view_bindings": ["*.so", "*.pyd", "*.pyi", "py.typed"],
+        "chunked_data_view_bindings": ["*.so", "*.pyd"],
+        "chunked_data_view_bindings-stubs": ["*.pyi", "py.typed"],
     },
     license="Apache 2.0",
     license_files=["LICENSE"],
@@ -65,7 +83,10 @@ setup(
         "Operating System :: OS Independent",
         "Topic :: Software Development :: Libraries",
     ],
-    install_requires=["numpy", "zarr~=3.1.6", "findlibs>=0.1.2"] + requires_extra,
+    # Runtime dependencies live in python/z3fdb-requirements.txt, staged next to this file by
+    # src/CMakeLists.txt. Build tooling stays out of the wheel metadata; it is provisioned into
+    # the uvx environment from python/z3fdb-build-requirements.txt instead.
+    install_requires=_read_requirements("z3fdb-requirements.txt") + requires_extra,
     python_requires=python_requires,
     has_ext_modules=lambda: True,
     **ext_kwargs[sys.platform],

docs/z3fdb/api.rst

@@ -21,6 +21,42 @@ z3fdb.Z3fdbError
 
 .. autoapiexception:: z3fdb.Z3fdbError
 
+Extractor errors
+^^^^^^^^^^^^^^^^
+
+Raised by the extractor backends and re-exported from
+:mod:`pychunked_data_view`, so they can be caught by type:
+
+``GribExtractorError``
+   A GRIB field could not be retrieved or decoded. For example, FDB returned no field
+   for a sub-request, or a field's size does not match the rest of the view.
+
+``GribJumpExtractorError``
+   GribJump extraction failed. For example, a field location carried no usable file
+   offset, or the request matched nothing.
+
+``MarsRequestFormattingError``
+   A malformed MARS request string: a trailing comma, a missing comma between keys, or a
+   misspelled key. Raised from ``build()``; a subclass of ``RuntimeError``.
+
+``InternalError``
+   Something inside ``pychunked_data_view`` is inconsistent. You should not see this.
+
+Note that other misconfiguration detected by the builder (unmapped axes, incompatible parts, an
+invalid chunk size, parts disagreeing about the grid) surfaces as a plain ``RuntimeError``,
+since it originates as an ``eckit::UserError``.
+
+Build capability
+^^^^^^^^^^^^^^^^
+
+.. py:data:: pychunked_data_view.has_gribjump_extractor
+   :type: bool
+
+   Whether this build compiled the GribJump extractor.
+   :class:`~pychunked_data_view.ExtractorType.GribJump` can always be constructed, so this is
+   the way to find out whether it can actually be used. See
+   :ref:`z3fdb_gribjump_availability`.
+
 Type aliases
 ------------
 
@@ -32,9 +68,9 @@ z3fdb.MarsSelection
    A MARS request expressed as a mapping.  Keys are MARS keyword names
    (strings).  Values may be:
 
-   * a single ``str``, ``int``, or ``float`` — e.g. ``"step": 0``
-   * a list of ``str``, ``int``, or ``float`` — e.g. ``"param": [165, 166]``
-   * a MARS range expression passed as a ``str`` — e.g.
+   * a single ``str``, ``int``, or ``float``, e.g. ``"step": 0``
+   * a list of ``str``, ``int``, or ``float``, e.g. ``"param": [165, 166]``
+   * a MARS range expression passed as a ``str``, e.g.
      ``"date": "2020-01-01/to/2020-01-04"``
 
    Example::
@@ -58,9 +94,40 @@ Classes
 z3fdb.SimpleStoreBuilder
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
+Creates a store whose root *is* the array. Equivalent to
+:class:`~z3fdb.CustomStoreBuilder` restricted to ``path=None``, which is what it
+delegates to.
+
 .. autoapiclass:: z3fdb.SimpleStoreBuilder
    :members:
 
+z3fdb.ChunkedDataView
+^^^^^^^^^^^^^^^^^^^^^
+
+The read-only array returned by ``build()`` on the lower-level
+:class:`~pychunked_data_view.ChunkedDataViewBuilder`. Zarr normally drives it for you.
+
+.. note::
+
+   ``chunkShape()`` is deprecated in favour of ``chunk_shape()``; it still works but emits a
+   ``DeprecationWarning``. Every other accessor on the class is already snake_case.
+
+.. autoapiclass:: pychunked_data_view.ChunkedDataView
+   :members:
+
+z3fdb.CustomStoreBuilder
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Creates a store with an arbitrary group/array hierarchy: every method takes a
+zarr-style *path* naming the array it applies to, and ``path=None`` addresses a
+root array (mutually exclusive with any named path).
+
+.. seealso:: :ref:`tutorial_custom_store_mixed_extractors` for a worked example
+   building several arrays with different extractors in one store.
+
+.. autoapiclass:: z3fdb.CustomStoreBuilder
+   :members:
+
 z3fdb.AxisDefinition
 ^^^^^^^^^^^^^^^^^^^^
 
@@ -70,21 +137,21 @@ to Zarr dimensions.
 .. autoapiclass:: pychunked_data_view.AxisDefinition
    :members:
 
-Enums
------
+Chunking
+--------
 
 z3fdb.Chunking
-^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^
 
 .. autoapiclass:: pychunked_data_view.Chunking
    :members:
 
-.. py:class:: pychunked_data_view.Chunking.FixedSizeChunk(chunkShape)
+.. py:class:: pychunked_data_view.Chunking.FixedSizeChunk(chunk_shape)
 
    Specifies a custom chunk size along a single axis. This is a frozen
    dataclass nested inside :class:`~pychunked_data_view.Chunking`.
 
-   .. py:attribute:: chunkShape
+   .. py:attribute:: chunk_shape
       :type: int
 
       Number of consecutive axis values grouped into each chunk.
@@ -97,14 +164,82 @@ z3fdb.Chunking
    .. code-block:: python
 
       # Chunk a 12-date axis into groups of 3 (gives 4 chunks)
-      AxisDefinition(["date"], Chunking.FixedSizeChunk(chunkShape=3))
+      AxisDefinition(["date"], Chunking.FixedSizeChunk(chunk_shape=3))
 
-   See :ref:`dimension_mapping:Chunking` for a full comparison of
+   See :ref:`z3fdb_chunking` for a full comparison of
    chunking modes and guidance on when to use each one.
 
-z3fdb.ExtractorType
-^^^^^^^^^^^^^^^^^^^
+Extractors
+----------
 
-.. autoapiclass:: pychunked_data_view.ExtractorType
-   :members:
+``ExtractorType`` is a namespace class, not an enum. Its nested classes
+carry per-extractor configuration.  Pass an *instance* to
+:meth:`~z3fdb.SimpleStoreBuilder.add_part`.
+
+``add_part`` stores a *copy* of the configuration, so one instance can be reused across as many
+parts and builders as you like, and the ``fdb_config`` a builder fills in for you is never
+written back into your object.
+
+.. seealso:: :ref:`z3fdb_extractor_backends` for what the two backends do, their constraints,
+   and which builds provide GribJump.
+
+z3fdb.ExtractorType.Grib
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. py:class:: pychunked_data_view.ExtractorType.Grib(*, fdb_config=None)
+
+   Reads full GRIB fields from FDB and decodes them to ``float32`` via eccodes.
+   This is the default extractor for standard GRIB data.
+
+   :param fdb_config: Path to an FDB configuration YAML file.
+       ``None`` (default) uses the path passed to :class:`~z3fdb.SimpleStoreBuilder`.
+   :type fdb_config: pathlib.Path or None
+
+   **Example**
+
+   .. code-block:: python
+
+      builder.add_part(mars_request, axes, ExtractorType.Grib())
+
+      # With an explicit FDB config
+      builder.add_part(mars_request, axes, ExtractorType.Grib(fdb_config=Path("/etc/fdb/config.yaml")))
+
+z3fdb.ExtractorType.GribJump
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. py:class:: pychunked_data_view.ExtractorType.GribJump(*, fdb_config=None, gribjump_config=None, field_chunking=None)
+
+   Reads grid-point values from FDB using GribJump, a library that jumps
+   directly to the values inside the GRIB message without performing a full
+   decode.
+
+   :param fdb_config: Path to an FDB configuration YAML file.
+       ``None`` (default) uses the path passed to :class:`~z3fdb.SimpleStoreBuilder`.
+   :type fdb_config: pathlib.Path or None
+
+   :param gribjump_config: Path to a GribJump configuration YAML file.
+       ``None`` (default) reads the ``GRIBJUMP_CONFIG_FILE`` environment variable.
+   :type gribjump_config: pathlib.Path or None
+
+   :param field_chunking: How to sub-divide the implicit (grid-point) dimension into
+       Zarr chunks.  ``None`` (default) produces a single chunk covering the
+       full field.  Pass :class:`pychunked_data_view.Chunking.FixedSizeChunk`
+       to split the implicit axis into equal-sized pieces; the size must divide
+       the grid exactly, as that dimension cannot be left ragged.
+   :type field_chunking: pychunked_data_view.Chunking.FixedSizeChunk or None
+
+   **Example**
+
+   .. code-block:: python
+
+      # Full field: avoids eccodes decode
+      builder.add_part(mars_request, axes, ExtractorType.GribJump())
+
+      # Split the implicit grid-point axis into chunks of 1312
+      builder.add_part(mars_request, axes,
+                       ExtractorType.GribJump(field_chunking=Chunking.FixedSizeChunk(1312)))
+
+   .. seealso:: :ref:`z3fdb_extractor_backends` for how the two backends differ, their
+      constraints, and which builds provide GribJump; and
+      :ref:`tutorial_custom_store_mixed_extractors` for a worked example using both.