diff --git a/.github/ci-config.yml b/.github/ci-config.yml index a6bd79cc6..c37e82c04 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -2,7 +2,7 @@ dependencies: | ecmwf/ecbuild Deutsches-Klimarechenzentrum/libaec@refs/tags/v1.1.6 ecmwf/eccodes - ecmwf/eckit + ecmwf/eckit@feature/ECKIT-684-ceph-backend ecmwf/metkit dependency_branch: develop cmake_options: -DENABLE_MEMFS=ON -DENABLE_DUMMY_DAOS=ON -DENABLE_FAMFDB=ON diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index faaa19ad0..871902b0b 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -4,7 +4,7 @@ build: dependencies: - ecmwf/ecbuild@develop - ecmwf/eccodes@develop - - ecmwf/eckit@develop + - ecmwf/eckit@feature/ECKIT-684-ceph-backend - ecmwf/metkit@develop cmake_options: - -DENABLE_LUSTRE=OFF diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml new file mode 100644 index 000000000..ce02bbcbd --- /dev/null +++ b/.github/workflows/ci-rados.yml @@ -0,0 +1,253 @@ +name: ci-rados + +on: + push: + branches: + - "master" + - "develop" + tags-ignore: + - "**" + + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger on public pull request approval + pull_request_target: + types: [labeled] + + # Trigger the workflow manually + workflow_dispatch: ~ + +# avoid duplicate runs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + rados: + name: Test Ceph/RADOS Backend + runs-on: ubuntu-latest + + env: + CEPH_ETC: /etc/ceph + ECKIT_RADOS_CLUSTER_NAME: ceph + ECKIT_RADOS_CLUSTER_USER: client.admin + FDB_RADOS_TEST_POOL: fdb_test + # Install prefix shared by all bundle projects (eckit, eccodes, metkit, fdb). + INSTALL_PREFIX: ${{ github.workspace }}/install + # Local dirs used to cache apt archives and the Ceph docker image. + APT_CACHE: ${{ github.workspace }}/.apt-cache + CEPH_IMAGE: quay.io/ceph/demo@sha256:522483cf07cfce6386b8e18a3edfa88a1b32c688dee231d0a6962b1513557723 + CEPH_IMAGE_CACHE: ${{ github.workspace }}/.docker-ceph + CMAKE_FLAGS: -DENABLE_AEC=OFF -DENABLE_EXAMPLES=OFF -DENABLE_EXPERIMENTAL=OFF -DENABLE_NETCDF=OFF + + steps: + - name: Checkout ecbuild + uses: actions/checkout@v4 + with: + repository: ecmwf/ecbuild + path: ecbuild + + - name: Checkout eckit + uses: actions/checkout@v4 + with: + repository: ecmwf/eckit + ref: feature/ECKIT-684-ceph-backend + path: eckit + + - name: Checkout eccodes + uses: actions/checkout@v4 + with: + repository: ecmwf/eccodes + path: eccodes + + - name: Checkout metkit + uses: actions/checkout@v4 + with: + repository: ecmwf/metkit + path: metkit + + - name: Checkout fdb + uses: actions/checkout@v4 + with: + path: fdb + + - name: Cache apt packages + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.apt-cache + # Bump the suffix to invalidate when the package list changes. + key: apt-rados-${{ runner.os }}-v2 + + - name: Install build dependencies + run: | + mkdir -p "${APT_CACHE}/archives/partial" + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + -o Dir::Cache::archives="${APT_CACHE}/archives" \ + -o APT::Keep-Downloaded-Packages=true \ + build-essential \ + cmake \ + jq \ + ninja-build \ + gfortran \ + libopenmpi-dev \ + libssl-dev \ + uuid-dev \ + libbz2-dev \ + libcurl4-openssl-dev \ + libaec-dev \ + librados-dev + # Make the cached .deb archives readable/writable by the runner user. + sudo chown -R "$(id -u):$(id -g)" "${APT_CACHE}" + + - name: Cache Ceph image + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.docker-ceph + # Bump the suffix to refresh the pinned image snapshot. + key: ceph-image-522483cf07cf-v2 + + - name: Load or pull Ceph image + run: | + if [ -f "${CEPH_IMAGE_CACHE}/ceph-demo.tar" ]; then + docker load -i "${CEPH_IMAGE_CACHE}/ceph-demo.tar" + else + docker pull "${CEPH_IMAGE}" + mkdir -p "${CEPH_IMAGE_CACHE}" + docker save "${CEPH_IMAGE}" -o "${CEPH_IMAGE_CACHE}/ceph-demo.tar" + fi + + - name: Start Ceph cluster + run: | + sudo mkdir -p "${CEPH_ETC}" + docker run -d --name ceph-demo \ + --network host \ + -e MON_IP=127.0.0.1 \ + -e CEPH_PUBLIC_NETWORK=0.0.0.0/0 \ + -e CEPH_DEMO_UID=ci \ + -e DEMO_DAEMONS=mon,mgr,osd \ + -v "${CEPH_ETC}:/etc/ceph" \ + "${CEPH_IMAGE}" demo + + - name: Wait for Ceph and create test pool + run: | + echo "Waiting for Ceph to become ready..." + for i in $(seq 1 60); do + if docker exec ceph-demo ceph osd pool ls >/dev/null 2>&1; then + ready=1 + break + fi + sleep 5 + done + if [ "${ready:-0}" != "1" ]; then + echo "Ceph did not become ready in time" >&2 + docker logs ceph-demo || true + exit 1 + fi + echo "Waiting for Ceph cluster health..." + for i in $(seq 1 60); do + if docker exec ceph-demo ceph health 2>/dev/null | grep -q '^HEALTH_OK'; then + health_ready=1 + break + fi + sleep 5 + done + if [ "${health_ready:-0}" != "1" ]; then + echo "Ceph cluster did not become healthy in time" >&2 + docker exec ceph-demo ceph status || true + docker logs ceph-demo || true + exit 1 + fi + docker exec ceph-demo ceph config set mon mon_allow_pool_size_one true + docker exec ceph-demo ceph osd pool create "${FDB_RADOS_TEST_POOL}" 8 8 + # The demo cluster has one OSD; its test pool must use a single replica to become active+clean. + docker exec ceph-demo ceph osd pool set "${FDB_RADOS_TEST_POOL}" size 1 --yes-i-really-mean-it + docker exec ceph-demo ceph osd pool set "${FDB_RADOS_TEST_POOL}" min_size 1 + docker exec ceph-demo ceph osd pool application enable "${FDB_RADOS_TEST_POOL}" rados + echo "Waiting for ${FDB_RADOS_TEST_POOL} placement groups to become active+clean..." + for i in $(seq 1 60); do + if docker exec ceph-demo ceph pg stat 2>/dev/null | \ + awk '$2 == "pgs:" { gsub("[,;]", "", $4); exit !($1 == $3 && $4 == "active+clean") }'; then + pool_ready=1 + break + fi + sleep 5 + done + if [ "${pool_ready:-0}" != "1" ]; then + echo "Ceph test pool did not become active+clean in time" >&2 + docker exec ceph-demo ceph status || true + docker exec ceph-demo ceph pg stat || true + exit 1 + fi + sudo chmod 0644 "${CEPH_ETC}/ceph.client.admin.keyring" + + - name: Build eckit + run: | + # eckit locates ecbuild via find_package(ecbuild ... HINTS .../../ecbuild), + # which resolves to the sibling ecbuild checkout above. + cmake -S eckit -B build/eckit -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + ${CMAKE_FLAGS} \ + -DENABLE_MPI=ON \ + -DENABLE_RADOS=ON \ + -DENABLE_RADOS_TESTS_MANAGE_POOLS=OFF + cmake --build build/eckit --parallel + cmake --install build/eckit + + - name: Build eccodes + run: | + cmake -S eccodes -B build/eccodes -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ + ${CMAKE_FLAGS} \ + -DENABLE_JPG=OFF \ + -DENABLE_FORTRAN=ON \ + -DENABLE_MEMFS=ON \ + -DENABLE_ECCODES_THREADS=ON + cmake --build build/eccodes --parallel + cmake --install build/eccodes + + - name: Build metkit + run: | + cmake -S metkit -B build/metkit -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ + ${CMAKE_FLAGS} + cmake --build build/metkit --parallel + cmake --install build/metkit + + - name: Build fdb + run: | + cmake -S fdb -B build/fdb -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ + ${CMAKE_FLAGS} \ + -DENABLE_RADOSFDB=ON \ + -DENABLE_RADOS_TESTS_MANAGE_POOLS=OFF \ + -DFDB_RADOS_TEST_POOL="${FDB_RADOS_TEST_POOL}" + cmake --build build/fdb --parallel + + - name: Run RADOS FDB tests + env: + ECKIT_RADOS_CLUSTER_CONF: /etc/ceph/ceph.conf + run: | + ctest --test-dir build/fdb -L rados --output-on-failure + + - name: Dump Ceph diagnostics on failure + if: failure() + run: | + echo "=== Ceph status ===" + docker exec ceph-demo ceph status || true + echo "=== Ceph capacity ===" + docker exec ceph-demo ceph df || true + echo "=== Test-pool objects ===" + docker exec ceph-demo rados -p "${FDB_RADOS_TEST_POOL}" ls || true + echo "=== CTest temporary files ===" + ls -la build/fdb/Testing/Temporary/ || true + echo "=== Ceph container logs ===" + docker logs ceph-demo || true diff --git a/.gitignore b/.gitignore index 48937c521..efa7b4f29 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ __pycache__/ # Rust rust/target/ rust/Cargo.lock +doc-build diff --git a/CMakeLists.txt b/CMakeLists.txt index fbda0b42c..6a88d231b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,11 +49,16 @@ ecbuild_find_package( NAME metkit VERSION 1.15 REQUIRED ) ### FDB backend in CEPH object store (using Rados) find_package( RADOS QUIET ) + ecbuild_add_option( FEATURE RADOSFDB # option defined in fdb5_config.h CONDITION eckit_HAVE_RADOS AND RADOS_FOUND DEFAULT OFF DESCRIPTION "Ceph/Rados support for FDB Store" ) +ecbuild_add_option( FEATURE RADOS_TESTS_MANAGE_POOLS + DEFAULT OFF + DESCRIPTION "Have unit tests create pools automatically rather than using an existing pool specified in the FDB_RADOS_TEST_POOL cmake variable." ) + ### FDB backend in indexed filesystem with table-of-contents, i.e. TOC ### Supports Lustre parallel filesystem stripping control diff --git a/cmake/FindRADOS.cmake b/cmake/FindRADOS.cmake index 54eb0fc54..9c1783e9a 100644 --- a/cmake/FindRADOS.cmake +++ b/cmake/FindRADOS.cmake @@ -1,49 +1,51 @@ -# (C) Copyright 2011- ECMWF. +# (C) Copyright 2026- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. - -# - Try to find Rados -# Once done this will define -# -# RADOS_FOUND - system has Armadillo -# RADOS_INCLUDE_DIRS - the Armadillo include directory -# RADOS_LIBRARIES - the Armadillo library -# RADOS_VERSION - This is set to $major.$minor.$patch (eg. 0.9.8) # -# The following paths will be searched with priority if set in CMake or env +# This module defines the following variables: +# RADOS_INCLUDE_DIRS - Where to find rados/librados.h +# RADOS_LIBRARIES - The libraries needed to use Rados +# RADOS_FOUND - True if Rados was found # -# RADOS_PATH - prefix path of the Armadillo installation -# RADOS_ROOT - Set this variable to the root installation - -# Search with priority for RADOS_PATH if given as CMake or env var - -find_path(RADOS_INCLUDE_DIR rados/librados.hpp - HINTS $ENV{RADOS_ROOT} ${RADOS_ROOT} - PATHS ${RADOS_PATH} ENV RADOS_PATH - PATH_SUFFIXES include NO_DEFAULT_PATH) - -find_path(RADOS_INCLUDE_DIR rados/librados.hpp PATH_SUFFIXES include ) - -# Search with priority for RADOS_PATH if given as CMake or env var -find_library(RADOS_LIBRARY rados - HINTS $ENV{RADOS_ROOT} ${RADOS_ROOT} - PATHS ${RADOS_PATH} ENV RADOS_PATH - PATH_SUFFIXES lib64 lib NO_DEFAULT_PATH) - -find_library( RADOS_LIBRARY rados PATH_SUFFIXES lib64 lib ) - -set( RADOS_LIBRARIES ${RADOS_LIBRARY} ) -set( RADOS_INCLUDE_DIRS ${RADOS_INCLUDE_DIR} ) - +# This module also defines the following IMPORTED target: +# Ceph::RADOS + +# Find the header path by looking for the subdirectory file +find_path(RADOS_INCLUDE_DIR + NAMES rados/librados.h + DOC "Path to Rados include directory" +) + +# Find the library +find_library(RADOS_LIBRARY + NAMES rados + DOC "Path to Rados library" +) + +# Handle the QUIETLY and REQUIRED arguments and set RADOS_FOUND to TRUE if +# all listed variables are TRUE. include(FindPackageHandleStandardArgs) - -# handle the QUIET and REQUIRED arguments and set RADOS_FOUND to TRUE -# if all listed variables are TRUE -# Note: capitalisation of the package name must be the same as in the file name -find_package_handle_standard_args(RADOS DEFAULT_MSG RADOS_LIBRARY RADOS_INCLUDE_DIR) - +find_package_handle_standard_args(RADOS + REQUIRED_VARS RADOS_LIBRARY RADOS_INCLUDE_DIR +) + +if(RADOS_FOUND) + set(RADOS_LIBRARIES ${RADOS_LIBRARY}) + set(RADOS_INCLUDE_DIRS ${RADOS_INCLUDE_DIR}) + + # Create an modern generic imported target + if(NOT TARGET Ceph::RADOS) + add_library(Ceph::RADOS UNKNOWN IMPORTED) + set_target_properties(Ceph::RADOS PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${RADOS_INCLUDE_DIRS}" + IMPORTED_LOCATION "${RADOS_LIBRARY}" + ) + endif() +endif() + +# Hide these variables from the GUI cache view mark_as_advanced(RADOS_INCLUDE_DIR RADOS_LIBRARY) diff --git a/docs/fdb/content/rados-backend.rst b/docs/fdb/content/rados-backend.rst new file mode 100644 index 000000000..895a25dfd --- /dev/null +++ b/docs/fdb/content/rados-backend.rst @@ -0,0 +1,346 @@ +=================== +FDB RADOS backend +=================== + +Overview +======== + +The RADOS backend stores an FDB database in a Ceph object store through +eckit's RADOS API. It separates the database into two kinds of persistent +state: + +* **Catalogue metadata**: database identity, schema, index references, index + entries, and axis values. +* **Field data**: the encoded field payloads archived by FDB. + +The backend is selected with the ``rados`` store and catalogue type. A +database key is mapped to one Ceph pool and one RADOS namespace. The namespace +contains both the catalogue metadata and the field objects for that database. + +.. mermaid:: + + flowchart TD + FDB[FDB API] --> C[Catalogue writer/reader] + FDB --> S[Store writer/reader] + C --> KV[Catalogue RADOS KV] + C --> IKV[Index and axis RADOS KVs] + S --> OBJ[Field RADOS objects] + KV --> NS[Ceph pool and database namespace] + IKV --> NS + OBJ --> NS + +Build-time enablement +===================== + +RADOS support is optional. FDB enables it when both eckit RADOS support and +the Ceph RADOS development library are available: + +.. code-block:: console + + cmake \ + -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON + +``HAVE_RADOSFDB`` controls whether the RADOS source files and tests are added +to the build. The generated ``fdb5_config.h`` exposes the corresponding +``fdb5_HAVE_RADOSFDB`` feature macro. RADOS tests are compiled only when the +backend is enabled. + +The separate ``RADOS_TESTS_MANAGE_POOLS`` option affects test setup only. It +allows tests to create and destroy their own pool; it does not change the +production backend. + +Placement and configuration +=========================== + +RADOS placement is configured through ``spaces[].roots[]``. The first +``spaces`` entry whose ``regex`` matches the database key is selected. The +current implementation requires exactly one root in the matching space. + +.. code-block:: yaml + + spaces: + - regex: ".*" + roots: + - pool: fdb-rados + root_namespace: fdb-root + namespace_prefix: fdb + +The root attributes have these meanings: + +* ``pool``: Ceph pool used for the database. +* ``root_namespace``: the **registry namespace**, a shared RADOS namespace + containing the ``main_kv`` registry for the space. +* ``namespace_prefix``: prefix used to derive the **database namespace**, the + RADOS namespace containing one database's catalogue and field objects. + +Unlike filesystem-backed FDB engines, the RADOS backend does not use a root +filesystem path. The generic FDB configuration model permits ``path`` in +``spaces[].roots[]``, but it is not needed for RADOS placement and is not read +by the RADOS engine. + +For a database key whose values serialize as ``11:22``, the database namespace +is ``fdb_11:22``. The namespace prefix must not contain ``_``, because the +underscore separates the prefix from the serialized database key. The +``root_namespace`` and the derived database namespace are different: the +registry namespace contains ``main_kv``, while the database namespace contains +``catalogue_kv``, index KVs, and field objects. + +The optional ``rados`` block currently provides the maximum multipart object part size: + +.. code-block:: yaml + + rados: + maxPartSize: 67108864 + +The value is expressed in bytes. A value of zero uses eckit's default behavior +for the multipart write handle. + +RADOS layout +============ + +For the example above, the catalogue is represented by:: + + rados:fdb-rados/fdb_11:22/catalogue_kv + +The ``main_kv`` object in the registry namespace ``fdb-root`` maps the database +namespace to this catalogue URI. This registry allows a database to be found +again when a catalogue is opened by its FDB key. + +The ``catalogue_kv`` object contains: + +* ``key``: serialized FDB database key. +* ``schema``: serialized schema used by the database. +* One entry per index key, whose value is the URI of the index RADOS KV. +* ``control.*`` entries for persisted control state, such as list and + retrieve visibility. + +Each index is a RADOS key/value object in the database namespace. Its omap +entries contain: + +* ``key``: serialized index key. +* Datum keys mapped to serialized timestamps and ``FieldLocation`` values. +* ``axis.`` markers and per-axis key/value objects used for axis + enumeration. + +Field payloads are RADOS objects in the same database namespace. Their names +are generated from the field key and a unique timestamp/host/process value +hashed with MD5, for example:: + + ..data + +Multipart object writes +---------------------- + +The RADOS multipart handle used by FDB is an eckit abstraction over several +ordinary RADOS objects. It is not the multipart-upload protocol of an S3 +gateway. It allows one logical FDB field object to be split into independently +stored RADOS objects when the payload is larger than the configured part size. + +For a logical object named ````, eckit uses this naming convention: + +* the first part is ````; +* subsequent parts are ``;part-1``, ``;part-2``, and so on. + +The writer keeps the current part open until it reaches ``maxPartSize``. A +write that crosses a part boundary is divided between the current part and +the next one. FDB supplies ``rados.maxPartSize`` to the writer; the value is +in bytes. When it is zero, eckit uses the Ceph cluster's maximum object size. + +On flush, eckit stores attributes on the base object describing the logical +object, including its total ``length``, number of ``parts``, and ``maxsize``. +The field location recorded by FDB refers to the logical base-object URI and +contains a byte offset and length. It does not expose the individual part +names to the catalogue. + +On read, eckit reads those attributes, opens the base object followed by its +``;part-N`` objects, and presents them as one contiguous, seekable stream. +This means a field can be retrieved normally even when its bytes span several +RADOS objects. The stored offset and length still allow FDB to retrieve only +the field range within a collocated logical object. + +The base object and all of its parts must be managed together. FDB therefore +uses eckit's ``ensureAllDestroyed()`` operation when removing a field and +ignores names containing ``;part-`` during object enumeration and full-wipe +discovery. A part must not be deleted independently, or the logical object +will be incomplete. + +Catalogue operation +=================== + +Creation and reopening +---------------------- + +When a ``RadosCatalogueWriter`` is created from an FDB key: + +#. The matching RADOS space is selected. +#. The root namespace and database namespace are opened. +#. ``main_kv`` is created if necessary. +#. A new ``catalogue_kv`` is created when the database does not yet exist. +#. The configured schema and serialized database key are stored in the + catalogue KV. +#. The catalogue URI is registered in ``main_kv`` under the database + namespace. + +When opened from a ``rados:`` URI, the database key and schema are read from +the catalogue KV. A missing ``key`` entry is reported as a database-not-found +error. + +Indexing and archiving metadata +------------------------------- + +Selecting an index creates or reopens the corresponding index KV. Archiving a +datum stores its serialized field location under the datum key and updates +axis values for newly observed values. Index enumeration reads the index +references from the catalogue KV and reconstructs the RADOS indexes. + +The backend intentionally does not require sorted index enumeration; the +``sorted`` argument to ``indexes()`` is ignored because RADOS key enumeration +is used directly. + +Catalogue features +------------------ + +Implemented catalogue behavior includes: + +* schema loading and persistence; +* index creation, selection, lookup, and enumeration; +* axis value persistence; +* hiding contents through persisted control entries; +* URI ownership and existence checks; +* catalogue-driven wipe and cleanup. + +The catalogue's purge, move, mount, and overlay operations are not +implemented. Statistics visitors and catalogue purge/move visitors are also +unavailable for this backend. + +Store operation +=============== + +Writing fields +-------------- + +``RadosStore::archive()`` obtains one generated RADOS object per FDB key and +reuses it for subsequent writes for that key during the store lifetime. It +obtains an eckit multipart write handle, writes the field bytes, and returns +an ``RadosFieldLocation`` containing the object URI, byte offset, and length. + +``flush()`` flushes all open data handles. ``close()`` closes them. The +catalogue subsequently stores the returned field locations in its index KVs. + +The generated field objects allow multiple writer instances to archive +concurrently to the same database. A single ``RadosStore`` instance is not +thread-safe: calls to ``archive``, ``flush``, and ``close`` must be serialized +by the caller. + +Reading fields +-------------- + +A field location points directly to a RADOS object and byte range. The store's +retrieve path returns the field's data handle, allowing FDB to read the stored +payload using the location recorded in the index. + +Store URIs use the form:: + + rados:/ + +Field object URIs add the object name as a third component. The backend checks +the URI scheme, pool, and database namespace before treating a URI as +belonging to a store. + +Store features +-------------- + +Implemented store behavior includes: + +* archive and retrieve; +* flush and close; +* object and namespace existence checks; +* listing collocated field objects; +* removing individual objects or a database namespace; +* catalogue-aware and full wipes; +* detection and removal of unrecognised objects during a full wipe; +* statistics through the normal FDB store interfaces where supported by the + caller. + +The store does not expose auxiliary URIs; ``getAuxiliaryURIs()`` returns an +empty set of results. + +Use of RADOS features +===================== + +The backend relies on the following Ceph/eckit RADOS features: + +* **Pools** provide the physical Ceph storage boundary selected by FDB space + placement. +* **Namespaces** isolate each FDB database within a pool. Multiple FDB + spaces may share a pool if their root namespaces and namespace prefixes are + distinct. +* **RADOS objects** hold field payloads and provide object existence, deletion, + enumeration, and URI addressing. +* **Object-map key/value entries** provide compact catalogue and index metadata + without creating a separate object for every metadata property. +* **Multipart writes** allow large field payloads to be written in parts, + controlled by ``rados.maxPartSize``. +* **RADOS object listing** supports collocated-data discovery and detection of + unrecognised objects during wipes. +* **URI addressing** permits stores and catalogues to be reopened from + persisted RADOS locations. + +The eckit RADOS API also provides asynchronous handles and range-read handles, +but the current FDB implementation uses synchronous data-handle operations for +retrieval and multipart write handles for archival. It does not currently use +the asynchronous or range-read APIs directly. + +Wipe and cleanup safety +======================= + +A catalogue-driven wipe first determines which index and data URIs are +included and which are safe. It removes only the selected catalogue entries, +index/axis KVs, and field objects. When the complete database is selected, the +database namespace and its root registry entry are removed after the contents +have been removed. + +A full store wipe scans only the database namespace. Objects named as +multipart parts are handled with their main object and are not independently +treated as data records. If the namespace also contains a catalogue, the +catalogue owns the namespace cleanup and the store avoids deleting it during +an unsafe full wipe. + +Limitations and operational requirements +========================================= + +* Ceph and eckit RADOS support must be present at configure time. +* The target pool must exist and the configured Ceph identity must have + permissions to access the pool and namespaces. +* RADOS placement requires at least one matching ``spaces[]`` entry and + exactly one root in that entry. +* Operations on one ``RadosStore`` instance must be serialized by the caller. +* Separate writer instances can archive concurrently, but concurrent writers + targeting the same index entry use last-write-wins semantics for that entry. +* Catalogue-side purge, move, mount, and overlay operations are not + implemented. +* The RADOS backend does not persist masking metadata; wipe removes entries + directly. +* The ``sorted`` index enumeration request is ignored. +* Auxiliary store URIs are not provided. +* Runtime tests require a reachable Ceph cluster and an existing test pool + unless ``RADOS_TESTS_MANAGE_POOLS`` is enabled. + +Minimal test setup +================== + +With an existing Ceph pool: + +.. code-block:: console + + cmake \ + -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON \ + -DFDB_RADOS_TEST_POOL=fdb_test + + ctest -R 'fdb_test_rados_(store|catalogue)' + +The RADOS test environment must also provide the Ceph configuration and +credentials expected by eckit, for example through the standard Ceph +configuration directory and the configured RADOS cluster/user settings. diff --git a/docs/fdb/index.rst b/docs/fdb/index.rst index d18c5e0cc..3631353d7 100644 --- a/docs/fdb/index.rst +++ b/docs/fdb/index.rst @@ -22,6 +22,7 @@ the MARS Archive. content/mars content/config-schema content/environment-variables + content/rados-backend cli_tools/index content/api content/license diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 9e2a16560..7b69c239d 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -351,6 +351,25 @@ if( HAVE_RADOSFDB ) rados/RadosFieldLocation.h rados/RadosStore.cc rados/RadosStore.h + rados/RadosCommon.cc + rados/RadosCommon.h + rados/RadosCleanup.h + rados/RadosCatalogue.cc + rados/RadosCatalogue.h + rados/RadosCatalogueWriter.cc + rados/RadosCatalogueWriter.h + rados/RadosCatalogueReader.cc + rados/RadosCatalogueReader.h + rados/RadosIndex.cc + rados/RadosIndex.h + rados/RadosIndexLocation.cc + rados/RadosIndexLocation.h + rados/RadosLazyFieldLocation.cc + rados/RadosLazyFieldLocation.h + rados/RadosEngine.cc + rados/RadosEngine.h + rados/RadosStats.cc + rados/RadosStats.h ) endif() diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index fd2dcedff..123daa43f 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -1,18 +1,18 @@ #ifndef fdb5_fdb5_config_h #define fdb5_fdb5_config_h -#include "fdb5_ecbuild_config.h" // generated by ecbuild_generate_config_headers() - -#include "fdb5_version.h" // generated by ecbuild_generate_config_headers() +#include "fdb5_ecbuild_config.h" // generated by ecbuild_generate_config_headers() +#include "fdb5_version.h" // generated by ecbuild_generate_config_headers() // features #cmakedefine fdb5_HAVE_LUSTRE #cmakedefine fdb5_HAVE_RADOSFDB +#cmakedefine fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS #cmakedefine fdb5_HAVE_TOCFDB #cmakedefine fdb5_HAVE_DUMMY_DAOS #cmakedefine fdb5_HAVE_DAOSFDB #cmakedefine fdb5_HAVE_DAOS_ADMIN #cmakedefine01 fdb5_HAVE_GRIB -#endif // fdb5_fdb5_config_h +#endif // fdb5_fdb5_config_h diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README new file mode 100644 index 000000000..f7ed7b9b1 --- /dev/null +++ b/src/fdb5/rados/README @@ -0,0 +1,85 @@ +Running RadosStore unit tests against Ceph on Docker on mac: +============================================================ + +Supported RADOS backend scope: +============================== + +The backend supports archive, retrieve, list, wipe, statistics, hide, and reopening catalogues by +URI. Operations on a single RadosStore instance must be serialised by the caller; separate writer +instances can archive concurrently to the same database. Catalogue-side purge, move, mount, and +overlay operations are not implemented. + +RADOS placement is configured on `spaces[].roots[]`; legacy `rados.pool`, `rados.root_namespace`, +and `rados.namespace_prefix` values are not used. The first matching space must have exactly one +root with all of these required attributes: + +```yaml +spaces: +- regex: ".*" + roots: + - path: /local/fdb-root + pool: fdb-rados + root_namespace: fdb-root + namespace_prefix: fdb +``` + +- `pool` is the Ceph pool for the selected space. +- `root_namespace` holds the space's `main_kv` registry, which maps database namespaces to catalogue URIs. +- `namespace_prefix` forms each database namespace: a DB key with values `11:22` is stored in + `fdb_11:22`. It must not contain `_`, the prefix/key separator. + +The catalogue KV is `rados:fdb-rados/fdb_11:22/catalogue_kv`; index and data objects share the +`fdb_11:22` namespace. Multiple spaces may share a pool when their root namespaces and namespace +prefixes are distinct. + +RADOS tests require eckit to be built with RADOS support. + +git clone https://github.com/datenkollektiv/ceph-playground.git +cd ceph-playground +sed -i '' 's#volumes:#volumes:\n - < PATH TO YOUR LOCAL FDB BUNDLE SOURCE >:/root/git/fdb-bundle#g' docker-compose.yaml +sed -i '' 's#volumes:#volumes:\n - < PATH TO YOUR LOCAL CEPH SOURCE >/src/include/rados:/usr/include/rados#g' docker-compose.yaml +sed -i '' 's#5000:5000#7777:5000#g' docker-compose.yaml + +docker-compose down +rm -rf docker/ceph/etc/* +rm -rf docker/ceph/var/* +docker-compose up -d + +docker exec -it ceph-playground_ceph_1 /bin/bash + +# --- + +sed -i -e "s|mirrorlist=|#mirrorlist=|g" -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Linux-* + +yum install -y gcc gcc-c++ gcc-gfortran make cmake openssl openssl-devel git vim libuuid-devel +yum update -y libarchive + +ln -s /usr/lib64/librados.so.2 /usr/lib64/librados.so + +cd + +mkdir .ceph +cat /etc/ceph/ceph.conf | grep -e "global" -e "mon host" > .ceph/ceph.conf +ceph config set mon mon_allow_pool_delete true + +git clone https://github.com/ecmwf/ecbuild.git +export PATH=$HOME/ecbuild/bin:$PATH + +mkdir build +cd build +src_dir=$HOME/git/fdb-bundle +build_dir=$HOME/build/fdb-bundle +mkdir -p $build_dir +cd $build_dir +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON -DENABLE_RADOSFDB=ON +cmake --build . + +ctest -R rados_store + + + +cmake options: +============== + +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc new file mode 100644 index 000000000..f0d81b4a6 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -0,0 +1,365 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosCatalogue.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/DatabaseNotFoundException.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/database/WipeState.h" +#include "fdb5/rados/RadosCommon.h" +#include "fdb5/rados/RadosIndex.h" +#include "fdb5/rules/Rule.h" +#include "fdb5/rules/Schema.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosObject.h" +#include "eckit/log/Log.h" +#include "eckit/log/Timer.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Tokenizer.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : + CatalogueImpl(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) {} + +RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, + const fdb5::Config& config) : + CatalogueImpl(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { + try { + dbKey_ = read_db_key(*db_kv_); + } + catch (eckit::RadosEntityNotFoundException& e) { + throw fdb5::DatabaseNotFoundException(std::string("RadosCatalogue database not found ") + "(pool: '" + pool_ + + "', namespace: '" + db_namespace_ + "')"); + } +} + +bool RadosCatalogue::exists() const { + return db_kv_->exists(); +} + +eckit::URI RadosCatalogue::uri() const { + return db_kv_->nspace().uri(); +} + +const Schema& RadosCatalogue::schema() const { + return schema_; +} + +const Rule& RadosCatalogue::rule() const { + ASSERT(rule_); + return *rule_; +} + +void RadosCatalogue::loadSchema() { + + eckit::Timer timer("RadosCatalogue::loadSchema()", eckit::Log::debug()); + + std::vector data; + db_kv_->getMemoryStream(data, "schema", "DB Key-Value"); + + std::istringstream stream{std::string(data.begin(), data.end())}; + schema_.load(stream); + + rule_ = &schema_.matchingRule(dbKey_); +} + +std::vector RadosCatalogue::indexes(bool) const { + + // `sorted` is intentionally ignored; the RADOS backend does not need ordered enumeration. + std::vector res; + + for (const auto& key : db_kv_->keys()) { + + if (key == "schema" || key == "key" || key.rfind("control.", 0) == 0) { + continue; + } + + std::vector v; + auto m = db_kv_->getMemoryStream(v, key, "DB kv"); + + eckit::URI uri(std::string(v.begin(), v.end())); + + eckit::RadosKeyValue index_kv{uri}; + std::optional index_key; + try { + std::vector data; + eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); + index_key.emplace(ms); + } + catch (eckit::RadosEntityNotFoundException& e) { + continue; + } + + res.push_back(Index(new fdb5::RadosIndex(index_key.value(), index_kv, false))); + } + + return res; +} + +std::string RadosCatalogue::type() const { + + return RadosCatalogue::catalogueTypeName(); +} + +bool RadosCatalogue::uriBelongs(const eckit::URI& uri) const { + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + + return (uri.scheme() == type()) && (n >= 2) && (parts[0] == pool_) && (parts[1] == db_namespace_); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CatalogueWipeState RadosCatalogue::wipeInit() const { + return {dbKey_, config()}; +} + +void RadosCatalogue::maskIndexEntries(const std::set& indexes) const { + + for (const auto& index : indexes) { + std::string key = index.key().valuesToString(); + if (db_kv_->has(key)) { + db_kv_->remove(key); + } + } +} + +bool RadosCatalogue::markIndexForWipe(const Index& index, bool include, CatalogueWipeState& wipeState) const { + + eckit::RadosKeyValue index_kv{index.location().uri()}; + + // A cross fdb-mount must never delete another DB's index/axis KVs. + if (index_kv.nspace().pool().name() != pool_ || index_kv.nspace().name() != db_namespace_) { + include = false; + } + + std::vector axis_uris; + try { + std::vector axes_data; + index_kv.getMemoryStream(axes_data, "axes", "index kv"); + std::vector axis_names; + eckit::Tokenizer parse(","); + parse(std::string(axes_data.begin(), axes_data.end()), axis_names); + const std::string idx_key = index.key().valuesToString(); + for (const auto& axis : axis_names) { + axis_uris.push_back( + eckit::RadosKeyValue{index_kv.nspace().pool().name(), index_kv.nspace().name(), idx_key + "." + axis} + .uri()); + } + } + catch (const eckit::RadosEntityNotFoundException& e) { + LOG_DEBUG_LIB(LibFdb5) << "RadosCatalogue::markIndexForWipe: axes lookup missing for index " << index.key() + << " (assuming stale index kv): " << e.what() << std::endl; + } + + const eckit::URI index_uri = index.location().uri(); + + if (include) { + wipeState.markForMasking(index); + wipeState.markForDeletion(WipeElementType::CATALOGUE_INDEX, index_uri); + for (const auto& uri : axis_uris) { + wipeState.markForDeletion(WipeElementType::CATALOGUE_INDEX, uri); + } + } + else { + wipeState.markAsSafe({index_uri}); + for (const auto& uri : axis_uris) { + wipeState.markAsSafe({uri}); + } + } + + return include; +} + +void RadosCatalogue::finaliseWipeState(CatalogueWipeState& wipeState) const { + + const eckit::URI db_kv_uri = db_kv_->uri(); + + const bool wipeAll = wipeState.safeURIs().empty(); + if (wipeAll) { + wipeState.markForDeletion(WipeElementType::CATALOGUE, db_kv_uri); + } + else { + wipeState.markAsSafe({db_kv_uri}); + return; + } + + eckit::RadosNamespace db{pool_, db_namespace_}; + if (!db.exists()) { + return; + } + + for (const auto& obj : db.listObjects()) { + if (obj.name().find(";part-") != std::string::npos) { + continue; + } + const eckit::URI uri = obj.uri(); + if (!wipeState.isMarkedForDeletion(uri)) { + wipeState.insertUnrecognised(uri); + } + } +} + +namespace { + +void remove_catalogue_uri(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) { + + eckit::RadosObject obj{uri}; + logVerbose << "destroy Rados object: "; + logAlways << obj.str() << std::endl; + if (doit) { + obj.ensureAllDestroyed(); + } +} + +} // namespace + +bool RadosCatalogue::doWipeUnknowns(const std::set& unknownURIs) const { + + for (const auto& uri : unknownURIs) { + if (eckit::RadosObject{uri}.exists()) { + remove_catalogue_uri(uri, std::cout, std::cout, true); + } + } + return true; +} + +bool RadosCatalogue::doWipeURIs(const CatalogueWipeState& wipeState) const { + + const bool wipeAll = wipeState.safeURIs().empty(); + + for (const auto& [type, uris] : wipeState.deleteMap()) { + for (const auto& uri : uris) { + remove_catalogue_uri(uri, std::cout, std::cout, true); + } + } + + if (wipeAll) { + cleanupEmptyDatabase_ = true; + } + + return true; +} + +void RadosCatalogue::doWipeEmptyDatabase() const { + + if (!cleanupEmptyDatabase_) { + return; + } + + eckit::RadosNamespace db{pool_, db_namespace_}; + if (db.exists()) { + db.destroy(); + } + + if (root_kv_ && root_kv_->exists() && root_kv_->has(db_namespace_)) { + root_kv_->remove(db_namespace_); + } + + cleanupEmptyDatabase_ = false; +} + +bool RadosCatalogue::doUnsafeFullWipe() const { + + eckit::RadosNamespace db{pool_, db_namespace_}; + if (db.exists()) { + db.destroy(); + } + + if (root_kv_ && root_kv_->exists() && root_kv_->has(db_namespace_)) { + root_kv_->remove(db_namespace_); + } + + return true; +} + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +// Reserved KV entry name for a given control identifier; must be filtered out of index enumeration. +std::string control_kv_key(ControlIdentifier id) { + switch (id) { + case ControlIdentifier::List: + return "control.list"; + case ControlIdentifier::Retrieve: + return "control.retrieve"; + case ControlIdentifier::Archive: + return "control.archive"; + case ControlIdentifier::Wipe: + return "control.wipe"; + case ControlIdentifier::UniqueRoot: + return "control.unique_root"; + default: + return ""; + } +} + +} // namespace + +void RadosCatalogue::control(const ControlAction& action, const ControlIdentifiers& identifiers) const { + + for (ControlIdentifier id : identifiers) { + const std::string key = control_kv_key(id); + if (key.empty()) { + continue; + } + switch (action) { + case ControlAction::Disable: { + const char flag = '1'; + db_kv_->put(key, &flag, 1); + break; + } + case ControlAction::Enable: + if (db_kv_->has(key)) { + db_kv_->remove(key); + } + break; + default: + eckit::Log::warning() << "RadosCatalogue::control: unexpected action " << static_cast(action) + << std::endl; + } + } +} + +bool RadosCatalogue::enabled(const ControlIdentifier& controlIdentifier) const { + const std::string key = control_kv_key(controlIdentifier); + if (key.empty()) { + return true; + } + return !db_kv_->has(key); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h new file mode 100644 index 000000000..87d34faf5 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogue.h @@ -0,0 +1,118 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/api/helpers/MoveIterator.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/MoveVisitor.h" +#include "fdb5/database/PurgeVisitor.h" +#include "fdb5/database/StatsReportVisitor.h" +#include "fdb5/rados/RadosCommon.h" +#include "fdb5/rules/Schema.h" + +#include "eckit/config/Configuration.h" +#include "eckit/container/Queue.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Offset.h" + +#include +#include +#include +#include +#include + +namespace fdb5 { + +class Rule; +class RuleDatabase; +class CatalogueWipeState; + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on Rados + +class RadosCatalogue : public CatalogueImpl, public RadosCommon { + +public: // methods + + RadosCatalogue(const Key& key, const fdb5::Config& config); + RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config); + + static const char* catalogueTypeName() { return "rados"; } + + eckit::URI uri() const override; + const Key& indexKey() const override { return currentIndexKey_; } + + std::string type() const override; + + void checkUID() const override { /* nothing to do */ } + bool exists() const override; + void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override { + out << "RadosCatalogue(" << type() << ":" << dbKey_ << ")"; + } + const Schema& schema() const override; + + StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; + PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }; + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, + eckit::Queue& queue) const override { + NOTIMP; + }; + + void loadSchema() override; + + std::vector indexes(bool sorted = false) const override; + + // No masking metadata is persisted for this backend; wipe removes entries directly, so there is + // nothing to enumerate here. + void allMasked(std::set>& metadata, + std::set& data) const override {} + + // Control access properties of the DB. Persisted as per-identifier reserved KV entries (`control.list`, etc.) + // in the catalogue KV; absence of the entry means the identifier is enabled. + void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override; + bool enabled(const ControlIdentifier& controlIdentifier) const override; + + const Rule& rule() const override; + + bool uriBelongs(const eckit::URI& uri) const override; + + void maskIndexEntries(const std::set& indexes) const override; + + // Wipe-related methods + CatalogueWipeState wipeInit() const override; + bool markIndexForWipe(const Index& index, bool include, CatalogueWipeState& wipeState) const override; + void finaliseWipeState(CatalogueWipeState& wipeState) const override; + bool doWipeUnknowns(const std::set& unknownURIs) const override; + bool doWipeURIs(const CatalogueWipeState& wipeState) const override; + void doWipeEmptyDatabase() const override; + bool doUnsafeFullWipe() const override; + +protected: // members + + Key currentIndexKey_; + +private: // members + + Schema schema_; + const RuleDatabase* rule_{nullptr}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc new file mode 100644 index 000000000..4805631e7 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -0,0 +1,134 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosCatalogueReader.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/DbStats.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCatalogue.h" +#include "fdb5/rados/RadosIndex.h" +#include "fdb5/rados/RadosStats.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/Log.h" + +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosCatalogueReader::RadosCatalogueReader(const Key& key, const Config& config) : RadosCatalogue(key, config) {} + +RadosCatalogueReader::RadosCatalogueReader(const eckit::URI& uri, const Config& config) : + RadosCatalogue(uri, ControlIdentifiers{}, config) {} + +bool RadosCatalogueReader::selectIndex(const Key& key) { + + if (currentIndexKey_ == key) { + return true; + } + + if (indexes_.find(key) == indexes_.end()) { + try { + std::vector data; + db_kv_->getMemoryStream(data, key.valuesToString(), "DB kv"); + eckit::URI uri{std::string{data.begin(), data.end()}}; + eckit::RadosKeyValue index_kv{uri}; + indexes_[key] = Index(new RadosIndex(key, index_kv, true)); + } + catch (eckit::RadosEntityNotFoundException& e) { + return false; + } + } + + currentIndexKey_ = key; + current_ = indexes_[key]; + + return true; +} + +void RadosCatalogueReader::deselectIndex() { + current_ = Index(); + currentIndexKey_ = Key(); +} + +bool RadosCatalogueReader::open() { + if (!RadosCatalogue::exists()) { + return false; + } + RadosCatalogue::loadSchema(); + return true; +} + +DbStats RadosCatalogueReader::stats() const { + + auto* content = new RadosDbStats(); + content->dbCount_ = 1; + + for (const auto& indexEntry : indexes(false)) { + content->indexCount_++; + const auto* radosIndex = dynamic_cast(indexEntry.content()); + ASSERT(radosIndex); + for (const auto& key : radosIndex->idx_kv().keys()) { + if (key == "axes" || key == "key" || key.rfind("axis.", 0) == 0) { + continue; + } + content->fieldCount_++; + } + } + + return {content}; +} + +std::optional RadosCatalogueReader::computeAxis(const std::string& keyword) const { + + Axis s; + + bool found = false; + if (current_.axes().has(keyword)) { + found = true; + s.merge(current_.axes().values(keyword)); + } + + if (found) { + return s; + } + return std::nullopt; +} + +bool RadosCatalogueReader::retrieve(const Key& key, Field& field) const { + + eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; + eckit::Log::debug() << "Scanning index " << current_.location() << std::endl; + + if (!current_.mayContain(key)) { + return false; + } + + return current_.get(key, Key(), field); +} + +static CatalogueReaderBuilder builder("rados"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueReader.h b/src/fdb5/rados/RadosCatalogueReader.h new file mode 100644 index 000000000..dcac4f97a --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueReader.h @@ -0,0 +1,75 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/DbStats.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCatalogue.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" + +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on Rados + +class RadosCatalogueReader : public RadosCatalogue, public CatalogueReader { + +public: // methods + + RadosCatalogueReader(const Key& key, const fdb5::Config& config); + RadosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config); + + DbStats stats() const override; + + bool selectIndex(const Key& key) override; + void deselectIndex() override; + + bool open() override; + void flush(size_t archivedFields) override {} + void clean() override {} + void close() override {} + + bool retrieve(const Key& key, Field& field) const override; + + void print(std::ostream& out) const override { out << "RadosCatalogueReader(" << uri() << ")"; } + +private: // methods + + std::optional computeAxis(const std::string& keyword) const override; + +private: // types + + using IndexStore = std::map; + +private: // members + + IndexStore indexes_; + Index current_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc new file mode 100644 index 000000000..129c71370 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -0,0 +1,215 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "fdb5/rados/RadosCatalogueWriter.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/IndexAxis.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCatalogue.h" +#include "fdb5/rados/RadosCleanup.h" +#include "fdb5/rados/RadosIndex.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" +#include "eckit/io/FileHandle.h" +#include "eckit/io/Length.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/log/Log.h" +#include "eckit/serialisation/HandleStream.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& config) : RadosCatalogue(key, config) { + + std::string db_name = db_namespace_; + ASSERT(root_kv_->nspace().pool().exists()); + + root_kv_->ensureCreated(); + if (!root_kv_->has(db_name)) { + + db_kv_->ensureCreated(); + + eckit::Log::debug() << "Copy schema from " << config_.schemaPath() << " to " + << db_kv_->uri().asString() << " at key 'schema'." << std::endl; + + eckit::FileHandle in(config_.schemaPath()); + std::vector data; + data.resize(in.size()); + { + eckit::AutoClose ac{in}; + in.openForRead(); + in.read(&data[0], in.size()); + } + db_kv_->put("schema", &data[0], data.size()); + + eckit::MemoryHandle h{(size_t)PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + hs << dbKey_; + } + + db_kv_->put("key", h.data(), hs.bytesWritten()); + + std::string nstr = db_kv_->uri().asString(); + root_kv_->put(db_name, nstr.data(), nstr.length()); + } + + RadosCatalogue::loadSchema(); +} + +RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config) : + RadosCatalogue(uri, ControlIdentifiers{}, config) { + RadosCatalogue::loadSchema(); +} + +RadosCatalogueWriter::~RadosCatalogueWriter() { + std::exception_ptr ignored; + best_effort(ignored, "~RadosCatalogueWriter::clean", [&] { clean(); }); + best_effort(ignored, "~RadosCatalogueWriter::close", [&] { close(); }); +} + +bool RadosCatalogueWriter::createIndex(const Key& /* idxKey */, size_t /* datumKeySize */) { + return true; +} + +void RadosCatalogueWriter::hideContents() { + control(ControlAction::Disable, ControlIdentifier::List | ControlIdentifier::Retrieve); +} + +bool RadosCatalogueWriter::selectIndex(const Key& key) { + + currentIndexKey_ = key; + + if (indexes_.find(key) == indexes_.end()) { + + try { + std::vector data; + db_kv_->getMemoryStream(data, key.valuesToString(), "DB kv"); + + indexes_[key] = Index(new fdb5::RadosIndex( + key, eckit::RadosKeyValue{eckit::URI{std::string{data.begin(), data.end()}}}, false)); + } + catch (eckit::RadosEntityNotFoundException& e) { + + indexes_[key] = Index(new fdb5::RadosIndex(key, eckit::RadosNamespace{pool_, db_namespace_})); + + std::string nstr{indexes_[key].location().uri().asString()}; + db_kv_->put(key.valuesToString(), nstr.data(), nstr.length()); + } + } + + current_ = indexes_[key]; + + return true; +} + +void RadosCatalogueWriter::deselectIndex() { + current_ = Index(); + currentIndexKey_ = Key(); +} + +void RadosCatalogueWriter::clean() { + flush(0); + deselectIndex(); +} + +void RadosCatalogueWriter::close() { + closeIndexes(); +} + +const Index& RadosCatalogueWriter::currentIndex() { + if (current_.null()) { + ASSERT(!currentIndexKey_.empty()); + selectIndex(currentIndexKey_); + } + return current_; +} + +void RadosCatalogueWriter::archive(const Key& /* idxKey */, const Key& datumKey, + std::shared_ptr fieldLocation) { + + if (current_.null()) { + ASSERT(!currentIndexKey_.empty()); + selectIndex(currentIndexKey_); + } + + Field field(std::move(fieldLocation), currentIndex().timestamp()); + + const_cast(current_.axes()).sort(); + + std::vector axesToExpand; + std::vector valuesToAdd; + + for (const auto& [keyword, value] : datumKey) { + + if (value.length() == 0) { + continue; + } + + const auto& axis_set = current_.axes().values(keyword); + + if (!axis_set.contains(value)) { + + axesToExpand.push_back(keyword); + valuesToAdd.push_back(value); + } + } + + current_.put(datumKey, field); + + auto* radosIndex = dynamic_cast(current_.content()); + ASSERT(radosIndex); + + while (!axesToExpand.empty()) { + radosIndex->putAxisValue(axesToExpand.back(), valuesToAdd.back()); + axesToExpand.pop_back(); + valuesToAdd.pop_back(); + } +} + +void RadosCatalogueWriter::flush(size_t /* archivedFields */) { + if (!current_.null()) { + current_ = Index(); + } +} + +void RadosCatalogueWriter::closeIndexes() { + indexes_.clear(); +} + +static fdb5::CatalogueWriterBuilder builder("rados"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h new file mode 100644 index 000000000..304e1124e --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -0,0 +1,94 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCatalogue.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" + +#include +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB writer that implements the FDB on Rados. +/// Not thread-safe per instance; separate writer instances may archive to the same database. + +class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { + +public: // methods + + RadosCatalogueWriter(const Key& key, const fdb5::Config& config); + RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config); + ~RadosCatalogueWriter() override; + + void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; + + void reconsolidate() override { NOTIMP; } + + void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { + NOTIMP; + }; + + void hideContents() override; + + const Index& currentIndex() override; + +protected: // methods + + bool selectIndex(const Key& key) override; + bool createIndex(const Key& idxKey, size_t datumKeySize) override; + void deselectIndex() override; + + bool open() override { NOTIMP; } + void flush(size_t archivedFields) override; + void clean() override; + void close() override; + + void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; + + void print(std::ostream& out) const override { out << "RadosCatalogueWriter(" << uri() << ")"; } + +private: // methods + + void closeIndexes(); + +private: // types + + using IndexStore = std::map; + +private: // members + + IndexStore indexes_; + + Index current_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCleanup.h b/src/fdb5/rados/RadosCleanup.h new file mode 100644 index 000000000..7e1fcc5e6 --- /dev/null +++ b/src/fdb5/rados/RadosCleanup.h @@ -0,0 +1,47 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Metin Cakircali +/// @date Aug 2026 + +#pragma once + +#include "eckit/log/Log.h" + +#include +#include +#include + +namespace fdb5 { + +// Runs `op`; on failure logs via Log::error and records the first exception into `first`. +// Intended for destructor-safe cleanup paths where all steps must be attempted. +template +void best_effort(std::exception_ptr& first, const char* context, Op&& op) { + try { + std::forward(op)(); + } + catch (...) { + if (!first) { + first = std::current_exception(); + } + try { + throw; + } + catch (const std::exception& e) { + eckit::Log::error() << context << ": " << e.what() << std::endl; + } + catch (...) { + eckit::Log::error() << context << ": unknown exception" << std::endl; + } + } +} + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc new file mode 100644 index 000000000..1d979a5f1 --- /dev/null +++ b/src/fdb5/rados/RadosCommon.cc @@ -0,0 +1,155 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosCommon.h" + +#include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Regex.h" +#include "eckit/utils/Tokenizer.h" + +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +RadosSpace space_from_root(const eckit::LocalConfiguration& root) { + RadosSpace space{root.getString("pool"), root.getString("root_namespace"), root.getString("namespace_prefix")}; + if (space.namespacePrefix.find('_') != std::string::npos) { + throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + space.namespacePrefix + "'", + Here()); + } + return space; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +fdb5::Key read_db_key(const eckit::RadosKeyValue& db_kv) { + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + return fdb5::Key(ms); +} + +std::string RadosSpace::databaseNamespace(const Key& key) const { + return namespacePrefix + "_" + key.valuesToString(); +} + +std::vector rados_spaces(const Config& config) { + if (!config.has("spaces")) { + throw eckit::UserError("RADOS placement requires at least one spaces[] entry", Here()); + } + + std::vector spaces; + for (const auto& space : config.getSubConfigurations("spaces")) { + if (!space.has("roots")) { + throw eckit::UserError("RADOS placement requires roots[] in every spaces[] entry", Here()); + } + for (const auto& root : space.getSubConfigurations("roots")) { + spaces.emplace_back(space_from_root(root)); + } + } + return spaces; +} + +RadosSpace rados_space(const Config& config, const Key& key) { + if (!config.has("spaces")) { + throw eckit::UserError("RADOS placement requires at least one spaces[] entry", Here()); + } + + const std::string keyString = key.valuesToString(); + for (const auto& space : config.getSubConfigurations("spaces")) { + if (!eckit::Regex{space.getString("regex", ".*")}.match(keyString)) { + continue; + } + if (!space.has("roots")) { + throw eckit::UserError("RADOS placement requires roots[] in matching spaces[] entry", Here()); + } + const auto roots = space.getSubConfigurations("roots"); + if (roots.size() != 1) { + throw eckit::UserError("RADOS placement requires exactly one root in matching spaces[] entry", Here()); + } + return space_from_root(roots.front()); + } + + throw eckit::UserError("No RADOS placement matches database key " + keyString, Here()); +} + +RadosSpace rados_space(const Config& config, const eckit::URI& uri) { + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + ASSERT(parts.size() == 2 || parts.size() == 3); + + for (const auto& space : rados_spaces(config)) { + if (space.pool == parts[0] && parts[1].rfind(space.namespacePrefix + "_", 0) == 0) { + return space; + } + } + + throw eckit::UserError("No RADOS placement matches URI " + uri.asString(), Here()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +RadosCommon::RadosCommon(const Config& config, const std::string& component, const Key& key) { + + std::vector valid{"catalogue", "store"}; + ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); + + const RadosSpace space = rados_space(config, key); + pool_ = space.pool; + db_namespace_ = space.databaseNamespace(key); + readConfig(config, component); + + root_kv_.emplace(pool_, space.rootNamespace, "main_kv"); + db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); +} + +RadosCommon::RadosCommon(const Config& config, const std::string& component, const eckit::URI& uri) { + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + ASSERT(parts.size() == 2 || parts.size() == 3); + + const RadosSpace space = rados_space(config, uri); + pool_ = parts[0]; + db_namespace_ = parts[1]; + readConfig(config, component); + + root_kv_.emplace(pool_, space.rootNamespace, "main_kv"); + db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); +} + +void RadosCommon::readConfig(const Config& config, const std::string& component) { + + eckit::LocalConfiguration c{}; + + if (config.has("rados")) { + c = config.getSubConfiguration("rados"); + } + + maxPartSize_ = c.getInt("maxPartSize", 0); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h new file mode 100644 index 000000000..8b756f557 --- /dev/null +++ b/src/fdb5/rados/RadosCommon.h @@ -0,0 +1,68 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2024 + +#pragma once + +#include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" + +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/rados/RadosKeyValue.h" + +#include +#include +#include + +namespace fdb5 { + +struct RadosSpace { + std::string pool; + std::string rootNamespace; + std::string namespacePrefix; + + std::string databaseNamespace(const Key& key) const; +}; + +// Reads the persisted `key` entry from a RADOS DB KV and deserialises it into an fdb5::Key. +// Throws eckit::RadosEntityNotFoundException if the DB KV or the `key` entry is missing. +fdb5::Key read_db_key(const eckit::RadosKeyValue& db_kv); + +// RADOS space is selected from the sole root of the first matching `spaces[]` entry. +RadosSpace rados_space(const Config&, const Key&); +RadosSpace rados_space(const Config&, const eckit::URI&); +std::vector rados_spaces(const Config&); + +class RadosCommon { + +public: // methods + + RadosCommon(const Config&, const std::string& component, const Key&); + RadosCommon(const Config&, const std::string& component, const eckit::URI&); + +private: // methods + + void readConfig(const Config& config, const std::string& component); + +protected: // members + + std::string pool_; + std::string db_namespace_; + + std::optional root_kv_; + std::optional db_kv_; + + eckit::Length maxPartSize_; +}; + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc new file mode 100644 index 000000000..0d1c8e743 --- /dev/null +++ b/src/fdb5/rados/RadosEngine.cc @@ -0,0 +1,118 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "fdb5/rados/RadosEngine.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/database/Engine.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCommon.h" + +#include "metkit/mars/MarsRequest.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/Log.h" +#include "eckit/utils/Tokenizer.h" + +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +std::string RadosEngine::name() const { + return RadosEngine::typeName(); +} + +eckit::URI RadosEngine::location(const Key& key, const Config& config) const { + const RadosSpace space = rados_space(config, key); + return eckit::RadosKeyValue{space.pool, space.databaseNamespace(key), "catalogue_kv"}.uri(); +} + +bool RadosEngine::canHandle(const eckit::URI& uri, const Config&) const { + + if (uri.scheme() != typeName()) { + return false; + } + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + if (parts.size() != 2 && parts.size() != 3) { + return false; + } + + try { + return eckit::RadosKeyValue{parts[0], parts[1], "catalogue_kv"}.exists(); + } + catch (const eckit::Exception& e) { + eckit::Log::debug() << "RadosEngine::canHandle: exception checking URI " << uri << ": " << e.what() + << std::endl; + return false; + } +} + +std::vector RadosEngine::visitableLocations(const std::function& matches, + const Config& config) const { + + std::vector res{}; + + for (const auto& space : rados_spaces(config)) { + eckit::RadosKeyValue rootKv{space.pool, space.rootNamespace, "main_kv"}; + if (!rootKv.exists()) { + continue; + } + for (const auto& key : rootKv.keys()) { + try { + + std::vector val; + rootKv.getMemoryStream(val, key, "root kv"); + + eckit::URI uri(std::string(val.begin(), val.end())); + ASSERT(uri.scheme() == typeName()); + + eckit::RadosKeyValue db_kv{uri}; + fdb5::Key db_key = read_db_key(db_kv); + + if (matches(db_key)) { + eckit::Log::debug() + << " found match with " << rootKv.uri() << " at key " << key << std::endl; + res.push_back(uri); + } + } + catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << key << " from " << rootKv.uri() << std::endl; + eckit::Log::error() << e.what() << std::endl; + } + } + } + + return res; +} + +std::vector RadosEngine::visitableLocations(const Key& key, const Config& config) const { + return visitableLocations([&key](const fdb5::Key& dbKey) { return dbKey.match(key); }, config); +} + +std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest& request, + const Config& config) const { + return visitableLocations([&request](const fdb5::Key& dbKey) { return dbKey.partialMatch(request); }, config); +} + +static EngineBuilder rados_builder; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h new file mode 100644 index 000000000..0c70af155 --- /dev/null +++ b/src/fdb5/rados/RadosEngine.h @@ -0,0 +1,66 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/database/Engine.h" + +#include "metkit/mars/MarsRequest.h" + +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" + +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class RadosEngine : public Engine { + +public: // methods + + RadosEngine() = default; + + static const char* typeName() { return "rados"; } + +protected: // methods + + std::string name() const override; + + std::string dbType() const override { return typeName(); }; + + eckit::URI location(const Key& key, const Config& config) const override; + + bool canHandle(const eckit::URI& uri, const Config& config) const override; + + std::vector visitableLocations(const Key& key, const Config& config) const override; + std::vector visitableLocations(const metkit::mars::MarsRequest& rq, + const Config& config) const override; + + void print(std::ostream& out) const override { out << "RadosEngine"; } + +private: // methods + + std::vector visitableLocations(const std::function& matches, + const Config& config) const; +}; + +//---------------------------------------------------------------------------------------------------------------------- + + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index 0d847973e..d9e23fadf 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -9,9 +9,19 @@ */ #include "fdb5/rados/RadosFieldLocation.h" -#include "eckit/io/rados/RadosReadHandle.h" -#include "fdb5/LibFdb5.h" -#include "fdb5/io/SingleGribMungePartFileHandle.h" + +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" + +#include "eckit/filesystem/URIManager.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" +#include "eckit/io/rados/RadosObject.h" +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" + +#include +#include namespace fdb5 { @@ -23,31 +33,31 @@ ::eckit::Reanimator RadosFieldLocation::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length) : - FieldLocation(eckit::URI("rados", path), offset, length) {} +static FieldLocationBuilder builder("rados"); + +RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : + FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri) : FieldLocation(uri) {} RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length) : - FieldLocation(uri, offset, length) {} + FieldLocation(uri, offset, length, Key{}) {} -RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : FieldLocation(rhs.uri_) {} +// Kept for FieldLocationBuilder factory compatibility; `remapKey` is unused because the RADOS +// backend does not support key remapping. +RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, + const Key& /* remapKey */) : + RadosFieldLocation(uri, offset, length) {} RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) : FieldLocation(s) {} - std::shared_ptr RadosFieldLocation::make_shared() const { - return std::make_shared(std::move(*this)); + return std::make_shared(*this); } eckit::DataHandle* RadosFieldLocation::dataHandle() const { - eckit::RadosReadHandle* g = new eckit::RadosReadHandle(uri_.name(), offset(), length()); - return g; -} - -eckit::DataHandle* RadosFieldLocation::dataHandle(const Key& remapKey) const { - return new SingleGribMungePartFileHandle(path(), offset(), length(), remapKey); + return eckit::RadosObject(uri_).multipartRangeReadHandle(offset(), length()); } void RadosFieldLocation::print(std::ostream& out) const { @@ -58,12 +68,6 @@ void RadosFieldLocation::visit(FieldLocationVisitor& visitor) const { visitor(*this); } -eckit::URI RadosFieldLocation::uri(const eckit::PathName& path) { - return eckit::URI("rados", path); -} - -static FieldLocationBuilder builder("rados"); - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h index 3178cb3c8..5e18dd7f3 100644 --- a/src/fdb5/rados/RadosFieldLocation.h +++ b/src/fdb5/rados/RadosFieldLocation.h @@ -9,17 +9,21 @@ */ /// @author Emanuele Danovaro -/// @date Jan 2020 +/// @author Nicolau Manubens +/// @date Feb 2024 -#ifndef fdb5_RadosFieldLocation_H -#define fdb5_RadosFieldLocation_H +#pragma once -#include "eckit/filesystem/PathName.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" + +#include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" +#include "eckit/serialisation/Reanimator.h" -#include "fdb5/database/FieldLocation.h" -#include "fdb5/database/FileStore.h" +#include +#include namespace fdb5 { @@ -29,13 +33,16 @@ class RadosFieldLocation : public FieldLocation { public: RadosFieldLocation(const RadosFieldLocation& rhs); - RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length); RadosFieldLocation(const eckit::URI& uri); RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length); + // Factory-only overload; `remapKey` is ignored (see RadosFieldLocation.cc). + RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); RadosFieldLocation(eckit::Stream&); + eckit::DataHandle* dataHandle() const override; - eckit::DataHandle* dataHandle(const Key& remapKey) const override; + std::shared_ptr make_shared() const override; + void visit(FieldLocationVisitor& visitor) const override; public: // For Streamable @@ -45,18 +52,16 @@ class RadosFieldLocation : public FieldLocation { protected: // For Streamable const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods void print(std::ostream& out) const override; - eckit::URI uri(const eckit::PathName& path); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - -#endif // fdb5_RadosFieldLocation_H diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc new file mode 100644 index 000000000..b9451f33c --- /dev/null +++ b/src/fdb5/rados/RadosIndex.cc @@ -0,0 +1,211 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosIndex.h" + +#include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldDetails.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosLazyFieldLocation.h" + +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" +#include "eckit/io/Length.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/serialisation/HandleStream.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/Reanimator.h" +#include "eckit/utils/Tokenizer.h" + +#include // for PATH_MAX +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : + IndexBase(key, "radosKeyValue"), + location_(eckit::RadosKeyValue{name.pool().name(), name.name(), key.valuesToString()}, 0), + idx_kv_(location_.radosName().uri()) { + + // Persist indexKey under "key" so the index KV can later be identified when reopened. + eckit::MemoryHandle h{(size_t)PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + hs << key; + } + + idx_kv_.put("key", h.data(), hs.bytesWritten()); +} + +RadosIndex::RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes) : + IndexBase(key, "radosKeyValue"), location_(name, 0), idx_kv_(name.uri()) { + + if (readAxes) { + updateAxes(); + } +} + +void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) { + + const std::string axis_marker = "axis." + axis; + const char marker = '1'; + idx_kv_.put(axis_marker, &marker, 1); + + auto axis_kv = axis_kvs_.find(axis); + + if (axis_kv == axis_kvs_.end()) { + std::string kv_name = key().valuesToString() + std::string{"."} + axis; + axis_kvs_.emplace(std::piecewise_construct, std::forward_as_tuple(axis), + std::forward_as_tuple(location_.radosName().nspace().pool().name(), + location_.radosName().nspace().name(), kv_name)); + + axis_kv = axis_kvs_.find(axis); + } + + std::string v{"1"}; + axis_kv->second.put(value, v.data(), v.length()); +} + +void RadosIndex::updateAxes() { + + std::set axis_names; + for (const auto& key : idx_kv_.keys()) { + if (key.rfind("axis.", 0) == 0) { + axis_names.insert(key.substr(5)); + } + } + + // Compatibility with catalogues written before per-axis markers were introduced. + if (axis_names.empty() && idx_kv_.has("axes")) { + std::vector axes_data; + idx_kv_.getMemoryStream(axes_data, "axes", "index kv"); + std::vector legacy_axis_names; + eckit::Tokenizer parse(","); + parse(std::string(axes_data.begin(), axes_data.end()), legacy_axis_names); + axis_names.insert(legacy_axis_names.begin(), legacy_axis_names.end()); + } + + std::string indexKey{key_.valuesToString()}; + for (const auto& name : axis_names) { + eckit::RadosKeyValue axis_kv{idx_kv_.nspace().pool().name(), idx_kv_.nspace().name(), + indexKey + std::string{"."} + name}; + + axes_.insert(name, axis_kv.keys()); + } + + axes_.sort(); +} + +bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { + + std::string query{key.valuesToString()}; + + try { + std::vector loc_data; + eckit::MemoryStream ms = idx_kv_.getMemoryStream(loc_data, query, "index kv"); + + // Timestamp is read for informational purposes only; see note in RadosIndex::add. + time_t ts; + ms >> ts; + + fdb5::FieldLocation* loc = eckit::Reanimator::reanimate(ms); + field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); + } + catch (eckit::RadosEntityNotFoundException& e) { + return false; + } + + return true; +} + +void RadosIndex::add(const Key& key, const Field& field) { + + eckit::MemoryHandle h{(size_t)PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + // Timestamp kept per-entry for informational purposes; correctness does not depend on it because + // parallel writers targeting the same index key share this KV and last-write-wins. + takeTimestamp(); + hs << timestamp(); + hs << field.location(); + } + + idx_kv_.put(key.valuesToString(), h.data(), hs.bytesWritten()); +} + +void RadosIndex::entries(EntryVisitor& visitor) const { + + Index instantIndex(const_cast(this)); + + // Allow the visitor to selectively decline to visit the entries in this index + if (visitor.visitIndex(instantIndex)) { + + for (const auto& key : idx_kv_.keys()) { + + if (key == "axes" || key == "key" || key.rfind("axis.", 0) == 0) { + continue; + } + + // Build a lazy location so ListVisitor::visitDatum can filter without triggering a KV read. + // The real FieldLocation is retrieved and reanimated only when stableLocation() is called. + auto loc = std::make_shared(location_.radosName(), key); + fdb5::Field field(loc, time_t(), fdb5::FieldDetails()); + visitor.visitDatum(field, key); + } + } +} + +std::vector RadosIndex::dataURIs() const { + + // Iterates the index KV; each entry is a serialised RadosFieldLocation. + + std::set res; + + for (const auto& key : idx_kv_.keys()) { + + if (key == "axes" || key == "key" || key.rfind("axis.", 0) == 0) { + continue; + } + + std::vector data; + eckit::MemoryStream ms = idx_kv_.getMemoryStream(data, key, "index kv"); + + time_t ts; + ms >> ts; + + std::unique_ptr fl(eckit::Reanimator::reanimate(ms)); + res.insert(fl->uri()); + } + + return {res.begin(), res.end()}; +} + +//----------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h new file mode 100644 index 000000000..43724d008 --- /dev/null +++ b/src/fdb5/rados/RadosIndex.h @@ -0,0 +1,97 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/IndexStats.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosIndexLocation.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" + +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + + +class RadosIndex : public IndexBase { + +public: // methods + + // Creates a new index KV under `name`. + RadosIndex(const Key& key, const eckit::RadosNamespace& name); + // Wraps an already-existing index KV. + RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes = true); + + void flock() const override { NOTIMP; } + void funlock() const override { NOTIMP; } + + // Exposed so RadosCatalogueWriter can persist axis metadata into idx_kv_ / axis_kvs_. + void putAxisValue(const std::string& axis, const std::string& value); + + // Exposed so RadosCatalogueReader can enumerate field entries directly for stats. + const eckit::RadosKeyValue& idx_kv() const { return idx_kv_; } + +private: // methods + + const IndexLocation& location() const override { return location_; } + std::vector dataURIs() const override; + + bool dirty() const override { NOTIMP; } + + void open() override { NOTIMP; }; + // The RADOS KV index holds no open file/handle state, so closing is a no-op. + // Must not throw: invoked during normal read/list flows via eckit::AutoCloser. + void close() override {} + void reopen() override { NOTIMP; } + + void visit(IndexLocationVisitor& visitor) const override { NOTIMP; } + + bool get(const Key& key, const Key& remapKey, Field& field) const override; + void add(const Key& key, const Field& field) override; + void flush() override { NOTIMP; } + void encode(eckit::Stream& s, const int version) const override { NOTIMP; } + void entries(EntryVisitor& visitor) const override; + + void print(std::ostream& out) const override { NOTIMP; } + void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override { + NOTIMP; + } + + IndexStats statistics() const override { NOTIMP; } + + // Rehydrates the complete axis info from RADOS. + void updateAxes(); + +private: // members + + fdb5::RadosIndexLocation location_; + + eckit::RadosKeyValue idx_kv_; + std::map axis_kvs_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndexLocation.cc b/src/fdb5/rados/RadosIndexLocation.cc new file mode 100644 index 000000000..932944ac3 --- /dev/null +++ b/src/fdb5/rados/RadosIndexLocation.cc @@ -0,0 +1,26 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosIndexLocation.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosIndexLocation::RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset) : name_(name), offset_(offset) {} + +void RadosIndexLocation::print(std::ostream& out) const { + + out << "(" << name_.uri().asString() << ":" << offset_ << ")"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndexLocation.h b/src/fdb5/rados/RadosIndexLocation.h new file mode 100644 index 000000000..142ed9ace --- /dev/null +++ b/src/fdb5/rados/RadosIndexLocation.h @@ -0,0 +1,55 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/database/IndexLocation.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/io/rados/RadosKeyValue.h" + + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class RadosIndexLocation : public IndexLocation { + +public: // methods + + RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset); + + const eckit::RadosKeyValue& radosName() const { return name_; }; + + eckit::URI uri() const override { return name_.uri(); } + + IndexLocation* clone() const override { NOTIMP; } + +protected: // For Streamable + + void encode(eckit::Stream&) const override { NOTIMP; } + +private: // methods + + void print(std::ostream& out) const override; + +private: // members + + eckit::RadosKeyValue name_; + + off_t offset_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosLazyFieldLocation.cc b/src/fdb5/rados/RadosLazyFieldLocation.cc new file mode 100644 index 000000000..cd58342c2 --- /dev/null +++ b/src/fdb5/rados/RadosLazyFieldLocation.cc @@ -0,0 +1,77 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosLazyFieldLocation.h" + +#include "fdb5/database/FieldLocation.h" + +#include "eckit/filesystem/PathName.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/Reanimator.h" + +#include +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosLazyFieldLocation::RadosLazyFieldLocation(const fdb5::RadosLazyFieldLocation& rhs) : + index_(rhs.index_), key_(rhs.key_) {} + +RadosLazyFieldLocation::RadosLazyFieldLocation(const eckit::RadosKeyValue& index, const std::string& key) : + index_(index), key_(key) {} + +std::shared_ptr RadosLazyFieldLocation::make_shared() const { + return std::make_shared(*this); +} + +eckit::DataHandle* RadosLazyFieldLocation::dataHandle() const { + + return realise()->dataHandle(); +} + +void RadosLazyFieldLocation::print(std::ostream& out) const { + out << *realise(); +} + +void RadosLazyFieldLocation::visit(FieldLocationVisitor& visitor) const { + realise()->visit(visitor); +} + +std::shared_ptr RadosLazyFieldLocation::stableLocation() const { + return realise()->make_shared(); +} + +std::unique_ptr& RadosLazyFieldLocation::realise() const { + + if (fl_) { + return fl_; + } + + std::vector data; + eckit::MemoryStream ms = index_.getMemoryStream(data, key_, "index kv"); + + // Timestamp is read for informational purposes only; see note in RadosIndex::add. + time_t ts; + ms >> ts; + + fl_.reset(eckit::Reanimator::reanimate(ms)); + + return fl_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosLazyFieldLocation.h b/src/fdb5/rados/RadosLazyFieldLocation.h new file mode 100644 index 000000000..73f3d5fdc --- /dev/null +++ b/src/fdb5/rados/RadosLazyFieldLocation.h @@ -0,0 +1,61 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date June 2024 + +#pragma once + +#include "fdb5/database/FieldLocation.h" + +#include "eckit/io/rados/RadosKeyValue.h" + +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +// Used by fdb-list index visiting in RadosIndex::entries. Instances remain empty until the +// visitor accepts the enclosing key; only then does stableLocation() trigger the RADOS read +// and reanimate the concrete RadosFieldLocation. This avoids RPCs for unmatched keys. +class RadosLazyFieldLocation : public FieldLocation { +public: + + RadosLazyFieldLocation(const fdb5::RadosLazyFieldLocation& rhs); + RadosLazyFieldLocation(const eckit::RadosKeyValue& index, const std::string& key); + + eckit::DataHandle* dataHandle() const override; + + virtual std::shared_ptr make_shared() const override; + + virtual void visit(FieldLocationVisitor& visitor) const override; + + virtual std::shared_ptr stableLocation() const override; + +private: // methods + + std::unique_ptr& realise() const; + + void print(std::ostream& out) const override; + +private: // members + + eckit::RadosKeyValue index_; + std::string key_; + mutable std::unique_ptr fl_; +}; + + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosStats.cc b/src/fdb5/rados/RadosStats.cc new file mode 100644 index 000000000..1d943268f --- /dev/null +++ b/src/fdb5/rados/RadosStats.cc @@ -0,0 +1,65 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosStats.h" + +#include "fdb5/database/DbStats.h" + +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" + +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +::eckit::ClassSpec RadosDbStats::classSpec_ = { + &DbStatsContent::classSpec(), + "RadosDbStats", +}; +::eckit::Reanimator RadosDbStats::reanimator_; + +//---------------------------------------------------------------------------------------------------------------------- + +RadosDbStats::RadosDbStats() : dbCount_(0), indexCount_(0), fieldCount_(0) {} + +RadosDbStats::RadosDbStats(eckit::Stream& out) { + out >> dbCount_; + out >> indexCount_; + out >> fieldCount_; +} + +RadosDbStats& RadosDbStats::operator+=(const RadosDbStats& rhs) { + dbCount_ += rhs.dbCount_; + indexCount_ += rhs.indexCount_; + fieldCount_ += rhs.fieldCount_; + return *this; +} + +void RadosDbStats::add(const DbStatsContent& rhs) { + *this += dynamic_cast(rhs); +} + +void RadosDbStats::report(std::ostream& out, const char* indent) const { + reportCount(out, "Databases", dbCount_, indent); + reportCount(out, "Indexes", indexCount_, indent); + reportCount(out, "Fields", fieldCount_, indent); +} + +void RadosDbStats::encode(eckit::Stream& out) const { + out << dbCount_; + out << indexCount_; + out << fieldCount_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosStats.h b/src/fdb5/rados/RadosStats.h new file mode 100644 index 000000000..080683f47 --- /dev/null +++ b/src/fdb5/rados/RadosStats.h @@ -0,0 +1,62 @@ +/* + * (C) Copyright 2026- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Metin Cakircali +/// @date Aug 2026 + +#pragma once + +#include "fdb5/database/DbStats.h" + +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" + +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +// Byte totals are intentionally omitted; adding them would require per-object HEAD reads. +class RadosDbStats : public DbStatsContent { +public: + + RadosDbStats(); + RadosDbStats(eckit::Stream&); + + static DbStats make() { return DbStats(new RadosDbStats()); } + + size_t dbCount_; + size_t indexCount_; + size_t fieldCount_; + + RadosDbStats& operator+=(const RadosDbStats& rhs); + + void add(const DbStatsContent& rhs) override; + + void report(std::ostream& out, const char* indent) const override; + +public: // For Streamable + + static const eckit::ClassSpec& classSpec() { return classSpec_; } + +protected: // For Streamable + + void encode(eckit::Stream&) const override; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index d54725544..7c4f8a27d 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -8,74 +8,158 @@ * does it submit to any jurisdiction. */ -#include "eckit/log/Bytes.h" -#include "eckit/log/Timer.h" - -#include "eckit/config/Resource.h" -#include "eckit/io/EmptyHandle.h" -#include "eckit/io/rados/RadosWriteHandle.h" +#include "fdb5/rados/RadosStore.h" #include "fdb5/LibFdb5.h" +#include "fdb5/database/Field.h" #include "fdb5/database/FieldLocation.h" -#include "fdb5/io/FDBFileHandle.h" +#include "fdb5/database/Store.h" +#include "fdb5/database/WipeState.h" +#include "fdb5/rados/RadosCleanup.h" +#include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosFieldLocation.h" -#include "fdb5/rados/RadosStore.h" #include "fdb5/rules/Rule.h" -using namespace eckit; +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosObject.h" +#include "eckit/io/rados/RadosPool.h" +#include "eckit/log/Log.h" +#include "eckit/log/TimeStamp.h" +#include "eckit/runtime/Main.h" +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/StaticMutex.h" +#include "eckit/utils/MD5.h" +#include "eckit/utils/Tokenizer.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RadosStore::RadosStore(const Key& key, const Config& config) : - Store(), directory_("mars:" + key.valuesToString()), archivedFields_(0) {} +static StoreBuilder builder("rados"); + +RadosStore::RadosStore(const Key& key, const Config& config) : RadosCommon(config, "store", key) {} -RadosStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint) : - Store(), directory_("mars:" + key.valuesToString()), archivedFields_(0) { - NOTIMP; -} +RadosStore::RadosStore(const Schema& /*schema*/, const Key& key, const Config& config) : RadosStore(key, config) {} -RadosStore::RadosStore(const eckit::URI& uri) : - Store(), directory_("mars:" + uri.path().dirName()), archivedFields_(0) {} +RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : RadosCommon(config, "store", uri) {} eckit::URI RadosStore::uri() const { - return URI("rados", directory_); + return eckit::RadosNamespace(pool_, db_namespace_).uri(); +} + +eckit::URI RadosStore::uri(const eckit::URI& dataURI) { + eckit::RadosObject o{dataURI}; + return o.nspace().uri(); +} + +bool RadosStore::uriBelongs(const eckit::URI& uri) const { + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + + ASSERT(n == 2 || n == 3); + return ((uri.scheme() == type()) && (parts[0] == pool_) && (parts[1] == db_namespace_)); +} + +bool RadosStore::uriExists(const eckit::URI& uri) const { + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + + ASSERT(uri.scheme() == type()); + + ASSERT(n == 2 || n == 3); + ASSERT(parts[0] == pool_); + ASSERT(parts[1] == db_namespace_); + + if (n == 2) { + return eckit::RadosNamespace(uri).exists(); + } + + return eckit::RadosObject(uri).exists(); +} + +std::set RadosStore::collocatedDataURIs() const { + + std::set store_unit_uris; + + eckit::RadosNamespace n{pool_, db_namespace_}; + + if (!n.exists()) { + return store_unit_uris; + } + + for (const auto& obj : n.listObjects()) { + if (obj.name().find(";part-") != std::string::npos) { + continue; + } + store_unit_uris.insert(obj.uri()); + } + + return store_unit_uris; +} + +std::set RadosStore::asCollocatedDataURIs(const std::set& uris) const { + std::set res; + for (const auto& uri : uris) { + res.insert(uri); + } + return res; } bool RadosStore::exists() const { - return true; + return eckit::RadosNamespace(pool_, db_namespace_).exists(); } -eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const { - return remapKey.empty() ? field.dataHandle() : field.dataHandle(remapKey); +eckit::DataHandle* RadosStore::retrieve(Field& field) const { + return field.dataHandle(); } -std::unique_ptr RadosStore::archive(const uint32_t, const Key& key, const void* data, - eckit::Length length) { +std::unique_ptr RadosStore::archive(const Key& key, const void* data, eckit::Length length) { + archivedFields_++; - eckit::PathName dataPath = getDataPath(key); - eckit::URI dataUri("rados", dataPath); + const eckit::RadosObject& o = getDataObject(key); - eckit::DataHandle& dh = getDataHandle(dataPath); + eckit::DataHandle& h = getDataHandle(key, o); - eckit::Offset position = dh.position(); + eckit::Offset offset{h.position()}; - long len = dh.write(data, length); + long len = h.write(data, length); ASSERT(len == length); - return std::make_unique(dataUri, position, length); + return std::make_unique(o.uri(), offset, length); +} + +RadosStore::~RadosStore() { + std::exception_ptr ignored; + best_effort(ignored, "~RadosStore::closeDataHandles", [&] { closeDataHandles(); }); } size_t RadosStore::flush() { + if (archivedFields_ == 0) { return 0; } - // ensure consistent state before writing Toc entry - flushDataHandles(); size_t out = archivedFields_; @@ -88,125 +172,228 @@ void RadosStore::close() { } void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { + ASSERT(uri.scheme() == type()); - eckit::PathName path = uri.path(); - if (path.isDir()) { - logVerbose << "rmdir: "; - logAlways << path << std::endl; - if (doit) { - path.rmdir(false); + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + + ASSERT(n == 2 || n == 3); + + ASSERT(parts[0] == pool_); + ASSERT(parts[1] == db_namespace_); + + if (n == 2) { // namespace + + eckit::RadosNamespace ns{uri}; + + logVerbose << "destroy Rados namespace: "; + logAlways << ns.str() << std::endl; + + if (doit && ns.exists()) { + ns.destroy(); } } - else { - logVerbose << "Unlinking: "; - logAlways << path << std::endl; + else { // object + + eckit::RadosObject obj{uri}; + + logVerbose << "destroy Rados object: "; + logAlways << obj.str() << std::endl; + if (doit) { - path.unlink(false); + obj.ensureAllDestroyed(); } } } -eckit::DataHandle* RadosStore::getCachedHandle(const eckit::PathName& path) const { - HandleStore::const_iterator j = handles_.find(path); - if (j != handles_.end()) { - return j->second; +void RadosStore::print(std::ostream& out) const { + + out << "RadosStore(" << pool_ << "/" << db_namespace_ << ")"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +// The database maps to a RADOS namespace; only the namespace holding this database's objects is +// ever touched by the wipe-related methods below. + +void RadosStore::finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) { + + // `doit` and `unsafeWipeAll` do not affect the preparation of a RADOS store wipe. + + const std::set& dataURIs = storeState.includedDataURIs(); // included according to cat + const std::set& safeURIs = storeState.safeURIs(); // excluded according to cat + + // Objects included by the catalogue may no longer exist (e.g. due to a prior incomplete wipe). + std::set nonExistingURIs; + for (const auto& uri : dataURIs) { + if (!eckit::RadosObject{uri}.exists()) { + nonExistingURIs.insert(uri); + } + } + for (const auto& uri : nonExistingURIs) { + storeState.markAsMissing(uri); + } + + const bool all = safeURIs.empty(); + if (!all) { + return; + } + + // Full wipe: scan the database namespace for any objects unaccounted for by the catalogue. + eckit::RadosNamespace db{pool_, db_namespace_}; + + if (!db.exists()) { + return; } - else { - return nullptr; + + for (const auto& obj : db.listObjects()) { + + // Parts belong to a main object and are removed together with it. + if (obj.name().find(";part-") != std::string::npos) { + continue; + } + + const eckit::URI uri = obj.uri(); + if (dataURIs.find(uri) == dataURIs.end() && safeURIs.find(uri) == safeURIs.end()) { + storeState.insertUnrecognised(uri); + } } } -void RadosStore::closeDataHandles() { - for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle* dh = j->second; - dh->close(); - delete dh; +bool RadosStore::doWipeUnknowns(const std::set& unknownURIs) const { + for (const auto& uri : unknownURIs) { + if (eckit::RadosObject{uri}.exists()) { + remove(uri, std::cout, std::cout, true); + } } - handles_.clear(); + return true; } -eckit::DataHandle* RadosStore::createFileHandle(const eckit::PathName& path) { +bool RadosStore::doWipeURIs(const StoreWipeState& wipeState) const { + const bool wipeAll = wipeState.safeURIs().empty(); - // static size_t sizeBuffer = eckit::Resource("fdbBufferSize", 64 * 1024 * 1024); + for (const auto& uri : wipeState.includedDataURIs()) { + remove(uri, std::cout, std::cout, true); + } - LOG_DEBUG_LIB(LibFdb5) << "Creating RadosWriteHandle to " - << path - // << " with buffer of " << eckit::Bytes(sizeBuffer) - << std::endl; + if (wipeAll) { + cleanupEmptyDatabase_ = true; + } - return new RadosWriteHandle(path, 0); + return true; } -eckit::DataHandle* RadosStore::createAsyncHandle(const eckit::PathName& path) { - NOTIMP; +void RadosStore::doWipeEmptyDatabase() const { + + if (!cleanupEmptyDatabase_) { + return; + } - /* static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); - static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); + eckit::RadosNamespace db{pool_, db_namespace_}; - return new eckit::AIOHandle(path, nbBuffers, sizeBuffer);*/ + if (db.exists()) { + remove(db.uri(), std::cout, std::cout, true); + } } -eckit::DataHandle* RadosStore::createDataHandle(const eckit::PathName& path) { +bool RadosStore::doUnsafeFullWipe() const { - static bool fdbWriteToNull = eckit::Resource("fdbWriteToNull;$FDB_WRITE_TO_NULL", false); - if (fdbWriteToNull) { - return new eckit::EmptyHandle(); - } + // If the database namespace also holds a catalogue, skip: the catalogue-driven wipe owns the + // namespace. Presence of a "key" entry in the DB KV is used as the catalogue-exists signal. + if (db_kv_ && (!db_kv_->exists() || !db_kv_->has("key"))) { + + eckit::RadosNamespace db{pool_, db_namespace_}; - static bool fdbAsyncWrite = eckit::Resource("fdbAsyncWrite;$FDB_ASYNC_WRITE", false); - if (fdbAsyncWrite) { - return createAsyncHandle(path); + if (db.exists()) { + remove(db.uri(), std::cout, std::cout, true); + } } - return createFileHandle(path); + return true; } -eckit::DataHandle& RadosStore::getDataHandle(const eckit::PathName& path) { - eckit::DataHandle* dh = getCachedHandle(path); - if (!dh) { - dh = createDataHandle(path); - ASSERT(dh); - handles_[path] = dh; - dh->openForWrite(0); - } - return *dh; +std::vector RadosStore::getAuxiliaryURIs(const eckit::URI& /*uri*/, bool /*onlyExisting*/) const { + return {}; +} + + +//---------------------------------------------------------------------------------------------------------------------- + +// Unique name generation copied from eckit::LocalPathName::unique. +static eckit::StaticMutex local_mutex; + +eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { + + eckit::AutoLock lock(local_mutex); + + std::string hostname = eckit::Main::hostname(); + + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + + std::string name = os.str(); + + eckit::MD5 md5(name); + + return eckit::RadosObject{pool_, db_namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; } -eckit::PathName RadosStore::generateDataPath(const Key& key) const { +const eckit::RadosObject& RadosStore::getDataObject(const Key& key) const { - eckit::PathName dpath(directory_); - dpath /= key.valuesToString(); - dpath = eckit::PathName::unique(dpath) + ".data"; - return dpath; + auto it = dataObjects_.find(key); + if (it == dataObjects_.end()) { + it = dataObjects_.emplace(key, generateDataObject(key)).first; + } + return it->second; } -eckit::PathName RadosStore::getDataPath(const Key& key) { - PathStore::const_iterator j = dataPaths_.find(key); - if (j != dataPaths_.end()) { - return j->second; +eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosObject& name) { + + auto iter = handles_.find(key); + if (iter != handles_.end()) { + return *(iter->second); } - eckit::PathName dataPath = generateDataPath(key); + auto handle = std::unique_ptr{name.multipartWriteHandle(maxPartSize_)}; + ASSERT(handle); - dataPaths_[key] = dataPath; + handle->openForWrite(0); + auto [inserted, success] = handles_.emplace(key, std::move(handle)); + ASSERT(success); - return dataPath; + return *inserted->second; } -void RadosStore::flushDataHandles() { +void RadosStore::closeDataHandles() { + + // Detach the map first so partial failures never leave the destructor retrying the same handle. + HandleStore handles; + handles.swap(handles_); + dataObjects_.clear(); - for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle* dh = j->second; - dh->flush(); + std::exception_ptr first; + for (auto& [key, handle] : handles) { + best_effort(first, "RadosStore::closeDataHandles", [&] { handle->close(); }); + } + if (first) { + std::rethrow_exception(first); } } -void RadosStore::print(std::ostream& out) const { - out << "RadosStore(" << directory_ << ")"; -} +void RadosStore::flushDataHandles() { -static StoreBuilder builder("rados"); + std::exception_ptr first; + for (auto& [key, handle] : handles_) { + best_effort(first, "RadosStore::flushDataHandles", [&] { handle->flush(); }); + } + if (first) { + std::rethrow_exception(first); + } +} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 300475a03..3dd401352 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -8,34 +8,58 @@ * does it submit to any jurisdiction. */ -/// @file RadosStore.h /// @author Emanuele Danovaro -/// @date Jan 2020 +/// @author Nicolau Manubens +/// @date Feb 2024 -#ifndef fdb5_RadosStore_H -#define fdb5_RadosStore_H +#pragma once -#include "fdb5/database/Catalogue.h" -#include "fdb5/database/Index.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" #include "fdb5/database/Store.h" +#include "fdb5/rados/RadosCommon.h" #include "fdb5/rules/Schema.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/rados/RadosObject.h" + +#include +#include +#include +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// Store that implements the FDB on CEPH object store +/// Store that implements the FDB on CEPH object store. +/// Not thread-safe: archive/flush/close calls on a single instance must be serialised by the caller. -class RadosStore : public Store { +class RadosStore : public Store, public RadosCommon { public: // methods + RadosStore(const Key& key, const Config& config); RadosStore(const Schema& schema, const Key& key, const Config& config); - RadosStore(const eckit::URI& uri); + RadosStore(const eckit::URI& uri, const Config& config); + ~RadosStore() override; - ~RadosStore() override {} + RadosStore(const RadosStore&) = delete; + RadosStore& operator=(const RadosStore&) = delete; + RadosStore(RadosStore&&) = delete; + RadosStore& operator=(RadosStore&&) = delete; eckit::URI uri() const override; + static eckit::URI uri(const eckit::URI& dataURI); + bool uriBelongs(const eckit::URI&) const override; + bool uriExists(const eckit::URI&) const override; + std::set collocatedDataURIs() const override; + std::set asCollocatedDataURIs(const std::set&) const override; bool open() override { return true; } size_t flush() override; @@ -43,47 +67,48 @@ class RadosStore : public Store { void checkUID() const override { /* nothing to do */ } + // Wipe-related methods + void finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) override; + bool doWipeUnknowns(const std::set& unknownURIs) const override; + bool doWipeURIs(const StoreWipeState& wipeState) const override; + void doWipeEmptyDatabase() const override; + bool doUnsafeFullWipe() const override; + + std::vector getAuxiliaryURIs(const eckit::URI& uri, bool onlyExisting) const override; + protected: // methods std::string type() const override { return "rados"; } bool exists() const override; - eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override; - std::unique_ptr archive(const uint32_t, const Key& key, const void* data, - eckit::Length length) override; + eckit::DataHandle* retrieve(Field& field) const override; + std::unique_ptr archive(const Key& key, const void* data, eckit::Length length) override; using Store::remove; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; - eckit::DataHandle* getCachedHandle(const eckit::PathName& path) const; + void print(std::ostream& out) const override; + + eckit::RadosObject generateDataObject(const Key& key) const; + + const eckit::RadosObject& getDataObject(const Key& key) const; + eckit::DataHandle& getDataHandle(const Key& key, const eckit::RadosObject& name); void closeDataHandles(); - eckit::DataHandle* createFileHandle(const eckit::PathName& path); - eckit::DataHandle* createAsyncHandle(const eckit::PathName& path); - eckit::DataHandle* createDataHandle(const eckit::PathName& path); - eckit::DataHandle& getDataHandle(const eckit::PathName& path); - eckit::PathName generateDataPath(const Key& key) const; - eckit::PathName getDataPath(const Key& key); void flushDataHandles(); - void print(std::ostream& out) const override; - private: // types - typedef std::map HandleStore; - typedef std::map PathStore; + using HandleStore = std::map>; + using ObjectStore = std::map; private: // members - HandleStore handles_; ///< stores the DataHandles being used by the Session + size_t archivedFields_{0}; - PathStore dataPaths_; - eckit::PathName directory_; - - size_t archivedFields_; + HandleStore handles_; + mutable ObjectStore dataObjects_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - -#endif // fdb5_RadosStore_H diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index e6cecbba1..7276ad2be 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -69,6 +69,7 @@ endforeach() add_subdirectory( api ) add_subdirectory( database ) add_subdirectory( type ) +add_subdirectory( rados ) add_subdirectory( daos ) add_subdirectory( fam ) add_subdirectory( concurrent ) diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt new file mode 100644 index 000000000..374374c6d --- /dev/null +++ b/tests/fdb/rados/CMakeLists.txt @@ -0,0 +1,38 @@ +if (HAVE_RADOSFDB) + + list( APPEND rados_tests + rados_store + rados_catalogue + ) + + list( APPEND unit_test_libraries fdb5 ) + + # The Rados unit tests need a pool to run against (with RADOS_TESTS_MANAGE_POOLS=OFF the pool + # must already exist, e.g. one created in the Ceph service). The pool name can be + # provided at configure time via -DFDB_RADOS_TEST_POOL=, or inherited from + # the FDB_RADOS_TEST_POOL environment variable (e.g. exported in the dev container + # that talks to the Ceph service). + if( NOT FDB_RADOS_TEST_POOL AND DEFINED ENV{FDB_RADOS_TEST_POOL} ) + set( FDB_RADOS_TEST_POOL "$ENV{FDB_RADOS_TEST_POOL}" ) + endif() + + # Only inject the variable into the test environment when we have a value: + # passing an empty "FDB_RADOS_TEST_POOL=" would clobber any value already present + # in the runtime environment when the test executes. + unset( _rados_test_environment ) + if( FDB_RADOS_TEST_POOL ) + set( _rados_test_environment ENVIRONMENT FDB_RADOS_TEST_POOL=${FDB_RADOS_TEST_POOL} ) + endif() + + foreach( _test ${rados_tests} ) + + ecbuild_add_test( TARGET fdb_test_${_test} + SOURCES test_${_test}.cc + LABELS rados + LIBS "${unit_test_libraries}" + INCLUDES "${unit_test_include_dirs}" + ${_rados_test_environment} ) + + endforeach() + +endif() diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc new file mode 100644 index 000000000..48f93b36b --- /dev/null +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -0,0 +1,1004 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" +#include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/DatabaseNotFoundException.h" +#include "fdb5/database/DbStats.h" +#include "fdb5/database/Engine.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/rados/RadosCatalogueReader.h" +#include "fdb5/rados/RadosCatalogueWriter.h" +#include "fdb5/rados/RadosFieldLocation.h" +#include "fdb5/rados/RadosStore.h" +#include "fdb5/rules/Schema.h" + +#include "metkit/mars/MarsRequest.h" + +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/io/Offset.h" +#include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosCluster.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosPool.h" +#include "eckit/log/Log.h" +#include "eckit/testing/Test.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace eckit; + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); +}; + +// Guard against clobbering unrelated namespaces in a shared CI pool. +void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); + } + } +} + +// Count only URIs that would actually be deleted, filtering out safe/info/error records so that +// too-specific requests yield zero. +size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { + size_t count = 0; + fdb5::WipeElement elem; + while (wipeObject.next(elem)) { + if (print) { + std::cout << elem << std::endl; + } + if (elem.type() != fdb5::WipeElementType::ERROR && elem.type() != fdb5::WipeElementType::CATALOGUE_INFO && + elem.type() != fdb5::WipeElementType::CATALOGUE_SAFE && elem.type() != fdb5::WipeElementType::STORE_SAFE) { + count += elem.uris().size(); + } + } + return count; +} + +// temporary schema,spaces,root files common to all RADOS Catalogue tests + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& opt_schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& catalogue_tests_tmp_root() { + static eckit::PathName cd("./rados_catalogue_tests_fdb_root"); + return cd; +} + +void cleanupRados() noexcept { + try { +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + eckit::RadosPool{"test-catalogue"}.ensureDestroyed(); +#else + ensureCleanNamespaces(eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", ""), + "test-catalogue"); +#endif + if (catalogue_tests_tmp_root().exists()) { + deldir(catalogue_tests_tmp_root()); + } + } + catch (...) { + eckit::Log::error() << "FDB RADOS catalogue cleanup failed" << std::endl; + } +} + +} // namespace + +namespace fdb::test { + +CASE("Setup") { + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Catalogue/Store tests fail. + if (catalogue_tests_tmp_root().exists()) { + deldir(catalogue_tests_tmp_root()); + } + catalogue_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + std::string opt_schema_str{"[ a, b [ c?, d [ e?, f ]]]"}; + + std::unique_ptr hs_opt(opt_schema_file().fileHandle()); + hs_opt->openForWrite(opt_schema_str.size()); + { + eckit::AutoClose closer(*hs_opt); + hs_opt->write(opt_schema_str.data(), opt_schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); +} + +CASE("RadosCatalogue tests") { + + std::string test_id = "test-catalogue"; +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + std::string pool = test_id; + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer +#else + std::string pool; + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); + EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); +#endif + + SECTION("RadosCatalogue archive (index) and retrieve without a Store") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Schema schema{schema_file()}; + + /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using + /// a=1,b=2 + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // archive + + std::unique_ptr loc( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1))); + + eckit::URI catalogue_uri; + { + fdb5::RadosCatalogueWriter dcatw{db_key, config}; + + fdb5::Catalogue& cat = dcatw; + cat.selectIndex(index_key); + + fdb5::CatalogueWriter& catw = dcatw; + catw.archive(index_key, field_key, std::move(loc)); + cat.flush(0); + catalogue_uri = cat.uri(); + } + + { + auto reopened = fdb5::CatalogueWriterFactory::instance().build(catalogue_uri, config); + EXPECT(reopened->key() == db_key); + EXPECT_NOT(reopened->schema().empty()); + } + + // retrieve + + { + fdb5::RadosCatalogueReader dcatr{db_key, config}; + + fdb5::Catalogue& cat = dcatr; + EXPECT(cat.selectIndex(index_key)); + + fdb5::Key missing_index_key({{"c", "missing"}, {"d", "missing"}}); + EXPECT_NOT(cat.selectIndex(missing_index_key)); + EXPECT_NOT(cat.selectIndex(missing_index_key)); + + EXPECT(cat.selectIndex(index_key)); + cat.deselectIndex(); + EXPECT(cat.selectIndex(index_key)); + + fdb5::Field f; + fdb5::CatalogueReader& catr = dcatr; + catr.retrieve(field_key, f); + EXPECT(f.location().uri().name() == eckit::URI("rados", "test_uri").name()); + EXPECT(f.location().offset() == eckit::Offset(0)); + EXPECT(f.location().length() == eckit::Length(1)); + } + } + + SECTION("RadosCatalogue archive (index) and retrieve with a RadosStore") { + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // store data + + char data[] = "test"; + + fdb5::RadosStore rstore{schema, db_key, config}; + fdb5::Store& store = static_cast(rstore); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + // index data + + { + fdb5::RadosCatalogueWriter rcatw{db_key, config}; + fdb5::Catalogue& cat = rcatw; + cat.deselectIndex(); + cat.selectIndex(index_key); + fdb5::CatalogueWriter& catw = rcatw; + catw.archive(index_key, field_key, std::move(loc)); + + /// flush store before flushing catalogue + rstore.flush(); // not necessary if using a RADOS store + } + + // find data + + fdb5::Field field; + { + fdb5::RadosCatalogueReader rcatr{db_key, config}; + fdb5::Catalogue& cat = rcatr; + cat.selectIndex(index_key); + fdb5::CatalogueReader& catr = rcatr; + catr.retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + /// @note: the field spans potentially several objects and is returned as an + /// eckit::PartHandle wrapping a RadosMultiObjReadHandle. + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + } + + SECTION("RadosCatalogue reports missing databases via factory paths") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // A key-based reader over a DB that was never written must fail to open. + fdb5::Key missing_db_key({{"a", "99"}, {"b", "99"}}); + { + fdb5::RadosCatalogueReader reader{missing_db_key, config}; + fdb5::CatalogueReader& cr = reader; + EXPECT_NOT(cr.open()); + } + + // A URI-based reader/writer over a DB whose namespace has no catalogue KV must throw + // DatabaseNotFoundException at construction so the caller does not proceed on empty state. + const std::string missing_ns = test_id + "_" + missing_db_key.valuesToString(); + const eckit::URI missing_uri = eckit::RadosKeyValue{pool, missing_ns, "catalogue_kv"}.uri(); + + EXPECT_THROWS_AS(fdb5::CatalogueReaderFactory::instance().build(missing_uri, config), + fdb5::DatabaseNotFoundException); + EXPECT_THROWS_AS(fdb5::CatalogueWriterFactory::instance().build(missing_uri, config), + fdb5::DatabaseNotFoundException); + } + + SECTION("RadosCatalogueReader::stats reports index and field counts") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Key db_key({{"a", "77"}, {"b", "77"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key_1({{"e", "5"}, {"f", "6"}}); + fdb5::Key field_key_2({{"e", "5"}, {"f", "7"}}); + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& cat = writer; + cat.selectIndex(index_key); + std::unique_ptr loc1( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, eckit::Offset(0), eckit::Length(1))); + std::unique_ptr loc2( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, eckit::Offset(1), eckit::Length(1))); + static_cast(writer).archive(index_key, field_key_1, std::move(loc1)); + static_cast(writer).archive(index_key, field_key_2, std::move(loc2)); + cat.flush(0); + } + + { + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::CatalogueReader& cr = reader; + EXPECT(cr.open()); + fdb5::DbStats stats = cr.stats(); + std::ostringstream oss; + stats.report(oss); + const std::string report = oss.str(); + // Must expose non-empty output rather than throwing NOTIMP. + EXPECT(!report.empty()); + EXPECT(report.find("Indexes") != std::string::npos); + EXPECT(report.find("Fields") != std::string::npos); + } + } + + SECTION("RadosCatalogue persists ControlIdentifiers across processes") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Key db_key({{"a", "88"}, {"b", "88"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + eckit::URI catalogue_uri; + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& cat = writer; + cat.selectIndex(index_key); + std::unique_ptr loc( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, eckit::Offset(0), eckit::Length(1))); + static_cast(writer).archive(index_key, field_key, std::move(loc)); + cat.flush(0); + + // Default: everything enabled. + EXPECT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::List)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); + + // hideContents disables List and Retrieve, leaves Archive. + cat.hideContents(); + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::List)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); + catalogue_uri = cat.uri(); + } + + { + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::Catalogue& cat = reader; + EXPECT(static_cast(reader).open()); + // State survives process boundary. + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::List)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); + } + + { + auto reader = fdb5::CatalogueReaderFactory::instance().build(catalogue_uri, config); + EXPECT_NOT(reader->enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT_NOT(reader->enabled(fdb5::ControlIdentifier::List)); + EXPECT(reader->enabled(fdb5::ControlIdentifier::Archive)); + } + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& cat = writer; + fdb5::ControlIdentifiers ids = fdb5::ControlIdentifier::List | fdb5::ControlIdentifier::Retrieve; + cat.control(fdb5::ControlAction::Enable, ids); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::List)); + } + } + + SECTION("RadosCatalogueWriter supports concurrent writers on the same database") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Key db_key({{"a", "66"}, {"b", "66"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key first_field_key({{"e", "5"}}); + fdb5::Key second_field_key({{"g", "6"}}); + + std::promise start; + const std::shared_future ready = start.get_future().share(); + std::mutex error_mutex; + std::exception_ptr error; + + // + const auto archive = [&](const fdb5::Key& field_key, eckit::Offset offset) { + try { + ready.wait(); + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& catalogue = writer; + catalogue.selectIndex(index_key); + std::unique_ptr location( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, offset, eckit::Length(1))); + static_cast(writer).archive(index_key, field_key, std::move(location)); + catalogue.flush(0); + } + catch (...) { + std::lock_guard lock{error_mutex}; + if (!error) { + error = std::current_exception(); + } + } + }; + + std::thread first{archive, std::cref(first_field_key), eckit::Offset(0)}; + std::thread second{archive, std::cref(second_field_key), eckit::Offset(1)}; + start.set_value(); + first.join(); + second.join(); + EXPECT(!error); + + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::Catalogue& catalogue = reader; + EXPECT(static_cast(reader).open()); + EXPECT(catalogue.selectIndex(index_key)); + const auto e_axis = static_cast(reader).axis("e"); + const auto g_axis = static_cast(reader).axis("g"); + EXPECT(e_axis && e_axis->get().contains("5")); + EXPECT(g_axis && g_axis->get().contains("6")); + } + + SECTION("Rados placements are selected from matching space roots") { + + const std::string alpha_prefix = test_id + "alpha"; + const std::string beta_prefix = test_id + "beta"; + std::string config_str{ + "spaces:\n" + "- regex: 11:11\n" + " roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_alpha_root\n" + " namespace_prefix: " + + alpha_prefix + + "\n" + "- regex: 22:22\n" + " roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_beta_root\n" + " namespace_prefix: " + + beta_prefix + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_legacy_root\n" + " namespace_prefix: legacy\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Key alpha_key({{"a", "11"}, {"b", "11"}}); + fdb5::Key beta_key({{"a", "22"}, {"b", "22"}}); + + fdb5::RadosCatalogueWriter alpha{alpha_key, config}; + fdb5::RadosCatalogueWriter beta{beta_key, config}; + + const std::string alpha_namespace = pool + "/" + alpha_prefix + "_" + alpha_key.valuesToString(); + const std::string beta_namespace = pool + "/" + beta_prefix + "_" + beta_key.valuesToString(); + EXPECT(alpha.uri().name() == alpha_namespace); + EXPECT(beta.uri().name() == beta_namespace); + EXPECT(fdb5::Engine::backend("rados").location(alpha_key, config).name() == alpha_namespace + "/catalogue_kv"); + EXPECT(fdb5::Engine::backend("rados").location(beta_key, config).name() == beta_namespace + "/catalogue_kv"); + + const auto alpha_locations = fdb5::Engine::backend("rados").visitableLocations(alpha_key, config); + const auto beta_locations = fdb5::Engine::backend("rados").visitableLocations(beta_key, config); + EXPECT(alpha_locations.size() == 1); + EXPECT(beta_locations.size() == 1); + EXPECT(alpha_locations.front().name() == alpha_namespace + "/catalogue_kv"); + EXPECT(beta_locations.front().name() == beta_namespace + "/catalogue_kv"); + } + + SECTION("RadosCatalogue supports large serialised field locations") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Key db_key({{"a", "large"}, {"b", "large"}}); + fdb5::Key index_key({{"c", "large"}, {"d", "large"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + auto location = std::make_unique(eckit::URI{"rados", std::string(600, 'x')}, + eckit::Offset(0), eckit::Length(1)); + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& catalogue = writer; + EXPECT(catalogue.selectIndex(index_key)); + static_cast(writer).archive(index_key, field_key, std::move(location)); + catalogue.flush(0); + } + + { + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::Catalogue& catalogue = reader; + EXPECT(catalogue.selectIndex(index_key)); + + fdb5::Field field; + EXPECT(static_cast(reader).retrieve(field_key, field)); + EXPECT(field.location().uri().name() == std::string(600, 'x')); + } + } + + SECTION("Via FDB API with a Rados catalogue and store") { + +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); +#else + ensureCleanNamespaces(pool, test_id); +#endif + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "type: local\n" + "schema : " + + schema_file().path() + + "\n" + "engine: rados\n" + "store: rados\n" + "rados:\n"}; + + config_str += " pool: " + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest all_req{metkit::mars::MarsRequest{}, true, std::vector{}}; + + // initialise FDB + + fdb5::FDB fdb(config); + + // check FDB is empty + + size_t count; + fdb5::ListElement info; + + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true, false, " "); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // archive data + + char data[] = "test"; + + fdb.archive(request_key, data, sizeof(data)); + fdb.flush(); + + // retrieve data + + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + fdb5::FDB reopened(config); + std::unique_ptr reopened_handle(reopened.retrieve(r)); + + eckit::MemoryHandle reopened_data; + reopened_handle->copyTo(reopened_data); + EXPECT(reopened_data.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(reopened_data.data(), data, sizeof(data)) == 0); + + // list all + + listObject = fdb.list(all_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 1); + + // wipe data + + // dry run attempt to wipe with too specific request + + auto wipeObject = fdb.wipe(full_req); + EXPECT(countWipeable(wipeObject) == 0); + + // dry run wipe index and store unit + wipeObject = fdb.wipe(index_req); + EXPECT(countWipeable(wipeObject) > 0); + + // dry run wipe database + wipeObject = fdb.wipe(db_req); + EXPECT(countWipeable(wipeObject) > 0); + + // ensure field still exists + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 1); + + // attempt to wipe with too specific request + wipeObject = fdb.wipe(full_req, true); + EXPECT(countWipeable(wipeObject) == 0); + fdb.flush(); + + // wipe index and store unit + wipeObject = fdb.wipe(index_req, true); + EXPECT(countWipeable(wipeObject) > 0); + fdb.flush(); + + // ensure field does not exist + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 0); + + // re-archive data + + // FDB caches open DBs. Once a full DB is wiped, a fresh FDB instance is needed + // to re-create the top-level catalogue KV. + fdb5::FDB fdb2(config); + + fdb2.archive(request_key, data, sizeof(data)); + fdb2.flush(); + + listObject = fdb2.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 1); + + // Wipe remains enabled after hideContents disables only List and Retrieve. + fdb2.control(db_req, fdb5::ControlAction::Disable, + fdb5::ControlIdentifier::List | fdb5::ControlIdentifier::Retrieve); + wipeObject = fdb2.wipe(db_req, true); + EXPECT(countWipeable(wipeObject) > 0); + fdb2.flush(); + + fdb5::RadosCatalogueReader hidden_reader{db_key, config}; + EXPECT_NOT(static_cast(hidden_reader).open()); + + // ensure field does not exist + listObject = fdb2.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 0); + + // Wipe an already-wiped DB. The store-side namespace destroy path must be idempotent so + // recovering from a partial wipe does not raise. + EXPECT_NO_THROW(fdb2.wipe(db_req, true)); + fdb2.flush(); + } + + // teardown rados + +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + eckit::RadosPool{pool}.ensureDestroyed(); +#else + ensureCleanNamespaces(pool, test_id); +#endif +} + +} // namespace fdb::test + +//---------------------------------------------------------------------------------------------------------------------- + +int main(int argc, char** argv) { + int ret = -1; + try { + ret = eckit::testing::run_tests(argc, argv); + } + catch (...) { + eckit::Log::error() << "FDB RADOS catalogue tests terminated with an exception" << std::endl; + } + + cleanupRados(); + return ret; +} diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc new file mode 100644 index 000000000..3d3ad1ea7 --- /dev/null +++ b/tests/fdb/rados/test_rados_store.cc @@ -0,0 +1,705 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" +#include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/Engine.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Store.h" +#include "fdb5/fdb5_config.h" +#include "fdb5/rados/RadosFieldLocation.h" +#include "fdb5/rados/RadosStore.h" +#include "fdb5/rules/Schema.h" +#include "fdb5/toc/TocCatalogueReader.h" +#include "fdb5/toc/TocCatalogueWriter.h" + +#include "metkit/mars/MarsRequest.h" + +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/io/Offset.h" +#include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosCluster.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosObject.h" +#include "eckit/io/rados/RadosPool.h" +#include "eckit/log/Log.h" +#include "eckit/testing/Test.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace eckit; + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); +}; + +void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); + } + } +} + +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS +void ensureCleanPools(const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listPools()) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosPool{name}.destroy(); + } + } +} +#endif + +} // namespace + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& store_tests_tmp_root() { + static eckit::PathName sd("./rados_store_tests_fdb_root"); + return sd; +} + +void cleanupRados() noexcept { + try { +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + ensureCleanPools("test-store"); +#else + const std::string pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", ""); + for (const std::string& prefix : {"test-store1", "test-store2", "test-store3", "test-store4"}) { + ensureCleanNamespaces(pool, prefix); + } +#endif + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } + } + catch (...) { + eckit::Log::error() << "FDB RADOS store cleanup failed" << std::endl; + } +} + +/// @note: counts only the URIs that would actually be deleted, filtering out purely +/// informational wipe elements (safe/info/error) so a too-specific request yields 0. +size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { + size_t count = 0; + fdb5::WipeElement elem; + while (wipeObject.next(elem)) { + if (print) { + std::cout << elem << std::endl; + } + if (elem.type() != fdb5::WipeElementType::ERROR && elem.type() != fdb5::WipeElementType::CATALOGUE_INFO && + elem.type() != fdb5::WipeElementType::CATALOGUE_SAFE && elem.type() != fdb5::WipeElementType::STORE_SAFE) { + count += elem.uris().size(); + } + } + return count; +} + +//---------------------------------------------------------------------------------------------------------------------- + +namespace fdb::test { + +CASE("Setup") { + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Store tests fail. + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } + store_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", store_tests_tmp_root().path().c_str(), 1); + + // prepare schema for tests involving S3Store + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); +} + +CASE("RadosStore tests") { + + SECTION("archive and retrieve") { + + std::string test_id = "test-store1"; +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + std::string pool = test_id; + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer +#else + std::string pool; + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); + EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); +#endif + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "rados:\n" + " maxPartSize: 16\n" + " store:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Schema schema{schema_file()}; + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + + const std::string data{"0123456789abcdef0123456789abcdef"}; + + // archive + + fdb5::RadosStore rados_store{schema, db_key, config}; + fdb5::Store& store = rados_store; + std::unique_ptr loc(store.archive(index_key, data.data(), data.size())); + + rados_store.close(); + + // retrieve + fdb5::Field field(std::move(loc), std::time(nullptr)); + std::cout << "Read location: " << field.location() << std::endl; + std::unique_ptr dh(store.retrieve(field)); + /// @note: the field spans potentially several objects and is returned as an + /// eckit::PartHandle wrapping a RadosMultiObjReadHandle. + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(data.size())); + EXPECT(::memcmp(mh.data(), data.data(), data.size()) == 0); + + // remove + eckit::RadosObject field_name{field.location().uri()}; + eckit::RadosNamespace store_name = field_name.nspace(); + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT(store_name.listObjects().size() == 0); + + std::unique_ptr expiring_location; + { + fdb5::RadosStore expiring_store{schema, db_key, config}; + fdb5::Store& store = expiring_store; + expiring_location = store.archive(index_key, data.data(), data.size()); + } + + fdb5::Field expiring_field(std::move(expiring_location), std::time(nullptr)); + std::unique_ptr expiring_handle(expiring_field.dataHandle()); + eckit::MemoryHandle expiring_data; + expiring_handle->copyTo(expiring_data); + EXPECT(expiring_data.size() == eckit::Length(data.size())); + EXPECT(::memcmp(expiring_data.data(), data.data(), data.size()) == 0); + + eckit::RadosObject{expiring_field.location().uri()}.nspace().destroy(); + } + + SECTION("rejects namespace prefixes containing underscores") { + + fdb5::Schema schema{schema_file()}; + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: unused\n" + " pool: unused\n" + " root_namespace: unused\n" + " namespace_prefix: invalid_prefix\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + EXPECT_THROWS_AS((fdb5::RadosStore{schema, db_key, config}), eckit::UserError); + EXPECT_THROWS_AS((fdb5::Engine::backend("rados").location(db_key, config)), eckit::UserError); + } + + SECTION("RadosFieldLocation three-argument constructor forwards an empty remapKey") { + fdb5::RadosFieldLocation loc{eckit::URI{"rados", "pool/ns/obj"}, eckit::Offset(0), eckit::Length(1)}; + EXPECT(loc.remapKey().empty()); + EXPECT(loc.uri().name() == "pool/ns/obj"); + EXPECT(loc.offset() == eckit::Offset(0)); + EXPECT(loc.length() == eckit::Length(1)); + } + + SECTION("with POSIX Catalogue") { + + std::string test_id = "test-store2"; +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + std::string pool = test_id; + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer +#else + std::string pool; + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); + EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); +#endif + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " store:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // store data + + char data[] = "test"; + + fdb5::RadosStore rados_store{schema, db_key, config}; + fdb5::Store& store = static_cast(rados_store); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + // index data + + { + /// @todo: could have a unique ptr here, might not need a static cast + fdb5::TocCatalogueWriter tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.deselectIndex(); + cat.selectIndex(index_key); + // const fdb5::Index& idx = tcat.currentIndex(); + static_cast(tcat).archive(index_key, field_key, std::move(loc)); + + /// flush store before flushing catalogue + rados_store.flush(); + } + + // find data + + fdb5::Field field; + { + fdb5::TocCatalogueReader tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.selectIndex(index_key); + static_cast(tcat).retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + /// @note: the field spans potentially several objects and is returned as an + /// eckit::PartHandle wrapping a RadosMultiObjReadHandle. + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // remove data + eckit::RadosObject field_name{field.location().uri()}; + eckit::RadosNamespace store_name{field_name.nspace()}; + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT(store_name.listObjects().size() == 0); + } + + SECTION("VIA FDB API") { + + std::string test_id = "test-store3"; + + /// @note: the POSIX toc catalogue root is shared across sections; reset it so this + /// section is not polluted by entries left behind by previous sections. + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } + store_tests_tmp_root().mkdir(); +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + std::string pool = test_id; + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer +#else + std::string pool; + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); + EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); +#endif + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "type: local\n" + "schema : " + + schema_file().path() + + "\n" + "engine: toc\n" + "store: rados\n" + "rados:\n"}; + + config_str += " maxPartSize: 16\n"; + + config_str += " store:\n"; + + config_str += " pool: " + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; + + // initialise store + + fdb5::FDB fdb(config); + + // check store is empty + + size_t count; + fdb5::ListElement info; + + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true, false, " "); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // store data + + char data[] = "test123456"; + + /// @note: maxPartSize is set to 16, and four 10-byte fields are archived, spanning 3 objects + for (int i = 0; i < 4; i++) { + std::cout << "Archive field " << i << std::endl; + fdb5::Key request_key_i( + {{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); + fdb.archive(request_key_i, data, sizeof(data)); + } + + fdb.flush(); + + // retrieve data + + for (int i = 0; i < 4; i++) { + std::cout << "Retrieve field " << i << std::endl; + fdb5::Key request_key_i( + {{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); + metkit::mars::MarsRequest r_i = request_key_i.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r_i)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + } + + // wipe data + + // dry run attempt to wipe with too specific request + + auto wipeObject = fdb.wipe(full_req); + EXPECT(countWipeable(wipeObject) == 0); + + // dry run wipe index and store unit + wipeObject = fdb.wipe(index_req); + EXPECT(countWipeable(wipeObject) > 0); + + // dry run wipe database + wipeObject = fdb.wipe(db_req); + EXPECT(countWipeable(wipeObject) > 0); + + // ensure field still exists + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // attempt to wipe with too specific request + wipeObject = fdb.wipe(full_req, true); + EXPECT(countWipeable(wipeObject) == 0); + /// @todo: really needed? + fdb.flush(); + + // wipe index and store unit (and DB pool or namespace as there is only one index) + wipeObject = fdb.wipe(index_req, true); + EXPECT(countWipeable(wipeObject) > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 0); + } + + /// @todo: if doing what's in this section at the end of the previous section reusing the same FDB object, + // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe + SECTION("FDB API RE-STORE AND WIPE DB") { + + std::string test_id = "test-store4"; + + /// @note: the POSIX toc catalogue root is shared across sections; reset it so this + /// section is not polluted by entries left behind by previous sections. + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } + store_tests_tmp_root().mkdir(); +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + std::string pool = test_id; + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer +#else + std::string pool; + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); + EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); +#endif + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" + "type: local\n" + "schema : " + + schema_file().path() + + "\n" + "engine: toc\n" + "store: rados\n" + "rados:\n"}; + + config_str += " maxPartSize: 16\n"; + + config_str += " store:\n"; + + config_str += " pool: " + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; + + // initialise store + + fdb5::FDB fdb(config); + + // store again + + char data[] = "test"; + + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + size_t count; + + // wipe all database + + auto wipeObject = fdb.wipe(db_req, true); + EXPECT(countWipeable(wipeObject) > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + + fdb5::ListElement info; + auto listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 0); + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb::test + + +int main(int argc, char** argv) { + + int ret = -1; + try { + ret = eckit::testing::run_tests(argc, argv); + } + catch (...) { + eckit::Log::error() << "FDB RADOS store tests terminated with an exception" << std::endl; + } + + cleanupRados(); + + return ret; +}