diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml deleted file mode 100644 index 48efac09a..000000000 --- a/.github/workflows/ci-rust.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: rust - -on: - push: - branches: - - 'master' - - 'develop' - - 'rust-bindings' - tags-ignore: - - '**' - paths: - - 'rust/**' - - '.github/workflows/ci-rust.yml' - - pull_request: - paths: - - 'rust/**' - - '.github/workflows/ci-rust.yml' - - workflow_dispatch: ~ - -env: - CARGO_TERM_COLOR: always - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - -jobs: - fmt: - name: fmt - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - - name: Format check - run: cargo fmt --check - - clippy: - name: clippy - if: ${{ !github.event.pull_request.head.repo.fork }} - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - - name: Configure git for private repos - run: git config --global url."https://x-access-token:${{ secrets.GH_REPO_READ_TOKEN }}@github.com/".insteadOf "ssh://git@github.com/" - - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - - name: Clippy - run: cargo clippy --features vendored --all-targets -- -D warnings - - test: - name: test - if: ${{ !github.event.pull_request.head.repo.fork }} - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - - name: Configure git for private repos - run: git config --global url."https://x-access-token:${{ secrets.GH_REPO_READ_TOKEN }}@github.com/".insteadOf "ssh://git@github.com/" - - - uses: dtolnay/rust-toolchain@stable - - - name: Test - run: cargo test --features vendored diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ef9b327d..4a8b17b5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,17 @@ jobs: third_party secrets: inherit + # Run Rust CI (fmt + clippy + doc + test) on the rust/ workspace + ci-rust: + name: ci-rust + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf/reusable-workflows/.github/workflows/ci-rust.yml@main + with: + manifest-path: rust/Cargo.toml + features: --features fdb-sys/vendored + secrets: + private_repos_token: ${{ secrets.GH_REPO_READ_TOKEN }} + # Run CI of private downstream packages on self-hosted runners private-downstream-ci: name: private-downstream-ci @@ -83,6 +94,7 @@ jobs: runs-on: ubuntu-latest needs: - downstream-ci + - ci-rust - private-downstream-ci - downstream-ci-hpc - private-downstream-ci-hpc diff --git a/.cargo/config.toml b/rust/.cargo/config.toml similarity index 100% rename from .cargo/config.toml rename to rust/.cargo/config.toml diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 666a66b3c..53cea9366 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -17,14 +17,20 @@ fdb-sys = { path = "crates/fdb-sys" } fdb = { path = "crates/fdb" } # Foundation crates -eckit-sys = { git = "ssh://git@github.com/ecmwf/rust-wrappers-playground.git", default-features = false } -metkit-sys = { git = "ssh://git@github.com/ecmwf/rust-wrappers-playground.git", default-features = false } -eccodes-sys = { git = "ssh://git@github.com/ecmwf/rust-wrappers-playground.git", default-features = false } +eckit-sys = { git = "ssh://git@github.com/ecmwf/eckit.git", branch = "develop", default-features = false } +eckit = { git = "ssh://git@github.com/ecmwf/eckit.git", branch = "develop" } +metkit-sys = { git = "ssh://git@github.com/ecmwf/metkit.git", branch = "develop", default-features = false } +metkit = { git = "ssh://git@github.com/ecmwf/metkit.git", branch = "develop" } +# Pinned to match metkit develop: the playground HEAD and the eccodes-repo +# crate both consume eckit-sys from the rust-bindings branch, which conflicts +# with our eckit-sys develop dependency (links collision). Drop the pin and +# switch to eccodes.git once its eckit-sys dependency moves to develop. +eccodes-sys = { git = "ssh://git@github.com/ecmwf/rust-wrappers-playground.git", rev = "d6b582fd", default-features = false, features = ["vendored", "eccodes-threads"] } # Build tools -bindman = { git = "ssh://git@github.com/ecmwf/bindman.git" } -bindman-build = { git = "ssh://git@github.com/ecmwf/bindman.git" } -bindman-utils = { git = "ssh://git@github.com/ecmwf/bindman.git" } +bindman = { git = "ssh://git@github.com/ecmwf/bindman.git", rev = "47edf68" } +bindman-build = { git = "ssh://git@github.com/ecmwf/bindman.git", rev = "47edf68" } +bindman-utils = { git = "ssh://git@github.com/ecmwf/bindman.git", rev = "47edf68" } # External thiserror = "2" diff --git a/rust/crates/fdb-sys/build.rs b/rust/crates/fdb-sys/build.rs index 49afc27ad..9a311d2fc 100644 --- a/rust/crates/fdb-sys/build.rs +++ b/rust/crates/fdb-sys/build.rs @@ -6,16 +6,33 @@ //! //! Both modes build the CXX bridge for C++ to Rust bindings. -use std::env; -use std::path::PathBuf; - -const FDB_VERSION: &str = "5.19.1"; - fn main() { println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-changed=src/lib.rs"); - println!("cargo:rerun-if-changed=cpp/fdb_bridge.h"); - println!("cargo:rerun-if-changed=cpp/fdb_bridge.cpp"); + println!("cargo:rerun-if-changed=cpp/FdbBridge.h"); + println!("cargo:rerun-if-changed=cpp/Types.h"); + println!("cargo:rerun-if-changed=cpp/Key.h"); + println!("cargo:rerun-if-changed=cpp/Key.cc"); + println!("cargo:rerun-if-changed=cpp/Library.h"); + println!("cargo:rerun-if-changed=cpp/Library.cc"); + println!("cargo:rerun-if-changed=cpp/FdbHandle.h"); + println!("cargo:rerun-if-changed=cpp/FdbHandle.cc"); + println!("cargo:rerun-if-changed=cpp/ListIteratorHandle.h"); + println!("cargo:rerun-if-changed=cpp/ListIteratorHandle.cc"); + println!("cargo:rerun-if-changed=cpp/DumpIteratorHandle.h"); + println!("cargo:rerun-if-changed=cpp/DumpIteratorHandle.cc"); + println!("cargo:rerun-if-changed=cpp/StatusIteratorHandle.h"); + println!("cargo:rerun-if-changed=cpp/StatusIteratorHandle.cc"); + println!("cargo:rerun-if-changed=cpp/WipeIteratorHandle.h"); + println!("cargo:rerun-if-changed=cpp/WipeIteratorHandle.cc"); + println!("cargo:rerun-if-changed=cpp/PurgeIteratorHandle.h"); + println!("cargo:rerun-if-changed=cpp/PurgeIteratorHandle.cc"); + println!("cargo:rerun-if-changed=cpp/StatsIteratorHandle.h"); + println!("cargo:rerun-if-changed=cpp/StatsIteratorHandle.cc"); + println!("cargo:rerun-if-changed=cpp/ControlIteratorHandle.h"); + println!("cargo:rerun-if-changed=cpp/ControlIteratorHandle.cc"); + println!("cargo:rerun-if-changed=cpp/MessageArchiverWrapper.h"); + println!("cargo:rerun-if-changed=cpp/MessageArchiverWrapper.cc"); println!("cargo:rerun-if-env-changed=FDB_DIR"); println!("cargo:rerun-if-env-changed=CMAKE_PREFIX_PATH"); println!("cargo:rerun-if-env-changed=DOCS_RS"); @@ -26,6 +43,8 @@ fn main() { bindman_utils::validate_build_mode(cfg!(feature = "system"), cfg!(feature = "vendored")); + generate_exceptions(); + if cfg!(feature = "system") { build_system(); } else { @@ -33,44 +52,82 @@ fn main() { } } +/// Generate `fdb_exceptions.{h,rs}` for fdb-sys's cxx bridge. +/// +/// fdb-sys does not introduce its own exception subclasses (the higher-level +/// `fdb` crate maps `cxx::Exception` directly), so the `own` list is empty +/// and we inherit C++ catch blocks from upstream `-sys` crates (eckit-sys, +/// metkit-sys) via [`bindman_build::collect_dep_exception_sources`]. +fn generate_exceptions() { + let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set")); + let inherited = bindman_build::collect_dep_exception_sources(); + + bindman_build::generate_exception_bridge(&bindman_build::ExceptionBridgeConfig { + primary_namespace: "fdb", + out_dir: &out_dir, + own: &[], + inherited: &inherited, + }); +} + /// Build using system-installed fdb5 via `CMake` `find_package` #[cfg(feature = "system")] fn build_system() { + use std::env; + use std::path::PathBuf; + let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")); // Get dependency paths from -sys crates - let eckit_include = env::var("DEP_ECKIT_INCLUDE") - .expect("DEP_ECKIT_INCLUDE not set - eckit-sys must be a dependency"); - let metkit_include = env::var("DEP_METKIT_INCLUDE") - .expect("DEP_METKIT_INCLUDE not set - metkit-sys must be a dependency"); - let eccodes_include = env::var("DEP_ECCODES_INCLUDE") - .expect("DEP_ECCODES_INCLUDE not set - eccodes-sys must be a dependency"); + let eckit_include = env::var("DEP_ECKIT_SYS_INCLUDE") + .expect("DEP_ECKIT_SYS_INCLUDE not set - eckit-sys must be a dependency"); + let eckit_cpp_dir = env::var("DEP_ECKIT_SYS_CPP_DIR") + .expect("DEP_ECKIT_SYS_CPP_DIR not set - eckit-sys must be a dependency"); + let metkit_include = env::var("DEP_METKIT_SYS_INCLUDE") + .expect("DEP_METKIT_SYS_INCLUDE not set - metkit-sys must be a dependency"); + let metkit_cpp_dir = env::var("DEP_METKIT_SYS_CPP_DIR") + .expect("DEP_METKIT_SYS_CPP_DIR not set - metkit-sys must be a dependency"); + let eccodes_include = env::var("DEP_ECCODES_SYS_INCLUDE") + .expect("DEP_ECCODES_SYS_INCLUDE not set - eccodes-sys must be a dependency"); + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); - let (root, fdb_include, lib_dir) = - bindman_utils::cmake_find_package("fdb5", FDB_VERSION, Some("FDB_DIR")); + let (root, fdb_include, lib_dir) = bindman_utils::cmake_find_package("fdb5", "5.10.0"); println!("cargo:rustc-link-search=native={}", lib_dir.display()); println!("cargo:rustc-link-lib=dylib=fdb5"); // Build the CXX bridge cxx_build::bridge("src/lib.rs") - .file(crate_dir.join("cpp/fdb_bridge.cpp")) + .file(crate_dir.join("cpp/Key.cc")) + .file(crate_dir.join("cpp/Library.cc")) + .file(crate_dir.join("cpp/FdbHandle.cc")) + .file(crate_dir.join("cpp/ListIteratorHandle.cc")) + .file(crate_dir.join("cpp/DumpIteratorHandle.cc")) + .file(crate_dir.join("cpp/StatusIteratorHandle.cc")) + .file(crate_dir.join("cpp/WipeIteratorHandle.cc")) + .file(crate_dir.join("cpp/PurgeIteratorHandle.cc")) + .file(crate_dir.join("cpp/StatsIteratorHandle.cc")) + .file(crate_dir.join("cpp/ControlIteratorHandle.cc")) + .file(crate_dir.join("cpp/MessageArchiverWrapper.cc")) .include(&fdb_include) .include(&eckit_include) + .include(&eckit_cpp_dir) // for EckitBridge.h .include(&metkit_include) + .include(&metkit_cpp_dir) // for MetkitBridge.h .include(&eccodes_include) .include(crate_dir.join("cpp")) - .flag_if_supported("-std=c++17") + .include(&out_dir) // for fdb_exceptions.h (generated) + .std("c++17") .compile("fdb_sys_bridge"); // Link to eckit and metkit (bridge uses their symbols) - let eckit_root = env::var("DEP_ECKIT_ROOT") - .expect("DEP_ECKIT_ROOT not set - eckit-sys must be a dependency"); - let metkit_root = env::var("DEP_METKIT_ROOT") - .expect("DEP_METKIT_ROOT not set - metkit-sys must be a dependency"); - let eccodes_root = env::var("DEP_ECCODES_ROOT") - .expect("DEP_ECCODES_ROOT not set - eccodes-sys must be a dependency"); + let eckit_root = env::var("DEP_ECKIT_SYS_ROOT") + .expect("DEP_ECKIT_SYS_ROOT not set - eckit-sys must be a dependency"); + let metkit_root = env::var("DEP_METKIT_SYS_ROOT") + .expect("DEP_METKIT_SYS_ROOT not set - metkit-sys must be a dependency"); + let eccodes_root = env::var("DEP_ECCODES_SYS_ROOT") + .expect("DEP_ECCODES_SYS_ROOT not set - eccodes-sys must be a dependency"); println!("cargo:rustc-link-search=native={eckit_root}/lib"); println!("cargo:rustc-link-lib=dylib=eckit"); @@ -105,13 +162,16 @@ fn build_system() { #[cfg(feature = "vendored")] #[allow(clippy::too_many_lines)] fn build_vendored() { + use std::env; use std::fs; + use std::path::PathBuf; use std::process::Command; const ECBUILD_REPO: &str = "https://github.com/ecmwf/ecbuild.git"; const ECBUILD_TAG: &str = "3.13.1"; const FDB_REPO: &str = "https://github.com/ecmwf/fdb.git"; + const FDB_TAG: &str = "5.19.1"; let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); let src_dir = out_dir.join("src"); @@ -122,16 +182,16 @@ fn build_vendored() { fs::create_dir_all(&build_dir).expect("Failed to create build directory"); // Get dependency paths from -sys crates - let eckit_root = env::var("DEP_ECKIT_ROOT") - .expect("DEP_ECKIT_ROOT not set - eckit-sys must be a dependency"); - let metkit_root = env::var("DEP_METKIT_ROOT") - .expect("DEP_METKIT_ROOT not set - metkit-sys must be a dependency"); - let eccodes_root = env::var("DEP_ECCODES_ROOT") - .expect("DEP_ECCODES_ROOT not set - eccodes-sys must be a dependency"); + let eckit_root = env::var("DEP_ECKIT_SYS_ROOT") + .expect("DEP_ECKIT_SYS_ROOT not set - eckit-sys must be a dependency"); + let metkit_root = env::var("DEP_METKIT_SYS_ROOT") + .expect("DEP_METKIT_SYS_ROOT not set - metkit-sys must be a dependency"); + let eccodes_root = env::var("DEP_ECCODES_SYS_ROOT") + .expect("DEP_ECCODES_SYS_ROOT not set - eccodes-sys must be a dependency"); // Clone sources let ecbuild_src = bindman_utils::git_clone(ECBUILD_REPO, ECBUILD_TAG, &src_dir.join("ecbuild")); - let fdb_src = bindman_utils::git_clone(FDB_REPO, FDB_VERSION, &src_dir.join("fdb")); + let fdb_src = bindman_utils::git_clone(FDB_REPO, FDB_TAG, &src_dir.join("fdb")); // Patch CMakeLists.txt to remove tests subdirectory (buggy when ENABLE_TESTS=OFF) let cmakelists = fdb_src.join("CMakeLists.txt"); @@ -211,15 +271,9 @@ fn build_vendored() { bindman_utils::on_off(cfg!(feature = "sandbox")) )); - // Portable install names for dynamic libraries + // Use @rpath install names — the leaf binary sets rpaths via bindman_utils::emit_rpaths() #[cfg(target_os = "macos")] - cmd.arg("-DCMAKE_INSTALL_NAME_DIR=@executable_path/fdb_libs"); - - #[cfg(target_os = "linux")] - { - cmd.arg("-DCMAKE_INSTALL_RPATH=$ORIGIN:$ORIGIN/../fdb_libs"); - cmd.arg("-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON"); - } + cmd.arg("-DCMAKE_INSTALL_NAME_DIR=@rpath"); bindman_utils::run_command(&mut cmd, "ecbuild configure fdb"); @@ -244,35 +298,46 @@ fn build_vendored() { // FDB source directory contains private headers that may be needed let fdb_src_include = fdb_src.join("src"); - // IMPORTANT: Copy resources FIRST, then link against the copied location. - let libs_dest = copy_resources_to_output(&install_dir, &eckit_root, &metkit_root); + let eckit_cpp_dir = env::var("DEP_ECKIT_SYS_CPP_DIR") + .expect("DEP_ECKIT_SYS_CPP_DIR not set - eckit-sys must be a dependency"); + let metkit_cpp_dir = env::var("DEP_METKIT_SYS_CPP_DIR") + .expect("DEP_METKIT_SYS_CPP_DIR not set - metkit-sys must be a dependency"); // Build the CXX bridge cxx_build::bridge("src/lib.rs") - .file(crate_dir.join("cpp/fdb_bridge.cpp")) + .file(crate_dir.join("cpp/Key.cc")) + .file(crate_dir.join("cpp/Library.cc")) + .file(crate_dir.join("cpp/FdbHandle.cc")) + .file(crate_dir.join("cpp/ListIteratorHandle.cc")) + .file(crate_dir.join("cpp/DumpIteratorHandle.cc")) + .file(crate_dir.join("cpp/StatusIteratorHandle.cc")) + .file(crate_dir.join("cpp/WipeIteratorHandle.cc")) + .file(crate_dir.join("cpp/PurgeIteratorHandle.cc")) + .file(crate_dir.join("cpp/StatsIteratorHandle.cc")) + .file(crate_dir.join("cpp/ControlIteratorHandle.cc")) + .file(crate_dir.join("cpp/MessageArchiverWrapper.cc")) .include(&include_dir) .include(&fdb_src_include) .include(format!("{eckit_root}/include")) + .include(&eckit_cpp_dir) // for EckitBridge.h .include(format!("{metkit_root}/include")) + .include(&metkit_cpp_dir) // for MetkitBridge.h .include(format!("{eccodes_root}/include")) .include(crate_dir.join("cpp")) - .flag_if_supported("-std=c++17") + .include(&out_dir) // for fdb_exceptions.h (generated) + .std("c++17") .compile("fdb_sys_bridge"); - // Link against the copied location in target directory - println!("cargo:rustc-link-search=native={}", libs_dest.display()); + // Link against the install directory + let fdb_lib_dir = bindman_utils::resolve_lib_dir(&install_dir); + println!("cargo:rustc-link-search=native={}", fdb_lib_dir.display()); println!("cargo:rustc-link-lib=dylib=fdb5"); - println!("cargo:rustc-link-lib=dylib=eckit"); - println!("cargo:rustc-link-lib=dylib=metkit"); bindman_utils::link_cpp_stdlib(); // Export for downstream crates (still point to install dir for headers) println!("cargo:root={}", install_dir.display()); println!("cargo:include={}", include_dir.display()); - // Emit RPATH flags for runtime library discovery - bindman_utils::emit_rpath_flags(&["fdb_libs"]); - // Check C++ API bindman_build::check_cpp_api(&fdb_src_include, &crate_dir.join("src/lib.rs")); } @@ -281,30 +346,3 @@ fn build_vendored() { fn build_vendored() { unreachable!("build_vendored called without vendored feature"); } - -/// Copy libraries to target directory for portable binaries. -/// Returns the path to the libs directory where libraries were copied. -#[cfg(feature = "vendored")] -fn copy_resources_to_output( - fdb_install_dir: &std::path::Path, - eckit_root: &str, - metkit_root: &str, -) -> PathBuf { - use std::path::Path; - - let target_dir = bindman_utils::target_profile_dir(); - let libs_dest = target_dir.join("fdb_libs"); - - let fdb_lib_dir = bindman_utils::resolve_lib_dir(fdb_install_dir); - let eckit_lib_dir = Path::new(eckit_root).join("lib"); - let metkit_lib_dir = Path::new(metkit_root).join("lib"); - - bindman_utils::copy_shared_libs(&fdb_lib_dir, &libs_dest, "fdb5"); - bindman_utils::copy_shared_libs(&eckit_lib_dir, &libs_dest, "eckit"); - bindman_utils::copy_shared_libs(&metkit_lib_dir, &libs_dest, "metkit"); - - // Export resource directory name for runtime discovery - println!("cargo:rustc-env=FDB_LIBS_DIR=fdb_libs"); - - libs_dest -} diff --git a/rust/crates/fdb-sys/cpp/ControlIteratorHandle.cc b/rust/crates/fdb-sys/cpp/ControlIteratorHandle.cc new file mode 100644 index 000000000..672487952 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/ControlIteratorHandle.cc @@ -0,0 +1,51 @@ +// fdb ControlIteratorHandle bridge — implementation. + +#include "fdb_exceptions.h" + +#include "ControlIteratorHandle.h" +#include "fdb-sys/src/lib.rs.h" + +#include "eckit/exception/Exceptions.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +ControlIteratorHandle::ControlIteratorHandle(fdb5::ControlIterator&& it) : impl_(std::move(it)) {} + +ControlIteratorHandle::~ControlIteratorHandle() = default; + +bool ControlIteratorHandle::hasNext() { + if (exhausted_) { + return false; + } + if (has_current_) { + return true; + } + + if (impl_.next(current_)) { + has_current_ = true; + return true; + } + exhausted_ = true; + return false; +} + +ControlElementData ControlIteratorHandle::next() { + if (!has_current_ && !hasNext()) { + throw eckit::OutOfRange("Iterator exhausted", Here()); + } + + has_current_ = false; + + ControlElementData data; + data.location = rust::String(current_.location.asString()); + for (const auto& id : current_.controlIdentifiers) { + data.identifiers.push_back(id); + } + return data; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/ControlIteratorHandle.h b/rust/crates/fdb-sys/cpp/ControlIteratorHandle.h new file mode 100644 index 000000000..c70e4ed4a --- /dev/null +++ b/rust/crates/fdb-sys/cpp/ControlIteratorHandle.h @@ -0,0 +1,37 @@ +// fdb ControlIteratorHandle bridge — wraps `fdb5::ControlIterator`. +#pragma once + +#include "Types.h" + +#include "fdb5/api/helpers/ControlIterator.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::ControlIterator` for Rust FFI. +class ControlIteratorHandle { +public: + + explicit ControlIteratorHandle(fdb5::ControlIterator&& it); + ~ControlIteratorHandle(); + + ControlIteratorHandle(const ControlIteratorHandle&) = delete; + ControlIteratorHandle& operator=(const ControlIteratorHandle&) = delete; + ControlIteratorHandle(ControlIteratorHandle&&) = default; + ControlIteratorHandle& operator=(ControlIteratorHandle&&) = default; + + bool hasNext(); + ControlElementData next(); + +private: + + fdb5::ControlIterator impl_; + fdb5::ControlElement current_; + bool has_current_ = false; + bool exhausted_ = false; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/DumpIteratorHandle.cc b/rust/crates/fdb-sys/cpp/DumpIteratorHandle.cc new file mode 100644 index 000000000..4df5af319 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/DumpIteratorHandle.cc @@ -0,0 +1,48 @@ +// fdb DumpIteratorHandle bridge — implementation. + +#include "fdb_exceptions.h" + +#include "DumpIteratorHandle.h" +#include "fdb-sys/src/lib.rs.h" + +#include "eckit/exception/Exceptions.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +DumpIteratorHandle::DumpIteratorHandle(fdb5::DumpIterator&& it) : impl_(std::move(it)) {} + +DumpIteratorHandle::~DumpIteratorHandle() = default; + +bool DumpIteratorHandle::hasNext() { + if (exhausted_) { + return false; + } + if (has_current_) { + return true; + } + + if (impl_.next(current_)) { + has_current_ = true; + return true; + } + exhausted_ = true; + return false; +} + +DumpElementData DumpIteratorHandle::next() { + if (!has_current_ && !hasNext()) { + throw eckit::OutOfRange("Iterator exhausted", Here()); + } + + has_current_ = false; + + DumpElementData data; + data.content = rust::String(current_); + return data; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/DumpIteratorHandle.h b/rust/crates/fdb-sys/cpp/DumpIteratorHandle.h new file mode 100644 index 000000000..174a6daaa --- /dev/null +++ b/rust/crates/fdb-sys/cpp/DumpIteratorHandle.h @@ -0,0 +1,37 @@ +// fdb DumpIteratorHandle bridge — wraps `fdb5::DumpIterator`. +#pragma once + +#include "Types.h" + +#include "fdb5/api/helpers/DumpIterator.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::DumpIterator` for Rust FFI. +class DumpIteratorHandle { +public: + + explicit DumpIteratorHandle(fdb5::DumpIterator&& it); + ~DumpIteratorHandle(); + + DumpIteratorHandle(const DumpIteratorHandle&) = delete; + DumpIteratorHandle& operator=(const DumpIteratorHandle&) = delete; + DumpIteratorHandle(DumpIteratorHandle&&) = default; + DumpIteratorHandle& operator=(DumpIteratorHandle&&) = default; + + bool hasNext(); + DumpElementData next(); + +private: + + fdb5::DumpIterator impl_; + fdb5::DumpElement current_; + bool has_current_ = false; + bool exhausted_ = false; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/FdbBridge.h b/rust/crates/fdb-sys/cpp/FdbBridge.h new file mode 100644 index 000000000..c003732a4 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/FdbBridge.h @@ -0,0 +1,24 @@ +// fdb C++ bridge for Rust FFI — umbrella header pulled in by the +// cxx-generated bridge (`include!("FdbBridge.h")` in lib.rs) and by +// downstream `-sys` crates. Real declarations live in the per-topic headers +// below. +#pragma once + +// Note: the auto-generated `rust::behavior::trycatch` lives in +// `fdb_exceptions.h`, which is included by each per-topic `.cc` directly +// (not from this header). Downstream `-sys` crates have their own generated +// `_exceptions.h` and must not see fdb's transitively through here, or +// they would have two `trycatch` specializations in one translation unit. + +#include "ControlIteratorHandle.h" +#include "DumpIteratorHandle.h" +#include "FdbHandle.h" +#include "Key.h" +#include "Library.h" +#include "ListIteratorHandle.h" +#include "MessageArchiverWrapper.h" +#include "PurgeIteratorHandle.h" +#include "StatsIteratorHandle.h" +#include "StatusIteratorHandle.h" +#include "Types.h" +#include "WipeIteratorHandle.h" diff --git a/rust/crates/fdb-sys/cpp/FdbHandle.cc b/rust/crates/fdb-sys/cpp/FdbHandle.cc new file mode 100644 index 000000000..1060cfd95 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/FdbHandle.cc @@ -0,0 +1,301 @@ +// fdb FDB bridge — implementation. + +#include "fdb_exceptions.h" + +#include "ControlIteratorHandle.h" +#include "DumpIteratorHandle.h" +#include "FdbHandle.h" +#include "Key.h" +#include "ListIteratorHandle.h" +#include "PurgeIteratorHandle.h" +#include "StatsIteratorHandle.h" +#include "StatusIteratorHandle.h" +#include "WipeIteratorHandle.h" + +#include "fdb-sys/src/lib.rs.h" +#include "metkit-sys/src/lib.rs.h" + +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/config/Config.h" +#include "fdb5/fdb5_version.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/io/DataHandle.h" +#include "eckit/runtime/Main.h" +#include "metkit/mars/MarsRequest.h" + +#include +#include +#include +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +FdbHandle::FdbHandle() = default; + +FdbHandle::FdbHandle(const eckit_bridge::ConfigWrapper& config) : impl_(fdb5::Config(config.inner())) {} + +FdbHandle::FdbHandle(const eckit_bridge::ConfigWrapper& config, const eckit_bridge::ConfigWrapper& user_config) : + impl_(fdb5::Config(config.inner(), user_config.inner())) {} + +FdbHandle::~FdbHandle() = default; + +bool FdbHandle::dirty() const { + return impl_.dirty(); +} + +void FdbHandle::flush() { + impl_.flush(); +} + +FdbStatsData FdbHandle::stats() const { + auto s = impl_.stats(); + FdbStatsData data; + data.num_archive = s.numArchive(); + data.num_location = s.numLocation(); + data.num_flush = s.numFlush(); + return data; +} + +bool FdbHandle::enabled(fdb5::ControlIdentifier identifier) const { + return impl_.enabled(identifier); +} + +rust::String FdbHandle::id() const { + return rust::String(impl_.id()); +} + +rust::String FdbHandle::name() const { + return rust::String(impl_.name()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +void FdbHandle::archive(const KeyData& key, rust::Slice data) { + fdb5::Key fdb_key = Key::from_data(key); + impl_.archive(fdb_key, data.data(), data.size()); +} + +void FdbHandle::archive_raw(rust::Slice data) { + impl_.archive(data.data(), data.size()); +} + +namespace { + +/// `eckit::DataHandle` adapter that pulls bytes from a Rust `std::io::Read` +/// source via the cxx callback `invoke_reader_read`. Used by +/// `FdbHandle::archive_reader` to stream Rust-side data into +/// `fdb5::FDB::archive(eckit::DataHandle&)` without buffering the whole +/// payload in memory first. +class RustReaderHandle : public eckit::DataHandle { +public: + + explicit RustReaderHandle(rust::Box reader) : reader_(std::move(reader)) {} + + void print(std::ostream& s) const override { s << "RustReaderHandle[]"; } + + eckit::Length openForRead() override { return eckit::Length(0); } + + long read(void* buffer, long length) override { + if (length <= 0) { + return 0; + } + auto* bytes = static_cast(buffer); + rust::Slice slice{bytes, static_cast(length)}; + int64_t n = invoke_reader_read(*reader_, slice); + if (n < 0) { + throw eckit::ReadError("RustReaderHandle: error reading from Rust source"); + } + return static_cast(n); + } + + void close() override {} + + eckit::Length estimate() override { return eckit::Length(0); } + + eckit::Length size() override { return eckit::Length(0); } + +private: + + rust::Box reader_; +}; + +} // namespace + +void FdbHandle::archive_reader(rust::Box reader) { + RustReaderHandle adapter(std::move(reader)); + impl_.archive(adapter); +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::unique_ptr FdbHandle::retrieve(const metkit_bridge::MarsRequestWrapper& request) { + return std::make_unique(impl_.retrieve(request.inner())); +} + +std::unique_ptr FdbHandle::read_uri(rust::Str uri) { + std::string uri_str{uri}; + eckit::URI eckit_uri{uri_str}; + return std::make_unique(impl_.read(eckit_uri)); +} + +std::unique_ptr FdbHandle::read_uris(const rust::Vec& uris, + bool in_storage_order) { + std::vector eckit_uris; + eckit_uris.reserve(uris.size()); + for (const auto& uri : uris) { + eckit_uris.emplace_back(std::string(uri)); + } + return std::make_unique(impl_.read(eckit_uris, in_storage_order)); +} + +std::unique_ptr FdbHandle::read_list_iterator(ListIteratorHandle& iterator, + bool in_storage_order) { + return std::make_unique(impl_.read(iterator.inner(), in_storage_order)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::unique_ptr FdbHandle::list(const metkit_bridge::MarsRequestWrapper& request, bool deduplicate, + int32_t level) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + auto it = impl_.list(tool_request, deduplicate, level); + return std::make_unique(std::move(it)); +} + +rust::Vec FdbHandle::axes(const metkit_bridge::MarsRequestWrapper& request, int32_t level) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + auto index_axis = impl_.axes(tool_request, level); + + rust::Vec result; + auto axes_map = index_axis.map(); + for (const auto& [axis_name, values_set] : axes_map) { + AxisEntry entry; + entry.key = rust::String(axis_name); + for (const auto& v : values_set) { + entry.values.push_back(rust::String(v)); + } + result.push_back(std::move(entry)); + } + return result; +} + +std::unique_ptr FdbHandle::dump(const metkit_bridge::MarsRequestWrapper& request, bool simple) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + auto it = impl_.dump(tool_request, simple); + return std::make_unique(std::move(it)); +} + +std::unique_ptr FdbHandle::status(const metkit_bridge::MarsRequestWrapper& request) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + auto it = impl_.status(tool_request); + return std::make_unique(std::move(it)); +} + +std::unique_ptr FdbHandle::wipe(const metkit_bridge::MarsRequestWrapper& request, bool doit, + bool porcelain, bool unsafe_wipe_all) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + auto it = impl_.wipe(tool_request, doit, porcelain, unsafe_wipe_all); + return std::make_unique(std::move(it)); +} + +std::unique_ptr FdbHandle::purge(const metkit_bridge::MarsRequestWrapper& request, bool doit, + bool porcelain) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + auto it = impl_.purge(tool_request, doit, porcelain); + return std::make_unique(std::move(it)); +} + +std::unique_ptr FdbHandle::stats_iterator(const metkit_bridge::MarsRequestWrapper& request) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + auto it = impl_.stats(tool_request); + return std::make_unique(std::move(it)); +} + +std::unique_ptr FdbHandle::control(const metkit_bridge::MarsRequestWrapper& request, + fdb5::ControlAction action, + rust::Slice identifiers) { + const auto& mars = request.inner(); + auto tool_request = fdb5::FDBToolRequest{mars, mars.empty(), std::vector{}}; + + fdb5::ControlIdentifiers ctrl_ids; + for (auto id : identifiers) { + ctrl_ids |= id; + } + + auto it = impl_.control(tool_request, action, ctrl_ids); + return std::make_unique(std::move(it)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +void FdbHandle::register_flush_callback(rust::Box callback) { + auto callback_ptr = std::make_shared>(std::move(callback)); + + fdb5::FlushCallback cpp_callback = [callback_ptr]() { invoke_flush_callback(**callback_ptr); }; + + impl_.registerFlushCallback(std::move(cpp_callback)); +} + +void FdbHandle::register_archive_callback(rust::Box callback) { + auto callback_ptr = std::make_shared>(std::move(callback)); + + fdb5::ArchiveCallback cpp_callback = [callback_ptr]( + const fdb5::Key& key, const void* data, size_t length, + std::future> location_future) { + rust::Vec key_vec = Key::to_data(key); + + rust::Slice data_slice{static_cast(data), length}; + + std::string location_uri; + uint64_t location_offset = 0; + uint64_t location_length = 0; + + try { + auto location = location_future.get(); + if (location) { + location_uri = location->uri().asRawString(); + location_offset = location->offset(); + location_length = location->length(); + } + } + catch (const std::exception&) { + // best-effort — leave location info empty on future failure + } + + rust::Slice key_slice{key_vec.data(), key_vec.size()}; + + invoke_archive_callback(**callback_ptr, key_slice, data_slice, rust::Str(location_uri), location_offset, + location_length); + }; + + impl_.registerArchiveCallback(std::move(cpp_callback)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::unique_ptr FdbHandle::create() { + return std::make_unique(); +} + +std::unique_ptr FdbHandle::from_config(const eckit_bridge::ConfigWrapper& config) { + return std::make_unique(config); +} + +std::unique_ptr FdbHandle::from_config_with_user(const eckit_bridge::ConfigWrapper& config, + const eckit_bridge::ConfigWrapper& user_config) { + return std::make_unique(config, user_config); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/FdbHandle.h b/rust/crates/fdb-sys/cpp/FdbHandle.h new file mode 100644 index 000000000..a4da76e90 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/FdbHandle.h @@ -0,0 +1,129 @@ +// fdb FDB bridge — wraps `fdb5::FDB`. +#pragma once + +#include "Types.h" + +#include "eckit-sys/src/lib.rs.h" // ConfigWrapper / DataHandleWrapper +#include "metkit-sys/src/lib.rs.h" // MarsRequestWrapper + +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/ControlIterator.h" + +#include "rust/cxx.h" + +#include +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +class ListIteratorHandle; +class DumpIteratorHandle; +class StatusIteratorHandle; +class WipeIteratorHandle; +class PurgeIteratorHandle; +class StatsIteratorHandle; +class ControlIteratorHandle; + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::FDB` for Rust FFI. +class FdbHandle { +public: + + FdbHandle(); + explicit FdbHandle(const eckit_bridge::ConfigWrapper& config); + FdbHandle(const eckit_bridge::ConfigWrapper& config, const eckit_bridge::ConfigWrapper& user_config); + + ~FdbHandle(); + + // Non-copyable + FdbHandle(const FdbHandle&) = delete; + FdbHandle& operator=(const FdbHandle&) = delete; + + // Movable + FdbHandle(FdbHandle&&) = default; + FdbHandle& operator=(FdbHandle&&) = default; + + /// Access the underlying FDB instance. + fdb5::FDB& inner() { return impl_; } + const fdb5::FDB& inner() const { return impl_; } + + // ============== Query / status ============== + + bool dirty() const; + void flush(); + FdbStatsData stats() const; + bool enabled(fdb5::ControlIdentifier identifier) const; + rust::String id() const; + rust::String name() const; + + // ============== Archive ============== + + /// Archive data with an explicit key. + void archive(const KeyData& key, rust::Slice data); + + /// Archive raw GRIB data (key is extracted from the message). + void archive_raw(rust::Slice data); + + /// Archive raw GRIB data streamed from a Rust `std::io::Read` source. + void archive_reader(rust::Box reader); + + // ============== Retrieve / read ============== + + /// Retrieve data matching a MARS request. + std::unique_ptr retrieve(const metkit_bridge::MarsRequestWrapper& request); + + /// Read data from a single URI. + std::unique_ptr read_uri(rust::Str uri); + + /// Read data from a list of URIs. + std::unique_ptr read_uris(const rust::Vec& uris, + bool in_storage_order); + + /// Read data from a list iterator — avoids URI round-tripping. + std::unique_ptr read_list_iterator(ListIteratorHandle& iterator, + bool in_storage_order); + + // ============== Query iterators ============== + + std::unique_ptr list(const metkit_bridge::MarsRequestWrapper& request, bool deduplicate, + int32_t level); + rust::Vec axes(const metkit_bridge::MarsRequestWrapper& request, int32_t level); + std::unique_ptr dump(const metkit_bridge::MarsRequestWrapper& request, bool simple); + std::unique_ptr status(const metkit_bridge::MarsRequestWrapper& request); + std::unique_ptr wipe(const metkit_bridge::MarsRequestWrapper& request, bool doit, + bool porcelain, bool unsafe_wipe_all); + std::unique_ptr purge(const metkit_bridge::MarsRequestWrapper& request, bool doit, + bool porcelain); + std::unique_ptr stats_iterator(const metkit_bridge::MarsRequestWrapper& request); + std::unique_ptr control(const metkit_bridge::MarsRequestWrapper& request, + fdb5::ControlAction action, + rust::Slice identifiers); + + // ============== Callbacks ============== + + void register_flush_callback(rust::Box callback); + void register_archive_callback(rust::Box callback); + + // ============== Factories ============== + + /// Create a new FDB handle with default configuration. + static std::unique_ptr create(); + + /// Create a new FDB handle from an eckit config. + static std::unique_ptr from_config(const eckit_bridge::ConfigWrapper& config); + + /// Create a new FDB handle from an eckit config with a user-config overlay. + static std::unique_ptr from_config_with_user(const eckit_bridge::ConfigWrapper& config, + const eckit_bridge::ConfigWrapper& user_config); + +private: + + fdb5::FDB impl_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/Key.cc b/rust/crates/fdb-sys/cpp/Key.cc new file mode 100644 index 000000000..b603997e5 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/Key.cc @@ -0,0 +1,33 @@ +// fdb Key bridge — implementation. + +#include "fdb_exceptions.h" + +#include "Key.h" +#include "fdb-sys/src/lib.rs.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +fdb5::Key Key::from_data(const KeyData& data) { + fdb5::Key key; + for (const auto& entry : data.entries) { + key.set(std::string(entry.key), std::string(entry.value)); + } + return key; +} + +rust::Vec Key::to_data(const fdb5::Key& key) { + rust::Vec result; + for (const auto& [k, v] : key) { + KeyValue kv; + kv.key = rust::String(k); + kv.value = rust::String(v); + result.push_back(std::move(kv)); + } + return result; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/Key.h b/rust/crates/fdb-sys/cpp/Key.h new file mode 100644 index 000000000..317bcf01e --- /dev/null +++ b/rust/crates/fdb-sys/cpp/Key.h @@ -0,0 +1,28 @@ +// fdb Key bridge — converters between the FFI `KeyData` struct and the +// underlying `fdb5::Key`. +#pragma once + +#include "Types.h" +#include "fdb5/database/Key.h" + +#include "rust/cxx.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Conversion helpers between the FFI `KeyData` shared struct and the C++ +/// `fdb5::Key` type. +class Key { +public: + + /// Build an `fdb5::Key` from the FFI `KeyData` carrier. + static fdb5::Key from_data(const KeyData& data); + + /// Render an `fdb5::Key` as a `Vec` for return over the bridge. + static rust::Vec to_data(const fdb5::Key& key); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/Library.cc b/rust/crates/fdb-sys/cpp/Library.cc new file mode 100644 index 000000000..842b7e4be --- /dev/null +++ b/rust/crates/fdb-sys/cpp/Library.cc @@ -0,0 +1,42 @@ +// fdb library metadata + runtime initialisation bridge — implementation. + +#include "fdb_exceptions.h" + +#include "Library.h" + +#include "fdb5/fdb5_version.h" + +#include "eckit/runtime/Main.h" + +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +std::once_flag g_init_flag; + +} // namespace + +void Library::initialise() { + std::call_once(g_init_flag, []() { + if (!eckit::Main::ready()) { + static const char* argv[] = {"fdb-sys", nullptr}; + eckit::Main::initialise(1, const_cast(argv)); + } + }); +} + +rust::String Library::version() { + return rust::String(fdb5_version_str()); +} + +rust::String Library::git_sha1() { + return rust::String(fdb5_git_sha1()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/Library.h b/rust/crates/fdb-sys/cpp/Library.h new file mode 100644 index 000000000..5af177a38 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/Library.h @@ -0,0 +1,25 @@ +// fdb library metadata + runtime initialisation bridge. +#pragma once + +#include "rust/cxx.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +class Library { +public: + + /// Initialise the FDB library (sets up `eckit::Main`). Idempotent. + static void initialise(); + + /// Get the FDB library version string. + static rust::String version(); + + /// Get the FDB git SHA1 hash. + static rust::String git_sha1(); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/ListIteratorHandle.cc b/rust/crates/fdb-sys/cpp/ListIteratorHandle.cc new file mode 100644 index 000000000..deb59eb45 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/ListIteratorHandle.cc @@ -0,0 +1,81 @@ +// fdb ListIteratorHandle bridge — implementation. + +#include "fdb_exceptions.h" + +#include "Key.h" +#include "ListIteratorHandle.h" +#include "fdb-sys/src/lib.rs.h" + +#include "eckit/exception/Exceptions.h" + +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +ListIteratorHandle::ListIteratorHandle(fdb5::ListIterator&& it) : impl_(std::move(it)) {} + +ListIteratorHandle::~ListIteratorHandle() = default; + +bool ListIteratorHandle::hasNext() { + if (exhausted_) { + return false; + } + if (has_current_) { + return true; + } + + if (impl_.next(current_)) { + has_current_ = true; + return true; + } + exhausted_ = true; + return false; +} + +ListElementData ListIteratorHandle::next() { + if (!has_current_ && !hasNext()) { + throw eckit::OutOfRange("Iterator exhausted", Here()); + } + + has_current_ = false; + + ListElementData data; + // `fullUri()` encodes the entry's offset in the URI fragment and its + // length in the `length` query parameter — round-trippable through + // `FdbHandle::read_uri` without manual seeking, mirroring upstream + // `fdb-url` / `fdb-hammer`. + data.uri = rust::String(current_.location().fullUri().asRawString()); + data.offset = current_.location().offset(); + data.length = current_.location().length(); + + const auto& keys = current_.keys(); + if (keys.size() > 0) { + data.db_key = Key::to_data(keys[0]); + } + if (keys.size() > 1) { + data.index_key = Key::to_data(keys[1]); + } + if (keys.size() > 2) { + data.datum_key = Key::to_data(keys[2]); + } + + data.timestamp = static_cast(current_.timestamp()); + + return data; +} + +CompactListingData ListIteratorHandle::dump_compact() { + std::ostringstream os; + auto [fields, length] = impl_.dumpCompact(os); + CompactListingData data; + data.text = rust::String(os.str()); + data.fields = static_cast(fields); + data.total_bytes = static_cast(length); + return data; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/ListIteratorHandle.h b/rust/crates/fdb-sys/cpp/ListIteratorHandle.h new file mode 100644 index 000000000..17c92399a --- /dev/null +++ b/rust/crates/fdb-sys/cpp/ListIteratorHandle.h @@ -0,0 +1,44 @@ +// fdb ListIteratorHandle bridge — wraps `fdb5::ListIterator`. +#pragma once + +#include "Types.h" + +#include "fdb5/api/helpers/ListIterator.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::ListIterator` for Rust FFI. +class ListIteratorHandle { +public: + + explicit ListIteratorHandle(fdb5::ListIterator&& it); + ~ListIteratorHandle(); + + ListIteratorHandle(const ListIteratorHandle&) = delete; + ListIteratorHandle& operator=(const ListIteratorHandle&) = delete; + ListIteratorHandle(ListIteratorHandle&&) = default; + ListIteratorHandle& operator=(ListIteratorHandle&&) = default; + + bool hasNext(); + ListElementData next(); + + /// Drain the iterator via `fdb5::ListIterator::dumpCompact`, returning the + /// aggregated MARS-request text plus the field/byte counters. + CompactListingData dump_compact(); + + /// Access the underlying `fdb5::ListIterator` for other bridge code. + fdb5::ListIterator& inner() { return impl_; } + +private: + + fdb5::ListIterator impl_; + fdb5::ListElement current_; + bool has_current_ = false; + bool exhausted_ = false; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/MessageArchiverWrapper.cc b/rust/crates/fdb-sys/cpp/MessageArchiverWrapper.cc new file mode 100644 index 000000000..1cd267d69 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/MessageArchiverWrapper.cc @@ -0,0 +1,38 @@ +// fdb MessageArchiver bridge — implementation. + +#include "fdb_exceptions.h" + +#include "Key.h" +#include "MessageArchiverWrapper.h" +#include "fdb-sys/src/lib.rs.h" + +#include "fdb5/config/Config.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +MessageArchiverWrapper::MessageArchiverWrapper(const KeyData& key, bool complete_transfers, bool verbose, + const eckit_bridge::ConfigWrapper& config) : + archiver_(Key::from_data(key), complete_transfers, verbose, fdb5::Config(config.inner())) {} + +int64_t MessageArchiverWrapper::archive(eckit_bridge::DataHandleWrapper& source) { + auto length = archiver_.archive(source.inner()); + return static_cast(static_cast(length)); +} + +void MessageArchiverWrapper::flush() { + archiver_.flush(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::unique_ptr MessageArchiverWrapper::create(const KeyData& key, bool complete_transfers, + bool verbose, + const eckit_bridge::ConfigWrapper& config) { + return std::make_unique(key, complete_transfers, verbose, config); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/MessageArchiverWrapper.h b/rust/crates/fdb-sys/cpp/MessageArchiverWrapper.h new file mode 100644 index 000000000..c4bd8daaf --- /dev/null +++ b/rust/crates/fdb-sys/cpp/MessageArchiverWrapper.h @@ -0,0 +1,44 @@ +// fdb MessageArchiver bridge — wraps `fdb5::MessageArchiver`. +#pragma once + +#include "Types.h" + +#include "eckit-sys/src/lib.rs.h" + +#include "fdb5/message/MessageArchiver.h" + +#include +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::MessageArchiver` — used by mars-client-cpp's +/// `FDBBase::archive`. The C++ ctor takes `(key, completeTransfers, verbose, +/// config)`; the wrapper exposes all four so the caller picks values +/// (mars-client-cpp uses an empty key + both flags `false`). +class MessageArchiverWrapper { + fdb5::MessageArchiver archiver_; + +public: + + MessageArchiverWrapper(const KeyData& key, bool complete_transfers, bool verbose, + const eckit_bridge::ConfigWrapper& config); + + /// `fdb5::MessageArchiver::archive(eckit::DataHandle&)` — returns bytes + /// archived (eckit::Length cast to int64). + int64_t archive(eckit_bridge::DataHandleWrapper& source); + + /// `fdb5::MessageArchiver::flush()`. + void flush(); + + // ============== Factories ============== + + static std::unique_ptr create(const KeyData& key, bool complete_transfers, bool verbose, + const eckit_bridge::ConfigWrapper& config); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/PurgeIteratorHandle.cc b/rust/crates/fdb-sys/cpp/PurgeIteratorHandle.cc new file mode 100644 index 000000000..29685fe2c --- /dev/null +++ b/rust/crates/fdb-sys/cpp/PurgeIteratorHandle.cc @@ -0,0 +1,52 @@ +// fdb PurgeIteratorHandle bridge — implementation. + +#include "fdb_exceptions.h" + +#include "PurgeIteratorHandle.h" +#include "fdb-sys/src/lib.rs.h" + +#include "eckit/exception/Exceptions.h" + +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +PurgeIteratorHandle::PurgeIteratorHandle(fdb5::PurgeIterator&& it) : impl_(std::move(it)) {} + +PurgeIteratorHandle::~PurgeIteratorHandle() = default; + +bool PurgeIteratorHandle::hasNext() { + if (exhausted_) { + return false; + } + if (has_current_) { + return true; + } + + if (impl_.next(current_)) { + has_current_ = true; + return true; + } + exhausted_ = true; + return false; +} + +PurgeElementData PurgeIteratorHandle::next() { + if (!has_current_ && !hasNext()) { + throw eckit::OutOfRange("Iterator exhausted", Here()); + } + + has_current_ = false; + + PurgeElementData data; + std::ostringstream ss; + ss << current_; + data.content = rust::String(ss.str()); + return data; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/PurgeIteratorHandle.h b/rust/crates/fdb-sys/cpp/PurgeIteratorHandle.h new file mode 100644 index 000000000..8871364eb --- /dev/null +++ b/rust/crates/fdb-sys/cpp/PurgeIteratorHandle.h @@ -0,0 +1,37 @@ +// fdb PurgeIteratorHandle bridge — wraps `fdb5::PurgeIterator`. +#pragma once + +#include "Types.h" + +#include "fdb5/api/helpers/PurgeIterator.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::PurgeIterator` for Rust FFI. +class PurgeIteratorHandle { +public: + + explicit PurgeIteratorHandle(fdb5::PurgeIterator&& it); + ~PurgeIteratorHandle(); + + PurgeIteratorHandle(const PurgeIteratorHandle&) = delete; + PurgeIteratorHandle& operator=(const PurgeIteratorHandle&) = delete; + PurgeIteratorHandle(PurgeIteratorHandle&&) = default; + PurgeIteratorHandle& operator=(PurgeIteratorHandle&&) = default; + + bool hasNext(); + PurgeElementData next(); + +private: + + fdb5::PurgeIterator impl_; + fdb5::PurgeElement current_; + bool has_current_ = false; + bool exhausted_ = false; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/StatsIteratorHandle.cc b/rust/crates/fdb-sys/cpp/StatsIteratorHandle.cc new file mode 100644 index 000000000..298400721 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/StatsIteratorHandle.cc @@ -0,0 +1,63 @@ +// fdb StatsIteratorHandle bridge — implementation. + +#include "fdb_exceptions.h" + +#include "StatsIteratorHandle.h" +#include "fdb-sys/src/lib.rs.h" + +#include "eckit/exception/Exceptions.h" + +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +StatsIteratorHandle::StatsIteratorHandle(fdb5::StatsIterator&& it) : impl_(std::move(it)) {} + +StatsIteratorHandle::~StatsIteratorHandle() = default; + +bool StatsIteratorHandle::hasNext() { + if (exhausted_) { + return false; + } + if (has_current_) { + return true; + } + + if (impl_.next(current_)) { + has_current_ = true; + return true; + } + exhausted_ = true; + return false; +} + +StatsElementData StatsIteratorHandle::next() { + if (!has_current_ && !hasNext()) { + throw eckit::OutOfRange("Iterator exhausted", Here()); + } + + has_current_ = false; + + StatsElementData data; + data.index_statistics.fields_count = current_.indexStatistics.fieldsCount(); + data.index_statistics.fields_size = current_.indexStatistics.fieldsSize(); + data.index_statistics.duplicates_count = current_.indexStatistics.duplicatesCount(); + data.index_statistics.duplicates_size = current_.indexStatistics.duplicatesSize(); + { + std::ostringstream os; + current_.indexStatistics.report(os); + data.index_statistics.report = os.str(); + } + { + std::ostringstream os; + current_.dbStatistics.report(os); + data.db_statistics.report = os.str(); + } + return data; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/StatsIteratorHandle.h b/rust/crates/fdb-sys/cpp/StatsIteratorHandle.h new file mode 100644 index 000000000..12e8baca6 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/StatsIteratorHandle.h @@ -0,0 +1,37 @@ +// fdb StatsIteratorHandle bridge — wraps `fdb5::StatsIterator`. +#pragma once + +#include "Types.h" + +#include "fdb5/api/helpers/StatsIterator.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::StatsIterator` for Rust FFI. +class StatsIteratorHandle { +public: + + explicit StatsIteratorHandle(fdb5::StatsIterator&& it); + ~StatsIteratorHandle(); + + StatsIteratorHandle(const StatsIteratorHandle&) = delete; + StatsIteratorHandle& operator=(const StatsIteratorHandle&) = delete; + StatsIteratorHandle(StatsIteratorHandle&&) = default; + StatsIteratorHandle& operator=(StatsIteratorHandle&&) = default; + + bool hasNext(); + StatsElementData next(); + +private: + + fdb5::StatsIterator impl_; + fdb5::StatsElement current_; + bool has_current_ = false; + bool exhausted_ = false; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/StatusIteratorHandle.cc b/rust/crates/fdb-sys/cpp/StatusIteratorHandle.cc new file mode 100644 index 000000000..50396764f --- /dev/null +++ b/rust/crates/fdb-sys/cpp/StatusIteratorHandle.cc @@ -0,0 +1,48 @@ +// fdb StatusIteratorHandle bridge — implementation. + +#include "fdb_exceptions.h" + +#include "StatusIteratorHandle.h" +#include "fdb-sys/src/lib.rs.h" + +#include "eckit/exception/Exceptions.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +StatusIteratorHandle::StatusIteratorHandle(fdb5::StatusIterator&& it) : impl_(std::move(it)) {} + +StatusIteratorHandle::~StatusIteratorHandle() = default; + +bool StatusIteratorHandle::hasNext() { + if (exhausted_) { + return false; + } + if (has_current_) { + return true; + } + + if (impl_.next(current_)) { + has_current_ = true; + return true; + } + exhausted_ = true; + return false; +} + +StatusElementData StatusIteratorHandle::next() { + if (!has_current_ && !hasNext()) { + throw eckit::OutOfRange("Iterator exhausted", Here()); + } + + has_current_ = false; + + StatusElementData data; + data.location = rust::String(current_.location.asString()); + return data; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/StatusIteratorHandle.h b/rust/crates/fdb-sys/cpp/StatusIteratorHandle.h new file mode 100644 index 000000000..64c401dcb --- /dev/null +++ b/rust/crates/fdb-sys/cpp/StatusIteratorHandle.h @@ -0,0 +1,37 @@ +// fdb StatusIteratorHandle bridge — wraps `fdb5::StatusIterator`. +#pragma once + +#include "Types.h" + +#include "fdb5/api/helpers/StatusIterator.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::StatusIterator` for Rust FFI. +class StatusIteratorHandle { +public: + + explicit StatusIteratorHandle(fdb5::StatusIterator&& it); + ~StatusIteratorHandle(); + + StatusIteratorHandle(const StatusIteratorHandle&) = delete; + StatusIteratorHandle& operator=(const StatusIteratorHandle&) = delete; + StatusIteratorHandle(StatusIteratorHandle&&) = default; + StatusIteratorHandle& operator=(StatusIteratorHandle&&) = default; + + bool hasNext(); + StatusElementData next(); + +private: + + fdb5::StatusIterator impl_; + fdb5::StatusElement current_; + bool has_current_ = false; + bool exhausted_ = false; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/Types.h b/rust/crates/fdb-sys/cpp/Types.h new file mode 100644 index 000000000..6e8eef623 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/Types.h @@ -0,0 +1,35 @@ +// fdb C++ bridge — forward declarations for the cxx-shared data structs. +// +// All structs are defined on the Rust side via the cxx bridge in `lib.rs`; +// this header just exposes them to the C++ wrapper code that consumes them. +#pragma once + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +struct KeyValue; +struct KeyData; +struct ListElementData; +struct CompactListingData; +struct AxisEntry; +struct FdbStatsData; +struct DumpElementData; +struct StatusElementData; +struct WipeElementData; +struct PurgeElementData; +struct IndexStatsData; +struct DbStatsData; +struct StatsElementData; +struct ControlElementData; + +//---------------------------------------------------------------------------------------------------------------------- + +// Forward declarations for Rust-side opaque boxes; defined in `lib.rs`. +struct FlushCallbackBox; +struct ArchiveCallbackBox; +struct ReaderBox; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/WipeIteratorHandle.cc b/rust/crates/fdb-sys/cpp/WipeIteratorHandle.cc new file mode 100644 index 000000000..f4e750847 --- /dev/null +++ b/rust/crates/fdb-sys/cpp/WipeIteratorHandle.cc @@ -0,0 +1,52 @@ +// fdb WipeIteratorHandle bridge — implementation. + +#include "fdb_exceptions.h" + +#include "WipeIteratorHandle.h" +#include "fdb-sys/src/lib.rs.h" + +#include "eckit/exception/Exceptions.h" + +#include + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +WipeIteratorHandle::WipeIteratorHandle(fdb5::WipeIterator&& it) : impl_(std::move(it)) {} + +WipeIteratorHandle::~WipeIteratorHandle() = default; + +bool WipeIteratorHandle::hasNext() { + if (exhausted_) { + return false; + } + if (has_current_) { + return true; + } + + if (impl_.next(current_)) { + has_current_ = true; + return true; + } + exhausted_ = true; + return false; +} + +WipeElementData WipeIteratorHandle::next() { + if (!has_current_ && !hasNext()) { + throw eckit::OutOfRange("Iterator exhausted", Here()); + } + + has_current_ = false; + + WipeElementData data; + std::ostringstream ss; + ss << current_; + data.content = rust::String(ss.str()); + return data; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/WipeIteratorHandle.h b/rust/crates/fdb-sys/cpp/WipeIteratorHandle.h new file mode 100644 index 000000000..4dba1250b --- /dev/null +++ b/rust/crates/fdb-sys/cpp/WipeIteratorHandle.h @@ -0,0 +1,37 @@ +// fdb WipeIteratorHandle bridge — wraps `fdb5::WipeIterator`. +#pragma once + +#include "Types.h" + +#include "fdb5/api/helpers/WipeIterator.h" + +namespace fdb_bridge { + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wraps `fdb5::WipeIterator` for Rust FFI. +class WipeIteratorHandle { +public: + + explicit WipeIteratorHandle(fdb5::WipeIterator&& it); + ~WipeIteratorHandle(); + + WipeIteratorHandle(const WipeIteratorHandle&) = delete; + WipeIteratorHandle& operator=(const WipeIteratorHandle&) = delete; + WipeIteratorHandle(WipeIteratorHandle&&) = default; + WipeIteratorHandle& operator=(WipeIteratorHandle&&) = default; + + bool hasNext(); + WipeElementData next(); + +private: + + fdb5::WipeIterator impl_; + fdb5::WipeElement current_; + bool has_current_ = false; + bool exhausted_ = false; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb_bridge diff --git a/rust/crates/fdb-sys/cpp/fdb_bridge.cpp b/rust/crates/fdb-sys/cpp/fdb_bridge.cpp deleted file mode 100644 index 703608b2d..000000000 --- a/rust/crates/fdb-sys/cpp/fdb_bridge.cpp +++ /dev/null @@ -1,887 +0,0 @@ -// fdb_bridge.cpp - C++ bridge implementation -// -// This file implements the shim functions that convert between the native -// FDB5 C++ API and cxx-compatible types. - -#include "fdb_bridge.h" - -#include "fdb5/api/helpers/FDBToolRequest.h" -#include "fdb5/config/Config.h" -#include "fdb5/database/Key.h" -#include "fdb5/fdb5_version.h" - -#include "eckit/config/YAMLConfiguration.h" -#include "eckit/exception/Exceptions.h" -#include "eckit/filesystem/PathName.h" -#include "eckit/runtime/Main.h" -#include "metkit/mars/MarsExpansion.h" -#include "metkit/mars/MarsParsedRequest.h" -#include "metkit/mars/MarsParser.h" -#include "metkit/mars/MarsRequest.h" - -#include -#include -#include - -// Include the cxx-generated header for our bridge types -#include "fdb-sys/src/lib.rs.h" - -namespace fdb::ffi { - -// ============================================================================ -// Initialization -// ============================================================================ - -static std::once_flag init_flag; - -void fdb_init() { - std::call_once(init_flag, []() { - // Initialize eckit::Main if not already initialized - if (!eckit::Main::ready()) { - static const char* argv[] = {"fdb-sys", nullptr}; - eckit::Main::initialise(1, const_cast(argv)); - } - }); -} - -// ============================================================================ -// Helper functions for type conversion -// ============================================================================ - -/// Convert KeyData to fdb5::Key -static fdb5::Key to_fdb_key(const KeyData& data) { - fdb5::Key key; - for (const auto& entry : data.entries) { - key.set(std::string(entry.key), std::string(entry.value)); - } - return key; -} - -/// Convert fdb5::Key to Vec -static rust::Vec from_fdb_key(const fdb5::Key& key) { - rust::Vec result; - for (const auto& [k, v] : key) { - KeyValue kv; - kv.key = rust::String(k); - kv.value = rust::String(v); - result.push_back(std::move(kv)); - } - return result; -} - -/// Parse a MARS request string into a fully-expanded `metkit::mars::MarsRequest`. -/// -/// Uses the same parser + expansion pipeline as upstream FDB tools (see -/// `fdb5::FDBToolRequest::requestsFromString`): -/// -/// 1. Prepend a dummy verb (`retrieve`) so `MarsParser` accepts the input. -/// 2. Run `MarsParser::parse()` to produce a `MarsParsedRequest`. -/// 3. Run `MarsExpansion::expand()` to apply `to`/`by` ranges, type -/// expansion, optional fields, etc. -/// -/// An empty request string is returned as a default-constructed -/// `MarsRequest` (matches everything) without invoking the parser. -/// -/// Throws on any parser/expansion error; the global `rust::behavior::trycatch` -/// turns the exception into a Rust `Result::Err`. -static metkit::mars::MarsRequest parse_to_mars_request(const std::string& request_str) { - if (request_str.empty()) { - return metkit::mars::MarsRequest{}; - } - - // MarsParser requires a verb at the start of the input. Use "retrieve" - // as the canonical verb (matches what `FDBToolRequest::requestsFromString` - // defaults to). The verb itself is discarded by MarsExpansion. - std::string full = "retrieve," + request_str; - std::istringstream in(full); - metkit::mars::MarsParser parser(in); - auto parsed = parser.parse(); - ASSERT(parsed.size() == 1); - - metkit::mars::MarsExpansion expand(/*inherit*/ false, /*strict*/ true); - auto expanded = expand.expand(parsed); - ASSERT(expanded.size() == 1); - return std::move(expanded.front()); -} - -/// Create an `FDBToolRequest` from a MARS request string. -static fdb5::FDBToolRequest make_tool_request(const std::string& request_str) { - auto mars = parse_to_mars_request(request_str); - // If the request is empty, match all; otherwise filter by request. - bool all = mars.empty(); - return fdb5::FDBToolRequest{mars, all, std::vector{}}; -} - -// ============================================================================ -// FdbHandle implementation -// ============================================================================ - -FdbHandle::FdbHandle() = default; - -FdbHandle::FdbHandle(const std::string& yaml_config) : - impl_([&] { - eckit::YAMLConfiguration config(yaml_config); - fdb5::Config fdb_config(config); - return fdb5::FDB(fdb_config); - }()) {} - -FdbHandle::FdbHandle(const std::string& yaml_config, const std::string& yaml_user_config) : - impl_([&] { - eckit::YAMLConfiguration config(yaml_config); - eckit::YAMLConfiguration user_config(yaml_user_config); - fdb5::Config fdb_config(config, user_config); - return fdb5::FDB(fdb_config); - }()) {} - -FdbHandle::FdbHandle(FromPathTag, const std::string& path) : - impl_([&] { - // `Config::make` loads YAML/JSON from the given path, expands - // `~fdb` and `fdb_home` references, and returns a fully-resolved - // `fdb5::Config`. This is the same entry point upstream FDB tools - // use when handed a `--config-file` / `FDB_CONFIG_FILE`. - return fdb5::FDB(fdb5::Config::make(eckit::PathName(path))); - }()) {} - -FdbHandle::FdbHandle(FromPathTag, const std::string& path, const std::string& yaml_user_config) : - impl_([&] { - eckit::YAMLConfiguration user_config(yaml_user_config); - return fdb5::FDB(fdb5::Config::make(eckit::PathName(path), user_config)); - }()) {} - -FdbHandle::~FdbHandle() = default; - -bool FdbHandle::dirty() const { - return impl_.dirty(); -} - -void FdbHandle::flush() { - impl_.flush(); -} - -FdbStatsData FdbHandle::stats() const { - auto s = impl_.stats(); - FdbStatsData data; - data.num_archive = s.numArchive(); - data.num_location = s.numLocation(); - data.num_flush = s.numFlush(); - return data; -} - -bool FdbHandle::enabled(fdb5::ControlIdentifier identifier) const { - return impl_.enabled(identifier); -} - -rust::String FdbHandle::id() const { - return rust::String(impl_.id()); -} - -rust::String FdbHandle::name() const { - return rust::String(impl_.name()); -} - -// ============================================================================ -// eckit::DataHandle shim functions -// ============================================================================ - -uint64_t data_handle_open(eckit::DataHandle& handle) { - return static_cast(handle.openForRead()); -} - -void data_handle_close(eckit::DataHandle& handle) { - handle.close(); -} - -size_t data_handle_read(eckit::DataHandle& handle, rust::Slice buffer) { - long n = handle.read(buffer.data(), static_cast(buffer.size())); - return n < 0 ? 0 : static_cast(n); -} - -void data_handle_seek(eckit::DataHandle& handle, uint64_t position) { - handle.seek(eckit::Offset(position)); -} - -uint64_t data_handle_tell(eckit::DataHandle& handle) { - return static_cast(handle.position()); -} - -uint64_t data_handle_size(eckit::DataHandle& handle) { - return static_cast(handle.size()); -} - -// ============================================================================ -// ListIteratorHandle implementation -// ============================================================================ - -ListIteratorHandle::ListIteratorHandle(fdb5::ListIterator&& it) : impl_(std::move(it)) {} - -ListIteratorHandle::~ListIteratorHandle() = default; - -bool ListIteratorHandle::hasNext() { - if (exhausted_) { - return false; - } - if (has_current_) { - return true; - } - - // Try to fetch next element - if (impl_.next(current_)) { - has_current_ = true; - return true; - } - else { - exhausted_ = true; - return false; - } -} - -ListElementData ListIteratorHandle::next() { - if (!has_current_ && !hasNext()) { - throw eckit::OutOfRange("Iterator exhausted", Here()); - } - - has_current_ = false; - - ListElementData data; - // Use `fullUri()` (not `uri()`) so the resulting string encodes the - // entry's offset in the URI fragment and its length in the `length` query - // parameter. This matches what `FieldLocation(const eckit::URI&)` parses - // back, so the URI is round-trippable through `read_uri()` without the - // caller having to seek manually. Same pattern as the upstream - // `fdb-url`/`fdb-hammer` tools. - data.uri = rust::String(current_.location().fullUri().asRawString()); - data.offset = current_.location().offset(); - data.length = current_.location().length(); - - // Extract keys - const auto& keys = current_.keys(); - if (keys.size() > 0) { - data.db_key = from_fdb_key(keys[0]); - } - if (keys.size() > 1) { - data.index_key = from_fdb_key(keys[1]); - } - if (keys.size() > 2) { - data.datum_key = from_fdb_key(keys[2]); - } - - // Convert timestamp to epoch seconds - data.timestamp = static_cast(current_.timestamp()); - - return data; -} - -// ============================================================================ -// DumpIteratorHandle implementation -// ============================================================================ - -DumpIteratorHandle::DumpIteratorHandle(fdb5::DumpIterator&& it) : impl_(std::move(it)) {} - -DumpIteratorHandle::~DumpIteratorHandle() = default; - -bool DumpIteratorHandle::hasNext() { - if (exhausted_) { - return false; - } - if (has_current_) { - return true; - } - - if (impl_.next(current_)) { - has_current_ = true; - return true; - } - else { - exhausted_ = true; - return false; - } -} - -DumpElementData DumpIteratorHandle::next() { - if (!has_current_ && !hasNext()) { - throw eckit::OutOfRange("Iterator exhausted", Here()); - } - - has_current_ = false; - - DumpElementData data; - // DumpElement is a string - data.content = rust::String(current_); - return data; -} - -// ============================================================================ -// StatusIteratorHandle implementation -// ============================================================================ - -StatusIteratorHandle::StatusIteratorHandle(fdb5::StatusIterator&& it) : impl_(std::move(it)) {} - -StatusIteratorHandle::~StatusIteratorHandle() = default; - -bool StatusIteratorHandle::hasNext() { - if (exhausted_) { - return false; - } - if (has_current_) { - return true; - } - - if (impl_.next(current_)) { - has_current_ = true; - return true; - } - else { - exhausted_ = true; - return false; - } -} - -StatusElementData StatusIteratorHandle::next() { - if (!has_current_ && !hasNext()) { - throw eckit::OutOfRange("Iterator exhausted", Here()); - } - - has_current_ = false; - - StatusElementData data; - data.location = rust::String(current_.location.asString()); - return data; -} - -// ============================================================================ -// WipeIteratorHandle implementation -// ============================================================================ - -WipeIteratorHandle::WipeIteratorHandle(fdb5::WipeIterator&& it) : impl_(std::move(it)) {} - -WipeIteratorHandle::~WipeIteratorHandle() = default; - -bool WipeIteratorHandle::hasNext() { - if (exhausted_) { - return false; - } - if (has_current_) { - return true; - } - - if (impl_.next(current_)) { - has_current_ = true; - return true; - } - else { - exhausted_ = true; - return false; - } -} - -WipeElementData WipeIteratorHandle::next() { - if (!has_current_ && !hasNext()) { - throw eckit::OutOfRange("Iterator exhausted", Here()); - } - - has_current_ = false; - - WipeElementData data; - std::ostringstream ss; - ss << current_; - data.content = rust::String(ss.str()); - return data; -} - -// ============================================================================ -// PurgeIteratorHandle implementation -// ============================================================================ - -PurgeIteratorHandle::PurgeIteratorHandle(fdb5::PurgeIterator&& it) : impl_(std::move(it)) {} - -PurgeIteratorHandle::~PurgeIteratorHandle() = default; - -bool PurgeIteratorHandle::hasNext() { - if (exhausted_) { - return false; - } - if (has_current_) { - return true; - } - - if (impl_.next(current_)) { - has_current_ = true; - return true; - } - else { - exhausted_ = true; - return false; - } -} - -PurgeElementData PurgeIteratorHandle::next() { - if (!has_current_ && !hasNext()) { - throw eckit::OutOfRange("Iterator exhausted", Here()); - } - - has_current_ = false; - - PurgeElementData data; - std::ostringstream ss; - ss << current_; - data.content = rust::String(ss.str()); - return data; -} - -// ============================================================================ -// StatsIteratorHandle implementation -// ============================================================================ - -StatsIteratorHandle::StatsIteratorHandle(fdb5::StatsIterator&& it) : impl_(std::move(it)) {} - -StatsIteratorHandle::~StatsIteratorHandle() = default; - -bool StatsIteratorHandle::hasNext() { - if (exhausted_) { - return false; - } - if (has_current_) { - return true; - } - - if (impl_.next(current_)) { - has_current_ = true; - return true; - } - else { - exhausted_ = true; - return false; - } -} - -StatsElementData StatsIteratorHandle::next() { - if (!has_current_ && !hasNext()) { - throw eckit::OutOfRange("Iterator exhausted", Here()); - } - - has_current_ = false; - - // Mirror `fdb5::StatsElement { IndexStats; DbStats; }` directly. - // For `IndexStats` we can read every numeric accessor; for - // `DbStats` upstream only exposes `report(ostream&)`, so the - // captured text is the only thing we can surface. - StatsElementData data; - data.index_statistics.fields_count = current_.indexStatistics.fieldsCount(); - data.index_statistics.fields_size = current_.indexStatistics.fieldsSize(); - data.index_statistics.duplicates_count = current_.indexStatistics.duplicatesCount(); - data.index_statistics.duplicates_size = current_.indexStatistics.duplicatesSize(); - { - std::ostringstream os; - current_.indexStatistics.report(os); - data.index_statistics.report = os.str(); - } - { - std::ostringstream os; - current_.dbStatistics.report(os); - data.db_statistics.report = os.str(); - } - return data; -} - -// ============================================================================ -// ControlIteratorHandle implementation -// ============================================================================ - -ControlIteratorHandle::ControlIteratorHandle(fdb5::ControlIterator&& it) : impl_(std::move(it)) {} - -ControlIteratorHandle::~ControlIteratorHandle() = default; - -bool ControlIteratorHandle::hasNext() { - if (exhausted_) { - return false; - } - if (has_current_) { - return true; - } - - if (impl_.next(current_)) { - has_current_ = true; - return true; - } - else { - exhausted_ = true; - return false; - } -} - -ControlElementData ControlIteratorHandle::next() { - if (!has_current_ && !hasNext()) { - throw eckit::OutOfRange("Iterator exhausted", Here()); - } - - has_current_ = false; - - ControlElementData data; - data.location = rust::String(current_.location.asString()); - for (const auto& id : current_.controlIdentifiers) { - data.identifiers.push_back(id); - } - return data; -} - -// ============================================================================ -// Library metadata functions -// ============================================================================ - -rust::String fdb_version() { - return rust::String(fdb5_version_str()); -} - -rust::String fdb_git_sha1() { - return rust::String(fdb5_git_sha1()); -} - -// ============================================================================ -// MARS request parsing -// ============================================================================ - -RequestData parse_mars_request(rust::Str request) { - // Parsing requires eckit to be initialised (type registries, log levels, - // etc.), but `parse_mars_request` is a free function that may be called - // before the user constructs an `Fdb`. Make it self-sufficient. - fdb_init(); - - auto mars = parse_to_mars_request(std::string(request)); - - RequestData out; - for (const auto& key : mars.params()) { - RequestParam param; - param.key = rust::String(key); - for (const auto& v : mars.values(key)) { - param.values.push_back(rust::String(v)); - } - out.params.push_back(std::move(param)); - } - return out; -} - -// ============================================================================ -// Handle lifecycle functions -// ============================================================================ - -std::unique_ptr new_fdb() { - return std::make_unique(); -} - -std::unique_ptr new_fdb_from_yaml(rust::Str config) { - return std::make_unique(std::string(config)); -} - -std::unique_ptr new_fdb_from_yaml_with_user_config(rust::Str config, rust::Str user_config) { - return std::make_unique(std::string(config), std::string(user_config)); -} - -std::unique_ptr new_fdb_from_path(rust::Str path) { - return std::make_unique(FdbHandle::FromPathTag{}, std::string(path)); -} - -std::unique_ptr new_fdb_from_path_with_user_config(rust::Str path, rust::Str user_config) { - return std::make_unique(FdbHandle::FromPathTag{}, std::string(path), std::string(user_config)); -} - -// ============================================================================ -// Archive functions -// ============================================================================ - -void FdbHandle::archive(const KeyData& key, rust::Slice data) { - fdb5::Key fdb_key = to_fdb_key(key); - inner().archive(fdb_key, data.data(), data.size()); -} - -void FdbHandle::archive_raw(rust::Slice data) { - inner().archive(data.data(), data.size()); -} - -namespace { - -/// `eckit::DataHandle` adapter that pulls bytes from a Rust `std::io::Read` -/// source via the cxx callback `invoke_reader_read`. Used by -/// `archive_reader` to stream Rust-side data into -/// `fdb5::FDB::archive(eckit::DataHandle&)` without buffering the whole -/// payload in memory first. -/// -/// Only the methods that `fdb5::FDB::archive(DataHandle&)` actually -/// touches are overridden — `openForRead`, `read`, `close`, `estimate`, -/// `size`, plus the abstract `print`. Everything else inherits the base -/// behaviour (which throws `NotImplemented` for the seek/write paths -/// `archive` never reaches). -class RustReaderHandle : public eckit::DataHandle { -public: - - explicit RustReaderHandle(rust::Box reader) : reader_(std::move(reader)) {} - - void print(std::ostream& s) const override { s << "RustReaderHandle[]"; } - - eckit::Length openForRead() override { return eckit::Length(0); } - - long read(void* buffer, long length) override { - if (length <= 0) { - return 0; - } - auto* bytes = static_cast(buffer); - rust::Slice slice{bytes, static_cast(length)}; - int64_t n = invoke_reader_read(*reader_, slice); - if (n < 0) { - throw eckit::ReadError("RustReaderHandle: error reading from Rust source"); - } - return static_cast(n); - } - - void close() override {} - - eckit::Length estimate() override { return eckit::Length(0); } - - eckit::Length size() override { return eckit::Length(0); } - -private: - - rust::Box reader_; -}; - -} // namespace - -void FdbHandle::archive_reader(rust::Box reader) { - RustReaderHandle adapter(std::move(reader)); - inner().archive(adapter); -} - -// ============================================================================ -// Retrieve functions -// ============================================================================ - -std::unique_ptr FdbHandle::retrieve(rust::Str request) { - auto mars = parse_to_mars_request(std::string(request)); - return std::unique_ptr(inner().retrieve(mars)); -} - -// ============================================================================ -// Read functions (by URI) -// ============================================================================ - -std::unique_ptr FdbHandle::read_uri(rust::Str uri) { - std::string uri_str{uri}; - eckit::URI eckit_uri{uri_str}; - return std::unique_ptr(inner().read(eckit_uri)); -} - -std::unique_ptr FdbHandle::read_uris(const rust::Vec& uris, bool in_storage_order) { - std::vector eckit_uris; - eckit_uris.reserve(uris.size()); - for (const auto& uri : uris) { - eckit_uris.emplace_back(std::string(uri)); - } - return std::unique_ptr(inner().read(eckit_uris, in_storage_order)); -} - -std::unique_ptr FdbHandle::read_list_iterator(ListIteratorHandle& iterator, bool in_storage_order) { - // Calls FDB::read(ListIterator&, bool) directly - most efficient path - return std::unique_ptr(inner().read(iterator.inner(), in_storage_order)); -} - -// ============================================================================ -// List functions -// ============================================================================ - -std::unique_ptr FdbHandle::list(rust::Str request, bool deduplicate, int32_t level) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - auto it = inner().list(tool_request, deduplicate, level); - return std::make_unique(std::move(it)); -} - -CompactListingData list_iterator_dump_compact(ListIteratorHandle& iterator) { - std::ostringstream os; - auto [fields, length] = iterator.inner().dumpCompact(os); - CompactListingData data; - data.text = rust::String(os.str()); - data.fields = static_cast(fields); - data.total_bytes = static_cast(length); - return data; -} - -// ============================================================================ -// Axes query functions -// ============================================================================ - -rust::Vec FdbHandle::axes(rust::Str request, int32_t level) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - auto index_axis = inner().axes(tool_request, level); - - rust::Vec result; - // Iterate over all axes using map() instead of hardcoded list - auto axes_map = index_axis.map(); - for (const auto& [axis_name, values_set] : axes_map) { - AxisEntry entry; - entry.key = rust::String(axis_name); - for (const auto& v : values_set) { - entry.values.push_back(rust::String(v)); - } - result.push_back(std::move(entry)); - } - return result; -} - -// ============================================================================ -// Dump functions -// ============================================================================ - -std::unique_ptr FdbHandle::dump(rust::Str request, bool simple) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - auto it = inner().dump(tool_request, simple); - return std::make_unique(std::move(it)); -} - -// ============================================================================ -// Status functions -// ============================================================================ - -std::unique_ptr FdbHandle::status(rust::Str request) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - auto it = inner().status(tool_request); - return std::make_unique(std::move(it)); -} - -// ============================================================================ -// Wipe functions -// ============================================================================ - -std::unique_ptr FdbHandle::wipe(rust::Str request, bool doit, bool porcelain, - bool unsafe_wipe_all) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - auto it = inner().wipe(tool_request, doit, porcelain, unsafe_wipe_all); - return std::make_unique(std::move(it)); -} - -// ============================================================================ -// Purge functions -// ============================================================================ - -std::unique_ptr FdbHandle::purge(rust::Str request, bool doit, bool porcelain) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - auto it = inner().purge(tool_request, doit, porcelain); - return std::make_unique(std::move(it)); -} - -// ============================================================================ -// Stats functions -// ============================================================================ - -std::unique_ptr FdbHandle::stats_iterator(rust::Str request) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - auto it = inner().stats(tool_request); - return std::make_unique(std::move(it)); -} - -// ============================================================================ -// Control functions -// ============================================================================ - -std::unique_ptr FdbHandle::control(rust::Str request, fdb5::ControlAction action, - rust::Slice identifiers) { - std::string request_str{request}; - auto tool_request = make_tool_request(request_str); - - fdb5::ControlIdentifiers ctrl_ids; - for (auto id : identifiers) { - ctrl_ids |= id; - } - - auto it = inner().control(tool_request, action, ctrl_ids); - return std::make_unique(std::move(it)); -} - -// ============================================================================ -// Callback registration functions -// ============================================================================ - -void FdbHandle::register_flush_callback(rust::Box callback) { - // Create a shared_ptr to hold the callback box so it can be captured by the lambda - auto callback_ptr = std::make_shared>(std::move(callback)); - - fdb5::FlushCallback cpp_callback = [callback_ptr]() { invoke_flush_callback(**callback_ptr); }; - - inner().registerFlushCallback(std::move(cpp_callback)); -} - -void FdbHandle::register_archive_callback(rust::Box callback) { - // Create a shared_ptr to hold the callback box so it can be captured by the lambda - auto callback_ptr = std::make_shared>(std::move(callback)); - - fdb5::ArchiveCallback cpp_callback = [callback_ptr]( - const fdb5::Key& key, const void* data, size_t length, - std::future> location_future) { - // Convert key to Vec - rust::Vec key_vec; - for (const auto& [k, v] : key) { - KeyValue kv; - kv.key = rust::String(k); - kv.value = rust::String(v); - key_vec.push_back(std::move(kv)); - } - - // Create a slice from the data - rust::Slice data_slice{static_cast(data), length}; - - // Wait for the location future and extract info - std::string location_uri; - uint64_t location_offset = 0; - uint64_t location_length = 0; - - try { - auto location = location_future.get(); - if (location) { - location_uri = location->uri().asRawString(); - location_offset = location->offset(); - location_length = location->length(); - } - } - catch (const std::exception&) { - // If future fails, leave location info empty (best-effort) - } - - // Create a slice from key_vec - rust::Slice key_slice{key_vec.data(), key_vec.size()}; - - invoke_archive_callback(**callback_ptr, key_slice, data_slice, rust::Str(location_uri), location_offset, - location_length); - }; - - inner().registerArchiveCallback(std::move(cpp_callback)); -} - -// ============================================================================ -// Test functions (for verifying exception handling) -// ============================================================================ - -void test_throw_eckit_exception() { - throw eckit::Exception("test eckit exception"); -} - -void test_throw_eckit_serious_bug() { - throw eckit::SeriousBug("test serious bug"); -} - -void test_throw_eckit_user_error() { - throw eckit::UserError("test user error"); -} - -void test_throw_std_exception() { - throw std::runtime_error("test std exception"); -} - -void test_throw_int() { - throw 42; -} - -} // namespace fdb::ffi diff --git a/rust/crates/fdb-sys/cpp/fdb_bridge.h b/rust/crates/fdb-sys/cpp/fdb_bridge.h deleted file mode 100644 index 5728be78c..000000000 --- a/rust/crates/fdb-sys/cpp/fdb_bridge.h +++ /dev/null @@ -1,459 +0,0 @@ -// fdb_bridge.h - C++ bridge declarations for cxx -// -// This header declares wrapper types and shim functions that convert between -// the native FDB5 C++ API and cxx-compatible types. - -#pragma once - -#include "rust/cxx.h" - -#include -#include -#include - -// Include eckit exception for the global trycatch handler -#include "eckit/exception/Exceptions.h" - -// Custom exception handler for cxx - catches eckit exceptions globally -// This replaces per-function try-catch blocks throughout the bridge -// Exception messages are prefixed with type for Rust-side discrimination -// Order matters: catch specific exceptions before base classes -namespace rust::behavior { -template -static void trycatch(Try&& func, Fail&& fail) noexcept try { - func(); -} -catch (const eckit::SeriousBug& e) { - fail((std::string("ECKIT_SERIOUS_BUG: ") + e.what()).c_str()); -} -catch (const eckit::UserError& e) { - fail((std::string("ECKIT_USER_ERROR: ") + e.what()).c_str()); -} -catch (const eckit::BadParameter& e) { - fail((std::string("ECKIT_BAD_PARAMETER: ") + e.what()).c_str()); -} -catch (const eckit::NotImplemented& e) { - fail((std::string("ECKIT_NOT_IMPLEMENTED: ") + e.what()).c_str()); -} -catch (const eckit::OutOfRange& e) { - fail((std::string("ECKIT_OUT_OF_RANGE: ") + e.what()).c_str()); -} -catch (const eckit::FileError& e) { - fail((std::string("ECKIT_FILE_ERROR: ") + e.what()).c_str()); -} -catch (const eckit::AssertionFailed& e) { - fail((std::string("ECKIT_ASSERTION_FAILED: ") + e.what()).c_str()); -} -catch (const eckit::Exception& e) { - fail((std::string("ECKIT: ") + e.what()).c_str()); -} -catch (const std::exception& e) { - fail(e.what()); -} -// REQUIRED: catch(...) is necessary at FFI boundary to prevent undefined behavior. -catch (...) { - fail("unknown C++ exception (non-std::exception type)"); -} -} // namespace rust::behavior - -#include "fdb5/api/FDB.h" -#include "fdb5/api/helpers/ControlIterator.h" -#include "fdb5/api/helpers/DumpIterator.h" -#include "fdb5/api/helpers/ListIterator.h" -#include "fdb5/api/helpers/PurgeIterator.h" -#include "fdb5/api/helpers/StatsIterator.h" -#include "fdb5/api/helpers/StatusIterator.h" -#include "fdb5/api/helpers/WipeIterator.h" - -#include "eckit/io/DataHandle.h" - -namespace fdb::ffi { - -// ============================================================================ -// Shared struct forward declarations (defined by cxx in generated code) -// ============================================================================ - -struct KeyValue; -struct KeyData; -struct RequestData; -struct ListElementData; -struct CompactListingData; -struct AxisEntry; -struct FdbStatsData; -struct DumpElementData; -struct StatusElementData; -struct WipeElementData; -struct PurgeElementData; -struct IndexStatsData; -struct DbStatsData; -struct StatsElementData; -struct ControlElementData; - -// Forward declarations for types used by FdbHandle methods. -class ListIteratorHandle; -class DumpIteratorHandle; -class StatusIteratorHandle; -class WipeIteratorHandle; -class PurgeIteratorHandle; -class StatsIteratorHandle; -class ControlIteratorHandle; -struct ReaderBox; -struct FlushCallbackBox; -struct ArchiveCallbackBox; - -// ============================================================================ -// Wrapper classes for opaque C++ types -// ============================================================================ - -/// Wrapper around fdb5::FDB that can be passed through cxx. -class FdbHandle { -public: - - FdbHandle(); - explicit FdbHandle(const std::string& yaml_config); - FdbHandle(const std::string& yaml_config, const std::string& yaml_user_config); - - /// Tag type to disambiguate the path-loading constructor from the - /// YAML-string constructor (both take a `std::string`). - struct FromPathTag {}; - FdbHandle(FromPathTag, const std::string& path); - FdbHandle(FromPathTag, const std::string& path, const std::string& yaml_user_config); - - ~FdbHandle(); - - // Non-copyable - FdbHandle(const FdbHandle&) = delete; - FdbHandle& operator=(const FdbHandle&) = delete; - - // Movable - FdbHandle(FdbHandle&&) = default; - FdbHandle& operator=(FdbHandle&&) = default; - - /// Access the underlying FDB instance. - fdb5::FDB& inner() { return impl_; } - const fdb5::FDB& inner() const { return impl_; } - - // ------------------------------------------------------------------------- - // Methods exposed to Rust via cxx - // ------------------------------------------------------------------------- - - /// Check if the FDB has unflushed data. - bool dirty() const; - - /// Flush pending writes to disk. - void flush(); - - /// Get aggregate statistics. - FdbStatsData stats() const; - - /// Check if a control identifier is enabled. - bool enabled(fdb5::ControlIdentifier identifier) const; - - /// Get the FDB configuration ID. - rust::String id() const; - - /// Get the FDB type name. - rust::String name() const; - - // ------------------------------------------------------------------------- - // Operations (exposed to Rust as methods via cxx) - // ------------------------------------------------------------------------- - - void archive(const KeyData& key, rust::Slice data); - void archive_raw(rust::Slice data); - void archive_reader(rust::Box reader); - - std::unique_ptr retrieve(rust::Str request); - std::unique_ptr read_uri(rust::Str uri); - std::unique_ptr read_uris(const rust::Vec& uris, bool in_storage_order); - std::unique_ptr read_list_iterator(ListIteratorHandle& iterator, bool in_storage_order); - - std::unique_ptr list(rust::Str request, bool deduplicate, int32_t level); - rust::Vec axes(rust::Str request, int32_t level); - std::unique_ptr dump(rust::Str request, bool simple); - std::unique_ptr status(rust::Str request); - std::unique_ptr wipe(rust::Str request, bool doit, bool porcelain, bool unsafe_wipe_all); - std::unique_ptr purge(rust::Str request, bool doit, bool porcelain); - std::unique_ptr stats_iterator(rust::Str request); - std::unique_ptr control(rust::Str request, fdb5::ControlAction action, - rust::Slice identifiers); - - void register_flush_callback(rust::Box callback); - void register_archive_callback(rust::Box callback); - -private: - - fdb5::FDB impl_; -}; - -/// Wrapper around fdb5::ListIterator. -class ListIteratorHandle { -public: - - explicit ListIteratorHandle(fdb5::ListIterator&& it); - ~ListIteratorHandle(); - - // Non-copyable - ListIteratorHandle(const ListIteratorHandle&) = delete; - ListIteratorHandle& operator=(const ListIteratorHandle&) = delete; - - // Movable - ListIteratorHandle(ListIteratorHandle&&) = default; - ListIteratorHandle& operator=(ListIteratorHandle&&) = default; - - // Methods exposed to Rust via cxx - bool hasNext(); - ListElementData next(); - - /// Access the underlying ListIterator (for read_list_iterator). - fdb5::ListIterator& inner() { return impl_; } - -private: - - fdb5::ListIterator impl_; - fdb5::ListElement current_; - bool has_current_ = false; - bool exhausted_ = false; -}; - -/// Wrapper around fdb5::DumpIterator. -class DumpIteratorHandle { -public: - - explicit DumpIteratorHandle(fdb5::DumpIterator&& it); - ~DumpIteratorHandle(); - - DumpIteratorHandle(const DumpIteratorHandle&) = delete; - DumpIteratorHandle& operator=(const DumpIteratorHandle&) = delete; - DumpIteratorHandle(DumpIteratorHandle&&) = default; - DumpIteratorHandle& operator=(DumpIteratorHandle&&) = default; - - // Methods exposed to Rust via cxx - bool hasNext(); - DumpElementData next(); - -private: - - fdb5::DumpIterator impl_; - fdb5::DumpElement current_; - bool has_current_ = false; - bool exhausted_ = false; -}; - -/// Wrapper around fdb5::StatusIterator. -class StatusIteratorHandle { -public: - - explicit StatusIteratorHandle(fdb5::StatusIterator&& it); - ~StatusIteratorHandle(); - - StatusIteratorHandle(const StatusIteratorHandle&) = delete; - StatusIteratorHandle& operator=(const StatusIteratorHandle&) = delete; - StatusIteratorHandle(StatusIteratorHandle&&) = default; - StatusIteratorHandle& operator=(StatusIteratorHandle&&) = default; - - // Methods exposed to Rust via cxx - bool hasNext(); - StatusElementData next(); - -private: - - fdb5::StatusIterator impl_; - fdb5::StatusElement current_; - bool has_current_ = false; - bool exhausted_ = false; -}; - -/// Wrapper around fdb5::WipeIterator. -class WipeIteratorHandle { -public: - - explicit WipeIteratorHandle(fdb5::WipeIterator&& it); - ~WipeIteratorHandle(); - - WipeIteratorHandle(const WipeIteratorHandle&) = delete; - WipeIteratorHandle& operator=(const WipeIteratorHandle&) = delete; - WipeIteratorHandle(WipeIteratorHandle&&) = default; - WipeIteratorHandle& operator=(WipeIteratorHandle&&) = default; - - // Methods exposed to Rust via cxx - bool hasNext(); - WipeElementData next(); - -private: - - fdb5::WipeIterator impl_; - fdb5::WipeElement current_; - bool has_current_ = false; - bool exhausted_ = false; -}; - -/// Wrapper around fdb5::PurgeIterator. -class PurgeIteratorHandle { -public: - - explicit PurgeIteratorHandle(fdb5::PurgeIterator&& it); - ~PurgeIteratorHandle(); - - PurgeIteratorHandle(const PurgeIteratorHandle&) = delete; - PurgeIteratorHandle& operator=(const PurgeIteratorHandle&) = delete; - PurgeIteratorHandle(PurgeIteratorHandle&&) = default; - PurgeIteratorHandle& operator=(PurgeIteratorHandle&&) = default; - - // Methods exposed to Rust via cxx - bool hasNext(); - PurgeElementData next(); - -private: - - fdb5::PurgeIterator impl_; - fdb5::PurgeElement current_; - bool has_current_ = false; - bool exhausted_ = false; -}; - -/// Wrapper around fdb5::StatsIterator. -class StatsIteratorHandle { -public: - - explicit StatsIteratorHandle(fdb5::StatsIterator&& it); - ~StatsIteratorHandle(); - - StatsIteratorHandle(const StatsIteratorHandle&) = delete; - StatsIteratorHandle& operator=(const StatsIteratorHandle&) = delete; - StatsIteratorHandle(StatsIteratorHandle&&) = default; - StatsIteratorHandle& operator=(StatsIteratorHandle&&) = default; - - // Methods exposed to Rust via cxx - bool hasNext(); - StatsElementData next(); - -private: - - fdb5::StatsIterator impl_; - fdb5::StatsElement current_; - bool has_current_ = false; - bool exhausted_ = false; -}; - -/// Wrapper around fdb5::ControlIterator. -class ControlIteratorHandle { -public: - - explicit ControlIteratorHandle(fdb5::ControlIterator&& it); - ~ControlIteratorHandle(); - - ControlIteratorHandle(const ControlIteratorHandle&) = delete; - ControlIteratorHandle& operator=(const ControlIteratorHandle&) = delete; - ControlIteratorHandle(ControlIteratorHandle&&) = default; - ControlIteratorHandle& operator=(ControlIteratorHandle&&) = default; - - // Methods exposed to Rust via cxx - bool hasNext(); - ControlElementData next(); - -private: - - fdb5::ControlIterator impl_; - fdb5::ControlElement current_; - bool has_current_ = false; - bool exhausted_ = false; -}; - -// ============================================================================ -// Initialization functions -// ============================================================================ - -/// Initialize the FDB library. -/// Must be called before any other FDB operations. -void fdb_init(); - -// ============================================================================ -// Library metadata functions -// ============================================================================ - -/// Get the FDB library version string. -rust::String fdb_version(); - -/// Get the FDB git SHA1 hash. -rust::String fdb_git_sha1(); - -// ============================================================================ -// MARS request parsing -// ============================================================================ - -/// Parse a MARS request string with metkit's parser + expansion. Handles -/// `to`/`by` ranges, type expansion, optional fields, etc. Throws an -/// `eckit::Exception` on parse failure (which the global trycatch turns -/// into a Rust `Result::Err`). -RequestData parse_mars_request(rust::Str request); - -// ============================================================================ -// Handle lifecycle functions -// ============================================================================ - -/// Create a new FDB handle with default configuration. -std::unique_ptr new_fdb(); - -/// Create a new FDB handle from YAML configuration. -std::unique_ptr new_fdb_from_yaml(rust::Str config); - -/// Create a new FDB handle from YAML configuration plus a YAML "user config" -/// (per-instance overrides such as `useSubToc`, `preloadTocBTree`, etc.). -std::unique_ptr new_fdb_from_yaml_with_user_config(rust::Str config, rust::Str user_config); - -/// Create a new FDB handle by loading the configuration file at `path`. -/// Delegates to `fdb5::Config::make`, which is the same entry point upstream -/// FDB tools use when given `--config-file` / `FDB_CONFIG_FILE`. Loads -/// YAML or JSON, resolves `~fdb`-style paths, and honours `fdb_home`. -std::unique_ptr new_fdb_from_path(rust::Str path); - -/// Same as `new_fdb_from_path` but also applies a YAML "user config". -std::unique_ptr new_fdb_from_path_with_user_config(rust::Str path, rust::Str user_config); - -// ============================================================================ -// eckit::DataHandle shim functions -// ============================================================================ - -/// Open the handle for reading. Returns the estimated length. -uint64_t data_handle_open(eckit::DataHandle& handle); - -/// Read up to `buffer.size()` bytes into `buffer`. Returns the byte count. -size_t data_handle_read(eckit::DataHandle& handle, rust::Slice buffer); - -/// Seek to an absolute byte position in the underlying stream. -void data_handle_seek(eckit::DataHandle& handle, uint64_t position); - -/// Current read position. -uint64_t data_handle_tell(eckit::DataHandle& handle); - -/// Total size of the underlying data, in bytes. -uint64_t data_handle_size(eckit::DataHandle& handle); - -/// Close the handle. Safe to call more than once. -void data_handle_close(eckit::DataHandle& handle); - -/// Drain a `ListIteratorHandle` via `fdb5::ListIterator::dumpCompact` and -/// return the aggregated MARS-request text plus the two counters. -CompactListingData list_iterator_dump_compact(ListIteratorHandle& iterator); - -// ============================================================================ -// Test functions (for verifying exception handling) -// ============================================================================ - -/// Test function that throws eckit::Exception -void test_throw_eckit_exception(); - -/// Test function that throws eckit::SeriousBug -void test_throw_eckit_serious_bug(); - -/// Test function that throws eckit::UserError -void test_throw_eckit_user_error(); - -/// Test function that throws std::runtime_error -void test_throw_std_exception(); - -/// Test function that throws an int (non-std::exception type) -void test_throw_int(); - -} // namespace fdb::ffi diff --git a/rust/crates/fdb-sys/src/lib.rs b/rust/crates/fdb-sys/src/lib.rs index 0e3d3aa12..c55b677c0 100644 --- a/rust/crates/fdb-sys/src/lib.rs +++ b/rust/crates/fdb-sys/src/lib.rs @@ -42,8 +42,9 @@ pub struct ArchiveCallbackBox(Box); /// Opaque wrapper for an arbitrary Rust [`std::io::Read`] source. /// /// Exposed to the C++ side as an `eckit::DataHandle` by -/// [`archive_reader`] to stream GRIB data from a Rust source into FDB -/// without buffering the entire payload in memory first. +/// [`FdbHandle::archive_reader`](ffi::FdbHandle::archive_reader) to stream +/// GRIB data from a Rust source into FDB without buffering the entire +/// payload in memory first. pub struct ReaderBox(Box); // Methods intentionally not exposed: @@ -64,7 +65,7 @@ pub struct ReaderBox(Box); class = "FDB", ignore = ["inspect", "reindex", "axesIterator", "config", "move"] )] -#[cxx::bridge(namespace = "fdb::ffi")] +#[cxx::bridge(namespace = "fdb_bridge")] mod ffi { // ========================================================================= // Shared structs (POD-like types that can cross the FFI boundary) @@ -83,21 +84,6 @@ mod ffi { pub entries: Vec, } - /// A single key in a parsed MARS request, paired with all of its values. - #[derive(Debug, Clone, Default)] - pub struct RequestParam { - pub key: String, - pub values: Vec, - } - - /// A fully-expanded MARS request, as produced by `parse_mars_request`. - /// `to`/`by` ranges, type expansions, etc. have already been applied by - /// `metkit::mars::MarsExpansion`. - #[derive(Debug, Clone, Default)] - pub struct RequestData { - pub params: Vec, - } - /// Data returned from list iteration. #[derive(Debug, Clone, Default)] pub struct ListElementData { @@ -258,7 +244,25 @@ mod ffi { // ========================================================================= unsafe extern "C++" { - include!("fdb_bridge.h"); + include!("FdbBridge.h"); + + // ===================================================================== + // Library — runtime initialisation and metadata + // ===================================================================== + + type Library; + + /// Initialise the FDB library (sets up `eckit::Main`). Idempotent. + #[Self = "Library"] + fn initialise(); + + /// Get the FDB library version string. + #[Self = "Library"] + fn version() -> String; + + /// Get the FDB git SHA1 hash. + #[Self = "Library"] + fn git_sha1() -> String; // ===================================================================== // FdbHandle - Main FDB handle @@ -286,127 +290,14 @@ mod ffi { fn name(self: &FdbHandle) -> String; // ===================================================================== - // FdbHandle operations - // ===================================================================== - - /// Archive data with an explicit key. - fn archive(self: Pin<&mut FdbHandle>, key: &KeyData, data: &[u8]) -> Result<()>; - - /// Archive raw GRIB data (key is extracted from the message). - fn archive_raw(self: Pin<&mut FdbHandle>, data: &[u8]) -> Result<()>; - - /// Archive raw GRIB data streamed from a Rust `std::io::Read`. - fn archive_reader(self: Pin<&mut FdbHandle>, reader: Box) -> Result<()>; - - /// Retrieve data matching a request. - fn retrieve(self: Pin<&mut FdbHandle>, request: &str) -> Result>; - - /// Read data from a single URI. - fn read_uri(self: Pin<&mut FdbHandle>, uri: &str) -> Result>; - - /// Read data from a list of URIs. - fn read_uris( - self: Pin<&mut FdbHandle>, - uris: &Vec, - in_storage_order: bool, - ) -> Result>; - - /// Read data from a list iterator (most efficient). - fn read_list_iterator( - self: Pin<&mut FdbHandle>, - iterator: Pin<&mut ListIteratorHandle>, - in_storage_order: bool, - ) -> Result>; - - /// List data matching a request. - fn list( - self: Pin<&mut FdbHandle>, - request: &str, - deduplicate: bool, - level: i32, - ) -> Result>; - - /// Get axes for a request. - fn axes(self: Pin<&mut FdbHandle>, request: &str, level: i32) -> Result>; - - /// Dump database structure. - fn dump( - self: Pin<&mut FdbHandle>, - request: &str, - simple: bool, - ) -> Result>; - - /// Get database status. - fn status( - self: Pin<&mut FdbHandle>, - request: &str, - ) -> Result>; - - /// Wipe data matching a request. - fn wipe( - self: Pin<&mut FdbHandle>, - request: &str, - doit: bool, - porcelain: bool, - unsafe_wipe_all: bool, - ) -> Result>; - - /// Purge duplicate data. - fn purge( - self: Pin<&mut FdbHandle>, - request: &str, - doit: bool, - porcelain: bool, - ) -> Result>; - - /// Get statistics iterator. - fn stats_iterator( - self: Pin<&mut FdbHandle>, - request: &str, - ) -> Result>; - - /// Control database features. - fn control( - self: Pin<&mut FdbHandle>, - request: &str, - action: ControlAction, - identifiers: &[ControlIdentifier], - ) -> Result>; - - /// Register a flush callback. - fn register_flush_callback(self: Pin<&mut FdbHandle>, callback: Box); - - /// Register an archive callback. - fn register_archive_callback(self: Pin<&mut FdbHandle>, callback: Box); - - // ===================================================================== - // eckit::DataHandle - For reading retrieved data + // eckit::DataHandleWrapper (from eckit-sys, cross-crate ExternType) // ===================================================================== - /// Opaque handle to an `eckit::DataHandle` (the upstream abstract - /// base for byte streams). Owned via `UniquePtr`; - /// `eckit::DataHandle` has a virtual destructor so cxx's - /// generated `delete` is correct for any concrete subclass. - #[namespace = "eckit"] - type DataHandle; + #[namespace = "eckit_bridge"] + type DataHandleWrapper = eckit_sys::DataHandleWrapper; - /// Open the handle for reading. Returns the estimated length. - fn data_handle_open(handle: Pin<&mut DataHandle>) -> Result; - - /// Close the handle. - fn data_handle_close(handle: Pin<&mut DataHandle>) -> Result<()>; - - /// Read up to `buffer.len()` bytes into `buffer`. - fn data_handle_read(handle: Pin<&mut DataHandle>, buffer: &mut [u8]) -> Result; - - /// Seek to an absolute byte position. - fn data_handle_seek(handle: Pin<&mut DataHandle>, position: u64) -> Result<()>; - - /// Current read position. - fn data_handle_tell(handle: Pin<&mut DataHandle>) -> u64; - - /// Total size of the underlying data, in bytes. - fn data_handle_size(handle: Pin<&mut DataHandle>) -> u64; + #[namespace = "eckit_bridge"] + type ConfigWrapper = eckit_sys::ConfigWrapper; // ===================================================================== // ListIteratorHandle @@ -424,9 +315,7 @@ mod ffi { /// Drain the iterator via `fdb5::ListIterator::dumpCompact`, /// returning the aggregated MARS-request text and the two /// counters. Mirrors `fdb-list --compact`. - fn list_iterator_dump_compact( - iterator: Pin<&mut ListIteratorHandle>, - ) -> Result; + fn dump_compact(self: Pin<&mut ListIteratorHandle>) -> Result; // ===================================================================== // DumpIteratorHandle @@ -507,84 +396,154 @@ mod ffi { fn next(self: Pin<&mut ControlIteratorHandle>) -> Result; // ===================================================================== - // Initialization (free functions) + // Cross-crate ExternType for MARS request // ===================================================================== - /// Initialize the FDB library. - /// Must be called before any other FDB operations. - fn fdb_init(); + #[namespace = "metkit_bridge"] + type MarsRequestWrapper = metkit_sys::ffi::MarsRequestWrapper; // ===================================================================== - // Library metadata (free functions) + // FdbHandle — archive / retrieve / read / query / callbacks / factories // ===================================================================== - /// Get the FDB library version string. - fn fdb_version() -> String; + /// Archive data with an explicit key. + fn archive(self: Pin<&mut FdbHandle>, key: &KeyData, data: &[u8]) -> Result<()>; - /// Get the FDB git SHA1 hash. - fn fdb_git_sha1() -> String; + /// Archive raw GRIB data (key is extracted from the message). + fn archive_raw(self: Pin<&mut FdbHandle>, data: &[u8]) -> Result<()>; - // ===================================================================== - // MARS request parsing (free functions) - // ===================================================================== + /// Archive raw GRIB data streamed from a Rust `std::io::Read` source. + fn archive_reader(self: Pin<&mut FdbHandle>, reader: Box) -> Result<()>; - /// Parse a MARS request string using metkit's parser and expansion - /// machinery. Handles `to`/`by` ranges, type expansion, optional - /// fields, and any other syntax the upstream MARS language supports. - /// - /// On success, returns the fully-expanded request as a sequence of - /// `(key, [values])` pairs. On parse failure, returns an `Err` whose - /// message comes from the underlying eckit/metkit exception. - fn parse_mars_request(request: &str) -> Result; + /// Retrieve data matching a MarsRequest. + fn retrieve( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + ) -> Result>; - // ===================================================================== - // Handle lifecycle (free functions) - // ===================================================================== + /// Read data from a single URI. + fn read_uri(self: Pin<&mut FdbHandle>, uri: &str) -> Result>; - /// Create a new FDB handle with default configuration. - fn new_fdb() -> Result>; + /// Read data from a list of URIs. + fn read_uris( + self: Pin<&mut FdbHandle>, + uris: &Vec, + in_storage_order: bool, + ) -> Result>; - /// Create a new FDB handle from YAML configuration. - fn new_fdb_from_yaml(config: &str) -> Result>; + /// Read data from a list iterator (most efficient). + fn read_list_iterator( + self: Pin<&mut FdbHandle>, + iterator: Pin<&mut ListIteratorHandle>, + in_storage_order: bool, + ) -> Result>; - /// Create a new FDB handle from YAML configuration plus a YAML - /// per-instance "user config" (e.g. `useSubToc`, `preloadTocBTree`). - fn new_fdb_from_yaml_with_user_config( - config: &str, - user_config: &str, - ) -> Result>; + /// List data matching a request. + fn list( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + deduplicate: bool, + level: i32, + ) -> Result>; - /// Create a new FDB handle by loading the configuration file at - /// `path`. Delegates to `fdb5::Config::make`, which loads YAML or - /// JSON, expands `~fdb` and `fdb_home` references, and resolves - /// transitive sub-configurations. - fn new_fdb_from_path(path: &str) -> Result>; - - /// Same as `new_fdb_from_path` but additionally applies a YAML - /// per-instance "user config" (e.g. `useSubToc`). - fn new_fdb_from_path_with_user_config( - path: &str, - user_config: &str, - ) -> Result>; + /// Get axes (available metadata dimensions) for a request. + fn axes( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + level: i32, + ) -> Result>; - // ===================================================================== - // Test functions (for verifying exception handling) - // ===================================================================== + /// Dump database structure. + fn dump( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + simple: bool, + ) -> Result>; + + /// Get database status. + fn status( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + ) -> Result>; + + /// Wipe (delete) data matching a request. + fn wipe( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + doit: bool, + porcelain: bool, + unsafe_wipe_all: bool, + ) -> Result>; - /// Test function that throws eckit::Exception - fn test_throw_eckit_exception() -> Result<()>; + /// Purge duplicate data. + fn purge( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + doit: bool, + porcelain: bool, + ) -> Result>; - /// Test function that throws eckit::SeriousBug - fn test_throw_eckit_serious_bug() -> Result<()>; + /// Get statistics iterator. + fn stats_iterator( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + ) -> Result>; + + /// Control database features. + fn control( + self: Pin<&mut FdbHandle>, + request: &MarsRequestWrapper, + action: ControlAction, + identifiers: &[ControlIdentifier], + ) -> Result>; - /// Test function that throws eckit::UserError - fn test_throw_eckit_user_error() -> Result<()>; + /// Register a flush callback. Invoked when `flush()` is called. + fn register_flush_callback(self: Pin<&mut FdbHandle>, callback: Box); - /// Test function that throws std::runtime_error - fn test_throw_std_exception() -> Result<()>; + /// Register an archive callback. Invoked for each field archived. + fn register_archive_callback(self: Pin<&mut FdbHandle>, callback: Box); - /// Test function that throws an int (non-std::exception type) - fn test_throw_int() -> Result<()>; + /// Create a new FDB handle with default configuration. + #[Self = "FdbHandle"] + fn create() -> Result>; + + /// Create a new FDB handle from an `eckit::Config`. + #[Self = "FdbHandle"] + fn from_config(config: &ConfigWrapper) -> Result>; + + /// Create a new FDB handle from an `eckit::Config` with a user-config + /// overlay. + #[Self = "FdbHandle"] + fn from_config_with_user( + config: &ConfigWrapper, + user_config: &ConfigWrapper, + ) -> Result>; + + // ===================================================================== + // MessageArchiver — direct wrapper of `fdb5::MessageArchiver` + // ===================================================================== + + type MessageArchiverWrapper; + + /// `fdb5::MessageArchiver::archive(eckit::DataHandle&)` — returns + /// total bytes archived. + fn archive( + self: Pin<&mut MessageArchiverWrapper>, + source: Pin<&mut DataHandleWrapper>, + ) -> Result; + + /// `fdb5::MessageArchiver::flush()`. + fn flush(self: Pin<&mut MessageArchiverWrapper>) -> Result<()>; + + /// Construct an `fdb5::MessageArchiver` with the given key modifier, + /// flags, and configuration. + #[Self = "MessageArchiverWrapper"] + fn create( + key: &KeyData, + complete_transfers: bool, + verbose: bool, + config: &ConfigWrapper, + ) -> Result>; } // ========================================================================= @@ -732,93 +691,3 @@ pub use ffi::*; // Re-export cxx types needed by downstream crates pub use cxx::{Exception, UniquePtr}; - -#[cfg(test)] -mod tests { - use super::ffi; - - #[test] - fn test_eckit_exception_caught_by_trycatch() { - let result = ffi::test_throw_eckit_exception(); - assert!(result.is_err()); - let err = result.expect_err("expected error"); - // Generic eckit::Exception gets ECKIT: prefix - assert!( - err.what().starts_with("ECKIT: "), - "Expected ECKIT: prefix, got: {}", - err.what() - ); - assert!( - err.what().contains("test eckit exception"), - "Expected eckit exception message, got: {}", - err.what() - ); - } - - #[test] - fn test_eckit_serious_bug_caught_by_trycatch() { - let result = ffi::test_throw_eckit_serious_bug(); - assert!(result.is_err()); - let err = result.expect_err("expected error"); - // SeriousBug gets specific prefix - assert!( - err.what().starts_with("ECKIT_SERIOUS_BUG: "), - "Expected ECKIT_SERIOUS_BUG: prefix, got: {}", - err.what() - ); - assert!( - err.what().contains("test serious bug"), - "Expected serious bug message, got: {}", - err.what() - ); - } - - #[test] - fn test_eckit_user_error_caught_by_trycatch() { - let result = ffi::test_throw_eckit_user_error(); - assert!(result.is_err()); - let err = result.expect_err("expected error"); - // UserError gets specific prefix - assert!( - err.what().starts_with("ECKIT_USER_ERROR: "), - "Expected ECKIT_USER_ERROR: prefix, got: {}", - err.what() - ); - assert!( - err.what().contains("test user error"), - "Expected user error message, got: {}", - err.what() - ); - } - - #[test] - fn test_std_exception_caught_by_trycatch() { - let result = ffi::test_throw_std_exception(); - assert!(result.is_err()); - let err = result.expect_err("expected error"); - // std::exception should NOT have any ECKIT prefix - assert!( - !err.what().starts_with("ECKIT"), - "std::exception should not have ECKIT prefix, got: {}", - err.what() - ); - assert!( - err.what().contains("test std exception"), - "Expected std exception message, got: {}", - err.what() - ); - } - - #[test] - fn test_non_std_exception_caught_by_trycatch() { - let result = ffi::test_throw_int(); - assert!(result.is_err()); - let err = result.expect_err("expected error"); - // Non-std exceptions get a generic message - assert!( - err.what().contains("non-std::exception"), - "Expected non-std::exception message, got: {}", - err.what() - ); - } -} diff --git a/rust/crates/fdb/Cargo.toml b/rust/crates/fdb/Cargo.toml index 3cf0a571e..abd43ee2f 100644 --- a/rust/crates/fdb/Cargo.toml +++ b/rust/crates/fdb/Cargo.toml @@ -9,7 +9,7 @@ readme.workspace = true keywords.workspace = true categories.workspace = true description = "Safe Rust wrapper for ECMWF's FDB (Fields DataBase)" -links = "fdb_rpath" +links = "fdb_rs" build = "build.rs" [features] @@ -22,11 +22,15 @@ bindman-utils.workspace = true [dependencies] fdb-sys.workspace = true -indexmap.workspace = true +eckit.workspace = true +eckit-sys = { workspace = true, default-features = false } +metkit.workspace = true parking_lot.workspace = true thiserror.workspace = true [dev-dependencies] +eckit.workspace = true +metkit.workspace = true clap = { version = "4", features = ["derive"] } criterion = { version = "0.5", features = ["html_reports"] } tempfile.workspace = true diff --git a/rust/crates/fdb/README.md b/rust/crates/fdb/README.md index 8553e56c3..bce97912c 100644 --- a/rust/crates/fdb/README.md +++ b/rust/crates/fdb/README.md @@ -68,15 +68,26 @@ the filesystem TOC backend, and remote FDB client support. ## Running Binaries and `cargo run` work out of the box on both macOS and Linux — -no `LD_LIBRARY_PATH` / `DYLD_LIBRARY_PATH` setup required. The build -script stamps RPATH entries onto the final binary so the dynamic linker -finds the libraries at runtime automatically. +no `LD_LIBRARY_PATH` / `DYLD_LIBRARY_PATH` setup required. + +All C++ shared libraries use `@rpath` install names. Binary crates that +depend on `fdb` should add a one-line `build.rs`: + +```rust +fn main() { + bindman_utils::emit_rpaths(); +} +``` + +This stamps absolute RPATH entries onto the final binary pointing at each +dependency's build output, so the dynamic linker finds them automatically. ### System / FHS-packaged installs (e.g. RPM, deb) When the target system already provides FDB and its dependencies — typically via separate distro packages installed under `/usr/lib{,64}` -— build against them with: +with headers under `/usr/include` — you don't need the colocated +layout at all. Build against the system libraries with: ```bash cargo build --release --no-default-features --features system @@ -84,23 +95,23 @@ cargo build --release --no-default-features --features system The build script calls `find_package(fdb5)` (and the same for eckit / metkit / eccodes), links the Rust binary against those system -libraries, and stamps absolute RPATH entries pointing at the resolved -lib directories. Install the binary to `/usr/bin` (or any standard -location) and rely on the distro's own packages for the shared -libraries — no need to copy anything extra. - -### Vendored / self-contained builds - -With the default `vendored` feature the build compiles FDB and all its -dependencies from source and copies the resulting shared libraries next -to the binary. The RPATH is set to find them there, so the binary is -portable as-is. - -The eccodes definition/sample tables are baked into `libeccodes` via -the default `memfs` feature, so there are no extra resource directories -to ship. (If you opt out of `memfs`, you also need to ship -`eccodes_resources/{definitions,samples}/` and point -`ECCODES_DEFINITION_PATH`/`ECCODES_SAMPLES_PATH` at them.) +libraries, and stamps absolute RPATH entries pointing at the lib +directories the CMake search resolved. A downstream package can then +install the binary to a standard location such as `/usr/bin` and rely +on the distro's own `libfdb5` / `libeckit` / `libmetkit` / `libeccodes` +packages for the shared libraries — no need to copy any directories +around or set environment variables. + +Typical packaging setups: + +- **RPM / deb**: depend on the distro's FDB `-devel` packages at build + time, depend on the runtime packages at install time, and build with + `--features system`. Binary goes to `/usr/bin`, libs stay where the + distro packages put them. +- **Custom prefix**: point `CMAKE_PREFIX_PATH` at your install tree + before running cargo (e.g. + `CMAKE_PREFIX_PATH=/opt/ecmwf cargo build --features system`). + Everything else is automatic. ## License diff --git a/rust/crates/fdb/benches/fdb_bench.rs b/rust/crates/fdb/benches/fdb_bench.rs index 97e73961a..84304195f 100644 --- a/rust/crates/fdb/benches/fdb_bench.rs +++ b/rust/crates/fdb/benches/fdb_bench.rs @@ -6,7 +6,7 @@ //! Some benchmarks require FDB setup and will be skipped if setup fails. use criterion::{Criterion, black_box, criterion_group, criterion_main}; -use fdb::{Fdb, Key, ListOptions, Request}; +use fdb::{Fdb, Key, ListOptions}; use std::sync::OnceLock; // FDB setup for benchmarks that need data @@ -60,7 +60,8 @@ mod fdb_setup { env::set_var("FDB5_CONFIG", &config); } - let fdb = Fdb::open(Some(&config), None).ok()?; + let eckit_config: eckit::Config = config.parse().ok()?; + let fdb = Fdb::open(Some(&eckit_config), None).ok()?; // Read test GRIB data let grib_path = fixtures_dir.join("synth11.grib"); @@ -114,31 +115,31 @@ fn bench_key_creation(c: &mut Criterion) { }); } -/// Benchmark Request creation with builder pattern. +/// Benchmark `MarsRequest` creation with builder. fn bench_request_creation(c: &mut Criterion) { + eckit::init(); c.bench_function("fdb_request_creation", |b| { b.iter(|| { - black_box( - Request::new() - .with("class", "rd") - .with("expver", "xxxx") - .with("stream", "oper") - .with("date", "20230508") - .with("time", "1200"), - ); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); + request.set("stream", "oper"); + request.set("date", "20230508"); + request.set("time", "1200"); + black_box(request); }); }); } -/// Benchmark Request creation with multiple values. +/// Benchmark `MarsRequest` creation with multiple values. fn bench_request_multi_values(c: &mut Criterion) { + eckit::init(); c.bench_function("fdb_request_multi_values", |b| { b.iter(|| { - black_box( - Request::new() - .with("class", "rd") - .with_values("step", &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]), - ); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("step", ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); + black_box(request); }); }); } @@ -150,11 +151,12 @@ fn bench_list(c: &mut Criterion) { return; }; + eckit::init(); let fdb = Fdb::open_default().expect("failed to create FDB handle"); - let request = Request::new() - .with("class", "rd") - .with("expver", "xxxx") - .with("stream", "oper"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); + request.set("stream", "oper"); c.bench_function("fdb_list", |b| { b.iter(|| { @@ -180,11 +182,12 @@ fn bench_axes(c: &mut Criterion) { return; }; + eckit::init(); let fdb = Fdb::open_default().expect("failed to create FDB handle"); - let request = Request::new() - .with("class", "rd") - .with("expver", "xxxx") - .with("stream", "oper"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); + request.set("stream", "oper"); c.bench_function("fdb_axes", |b| { b.iter(|| { diff --git a/rust/crates/fdb/build.rs b/rust/crates/fdb/build.rs index c1e35c224..5f617a88f 100644 --- a/rust/crates/fdb/build.rs +++ b/rust/crates/fdb/build.rs @@ -1,40 +1,3 @@ -//! Build script for fdb crate. -//! -//! Emits RPATH linker flags so binaries can find dynamic libraries -//! at runtime without setting `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH`. -//! -//! Two layouts are supported: -//! -//! - **Vendored** (default): dynamic libs are copied into -//! `fdb_libs/` and `eccodes_libs/` subdirectories next to the -//! final binary. The rpath entries are binary-relative -//! (`@executable_path/fdb_libs` on macOS, `$ORIGIN/fdb_libs` on -//! Linux), so the binary is portable as long as the user ships -//! those two directories alongside it. -//! -//! - **System**: libraries live wherever `find_package` resolved -//! them (e.g. `/usr/lib`, `/opt/.../lib`, or a custom prefix). -//! `fdb-sys`'s build script re-publishes each dependency's lib dir -//! via `cargo:system_*_lib` metadata keys, and we emit an -//! absolute rpath entry for each one so the binary still loads -//! without `LD_LIBRARY_PATH` / `DYLD_LIBRARY_PATH`. fn main() { - println!("cargo:rerun-if-changed=build.rs"); - bindman_utils::emit_rpath_flags(&["fdb_libs", "eccodes_libs"]); - - // When fdb-sys is in system mode, it re-publishes each - // dependency's install lib dir so we can stamp matching - // absolute rpath entries onto the final binary. The vendored - // build leaves these unset, so this block is a no-op there. - for key in [ - "DEP_FDB_SYS_SYSTEM_FDB5_LIB", - "DEP_FDB_SYS_SYSTEM_ECKIT_LIB", - "DEP_FDB_SYS_SYSTEM_METKIT_LIB", - "DEP_FDB_SYS_SYSTEM_ECCODES_LIB", - ] { - println!("cargo:rerun-if-env-changed={key}"); - if let Ok(lib_dir) = std::env::var(key) { - println!("cargo:rustc-link-arg=-Wl,-rpath,{lib_dir}"); - } - } + bindman_utils::reexport_dep_root("FDB_SYS"); } diff --git a/rust/crates/fdb/examples/fdb_archive.rs b/rust/crates/fdb/examples/fdb_archive.rs index 108061106..f514db8a3 100644 --- a/rust/crates/fdb/examples/fdb_archive.rs +++ b/rust/crates/fdb/examples/fdb_archive.rs @@ -24,10 +24,9 @@ fn main() -> Result<(), Box> { let grib_path = &args[2]; let use_raw = args.get(3).is_some_and(|a| a == "--raw"); - // Open the FDB. Passing a `Path` (rather than a `&str`) routes through - // `fdb5::Config::make`, which loads YAML or JSON and expands `~fdb`/ - // `fdb_home` references — no need to slurp the file into a String first. - let fdb = Fdb::open(Some(config_path), None)?; + // Load the config from the YAML/JSON file path, then open the FDB. + let cfg = eckit::Config::from_path(config_path)?; + let fdb = Fdb::open(Some(&cfg), None)?; // Read GRIB data let data = fs::read(grib_path)?; @@ -55,7 +54,7 @@ fn main() -> Result<(), Box> { } // Flush to persist - fdb.flush()?; + let () = fdb.flush()?; println!("Data archived and flushed successfully"); // Show stats diff --git a/rust/crates/fdb/examples/fdb_axes.rs b/rust/crates/fdb/examples/fdb_axes.rs index 5486a183b..acb4dff22 100644 --- a/rust/crates/fdb/examples/fdb_axes.rs +++ b/rust/crates/fdb/examples/fdb_axes.rs @@ -10,7 +10,7 @@ use std::process::ExitCode; use clap::Parser; -use fdb::{Fdb, Request}; +use fdb::Fdb; /// Query the available axes (metadata dimensions) for a MARS request. #[derive(Parser, Debug)] @@ -22,7 +22,9 @@ struct Args { } fn run(args: &Args) -> Result<(), Box> { - let request: Request = args.request.parse()?; + eckit::init(); + let parsed = metkit::parse(&format!("retrieve, {}", args.request), false)?; + let request = parsed.at(0)?; let fdb = Fdb::open_default()?; // Full traversal (db + index + datum) mirrors the behaviour of diff --git a/rust/crates/fdb/examples/fdb_list.rs b/rust/crates/fdb/examples/fdb_list.rs index b21de2335..5d836fd4e 100644 --- a/rust/crates/fdb/examples/fdb_list.rs +++ b/rust/crates/fdb/examples/fdb_list.rs @@ -17,7 +17,7 @@ use std::io::{self, Write as _}; use std::process::ExitCode; use clap::Parser; -use fdb::{Fdb, ListElement, ListOptions, Request}; +use fdb::{Fdb, ListElement, ListOptions}; /// `fdb-list`-style listing tool. Reimplements a sensible subset of the /// upstream `fdb-list` CLI on top of the Rust `fdb` binding. @@ -100,7 +100,9 @@ fn format_item(item: &ListElement, args: &Args) -> Result Result<(), Box> { - let request: Request = args.request.parse()?; + eckit::init(); + let parsed = metkit::parse(&format!("retrieve, {}", args.request), false)?; + let request = parsed.at(0)?; let fdb = Fdb::open_default()?; if !args.porcelain { diff --git a/rust/crates/fdb/examples/fdb_read.rs b/rust/crates/fdb/examples/fdb_read.rs index 984e4deaa..2e42a4878 100644 --- a/rust/crates/fdb/examples/fdb_read.rs +++ b/rust/crates/fdb/examples/fdb_read.rs @@ -23,7 +23,7 @@ use std::path::{Path, PathBuf}; use std::process::ExitCode; use clap::Parser; -use fdb::{Fdb, Request}; +use fdb::Fdb; /// `fdb-read`-style retrieval tool. Reimplements a sensible subset of /// the upstream `fdb-read` CLI on top of the Rust `fdb` binding. @@ -38,24 +38,24 @@ struct Args { } fn run(args: &Args) -> Result<(), Box> { - let request: Request = args.request.parse()?; + eckit::init(); + let parsed = metkit::parse(&args.request, false)?; + let request = parsed.at(0)?; let fdb = Fdb::open_default()?; - // `retrieve` hands back a `DataReader` (which implements - // `std::io::Read`) — exactly the streaming retrieval path the - // reviewer redesign was meant to enable. - let mut reader = fdb.retrieve(&request)?; + let handle = fdb.retrieve(&request)?; + let (mut handle, _len) = handle.open_for_read()?; // Open the target. `-` means stdout, matching the convention of // `fdb-read`'s sibling tools and most Unix utilities. let bytes_copied = if args.target == Path::new("-") { let stdout = io::stdout(); let mut out = stdout.lock(); - io::copy(&mut reader, &mut out)? + io::copy(&mut handle, &mut out)? } else { let file = File::create(&args.target)?; let mut out = BufWriter::new(file); - let n = io::copy(&mut reader, &mut out)?; + let n = io::copy(&mut handle, &mut out)?; out.flush()?; n }; diff --git a/rust/crates/fdb/examples/fdb_retrieve.rs b/rust/crates/fdb/examples/fdb_retrieve.rs index 3e1feb300..6a2574b1b 100644 --- a/rust/crates/fdb/examples/fdb_retrieve.rs +++ b/rust/crates/fdb/examples/fdb_retrieve.rs @@ -3,35 +3,39 @@ //! Run with: `cargo run --example fdb_retrieve -p fdb -- [output.grib]` //! //! Examples: -//! cargo run --example `fdb_retrieve` -p fdb -- class=rd,expver=xxxx,date=20230508,... -//! cargo run --example `fdb_retrieve` -p fdb -- class=rd,expver=xxxx,... output.grib +//! cargo run --example `fdb_retrieve` -p fdb -- "retrieve, class=rd,expver=xxxx,..." +//! cargo run --example `fdb_retrieve` -p fdb -- "retrieve, class=rd,..." output.grib use std::env; use std::fs::File; use std::io::{Read, Write}; -use fdb::{Fdb, Request}; +use fdb::Fdb; fn main() -> Result<(), Box> { let args: Vec = env::args().collect(); if args.len() < 2 { eprintln!("Usage: {} [output.grib]", args[0]); eprintln!(); - eprintln!("Request format: key=value,key=value,..."); + eprintln!("Request format: retrieve, key=value, key=value, ..."); eprintln!( - "Example: class=rd,expver=xxxx,stream=oper,date=20230508,time=1200,type=fc,levtype=sfc,step=0,param=151130" + "Example: retrieve, class=rd,expver=xxxx,stream=oper,date=20230508,time=1200,type=fc,levtype=sfc,step=0,param=151130" ); std::process::exit(1); } + eckit::init(); + let fdb = Fdb::open_default()?; - let request: Request = args[1].parse()?; + let parsed = metkit::parse(&args[1], false)?; + let request = parsed.at(0)?; println!("Retrieving data..."); - let mut reader = fdb.retrieve(&request)?; + let handle = fdb.retrieve(&request)?; + let (mut handle, _len) = handle.open_for_read()?; let mut buffer = Vec::new(); - let bytes_read = reader.read_to_end(&mut buffer)?; + let bytes_read = handle.read_to_end(&mut buffer)?; println!("Retrieved {bytes_read} bytes"); // Write to file or show summary diff --git a/rust/crates/fdb/src/datareader.rs b/rust/crates/fdb/src/datareader.rs deleted file mode 100644 index 3a976d3dc..000000000 --- a/rust/crates/fdb/src/datareader.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! FDB data reader wrapper. - -use std::io::{Read, Seek, SeekFrom}; - -use fdb_sys::UniquePtr; - -use crate::error::Result; - -/// A reader for data retrieved from FDB. -/// -/// Implements [`std::io::Read`] and [`std::io::Seek`] for standard I/O operations. -pub struct DataReader { - handle: UniquePtr, -} - -impl DataReader { - /// Create a new data reader from a cxx handle. - pub(crate) fn new(mut handle: UniquePtr) -> Result { - fdb_sys::data_handle_open(handle.pin_mut())?; - Ok(Self { handle }) - } - - /// Get the total size of the data in bytes. - pub fn size(&mut self) -> u64 { - fdb_sys::data_handle_size(self.handle.pin_mut()) - } - - /// Get the current read position. - pub fn tell(&mut self) -> u64 { - fdb_sys::data_handle_tell(self.handle.pin_mut()) - } - - /// Seek to a position in the data. - /// - /// # Errors - /// - /// Returns an error if seeking fails. - pub fn seek_to(&mut self, pos: u64) -> Result<()> { - fdb_sys::data_handle_seek(self.handle.pin_mut(), pos)?; - Ok(()) - } - - /// Read all data into a vector. - /// - /// # Errors - /// - /// Returns an error if reading fails or if the data size exceeds platform capacity. - pub fn read_all(&mut self) -> Result> { - let size = usize::try_from(self.size())?; - let mut buf = vec![0u8; size]; - let mut total_read = 0; - - while total_read < size { - let n = fdb_sys::data_handle_read(self.handle.pin_mut(), &mut buf[total_read..])?; - if n == 0 { - break; - } - total_read += n; - } - - buf.truncate(total_read); - Ok(buf) - } - - /// Close the data reader. - /// - /// # Errors - /// - /// Returns an error if closing fails. - pub fn close(&mut self) -> Result<()> { - fdb_sys::data_handle_close(self.handle.pin_mut())?; - Ok(()) - } -} - -impl Read for DataReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - fdb_sys::data_handle_read(self.handle.pin_mut(), buf) - .map_err(|e| std::io::Error::other(e.to_string())) - } -} - -impl Seek for DataReader { - fn seek(&mut self, pos: SeekFrom) -> std::io::Result { - let new_pos = match pos { - SeekFrom::Start(offset) => offset, - SeekFrom::End(offset) => { - let size = i64::try_from(self.size()) - .map_err(|_| std::io::Error::other("file size exceeds i64::MAX"))?; - let new = size - .checked_add(offset) - .ok_or_else(|| std::io::Error::other("seek position overflow"))?; - if new < 0 { - return Err(std::io::Error::other("seek to negative position")); - } - new.cast_unsigned() - } - SeekFrom::Current(offset) => { - let current = i64::try_from(self.tell()) - .map_err(|_| std::io::Error::other("current position exceeds i64::MAX"))?; - let new = current - .checked_add(offset) - .ok_or_else(|| std::io::Error::other("seek position overflow"))?; - if new < 0 { - return Err(std::io::Error::other("seek to negative position")); - } - new.cast_unsigned() - } - }; - - fdb_sys::data_handle_seek(self.handle.pin_mut(), new_pos) - .map_err(|e| std::io::Error::other(e.to_string()))?; - - Ok(new_pos) - } -} - -impl Drop for DataReader { - fn drop(&mut self) { - let _ = fdb_sys::data_handle_close(self.handle.pin_mut()); - } -} - -// SAFETY: The underlying C++ DataHandle is accessed through &mut self only. -#[allow(clippy::non_send_fields_in_send_ty)] -unsafe impl Send for DataReader {} diff --git a/rust/crates/fdb/src/error.rs b/rust/crates/fdb/src/error.rs index 11246b6a6..c2da9130d 100644 --- a/rust/crates/fdb/src/error.rs +++ b/rust/crates/fdb/src/error.rs @@ -3,41 +3,9 @@ /// Error type for FDB operations. #[derive(Debug, thiserror::Error)] pub enum Error { - /// Internal programming error in the C++ library (`eckit::SeriousBug`). - #[error("serious bug: {0}")] - SeriousBug(String), - - /// User-caused error (`eckit::UserError`). - #[error("user error: {0}")] - UserError(String), - - /// Invalid parameter passed to C++ library (`eckit::BadParameter`). - #[error("bad parameter: {0}")] - BadParameter(String), - - /// Feature not implemented (`eckit::NotImplemented`). - #[error("not implemented: {0}")] - NotImplemented(String), - - /// Index or range out of bounds (`eckit::OutOfRange`). - #[error("out of range: {0}")] - OutOfRange(String), - - /// File operation error (`eckit::FileError`). - #[error("file error: {0}")] - FileError(String), - - /// Assertion failed in C++ library (`eckit::AssertionFailed`). - #[error("assertion failed: {0}")] - AssertionFailed(String), - - /// Generic eckit exception. - #[error("eckit error: {0}")] - Eckit(String), - - /// Generic error from the FDB C++ library. - #[error("fdb error: {0}")] - Fdb(String), + /// Error from eckit/metkit C++ libraries. + #[error(transparent)] + Eckit(#[from] eckit::Error), /// I/O error. #[error("I/O error: {0}")] @@ -52,122 +20,7 @@ pub enum Error { pub type Result = std::result::Result; impl From for Error { - #[allow(clippy::option_if_let_else)] fn from(e: fdb_sys::Exception) -> Self { - let msg = e.what(); - - // Parse prefixes added by rust::behavior::trycatch - if let Some(rest) = msg.strip_prefix("ECKIT_SERIOUS_BUG: ") { - Self::SeriousBug(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_USER_ERROR: ") { - Self::UserError(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_BAD_PARAMETER: ") { - Self::BadParameter(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_NOT_IMPLEMENTED: ") { - Self::NotImplemented(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_OUT_OF_RANGE: ") { - Self::OutOfRange(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_FILE_ERROR: ") { - Self::FileError(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_ASSERTION_FAILED: ") { - Self::AssertionFailed(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT: ") { - Self::Eckit(rest.to_string()) - } else { - Self::Fdb(msg.to_string()) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // Helper to create a mock exception-like message - #[allow(clippy::option_if_let_else)] - fn convert_message(msg: &str) -> Error { - // Simulate what From does by parsing the prefix - msg.strip_prefix("ECKIT_SERIOUS_BUG: ").map_or_else( - || { - if let Some(rest) = msg.strip_prefix("ECKIT_USER_ERROR: ") { - Error::UserError(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_BAD_PARAMETER: ") { - Error::BadParameter(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_NOT_IMPLEMENTED: ") { - Error::NotImplemented(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_OUT_OF_RANGE: ") { - Error::OutOfRange(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_FILE_ERROR: ") { - Error::FileError(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT_ASSERTION_FAILED: ") { - Error::AssertionFailed(rest.to_string()) - } else if let Some(rest) = msg.strip_prefix("ECKIT: ") { - Error::Eckit(rest.to_string()) - } else { - Error::Fdb(msg.to_string()) - } - }, - |rest| Error::SeriousBug(rest.to_string()), - ) - } - - #[test] - fn test_serious_bug_prefix() { - let err = convert_message("ECKIT_SERIOUS_BUG: something went wrong"); - assert!(matches!(err, Error::SeriousBug(msg) if msg == "something went wrong")); - } - - #[test] - fn test_user_error_prefix() { - let err = convert_message("ECKIT_USER_ERROR: invalid input"); - assert!(matches!(err, Error::UserError(msg) if msg == "invalid input")); - } - - #[test] - fn test_bad_parameter_prefix() { - let err = convert_message("ECKIT_BAD_PARAMETER: param must be positive"); - assert!(matches!(err, Error::BadParameter(msg) if msg == "param must be positive")); - } - - #[test] - fn test_not_implemented_prefix() { - let err = convert_message("ECKIT_NOT_IMPLEMENTED: feature X"); - assert!(matches!(err, Error::NotImplemented(msg) if msg == "feature X")); - } - - #[test] - fn test_out_of_range_prefix() { - let err = convert_message("ECKIT_OUT_OF_RANGE: index 10 out of bounds"); - assert!(matches!(err, Error::OutOfRange(msg) if msg == "index 10 out of bounds")); - } - - #[test] - fn test_file_error_prefix() { - let err = convert_message("ECKIT_FILE_ERROR: cannot open file"); - assert!(matches!(err, Error::FileError(msg) if msg == "cannot open file")); - } - - #[test] - fn test_assertion_failed_prefix() { - let err = convert_message("ECKIT_ASSERTION_FAILED: x > 0"); - assert!(matches!(err, Error::AssertionFailed(msg) if msg == "x > 0")); - } - - #[test] - fn test_generic_eckit_prefix() { - let err = convert_message("ECKIT: some eckit error"); - assert!(matches!(err, Error::Eckit(msg) if msg == "some eckit error")); - } - - #[test] - fn test_no_prefix_falls_through() { - let err = convert_message("plain error message"); - assert!(matches!(err, Error::Fdb(msg) if msg == "plain error message")); - } - - #[test] - fn test_std_exception_no_prefix() { - let err = convert_message("std::runtime_error message"); - assert!(matches!(err, Error::Fdb(msg) if msg == "std::runtime_error message")); + Self::Eckit(eckit::Error::from(e)) } } diff --git a/rust/crates/fdb/src/handle.rs b/rust/crates/fdb/src/handle.rs index b6f0b1da0..2557b02d0 100644 --- a/rust/crates/fdb/src/handle.rs +++ b/rust/crates/fdb/src/handle.rs @@ -1,13 +1,12 @@ //! FDB handle wrapper. use std::collections::HashMap; -use std::sync::{LazyLock, Once}; +use std::sync::Once; use fdb_sys::UniquePtr; use fdb_sys::{ControlAction, ControlIdentifier}; use parking_lot::Mutex; -use crate::datareader::DataReader; use crate::error::Result; use crate::iterator::{ ControlIterator, DumpIterator, ListIterator, PurgeIterator, StatsIterator, StatusIterator, @@ -15,38 +14,14 @@ use crate::iterator::{ }; use crate::key::Key; use crate::options::{DumpOptions, ListOptions, PurgeOptions, WipeOptions}; -use crate::request::Request; +use eckit::DataHandle; static INIT: Once = Once::new(); -/// Process-global mutex serializing GRIB ingest across `Fdb` -/// instances. -/// -/// Running `archive_raw` / `archive_reader` from two separate -/// instances on different threads crashes the process with `fatal -/// flex scanner internal error — end of buffer missed` + SIGSEGV — -/// non-reentrant state somewhere inside `libeccodes`' GRIB decoding -/// path. This lock serializes those two methods' FFI hops, which -/// empirically eliminates the crash. MARS-request methods -/// (`list`, `retrieve`, etc.) were confirmed safe under parallel -/// test pressure and remain lock-free. -static LEXER_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - /// Initialize the FDB library. /// Called automatically when creating any FDB handle. fn initialize() { - INIT.call_once(fdb_sys::fdb_init); -} - -/// Convert a path to a `&str`, returning a typed `UserError` if it isn't -/// valid UTF-8 (which the cxx bridge can't accept). -fn path_to_str(path: &std::path::Path) -> Result<&str> { - path.to_str().ok_or_else(|| { - crate::Error::UserError(format!( - "FDB config path is not valid UTF-8: {}", - path.display() - )) - }) + INIT.call_once(fdb_sys::Library::initialise); } // Private wrapper to make UniquePtr Send-safe for use with Mutex @@ -68,16 +43,18 @@ unsafe impl Send for HandleInner {} /// # Example /// /// ```no_run -/// use fdb::{Fdb, Request}; +/// use fdb::Fdb; /// use std::sync::Arc; /// use std::thread; /// +/// eckit::init(); /// let fdb = Arc::new(Fdb::open_default().expect("failed to create FDB handle")); /// /// let handles: Vec<_> = (0..4).map(|_| { /// let fdb = Arc::clone(&fdb); /// thread::spawn(move || { -/// let request = Request::new().with("class", "od"); +/// let mut request = metkit::MarsRequest::new("list"); +/// request.set("class", "od"); /// let _ = fdb.list(&request, fdb::ListOptions::default()); /// }) /// }).collect(); @@ -90,136 +67,60 @@ pub struct Fdb { handle: Mutex, } -/// One of the shapes the main FDB config can take when opening an `Fdb`. -/// -/// You generally don't construct this directly — [`Fdb::open`] accepts any -/// `Option>`, and the standard `From` impls let you -/// pass `&str`/`&String` (interpreted as inline YAML) or `&Path`/`&PathBuf` -/// (interpreted as a path to a config file on disk) directly. -/// -/// Mirrors the shape of pyfdb's `config: str | Path | None` argument. -/// -/// Note that this enum is for the *main* config only. The user config -/// (second argument of [`Fdb::open`]) takes only YAML strings — upstream -/// `fdb5::Config` does not have a path-based user-config entry point. -#[derive(Debug, Clone)] -pub enum FdbConfig<'a> { - /// Inline YAML. Goes through `eckit::YAMLConfiguration` on the C++ side. - Yaml(&'a str), - /// Path to a YAML/JSON config file. Goes through `fdb5::Config::make`, - /// which also expands `~fdb`/`fdb_home` references and resolves - /// transitive sub-configurations. - Path(&'a std::path::Path), -} - -impl<'a> From<&'a str> for FdbConfig<'a> { - fn from(s: &'a str) -> Self { - FdbConfig::Yaml(s) - } -} - -impl<'a> From<&'a String> for FdbConfig<'a> { - fn from(s: &'a String) -> Self { - FdbConfig::Yaml(s.as_str()) - } -} - -impl<'a> From<&'a std::path::Path> for FdbConfig<'a> { - fn from(p: &'a std::path::Path) -> Self { - FdbConfig::Path(p) - } -} - -impl<'a> From<&'a std::path::PathBuf> for FdbConfig<'a> { - fn from(p: &'a std::path::PathBuf) -> Self { - FdbConfig::Path(p.as_path()) - } -} - impl Fdb { /// Open an FDB. /// - /// `config` is the main FDB configuration. It accepts anything - /// convertible to [`FdbConfig`]: a `&str`/`&String` (inline YAML), a - /// `&Path`/`&PathBuf` (config file on disk), or `None` to use the - /// upstream's environment-driven defaults (`FDB_HOME` / - /// `FDB_CONFIG_FILE` / `~/.fdb`). - /// - /// `user_config` is an optional per-instance YAML overlay (e.g. - /// `useSubToc: true`, `preloadTocBTree: false`). It accepts only a - /// YAML string because upstream `fdb5::Config` itself only takes the - /// user config as an in-memory `eckit::Configuration`, never as a - /// path. A user config without a main config is rejected — there's - /// nothing for the overlay to apply to. - /// - /// Mirrors pyfdb's `FDB(config, user_config)` constructor shape, with - /// two improvements: (1) `(None, Some(user_config))` is rejected - /// instead of silently dropping the user config like pyfdb does, and - /// (2) the unsupported `Path` user-config shape is forbidden at the - /// type level rather than at runtime. + /// Matches C++ `fdb5::FDB(fdb5::Config)` / `fdb5::FDB(fdb5::Config(config, user_config))`. + /// + /// - `None, None` — use environment defaults (`FDB_HOME` / `FDB_CONFIG_FILE` / `~/.fdb`) + /// - `Some(config), None` — use the given config + /// - `Some(config), Some(user_config)` — config + per-instance overlay + /// + /// Build the `eckit::Config` however you want: `Config::from_path()`, + /// `"yaml".parse()`, or `Config::new()` + `.set()`. /// /// # Examples /// /// ```no_run - /// use fdb::Fdb; - /// use std::path::Path; + /// use fdb::{Fdb, UserConfig}; + /// + /// // Default config from environment: + /// let fdb = Fdb::open(None, None)?; /// - /// // Inline YAML, no user config: - /// let fdb = Fdb::open(Some("type: local\nschema: /tmp/schema\nspaces: []"), None)?; + /// // From a YAML file: + /// let cfg = eckit::Config::from_path("/etc/fdb/config.yaml")?; + /// let fdb = Fdb::open(Some(&cfg), None)?; /// - /// // Config file on disk: - /// let fdb = Fdb::open(Some(Path::new("/etc/fdb/config.yaml")), None)?; + /// // Inline YAML: + /// let cfg: eckit::Config = "type: local\nspaces: []".parse()?; + /// let fdb = Fdb::open(Some(&cfg), None)?; /// - /// // Path config + inline user config to enable sub-tocs: + /// // With user config: + /// let cfg = eckit::Config::from_path("/etc/fdb/config.yaml")?; /// let fdb = Fdb::open( - /// Some(Path::new("/etc/fdb/config.yaml")), - /// Some("useSubToc: true"), + /// Some(&cfg), + /// Some(UserConfig { use_sub_toc: true, ..Default::default() }), /// )?; /// # Ok::<(), fdb::Error>(()) /// ``` - /// - /// For the "use defaults from environment" case where neither argument - /// is supplied, prefer [`Self::open_default`] — it avoids Rust's - /// type-inference annoyance with `Fdb::open(None, None)`. - /// - /// # Errors - /// - /// - `UserError` if a non-UTF-8 path is supplied (the cxx bridge can't - /// accept it). - /// - `UserError` if `user_config` is supplied without a `config`. - /// - Whatever `eckit`/`fdb5` raises if the configuration can't be - /// parsed or the FDB instance can't be constructed. - pub fn open<'a, C>(config: Option, user_config: Option<&str>) -> Result - where - C: Into>, - { + pub fn open( + config: Option<&eckit::Config>, + user_config: Option, + ) -> Result { initialize(); - let config = config.map(Into::into); - - // Map (config, user_config) to one of the existing cxx-bridge - // entry points. The arms below cover exactly the combinations - // upstream `fdb5::Config` supports — there are no invented arms. - let handle = match (config, user_config) { - (None, None) => fdb_sys::new_fdb()?, - (Some(FdbConfig::Yaml(yaml)), None) => fdb_sys::new_fdb_from_yaml(yaml)?, - (Some(FdbConfig::Path(path)), None) => { - let path_str = path_to_str(path)?; - fdb_sys::new_fdb_from_path(path_str)? - } - (Some(FdbConfig::Yaml(yaml)), Some(user)) => { - fdb_sys::new_fdb_from_yaml_with_user_config(yaml, user)? - } - (Some(FdbConfig::Path(path)), Some(user)) => { - let path_str = path_to_str(path)?; - fdb_sys::new_fdb_from_path_with_user_config(path_str, user)? + + let user_eckit = user_config.map(eckit::Config::from); + + let handle = match (config, user_eckit.as_ref()) { + (None, None) => fdb_sys::FdbHandle::create()?, + (Some(cfg), None) => fdb_sys::FdbHandle::from_config(cfg.as_sys())?, + (Some(cfg), Some(user)) => { + fdb_sys::FdbHandle::from_config_with_user(cfg.as_sys(), user.as_sys())? } - // pyfdb silently drops `user_config` here. We don't — there's - // no upstream entry point that says "env-default config plus - // this user overlay", and silently dropping is a footgun. (None, Some(_)) => { - return Err(crate::Error::UserError( + return Err(crate::Error::Eckit(eckit::Error::UserError( "Fdb::open: user_config requires a main config".to_string(), - )); + ))); } }; @@ -228,12 +129,11 @@ impl Fdb { }) } - /// Open an FDB using the upstream's default configuration discovery - /// (`FDB_HOME` / `FDB_CONFIG_FILE` / `~/.fdb`). Equivalent to - /// `Fdb::open(None::<&str>, None)`, but avoids the type-inference - /// annoyance with the bare `Fdb::open(None, None)` form. + /// Open an FDB using environment defaults. + /// + /// Equivalent to `Fdb::open(None, None)`. pub fn open_default() -> Result { - Self::open(None::<&str>, None) + Self::open(None, None) } #[inline] @@ -281,46 +181,49 @@ impl Fdb { /// # Errors /// /// Returns an error if listing fails. - pub fn list(&self, request: &Request, options: ListOptions) -> Result { + pub fn list( + &self, + request: &metkit::MarsRequest, + options: ListOptions, + ) -> Result { let ListOptions { depth, deduplicate } = options; - let it = self.with_handle(|h| h.list(&request.to_request_string(), deduplicate, depth))?; + let it = self.with_handle(|h| h.list(request.as_sys(), deduplicate, depth))?; Ok(ListIterator::new(it)) } - /// Retrieve data from FDB. + /// Retrieve data from FDB using a `MarsRequest`. /// - /// # Arguments - /// - /// * `request` - The request specifying which data to retrieve + /// Returns an unopened `eckit::DataHandle`. Call + /// [`DataHandle::open_for_read`](eckit::DataHandle::open_for_read) before + /// reading, or wrap it in a reader type that does so. /// /// # Errors /// /// Returns an error if retrieval fails. - pub fn retrieve(&self, request: &Request) -> Result { - let handle = self.with_handle(|h| h.retrieve(&request.to_request_string()))?; - DataReader::new(handle) + pub fn retrieve( + &self, + request: &metkit::MarsRequest, + ) -> Result { + let handle = self.with_handle(|h| h.retrieve(request.as_sys()))?; + Ok(DataHandle::from_raw(handle)) } /// Read data from a single URI location. /// - /// This is more efficient than `retrieve()` when you already have + /// More efficient than `retrieve()` when you already have /// the field location from a previous `list()` operation. /// - /// # Arguments - /// - /// * `uri` - The URI to read from - /// /// # Errors /// /// Returns an error if reading fails. - pub fn read_uri(&self, uri: &str) -> Result { + pub fn read_uri(&self, uri: &str) -> Result { let handle = self.with_handle(|h| h.read_uri(uri))?; - DataReader::new(handle) + Ok(DataHandle::from_raw(handle)) } /// Read data from multiple URI locations. /// - /// This is more efficient than `retrieve()` when you already have + /// More efficient than `retrieve()` when you already have /// the field locations from a previous `list()` operation. /// /// # Arguments @@ -332,22 +235,17 @@ impl Fdb { /// # Errors /// /// Returns an error if reading fails. - pub fn read_uris(&self, uris: &[String], in_storage_order: bool) -> Result { + pub fn read_uris(&self, uris: &[String], in_storage_order: bool) -> Result { let uris_vec: Vec = uris.to_vec(); let handle = self.with_handle(|h| h.read_uris(&uris_vec, in_storage_order))?; - DataReader::new(handle) + Ok(DataHandle::from_raw(handle)) } /// Read data directly from a list iterator (most efficient). /// - /// This consumes the iterator and reads all matched fields. + /// Consumes the iterator and reads all matched fields. /// More efficient than `read_uris()` as it avoids URI string conversion. /// - /// # Arguments - /// - /// * `list` - `ListIterator` to read from (consumed) - /// * `in_storage_order` - If true, data is returned in storage order - /// /// # Errors /// /// Returns an error if reading fails. @@ -355,10 +253,10 @@ impl Fdb { &self, mut list: ListIterator, in_storage_order: bool, - ) -> Result { + ) -> Result { let handle = self.with_handle(|h| h.read_list_iterator(list.inner_mut(), in_storage_order))?; - DataReader::new(handle) + Ok(DataHandle::from_raw(handle)) } /// Flush any pending writes to FDB. @@ -414,7 +312,6 @@ impl Fdb { /// /// Returns an error if archiving fails. pub fn archive_raw(&self, data: &[u8]) -> Result<()> { - let _lexer = LEXER_LOCK.lock(); self.with_handle(|h| h.archive_raw(data))?; Ok(()) } @@ -438,7 +335,6 @@ impl Fdb { where R: std::io::Read + Send + 'static, { - let _lexer = LEXER_LOCK.lock(); let boxed = fdb_sys::make_reader_box(reader); self.with_handle(|h| h.archive_reader(boxed))?; Ok(()) @@ -456,8 +352,12 @@ impl Fdb { /// # Errors /// /// Returns an error if the query fails. - pub fn axes(&self, request: &Request, depth: i32) -> Result>> { - let axes = self.with_handle(|h| h.axes(&request.to_request_string(), depth))?; + pub fn axes( + &self, + request: &metkit::MarsRequest, + depth: i32, + ) -> Result>> { + let axes = self.with_handle(|h| h.axes(request.as_sys(), depth))?; Ok(axes.into_iter().map(|a| (a.key, a.values)).collect()) } @@ -472,9 +372,13 @@ impl Fdb { /// # Errors /// /// Returns an error if the dump fails. - pub fn dump(&self, request: &Request, options: DumpOptions) -> Result { + pub fn dump( + &self, + request: &metkit::MarsRequest, + options: DumpOptions, + ) -> Result { let DumpOptions { simple } = options; - let it = self.with_handle(|h| h.dump(&request.to_request_string(), simple))?; + let it = self.with_handle(|h| h.dump(request.as_sys(), simple))?; Ok(DumpIterator::new(it)) } @@ -487,8 +391,11 @@ impl Fdb { /// # Errors /// /// Returns an error if the status query fails. - pub fn status(&self, request: &Request) -> Result { - let it = self.with_handle(|h| h.status(&request.to_request_string()))?; + pub fn status( + &self, + request: &metkit::MarsRequest, + ) -> Result { + let it = self.with_handle(|h| h.status(request.as_sys()))?; Ok(StatusIterator::new(it)) } @@ -504,20 +411,18 @@ impl Fdb { /// # Errors /// /// Returns an error if the wipe fails. - pub fn wipe(&self, request: &Request, options: WipeOptions) -> Result { + pub fn wipe( + &self, + request: &metkit::MarsRequest, + options: WipeOptions, + ) -> Result { let WipeOptions { doit, porcelain, unsafe_wipe_all, } = options; - let it = self.with_handle(|h| { - h.wipe( - &request.to_request_string(), - doit, - porcelain, - unsafe_wipe_all, - ) - })?; + let it = + self.with_handle(|h| h.wipe(request.as_sys(), doit, porcelain, unsafe_wipe_all))?; Ok(WipeIterator::new(it)) } @@ -533,9 +438,13 @@ impl Fdb { /// # Errors /// /// Returns an error if the purge fails. - pub fn purge(&self, request: &Request, options: PurgeOptions) -> Result { + pub fn purge( + &self, + request: &metkit::MarsRequest, + options: PurgeOptions, + ) -> Result { let PurgeOptions { doit, porcelain } = options; - let it = self.with_handle(|h| h.purge(&request.to_request_string(), doit, porcelain))?; + let it = self.with_handle(|h| h.purge(request.as_sys(), doit, porcelain))?; Ok(PurgeIterator::new(it)) } @@ -548,8 +457,11 @@ impl Fdb { /// # Errors /// /// Returns an error if the stats query fails. - pub fn stats_iter(&self, request: &Request) -> Result { - let it = self.with_handle(|h| h.stats_iterator(&request.to_request_string()))?; + pub fn stats_iter( + &self, + request: &metkit::MarsRequest, + ) -> Result { + let it = self.with_handle(|h| h.stats_iterator(request.as_sys()))?; Ok(StatsIterator::new(it)) } @@ -565,14 +477,13 @@ impl Fdb { /// # Errors /// /// Returns an error if the control operation fails. - pub fn control( + pub fn control( &self, - request: &Request, + request: &metkit::MarsRequest, action: ControlAction, identifiers: &[ControlIdentifier], ) -> Result { - let it = - self.with_handle(|h| h.control(&request.to_request_string(), action, identifiers))?; + let it = self.with_handle(|h| h.control(request.as_sys(), action, identifiers))?; Ok(ControlIterator::new(it)) } @@ -625,3 +536,70 @@ pub struct FdbStats { /// Re-export callback data type. pub use fdb_sys::ArchiveCallbackData; + +/// Wrapper for `fdb5::MessageArchiver`. +/// +/// This is the same class used by mars-client-cpp's `FDBBase::archive`. Use +/// this when you want a literal port of the C++ archiving call path (filters, +/// modifiers, etc.) rather than going through `Fdb::archive_raw` / +/// `Fdb::archive_reader` which use `fdb5::FDB::archive`. +pub struct MessageArchiver { + inner: Mutex>, +} + +impl MessageArchiver { + /// Construct an archiver. `key` is the modifier key applied to every + /// message (use `Key::new()` for none, matching C++ `FDBBase`). + /// `complete_transfers` and `verbose` map directly to the + /// `fdb5::MessageArchiver` ctor flags (mars-client-cpp uses `false`). + pub fn new( + key: &Key, + complete_transfers: bool, + verbose: bool, + config: &eckit::Config, + ) -> Result { + initialize(); + let inner = fdb_sys::MessageArchiverWrapper::create( + key.to_cxx(), + complete_transfers, + verbose, + config.as_sys(), + )?; + Ok(Self { + inner: Mutex::new(inner), + }) + } + + /// `fdb5::MessageArchiver::archive(eckit::DataHandle&)` — streams + /// messages from a Rust [`Read`](std::io::Read) + [`Seek`](std::io::Seek) + /// source and returns total bytes archived. + /// + /// The source is wrapped in an `eckit::DataHandle` that calls back into + /// the Rust reader on each `read()`/`seek()`, so nothing is buffered up + /// front. + pub fn archive(&self, source: R) -> Result + where + R: std::io::Read + std::io::Seek + Send + 'static, + { + let mut handle = + eckit_sys::DataHandleWrapper::from_reader(eckit_sys::make_reader_box(source))?; + let mut guard = self.inner.lock(); + let bytes = guard.pin_mut().archive(handle.pin_mut())?; + drop(guard); + Ok(bytes) + } + + /// `fdb5::MessageArchiver::flush()`. + pub fn flush(&self) -> Result<()> { + let mut guard = self.inner.lock(); + guard.pin_mut().flush()?; + drop(guard); + Ok(()) + } +} + +// SAFETY: cxx UniquePtr is `!Send` by default; the Mutex serialises every +// access to the underlying C++ object so moving the wrapper between threads +// is safe. +#[allow(clippy::non_send_fields_in_send_ty)] +unsafe impl Send for MessageArchiver {} diff --git a/rust/crates/fdb/src/iterator.rs b/rust/crates/fdb/src/iterator.rs index f7a3db35e..f6f908fa8 100644 --- a/rust/crates/fdb/src/iterator.rs +++ b/rust/crates/fdb/src/iterator.rs @@ -56,7 +56,7 @@ impl ListIterator { where W: std::io::Write, { - let data = fdb_sys::list_iterator_dump_compact(self.handle.pin_mut())?; + let data = self.handle.pin_mut().dump_compact()?; // Mark exhausted so any stray subsequent use surfaces as // `None` rather than trying to touch the drained C++ iterator. self.exhausted = true; diff --git a/rust/crates/fdb/src/lib.rs b/rust/crates/fdb/src/lib.rs index a3914d6b1..401ea8dba 100644 --- a/rust/crates/fdb/src/lib.rs +++ b/rust/crates/fdb/src/lib.rs @@ -9,14 +9,15 @@ //! makes it the typical entry point for browsing what's archived. //! //! ```no_run -//! use fdb::{Fdb, ListOptions, Request}; +//! use fdb::{Fdb, ListOptions}; //! //! # fn main() -> Result<(), Box> { +//! eckit::init(); //! let fdb = Fdb::open_default()?; //! -//! let request = Request::new() -//! .with("class", "od") -//! .with("expver", "0001"); +//! let mut request = metkit::MarsRequest::new("retrieve"); +//! request.set("class", "od"); +//! request.set("expver", "0001"); //! //! // ListOptions::default() is depth=3 (full traversal), deduplicate=true //! for item in fdb.list(&request, ListOptions::default())? { @@ -33,25 +34,21 @@ //! # } //! ``` -mod datareader; mod error; mod handle; mod iterator; mod key; mod options; -mod request; -pub use datareader::DataReader; pub use error::{Error, Result}; -pub use handle::{ArchiveCallbackData, Fdb, FdbConfig, FdbStats}; +pub use handle::{ArchiveCallbackData, Fdb, FdbStats, MessageArchiver}; pub use iterator::{ CompactSummary, ControlElement, ControlIterator, DbStats, DumpElement, DumpIterator, IndexStats, ListElement, ListIterator, PurgeElement, PurgeIterator, StatsElement, StatsIterator, StatusElement, StatusIterator, WipeElement, WipeIterator, }; pub use key::Key; -pub use options::{DumpOptions, ListOptions, PurgeOptions, WipeOptions}; -pub use request::Request; +pub use options::{DumpOptions, ListOptions, PurgeOptions, UserConfig, WipeOptions}; // Re-export control enums from the cxx bindings pub use fdb_sys::{ControlAction, ControlIdentifier}; @@ -59,11 +56,11 @@ pub use fdb_sys::{ControlAction, ControlIdentifier}; /// Version string of the underlying FDB C++ library. #[must_use] pub fn version() -> String { - fdb_sys::fdb_version() + fdb_sys::Library::version() } /// Git SHA1 of the underlying FDB C++ library. #[must_use] pub fn git_sha1() -> String { - fdb_sys::fdb_git_sha1() + fdb_sys::Library::git_sha1() } diff --git a/rust/crates/fdb/src/options.rs b/rust/crates/fdb/src/options.rs index e15e620d8..1f1126065 100644 --- a/rust/crates/fdb/src/options.rs +++ b/rust/crates/fdb/src/options.rs @@ -13,11 +13,13 @@ //! `..Default::default()`: //! //! ```no_run -//! use fdb::{Fdb, Request, WipeOptions}; +//! use fdb::{Fdb, WipeOptions}; //! //! # fn main() -> fdb::Result<()> { +//! eckit::init(); //! let fdb = Fdb::open_default()?; -//! let request = Request::new().with("class", "od"); +//! let mut request = metkit::MarsRequest::new("retrieve"); +//! request.set("class", "od"); //! //! // Dry run with safe defaults — clearly the safe case. //! for entry in fdb.wipe(&request, WipeOptions::default())? { let _ = entry?; } @@ -38,6 +40,56 @@ //! - `DumpOptions`: `simple = false` — verbose dump by default, matching //! `fdb-dump`. +/// Per-instance FDB user configuration. +/// +/// Overlays the main FDB config with per-instance tuning parameters. +/// Pass to [`Fdb::open`](crate::Fdb::open) as the `user_config` argument. +/// +/// # Example +/// +/// ```no_run +/// use fdb::{Fdb, UserConfig}; +/// +/// let cfg: eckit::Config = "type: local\nspaces: []".parse()?; +/// let fdb = Fdb::open( +/// Some(&cfg), +/// Some(UserConfig { use_sub_toc: true, ..Default::default() }), +/// )?; +/// # Ok::<(), fdb::Error>(()) +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct UserConfig { + /// Enable sub-TOC files for improved write performance. + /// Default: `false`. + pub use_sub_toc: bool, + /// Preload `BTree` index into memory on open for faster lookups. + /// Default: `true`. + pub preload_toc_btree: bool, + /// Maximum read size limit for remote FDB (bytes). + /// Default: 1 GiB. + pub read_limit: i64, +} + +impl Default for UserConfig { + fn default() -> Self { + Self { + use_sub_toc: false, + preload_toc_btree: true, + read_limit: 1024 * 1024 * 1024, // 1 GiB + } + } +} + +impl From for eckit::Config { + fn from(cfg: UserConfig) -> Self { + let mut config = Self::new(); + config.set("useSubToc", cfg.use_sub_toc); + config.set("preloadTocBTree", cfg.preload_toc_btree); + config.set("limits.read", cfg.read_limit); + config + } +} + /// Options for [`Fdb::list`](crate::Fdb::list). /// /// Defaults match `fdb-list`'s defaults: full-depth traversal, masked diff --git a/rust/crates/fdb/src/request.rs b/rust/crates/fdb/src/request.rs deleted file mode 100644 index 9dc88ee5f..000000000 --- a/rust/crates/fdb/src/request.rs +++ /dev/null @@ -1,265 +0,0 @@ -//! FDB request wrapper. - -use std::str::FromStr; - -use indexmap::IndexMap; - -use crate::error::{Error, Result}; - -/// A request for FDB list/retrieve operations. -/// -/// Requests specify which fields to list or retrieve from FDB. Each MARS -/// key maps to exactly one value list — setting the same key twice -/// replaces the earlier list (last write wins). Insertion order is -/// preserved for predictable rendering via [`Self::to_request_string`]. -/// -/// # Example -/// -/// ``` -/// use fdb::Request; -/// -/// let request = Request::new() -/// .with("class", "od") -/// .with("expver", "0001") -/// .with_values("step", &["0", "6", "12"]); -/// ``` -#[derive(Debug, Clone, Default)] -pub struct Request { - entries: IndexMap>, -} - -impl Request { - /// Create a new empty request. - #[must_use] - pub fn new() -> Self { - Self::default() - } - - /// Set a single value for a key (builder pattern). - /// - /// If the key already exists, its value list is replaced — **last - /// write wins**. MARS requests have at most one value list per key, - /// so silently keeping two separate entries for the same key would - /// produce an invalid request string (`class=od,class=rd`). - #[must_use] - pub fn with(self, name: &str, value: &str) -> Self { - self.with_values(name, &[value]) - } - - /// Set multiple values for a key (builder pattern). - /// - /// If the key already exists, its value list is replaced. - #[must_use] - pub fn with_values(mut self, name: &str, values: &[&str]) -> Self { - self.set(name, values); - self - } - - /// Set a single value for a key (mutable reference). - /// - /// Same "last write wins" semantics as [`Self::with`]. - pub fn add(&mut self, name: &str, value: &str) -> &mut Self { - self.add_values(name, &[value]) - } - - /// Set multiple values for a key (mutable reference). - /// - /// Same "last write wins" semantics as [`Self::with_values`]. - pub fn add_values(&mut self, name: &str, values: &[&str]) -> &mut Self { - self.set(name, values); - self - } - - /// Shared implementation for the builder / mutable APIs. `IndexMap::insert` - /// replaces the value in place if the key already exists (preserving - /// its position), otherwise appends a new entry. - fn set(&mut self, name: &str, values: &[&str]) { - let vs: Vec = values.iter().map(ToString::to_string).collect(); - self.entries.insert(name.to_string(), vs); - } - - /// Get the number of entries in the request. - #[must_use] - pub fn len(&self) -> usize { - self.entries.len() - } - - /// Check if the request is empty. - #[must_use] - pub fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - /// Iterate the request entries in insertion order. - pub fn entries(&self) -> impl Iterator + '_ { - self.entries.iter().map(|(k, v)| (k.as_str(), v.as_slice())) - } - - /// Convert to MARS request string format. - /// - /// Format: `key1=val1/val2,key2=val3,...` - #[must_use] - pub fn to_request_string(&self) -> String { - self.entries - .iter() - .map(|(k, vs)| format!("{}={}", k, vs.join("/"))) - .collect::>() - .join(",") - } -} - -impl FromStr for Request { - type Err = Error; - - /// Parse a MARS request string using metkit's parser and expansion - /// machinery. - /// - /// Handles the full MARS language: `key=val1/val2` lists, `to`/`by` - /// ranges (e.g. `step=0/to/24/by/3`), type expansion, optional fields, - /// etc. Internally calls into the C++ bridge so the *exact same* parser - /// is used here as for `Fdb::list`/`retrieve`/etc. - /// - /// # Errors - /// - /// Returns an `Error` if metkit can't parse the request, with the - /// underlying eckit/metkit message attached. - /// - /// # Example - /// - /// ```no_run - /// use fdb::Request; - /// - /// let request: Request = "class=od,step=0/to/12/by/3".parse()?; - /// assert_eq!(request.len(), 2); - /// # Ok::<(), fdb::Error>(()) - /// ``` - fn from_str(s: &str) -> Result { - let parsed = fdb_sys::parse_mars_request(s)?; - let mut entries = IndexMap::with_capacity(parsed.params.len()); - for param in parsed.params { - entries.insert(param.key, param.values); - } - Ok(Self { entries }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_request_creation() { - let request = Request::new(); - assert!(request.is_empty()); - } - - #[test] - fn test_request_builder() { - let request = Request::new() - .with("class", "od") - .with("expver", "0001") - .with_values("step", &["0", "6", "12"]); - - assert_eq!(request.len(), 3); - } - - #[test] - fn test_request_add() { - let mut request = Request::new(); - request.add("class", "od").add("expver", "0001"); - assert_eq!(request.len(), 2); - } - - #[test] - fn test_request_string() { - let request = Request::new() - .with("class", "od") - .with_values("step", &["0", "6"]); - - assert_eq!(request.to_request_string(), "class=od,step=0/6"); - } - - /// Setting a key that already exists must replace the previous value - /// list — MARS has one value list per key, so producing - /// `class=od,class=rd` would be malformed. - #[test] - fn test_request_with_last_write_wins() { - let request = Request::new().with("class", "od").with("class", "rd"); - - assert_eq!(request.len(), 1); - assert_eq!(request.to_request_string(), "class=rd"); - } - - /// Multi-value overrides follow the same rule: the whole list is - /// replaced, not merged. - #[test] - fn test_request_with_values_last_write_wins() { - let request = Request::new() - .with_values("step", &["0", "6"]) - .with_values("step", &["12", "18"]); - - assert_eq!(request.len(), 1); - assert_eq!(request.to_request_string(), "step=12/18"); - } - - /// The mutable `add` / `add_values` APIs share the override semantics - /// with their builder counterparts. - #[test] - fn test_request_add_last_write_wins() { - let mut request = Request::new(); - request.add("class", "od"); - request.add("class", "rd"); - request.add_values("step", &["0", "6"]); - request.add_values("step", &["12"]); - - assert_eq!(request.len(), 2); - assert_eq!(request.to_request_string(), "class=rd,step=12"); - } - - /// Replacing a key in place must keep it in its original position, - /// so the rendered MARS string is stable across overrides. - #[test] - fn test_request_override_preserves_insertion_order() { - let request = Request::new() - .with("class", "od") - .with("expver", "0001") - .with("class", "rd"); - - assert_eq!(request.to_request_string(), "class=rd,expver=0001"); - } - - #[test] - fn test_request_from_str() { - let request: Request = "class=od,expver=0001" - .parse() - .expect("metkit should parse a trivial request"); - // Each key the user typed should be present after parsing. - let keys: Vec<&str> = request.entries().map(|(k, _)| k).collect(); - assert!(keys.contains(&"class")); - assert!(keys.contains(&"expver")); - } - - #[test] - fn test_request_from_str_with_to_by_range() { - // The whole point of routing through metkit: `to`/`by` should expand - // into a flat value list rather than being treated as literal strings. - let request: Request = "class=od,expver=0001,step=0/to/12/by/3" - .parse() - .expect("metkit should parse a to/by range"); - let step_values: Vec = request - .entries() - .find(|(k, _)| *k == "step") - .map(|(_, vs)| vs.to_vec()) - .expect("step key should be present"); - // step=0/to/12/by/3 expands to [0, 3, 6, 9, 12]. - assert_eq!(step_values, vec!["0", "3", "6", "9", "12"]); - } - - #[test] - fn test_request_from_str_invalid() { - // Garbage that even metkit can't make sense of should be a parse error, - // not a silent empty Request. - let result: Result = "this is not a mars request".parse(); - assert!(result.is_err(), "expected parse failure, got {result:?}"); - } -} diff --git a/rust/crates/fdb/tests/fdb_async.rs b/rust/crates/fdb/tests/fdb_async.rs index 53e759f6d..a59c3fadd 100644 --- a/rust/crates/fdb/tests/fdb_async.rs +++ b/rust/crates/fdb/tests/fdb_async.rs @@ -13,7 +13,7 @@ use std::io::Read; use std::path::PathBuf; use std::sync::Arc; -use fdb::{Fdb, Key, ListOptions, Request}; +use fdb::{Fdb, Key, ListOptions}; use tokio::task::JoinSet; /// Get the path to test fixtures directory. @@ -23,12 +23,12 @@ fn fixtures_dir() -> PathBuf { } /// Create a temporary FDB configuration for testing. -fn create_test_config(tmpdir: &std::path::Path) -> String { +fn create_test_config(tmpdir: &std::path::Path) -> eckit::Config { let schema_src = fixtures_dir().join("schema"); let schema_dst = tmpdir.join("schema"); fs::copy(&schema_src, &schema_dst).expect("failed to copy schema"); - format!( + let yaml = format!( r"--- type: local engine: toc @@ -39,7 +39,18 @@ spaces: ", tmpdir.display(), tmpdir.display() - ) + ); + yaml.parse().expect("failed to parse test config") +} + +/// Build a `MarsRequest` from a Key. +fn request_from_key(key: &Key) -> metkit::MarsRequest { + eckit::init(); + let mut request = metkit::MarsRequest::new("retrieve"); + for (k, v) in key.entries() { + request.set(k, v); + } + request } /// Archive test data and return the key used. @@ -130,7 +141,7 @@ async fn test_fdb_concurrent_retrieve() { let fdb = Arc::clone(&fdb); tasks.spawn(async move { - let request = Request::new() + let key = Key::new() .with("class", "rd") .with("expver", "xxxx") .with("stream", "oper") @@ -141,11 +152,14 @@ async fn test_fdb_concurrent_retrieve() { .with("step", &i.to_string()) .with("param", "151130"); - // Retrieve returns a DataReader that owns the data - let mut reader = fdb.retrieve(&request).expect("retrieve failed"); + let request = request_from_key(&key); + + // Retrieve returns an eckit::DataHandle + let handle = fdb.retrieve(&request).expect("retrieve failed"); + let (mut handle, _len) = handle.open_for_read().expect("open_for_read failed"); let mut buf = Vec::new(); - reader.read_to_end(&mut buf).expect("read failed"); + handle.read_to_end(&mut buf).expect("read failed"); (i, buf.len()) }); @@ -184,10 +198,10 @@ async fn test_fdb_concurrent_list() { let fdb = Arc::clone(&fdb); tasks.spawn(async move { - let request = Request::new() - .with("class", "rd") - .with("expver", "xxxx") - .with("stream", "oper"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); + request.set("stream", "oper"); let entries: Vec<_> = fdb .list( @@ -250,7 +264,7 @@ async fn test_fdb_spawn_blocking_pattern() { // Retrieve using spawn_blocking let fdb_clone = Arc::clone(&fdb); let result = tokio::task::spawn_blocking(move || { - let request = Request::new() + let key = Key::new() .with("class", "rd") .with("expver", "xxxx") .with("stream", "oper") @@ -261,10 +275,12 @@ async fn test_fdb_spawn_blocking_pattern() { .with("step", "1") .with("param", "151130"); - let mut reader = fdb_clone.retrieve(&request).expect("retrieve failed"); + let request = request_from_key(&key); + let handle = fdb_clone.retrieve(&request).expect("retrieve failed"); + let (mut handle, _len) = handle.open_for_read().expect("open_for_read failed"); let mut buf = Vec::new(); - reader.read_to_end(&mut buf).expect("read failed"); + handle.read_to_end(&mut buf).expect("read failed"); buf.len() }) .await diff --git a/rust/crates/fdb/tests/fdb_integration.rs b/rust/crates/fdb/tests/fdb_integration.rs index 86a1e9b83..0db1e25cd 100644 --- a/rust/crates/fdb/tests/fdb_integration.rs +++ b/rust/crates/fdb/tests/fdb_integration.rs @@ -5,10 +5,10 @@ use std::env; use std::fs; -use std::io::Read; +use std::io::{Read, Seek, SeekFrom}; use std::path::PathBuf; -use fdb::{DumpOptions, Fdb, Key, ListOptions, PurgeOptions, Request, WipeOptions}; +use fdb::{DumpOptions, Fdb, Key, ListOptions, PurgeOptions, WipeOptions}; /// Get the path to test fixtures directory. fn fixtures_dir() -> PathBuf { @@ -17,7 +17,7 @@ fn fixtures_dir() -> PathBuf { } /// Create a temporary FDB configuration for testing. -fn create_test_config(tmpdir: &std::path::Path) -> String { +fn create_test_config_yaml(tmpdir: &std::path::Path) -> String { // Copy schema to temp directory let schema_src = fixtures_dir().join("schema"); let schema_dst = tmpdir.join("schema"); @@ -37,6 +37,11 @@ spaces: ) } +fn create_test_config(tmpdir: &std::path::Path) -> eckit::Config { + let yaml = create_test_config_yaml(tmpdir); + yaml.parse().expect("failed to parse test config") +} + #[test] fn test_fdb_version() { let version = fdb::version(); @@ -55,7 +60,7 @@ fn test_fdb_git_sha1() { fn test_fdb_handle_from_yaml() { let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); - println!("Config:\n{config}"); + println!("Config loaded"); let fdb = Fdb::open(Some(&config), None); assert!(fdb.is_ok(), "failed to create FDB handle: {:?}", fdb.err()); @@ -64,13 +69,14 @@ fn test_fdb_handle_from_yaml() { #[test] fn test_fdb_handle_from_path() { let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); - let config = create_test_config(tmpdir.path()); + let yaml = create_test_config_yaml(tmpdir.path()); // Write the config to a file and load it via the path-based constructor. let config_path = tmpdir.path().join("fdb.yaml"); - fs::write(&config_path, &config).expect("failed to write config file"); + fs::write(&config_path, &yaml).expect("failed to write config file"); - let fdb = Fdb::open(Some(&config_path), None); + let config = eckit::Config::from_path(&config_path).expect("failed to load config from path"); + let fdb = Fdb::open(Some(&config), None); assert!( fdb.is_ok(), "failed to create FDB handle from path {:?}: {:?}", @@ -97,7 +103,9 @@ fn test_fdb_handle_from_path() { fdb.archive(&key, &grib_data).expect("archive failed"); fdb.flush().expect("flush failed"); - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let items: Vec<_> = fdb .list( &request, @@ -116,17 +124,16 @@ fn test_fdb_handle_from_path() { fn test_fdb_handle_from_path_invalid_utf8() { use std::os::unix::ffi::OsStrExt; use std::path::Path; - // Construct a path with a non-UTF-8 byte sequence. We don't need this - // file to exist — `from_path` should reject the path before touching - // the filesystem. + // Construct a path with a non-UTF-8 byte sequence. `Config::from_path` + // should reject the path before touching the filesystem. let bad = std::ffi::OsStr::from_bytes(b"/tmp/\xff-not-utf8"); - let result = Fdb::open(Some(Path::new(bad)), None); + let result = eckit::Config::from_path(Path::new(bad)); let err = result .err() .expect("from_path should reject a non-UTF-8 path"); assert!( - matches!(err, fdb::Error::UserError(_)), - "expected UserError for non-UTF-8 path, got {err:?}" + matches!(err, eckit::Error::Other(_)), + "expected Other error for non-UTF-8 path, got {err:?}" ); } @@ -137,9 +144,13 @@ fn test_fdb_key_creation() { } #[test] -fn test_fdb_request_creation() { - let request = Request::new().with("class", "rd").with("expver", "xxxx"); - assert_eq!(request.len(), 2); +fn test_mars_request_creation() { + eckit::init(); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); + assert!(request.has("class")); + assert!(request.has("expver")); } #[test] @@ -154,7 +165,9 @@ fn test_fdb_list_no_results() { // values it can type-check, so we can't pass a literal 'nonexistent' // class — we have to express "no results" via a value the schema // accepts but that doesn't appear in the database. - let request = Request::new().with("class", "rd").with("expver", "zzzz"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "zzzz"); let items: Vec<_> = fdb .list( @@ -174,12 +187,17 @@ fn test_fdb_list_no_results() { fn test_fdb_archive_simple() { let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); + println!("Temp dir: {}", tmpdir.path().display()); + println!("Config loaded"); let fdb = Fdb::open(Some(&config), None).expect("failed to create FDB from YAML"); + // Read test GRIB data let grib_path = fixtures_dir().join("template.grib"); let grib_data = fs::read(&grib_path).expect("failed to read template.grib"); + println!("GRIB data size: {} bytes", grib_data.len()); + // Create key matching schema: class, expver, stream, date, time, type, levtype, step, param let key = Key::new() .with("class", "rd") .with("expver", "xxxx") @@ -220,7 +238,9 @@ fn test_fdb_archive_retrieve_cycle() { fdb.flush().expect("flush failed"); // List with partial query - let list_request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut list_request = metkit::MarsRequest::new("retrieve"); + list_request.set("class", "rd"); + list_request.set("expver", "xxxx"); let items: Vec<_> = fdb .list( @@ -237,20 +257,21 @@ fn test_fdb_archive_retrieve_cycle() { assert!(!items.is_empty(), "no items found after archive"); // Retrieve with fully-specified request (FDB needs exact match for retrieve) - let retrieve_request = Request::new() - .with("class", "rd") - .with("expver", "xxxx") - .with("stream", "oper") - .with("date", "20230508") - .with("time", "1200") - .with("type", "fc") - .with("levtype", "sfc") - .with("step", "0") - .with("param", "151130"); - - let mut reader = fdb.retrieve(&retrieve_request).expect("failed to retrieve"); + let mut retrieve_request = metkit::MarsRequest::new("retrieve"); + retrieve_request.set("class", "rd"); + retrieve_request.set("expver", "xxxx"); + retrieve_request.set("stream", "oper"); + retrieve_request.set("date", "20230508"); + retrieve_request.set("time", "1200"); + retrieve_request.set("type", "fc"); + retrieve_request.set("levtype", "sfc"); + retrieve_request.set("step", "0"); + retrieve_request.set("param", "151130"); + + let handle = fdb.retrieve(&retrieve_request).expect("failed to retrieve"); + let (mut handle, _len) = handle.open_for_read().expect("open_for_read failed"); let mut retrieved_data = Vec::new(); - reader + handle .read_to_end(&mut retrieved_data) .expect("failed to read"); @@ -264,33 +285,33 @@ fn test_fdb_axes() { let fdb = Fdb::open(Some(&config), None).expect("failed to create FDB from YAML"); + // Archive some data first let grib_path = fixtures_dir().join("template.grib"); let grib_data = fs::read(&grib_path).expect("failed to read template.grib"); - // Archive four fields that share every key except `step`, so the - // axes query returns a real span for at least one keyword. - let steps = ["0", "3", "6", "9"]; - for step in &steps { - let key = Key::new() - .with("class", "rd") - .with("expver", "xxxx") - .with("stream", "oper") - .with("date", "20230508") - .with("time", "1200") - .with("type", "fc") - .with("levtype", "sfc") - .with("step", step) - .with("param", "151130"); - fdb.archive(&key, &grib_data).expect("failed to archive"); - } + let key = Key::new() + .with("class", "rd") + .with("expver", "xxxx") + .with("stream", "oper") + .with("date", "20230508") + .with("time", "1200") + .with("type", "fc") + .with("levtype", "sfc") + .with("step", "0") + .with("param", "151130"); + + fdb.archive(&key, &grib_data).expect("failed to archive"); fdb.flush().expect("flush failed"); - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + // Query axes + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let axes = fdb.axes(&request, 3).expect("failed to get axes"); - // Single-valued axes: each must contain exactly one value matching - // the key we archived (no extra crud allowed). - let single_valued: &[(&str, &str)] = &[ + // We archived exactly one field, so each axis the schema covers + // should be present with exactly the value from the key. + let expected: &[(&str, &str)] = &[ ("class", "rd"), ("expver", "xxxx"), ("stream", "oper"), @@ -298,30 +319,19 @@ fn test_fdb_axes() { ("time", "1200"), ("type", "fc"), ("levtype", "sfc"), + ("step", "0"), ("param", "151130"), ]; - for (axis, value) in single_valued { + for (axis, value) in expected { let values = axes .get(*axis) .unwrap_or_else(|| panic!("axis {axis:?} missing from axes() result: {axes:#?}")); - assert_eq!( - values, - &[value.to_string()], - "axis {axis:?}: expected exactly [{value:?}], got {values:?}" + assert!( + values.iter().any(|v| v == value), + "axis {axis:?} does not contain expected value {value:?} (got {values:?})" ); } - - // Multi-valued axis: `step` should contain exactly the four values - // we archived, in any order. - let step_values = axes - .get("step") - .unwrap_or_else(|| panic!("axis \"step\" missing from axes() result: {axes:#?}")); - let mut got: Vec<&str> = step_values.iter().map(String::as_str).collect(); - got.sort_unstable(); - let mut want: Vec<&str> = steps.to_vec(); - want.sort_unstable(); - assert_eq!(got, want, "step axis: expected {want:?}, got {got:?}"); } #[test] @@ -350,7 +360,8 @@ fn test_fdb_dump() { fdb.flush().expect("flush failed"); // Dump database structure - let request = Request::new().with("class", "rd"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); let dump_items: Vec<_> = fdb .dump(&request, DumpOptions { simple: true }) .expect("failed to dump") @@ -399,7 +410,8 @@ fn test_fdb_status() { fdb.flush().expect("flush failed"); // Get status - let request = Request::new().with("class", "rd"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); let status_items: Vec<_> = fdb .status(&request) .expect("failed to get status") @@ -448,7 +460,8 @@ fn test_fdb_wipe_dry_run() { fdb.flush().expect("flush failed"); // Verify data exists - let list_request = Request::new().with("class", "rd"); + let mut list_request = metkit::MarsRequest::new("retrieve"); + list_request.set("class", "rd"); let items_before: Vec<_> = fdb .list( &list_request, @@ -465,7 +478,9 @@ fn test_fdb_wipe_dry_run() { ); // Dry-run wipe (doit=false) - let wipe_request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut wipe_request = metkit::MarsRequest::new("retrieve"); + wipe_request.set("class", "rd"); + wipe_request.set("expver", "xxxx"); let wipe_items: Vec<_> = fdb .wipe(&wipe_request, WipeOptions::default()) .expect("failed to wipe") @@ -525,7 +540,8 @@ fn test_fdb_purge_dry_run() { fdb.flush().expect("flush failed"); // Dry-run purge (doit=false) - let purge_request = Request::new().with("class", "rd"); + let mut purge_request = metkit::MarsRequest::new("retrieve"); + purge_request.set("class", "rd"); let purge_items: Vec<_> = fdb .purge(&purge_request, PurgeOptions::default()) .expect("failed to purge") @@ -566,7 +582,8 @@ fn test_fdb_stats_iterator() { fdb.flush().expect("flush failed"); // Get stats - let request = Request::new().with("class", "rd"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); let stats_items: Vec<_> = fdb .stats_iter(&request) .expect("failed to get stats") @@ -885,7 +902,8 @@ fn test_fdb_wipe_actual() { println!("Archived 2 fields to 2 databases"); // Verify FDB is populated - let list_request = Request::new().with("class", "rd"); + let mut list_request = metkit::MarsRequest::new("retrieve"); + list_request.set("class", "rd"); let items: Vec<_> = fdb .list( &list_request, @@ -900,7 +918,9 @@ fn test_fdb_wipe_actual() { println!("Listed {} fields", items.len()); // Wipe first database (doit=true) - let wipe_request1 = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut wipe_request1 = metkit::MarsRequest::new("retrieve"); + wipe_request1.set("class", "rd"); + wipe_request1.set("expver", "xxxx"); let wipe_items: Vec<_> = fdb .wipe( &wipe_request1, @@ -928,7 +948,8 @@ fn test_fdb_wipe_actual() { println!("Listed {} fields after wipe", items_after.len()); // Wipe remaining database - let wipe_request2 = Request::new().with("class", "rd"); + let mut wipe_request2 = metkit::MarsRequest::new("retrieve"); + wipe_request2.set("class", "rd"); let _: Vec<_> = fdb .wipe( &wipe_request2, @@ -986,7 +1007,8 @@ fn test_fdb_wipe_masked_data() { println!("Archived 2 fields (1 masked)"); // List including masked - let list_request = Request::new().with("class", "rd"); + let mut list_request = metkit::MarsRequest::new("retrieve"); + list_request.set("class", "rd"); let items_with_masked: Vec<_> = fdb .list( &list_request, @@ -1008,7 +1030,9 @@ fn test_fdb_wipe_masked_data() { assert_eq!(items_dedup.len(), 1, "expected 1 field when deduplicated"); // Wipe all - let wipe_request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut wipe_request = metkit::MarsRequest::new("retrieve"); + wipe_request.set("class", "rd"); + wipe_request.set("expver", "xxxx"); let wipe_items: Vec<_> = fdb .wipe( &wipe_request, @@ -1065,7 +1089,8 @@ fn test_fdb_purge_actual() { println!("Archived 2 fields (1 duplicate)"); // List including masked - let list_request = Request::new().with("class", "rd"); + let mut list_request = metkit::MarsRequest::new("retrieve"); + list_request.set("class", "rd"); let items_before: Vec<_> = fdb .list( &list_request, @@ -1079,7 +1104,8 @@ fn test_fdb_purge_actual() { println!("Listed {} fields before purge", items_before.len()); // Purge duplicates (doit=true) - let purge_request = Request::new().with("class", "rd"); + let mut purge_request = metkit::MarsRequest::new("retrieve"); + purge_request.set("class", "rd"); let purge_items: Vec<_> = fdb .purge( &purge_request, @@ -1122,7 +1148,7 @@ fn test_fdb_config_from_yaml() { fs::copy(&schema_src, &schema_dst).expect("failed to copy schema"); // Create YAML config (matching C++ test_config.cc format) - let config = format!( + let yaml = format!( r"--- type: local engine: toc @@ -1134,8 +1160,9 @@ spaces: tmpdir.path().display(), tmpdir.path().display() ); + let config: eckit::Config = yaml.parse().expect("failed to parse YAML config"); - let fdb = Fdb::open(Some(&config), None).expect("failed to create FDB from YAML"); + let fdb = Fdb::open(Some(&config), None).expect("failed to create FDB from config"); // Verify the FDB handle came up cleanly with the YAML we built. let name = fdb.name(); @@ -1170,92 +1197,99 @@ fn test_fdb_datareader_seek() { fdb.archive(&key, &grib_data).expect("failed to archive"); fdb.flush().expect("flush failed"); - // Retrieve to get a DataReader - let retrieve_request = Request::new() - .with("class", "rd") - .with("expver", "xxxx") - .with("stream", "oper") - .with("date", "20230508") - .with("time", "1200") - .with("type", "fc") - .with("levtype", "sfc") - .with("step", "0") - .with("param", "151130"); - - let mut reader = fdb.retrieve(&retrieve_request).expect("failed to retrieve"); - - // Test size() and tell() - let total_size = reader.size(); - assert!(total_size > 0, "expected non-zero size"); - assert_eq!(reader.tell(), 0, "expected initial position at 0"); + // Retrieve returns an eckit::DataHandle + let mut retrieve_request = metkit::MarsRequest::new("retrieve"); + retrieve_request.set("class", "rd"); + retrieve_request.set("expver", "xxxx"); + retrieve_request.set("stream", "oper"); + retrieve_request.set("date", "20230508"); + retrieve_request.set("time", "1200"); + retrieve_request.set("type", "fc"); + retrieve_request.set("levtype", "sfc"); + retrieve_request.set("step", "0"); + retrieve_request.set("param", "151130"); + + let handle = fdb.retrieve(&retrieve_request).expect("failed to retrieve"); + + // Open for reading and get estimated size + let (mut handle, estimated) = handle.open_for_read().expect("open_for_read failed"); + assert!(estimated > 0, "expected non-zero estimated size"); + let total_size: u64 = estimated.try_into().expect("negative size"); + assert_eq!( + handle.position().expect("position"), + 0, + "expected initial position at 0" + ); // Test SeekFrom::Start - let pos = reader + let pos = handle .seek(SeekFrom::Start(10)) .expect("seek to start+10 failed"); assert_eq!(pos, 10); - assert_eq!(reader.tell(), 10); + assert_eq!(handle.position().expect("position"), 10); // Test SeekFrom::Current (positive) - let pos = reader + let pos = handle .seek(SeekFrom::Current(5)) .expect("seek current+5 failed"); assert_eq!(pos, 15); - assert_eq!(reader.tell(), 15); + assert_eq!(handle.position().expect("position"), 15); // Test SeekFrom::Current (negative) - let pos = reader + let pos = handle .seek(SeekFrom::Current(-5)) .expect("seek current-5 failed"); assert_eq!(pos, 10); - assert_eq!(reader.tell(), 10); + assert_eq!(handle.position().expect("position"), 10); // Test SeekFrom::End - let pos = reader.seek(SeekFrom::End(-10)).expect("seek end-10 failed"); + let pos = handle.seek(SeekFrom::End(-10)).expect("seek end-10 failed"); assert_eq!(pos, total_size - 10); - assert_eq!(reader.tell(), total_size - 10); + assert_eq!( + u64::try_from(handle.position().expect("position")).expect("negative pos"), + total_size - 10 + ); // Test SeekFrom::End to get to end - let pos = reader.seek(SeekFrom::End(0)).expect("seek to end failed"); + let pos = handle.seek(SeekFrom::End(0)).expect("seek to end failed"); assert_eq!(pos, total_size); // Test SeekFrom::Start to rewind - let pos = reader.seek(SeekFrom::Start(0)).expect("rewind failed"); + let pos = handle.seek(SeekFrom::Start(0)).expect("rewind failed"); assert_eq!(pos, 0); - // Test seek_to() method - reader.seek_to(20).expect("seek_to failed"); - assert_eq!(reader.tell(), 20); + // Test seek then read + handle.seek(SeekFrom::Start(20)).expect("seek failed"); + assert_eq!(handle.position().expect("position"), 20); - // Test read after seek let mut buf = [0u8; 10]; - let n = reader.read(&mut buf).expect("read after seek failed"); + let n = handle.read(&mut buf).expect("read after seek failed"); assert!(n > 0, "expected to read some bytes"); - // Test read_all() reads from current position - reader - .seek(SeekFrom::Start(0)) - .expect("rewind before read_all failed"); - let all_data = reader.read_all().expect("read_all failed"); + // Test read_to_end from start + handle.seek(SeekFrom::Start(0)).expect("rewind failed"); + let mut all_data = Vec::new(); + handle + .read_to_end(&mut all_data) + .expect("read_to_end failed"); assert_eq!(all_data.len(), grib_data.len()); assert_eq!(all_data, grib_data); // Test negative position errors - reader.seek(SeekFrom::Start(0)).expect("rewind failed"); - let err = reader.seek(SeekFrom::Current(-100)); + handle.seek(SeekFrom::Start(0)).expect("rewind failed"); + let err = handle.seek(SeekFrom::Current(-100)); assert!( err.is_err(), "expected error when seeking to negative position" ); - let err = reader.seek(SeekFrom::End(-(total_size.cast_signed() + 100))); + let err = handle.seek(SeekFrom::End(-(total_size.cast_signed() + 100))); assert!( err.is_err(), "expected error when seeking before start via End" ); - // Test close() explicitly - reader.close().expect("close failed"); + handle.close().expect("close failed"); } #[test] @@ -1284,7 +1318,9 @@ fn test_fdb_list_element_full_key() { fdb.flush().expect("flush failed"); // List and check full_key() - let list_request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut list_request = metkit::MarsRequest::new("retrieve"); + list_request.set("class", "rd"); + list_request.set("expver", "xxxx"); let items: Vec<_> = fdb .list( &list_request, @@ -1367,7 +1403,9 @@ fn test_fdb_list_dump_compact() { // Default ListOptions (depth=3, deduplicate=true) matches the mode // `dumpCompact` requires — it asserts `keys.size() == 3` internally. - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let list_iter = fdb .list(&request, fdb::ListOptions::default()) .expect("failed to list"); @@ -1435,7 +1473,9 @@ fn test_fdb_control_lock_unlock() { fdb.archive(&key, &grib_data).expect("failed to archive"); fdb.flush().expect("flush failed"); - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let identifiers = [ fdb::ControlIdentifier::Retrieve, fdb::ControlIdentifier::Archive, @@ -1532,7 +1572,9 @@ fn test_fdb_archive_raw() { // Verify the data actually landed in the database by listing it back // with the exact key the GRIB embeds, and check the field-level entry // matches. - let request = Request::new().with("class", "od").with("expver", "0001"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "od"); + request.set("expver", "0001"); let items: Vec<_> = fdb .list( &request, @@ -1601,7 +1643,9 @@ fn test_fdb_archive_reader() { fdb.flush().expect("flush failed"); // Verify the same key/length the slice-based test asserts on. - let request = Request::new().with("class", "od").with("expver", "0001"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "od"); + request.set("expver", "0001"); let items: Vec<_> = fdb .list( &request, @@ -1686,7 +1730,9 @@ fn test_fdb_read_uri() { fdb.flush().expect("flush failed"); // List to get the URI - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let items: Vec<_> = fdb .list( &request, @@ -1708,10 +1754,13 @@ fn test_fdb_read_uri() { println!("Reading from URI: {uri} (offset={offset}, length={length})"); // Read using the URI - let mut reader = fdb.read_uri(uri).expect("failed to read_uri"); + let reader = fdb.read_uri(uri).expect("failed to read_uri"); + let (mut reader, _len) = reader.open_for_read().expect("open_for_read failed"); // Seek to the offset and read the data - reader.seek_to(offset).expect("failed to seek"); + reader + .seek(SeekFrom::Start(offset)) + .expect("failed to seek"); let mut data = vec![0u8; usize::try_from(length).expect("length exceeds usize::MAX")]; reader.read_exact(&mut data).expect("failed to read"); @@ -1753,7 +1802,9 @@ fn test_fdb_read_uris() { fdb.flush().expect("flush failed"); // List to get URIs - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let items: Vec<_> = fdb .list( &request, @@ -1774,10 +1825,14 @@ fn test_fdb_read_uris() { println!("Reading from {} URIs", uris.len()); // Read using multiple URIs - let mut reader = fdb.read_uris(&uris, false).expect("failed to read_uris"); + let reader = fdb.read_uris(&uris, false).expect("failed to read_uris"); + let (mut reader, _len) = reader.open_for_read().expect("open_for_read failed"); // Read all data - let data = reader.read_all().expect("failed to read_all"); + let mut data = Vec::new(); + reader + .read_to_end(&mut data) + .expect("failed to read_to_end"); println!("read_uris returned {} bytes", data.len()); // Should have read data from both URIs @@ -1811,7 +1866,9 @@ fn test_fdb_read_from_list() { fdb.flush().expect("flush failed"); // Get a list iterator - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let list_iter = fdb .list( &request, @@ -1823,12 +1880,16 @@ fn test_fdb_read_from_list() { .expect("failed to list"); // Read from the list iterator - let mut reader = fdb + let reader = fdb .read_from_list(list_iter, false) .expect("failed to read_from_list"); + let (mut reader, _len) = reader.open_for_read().expect("open_for_read failed"); // Read all data - let data = reader.read_all().expect("failed to read_all"); + let mut data = Vec::new(); + reader + .read_to_end(&mut data) + .expect("failed to read_to_end"); println!("read_from_list returned {} bytes", data.len()); assert_eq!( @@ -1897,8 +1958,14 @@ fn test_fdb_subtoc_user_config() { let tmpdir_off = tempfile::tempdir().expect("failed to create temp dir"); let config_off = create_test_config(tmpdir_off.path()); { - let fdb_off = - Fdb::open(Some(&config_off), Some("useSubToc: false")).expect("from_yaml off"); + let fdb_off = Fdb::open( + Some(&config_off), + Some(fdb::UserConfig { + use_sub_toc: false, + ..Default::default() + }), + ) + .expect("from_yaml off"); archive_one_record(&fdb_off); } // drop handle so the TOC is fully closed before we walk the dir @@ -1912,7 +1979,14 @@ fn test_fdb_subtoc_user_config() { let tmpdir_on = tempfile::tempdir().expect("failed to create temp dir"); let config_on = create_test_config(tmpdir_on.path()); { - let fdb_on = Fdb::open(Some(&config_on), Some("useSubToc: true")).expect("from_yaml on"); + let fdb_on = Fdb::open( + Some(&config_on), + Some(fdb::UserConfig { + use_sub_toc: true, + ..Default::default() + }), + ) + .expect("from_yaml on"); archive_one_record(&fdb_on); } @@ -1932,17 +2006,24 @@ fn test_fdb_subtoc_user_config() { /// the C++ side and that an archive + list round-trip succeeds in each mode. #[test] fn test_fdb_preload_toc_btree_user_config() { - for preload in ["true", "false"] { + for preload in [true, false] { let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); - let user_config = format!("preloadTocBTree: {preload}"); - let fdb = Fdb::open(Some(&config), Some(&user_config)) - .unwrap_or_else(|e| panic!("from_yaml_with_user_config({user_config:?}) failed: {e}")); + let fdb = Fdb::open( + Some(&config), + Some(fdb::UserConfig { + preload_toc_btree: preload, + ..Default::default() + }), + ) + .unwrap_or_else(|e| panic!("preloadTocBTree={preload} failed: {e}")); archive_one_record(&fdb); - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let items: Vec<_> = fdb .list( &request, diff --git a/rust/crates/fdb/tests/fdb_thread_safety.rs b/rust/crates/fdb/tests/fdb_thread_safety.rs index 17d85aa34..bd9b2fd1f 100644 --- a/rust/crates/fdb/tests/fdb_thread_safety.rs +++ b/rust/crates/fdb/tests/fdb_thread_safety.rs @@ -12,28 +12,42 @@ //! //! Run with `cargo test --test fdb_thread_safety`. -use std::env; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::thread; -use fdb::{Fdb, Key, ListOptions, Request}; +use fdb::{Fdb, Key, ListOptions}; + +// ============================================================================= +// Test fixtures +// ============================================================================= fn fixtures_dir() -> PathBuf { - PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")) - .join("tests/fixtures") + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(manifest_dir).join("tests/fixtures") } -fn create_test_config(tmpdir: &std::path::Path) -> String { +/// Build a tempdir-backed FDB config so tests don't depend on `FDB_HOME` / +/// `FDB_CONFIG_FILE`. +fn create_test_config(tmpdir: &Path) -> eckit::Config { let schema_src = fixtures_dir().join("schema"); let schema_dst = tmpdir.join("schema"); - fs::copy(&schema_src, &schema_dst).expect("copy schema"); - format!( - "---\ntype: local\nengine: toc\nschema: {}/schema\nspaces:\n- handler: Default\n roots:\n - path: {}\n", + fs::copy(&schema_src, &schema_dst).expect("failed to copy schema"); + + let yaml = format!( + r"--- +type: local +engine: toc +schema: {}/schema +spaces: + - roots: + - path: {} +", tmpdir.display(), tmpdir.display() - ) + ); + yaml.parse().expect("failed to parse test config") } // ============================================================================= @@ -64,24 +78,15 @@ fn test_key_traits() { assert_sync::(); } -/// Test: `Request` is Send + Sync -#[test] -fn test_request_traits() { - fn assert_send() {} - fn assert_sync() {} - - assert_send::(); - assert_sync::(); -} - // ============================================================================= -// Runtime tests (require FDB libraries and configuration) +// Runtime tests (use a tempdir-backed config) // ============================================================================= /// Test: `Fdb` handle can be created #[test] fn test_handle_creation() { - let tmpdir = tempfile::tempdir().expect("tmpdir"); + eckit::init(); + let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Fdb::open(Some(&config), None); assert!(fdb.is_ok(), "Failed to create Fdb: {:?}", fdb.err()); @@ -90,7 +95,8 @@ fn test_handle_creation() { /// Test: `Fdb` can be shared via Arc for concurrent access #[test] fn test_arc_sharing_readonly() { - let tmpdir = tempfile::tempdir().expect("tmpdir"); + eckit::init(); + let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); @@ -116,7 +122,8 @@ fn test_arc_sharing_readonly() { /// Test: Concurrent read-only operations (id, name, dirty, stats) #[test] fn test_concurrent_readonly_methods() { - let tmpdir = tempfile::tempdir().expect("tmpdir"); + eckit::init(); + let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); @@ -142,7 +149,8 @@ fn test_concurrent_readonly_methods() { /// Test: `Fdb` can be used for concurrent list operations #[test] fn test_concurrent_list_operations() { - let tmpdir = tempfile::tempdir().expect("tmpdir"); + eckit::init(); + let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); @@ -150,7 +158,8 @@ fn test_concurrent_list_operations() { .map(|_| { let fdb = Arc::clone(&fdb); thread::spawn(move || { - let request = Request::new().with("class", "rd"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); for _ in 0..10 { let _ = fdb.list( &request, @@ -172,7 +181,8 @@ fn test_concurrent_list_operations() { /// Test: Concurrent axes queries #[test] fn test_concurrent_axes() { - let tmpdir = tempfile::tempdir().expect("tmpdir"); + eckit::init(); + let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); @@ -180,7 +190,8 @@ fn test_concurrent_axes() { .map(|_| { let fdb = Arc::clone(&fdb); thread::spawn(move || { - let request = Request::new().with("class", "rd"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); for _ in 0..10 { let _ = fdb.axes(&request, 1); } @@ -196,7 +207,8 @@ fn test_concurrent_axes() { /// Test: Stress test with many threads #[test] fn test_stress_concurrent_access() { - let tmpdir = tempfile::tempdir().expect("tmpdir"); + eckit::init(); + let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); let iterations = 50; @@ -206,7 +218,8 @@ fn test_stress_concurrent_access() { .map(|i| { let fdb = Arc::clone(&fdb); thread::spawn(move || { - let request = Request::new().with("class", "rd"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); for j in 0..iterations { if (i + j) % 2 == 0 { // Read-only operations @@ -232,6 +245,47 @@ fn test_stress_concurrent_access() { } } +/// Note: FDB has a documented caveat about `flush()`: +/// "`flush()` has global semantics - it flushes ALL archived messages from +/// ALL threads, not just the calling thread. For finer control, instantiate +/// one FDB object per thread." +/// +/// This test verifies the basic behavior but users should be aware of +/// this limitation when using FDB in multi-threaded contexts with archiving. +#[test] +fn test_concurrent_errors_no_crash() { + eckit::init(); + let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); + let config = create_test_config(tmpdir.path()); + let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); + + let handles: Vec<_> = (0..8) + .map(|i| { + let fdb = Arc::clone(&fdb); + thread::spawn(move || { + // Use invalid requests to trigger errors + let value = format!("value_{i}"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("INVALID_KEY", &value); + for _ in 0..20 { + // Ignore the error - testing that concurrent errors don't crash + let _ = fdb.list( + &request, + ListOptions { + depth: 1, + deduplicate: false, + }, + ); + } + }) + }) + .collect(); + + for h in handles { + h.join().expect("Thread panicked"); + } +} + // ============================================================================= // Concurrent write tests (M15) // ============================================================================= @@ -243,17 +297,14 @@ fn test_stress_concurrent_access() { /// archive operations don't crash, but users should be aware of this behavior. #[test] fn test_concurrent_archive_operations() { - use std::fs; - use std::path::PathBuf; - + eckit::init(); let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); // Read GRIB data for archiving - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string()); - let grib_path = PathBuf::from(manifest_dir).join("tests/fixtures/template.grib"); + let grib_path = fixtures_dir().join("template.grib"); let grib_data = Arc::new(fs::read(&grib_path).expect("failed to read template.grib")); let thread_count = 4; @@ -297,7 +348,9 @@ fn test_concurrent_archive_operations() { fdb.flush().expect("flush failed"); // Verify data was archived by listing - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); let items: Vec<_> = fdb .list( &request, @@ -322,17 +375,14 @@ fn test_concurrent_archive_operations() { /// Test: Mixed concurrent read and write operations. #[test] fn test_concurrent_read_write_mix() { - use std::fs; - use std::path::PathBuf; - + eckit::init(); let tmpdir = tempfile::tempdir().expect("failed to create temp dir"); let config = create_test_config(tmpdir.path()); let fdb = Arc::new(Fdb::open(Some(&config), None).expect("failed to create handle")); // Pre-archive some data first - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string()); - let grib_path = PathBuf::from(manifest_dir).join("tests/fixtures/template.grib"); + let grib_path = fixtures_dir().join("template.grib"); let grib_data = Arc::new(fs::read(&grib_path).expect("failed to read template.grib")); // Archive initial data @@ -359,7 +409,9 @@ fn test_concurrent_read_write_mix() { let fdb = Arc::clone(&fdb); let grib_data = Arc::clone(&grib_data); thread::spawn(move || { - let request = Request::new().with("class", "rd").with("expver", "xxxx"); + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", "rd"); + request.set("expver", "xxxx"); for i in 0..iterations { if thread_id % 2 == 0 { diff --git a/rust/deny.toml b/rust/deny.toml new file mode 100644 index 000000000..106e7c404 --- /dev/null +++ b/rust/deny.toml @@ -0,0 +1,10 @@ +# Bans direct dependencies on *-sys crates outside the safe wrapper layer: +# examples, tools and applications must use the safe crates. `wrappers` lists +# the only crates allowed to depend on the banned crate. +[bans] +deny = [ + { name = "fdb-sys", wrappers = ["fdb"] }, + { name = "eckit-sys", wrappers = ["eckit", "fdb", "fdb-sys", "metkit-sys", "eccodes-sys"] }, + { name = "metkit-sys", wrappers = ["metkit", "fdb-sys"] }, + { name = "eccodes-sys", wrappers = ["fdb-sys", "metkit-sys"] }, +] diff --git a/rust/tools/fdb-hammer/Cargo.toml b/rust/tools/fdb-hammer/Cargo.toml index c589ec696..9b046e14d 100644 --- a/rust/tools/fdb-hammer/Cargo.toml +++ b/rust/tools/fdb-hammer/Cargo.toml @@ -11,16 +11,20 @@ description = "Benchmark and stress test tool for FDB" name = "fdb-hammer" path = "src/main.rs" +[build-dependencies] +bindman-utils.workspace = true + [features] default = ["vendored"] -vendored = ["fdb/vendored", "eccodes/vendored"] -system = ["fdb/system", "eccodes/system"] +vendored = ["fdb/vendored"] +system = ["fdb/system"] [dependencies] md-5 = "0.10" clap = { version = "4", features = ["derive"] } +eckit.workspace = true +metkit.workspace = true fdb = { path = "../../crates/fdb", default-features = false } -eccodes = { git = "ssh://git@github.com/ecmwf/rust-wrappers-playground.git", default-features = false } hostname = "0.4" rand = "0.9" nix = { version = "0.29", features = ["fs", "signal", "user"] } diff --git a/rust/tools/fdb-hammer/README.md b/rust/tools/fdb-hammer/README.md index 81257f0d2..84ba39ae3 100644 --- a/rust/tools/fdb-hammer/README.md +++ b/rust/tools/fdb-hammer/README.md @@ -24,9 +24,7 @@ cargo build -p fdb-hammer --release --no-default-features --features system ## Running Binaries work out of the box on both macOS and Linux — no -`LD_LIBRARY_PATH` / `DYLD_LIBRARY_PATH` setup needed. The build script -stamps a binary-relative RPATH so the dynamic linker finds the -vendored libraries automatically: +`LD_LIBRARY_PATH` / `DYLD_LIBRARY_PATH` setup needed: ```bash cd target/release diff --git a/rust/tools/fdb-hammer/build.rs b/rust/tools/fdb-hammer/build.rs new file mode 100644 index 000000000..937de09ba --- /dev/null +++ b/rust/tools/fdb-hammer/build.rs @@ -0,0 +1,3 @@ +fn main() { + bindman_utils::emit_rpaths(); +} diff --git a/rust/tools/fdb-hammer/src/main.rs b/rust/tools/fdb-hammer/src/main.rs index d825e7715..cecae557c 100644 --- a/rust/tools/fdb-hammer/src/main.rs +++ b/rust/tools/fdb-hammer/src/main.rs @@ -38,8 +38,8 @@ use clap::Parser; use crossbeam_channel::{Receiver, Sender, bounded}; use rand::Rng; -use eccodes::GribHandle; -use fdb::{Fdb, Key, ListOptions, Request}; +use fdb::{Fdb, Key, ListOptions, UserConfig}; +use metkit::CodesHandle; // ============================================================================= // Valid parameter IDs (from C++ fdb-hammer) @@ -175,8 +175,9 @@ struct Args { check_queue_size: usize, // FDB config - /// Disable use of subtocs - #[arg(long)] + /// Disable use of subtocs (requires `--config`, since user overrides + /// must be layered on top of a main config). + #[arg(long, requires = "config")] disable_subtocs: bool, // ITT mode options @@ -508,17 +509,17 @@ impl AsyncVerifier { while let Ok(job) = rx.recv() { // Parse GRIB to get data section offsets for verification - let handle = GribHandle::from_bytes(&job.data) + let handle = CodesHandle::from_message(&job.data) .map_err(|e| format!("Failed to parse GRIB: {e}"))?; #[allow(clippy::cast_sign_loss)] let offset_before = handle - .get_long("offsetBeforeData") + .get::("offsetBeforeData") .map_err(|e| format!("Failed to get offsetBeforeData: {e}"))? as usize; #[allow(clippy::cast_sign_loss)] let offset_after = handle - .get_long("offsetAfterData") + .get::("offsetAfterData") .map_err(|e| format!("Failed to get offsetAfterData: {e}"))? as usize; @@ -762,7 +763,7 @@ impl HammerConfig { // Build request string // ============================================================================= -fn build_request(config: &HammerConfig, step: u32, member: u32) -> Request { +fn build_request(config: &HammerConfig, step: u32, member: u32) -> metkit::MarsRequest { let levels_str = config .levels .iter() @@ -776,18 +777,19 @@ fn build_request(config: &HammerConfig, step: u32, member: u32) -> Request { .collect::>() .join("/"); - Request::new() - .with("class", &config.class) - .with("expver", &config.expver) - .with("stream", &config.stream) - .with("date", &config.date) - .with("time", &config.time) - .with("type", &config.type_) - .with("levtype", &config.levtype) - .with("step", &step.to_string()) - .with("levelist", &levels_str) - .with("param", ¶ms_str) - .with("number", &member.to_string()) + let mut request = metkit::MarsRequest::new("retrieve"); + request.set("class", config.class.as_str()); + request.set("expver", config.expver.as_str()); + request.set("stream", &config.stream); + request.set("date", &config.date); + request.set("time", &config.time); + request.set("type", &config.type_); + request.set("levtype", &config.levtype); + request.set("step", step.to_string()); + request.set("levelist", &levels_str); + request.set("param", ¶ms_str); + request.set("number", member.to_string()); + request } // ============================================================================= @@ -800,7 +802,7 @@ fn run_write(fdb: &Fdb, config: &HammerConfig) -> Result Result = (0..size).map(|_| rng.random::() * 100.0).collect(); - handle.set_double_array("values", &random_values)?; + handle.set("values", random_values.as_slice())?; } // Get data section offsets for verification embedding (like C++ fdb-hammer) #[allow(clippy::cast_sign_loss)] - let offset_before_data = handle.get_long("offsetBeforeData")? as usize; + let offset_before_data = handle.get::("offsetBeforeData")? as usize; #[allow(clippy::cast_sign_loss)] - let offset_after_data = handle.get_long("offsetAfterData")? as usize; + let offset_after_data = handle.get::("offsetAfterData")? as usize; // Get the GRIB message and embed verification data in data section - let mut grib_data = handle.message_copy()?; + let mut grib_data = handle.message_data()?.to_vec(); // Build FDB key for this field let key = Key::new() - .with("class", &config.class) - .with("expver", &config.expver) + .with("class", config.class.as_str()) + .with("expver", config.expver.as_str()) .with("stream", &config.stream) .with("date", &config.date) .with("time", &config.time) @@ -905,7 +907,7 @@ fn run_write_itt( let mut rng = rand::rng(); // Create template GribHandle from bytes - let template_handle = GribHandle::from_bytes(&config.template_data)?; + let template_handle = CodesHandle::from_message(&config.template_data)?; println!( "Writing {} fields (ITT mode, step_window={}s)", @@ -948,34 +950,34 @@ fn run_write_itt( let mut handle = template_handle.try_clone()?; // Set GRIB keys for this field (matching C++ fdb-hammer) - handle.set_string("expver", &config.expver)?; - handle.set_string("class", &config.class)?; - handle.set_long("step", i64::from(step))?; - handle.set_long("level", i64::from(level))?; - handle.set_long("paramId", i64::from(param))?; - handle.set_long("number", i64::from(member))?; + handle.set("expver", config.expver.as_str())?; + handle.set("class", config.class.as_str())?; + handle.set("step", i64::from(step))?; + handle.set("level", i64::from(level))?; + handle.set("paramId", i64::from(param))?; + handle.set("number", i64::from(member))?; // Randomize values if requested if config.randomise_data { - let size = handle.get_size("values")?; + let size = handle.value_count("values")?; let random_values: Vec = (0..size).map(|_| rng.random::() * 100.0).collect(); - handle.set_double_array("values", &random_values)?; + handle.set("values", random_values.as_slice())?; } // Get data section offsets for verification embedding (like C++ fdb-hammer) #[allow(clippy::cast_sign_loss)] - let offset_before_data = handle.get_long("offsetBeforeData")? as usize; + let offset_before_data = handle.get::("offsetBeforeData")? as usize; #[allow(clippy::cast_sign_loss)] - let offset_after_data = handle.get_long("offsetAfterData")? as usize; + let offset_after_data = handle.get::("offsetAfterData")? as usize; // Get the GRIB message and embed verification data in data section - let mut grib_data = handle.message_copy()?; + let mut grib_data = handle.message_data()?.to_vec(); // Build FDB key for this field let key = Key::new() - .with("class", &config.class) - .with("expver", &config.expver) + .with("class", config.class.as_str()) + .with("expver", config.expver.as_str()) .with("stream", &config.stream) .with("date", &config.date) .with("time", &config.time) @@ -1091,11 +1093,13 @@ fn run_read(fdb: &Fdb, config: &HammerConfig) -> Result Result("offsetBeforeData"), + handle.get::("offsetAfterData"), ) { if let Err(e) = verifier.verify_from_message( &key, @@ -1233,8 +1237,10 @@ fn run_read_itt( }, )?; stats.record_io_start(); - let mut reader = fdb.read_from_list(list_iter, false)?; - let data = reader.read_all()?; + let reader = fdb.read_from_list(list_iter, false)?; + let (mut reader, _len) = reader.open_for_read()?; + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut reader, &mut data)?; stats.record_io_end(); stats.bytes_processed += data.len() as u64; @@ -1291,8 +1297,10 @@ fn run_read_uri_file( ); stats.record_io_start(); - let mut reader = fdb.read_uris(&uris, false)?; - let data = reader.read_all()?; + let reader = fdb.read_uris(&uris, false)?; + let (mut reader, _len) = reader.open_for_read()?; + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut reader, &mut data)?; stats.record_io_end(); stats.fields_processed = uris.len() as u64; @@ -1367,6 +1375,8 @@ fn run_list(fdb: &Fdb, config: &HammerConfig) -> Result Result<(), Box> { + eckit::init(); + let args = Args::parse(); println!("FDB Hammer (Rust)"); @@ -1381,17 +1391,20 @@ fn main() -> Result<(), Box> { } // Create FDB handle with optional subtoc configuration + let user_config = if args.disable_subtocs { + Some(UserConfig { + use_sub_toc: false, + ..Default::default() + }) + } else { + None + }; + let fdb = if let Some(config_path) = &args.config { - let mut config_str = fs::read_to_string(config_path)?; - if args.disable_subtocs { - config_str.push_str("\nuseSubToc: false\n"); - } - Fdb::open(Some(config_str.as_str()), None)? - } else if args.disable_subtocs { - // Create config with subtoc disabled - Fdb::open(Some("useSubToc: false\n"), None)? + let cfg = eckit::Config::from_path(config_path)?; + Fdb::open(Some(&cfg), user_config)? } else { - Fdb::open_default()? + Fdb::open(None, user_config)? }; println!("FDB handle created: {}", fdb.name()); diff --git a/rust/tools/fdb-hammer/test_config/template.dat b/rust/tools/fdb-hammer/test_config/template.dat new file mode 100644 index 000000000..e11a5eff2 Binary files /dev/null and b/rust/tools/fdb-hammer/test_config/template.dat differ