From f07f07408c45dc8faf293aad08afbf3692bebbd5 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Fri, 9 Feb 2024 13:30:22 +0100 Subject: [PATCH 001/109] Picked/merged all Store-related modifications from feature/daos_handle. --- src/fdb5/database/FieldLocation.cc | 2 +- src/fdb5/database/Store.h | 4 + src/fdb5/toc/FieldRef.cc | 31 +++++-- src/fdb5/toc/TocStore.cc | 65 ++++++++++++++- src/fdb5/toc/TocStore.h | 4 + src/fdb5/toc/TocWipeVisitor.cc | 128 +++++++++++++++++++++++++---- src/fdb5/toc/TocWipeVisitor.h | 3 +- 7 files changed, 209 insertions(+), 28 deletions(-) diff --git a/src/fdb5/database/FieldLocation.cc b/src/fdb5/database/FieldLocation.cc index c626abd15..bc42aac10 100644 --- a/src/fdb5/database/FieldLocation.cc +++ b/src/fdb5/database/FieldLocation.cc @@ -107,7 +107,7 @@ FieldLocationBuilderBase::~FieldLocationBuilderBase() { //---------------------------------------------------------------------------------------------------------------------- -FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri) { +FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri.scheme() + ":" + uri.name()) { try { offset_ = eckit::Offset(std::stoll(uri.fragment())); } catch (std::invalid_argument& e) { diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h index 268359b39..5410eafc5 100644 --- a/src/fdb5/database/Store.h +++ b/src/fdb5/database/Store.h @@ -57,6 +57,10 @@ class Store { virtual void remove(const Key& key) const { NOTIMP; } virtual eckit::URI uri() const = 0; + virtual bool uriBelongs(const eckit::URI&) const = 0; + virtual bool uriExists(const eckit::URI& uri) const = 0; + virtual std::vector storeUnitURIs() const = 0; + virtual std::set asStoreUnitURIs(const std::vector&) const = 0; protected: // members const Schema& schema_; //<< schema is owned by catalogue which always outlives the store diff --git a/src/fdb5/toc/FieldRef.cc b/src/fdb5/toc/FieldRef.cc index ec5e0900c..3538f9210 100644 --- a/src/fdb5/toc/FieldRef.cc +++ b/src/fdb5/toc/FieldRef.cc @@ -14,11 +14,14 @@ #include "eckit/filesystem/URI.h" #include "eckit/serialisation/Stream.h" +#include "fdb5/fdb5_config.h" #include "fdb5/database/Field.h" #include "fdb5/database/UriStore.h" #include "fdb5/toc/TocFieldLocation.h" - +// #ifdef fdb5_HAVE_DAOSFDB +// #include "fdb5/daos/DaosFieldLocation.h" +// #endif namespace fdb5 { @@ -33,14 +36,30 @@ FieldRefLocation::FieldRefLocation() { FieldRefLocation::FieldRefLocation(UriStore &store, const Field& field) { const FieldLocation& loc = field.location(); + +// #ifdef fdb5_HAVE_DAOSFDB +// const TocFieldLocation* tocfloc = dynamic_cast(&loc); +// const DaosFieldLocation* daosfloc = dynamic_cast(&loc); +// if(!tocfloc && !daosfloc) { +// throw eckit::NotImplemented( +// "Field location is not of TocFieldLocation or DaosFieldLocation type " +// "-- indexing other locations is not supported", +// Here()); +// } +// #else const TocFieldLocation* tocfloc = dynamic_cast(&loc); if(!tocfloc) { - throw eckit::NotImplemented("Field location is not of TocFieldLocation type -- indexing other locations is not supported", Here()); + throw eckit::NotImplemented( + "Field location is not of TocFieldLocation type " + "-- indexing other locations is not supported", + Here()); } +// #endif + + uriId_ = store.insert(loc.uri()); + length_ = loc.length(); + offset_ = loc.offset(); - uriId_ = store.insert(tocfloc->uri()); - length_ = tocfloc->length(); - offset_ = tocfloc->offset(); } void FieldRefLocation::print(std::ostream &s) const { @@ -79,4 +98,4 @@ void FieldRef::print(std::ostream &s) const { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc index d4ef8abc9..079a38440 100644 --- a/src/fdb5/toc/TocStore.cc +++ b/src/fdb5/toc/TocStore.cc @@ -38,14 +38,71 @@ TocStore::TocStore(const Schema& schema, const Key& key, const Config& config) : Store(schema), TocCommon(StoreRootManager(config).directory(key).directory_) {} TocStore::TocStore(const Schema& schema, const eckit::URI& uri, const Config& config) : - Store(schema), TocCommon(uri.path().dirName()) {} + Store(schema), TocCommon(uri.path().dirName()) {} eckit::URI TocStore::uri() const { + return URI("file", directory_); + +} + +bool TocStore::uriBelongs(const eckit::URI& uri) const { + + // TODO: assert uri represents a (not necessarily existing) data file + return ((uri.scheme() == type()) && (uri.path().dirName().sameAs(directory_))); + +} + +bool TocStore::uriExists(const eckit::URI& uri) const { + + ASSERT(uri.scheme() == type()); + eckit::PathName p(uri.path()); + // ensure provided URI is either DB URI or Store file URI + if (!p.sameAs(directory_)) { + ASSERT(p.dirName().sameAs(directory_)); + ASSERT(p.extension() == ".data"); + } + + return p.exists(); + +} + +std::vector TocStore::storeUnitURIs() const { + + std::vector files; + std::vector dirs; + (directory_).children(files, dirs); + + std::vector res; + for (const auto& f : files) { + if (f.extension() == ".data") { + res.push_back(eckit::URI{type(), f}); + } + } + + return res; + +} + +std::set TocStore::asStoreUnitURIs(const std::vector& uris) const { + + std::set res; + + for (auto& uri : uris) { + + ASSERT(uri.path().extension() == ".data"); + res.insert(uri); + + } + + return res; + } bool TocStore::exists() const { + return directory_.exists(); + } eckit::DataHandle* TocStore::retrieve(Field& field) const { @@ -234,7 +291,7 @@ void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& de eckit::PathName destPath = dest.path(); for (const eckit::PathName& root: StoreRootManager(config).canMoveToRoots(key)) { if (root.sameAs(destPath)) { - eckit::PathName src_db = directory_ / key.valuesToString(); + eckit::PathName src_db = directory_; eckit::PathName dest_db = destPath / key.valuesToString(); dest_db.mkdir(); @@ -260,7 +317,7 @@ void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& de void TocStore::remove(const Key& key) const { - eckit::PathName src_db = directory_ / key.valuesToString(); + eckit::PathName src_db = directory_; DIR* dirp = ::opendir(src_db.asString().c_str()); struct dirent* dp; @@ -282,4 +339,4 @@ static StoreBuilder builder("file"); //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h index 10500748b..72a16fffc 100644 --- a/src/fdb5/toc/TocStore.h +++ b/src/fdb5/toc/TocStore.h @@ -39,6 +39,10 @@ class TocStore : public Store, public TocCommon { ~TocStore() override {} eckit::URI uri() const override; + bool uriBelongs(const eckit::URI&) const override; + bool uriExists(const eckit::URI&) const override; + std::vector storeUnitURIs() const override; + std::set asStoreUnitURIs(const std::vector&) const override; bool open() override { return true; } void flush() override; diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index eeba7204d..5492c20d8 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -24,7 +24,6 @@ #include #include - using namespace eckit; namespace fdb5 { @@ -163,11 +162,18 @@ bool TocWipeVisitor::visitIndex(const Index& index) { // Enumerate data files. std::vector indexDataPaths(index.dataPaths()); - for (const eckit::URI& uri : indexDataPaths) { - if (include && uri.path().dirName().sameAs(basePath)) { - dataPaths_.insert(uri.path()); + for (const eckit::URI& uri : store_.asStoreUnitURIs(indexDataPaths)) { + if (include) { + if (!store_.uriBelongs(uri)) { + Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." << std::endl; + Log::error() << "Configured Store URI: " << store_.uri().asString() << std::endl; + Log::error() << "Pointed Store unit URI: " << uri.asString() << std::endl; + Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." << std::endl; + NOTIMP; + } + dataPaths_.insert(eckit::PathName(uri.path())); } else { - safePaths_.insert(uri.path()); + safePaths_.insert(eckit::PathName(uri.path())); } } @@ -192,7 +198,7 @@ void TocWipeVisitor::addMaskedPaths() { } } for (const auto& uri : data) { - if (uri.path().dirName().sameAs(catalogue_.basePath())) dataPaths_.insert(uri.path()); + if (store_.uriBelongs(uri)) dataPaths_.insert(eckit::PathName(uri.path())); } } @@ -232,15 +238,30 @@ void TocWipeVisitor::calculateResidualPaths() { // Remove paths to non-existant files. This is reasonable as we may be recovering from a // previous failed, partial wipe. As such, referenced files may not exist any more. - for (std::set* fileset : {&subtocPaths_, &lockfilePaths_, &indexPaths_, &dataPaths_}) { + for (std::set* fileset : {&subtocPaths_, &lockfilePaths_, &indexPaths_}) { for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { + if (it->exists()) { ++it; } else { fileset->erase(it++); } + } } + + for (std::set* fileset : {&dataPaths_}) { + for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { + + if (store_.uriExists(eckit::URI(store_.type(), *it))) { + ++it; + } else { + fileset->erase(it++); + } + + } + } + if (tocPath_.asString().size() && !tocPath_.exists()) tocPath_ = ""; if (schemaPath_.asString().size() && !schemaPath_.exists()) @@ -252,7 +273,8 @@ void TocWipeVisitor::calculateResidualPaths() { deletePaths.insert(subtocPaths_.begin(), subtocPaths_.end()); deletePaths.insert(lockfilePaths_.begin(), lockfilePaths_.end()); deletePaths.insert(indexPaths_.begin(), indexPaths_.end()); - deletePaths.insert(dataPaths_.begin(), dataPaths_.end()); + if (store_.type() == "file") + deletePaths.insert(dataPaths_.begin(), dataPaths_.end()); if (tocPath_.asString().size()) deletePaths.insert(tocPath_); if (schemaPath_.asString().size()) deletePaths.insert(schemaPath_); @@ -273,10 +295,10 @@ void TocWipeVisitor::calculateResidualPaths() { std::inserter(paths, paths.begin())); if (!paths.empty()) { - Log::error() << "Paths not in existing paths set:" << std::endl; - for (const auto& p : paths) { - Log::error() << " - " << p << std::endl; - } + Log::error() << "Paths not in existing paths set:" << std::endl; + for (const auto& p : paths) { + Log::error() << " - " << p << std::endl; + } throw SeriousBug("Path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); } @@ -284,6 +306,44 @@ void TocWipeVisitor::calculateResidualPaths() { deletePaths.begin(), deletePaths.end(), std::inserter(residualPaths_, residualPaths_.begin())); } + + // if the store uses a backend other than POSIX (file), repeat the algorithm specialized + // for its store units + + if (store_.type() == "file") return; + + std::vector allStoreUnitURIs(store_.storeUnitURIs()); + std::vector allDataPathsVector; + for (const auto& u : allStoreUnitURIs) { + allDataPathsVector.push_back(eckit::PathName(u.path())); + } + + std::set allDataPaths(allDataPathsVector.begin(), allDataPathsVector.end()); + + ASSERT(residualDataPaths_.empty()); + + if (!(dataPaths_ == allDataPaths)) { + + // First we check if there are paths marked to delete that don't exist. This is an error + + std::set paths; + std::set_difference(dataPaths_.begin(), dataPaths_.end(), + allDataPaths.begin(), allDataPaths.end(), + std::inserter(paths, paths.begin())); + + if (!paths.empty()) { + Log::error() << "Store unit paths not in existing paths set:" << std::endl; + for (const auto& p : paths) { + Log::error() << " - " << p << std::endl; + } + throw SeriousBug("Store unit path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); + } + + std::set_difference(allDataPaths.begin(), allDataPaths.end(), + dataPaths_.begin(), dataPaths_.end(), + std::inserter(residualDataPaths_, residualDataPaths_.begin())); + } + } bool TocWipeVisitor::anythingToWipe() const { @@ -292,7 +352,7 @@ bool TocWipeVisitor::anythingToWipe() const { tocPath_.asString().size() || schemaPath_.asString().size()); } -void TocWipeVisitor::report() { +void TocWipeVisitor::report(bool wipeAll) { ASSERT(anythingToWipe()); @@ -329,6 +389,16 @@ void TocWipeVisitor::report() { } out_ << std::endl; + if (store_.type() != "file") { + out_ << "Store URI to delete:" << std::endl; + if (wipeAll) { + out_ << " " << store_.uri() << std::endl; + } else { + out_ << " - NONE -" << std::endl; + } + out_ << std::endl; + } + out_ << "Protected files (explicitly untouched):" << std::endl; if (safePaths_.empty()) out_ << " - NONE - " << std::endl; for (const auto& f : safePaths_) { @@ -387,6 +457,15 @@ void TocWipeVisitor::wipe(bool wipeAll) { // Now we want to do the actual deletion // n.b. We delete carefully in a order such that we can always access the DB by what is left + + /// @todo: are all these exist checks necessary? + + for (const PathName& path : residualDataPaths_) { + eckit::URI uri(store_.type(), path); + if (store_.uriExists(uri)) { + store_.remove(uri, logAlways, logVerbose, doit_); + } + } for (const PathName& path : residualPaths_) { if (path.exists()) { catalogue_.remove(path, logAlways, logVerbose, doit_); @@ -394,9 +473,17 @@ void TocWipeVisitor::wipe(bool wipeAll) { } for (const PathName& path : dataPaths_) { - store_.remove(eckit::URI(store_.type(), path), logAlways, logVerbose, doit_); + eckit::URI uri(store_.type(), path); + if (store_.uriExists(uri)) { + store_.remove(uri, logAlways, logVerbose, doit_); + } } + if (wipeAll && store_.type() != "file") + /// @todo: if the store is holding catalogue information (e.g. daos KVs) it + /// should not be removed + store_.remove(store_.uri(), logAlways, logVerbose, doit_); + for (const std::set& pathset : {indexPaths_, std::set{schemaPath_}, subtocPaths_, std::set{tocPath_}, lockfilePaths_, @@ -435,7 +522,7 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { if (anythingToWipe()) { if (wipeAll) calculateResidualPaths(); - if (!porcelain_) report(); + if (!porcelain_) report(wipeAll); // This is here as it needs to run whatever combination of doit/porcelain/... if (wipeAll && !residualPaths_.empty()) { @@ -444,6 +531,15 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { for (const auto& p : residualPaths_) out_ << " " << p << std::endl; out_ << std::endl; + } + if (wipeAll && !residualDataPaths_.empty()) { + + out_ << "Unexpected store units present in store: " << std::endl; + for (const auto& p : residualDataPaths_) out_ << " " << store_.type() << "://" << p << std::endl; + out_ << std::endl; + + } + if (wipeAll && (!residualPaths_.empty() || !residualDataPaths_.empty())) { if (!unsafeWipeAll_) { out_ << "Full wipe will not proceed without --unsafe-wipe-all" << std::endl; if (doit_) @@ -458,4 +554,4 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/toc/TocWipeVisitor.h b/src/fdb5/toc/TocWipeVisitor.h index be1352396..6b1f88c18 100644 --- a/src/fdb5/toc/TocWipeVisitor.h +++ b/src/fdb5/toc/TocWipeVisitor.h @@ -48,7 +48,7 @@ class TocWipeVisitor : public WipeVisitor { bool anythingToWipe() const; - void report(); + void report(bool wipeAll); void wipe(bool wipeAll); private: // members @@ -71,6 +71,7 @@ class TocWipeVisitor : public WipeVisitor { std::set safePaths_; std::set residualPaths_; + std::set residualDataPaths_; std::vector indexesToMask_; }; From 6e4de59c6ae0ec9f2bb6e472baff74016d3ac63c Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 11 Feb 2024 00:17:48 +0100 Subject: [PATCH 002/109] Draft S3Store. Includes code for design alternatives including object per field vs. object per index, and bucket per DB vs. single bucket. --- CMakeLists.txt | 6 + src/fdb5/CMakeLists.txt | 15 +- src/fdb5/fdb5_config.h.in | 1 + src/fdb5/s3/S3Common.cc | 96 ++++++++++ src/fdb5/s3/S3Common.h | 41 ++++ src/fdb5/s3/S3FieldLocation.cc | 108 +++++++++++ src/fdb5/s3/S3FieldLocation.h | 59 ++++++ src/fdb5/s3/S3Store.cc | 333 +++++++++++++++++++++++++++++++++ src/fdb5/s3/S3Store.h | 88 +++++++++ src/fdb5/toc/TocWipeVisitor.cc | 1 + 10 files changed, 747 insertions(+), 1 deletion(-) create mode 100644 src/fdb5/s3/S3Common.cc create mode 100644 src/fdb5/s3/S3Common.h create mode 100644 src/fdb5/s3/S3FieldLocation.cc create mode 100644 src/fdb5/s3/S3FieldLocation.h create mode 100644 src/fdb5/s3/S3Store.cc create mode 100644 src/fdb5/s3/S3Store.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 03d8592f5..4bdec2a55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,12 @@ ecbuild_add_option( FEATURE TOCFDB # option defined in fdb5_config.h DEFAULT ON DESCRIPTION "Filesystem TOC support for FDB" ) +### FDB S3 Store backend +ecbuild_add_option( FEATURE S3FDB + CONDITION eckit_S3_FOUND + DEFAULT ON + DESCRIPTION "S3 support for FDB Store" ) + ### support for Lustre API control of file stripping find_package( LUSTREAPI QUIET ) diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8ccf68784..8907714d3 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -364,6 +364,17 @@ if( HAVE_RADOSFDB ) ) endif() +if( HAVE_S3FDB ) + list( APPEND fdb5_srcs + s3/S3FieldLocation.h + s3/S3FieldLocation.cc + s3/S3Store.h + s3/S3Store.cc + s3/S3Common.h + s3/S3Common.cc + ) +endif() + ecbuild_add_library( TARGET fdb5 @@ -390,11 +401,13 @@ ecbuild_add_library( PRIVATE_INCLUDES "${PMEM_INCLUDE_DIRS}" "${LUSTREAPI_INCLUDE_DIRS}" + "${S3_INCLUDE_DIRS}" PRIVATE_LIBS ${grib_handling_pkg} ${PMEM_LIBRARIES} ${LUSTREAPI_LIBRARIES} + ${S3_LIBRARIES} ) if(HAVE_FDB_BUILD_TOOLS) @@ -441,7 +454,7 @@ foreach( _tool ${fdb5_tools} ) ecbuild_add_executable( TARGET ${_tool} CONDITION HAVE_FDB_BUILD_TOOLS SOURCES tools/${_tool}.cc - INCLUDES ${ECCODES_INCLUDE_DIRS} # Please don't remove me, I am needed + INCLUDES ${ECCODES_INCLUDE_DIRS} ${S3_INCLUDE_DIRS} # Please don't remove me, I am needed LIBS fdb5 ) endforeach() diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index 6fec3c5d5..e9100c21c 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -11,6 +11,7 @@ #cmakedefine fdb5_HAVE_PMEMFDB #cmakedefine fdb5_HAVE_RADOSFDB #cmakedefine fdb5_HAVE_TOCFDB +#cmakedefine fdb5_HAVE_S3FDB #cmakedefine01 fdb5_HAVE_GRIB #endif // fdb5_fdb5_config_h diff --git a/src/fdb5/s3/S3Common.cc b/src/fdb5/s3/S3Common.cc new file mode 100644 index 000000000..1b0805ed3 --- /dev/null +++ b/src/fdb5/s3/S3Common.cc @@ -0,0 +1,96 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +// #include + +#include "eckit/s3/S3Name.h" + +#include "fdb5/s3/S3Common.h" + +// #include "eckit/exception/Exceptions.h" +#include "eckit/config/Resource.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Common::S3Common(const fdb5::Config& config, const std::string& component, const fdb5::Key& key) { + + /// @note: code for bucket per DB + + db_bucket_ = key.valuesToString(); + + + /// @note: code for single bucket for all DBs + + // std::vector valid{"catalogue", "store"}; + // ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); + + // bucket_ = "default"; + + // eckit::LocalConfiguration c{}; + + // if (config.has("s3")) c = config.getSubConfiguration("s3"); + // if (c.has(component)) bucket_ = c.getSubConfiguration(component).getString("bucket", bucket_); + + // std::string first_cap{component}; + // first_cap[0] = toupper(component[0]); + + // std::string all_caps{component}; + // for (auto & c: all_caps) c = toupper(c); + + // bucket_ = eckit::Resource("fdbS3" + first_cap + "Bucket;$FDB_S3_" + all_caps + "_BUCKET", bucket_); + + // db_prefix_ = key.valuesToString(); + + // if (c.has("client")) + // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); + +} + +S3Common::S3Common(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { + + /// @note: validity of input URI is not checked here because this constructor is only triggered + /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand + + + /// @note: code for bucket per DB + + db_bucket_ = eckit::S3Name{uri}.bucketName(); + + + + /// @note: code for single bucket for all DBs + + // eckit::S3Name n{uri}; + + // bucket_ = n.bucketName(); + + // eckit::Tokenizer parse("_"); + // std::vector bits; + // parse(n.keyName(), bits); + + // ASSERT(bits.size() == 2); + + // db_prefix_ = bits[0]; + + + // // eckit::LocalConfiguration c{}; + + // // if (config.has("s3")) c = config.getSubConfiguration("s3"); + + // // if (c.has("client")) + // // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Common.h b/src/fdb5/s3/S3Common.h new file mode 100644 index 000000000..717b3ea11 --- /dev/null +++ b/src/fdb5/s3/S3Common.h @@ -0,0 +1,41 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file S3Common.h +/// @author Nicolau Manubens +/// @date Feb 2024 + +#pragma once + +#include "eckit/filesystem/URI.h" + +#include "fdb5/database/Key.h" +#include "fdb5/config/Config.h" + +namespace fdb5 { + +class S3Common { + +public: // methods + + S3Common(const fdb5::Config&, const std::string& component, const fdb5::Key&); + S3Common(const fdb5::Config&, const std::string& component, const eckit::URI&); + +protected: // members + + std::string db_bucket_; + + /// @note: code for single bucket for all DBs + // std::string bucket_; + // std::string db_prefix_; + +}; + +} \ No newline at end of file diff --git a/src/fdb5/s3/S3FieldLocation.cc b/src/fdb5/s3/S3FieldLocation.cc new file mode 100644 index 000000000..dd0c274e8 --- /dev/null +++ b/src/fdb5/s3/S3FieldLocation.cc @@ -0,0 +1,108 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +// #include "eckit/filesystem/URIManager.h" +#include "fdb5/daos/S3FieldLocation.h" +// #include "fdb5/LibFdb5.h" + +namespace fdb5 { + +::eckit::ClassSpec S3FieldLocation::classSpec_ = {&FieldLocation::classSpec(), "S3FieldLocation",}; +::eckit::Reanimator S3FieldLocation::reanimator_; + +//---------------------------------------------------------------------------------------------------------------------- + +S3FieldLocation::S3FieldLocation(const S3FieldLocation& rhs) : + FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} + +S3FieldLocation::S3FieldLocation(const eckit::URI &uri) : FieldLocation(uri) {} + +/// @todo: remove remapKey from signature and always pass empty Key to FieldLocation +S3FieldLocation::S3FieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) : + FieldLocation(uri, offset, length, remapKey) {} + +S3FieldLocation::S3FieldLocation(eckit::Stream& s) : + FieldLocation(s) {} + +std::shared_ptr S3FieldLocation::make_shared() const { + return std::make_shared(std::move(*this)); +} + +eckit::DataHandle* S3FieldLocation::dataHandle() const { + + return eckit::S3Name(uri_).dataHandle(offset(), length()); + +} + +void S3FieldLocation::print(std::ostream &out) const { + out << "S3FieldLocation[uri=" << uri_ << "]"; +} + +void S3FieldLocation::visit(FieldLocationVisitor& visitor) const { + visitor(*this); +} + +static FieldLocationBuilder builder("s3"); + +//---------------------------------------------------------------------------------------------------------------------- + +// class DaosURIManager : public eckit::URIManager { +// virtual bool query() override { return true; } +// virtual bool fragment() override { return true; } + +// virtual eckit::PathName path(const eckit::URI& f) const override { return f.name(); } + +// virtual bool exists(const eckit::URI& f) override { + +// return fdb5::DaosName(f).exists(); + +// } + +// virtual eckit::DataHandle* newWriteHandle(const eckit::URI& f) override { + +// if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; + +// return fdb5::DaosArrayName(f).dataHandle(); + +// } + +// virtual eckit::DataHandle* newReadHandle(const eckit::URI& f) override { + +// if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; + +// return fdb5::DaosArrayName(f).dataHandle(); + +// } + +// virtual eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { + +// if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; + +// return fdb5::DaosArrayName(f).dataHandle(); + +// } + +// virtual std::string asString(const eckit::URI& uri) const override { +// std::string q = uri.query(); +// if (!q.empty()) +// q = "?" + q; +// std::string f = uri.fragment(); +// if (!f.empty()) +// f = "#" + f; + +// return uri.scheme() + ":" + uri.name() + q + f; +// } +// public: +// DaosURIManager(const std::string& name) : eckit::URIManager(name) {} +// }; + +// static DaosURIManager daos_uri_manager("daos"); + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3FieldLocation.h b/src/fdb5/s3/S3FieldLocation.h new file mode 100644 index 000000000..c27604792 --- /dev/null +++ b/src/fdb5/s3/S3FieldLocation.h @@ -0,0 +1,59 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2024 + +#pragma once + +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" + +#include "fdb5/database/FieldLocation.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3FieldLocation : public FieldLocation { +public: + + S3FieldLocation(const S3FieldLocation& rhs); + S3FieldLocation(const eckit::URI &uri); + S3FieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); + S3FieldLocation(eckit::Stream&); + + eckit::DataHandle* dataHandle() const override; + + virtual std::shared_ptr make_shared() const override; + + virtual void visit(FieldLocationVisitor& visitor) const override; + +public: // For Streamable + + static const eckit::ClassSpec& classSpec() { return classSpec_;} + +protected: // For Streamable + + virtual const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + +private: // methods + + void print(std::ostream &out) const override; + +}; + + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Store.cc b/src/fdb5/s3/S3Store.cc new file mode 100644 index 000000000..8fb606a4b --- /dev/null +++ b/src/fdb5/s3/S3Store.cc @@ -0,0 +1,333 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/StaticMutex.h" +#include "eckit/log/TimeStamp.h" +#include "eckit/utils/MD5.h" + +// #include "eckit/config/Resource.h" +#include "eckit/s3/S3Name.h" + +#include "fdb5/s3/S3FieldLocation.h" +#include "fdb5/s3/S3Store.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +static StoreBuilder builder("s3"); + +S3Store::S3Store(const Schema& schema, const Key& key, const Config& config) : + Store(schema), S3Common(config, "store", key), config_(config) { + /// @note: code for single bucket + // Store(schema), S3Common(config, "store", key), config_(config) { + +} + +S3Store::S3Store(const Schema& schema, const eckit::URI& uri, const Config& config) : + Store(schema), S3Common(config, "store", uri), config_(config) { + /// @note: code for single bucket + // Store(schema), S3Common(config, "store", uri), config_(config) { + +} + +eckit::URI S3Store::uri() const { + + return eckit::S3Name(db_bucket_).URI(); + + +/// @note: code for single bucket for all DBs +// TODO +// // warning! here an incomplete uri is being returned. Where is this method +// // being called? Can that caller code accept incomplete uris? +// return eckit::S3Name(bucket_, db_prefix_).URI(); + +} + +bool S3Store::uriBelongs(const eckit::URI& uri) const { + + /// @todo: avoid building a S3Name as it makes uriBelongs expensive + return ( + (uri.scheme() == type()) && + (eckit::S3Name(uri).bucketName() == db_bucket_)); + + + /// @note: code for single bucket for all DBs + // return ( + // (uri.scheme() == type()) && + // (eckit::S3Name(uri).keyName().rfind(db_prefix_, 0) == 0)); + +} + +bool S3Store::uriExists(const eckit::URI& uri) const { + + /// @todo: revisit the name of this method + + ASSERT(uri.scheme() == type()); + eckit::S3Name n(uri); + ASSERT(n.bucketName() == db_bucket_); + return n.exists(); + + + /// @note: code for single bucket for all DBs + // ASSERT(uri.scheme() == type()); + // eckit::S3Name n(uri); + // ASSERT(n.bucketName() == bucket_); + // ASSERT(n.keyName().rfind(db_prefix_, 0) == 0); + // return n.exists(); + +} + +std::vector S3Store::storeUnitURIs() const { + + std::vector store_unit_uris; + + eckit::S3Name bucket{db_bucket_}; + + if (!bucket.exists()) return store_unit_uris; + + /// @note if an S3Catalogue is implemented, some filtering will need to + /// be done here to discriminate store keys from catalogue keys + for (const auto& key : bucket.listKeys()) { + + store_unit_uris.push_back(eckit::S3Name(db_bucket_, key).URI()); + + } + + return store_unit_uris; + + + /// @note: code for single bucket for all DBs + // std::vector store_unit_uris; + + // eckit::S3Name bucket{bucket_}; + + // if (!bucket.exists()) return store_unit_uris; + + // /// @note if an S3Catalogue is implemented, more filtering will need to + // /// be done here to discriminate store keys from catalogue keys + // for (const auto& key : bucket.listKeys(filter = "^" + db_prefix_ + "_.*")) { + + // store_unit_uris.push_back(eckit::S3Name(bucket_, key).URI()); + + // } + + // return store_unit_uris; + +} + +std::set S3Store::asStoreUnitURIs(const std::vector& uris) const { + + std::set res; + + /// @note: this is only uniquefying the input uris (coming from an index) + /// in case theres any duplicate. + for (auto& uri : uris) + res.insert(uri); + + return res; + +} + +bool S3Store::exists() const { + + return eckit::S3Name(db_bucket_).exists(); + +} + +/// @todo: never used in actual fdb-read? +eckit::DataHandle* S3Store::retrieve(Field& field) const { + + return field.dataHandle(); + +} + +std::unique_ptr S3Store::archive(const Key& key, const void * data, eckit::Length length) { + + /// @note: code for S3 object (key) per field: + + /// @note: generate unique key name + /// if single bucket, starting by dbkey_indexkey_ + /// if bucket per db, starting by indexkey_ + eckit::S3Name n = generateDataKey(key); + + std::unique_ptr h(n.dataHandle()) + + h->openForWrite(length); + eckit::AutoClose closer(*h); + + h->write(data, length); + + return std::unique_ptr(new S3FieldLocation(n.URI(), 0, length, fdb5::Key())); + + + /// @note: code for S3 object (key) per index store: + + // /// @note: get or generate unique key name + // /// if single bucket, starting by dbkey_indexkey_ + // /// if bucket per db, starting by indexkey_ + // eckit::S3Name n = getDataKey(key); + + // eckit::DataHandle &dh = getDataHandle(key, n); + + // eckit::Offset offset{dh.position()}; + + // h.write(data, length); + + // return std::unique_ptr(new S3FieldLocation(n.URI(), offset, length, fdb5::Key())); + +} + +void S3Store::flush() { + + /// @note: code for S3 object (key) per index store: + + // /// @note: clear cached data handles thus triggering consolidation of + // /// multipart objects, so that step data is made visible to readers. + // /// New S3 handles will be created on the next archive() call after + // /// flush(). + // closeDataHandles(); + +} + +void S3Store::close() { + + /// @note: code for S3 object (key) per index store: + + // closeDataHandles(); + +} + +void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { + + eckit::S3Name n{uri}; + + ASSERT(n.hasBucketName()); + ASSERT(n.bucketName() == db_bucket_); + + if (n.hasKeyName()) { + logVerbose << "destroy S3 key: "; + } else { + logVerbose << "destroy S3 bucket: "; + } + + logAlways << n.asString() << std::endl; + if (doit) n.destroy(); + + + // /// @note: code for single bucket for all DBs + // eckit::S3Name n{uri}; + + // ASSERT(n.hasBucketName()); + // ASSERT(n.bucketName() == bucket_); + // /// @note: if !n.hasKeyName, maybe this method should return without destroying anything. + // /// this way when TocWipeVisitor has wipeAll == true, the (only) bucket will not be destroyed + // ASSERT(n.hasKeyName()); + // ASSERT(n.keyName().rfind(db_prefix_, 0) == 0); + + // logVerbose << "destroy S3 key: "; + // logAlways << n.asString() << std::endl; + // if (doit) n.destroy(); + +} + +void S3Store::print(std::ostream& out) const { + + out << "S3Store(" << db_bucket_ << ")"; + + /// @note: code for single bucket for all DBs + // out << "S3Store(" << bucket_ << ")"; + +} + +/// @note: unique name generation copied from LocalPathName::unique. +static StaticMutex local_mutex; + +eckit::S3Name S3Store::generateDataKey(const Key& key) const { + + AutoLock lock(local_mutex); + + std::string hostname = eckit::Main::hostname(); + + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + + std::string name = os.str(); + + while (::access(name.c_str(), F_OK) == 0) { + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + name = os.str(); + } + + eckit::MD5 md5(name); + + return eckit::S3Name{db_bucket_, key.valuesToString() + "_" + md5.digest() + ".data"}; + + /// @note: code for single bucket for all DBs + // return eckit::S3Name{bucket_, db_prefix_ + "_" + key.valuesToString() + "_" + md5.digest() + ".data"}; + +} + +/// @note: code for S3 object (key) per index store: +// eckit::S3Name S3Store::getDataKey(const Key& key) const { + +// KeyStore::const_iterator j = dataKeys_.find(key); + +// if ( j != dataKeys_.end() ) +// return j->second; + +// eckit::S3Name dataKey = generateDataKey(key); + +// dataKeys_[ key ] = dataKey; + +// return dataKey; + +// } + +/// @note: code for S3 object (key) per index store: +// eckit::DataHandle& S3Store::getDataHandle(const Key& key, const eckit::S3Name& name) { + +// HandleStore::const_iterator j = handles_.find(key); +// if ( j != handles_.end() ) +// return j->second; + +// eckit::DataHandle *dh = name.dataHandle(multipart = true); + +// ASSERT(dh); + +// handles_[ key ] = dh; + +// dh->openForAppend(0); + +// return *dh; + +// } + +/// @note: code for S3 object (key) per index store: +// void S3Store::closeDataHandles() { + +// for ( HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j ) { +// eckit::DataHandle *dh = j->second; +// dh->close(); +// delete dh; +// } + +// handles_.clear(); + +// } + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Store.h b/src/fdb5/s3/S3Store.h new file mode 100644 index 000000000..34ffb92f7 --- /dev/null +++ b/src/fdb5/s3/S3Store.h @@ -0,0 +1,88 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2024 + +#pragma once + +#include "eckit/io/s3/S3Name.h" + +#include "fdb5/database/Store.h" +#include "fdb5/rules/Schema.h" + +#include "fdb5/s3/S3Common.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Store : public Store, public S3Common { + +public: // methods + + S3Store(const Schema& schema, const Key& key, const Config& config); + S3Store(const Schema& schema, const eckit::URI& uri, const Config& config); + + ~S3Store() override {} + + eckit::URI uri() const override; + bool uriBelongs(const eckit::URI&) const override; + bool uriExists(const eckit::URI&) const override; + std::vector storeUnitURIs() const override; + std::set asStoreUnitURIs(const std::vector&) const override; + + bool open() override { return true; } + void flush() override; + void close() override; + + void checkUID() const override { /* nothing to do */ } + +protected: // methods + + std::string type() const override { return "s3"; } + + bool exists() const override; + + eckit::DataHandle* retrieve(Field& field) const override; + std::unique_ptr archive(const Key& key, const void * data, eckit::Length length) override; + + void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; + + void print(std::ostream &out) const override; + + eckit::S3Name generateDataKey(const Key& key) const; + + /// @note: code for S3 object (key) per index store: + // eckit::S3Name getDataKey(const Key& key) const; + // eckit::DataHandle& getDataHandle(const Key& key, const eckit::S3Name& name); + // void closeDataHandles(); + + void print( std::ostream &out ) const override; + +private: // types + + /// @note: code for S3 object (key) per index store: + // typedef std::map HandleStore; + // typedef std::map KeyStore; + +private: // members + + const Config& config_; + + /// @note: code for S3 object (key) per index store: + // HandleStore handles_; + // mutable KeyStore dataKeys_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index 5492c20d8..a530cc660 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -479,6 +479,7 @@ void TocWipeVisitor::wipe(bool wipeAll) { } } + /// @todo: do not remove store uri if backend is S3 and uses a single bucket for all DBs if (wipeAll && store_.type() != "file") /// @todo: if the store is holding catalogue information (e.g. daos KVs) it /// should not be removed From e4ebfa452c27870f521c08b164df2110e1a7d196 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 11 Feb 2024 12:58:20 +0100 Subject: [PATCH 003/109] Tidying. --- src/fdb5/s3/S3Store.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/fdb5/s3/S3Store.cc b/src/fdb5/s3/S3Store.cc index 8fb606a4b..89ac0c720 100644 --- a/src/fdb5/s3/S3Store.cc +++ b/src/fdb5/s3/S3Store.cc @@ -27,15 +27,11 @@ static StoreBuilder builder("s3"); S3Store::S3Store(const Schema& schema, const Key& key, const Config& config) : Store(schema), S3Common(config, "store", key), config_(config) { - /// @note: code for single bucket - // Store(schema), S3Common(config, "store", key), config_(config) { } S3Store::S3Store(const Schema& schema, const eckit::URI& uri, const Config& config) : Store(schema), S3Common(config, "store", uri), config_(config) { - /// @note: code for single bucket - // Store(schema), S3Common(config, "store", uri), config_(config) { } From c3d4c3092b77f9270027779e7908895f589b00d5 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Mon, 12 Feb 2024 16:03:21 +0100 Subject: [PATCH 004/109] Specifying an endpoint in all S3Names. Adding a configurable prefix for buckets in the variant with one bucket per DB. Additional config. --- src/fdb5/s3/S3Common.cc | 46 +++++++++++++++++++++++++++-- src/fdb5/s3/S3Common.h | 9 ++++++ src/fdb5/s3/S3FieldLocation.cc | 54 ---------------------------------- src/fdb5/s3/S3Store.cc | 22 +++++++------- src/fdb5/toc/TocStore.cc | 9 ++++++ 5 files changed, 73 insertions(+), 67 deletions(-) diff --git a/src/fdb5/s3/S3Common.cc b/src/fdb5/s3/S3Common.cc index 1b0805ed3..c5d62c244 100644 --- a/src/fdb5/s3/S3Common.cc +++ b/src/fdb5/s3/S3Common.cc @@ -11,6 +11,8 @@ // #include #include "eckit/s3/S3Name.h" +#include "eckit/s3/S3Credential.h" +#include "eckit/s3/S3Session.h" #include "fdb5/s3/S3Common.h" @@ -23,9 +25,15 @@ namespace fdb5 { S3Common::S3Common(const fdb5::Config& config, const std::string& component, const fdb5::Key& key) { + parseConfig(config); + + + /// @note: code for bucket per DB - db_bucket_ = key.valuesToString(); + db_bucket_ = prefix_ + key.valuesToString(); + + /// @note: code for single bucket for all DBs @@ -53,13 +61,21 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con // if (c.has("client")) // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); + + + + /// @todo: check that the bucket name complies with name restrictions + } S3Common::S3Common(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { /// @note: validity of input URI is not checked here because this constructor is only triggered /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand - + + parseConfig(config); + + /// @note: code for bucket per DB @@ -91,6 +107,32 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con } +S3Common::parseConfig(const fdb5::Config& config) { + + eckit::LocalConfiguration cr{}, s3{}; + + if (config.has("s3")) { + s3 = config.getSubConfiguration("s3"); + if (s3.has("credential")) cr = s3.getSubConfiguration("credential"); + } + + const eckit::S3Credential cred{ + cr.getString("accessKeyID", "defaultKeyID"), + cr.getString("secretKey", "defaultSecretKey"), + cr.getString("host", "127.0.0.1") + }; + + eckit::S3Session::instance().addCredentials(cred); + + endpoint_ = s3.getString("endpoint", "127.0.0.1:9000"); + + + + /// @note: code for bucket per DB only + prefix_ = s3.getString("bucketPrefix", prefix_); + +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Common.h b/src/fdb5/s3/S3Common.h index 717b3ea11..67374380a 100644 --- a/src/fdb5/s3/S3Common.h +++ b/src/fdb5/s3/S3Common.h @@ -28,14 +28,23 @@ class S3Common { S3Common(const fdb5::Config&, const std::string& component, const fdb5::Key&); S3Common(const fdb5::Config&, const std::string& component, const eckit::URI&); +private: // methods + + void parseConfig(const fdb5::Config& config); + protected: // members + std::string endpoint_; std::string db_bucket_; /// @note: code for single bucket for all DBs // std::string bucket_; // std::string db_prefix_; +private: // members + + std::string prefix_; + }; } \ No newline at end of file diff --git a/src/fdb5/s3/S3FieldLocation.cc b/src/fdb5/s3/S3FieldLocation.cc index dd0c274e8..85d7a0a49 100644 --- a/src/fdb5/s3/S3FieldLocation.cc +++ b/src/fdb5/s3/S3FieldLocation.cc @@ -51,58 +51,4 @@ void S3FieldLocation::visit(FieldLocationVisitor& visitor) const { static FieldLocationBuilder builder("s3"); -//---------------------------------------------------------------------------------------------------------------------- - -// class DaosURIManager : public eckit::URIManager { -// virtual bool query() override { return true; } -// virtual bool fragment() override { return true; } - -// virtual eckit::PathName path(const eckit::URI& f) const override { return f.name(); } - -// virtual bool exists(const eckit::URI& f) override { - -// return fdb5::DaosName(f).exists(); - -// } - -// virtual eckit::DataHandle* newWriteHandle(const eckit::URI& f) override { - -// if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; - -// return fdb5::DaosArrayName(f).dataHandle(); - -// } - -// virtual eckit::DataHandle* newReadHandle(const eckit::URI& f) override { - -// if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; - -// return fdb5::DaosArrayName(f).dataHandle(); - -// } - -// virtual eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { - -// if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; - -// return fdb5::DaosArrayName(f).dataHandle(); - -// } - -// virtual std::string asString(const eckit::URI& uri) const override { -// std::string q = uri.query(); -// if (!q.empty()) -// q = "?" + q; -// std::string f = uri.fragment(); -// if (!f.empty()) -// f = "#" + f; - -// return uri.scheme() + ":" + uri.name() + q + f; -// } -// public: -// DaosURIManager(const std::string& name) : eckit::URIManager(name) {} -// }; - -// static DaosURIManager daos_uri_manager("daos"); - } // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Store.cc b/src/fdb5/s3/S3Store.cc index 89ac0c720..befd9ebf6 100644 --- a/src/fdb5/s3/S3Store.cc +++ b/src/fdb5/s3/S3Store.cc @@ -37,14 +37,14 @@ S3Store::S3Store(const Schema& schema, const eckit::URI& uri, const Config& conf eckit::URI S3Store::uri() const { - return eckit::S3Name(db_bucket_).URI(); + return eckit::S3Name(endpoint_, db_bucket_).URI(); /// @note: code for single bucket for all DBs // TODO // // warning! here an incomplete uri is being returned. Where is this method // // being called? Can that caller code accept incomplete uris? -// return eckit::S3Name(bucket_, db_prefix_).URI(); +// return eckit::S3Name(endpoint_, bucket_, db_prefix_).URI(); } @@ -86,7 +86,7 @@ std::vector S3Store::storeUnitURIs() const { std::vector store_unit_uris; - eckit::S3Name bucket{db_bucket_}; + eckit::S3Name bucket{endpoint_, db_bucket_}; if (!bucket.exists()) return store_unit_uris; @@ -94,7 +94,7 @@ std::vector S3Store::storeUnitURIs() const { /// be done here to discriminate store keys from catalogue keys for (const auto& key : bucket.listKeys()) { - store_unit_uris.push_back(eckit::S3Name(db_bucket_, key).URI()); + store_unit_uris.push_back(eckit::S3Name(endpoint_, db_bucket_, key).URI()); } @@ -104,7 +104,7 @@ std::vector S3Store::storeUnitURIs() const { /// @note: code for single bucket for all DBs // std::vector store_unit_uris; - // eckit::S3Name bucket{bucket_}; + // eckit::S3Name bucket{endpoint_, bucket_}; // if (!bucket.exists()) return store_unit_uris; @@ -112,7 +112,7 @@ std::vector S3Store::storeUnitURIs() const { // /// be done here to discriminate store keys from catalogue keys // for (const auto& key : bucket.listKeys(filter = "^" + db_prefix_ + "_.*")) { - // store_unit_uris.push_back(eckit::S3Name(bucket_, key).URI()); + // store_unit_uris.push_back(eckit::S3Name(endpoint_, bucket_, key).URI()); // } @@ -135,7 +135,7 @@ std::set S3Store::asStoreUnitURIs(const std::vector& uri bool S3Store::exists() const { - return eckit::S3Name(db_bucket_).exists(); + return eckit::S3Name(endpoint_, db_bucket_).exists(); } @@ -237,10 +237,10 @@ void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostrea void S3Store::print(std::ostream& out) const { - out << "S3Store(" << db_bucket_ << ")"; + out << "S3Store(" << endpoint_ << "/" << db_bucket_ << ")"; /// @note: code for single bucket for all DBs - // out << "S3Store(" << bucket_ << ")"; + // out << "S3Store(" << endpoint_ << "/" << bucket_ << ")"; } @@ -269,10 +269,10 @@ eckit::S3Name S3Store::generateDataKey(const Key& key) const { eckit::MD5 md5(name); - return eckit::S3Name{db_bucket_, key.valuesToString() + "_" + md5.digest() + ".data"}; + return eckit::S3Name{endpoint_, db_bucket_, key.valuesToString() + "_" + md5.digest() + ".data"}; /// @note: code for single bucket for all DBs - // return eckit::S3Name{bucket_, db_prefix_ + "_" + key.valuesToString() + "_" + md5.digest() + ".data"}; + // return eckit::S3Name{endpoint_, bucket_, db_prefix_ + "_" + key.valuesToString() + "_" + md5.digest() + ".data"}; } diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc index 079a38440..ef30a49da 100644 --- a/src/fdb5/toc/TocStore.cc +++ b/src/fdb5/toc/TocStore.cc @@ -239,6 +239,15 @@ eckit::PathName TocStore::generateDataPath(const Key &key) const { eckit::PathName dpath ( directory_ ); dpath /= key.valuesToString(); + /// @todo: in cases where a catalogue other than POSIX is used and a + /// POSIX store is used, the DB directory for the store is first + /// created within PathName::unique(). + /// DB directory creation should maybe be removed from there and + /// performed here, or as part of FDB/LustreFileHandle::openForAppend + /// if not exists. + /// If doing it in openForAppend, it should be ensured that the + /// existence of the database directory is not checked an excessive + /// amount of times. dpath = eckit::PathName::unique(dpath) + ".data"; return dpath; } From 2215a6c1e0651aabe1074cc4898f7197ae82fac7 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Mon, 12 Feb 2024 22:08:16 +0100 Subject: [PATCH 005/109] Minor changes. --- src/fdb5/s3/S3Store.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fdb5/s3/S3Store.cc b/src/fdb5/s3/S3Store.cc index befd9ebf6..1706536c7 100644 --- a/src/fdb5/s3/S3Store.cc +++ b/src/fdb5/s3/S3Store.cc @@ -206,7 +206,6 @@ void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostrea eckit::S3Name n{uri}; - ASSERT(n.hasBucketName()); ASSERT(n.bucketName() == db_bucket_); if (n.hasKeyName()) { @@ -222,7 +221,6 @@ void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostrea // /// @note: code for single bucket for all DBs // eckit::S3Name n{uri}; - // ASSERT(n.hasBucketName()); // ASSERT(n.bucketName() == bucket_); // /// @note: if !n.hasKeyName, maybe this method should return without destroying anything. // /// this way when TocWipeVisitor has wipeAll == true, the (only) bucket will not be destroyed From 53a7455a309cbf404e5a245485c96de12e0c2783 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Thu, 15 Feb 2024 00:14:34 +0100 Subject: [PATCH 006/109] Updated code to comply with agreed eckit S3 APIs. --- src/fdb5/s3/S3Common.cc | 20 ++++++--- src/fdb5/s3/S3FieldLocation.cc | 2 + src/fdb5/s3/S3Store.cc | 78 +++++++++++++++++++++++----------- 3 files changed, 68 insertions(+), 32 deletions(-) diff --git a/src/fdb5/s3/S3Common.cc b/src/fdb5/s3/S3Common.cc index c5d62c244..48d3949ef 100644 --- a/src/fdb5/s3/S3Common.cc +++ b/src/fdb5/s3/S3Common.cc @@ -10,9 +10,10 @@ // #include -#include "eckit/s3/S3Name.h" -#include "eckit/s3/S3Credential.h" -#include "eckit/s3/S3Session.h" +#include "eckit/io/s3/S3Name.h" +#include "eckit/io/s3/S3Bucket.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Session.h" #include "fdb5/s3/S3Common.h" @@ -75,11 +76,16 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con parseConfig(config); + endpoint_ = eckit::net::Endpoint{uri.host(), uri.port()}; + /// @note: code for bucket per DB - db_bucket_ = eckit::S3Name{uri}.bucketName(); + const auto parts = Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + ASSERT(n == 1 | n == 2); + db_bucket_ = parts[0]; @@ -87,11 +93,11 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con // eckit::S3Name n{uri}; - // bucket_ = n.bucketName(); + // bucket_ = n.bucket().name(); // eckit::Tokenizer parse("_"); // std::vector bits; - // parse(n.keyName(), bits); + // parse(n.name(), bits); // ASSERT(bits.size() == 2); @@ -124,7 +130,7 @@ S3Common::parseConfig(const fdb5::Config& config) { eckit::S3Session::instance().addCredentials(cred); - endpoint_ = s3.getString("endpoint", "127.0.0.1:9000"); + endpoint_ = eckit::net::Endpoint{s3.getString("endpoint", "127.0.0.1:9000")}; diff --git a/src/fdb5/s3/S3FieldLocation.cc b/src/fdb5/s3/S3FieldLocation.cc index 85d7a0a49..aa9af967b 100644 --- a/src/fdb5/s3/S3FieldLocation.cc +++ b/src/fdb5/s3/S3FieldLocation.cc @@ -9,6 +9,8 @@ */ // #include "eckit/filesystem/URIManager.h" +#include "eckit/io/s3/S3Name.h" + #include "fdb5/daos/S3FieldLocation.h" // #include "fdb5/LibFdb5.h" diff --git a/src/fdb5/s3/S3Store.cc b/src/fdb5/s3/S3Store.cc index 1706536c7..51a1fc10f 100644 --- a/src/fdb5/s3/S3Store.cc +++ b/src/fdb5/s3/S3Store.cc @@ -14,7 +14,6 @@ #include "eckit/utils/MD5.h" // #include "eckit/config/Resource.h" -#include "eckit/s3/S3Name.h" #include "fdb5/s3/S3FieldLocation.h" #include "fdb5/s3/S3Store.h" @@ -37,7 +36,7 @@ S3Store::S3Store(const Schema& schema, const eckit::URI& uri, const Config& conf eckit::URI S3Store::uri() const { - return eckit::S3Name(endpoint_, db_bucket_).URI(); + return eckit::S3Bucket(endpoint_, db_bucket_).URI(); /// @note: code for single bucket for all DBs @@ -50,16 +49,22 @@ eckit::URI S3Store::uri() const { bool S3Store::uriBelongs(const eckit::URI& uri) const { - /// @todo: avoid building a S3Name as it makes uriBelongs expensive + const auto parts = Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + + + /// @note: code for bucket per DB + ASSERT(n == 1 || n == 2); return ( (uri.scheme() == type()) && - (eckit::S3Name(uri).bucketName() == db_bucket_)); + (parts[0] == db_bucket_)); /// @note: code for single bucket for all DBs + // ASSERT(n == 2); // return ( // (uri.scheme() == type()) && - // (eckit::S3Name(uri).keyName().rfind(db_prefix_, 0) == 0)); + // (parts[1].rfind(db_prefix_, 0) == 0)); } @@ -67,17 +72,22 @@ bool S3Store::uriExists(const eckit::URI& uri) const { /// @todo: revisit the name of this method + + /// @note: code for bucket per DB + const auto parts = Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + ASSERT(n == 1 | n == 2); ASSERT(uri.scheme() == type()); - eckit::S3Name n(uri); - ASSERT(n.bucketName() == db_bucket_); + eckit::S3Bucket n{eckit::net::Endpoint{uri.host(), uri.port()}, parts[0]}; + ASSERT(n.name() == db_bucket_); return n.exists(); /// @note: code for single bucket for all DBs // ASSERT(uri.scheme() == type()); // eckit::S3Name n(uri); - // ASSERT(n.bucketName() == bucket_); - // ASSERT(n.keyName().rfind(db_prefix_, 0) == 0); + // ASSERT(n.bucket().name() == bucket_); + // ASSERT(n.name().rfind(db_prefix_, 0) == 0); // return n.exists(); } @@ -86,7 +96,7 @@ std::vector S3Store::storeUnitURIs() const { std::vector store_unit_uris; - eckit::S3Name bucket{endpoint_, db_bucket_}; + eckit::S3Bucket bucket{endpoint_, db_bucket_}; if (!bucket.exists()) return store_unit_uris; @@ -104,7 +114,7 @@ std::vector S3Store::storeUnitURIs() const { /// @note: code for single bucket for all DBs // std::vector store_unit_uris; - // eckit::S3Name bucket{endpoint_, bucket_}; + // eckit::S3Bucket bucket{endpoint_, bucket_}; // if (!bucket.exists()) return store_unit_uris; @@ -135,7 +145,7 @@ std::set S3Store::asStoreUnitURIs(const std::vector& uri bool S3Store::exists() const { - return eckit::S3Name(endpoint_, db_bucket_).exists(); + return eckit::S3Bucket(endpoint_, db_bucket_).exists(); } @@ -155,6 +165,13 @@ std::unique_ptr S3Store::archive(const Key& key, const void * dat /// if bucket per db, starting by indexkey_ eckit::S3Name n = generateDataKey(key); + /// @todo: ensure bucket if not yet seen by this process + static std::set knownBuckets; + if (knownBuckets.find(n.bucket().name()) != knownBuckets.end()) { + n.bucket().ensureCreated(); + knownBuckets.insert(n.bucket().name()); + } + std::unique_ptr h(n.dataHandle()) h->openForWrite(length); @@ -204,28 +221,39 @@ void S3Store::close() { void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { - eckit::S3Name n{uri}; + /// @note: code for bucket per DB + + const auto parts = Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + ASSERT(n == 1 | n == 2); - ASSERT(n.bucketName() == db_bucket_); + ASSERT(parts[0] == db_bucket_); - if (n.hasKeyName()) { - logVerbose << "destroy S3 key: "; - } else { - logVerbose << "destroy S3 bucket: "; - } + if (n == 2) { // object - logAlways << n.asString() << std::endl; - if (doit) n.destroy(); + eckit::S3Name key{uri}; + + logVerbose << "destroy S3 key: " << key.asString() << std::endl; + + if (doit) key.delete(); + + } else { // pool + + eckit::S3Bucket bucket{uri}; + + logVerbose << "destroy S3 bucket: " << bucket.asString() << std::endl; + + if (doit) bucket.ensureDestroyed(); + } // /// @note: code for single bucket for all DBs // eckit::S3Name n{uri}; - // ASSERT(n.bucketName() == bucket_); - // /// @note: if !n.hasKeyName, maybe this method should return without destroying anything. + // ASSERT(n.bucket().name() == bucket_); + // /// @note: if uri doesn't have key name, maybe this method should return without destroying anything. // /// this way when TocWipeVisitor has wipeAll == true, the (only) bucket will not be destroyed - // ASSERT(n.hasKeyName()); - // ASSERT(n.keyName().rfind(db_prefix_, 0) == 0); + // ASSERT(n.name().rfind(db_prefix_, 0) == 0); // logVerbose << "destroy S3 key: "; // logAlways << n.asString() << std::endl; From 6393852624bdcddca93480ec2a3dbdea72ae93c5 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sat, 17 Feb 2024 14:22:11 +0100 Subject: [PATCH 007/109] Modifications for the S3Store to fully build and run with the eckit S3 classes. Added S3Store unit tests with working archive, retrieve, list and wipe. --- CMakeLists.txt | 2 +- src/fdb5/CMakeLists.txt | 4 +- src/fdb5/api/helpers/ListIterator.h | 7 + src/fdb5/database/Catalogue.cc | 10 +- src/fdb5/s3/S3Common.cc | 8 +- src/fdb5/s3/S3Common.h | 2 +- src/fdb5/s3/S3FieldLocation.cc | 4 +- src/fdb5/s3/S3Store.cc | 43 ++- src/fdb5/s3/S3Store.h | 2 - src/fdb5/toc/FieldRef.cc | 28 +- src/fdb5/toc/TocWipeVisitor.cc | 99 ++--- src/fdb5/toc/TocWipeVisitor.h | 5 +- tests/fdb/CMakeLists.txt | 1 + tests/fdb/s3/CMakeLists.txt | 18 + tests/fdb/s3/test_s3_store.cc | 564 ++++++++++++++++++++++++++++ 15 files changed, 700 insertions(+), 97 deletions(-) create mode 100644 tests/fdb/s3/CMakeLists.txt create mode 100644 tests/fdb/s3/test_s3_store.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bdec2a55..e0cbe98cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,7 @@ ecbuild_add_option( FEATURE TOCFDB # option defined in fdb5_config.h ### FDB S3 Store backend ecbuild_add_option( FEATURE S3FDB - CONDITION eckit_S3_FOUND + CONDITION eckit_HAVE_AWS_S3 DEFAULT ON DESCRIPTION "S3 support for FDB Store" ) diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8907714d3..5d162696c 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -401,13 +401,11 @@ ecbuild_add_library( PRIVATE_INCLUDES "${PMEM_INCLUDE_DIRS}" "${LUSTREAPI_INCLUDE_DIRS}" - "${S3_INCLUDE_DIRS}" PRIVATE_LIBS ${grib_handling_pkg} ${PMEM_LIBRARIES} ${LUSTREAPI_LIBRARIES} - ${S3_LIBRARIES} ) if(HAVE_FDB_BUILD_TOOLS) @@ -454,7 +452,7 @@ foreach( _tool ${fdb5_tools} ) ecbuild_add_executable( TARGET ${_tool} CONDITION HAVE_FDB_BUILD_TOOLS SOURCES tools/${_tool}.cc - INCLUDES ${ECCODES_INCLUDE_DIRS} ${S3_INCLUDE_DIRS} # Please don't remove me, I am needed + INCLUDES ${ECCODES_INCLUDE_DIRS} # Please don't remove me, I am needed LIBS fdb5 ) endforeach() diff --git a/src/fdb5/api/helpers/ListIterator.h b/src/fdb5/api/helpers/ListIterator.h index 255fc8450..76ba1cfdc 100644 --- a/src/fdb5/api/helpers/ListIterator.h +++ b/src/fdb5/api/helpers/ListIterator.h @@ -102,6 +102,13 @@ class ListIterator : public APIIterator { ListIterator(ListIterator&& iter) : APIIterator(std::move(iter)), seenKeys_(std::move(iter.seenKeys_)), deduplicate_(iter.deduplicate_) {} + ListIterator& operator=(ListIterator&& iter) { + APIIterator::operator=(std::forward(iter)); + seenKeys_ = std::move(iter.seenKeys_); + deduplicate_ = iter.deduplicate_; + return *this; + } + bool next(ListElement& elem) { ListElement tmp; while (APIIterator::next(tmp)) { diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc index 5efb2857a..3cae35739 100644 --- a/src/fdb5/database/Catalogue.cc +++ b/src/fdb5/database/Catalogue.cc @@ -25,13 +25,9 @@ namespace fdb5 { std::unique_ptr Catalogue::buildStore() { - if (buildByKey_) - return StoreFactory::instance().build(schema(), key(), config_); - else { - std::string name = config_.getString("store", "file"); - - return StoreFactory::instance().build(schema(), eckit::URI(name, uri()), config_); - } + /// @todo: buildByKey_ and all Store constructors taking a URI + /// (and StoreFactory::build(..., uri, ...)) can be removed + return StoreFactory::instance().build(schema(), key(), config_); } bool Catalogue::enabled(const ControlIdentifier& controlIdentifier) const { diff --git a/src/fdb5/s3/S3Common.cc b/src/fdb5/s3/S3Common.cc index 48d3949ef..77d35fea2 100644 --- a/src/fdb5/s3/S3Common.cc +++ b/src/fdb5/s3/S3Common.cc @@ -32,7 +32,9 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con /// @note: code for bucket per DB - db_bucket_ = prefix_ + key.valuesToString(); + std::string keyStr = key.valuesToString(); + std::replace(keyStr.begin(), keyStr.end(), ':', '-'); + db_bucket_ = prefix_ + keyStr; @@ -82,7 +84,7 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con /// @note: code for bucket per DB - const auto parts = Tokenizer("/").tokenize(uri.name()); + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); ASSERT(n == 1 | n == 2); db_bucket_ = parts[0]; @@ -113,7 +115,7 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con } -S3Common::parseConfig(const fdb5::Config& config) { +void S3Common::parseConfig(const fdb5::Config& config) { eckit::LocalConfiguration cr{}, s3{}; diff --git a/src/fdb5/s3/S3Common.h b/src/fdb5/s3/S3Common.h index 67374380a..e0fc96a2b 100644 --- a/src/fdb5/s3/S3Common.h +++ b/src/fdb5/s3/S3Common.h @@ -34,7 +34,7 @@ class S3Common { protected: // members - std::string endpoint_; + eckit::net::Endpoint endpoint_; std::string db_bucket_; /// @note: code for single bucket for all DBs diff --git a/src/fdb5/s3/S3FieldLocation.cc b/src/fdb5/s3/S3FieldLocation.cc index aa9af967b..fbc1e74bb 100644 --- a/src/fdb5/s3/S3FieldLocation.cc +++ b/src/fdb5/s3/S3FieldLocation.cc @@ -11,7 +11,7 @@ // #include "eckit/filesystem/URIManager.h" #include "eckit/io/s3/S3Name.h" -#include "fdb5/daos/S3FieldLocation.h" +#include "fdb5/s3/S3FieldLocation.h" // #include "fdb5/LibFdb5.h" namespace fdb5 { @@ -39,7 +39,7 @@ std::shared_ptr S3FieldLocation::make_shared() const { eckit::DataHandle* S3FieldLocation::dataHandle() const { - return eckit::S3Name(uri_).dataHandle(offset(), length()); + return eckit::S3Name(uri_).dataHandle(offset()); } diff --git a/src/fdb5/s3/S3Store.cc b/src/fdb5/s3/S3Store.cc index 51a1fc10f..acf8ec163 100644 --- a/src/fdb5/s3/S3Store.cc +++ b/src/fdb5/s3/S3Store.cc @@ -8,10 +8,14 @@ * does it submit to any jurisdiction. */ +#include + +#include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" #include "eckit/log/TimeStamp.h" #include "eckit/utils/MD5.h" +#include "eckit/utils/Tokenizer.h" // #include "eckit/config/Resource.h" @@ -36,7 +40,7 @@ S3Store::S3Store(const Schema& schema, const eckit::URI& uri, const Config& conf eckit::URI S3Store::uri() const { - return eckit::S3Bucket(endpoint_, db_bucket_).URI(); + return eckit::S3Bucket(endpoint_, db_bucket_).uri(); /// @note: code for single bucket for all DBs @@ -49,7 +53,7 @@ eckit::URI S3Store::uri() const { bool S3Store::uriBelongs(const eckit::URI& uri) const { - const auto parts = Tokenizer("/").tokenize(uri.name()); + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); @@ -74,9 +78,9 @@ bool S3Store::uriExists(const eckit::URI& uri) const { /// @note: code for bucket per DB - const auto parts = Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); - ASSERT(n == 1 | n == 2); + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto pn = parts.size(); + ASSERT(pn == 1 | pn == 2); ASSERT(uri.scheme() == type()); eckit::S3Bucket n{eckit::net::Endpoint{uri.host(), uri.port()}, parts[0]}; ASSERT(n.name() == db_bucket_); @@ -102,9 +106,9 @@ std::vector S3Store::storeUnitURIs() const { /// @note if an S3Catalogue is implemented, some filtering will need to /// be done here to discriminate store keys from catalogue keys - for (const auto& key : bucket.listKeys()) { + for (const auto& key : bucket.listObjects()) { - store_unit_uris.push_back(eckit::S3Name(endpoint_, db_bucket_, key).URI()); + store_unit_uris.push_back(key.uri()); } @@ -120,9 +124,9 @@ std::vector S3Store::storeUnitURIs() const { // /// @note if an S3Catalogue is implemented, more filtering will need to // /// be done here to discriminate store keys from catalogue keys - // for (const auto& key : bucket.listKeys(filter = "^" + db_prefix_ + "_.*")) { + // for (const auto& key : bucket.listObjects(filter = "^" + db_prefix_ + "_.*")) { - // store_unit_uris.push_back(eckit::S3Name(endpoint_, bucket_, key).URI()); + // store_unit_uris.push_back(key.uri()); // } @@ -167,19 +171,19 @@ std::unique_ptr S3Store::archive(const Key& key, const void * dat /// @todo: ensure bucket if not yet seen by this process static std::set knownBuckets; - if (knownBuckets.find(n.bucket().name()) != knownBuckets.end()) { + if (knownBuckets.find(n.bucket().name()) == knownBuckets.end()) { n.bucket().ensureCreated(); knownBuckets.insert(n.bucket().name()); } - std::unique_ptr h(n.dataHandle()) - + std::unique_ptr h(n.dataHandle()); + h->openForWrite(length); eckit::AutoClose closer(*h); h->write(data, length); - return std::unique_ptr(new S3FieldLocation(n.URI(), 0, length, fdb5::Key())); + return std::unique_ptr(new S3FieldLocation(n.uri(), 0, length, fdb5::Key())); /// @note: code for S3 object (key) per index store: @@ -223,7 +227,7 @@ void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostrea /// @note: code for bucket per DB - const auto parts = Tokenizer("/").tokenize(uri.name()); + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); ASSERT(n == 1 | n == 2); @@ -235,7 +239,7 @@ void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostrea logVerbose << "destroy S3 key: " << key.asString() << std::endl; - if (doit) key.delete(); + if (doit) key.destroy(); } else { // pool @@ -271,11 +275,11 @@ void S3Store::print(std::ostream& out) const { } /// @note: unique name generation copied from LocalPathName::unique. -static StaticMutex local_mutex; +static eckit::StaticMutex local_mutex; eckit::S3Name S3Store::generateDataKey(const Key& key) const { - AutoLock lock(local_mutex); + eckit::AutoLock lock(local_mutex); std::string hostname = eckit::Main::hostname(); @@ -295,7 +299,10 @@ eckit::S3Name S3Store::generateDataKey(const Key& key) const { eckit::MD5 md5(name); - return eckit::S3Name{endpoint_, db_bucket_, key.valuesToString() + "_" + md5.digest() + ".data"}; + std::string keyStr = key.valuesToString(); + std::replace(keyStr.begin(), keyStr.end(), ':', '-'); + + return eckit::S3Name{endpoint_, db_bucket_, keyStr + "." + md5.digest() + ".data"}; /// @note: code for single bucket for all DBs // return eckit::S3Name{endpoint_, bucket_, db_prefix_ + "_" + key.valuesToString() + "_" + md5.digest() + ".data"}; diff --git a/src/fdb5/s3/S3Store.h b/src/fdb5/s3/S3Store.h index 34ffb92f7..bd7b69de0 100644 --- a/src/fdb5/s3/S3Store.h +++ b/src/fdb5/s3/S3Store.h @@ -65,8 +65,6 @@ class S3Store : public Store, public S3Common { // eckit::DataHandle& getDataHandle(const Key& key, const eckit::S3Name& name); // void closeDataHandles(); - void print( std::ostream &out ) const override; - private: // types /// @note: code for S3 object (key) per index store: diff --git a/src/fdb5/toc/FieldRef.cc b/src/fdb5/toc/FieldRef.cc index 3538f9210..967c12d2f 100644 --- a/src/fdb5/toc/FieldRef.cc +++ b/src/fdb5/toc/FieldRef.cc @@ -19,9 +19,9 @@ #include "fdb5/database/UriStore.h" #include "fdb5/toc/TocFieldLocation.h" -// #ifdef fdb5_HAVE_DAOSFDB -// #include "fdb5/daos/DaosFieldLocation.h" -// #endif +#ifdef fdb5_HAVE_S3FDB +#include "fdb5/s3/S3FieldLocation.h" +#endif namespace fdb5 { @@ -37,16 +37,16 @@ FieldRefLocation::FieldRefLocation(UriStore &store, const Field& field) { const FieldLocation& loc = field.location(); -// #ifdef fdb5_HAVE_DAOSFDB -// const TocFieldLocation* tocfloc = dynamic_cast(&loc); -// const DaosFieldLocation* daosfloc = dynamic_cast(&loc); -// if(!tocfloc && !daosfloc) { -// throw eckit::NotImplemented( -// "Field location is not of TocFieldLocation or DaosFieldLocation type " -// "-- indexing other locations is not supported", -// Here()); -// } -// #else +#ifdef fdb5_HAVE_S3FDB + const TocFieldLocation* tocfloc = dynamic_cast(&loc); + const S3FieldLocation* s3floc = dynamic_cast(&loc); + if(!tocfloc && !s3floc) { + throw eckit::NotImplemented( + "Field location is not of TocFieldLocation or S3FieldLocation type " + "-- indexing other locations is not supported", + Here()); + } +#else const TocFieldLocation* tocfloc = dynamic_cast(&loc); if(!tocfloc) { throw eckit::NotImplemented( @@ -54,7 +54,7 @@ FieldRefLocation::FieldRefLocation(UriStore &store, const Field& field) { "-- indexing other locations is not supported", Here()); } -// #endif +#endif uriId_ = store.insert(loc.uri()); length_ = loc.length(); diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index a530cc660..354a740c6 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -12,6 +12,7 @@ #include #include "eckit/os/Stat.h" +#include "eckit/io/s3/S3Bucket.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/database/DB.h" @@ -114,7 +115,7 @@ bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& stor ASSERT(subtocPaths_.empty()); ASSERT(lockfilePaths_.empty()); ASSERT(indexPaths_.empty()); - ASSERT(dataPaths_.empty()); + ASSERT(dataURIs_.empty()); ASSERT(safePaths_.empty()); ASSERT(indexesToMask_.empty()); @@ -161,8 +162,8 @@ bool TocWipeVisitor::visitIndex(const Index& index) { // Enumerate data files. - std::vector indexDataPaths(index.dataPaths()); - for (const eckit::URI& uri : store_.asStoreUnitURIs(indexDataPaths)) { + std::vector indexDataURIs(index.dataPaths()); + for (const eckit::URI& uri : store_.asStoreUnitURIs(indexDataURIs)) { if (include) { if (!store_.uriBelongs(uri)) { Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." << std::endl; @@ -171,9 +172,9 @@ bool TocWipeVisitor::visitIndex(const Index& index) { Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." << std::endl; NOTIMP; } - dataPaths_.insert(eckit::PathName(uri.path())); + dataURIs_.insert(uri); } else { - safePaths_.insert(eckit::PathName(uri.path())); + safeURIs_.insert(uri); } } @@ -198,7 +199,7 @@ void TocWipeVisitor::addMaskedPaths() { } } for (const auto& uri : data) { - if (store_.uriBelongs(uri)) dataPaths_.insert(eckit::PathName(uri.path())); + if (store_.uriBelongs(uri)) dataURIs_.insert(uri); } } @@ -228,11 +229,17 @@ void TocWipeVisitor::ensureSafePaths() { if (safePaths_.find(schemaPath_) != safePaths_.end()) schemaPath_ = ""; for (const auto& p : safePaths_) { - for (std::set* s : {&subtocPaths_, &lockfilePaths_, &indexPaths_, &dataPaths_}) { + for (std::set* s : {&subtocPaths_, &lockfilePaths_, &indexPaths_}) { + s->erase(p); + } + } + for (const auto& p : safeURIs_) { + for (std::set* s : {&dataURIs_}) { s->erase(p); } } } + void TocWipeVisitor::calculateResidualPaths() { // Remove paths to non-existant files. This is reasonable as we may be recovering from a @@ -250,13 +257,13 @@ void TocWipeVisitor::calculateResidualPaths() { } } - for (std::set* fileset : {&dataPaths_}) { - for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { + for (std::set* uriset : {&dataURIs_}) { + for (std::set::iterator it = uriset->begin(); it != uriset->end(); ) { - if (store_.uriExists(eckit::URI(store_.type(), *it))) { + if (store_.uriExists(*it)) { ++it; } else { - fileset->erase(it++); + uriset->erase(it++); } } @@ -270,11 +277,13 @@ void TocWipeVisitor::calculateResidualPaths() { // Consider the total sets of paths std::set deletePaths; + std::set deleteURIs; deletePaths.insert(subtocPaths_.begin(), subtocPaths_.end()); deletePaths.insert(lockfilePaths_.begin(), lockfilePaths_.end()); deletePaths.insert(indexPaths_.begin(), indexPaths_.end()); if (store_.type() == "file") - deletePaths.insert(dataPaths_.begin(), dataPaths_.end()); + for (auto u : dataURIs_) + deletePaths.insert(eckit::PathName{u.name()}); if (tocPath_.asString().size()) deletePaths.insert(tocPath_); if (schemaPath_.asString().size()) deletePaths.insert(schemaPath_); @@ -313,42 +322,38 @@ void TocWipeVisitor::calculateResidualPaths() { if (store_.type() == "file") return; std::vector allStoreUnitURIs(store_.storeUnitURIs()); - std::vector allDataPathsVector; - for (const auto& u : allStoreUnitURIs) { - allDataPathsVector.push_back(eckit::PathName(u.path())); - } - std::set allDataPaths(allDataPathsVector.begin(), allDataPathsVector.end()); + std::set allDataURIs(allStoreUnitURIs.begin(), allStoreUnitURIs.end()); - ASSERT(residualDataPaths_.empty()); + ASSERT(residualDataURIs_.empty()); - if (!(dataPaths_ == allDataPaths)) { + if (!(dataURIs_ == allDataURIs)) { // First we check if there are paths marked to delete that don't exist. This is an error - std::set paths; - std::set_difference(dataPaths_.begin(), dataPaths_.end(), - allDataPaths.begin(), allDataPaths.end(), - std::inserter(paths, paths.begin())); + std::set uris; + std::set_difference(dataURIs_.begin(), dataURIs_.end(), + allDataURIs.begin(), allDataURIs.end(), + std::inserter(uris, uris.begin())); - if (!paths.empty()) { - Log::error() << "Store unit paths not in existing paths set:" << std::endl; - for (const auto& p : paths) { - Log::error() << " - " << p << std::endl; + if (!uris.empty()) { + Log::error() << "Store unit uris not in existing uris set:" << std::endl; + for (const auto& u : uris) { + Log::error() << " - " << u << std::endl; } - throw SeriousBug("Store unit path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); + throw SeriousBug("Store unit uri to delete should be in existing uri set. Are multiple wipe commands running simultaneously?", Here()); } - std::set_difference(allDataPaths.begin(), allDataPaths.end(), - dataPaths_.begin(), dataPaths_.end(), - std::inserter(residualDataPaths_, residualDataPaths_.begin())); + std::set_difference(allDataURIs.begin(), allDataURIs.end(), + dataURIs_.begin(), dataURIs_.end(), + std::inserter(residualDataURIs_, residualDataURIs_.begin())); } } bool TocWipeVisitor::anythingToWipe() const { return (!subtocPaths_.empty() || !lockfilePaths_.empty() || !indexPaths_.empty() || - !dataPaths_.empty() || !indexesToMask_.empty() || + !dataURIs_.empty() || !indexesToMask_.empty() || tocPath_.asString().size() || schemaPath_.asString().size()); } @@ -382,9 +387,9 @@ void TocWipeVisitor::report(bool wipeAll) { } out_ << std::endl; - out_ << "Data files to delete: " << std::endl; - if (dataPaths_.empty()) out_ << " - NONE -" << std::endl; - for (const auto& f : dataPaths_) { + out_ << "Data URIs to delete: " << std::endl; + if (dataURIs_.empty()) out_ << " - NONE -" << std::endl; + for (const auto& f : dataURIs_) { out_ << " " << f << std::endl; } out_ << std::endl; @@ -406,6 +411,13 @@ void TocWipeVisitor::report(bool wipeAll) { } out_ << std::endl; + out_ << "Protected URIs (explicitly untouched):" << std::endl; + if (safeURIs_.empty()) out_ << " - NONE - " << std::endl; + for (const auto& u : safeURIs_) { + out_ << " " << u << std::endl; + } + out_ << std::endl; + if (!safePaths_.empty()) { out_ << "Indexes to mask:" << std::endl; if (indexesToMask_.empty()) out_ << " - NONE - " << std::endl; @@ -460,8 +472,7 @@ void TocWipeVisitor::wipe(bool wipeAll) { /// @todo: are all these exist checks necessary? - for (const PathName& path : residualDataPaths_) { - eckit::URI uri(store_.type(), path); + for (const URI& uri : residualDataURIs_) { if (store_.uriExists(uri)) { store_.remove(uri, logAlways, logVerbose, doit_); } @@ -472,8 +483,7 @@ void TocWipeVisitor::wipe(bool wipeAll) { } } - for (const PathName& path : dataPaths_) { - eckit::URI uri(store_.type(), path); + for (const URI& uri : dataURIs_) { if (store_.uriExists(uri)) { store_.remove(uri, logAlways, logVerbose, doit_); } @@ -483,7 +493,8 @@ void TocWipeVisitor::wipe(bool wipeAll) { if (wipeAll && store_.type() != "file") /// @todo: if the store is holding catalogue information (e.g. daos KVs) it /// should not be removed - store_.remove(store_.uri(), logAlways, logVerbose, doit_); + if (store_.uriExists(store_.uri())) + store_.remove(store_.uri(), logAlways, logVerbose, doit_); for (const std::set& pathset : {indexPaths_, std::set{schemaPath_}, subtocPaths_, @@ -505,7 +516,7 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { // We wipe everything if there is nothingn within safePaths - i.e. there is // no data that wasn't matched by the request - bool wipeAll = safePaths_.empty(); + bool wipeAll = safePaths_.empty() && safeURIs_.empty(); if (wipeAll) { addMaskedPaths(); @@ -533,14 +544,14 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { out_ << std::endl; } - if (wipeAll && !residualDataPaths_.empty()) { + if (wipeAll && !residualDataURIs_.empty()) { out_ << "Unexpected store units present in store: " << std::endl; - for (const auto& p : residualDataPaths_) out_ << " " << store_.type() << "://" << p << std::endl; + for (const auto& u : residualDataURIs_) out_ << " " << u << std::endl; out_ << std::endl; } - if (wipeAll && (!residualPaths_.empty() || !residualDataPaths_.empty())) { + if (wipeAll && (!residualPaths_.empty() || !residualDataURIs_.empty())) { if (!unsafeWipeAll_) { out_ << "Full wipe will not proceed without --unsafe-wipe-all" << std::endl; if (doit_) diff --git a/src/fdb5/toc/TocWipeVisitor.h b/src/fdb5/toc/TocWipeVisitor.h index 6b1f88c18..fb5d32752 100644 --- a/src/fdb5/toc/TocWipeVisitor.h +++ b/src/fdb5/toc/TocWipeVisitor.h @@ -67,11 +67,12 @@ class TocWipeVisitor : public WipeVisitor { std::set subtocPaths_; std::set lockfilePaths_; std::set indexPaths_; - std::set dataPaths_; + std::set dataURIs_; std::set safePaths_; + std::set safeURIs_; std::set residualPaths_; - std::set residualDataPaths_; + std::set residualDataURIs_; std::vector indexesToMask_; }; diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index 3f9748011..7820737e9 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -71,6 +71,7 @@ endforeach() # pmem tests make use of the test environment, so are added at the end add_subdirectory( pmem ) +add_subdirectory( s3 ) add_subdirectory( api ) add_subdirectory( tools ) add_subdirectory( type ) diff --git a/tests/fdb/s3/CMakeLists.txt b/tests/fdb/s3/CMakeLists.txt new file mode 100644 index 000000000..7c460fd53 --- /dev/null +++ b/tests/fdb/s3/CMakeLists.txt @@ -0,0 +1,18 @@ +if (HAVE_S3FDB) + + list( APPEND s3_tests + s3_store + ) + + list( APPEND unit_test_libraries fdb5 ) + + foreach( _test ${s3_tests} ) + + ecbuild_add_test( TARGET test_fdb5_s3_${_test} + SOURCES test_${_test}.cc + LIBS "${unit_test_libraries}" + INCLUDES "${unit_test_include_dirs}" ) + + endforeach() + +endif() \ No newline at end of file diff --git a/tests/fdb/s3/test_s3_store.cc b/tests/fdb/s3/test_s3_store.cc new file mode 100644 index 000000000..62ad2645e --- /dev/null +++ b/tests/fdb/s3/test_s3_store.cc @@ -0,0 +1,564 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +// #include +// #include + +// #include "eckit/config/Resource.h" +#include "eckit/testing/Test.h" +// #include "eckit/filesystem/URI.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +// #include "eckit/filesystem/TmpDir.h" +// #include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/config/YAMLConfiguration.h" + +// #include "metkit/mars/MarsRequest.h" + +// #include "fdb5/fdb5_config.h" +// #include "fdb5/config/Config.h" +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" + +#include "fdb5/toc/TocCatalogueWriter.h" +#include "fdb5/toc/TocCatalogueReader.h" + +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Handle.h" + +#include "fdb5/s3/S3Store.h" +#include "fdb5/s3/S3FieldLocation.h" +// #include "fdb5/daos/DaosException.h" + +using namespace eckit::testing; +using namespace eckit; + +namespace { + + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; + + S3Config cfg("eu-central-1", "127.0.0.1", 9000); + + void ensureClean(const std::string& prefix) { + auto client = S3Client::makeUnique(cfg); + auto&& tmp = client->listBuckets(); + std::set buckets(tmp.begin(), tmp.end()); + + for (const std::string& name : buckets) { + if (name.rfind(prefix, 0) == 0) { + client->emptyBucket(name); + client->deleteBucket(name); + } + } + } +} + +// #ifdef fdb5_HAVE_DUMMY_DAOS +// eckit::TmpDir& tmp_dummy_daos_root() { +// static eckit::TmpDir d{}; +// return d; +// } +// #endif + +// temporary schema,spaces,root files common to all DAOS Store tests + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& spaces_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& roots_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& store_tests_tmp_root() { + static eckit::PathName sd("./s3_store_tests_fdb_root"); + return sd; +} + +namespace fdb { +namespace test { + +CASE( "Setup" ) { + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Store tests fail. + if (store_tests_tmp_root().exists()) deldir(store_tests_tmp_root()); + store_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", store_tests_tmp_root().path().c_str(), 1); + + // prepare schema for tests involving S3Store + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); + + // prepare scpaces + + std::string spaces_str{".* all Default"}; + + std::unique_ptr hsp(spaces_file().fileHandle()); + hsp->openForWrite(spaces_str.size()); + { + eckit::AutoClose closer(*hsp); + hsp->write(spaces_str.data(), spaces_str.size()); + } + + ::setenv("FDB_SPACES_FILE", spaces_file().path().c_str(), 1); + + // prepare roots + + std::string roots_str{store_tests_tmp_root().asString() + " all yes yes"}; + + std::unique_ptr hr(roots_file().fileHandle()); + hr->openForWrite(roots_str.size()); + { + eckit::AutoClose closer(*hr); + hr->write(roots_str.data(), roots_str.size()); + } + + ::setenv("FDB_ROOTS_FILE", roots_file().path().c_str(), 1); + +} + +CASE("S3Store tests") { + + SECTION("archive and retrieve") { + + std::string prefix{"test1-"}; + + ensureClean(prefix); + + std::string config_str{ + "s3:\n" + " credential:\n" + " accessKeyID: minio\n" + " secretKey: minio1234\n" + " host: 127.0.0.1\n" + " endpoint: 127.0.0.1:9000\n" + " bucketPrefix: " + prefix + "\n" + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Schema schema{schema_file()}; + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key db_key{"a=1,b=2"}; + fdb5::Key index_key{"c=3,d=4"}; + + char data[] = "test"; + + // archive + + fdb5::S3Store s3store{schema, db_key, config}; + fdb5::Store& store = s3store; + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + s3store.flush(); + + // retrieve + fdb5::Field field(std::move(loc), std::time(nullptr)); + std::cout << "Read location: " << field.location() << std::endl; + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // remove + eckit::S3Name field_name{field.location().uri()}; + eckit::S3Bucket store_name{field_name.bucket()}; + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT_NOT(store_name.exists()); + + } + + SECTION("with POSIX Catalogue") { + + std::string prefix{"test2-"}; + + ensureClean(prefix); + + // FDB configuration + + std::string config_str{ + "schema : " + schema_file().path() + "\n" + "s3:\n" + " credential:\n" + " accessKeyID: minio\n" + " secretKey: minio1234\n" + " host: 127.0.0.1\n" + " endpoint: 127.0.0.1:9000\n" + " bucketPrefix: " + prefix + "\n" + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key db_key{"a=1,b=2"}; + fdb5::Key index_key{"c=3,d=4"}; + fdb5::Key field_key{"e=5,f=6"}; + + // store data + + char data[] = "test"; + + fdb5::S3Store s3store{schema, db_key, config}; + fdb5::Store& store = static_cast(s3store); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + // index data + + { + /// @todo: could have a unique ptr here, might not need a static cast + fdb5::TocCatalogueWriter tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.deselectIndex(); + cat.selectIndex(index_key); + //const fdb5::Index& idx = tcat.currentIndex(); + static_cast(tcat).archive(field_key, std::move(loc)); + + /// flush store before flushing catalogue + s3store.flush(); // not necessary if using a DAOS store + } + + // find data + + fdb5::Field field; + { + fdb5::TocCatalogueReader tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.selectIndex(index_key); + static_cast(tcat).retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // remove data + + eckit::S3Name field_name{field.location().uri()}; + eckit::S3Bucket store_name{field_name.bucket()}; + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT_NOT(store_name.exists()); + + // deindex data + + { + fdb5::TocCatalogueWriter tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + metkit::mars::MarsRequest r = db_key.request("retrieve"); + std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + cat.visitEntries(*wv, store, false); + } + + } + + SECTION("VIA FDB API") { + + std::string prefix{"test3-"}; + + ensureClean(prefix); + + // FDB configuration + + std::string config_str{ + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: toc\n" + "store: s3\n" + "s3:\n" + " credential:\n" + " accessKeyID: minio\n" + " secretKey: minio1234\n" + " host: 127.0.0.1\n" + " endpoint: 127.0.0.1:9000\n" + " bucketPrefix: " + prefix + "\n" + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; + fdb5::Key db_key{"a=1,b=2"}; + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + + // initialise store + + fdb5::FDB fdb(config); + + // check store is empty + + size_t count; + fdb5::ListElement info; + + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // store data + + char data[] = "test"; + + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + // retrieve data + + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // wipe data + + fdb5::WipeElement elem; + + // dry run attempt to wipe with too specific request + + auto wipeObject = fdb.wipe(full_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + + // dry run wipe index and store unit + wipeObject = fdb.wipe(index_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // dry run wipe database + wipeObject = fdb.wipe(db_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // ensure field still exists + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // attempt to wipe with too specific request + wipeObject = fdb.wipe(full_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + /// @todo: really needed? + fdb.flush(); + + // wipe index and store unit (and DB bucket as there is only one index) + wipeObject = fdb.wipe(index_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) count++; + EXPECT(count == 0); + + } + + /// @todo: if doing what's in this section at the end of the previous section reusing the same FDB object, + // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe + SECTION("FDB API RE-STORE AND WIPE DB") { + + std::string prefix{"test4-"}; + + // FDB configuration + + std::string config_str{ + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: toc\n" + "store: s3\n" + "s3:\n" + " credential:\n" + " accessKeyID: minio\n" + " secretKey: minio1234\n" + " host: 127.0.0.1\n" + " endpoint: 127.0.0.1:9000\n" + " bucketPrefix: " + prefix + "\n" + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; + fdb5::Key db_key{"a=1,b=2"}; + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + + // initialise store + + fdb5::FDB fdb(config); + + // store again + + char data[] = "test"; + + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + size_t count; + + // wipe all database + + fdb5::WipeElement elem; + auto wipeObject = fdb.wipe(db_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + + fdb5::ListElement info; + auto listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 0); + + } + +} + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + const eckit::S3Credential cred {"minio", "minio1234", "127.0.0.1"}; + eckit::S3Session::instance().addCredentials(cred); + + return run_tests ( argc, argv ); + + ensureClean(""); +} \ No newline at end of file From e503401b2d026e99c00972590eb8c07a0e2ba42a Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sat, 17 Feb 2024 15:01:25 +0100 Subject: [PATCH 008/109] Fixed authorship. --- src/fdb5/s3/S3Store.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fdb5/s3/S3Store.h b/src/fdb5/s3/S3Store.h index bd7b69de0..14aa2b43b 100644 --- a/src/fdb5/s3/S3Store.h +++ b/src/fdb5/s3/S3Store.h @@ -9,6 +9,8 @@ */ /// @author Nicolau Manubens +/// @author Metin Cakircali +/// @author Simon Smart /// @date Feb 2024 #pragma once @@ -83,4 +85,4 @@ class S3Store : public Store, public S3Common { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 From 23fdb9f54131e316b7872028bf584b50bf906eaf Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Thu, 29 Feb 2024 21:55:14 +0100 Subject: [PATCH 009/109] First feature/rados-backend branch commit. Removing content from S3 backend branch. --- CMakeLists.txt | 6 - src/fdb5/CMakeLists.txt | 11 - src/fdb5/fdb5_config.h.in | 1 - src/fdb5/s3/S3Common.cc | 146 --------- src/fdb5/s3/S3Common.h | 50 --- src/fdb5/s3/S3FieldLocation.cc | 56 ---- src/fdb5/s3/S3FieldLocation.h | 59 ---- src/fdb5/s3/S3Store.cc | 362 --------------------- src/fdb5/s3/S3Store.h | 88 ----- src/fdb5/toc/FieldRef.cc | 12 +- tests/fdb/s3/CMakeLists.txt | 18 -- tests/fdb/s3/test_s3_store.cc | 564 --------------------------------- 12 files changed, 6 insertions(+), 1367 deletions(-) delete mode 100644 src/fdb5/s3/S3Common.cc delete mode 100644 src/fdb5/s3/S3Common.h delete mode 100644 src/fdb5/s3/S3FieldLocation.cc delete mode 100644 src/fdb5/s3/S3FieldLocation.h delete mode 100644 src/fdb5/s3/S3Store.cc delete mode 100644 src/fdb5/s3/S3Store.h delete mode 100644 tests/fdb/s3/CMakeLists.txt delete mode 100644 tests/fdb/s3/test_s3_store.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index e0cbe98cf..03d8592f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,12 +55,6 @@ ecbuild_add_option( FEATURE TOCFDB # option defined in fdb5_config.h DEFAULT ON DESCRIPTION "Filesystem TOC support for FDB" ) -### FDB S3 Store backend -ecbuild_add_option( FEATURE S3FDB - CONDITION eckit_HAVE_AWS_S3 - DEFAULT ON - DESCRIPTION "S3 support for FDB Store" ) - ### support for Lustre API control of file stripping find_package( LUSTREAPI QUIET ) diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 5d162696c..8ccf68784 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -364,17 +364,6 @@ if( HAVE_RADOSFDB ) ) endif() -if( HAVE_S3FDB ) - list( APPEND fdb5_srcs - s3/S3FieldLocation.h - s3/S3FieldLocation.cc - s3/S3Store.h - s3/S3Store.cc - s3/S3Common.h - s3/S3Common.cc - ) -endif() - ecbuild_add_library( TARGET fdb5 diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index e9100c21c..6fec3c5d5 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -11,7 +11,6 @@ #cmakedefine fdb5_HAVE_PMEMFDB #cmakedefine fdb5_HAVE_RADOSFDB #cmakedefine fdb5_HAVE_TOCFDB -#cmakedefine fdb5_HAVE_S3FDB #cmakedefine01 fdb5_HAVE_GRIB #endif // fdb5_fdb5_config_h diff --git a/src/fdb5/s3/S3Common.cc b/src/fdb5/s3/S3Common.cc deleted file mode 100644 index 77d35fea2..000000000 --- a/src/fdb5/s3/S3Common.cc +++ /dev/null @@ -1,146 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -// #include - -#include "eckit/io/s3/S3Name.h" -#include "eckit/io/s3/S3Bucket.h" -#include "eckit/io/s3/S3Credential.h" -#include "eckit/io/s3/S3Session.h" - -#include "fdb5/s3/S3Common.h" - -// #include "eckit/exception/Exceptions.h" -#include "eckit/config/Resource.h" - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -S3Common::S3Common(const fdb5::Config& config, const std::string& component, const fdb5::Key& key) { - - parseConfig(config); - - - - /// @note: code for bucket per DB - - std::string keyStr = key.valuesToString(); - std::replace(keyStr.begin(), keyStr.end(), ':', '-'); - db_bucket_ = prefix_ + keyStr; - - - - - /// @note: code for single bucket for all DBs - - // std::vector valid{"catalogue", "store"}; - // ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); - - // bucket_ = "default"; - - // eckit::LocalConfiguration c{}; - - // if (config.has("s3")) c = config.getSubConfiguration("s3"); - // if (c.has(component)) bucket_ = c.getSubConfiguration(component).getString("bucket", bucket_); - - // std::string first_cap{component}; - // first_cap[0] = toupper(component[0]); - - // std::string all_caps{component}; - // for (auto & c: all_caps) c = toupper(c); - - // bucket_ = eckit::Resource("fdbS3" + first_cap + "Bucket;$FDB_S3_" + all_caps + "_BUCKET", bucket_); - - // db_prefix_ = key.valuesToString(); - - // if (c.has("client")) - // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); - - - - - /// @todo: check that the bucket name complies with name restrictions - -} - -S3Common::S3Common(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { - - /// @note: validity of input URI is not checked here because this constructor is only triggered - /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand - - parseConfig(config); - - endpoint_ = eckit::net::Endpoint{uri.host(), uri.port()}; - - - - /// @note: code for bucket per DB - - const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); - ASSERT(n == 1 | n == 2); - db_bucket_ = parts[0]; - - - - /// @note: code for single bucket for all DBs - - // eckit::S3Name n{uri}; - - // bucket_ = n.bucket().name(); - - // eckit::Tokenizer parse("_"); - // std::vector bits; - // parse(n.name(), bits); - - // ASSERT(bits.size() == 2); - - // db_prefix_ = bits[0]; - - - // // eckit::LocalConfiguration c{}; - - // // if (config.has("s3")) c = config.getSubConfiguration("s3"); - - // // if (c.has("client")) - // // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); - -} - -void S3Common::parseConfig(const fdb5::Config& config) { - - eckit::LocalConfiguration cr{}, s3{}; - - if (config.has("s3")) { - s3 = config.getSubConfiguration("s3"); - if (s3.has("credential")) cr = s3.getSubConfiguration("credential"); - } - - const eckit::S3Credential cred{ - cr.getString("accessKeyID", "defaultKeyID"), - cr.getString("secretKey", "defaultSecretKey"), - cr.getString("host", "127.0.0.1") - }; - - eckit::S3Session::instance().addCredentials(cred); - - endpoint_ = eckit::net::Endpoint{s3.getString("endpoint", "127.0.0.1:9000")}; - - - - /// @note: code for bucket per DB only - prefix_ = s3.getString("bucketPrefix", prefix_); - -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Common.h b/src/fdb5/s3/S3Common.h deleted file mode 100644 index e0fc96a2b..000000000 --- a/src/fdb5/s3/S3Common.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/// @file S3Common.h -/// @author Nicolau Manubens -/// @date Feb 2024 - -#pragma once - -#include "eckit/filesystem/URI.h" - -#include "fdb5/database/Key.h" -#include "fdb5/config/Config.h" - -namespace fdb5 { - -class S3Common { - -public: // methods - - S3Common(const fdb5::Config&, const std::string& component, const fdb5::Key&); - S3Common(const fdb5::Config&, const std::string& component, const eckit::URI&); - -private: // methods - - void parseConfig(const fdb5::Config& config); - -protected: // members - - eckit::net::Endpoint endpoint_; - std::string db_bucket_; - - /// @note: code for single bucket for all DBs - // std::string bucket_; - // std::string db_prefix_; - -private: // members - - std::string prefix_; - -}; - -} \ No newline at end of file diff --git a/src/fdb5/s3/S3FieldLocation.cc b/src/fdb5/s3/S3FieldLocation.cc deleted file mode 100644 index fbc1e74bb..000000000 --- a/src/fdb5/s3/S3FieldLocation.cc +++ /dev/null @@ -1,56 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -// #include "eckit/filesystem/URIManager.h" -#include "eckit/io/s3/S3Name.h" - -#include "fdb5/s3/S3FieldLocation.h" -// #include "fdb5/LibFdb5.h" - -namespace fdb5 { - -::eckit::ClassSpec S3FieldLocation::classSpec_ = {&FieldLocation::classSpec(), "S3FieldLocation",}; -::eckit::Reanimator S3FieldLocation::reanimator_; - -//---------------------------------------------------------------------------------------------------------------------- - -S3FieldLocation::S3FieldLocation(const S3FieldLocation& rhs) : - FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} - -S3FieldLocation::S3FieldLocation(const eckit::URI &uri) : FieldLocation(uri) {} - -/// @todo: remove remapKey from signature and always pass empty Key to FieldLocation -S3FieldLocation::S3FieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) : - FieldLocation(uri, offset, length, remapKey) {} - -S3FieldLocation::S3FieldLocation(eckit::Stream& s) : - FieldLocation(s) {} - -std::shared_ptr S3FieldLocation::make_shared() const { - return std::make_shared(std::move(*this)); -} - -eckit::DataHandle* S3FieldLocation::dataHandle() const { - - return eckit::S3Name(uri_).dataHandle(offset()); - -} - -void S3FieldLocation::print(std::ostream &out) const { - out << "S3FieldLocation[uri=" << uri_ << "]"; -} - -void S3FieldLocation::visit(FieldLocationVisitor& visitor) const { - visitor(*this); -} - -static FieldLocationBuilder builder("s3"); - -} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3FieldLocation.h b/src/fdb5/s3/S3FieldLocation.h deleted file mode 100644 index c27604792..000000000 --- a/src/fdb5/s3/S3FieldLocation.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/// @author Nicolau Manubens -/// @date Feb 2024 - -#pragma once - -#include "eckit/io/Length.h" -#include "eckit/io/Offset.h" - -#include "fdb5/database/FieldLocation.h" - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -class S3FieldLocation : public FieldLocation { -public: - - S3FieldLocation(const S3FieldLocation& rhs); - S3FieldLocation(const eckit::URI &uri); - S3FieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); - S3FieldLocation(eckit::Stream&); - - eckit::DataHandle* dataHandle() const override; - - virtual std::shared_ptr make_shared() const override; - - virtual void visit(FieldLocationVisitor& visitor) const override; - -public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} - -protected: // For Streamable - - virtual const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - - static eckit::ClassSpec classSpec_; - static eckit::Reanimator reanimator_; - -private: // methods - - void print(std::ostream &out) const override; - -}; - - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Store.cc b/src/fdb5/s3/S3Store.cc deleted file mode 100644 index acf8ec163..000000000 --- a/src/fdb5/s3/S3Store.cc +++ /dev/null @@ -1,362 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#include - -#include "eckit/runtime/Main.h" -#include "eckit/thread/AutoLock.h" -#include "eckit/thread/StaticMutex.h" -#include "eckit/log/TimeStamp.h" -#include "eckit/utils/MD5.h" -#include "eckit/utils/Tokenizer.h" - -// #include "eckit/config/Resource.h" - -#include "fdb5/s3/S3FieldLocation.h" -#include "fdb5/s3/S3Store.h" - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -static StoreBuilder builder("s3"); - -S3Store::S3Store(const Schema& schema, const Key& key, const Config& config) : - Store(schema), S3Common(config, "store", key), config_(config) { - -} - -S3Store::S3Store(const Schema& schema, const eckit::URI& uri, const Config& config) : - Store(schema), S3Common(config, "store", uri), config_(config) { - -} - -eckit::URI S3Store::uri() const { - - return eckit::S3Bucket(endpoint_, db_bucket_).uri(); - - -/// @note: code for single bucket for all DBs -// TODO -// // warning! here an incomplete uri is being returned. Where is this method -// // being called? Can that caller code accept incomplete uris? -// return eckit::S3Name(endpoint_, bucket_, db_prefix_).URI(); - -} - -bool S3Store::uriBelongs(const eckit::URI& uri) const { - - const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); - - - /// @note: code for bucket per DB - ASSERT(n == 1 || n == 2); - return ( - (uri.scheme() == type()) && - (parts[0] == db_bucket_)); - - - /// @note: code for single bucket for all DBs - // ASSERT(n == 2); - // return ( - // (uri.scheme() == type()) && - // (parts[1].rfind(db_prefix_, 0) == 0)); - -} - -bool S3Store::uriExists(const eckit::URI& uri) const { - - /// @todo: revisit the name of this method - - - /// @note: code for bucket per DB - const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto pn = parts.size(); - ASSERT(pn == 1 | pn == 2); - ASSERT(uri.scheme() == type()); - eckit::S3Bucket n{eckit::net::Endpoint{uri.host(), uri.port()}, parts[0]}; - ASSERT(n.name() == db_bucket_); - return n.exists(); - - - /// @note: code for single bucket for all DBs - // ASSERT(uri.scheme() == type()); - // eckit::S3Name n(uri); - // ASSERT(n.bucket().name() == bucket_); - // ASSERT(n.name().rfind(db_prefix_, 0) == 0); - // return n.exists(); - -} - -std::vector S3Store::storeUnitURIs() const { - - std::vector store_unit_uris; - - eckit::S3Bucket bucket{endpoint_, db_bucket_}; - - if (!bucket.exists()) return store_unit_uris; - - /// @note if an S3Catalogue is implemented, some filtering will need to - /// be done here to discriminate store keys from catalogue keys - for (const auto& key : bucket.listObjects()) { - - store_unit_uris.push_back(key.uri()); - - } - - return store_unit_uris; - - - /// @note: code for single bucket for all DBs - // std::vector store_unit_uris; - - // eckit::S3Bucket bucket{endpoint_, bucket_}; - - // if (!bucket.exists()) return store_unit_uris; - - // /// @note if an S3Catalogue is implemented, more filtering will need to - // /// be done here to discriminate store keys from catalogue keys - // for (const auto& key : bucket.listObjects(filter = "^" + db_prefix_ + "_.*")) { - - // store_unit_uris.push_back(key.uri()); - - // } - - // return store_unit_uris; - -} - -std::set S3Store::asStoreUnitURIs(const std::vector& uris) const { - - std::set res; - - /// @note: this is only uniquefying the input uris (coming from an index) - /// in case theres any duplicate. - for (auto& uri : uris) - res.insert(uri); - - return res; - -} - -bool S3Store::exists() const { - - return eckit::S3Bucket(endpoint_, db_bucket_).exists(); - -} - -/// @todo: never used in actual fdb-read? -eckit::DataHandle* S3Store::retrieve(Field& field) const { - - return field.dataHandle(); - -} - -std::unique_ptr S3Store::archive(const Key& key, const void * data, eckit::Length length) { - - /// @note: code for S3 object (key) per field: - - /// @note: generate unique key name - /// if single bucket, starting by dbkey_indexkey_ - /// if bucket per db, starting by indexkey_ - eckit::S3Name n = generateDataKey(key); - - /// @todo: ensure bucket if not yet seen by this process - static std::set knownBuckets; - if (knownBuckets.find(n.bucket().name()) == knownBuckets.end()) { - n.bucket().ensureCreated(); - knownBuckets.insert(n.bucket().name()); - } - - std::unique_ptr h(n.dataHandle()); - - h->openForWrite(length); - eckit::AutoClose closer(*h); - - h->write(data, length); - - return std::unique_ptr(new S3FieldLocation(n.uri(), 0, length, fdb5::Key())); - - - /// @note: code for S3 object (key) per index store: - - // /// @note: get or generate unique key name - // /// if single bucket, starting by dbkey_indexkey_ - // /// if bucket per db, starting by indexkey_ - // eckit::S3Name n = getDataKey(key); - - // eckit::DataHandle &dh = getDataHandle(key, n); - - // eckit::Offset offset{dh.position()}; - - // h.write(data, length); - - // return std::unique_ptr(new S3FieldLocation(n.URI(), offset, length, fdb5::Key())); - -} - -void S3Store::flush() { - - /// @note: code for S3 object (key) per index store: - - // /// @note: clear cached data handles thus triggering consolidation of - // /// multipart objects, so that step data is made visible to readers. - // /// New S3 handles will be created on the next archive() call after - // /// flush(). - // closeDataHandles(); - -} - -void S3Store::close() { - - /// @note: code for S3 object (key) per index store: - - // closeDataHandles(); - -} - -void S3Store::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { - - /// @note: code for bucket per DB - - const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); - ASSERT(n == 1 | n == 2); - - ASSERT(parts[0] == db_bucket_); - - if (n == 2) { // object - - eckit::S3Name key{uri}; - - logVerbose << "destroy S3 key: " << key.asString() << std::endl; - - if (doit) key.destroy(); - - } else { // pool - - eckit::S3Bucket bucket{uri}; - - logVerbose << "destroy S3 bucket: " << bucket.asString() << std::endl; - - if (doit) bucket.ensureDestroyed(); - } - - - // /// @note: code for single bucket for all DBs - // eckit::S3Name n{uri}; - - // ASSERT(n.bucket().name() == bucket_); - // /// @note: if uri doesn't have key name, maybe this method should return without destroying anything. - // /// this way when TocWipeVisitor has wipeAll == true, the (only) bucket will not be destroyed - // ASSERT(n.name().rfind(db_prefix_, 0) == 0); - - // logVerbose << "destroy S3 key: "; - // logAlways << n.asString() << std::endl; - // if (doit) n.destroy(); - -} - -void S3Store::print(std::ostream& out) const { - - out << "S3Store(" << endpoint_ << "/" << db_bucket_ << ")"; - - /// @note: code for single bucket for all DBs - // out << "S3Store(" << endpoint_ << "/" << bucket_ << ")"; - -} - -/// @note: unique name generation copied from LocalPathName::unique. -static eckit::StaticMutex local_mutex; - -eckit::S3Name S3Store::generateDataKey(const Key& key) const { - - eckit::AutoLock lock(local_mutex); - - std::string hostname = eckit::Main::hostname(); - - static unsigned long long n = (((unsigned long long)::getpid()) << 32); - - static std::string format = "%Y%m%d.%H%M%S"; - std::ostringstream os; - os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; - - std::string name = os.str(); - - while (::access(name.c_str(), F_OK) == 0) { - std::ostringstream os; - os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; - name = os.str(); - } - - eckit::MD5 md5(name); - - std::string keyStr = key.valuesToString(); - std::replace(keyStr.begin(), keyStr.end(), ':', '-'); - - return eckit::S3Name{endpoint_, db_bucket_, keyStr + "." + md5.digest() + ".data"}; - - /// @note: code for single bucket for all DBs - // return eckit::S3Name{endpoint_, bucket_, db_prefix_ + "_" + key.valuesToString() + "_" + md5.digest() + ".data"}; - -} - -/// @note: code for S3 object (key) per index store: -// eckit::S3Name S3Store::getDataKey(const Key& key) const { - -// KeyStore::const_iterator j = dataKeys_.find(key); - -// if ( j != dataKeys_.end() ) -// return j->second; - -// eckit::S3Name dataKey = generateDataKey(key); - -// dataKeys_[ key ] = dataKey; - -// return dataKey; - -// } - -/// @note: code for S3 object (key) per index store: -// eckit::DataHandle& S3Store::getDataHandle(const Key& key, const eckit::S3Name& name) { - -// HandleStore::const_iterator j = handles_.find(key); -// if ( j != handles_.end() ) -// return j->second; - -// eckit::DataHandle *dh = name.dataHandle(multipart = true); - -// ASSERT(dh); - -// handles_[ key ] = dh; - -// dh->openForAppend(0); - -// return *dh; - -// } - -/// @note: code for S3 object (key) per index store: -// void S3Store::closeDataHandles() { - -// for ( HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j ) { -// eckit::DataHandle *dh = j->second; -// dh->close(); -// delete dh; -// } - -// handles_.clear(); - -// } - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/s3/S3Store.h b/src/fdb5/s3/S3Store.h deleted file mode 100644 index 14aa2b43b..000000000 --- a/src/fdb5/s3/S3Store.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/// @author Nicolau Manubens -/// @author Metin Cakircali -/// @author Simon Smart -/// @date Feb 2024 - -#pragma once - -#include "eckit/io/s3/S3Name.h" - -#include "fdb5/database/Store.h" -#include "fdb5/rules/Schema.h" - -#include "fdb5/s3/S3Common.h" - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -class S3Store : public Store, public S3Common { - -public: // methods - - S3Store(const Schema& schema, const Key& key, const Config& config); - S3Store(const Schema& schema, const eckit::URI& uri, const Config& config); - - ~S3Store() override {} - - eckit::URI uri() const override; - bool uriBelongs(const eckit::URI&) const override; - bool uriExists(const eckit::URI&) const override; - std::vector storeUnitURIs() const override; - std::set asStoreUnitURIs(const std::vector&) const override; - - bool open() override { return true; } - void flush() override; - void close() override; - - void checkUID() const override { /* nothing to do */ } - -protected: // methods - - std::string type() const override { return "s3"; } - - bool exists() const override; - - eckit::DataHandle* retrieve(Field& field) const override; - std::unique_ptr archive(const Key& key, const void * data, eckit::Length length) override; - - void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; - - void print(std::ostream &out) const override; - - eckit::S3Name generateDataKey(const Key& key) const; - - /// @note: code for S3 object (key) per index store: - // eckit::S3Name getDataKey(const Key& key) const; - // eckit::DataHandle& getDataHandle(const Key& key, const eckit::S3Name& name); - // void closeDataHandles(); - -private: // types - - /// @note: code for S3 object (key) per index store: - // typedef std::map HandleStore; - // typedef std::map KeyStore; - -private: // members - - const Config& config_; - - /// @note: code for S3 object (key) per index store: - // HandleStore handles_; - // mutable KeyStore dataKeys_; - -}; - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 diff --git a/src/fdb5/toc/FieldRef.cc b/src/fdb5/toc/FieldRef.cc index 967c12d2f..ac2baa304 100644 --- a/src/fdb5/toc/FieldRef.cc +++ b/src/fdb5/toc/FieldRef.cc @@ -19,8 +19,8 @@ #include "fdb5/database/UriStore.h" #include "fdb5/toc/TocFieldLocation.h" -#ifdef fdb5_HAVE_S3FDB -#include "fdb5/s3/S3FieldLocation.h" +#ifdef fdb5_HAVE_RADOSFDB +#include "fdb5/rados/RadosFieldLocation.h" #endif namespace fdb5 { @@ -37,12 +37,12 @@ FieldRefLocation::FieldRefLocation(UriStore &store, const Field& field) { const FieldLocation& loc = field.location(); -#ifdef fdb5_HAVE_S3FDB +#ifdef fdb5_HAVE_RADOSFDB const TocFieldLocation* tocfloc = dynamic_cast(&loc); - const S3FieldLocation* s3floc = dynamic_cast(&loc); - if(!tocfloc && !s3floc) { + const RadosFieldLocation* radosfloc = dynamic_cast(&loc); + if(!tocfloc && !radosfloc) { throw eckit::NotImplemented( - "Field location is not of TocFieldLocation or S3FieldLocation type " + "Field location is not of TocFieldLocation or RadosFieldLocation type " "-- indexing other locations is not supported", Here()); } diff --git a/tests/fdb/s3/CMakeLists.txt b/tests/fdb/s3/CMakeLists.txt deleted file mode 100644 index 7c460fd53..000000000 --- a/tests/fdb/s3/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -if (HAVE_S3FDB) - - list( APPEND s3_tests - s3_store - ) - - list( APPEND unit_test_libraries fdb5 ) - - foreach( _test ${s3_tests} ) - - ecbuild_add_test( TARGET test_fdb5_s3_${_test} - SOURCES test_${_test}.cc - LIBS "${unit_test_libraries}" - INCLUDES "${unit_test_include_dirs}" ) - - endforeach() - -endif() \ No newline at end of file diff --git a/tests/fdb/s3/test_s3_store.cc b/tests/fdb/s3/test_s3_store.cc deleted file mode 100644 index 62ad2645e..000000000 --- a/tests/fdb/s3/test_s3_store.cc +++ /dev/null @@ -1,564 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -// #include -// #include - -// #include "eckit/config/Resource.h" -#include "eckit/testing/Test.h" -// #include "eckit/filesystem/URI.h" -#include "eckit/filesystem/PathName.h" -#include "eckit/filesystem/TmpFile.h" -// #include "eckit/filesystem/TmpDir.h" -// #include "eckit/io/FileHandle.h" -#include "eckit/io/MemoryHandle.h" -#include "eckit/config/YAMLConfiguration.h" - -// #include "metkit/mars/MarsRequest.h" - -// #include "fdb5/fdb5_config.h" -// #include "fdb5/config/Config.h" -#include "fdb5/api/FDB.h" -#include "fdb5/api/helpers/FDBToolRequest.h" - -#include "fdb5/toc/TocCatalogueWriter.h" -#include "fdb5/toc/TocCatalogueReader.h" - -#include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Session.h" -#include "eckit/io/s3/S3Credential.h" -#include "eckit/io/s3/S3Handle.h" - -#include "fdb5/s3/S3Store.h" -#include "fdb5/s3/S3FieldLocation.h" -// #include "fdb5/daos/DaosException.h" - -using namespace eckit::testing; -using namespace eckit; - -namespace { - - void deldir(eckit::PathName& p) { - if (!p.exists()) { - return; - } - - std::vector files; - std::vector dirs; - p.children(files, dirs); - - for (auto& f : files) { - f.unlink(); - } - for (auto& d : dirs) { - deldir(d); - } - - p.rmdir(); - }; - - S3Config cfg("eu-central-1", "127.0.0.1", 9000); - - void ensureClean(const std::string& prefix) { - auto client = S3Client::makeUnique(cfg); - auto&& tmp = client->listBuckets(); - std::set buckets(tmp.begin(), tmp.end()); - - for (const std::string& name : buckets) { - if (name.rfind(prefix, 0) == 0) { - client->emptyBucket(name); - client->deleteBucket(name); - } - } - } -} - -// #ifdef fdb5_HAVE_DUMMY_DAOS -// eckit::TmpDir& tmp_dummy_daos_root() { -// static eckit::TmpDir d{}; -// return d; -// } -// #endif - -// temporary schema,spaces,root files common to all DAOS Store tests - -eckit::TmpFile& schema_file() { - static eckit::TmpFile f{}; - return f; -} - -eckit::TmpFile& spaces_file() { - static eckit::TmpFile f{}; - return f; -} - -eckit::TmpFile& roots_file() { - static eckit::TmpFile f{}; - return f; -} - -eckit::PathName& store_tests_tmp_root() { - static eckit::PathName sd("./s3_store_tests_fdb_root"); - return sd; -} - -namespace fdb { -namespace test { - -CASE( "Setup" ) { - - // ensure fdb root directory exists. If not, then that root is - // registered as non existing and Store tests fail. - if (store_tests_tmp_root().exists()) deldir(store_tests_tmp_root()); - store_tests_tmp_root().mkdir(); - ::setenv("FDB_ROOT_DIRECTORY", store_tests_tmp_root().path().c_str(), 1); - - // prepare schema for tests involving S3Store - - std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; - - std::unique_ptr hs(schema_file().fileHandle()); - hs->openForWrite(schema_str.size()); - { - eckit::AutoClose closer(*hs); - hs->write(schema_str.data(), schema_str.size()); - } - - // this is necessary to avoid ~fdb/etc/fdb/schema being used where - // LibFdb5::instance().defaultConfig().schema() is called - // due to no specified schema file (e.g. in Key::registry()) - ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); - - // prepare scpaces - - std::string spaces_str{".* all Default"}; - - std::unique_ptr hsp(spaces_file().fileHandle()); - hsp->openForWrite(spaces_str.size()); - { - eckit::AutoClose closer(*hsp); - hsp->write(spaces_str.data(), spaces_str.size()); - } - - ::setenv("FDB_SPACES_FILE", spaces_file().path().c_str(), 1); - - // prepare roots - - std::string roots_str{store_tests_tmp_root().asString() + " all yes yes"}; - - std::unique_ptr hr(roots_file().fileHandle()); - hr->openForWrite(roots_str.size()); - { - eckit::AutoClose closer(*hr); - hr->write(roots_str.data(), roots_str.size()); - } - - ::setenv("FDB_ROOTS_FILE", roots_file().path().c_str(), 1); - -} - -CASE("S3Store tests") { - - SECTION("archive and retrieve") { - - std::string prefix{"test1-"}; - - ensureClean(prefix); - - std::string config_str{ - "s3:\n" - " credential:\n" - " accessKeyID: minio\n" - " secretKey: minio1234\n" - " host: 127.0.0.1\n" - " endpoint: 127.0.0.1:9000\n" - " bucketPrefix: " + prefix + "\n" - }; - - fdb5::Config config{YAMLConfiguration(config_str)}; - - fdb5::Schema schema{schema_file()}; - - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key db_key{"a=1,b=2"}; - fdb5::Key index_key{"c=3,d=4"}; - - char data[] = "test"; - - // archive - - fdb5::S3Store s3store{schema, db_key, config}; - fdb5::Store& store = s3store; - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); - - s3store.flush(); - - // retrieve - fdb5::Field field(std::move(loc), std::time(nullptr)); - std::cout << "Read location: " << field.location() << std::endl; - std::unique_ptr dh(store.retrieve(field)); - EXPECT(dynamic_cast(dh.get())); - - eckit::MemoryHandle mh; - dh->copyTo(mh); - EXPECT(mh.size() == eckit::Length(sizeof(data))); - EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - - // remove - eckit::S3Name field_name{field.location().uri()}; - eckit::S3Bucket store_name{field_name.bucket()}; - eckit::URI store_uri(store_name.uri()); - std::ostream out(std::cout.rdbuf()); - store.remove(store_uri, out, out, false); - EXPECT(field_name.exists()); - store.remove(store_uri, out, out, true); - EXPECT_NOT(field_name.exists()); - EXPECT_NOT(store_name.exists()); - - } - - SECTION("with POSIX Catalogue") { - - std::string prefix{"test2-"}; - - ensureClean(prefix); - - // FDB configuration - - std::string config_str{ - "schema : " + schema_file().path() + "\n" - "s3:\n" - " credential:\n" - " accessKeyID: minio\n" - " secretKey: minio1234\n" - " host: 127.0.0.1\n" - " endpoint: 127.0.0.1:9000\n" - " bucketPrefix: " + prefix + "\n" - }; - - fdb5::Config config{YAMLConfiguration(config_str)}; - - // schema - - fdb5::Schema schema{schema_file()}; - - // request - - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key db_key{"a=1,b=2"}; - fdb5::Key index_key{"c=3,d=4"}; - fdb5::Key field_key{"e=5,f=6"}; - - // store data - - char data[] = "test"; - - fdb5::S3Store s3store{schema, db_key, config}; - fdb5::Store& store = static_cast(s3store); - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); - - // index data - - { - /// @todo: could have a unique ptr here, might not need a static cast - fdb5::TocCatalogueWriter tcat{db_key, config}; - fdb5::Catalogue& cat = static_cast(tcat); - cat.deselectIndex(); - cat.selectIndex(index_key); - //const fdb5::Index& idx = tcat.currentIndex(); - static_cast(tcat).archive(field_key, std::move(loc)); - - /// flush store before flushing catalogue - s3store.flush(); // not necessary if using a DAOS store - } - - // find data - - fdb5::Field field; - { - fdb5::TocCatalogueReader tcat{db_key, config}; - fdb5::Catalogue& cat = static_cast(tcat); - cat.selectIndex(index_key); - static_cast(tcat).retrieve(field_key, field); - } - std::cout << "Read location: " << field.location() << std::endl; - - // retrieve data - - std::unique_ptr dh(store.retrieve(field)); - EXPECT(dynamic_cast(dh.get())); - - eckit::MemoryHandle mh; - dh->copyTo(mh); - EXPECT(mh.size() == eckit::Length(sizeof(data))); - EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - - // remove data - - eckit::S3Name field_name{field.location().uri()}; - eckit::S3Bucket store_name{field_name.bucket()}; - eckit::URI store_uri(store_name.uri()); - std::ostream out(std::cout.rdbuf()); - store.remove(store_uri, out, out, false); - EXPECT(field_name.exists()); - store.remove(store_uri, out, out, true); - EXPECT_NOT(field_name.exists()); - EXPECT_NOT(store_name.exists()); - - // deindex data - - { - fdb5::TocCatalogueWriter tcat{db_key, config}; - fdb5::Catalogue& cat = static_cast(tcat); - metkit::mars::MarsRequest r = db_key.request("retrieve"); - std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); - cat.visitEntries(*wv, store, false); - } - - } - - SECTION("VIA FDB API") { - - std::string prefix{"test3-"}; - - ensureClean(prefix); - - // FDB configuration - - std::string config_str{ - "type: local\n" - "schema : " + schema_file().path() + "\n" - "engine: toc\n" - "store: s3\n" - "s3:\n" - " credential:\n" - " accessKeyID: minio\n" - " secretKey: minio1234\n" - " host: 127.0.0.1\n" - " endpoint: 127.0.0.1:9000\n" - " bucketPrefix: " + prefix + "\n" - }; - - fdb5::Config config{YAMLConfiguration(config_str)}; - - // request - - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; - fdb5::Key db_key{"a=1,b=2"}; - - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - - // initialise store - - fdb5::FDB fdb(config); - - // check store is empty - - size_t count; - fdb5::ListElement info; - - auto listObject = fdb.list(db_req); - - count = 0; - while (listObject.next(info)) { - info.print(std::cout, true, true); - std::cout << std::endl; - ++count; - } - EXPECT(count == 0); - - // store data - - char data[] = "test"; - - fdb.archive(request_key, data, sizeof(data)); - - fdb.flush(); - - // retrieve data - - metkit::mars::MarsRequest r = request_key.request("retrieve"); - std::unique_ptr dh(fdb.retrieve(r)); - - eckit::MemoryHandle mh; - dh->copyTo(mh); - EXPECT(mh.size() == eckit::Length(sizeof(data))); - EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - - // wipe data - - fdb5::WipeElement elem; - - // dry run attempt to wipe with too specific request - - auto wipeObject = fdb.wipe(full_req); - count = 0; - while (wipeObject.next(elem)) count++; - EXPECT(count == 0); - - // dry run wipe index and store unit - wipeObject = fdb.wipe(index_req); - count = 0; - while (wipeObject.next(elem)) count++; - EXPECT(count > 0); - - // dry run wipe database - wipeObject = fdb.wipe(db_req); - count = 0; - while (wipeObject.next(elem)) count++; - EXPECT(count > 0); - - // ensure field still exists - listObject = fdb.list(full_req); - count = 0; - while (listObject.next(info)) { - // info.print(std::cout, true, true); - // std::cout << std::endl; - count++; - } - EXPECT(count == 1); - - // attempt to wipe with too specific request - wipeObject = fdb.wipe(full_req, true); - count = 0; - while (wipeObject.next(elem)) count++; - EXPECT(count == 0); - /// @todo: really needed? - fdb.flush(); - - // wipe index and store unit (and DB bucket as there is only one index) - wipeObject = fdb.wipe(index_req, true); - count = 0; - while (wipeObject.next(elem)) count++; - EXPECT(count > 0); - /// @todo: really needed? - fdb.flush(); - - // ensure field does not exist - listObject = fdb.list(full_req); - count = 0; - while (listObject.next(info)) count++; - EXPECT(count == 0); - - } - - /// @todo: if doing what's in this section at the end of the previous section reusing the same FDB object, - // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe - SECTION("FDB API RE-STORE AND WIPE DB") { - - std::string prefix{"test4-"}; - - // FDB configuration - - std::string config_str{ - "type: local\n" - "schema : " + schema_file().path() + "\n" - "engine: toc\n" - "store: s3\n" - "s3:\n" - " credential:\n" - " accessKeyID: minio\n" - " secretKey: minio1234\n" - " host: 127.0.0.1\n" - " endpoint: 127.0.0.1:9000\n" - " bucketPrefix: " + prefix + "\n" - }; - - fdb5::Config config{YAMLConfiguration(config_str)}; - - // request - - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; - fdb5::Key db_key{"a=1,b=2"}; - - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - - // initialise store - - fdb5::FDB fdb(config); - - // store again - - char data[] = "test"; - - fdb.archive(request_key, data, sizeof(data)); - - fdb.flush(); - - size_t count; - - // wipe all database - - fdb5::WipeElement elem; - auto wipeObject = fdb.wipe(db_req, true); - count = 0; - while (wipeObject.next(elem)) count++; - EXPECT(count > 0); - /// @todo: really needed? - fdb.flush(); - - // ensure field does not exist - - fdb5::ListElement info; - auto listObject = fdb.list(full_req); - count = 0; - while (listObject.next(info)) { - // info.print(std::cout, true, true); - // std::cout << std::endl; - count++; - } - EXPECT(count == 0); - - } - -} - -} // namespace test -} // namespace fdb - -int main(int argc, char **argv) -{ - const eckit::S3Credential cred {"minio", "minio1234", "127.0.0.1"}; - eckit::S3Session::instance().addCredentials(cred); - - return run_tests ( argc, argv ); - - ensureClean(""); -} \ No newline at end of file From 677935d72cc1e5b55f219e16b1bf5e419baaa91a Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Thu, 29 Feb 2024 21:58:17 +0100 Subject: [PATCH 010/109] Remove leftover s3 content. --- tests/fdb/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index 7820737e9..3f9748011 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -71,7 +71,6 @@ endforeach() # pmem tests make use of the test environment, so are added at the end add_subdirectory( pmem ) -add_subdirectory( s3 ) add_subdirectory( api ) add_subdirectory( tools ) add_subdirectory( type ) From 5ed2b661a6a775a901f7e1e00f25eedf63e6493d Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Thu, 29 Feb 2024 23:28:06 +0100 Subject: [PATCH 011/109] Remove leftover s3 content. --- src/fdb5/toc/TocWipeVisitor.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index 354a740c6..a8b231fcb 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -12,7 +12,6 @@ #include #include "eckit/os/Stat.h" -#include "eckit/io/s3/S3Bucket.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/database/DB.h" From 07f02fa705315ab928ba8bc012065a1609c17bcf Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 3 Mar 2024 00:30:12 +0100 Subject: [PATCH 012/109] Functional RadosStore backend. Supports several modes: pool per DB, single pool, object per field, object per collocation unit, span multiple objects if maxObjectSize is exceeded, no persist, persist on flush and persist on write. Includes compreunit tests for all modes and instructions to run on docker. --- CMakeLists.txt | 23 + src/fdb5/CMakeLists.txt | 2 + src/fdb5/fdb5_config.h.in | 5 + src/fdb5/rados/README | 110 +++++ src/fdb5/rados/RadosCommon.cc | 113 +++++ src/fdb5/rados/RadosCommon.h | 54 +++ src/fdb5/rados/RadosFieldLocation.cc | 107 +++-- src/fdb5/rados/RadosFieldLocation.h | 46 +- src/fdb5/rados/RadosStore.cc | 584 +++++++++++++++++++---- src/fdb5/rados/RadosStore.h | 78 ++-- tests/fdb/CMakeLists.txt | 1 + tests/fdb/rados/CMakeLists.txt | 18 + tests/fdb/rados/test_rados_store.cc | 675 +++++++++++++++++++++++++++ 13 files changed, 1628 insertions(+), 188 deletions(-) create mode 100644 src/fdb5/rados/README create mode 100644 src/fdb5/rados/RadosCommon.cc create mode 100644 src/fdb5/rados/RadosCommon.h create mode 100644 tests/fdb/rados/CMakeLists.txt create mode 100644 tests/fdb/rados/test_rados_store.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 03d8592f5..57ac4835d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,11 +43,34 @@ ecbuild_add_option( FEATURE PMEMFDB # option present in fdb5_config.h ### FDB backend in CEPH object store (using Rados) find_package( RADOS QUIET ) + ecbuild_add_option( FEATURE RADOSFDB # option defined in fdb5_config.h CONDITION eckit_HAVE_RADOS AND RADOS_FOUND DEFAULT OFF DESCRIPTION "Ceph/Rados support for FDB Store" ) +ecbuild_add_option( FEATURE RADOS_STORE_SINGLE_POOL + DEFAULT ON + DESCRIPTION "Use a single Rados pool with a namespace per database (ON) or a pool per database (OFF)" ) + +ecbuild_add_option( FEATURE RADOS_STORE_OBJ_PER_FIELD + DEFAULT OFF + DESCRIPTION "Use a Rados object per archived field (ON) or per collocation key (OFF)" ) + +ecbuild_add_option( FEATURE RADOS_STORE_MULTIPART + DEFAULT ON + DESCRIPTION "If RADOS_STORE_OBJ_PER_FIELD=OFF and the maximum object size is exceeded, use multiple Rados objects per collocation key (ON) or throw an exception (OFF)" ) + +ecbuild_add_option( FEATURE RADOS_STORE_PERSIST_ON_FLUSH + DEFAULT OFF + DESCRIPTION "Ensure writes are persisted in Rados storage on flush." ) + +ecbuild_add_option( FEATURE RADOS_STORE_PERSIST_ON_WRITE + CONDITION fdb5_HAVE_STORE_OBJ_PER_FIELD AND NOT fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + DEFAULT OFF + DESCRIPTION "If RADOS_STORE_OBJ_PER_FIELD=ON, ensure every object write is persisted immediately." ) + + ### FDB backend in indexed filesystem with table-of-contents, i.e. TOC ### Supports Lustre parallel filesystem stripping control diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8ccf68784..5b193ed18 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -361,6 +361,8 @@ if( HAVE_RADOSFDB ) rados/RadosFieldLocation.h rados/RadosStore.cc rados/RadosStore.h + rados/RadosCommon.cc + rados/RadosCommon.h ) endif() diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index 6fec3c5d5..98c2a8b5d 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -10,6 +10,11 @@ #cmakedefine fdb5_HAVE_LUSTRE #cmakedefine fdb5_HAVE_PMEMFDB #cmakedefine fdb5_HAVE_RADOSFDB +#cmakedefine fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#cmakedefine fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD +#cmakedefine fdb5_HAVE_RADOS_STORE_MULTIPART +#cmakedefine fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH +#cmakedefine fdb5_HAVE_RADOS_STORE_PERSIST_ON_WRITE #cmakedefine fdb5_HAVE_TOCFDB #cmakedefine01 fdb5_HAVE_GRIB diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README new file mode 100644 index 000000000..762bcd451 --- /dev/null +++ b/src/fdb5/rados/README @@ -0,0 +1,110 @@ +Running RadosStore unit tests against Ceph on Docker on mac: +============================================================ + +git clone https://github.com/datenkollektiv/ceph-playground.git +cd ceph-playground +sed -i '' 's#volumes:#volumes:\n - < PATH TO YOUR LOCAL FDB BUNDLE SOURCE >:/root/git/fdb-bundle#g' docker-compose.yaml + +docker-compose down +rm -rf docker/ceph/etc/* +rm -rf docker/ceph/var/* +docker-compose up -d + +docker exec -it ceph-playground_ceph_1 /bin/bash + +# --- + +sed -i -e "s|mirrorlist=|#mirrorlist=|g" -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Linux-* + +yum install -y gcc gcc-c++ gcc-gfortran make cmake openssl openssl-devel git vim +yum update -y libarchive + +ln -s /usr/lib64/librados.so.2 /usr/lib64/librados.so + +cd + +mkdir .ceph +cat /etc/ceph/ceph.conf | grep -e "global" -e "mon host" > .ceph/ceph.conf +ceph config set mon mon_allow_pool_delete true + +git clone https://github.com/ecmwf/ecbuild.git +export PATH=$HOME/ecbuild/bin:$PATH + +mkdir build +cd build +src_dir=$HOME/git/fdb-bundle +build_dir=$HOME/build/fdb-bundle +mkdir -p $build_dir +cd $build_dir +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON -DENABLE_RADOSFDB=ON +cmake --build . + +ctest -R rados_store + + + +cmake options: +============== + +# single pool, multiple fields per obj +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + +# pool per db, multiple fields per obj +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=OFF \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + +# single pool, field per obj +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + +# pool per db, field per obj +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=OFF \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + +# single pool, multiple fields per obj, multipart +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + +# single pool, multiple fields per obj, persist on flush +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=ON \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + +# single pool, field per obj, persist on flush +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=ON \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + +# single pool, field per obj, persist on write +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=ON + +# single pool, multiple fields per obj, multipart, persist on flush +cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ + -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=ON \ + -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF \ No newline at end of file diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc new file mode 100644 index 000000000..61be120f2 --- /dev/null +++ b/src/fdb5/rados/RadosCommon.cc @@ -0,0 +1,113 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/io/rados/RadosObject.h" +// #include "eckit/io/s3/S3Bucket.h" +// #include "eckit/io/s3/S3Credential.h" +// #include "eckit/io/s3/S3Session.h" + +#include "fdb5/rados/RadosCommon.h" + +#include "eckit/exception/Exceptions.h" +// #include "eckit/config/Resource.h" +#include "eckit/utils/Tokenizer.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& component, const fdb5::Key& key) { + + std::vector valid{"catalogue", "store"}; + ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); + + parseConfig(config, component); + + eckit::LocalConfiguration rados{}, comp_conf{}; + + if (config.has("rados")) { + rados = config.getSubConfiguration("rados"); + if (rados.has(component)) comp_conf = rados.getSubConfiguration(component); + } + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + pool_ = rados.getString("pool", pool_); + pool_ = comp_conf.getString("pool", pool_); + + // std::string first_cap{component}; + // first_cap[0] = toupper(component[0]); + // std::string all_caps{component}; + // for (auto & c: all_caps) c = toupper(c); + // bucket_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); + + ASSERT_MSG(pool_.length() > 0, "No pool configured for Rados " + component); + + db_namespace_ = key.valuesToString(); + +#else + + prefix_ = rados.getString("poolPrefix", prefix_); + prefix_ = comp_conf.getString("poolPrefix", prefix_); + ASSERT_MSG(prefix_.find("_") == std::string::npos, "The configured poolPrefix must not contain underscores."); + db_pool_ = prefix_ + "_" + key.valuesToString(); + namespace_ = "default"; + +#endif + +} + +RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { + + /// @note: validity of input URI is not checked here because this constructor is only triggered + /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand + + parseConfig(config, component); + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + eckit::RadosObject o{uri}; + pool_ = o.nspace().pool().name(); + db_namespace_ = o.nspace().name(); + +#else + + eckit::RadosObject o{uri}; + db_pool_ = o.nspace().pool().name(); + namespace_ = o.nspace().name(); + if (namespace_ != "default") + throw eckit::SeriousBug("Unexpected namespace name '" + namespace_ + "'. Expected 'default'."); + const auto parts = eckit::Tokenizer("_").tokenize(db_pool_); + const auto n = parts.size(); + ASSERT(n > 1); + prefix_ = parts[0]; + +#endif + +} + +void RadosCommon::parseConfig(const fdb5::Config& config, const std::string& component) { + + eckit::LocalConfiguration rados{}, comp_conf{}; + + if (config.has("rados")) { + rados = config.getSubConfiguration("rados"); + if (rados.has(component)) comp_conf = rados.getSubConfiguration(component); + } + + maxObjectSize_ = rados.getInt("maxObjectSize", 0); + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h new file mode 100644 index 000000000..af46df65d --- /dev/null +++ b/src/fdb5/rados/RadosCommon.h @@ -0,0 +1,54 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2024 + +#pragma once + +#include "eckit/filesystem/URI.h" + +#include "fdb5/fdb5_config.h" +#include "fdb5/database/Key.h" +#include "fdb5/config/Config.h" + +namespace fdb5 { + +class RadosCommon { + +public: // methods + + RadosCommon(const fdb5::Config&, const std::string& component, const fdb5::Key&); + RadosCommon(const fdb5::Config&, const std::string& component, const eckit::URI&); + +private: // methods + + void parseConfig(const fdb5::Config& config, const std::string& component); + +protected: // members + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + std::string pool_; + std::string db_namespace_; +#else + std::string db_pool_; + std::string namespace_; +#endif + eckit::Length maxObjectSize_; + +#ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +private: // members + + std::string prefix_; +#endif + +}; + +} \ No newline at end of file diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index 9c835769f..bafbbb99f 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -8,10 +8,13 @@ * does it submit to any jurisdiction. */ -#include "eckit/io/rados/RadosReadHandle.h" +#include "eckit/filesystem/URIManager.h" +#include "eckit/io/rados/RadosObject.h" +// #include "eckit/io/rados/RadosMultiObjReadHandle.h" + #include "fdb5/rados/RadosFieldLocation.h" -#include "fdb5/LibFdb5.h" -#include "fdb5/io/SingleGribMungePartFileHandle.h" +// #include "fdb5/LibFdb5.h" +// #include "fdb5/io/SingleGribMungePartFileHandle.h" namespace fdb5 { @@ -20,42 +23,48 @@ ::eckit::Reanimator RadosFieldLocation::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -//RadosFieldLocation::RadosFieldLocation() {} +static FieldLocationBuilder builder("rados"); -RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length ) : - FieldLocation(eckit::URI("rados", path), offset, length) {} +// RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length ) : +// FieldLocation(eckit::URI("rados", path), offset, length) {} -RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri) : - FieldLocation(uri) {} +RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : + FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} -RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length ) : - FieldLocation(uri, offset, length) { -} +RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri) : FieldLocation(uri) {} -RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : - FieldLocation(rhs.uri_) {} +/// @todo: remove remapKey from signature and always pass empty Key to FieldLocation +RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) : + FieldLocation(uri, offset, length, remapKey) {} -RadosFieldLocation::RadosFieldLocation(const FileStore &store, const FieldRef &ref) : - FieldLocation(store.get(ref.pathId()), ref.offset(), ref.length()) {} +// RadosFieldLocation::RadosFieldLocation(const FileStore &store, const FieldRef &ref) : +// FieldLocation(store.get(ref.pathId()), ref.offset(), ref.length()) {} RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) : FieldLocation(s) {} - std::shared_ptr RadosFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); } eckit::DataHandle* RadosFieldLocation::dataHandle() const { - eckit::RadosReadHandle* g = new eckit::RadosReadHandle(uri_.name(), offset(), length()); - return g; -} +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + + return eckit::RadosObject(uri_).multipartRangeReadHandle(offset(), length()); + +#else + + return eckit::RadosObject(uri_).rangeReadHandle(offset(), length()); + +#endif -eckit::DataHandle *RadosFieldLocation::dataHandle(const Key& remapKey) const { - return new SingleGribMungePartFileHandle(path(), offset(), length(), remapKey); } +// eckit::DataHandle *RadosFieldLocation::dataHandle(const Key& remapKey) const { +// return new SingleGribMungePartFileHandle(path(), offset(), length(), remapKey); +// } + void RadosFieldLocation::print(std::ostream &out) const { out << "RadosFieldLocation[uri=" << uri_ << "]"; } @@ -64,12 +73,56 @@ void RadosFieldLocation::visit(FieldLocationVisitor& visitor) const { visitor(*this); } -eckit::URI RadosFieldLocation::uri(const eckit::PathName &path) { - return eckit::URI("rados", path); -} - -static FieldLocationBuilder builder("rados"); +// eckit::URI RadosFieldLocation::uri(const eckit::PathName &path) { +// return eckit::URI("rados", path); +// } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +class RadosURIManager : public eckit::URIManager { + virtual bool query() override { return true; } + virtual bool fragment() override { return true; } + + // virtual eckit::PathName path(const eckit::URI& f) const override { return f.name(); } + + virtual bool exists(const eckit::URI& f) override { + + return eckit::RadosObject(f).exists(); + + } + + virtual eckit::DataHandle* newWriteHandle(const eckit::URI& f) override { + + return eckit::RadosObject(f).dataHandle(); + + } + + virtual eckit::DataHandle* newReadHandle(const eckit::URI& f) override { + + return eckit::RadosObject(f).dataHandle(); + + } + + virtual eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { + + NOTIMP; + + } + + virtual std::string asString(const eckit::URI& uri) const override { + std::string q = uri.query(); + if (!q.empty()) + q = "?" + q; + std::string f = uri.fragment(); + if (!f.empty()) + f = "#" + f; + + return uri.scheme() + ":" + uri.name() + q + f; + } +public: + RadosURIManager(const std::string& name) : eckit::URIManager(name) {} +}; + +static RadosURIManager rados_uri_manager("rados"); + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h index b8f308549..0a5c1dc28 100644 --- a/src/fdb5/rados/RadosFieldLocation.h +++ b/src/fdb5/rados/RadosFieldLocation.h @@ -9,18 +9,19 @@ */ /// @author Emanuele Danovaro -/// @date Jan 2020 +/// @author Nicolau Manubens +/// @date Feb 2024 -#ifndef fdb5_RadosFieldLocation_H -#define fdb5_RadosFieldLocation_H +#pragma once -#include "eckit/filesystem/PathName.h" +// #include "eckit/filesystem/PathName.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" +#include "fdb5/fdb5_config.h" #include "fdb5/database/FieldLocation.h" -#include "fdb5/database/FileStore.h" -#include "fdb5/toc/FieldRef.h" +// #include "fdb5/database/FileStore.h" +// #include "fdb5/toc/FieldRef.h" namespace fdb5 { @@ -29,21 +30,15 @@ namespace fdb5 { class RadosFieldLocation : public FieldLocation { public: - //RadosFieldLocation(); RadosFieldLocation(const RadosFieldLocation& rhs); - RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length); + // RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length); RadosFieldLocation(const eckit::URI &uri); - RadosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length); - RadosFieldLocation(const FileStore& store, const FieldRef& ref); + RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); + // RadosFieldLocation(const FileStore& store, const FieldRef& ref); RadosFieldLocation(eckit::Stream&); -// const eckit::PathName path() const { return uri_.name(); } -// const eckit::Offset& offset() const { return offset_; } - eckit::DataHandle* dataHandle() const override; - eckit::DataHandle* dataHandle(const Key& remapKey) const override; - - // eckit::URI uri() const override; + // eckit::DataHandle* dataHandle(const Key& remapKey) const override; virtual std::shared_ptr make_shared() const override; @@ -51,35 +46,24 @@ class RadosFieldLocation : public FieldLocation { public: // For Streamable - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_;} protected: // For Streamable virtual const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - //virtual void encode(eckit::Stream&) const override; - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods -// void dump(std::ostream &out) const override; - void print(std::ostream &out) const override; - eckit::URI uri(const eckit::PathName &path); + // eckit::URI uri(const eckit::PathName &path); -private: // members - -// eckit::PathName path_; -// eckit::Offset offset_; - - // For streamability }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 - -#endif // fdb5_RadosFieldLocation_H +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index c65c34fac..69b26a93f 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -8,173 +8,513 @@ * does it submit to any jurisdiction. */ -#include "eckit/log/Timer.h" -#include "eckit/log/Bytes.h" +// #include -#include "eckit/config/Resource.h" -#include "eckit/io/EmptyHandle.h" -#include "eckit/io/rados/RadosWriteHandle.h" +#include "eckit/runtime/Main.h" +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/StaticMutex.h" +#include "eckit/log/TimeStamp.h" +#include "eckit/utils/MD5.h" +#include "eckit/utils/Tokenizer.h" + +// // #include "eckit/config/Resource.h" + +#include "eckit/io/rados/RadosPool.h" +#include "eckit/io/rados/RadosNamespace.h" -#include "fdb5/LibFdb5.h" -#include "fdb5/rules/Rule.h" -#include "fdb5/database/FieldLocation.h" #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rados/RadosStore.h" -#include "fdb5/io/FDBFileHandle.h" -using namespace eckit; +// #include "eckit/log/Timer.h" +// #include "eckit/log/Bytes.h" + +// #include "eckit/io/EmptyHandle.h" +// #include "eckit/io/rados/RadosMultiObjWriteHandle.h" + +// #include "fdb5/LibFdb5.h" +// #include "fdb5/rules/Rule.h" +// #include "fdb5/database/FieldLocation.h" +// #include "fdb5/io/FDBFileHandle.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +static StoreBuilder builder("rados"); + RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : - Store(schema), directory_("mars:"+key.valuesToString()) {} + Store(schema), RadosCommon(config, "store", key), config_(config) { + + parseConfig(config_); + +} RadosStore::RadosStore(const Schema& schema, const eckit::URI& uri, const Config& config) : - Store(schema), directory_("mars:"+uri.path().dirName()) {} + Store(schema), RadosCommon(config, "store", uri), config_(config) { + + parseConfig(config_); + +} eckit::URI RadosStore::uri() const { - return URI("rados", directory_); + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + return eckit::RadosNamespace(pool_, db_namespace_).uri(); + +#else + + return eckit::RadosPool(db_pool_).uri(); + +#endif + +} + +bool RadosStore::uriBelongs(const eckit::URI& uri) const { + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + ASSERT(n == 2 || n == 3); + return ( + (uri.scheme() == type()) && + (parts[0] == pool_) && + (parts[1] == db_namespace_)); + +#else + + ASSERT(n == 2 || n == 3); + return ( + (uri.scheme() == type()) && + (parts[0] == db_pool_) && + (parts[1] == namespace_)); + +#endif + +} + +bool RadosStore::uriExists(const eckit::URI& uri) const { + + /// @todo: revisit the name of this method + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + + ASSERT(uri.scheme() == type()); + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + ASSERT(n == 2 || n == 3); + ASSERT(parts[0] == pool_); + ASSERT(parts[1] == db_namespace_); + + if (n == 2) return eckit::RadosNamespace(uri).exists(); + +#else + + ASSERT(n == 1 || n == 3); + ASSERT(parts[0] == db_pool_); + if (n > 1) ASSERT(parts[1] == namespace_); + + if (n == 1) return eckit::RadosPool(uri).exists(); + +#endif + + return eckit::RadosObject(uri).exists(); + +} + +std::vector RadosStore::storeUnitURIs() const { + + std::vector store_unit_uris; + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + eckit::RadosNamespace n{pool_, db_namespace_}; + +#else + + eckit::RadosNamespace n{db_pool_, namespace_}; + +#endif + + if (!n.exists()) return store_unit_uris; + + /// @note if a RadosCatalogue is implemented, some filtering will need to + /// be done here to discriminate store objects from catalogue objects + for (const auto& obj : n.listObjects()) { + +#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART + if (obj.name().find(";part-") != std::string::npos) continue; +#endif + + store_unit_uris.push_back(obj.uri()); + + } + + return store_unit_uris; + +} + +std::set RadosStore::asStoreUnitURIs(const std::vector& uris) const { + + std::set res; + + /// @note: this is only uniquefying the input uris (coming from an index) + /// in case theres any duplicate. + for (auto& uri : uris) + res.insert(uri); + + return res; + } bool RadosStore::exists() const { - return true; + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + return eckit::RadosNamespace(pool_, db_namespace_).exists(); + +#else + + return eckit::RadosNamespace(db_pool_, namespace_).exists(); + +#endif + } -eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const { - return remapKey.empty() ? - field.dataHandle() : - field.dataHandle(remapKey); +/// @todo: never used in actual fdb-read? +eckit::DataHandle* RadosStore::retrieve(Field& field) const { + + return field.dataHandle(); + + // return remapKey.empty() ? + // field.dataHandle() : + // field.dataHandle(remapKey); + } -FieldLocation* RadosStore::archive(const Key &key, const void *data, eckit::Length length) { - dirty_ = true; +std::unique_ptr RadosStore::archive(const Key& key, const void * data, eckit::Length length) { + +#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - eckit::PathName dataPath = getDataPath(key); - eckit::URI dataUri("rados", dataPath); + /// @note: generate unique object name starting by indexkey_ + eckit::RadosObject o = generateDataObject(key); + + #ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + /// @todo: ensure pool if not yet seen by this process + static std::set knownPools; + const eckit::RadosPool& p = o.nspace().pool(); + if (knownPools.find(p.name()) == knownPools.end()) { + p.ensureCreated(); + knownPools.insert(p.name()); + } - eckit::DataHandle &dh = getDataHandle(dataPath); + #endif - eckit::Offset position = dh.position(); + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + eckit::DataHandle* h = o.persistentDataHandle(); + ASSERT(handles_.size() < maxHandleBuffSize_); + handles_.push_back(h); + #elif fdb5_HAVE_RADOS_STORE_PERSIST_ON_WRITE + std::unique_ptr h(o.persistentDataHandle(true)); + #else + std::unique_ptr h(o.dataHandle()); + #endif - long len = dh.write( data, length ); + /// @todo: should throw here if object already exists + + h->openForWrite(length); + eckit::AutoClose closer(*h); + + h->write(data, length); + + + return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key())); + +#else + + /// @note: get or generate unique key name + const eckit::RadosObject& o = getDataObject(key); + + #ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + /// @todo: ensure pool if not yet seen by this process + static std::set knownPools; + const eckit::RadosPool& p = o.nspace().pool(); + if (knownPools.find(p.name()) == knownPools.end()) { + p.ensureCreated(); + knownPools.insert(p.name()); + } + + #endif + + eckit::DataHandle &h = getDataHandle(key, o); + + eckit::Offset offset{h.position()}; + + long len = h.write(data, length); ASSERT(len == length); - return new RadosFieldLocation(dataUri, position, length); + return std::unique_ptr(new RadosFieldLocation(o.uri(), offset, length, fdb5::Key())); + +#endif + } void RadosStore::flush() { - if (!dirty_) { - return; - } - // ensure consistent state before writing Toc entry +#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD + + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + for (const auto& h : handles_) h->flush(); + #else + // NOOP + #endif + +#else + + #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH flushDataHandles(); + #endif + closeDataHandles(); + + #else + + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + flushDataHandles(); + #else + // NOOP + #endif + + #endif + +#endif - dirty_ = false; } void RadosStore::close() { + +#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD + + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + for (const auto& h : handles_) h->close(); + #else + // NOOP + #endif + +#else + closeDataHandles(); + +#endif + } void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { - ASSERT(uri.scheme() == type()); - eckit::PathName path = uri.path(); - if (path.isDir()) { - logVerbose << "rmdir: "; - logAlways << path << std::endl; - if (doit) path.rmdir(false); - } else { - logVerbose << "Unlinking: "; - logAlways << path << std::endl; - if (doit) path.unlink(false); + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + ASSERT(n == 2 || n == 3); + + ASSERT(parts[0] == pool_); + ASSERT(parts[1] == db_namespace_); + + if (n == 2) { // namespace + + eckit::RadosNamespace ns{uri}; + + logVerbose << "destroy Rados namespace: " << ns.str() << std::endl; + + if (doit) ns.destroy(); /// @todo: ensureDestroyed? + + } else { // object + + eckit::RadosObject obj{uri}; + + logVerbose << "destroy Rados object: " << obj.str() << std::endl; + + #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + if (doit) obj.ensureAllDestroyed(); + #else + if (doit) obj.ensureDestroyed(); + #endif + } -} -eckit::DataHandle *RadosStore::getCachedHandle( const eckit::PathName &path ) const { - HandleStore::const_iterator j = handles_.find( path ); - if ( j != handles_.end() ) - return j->second; - else - return nullptr; -} +#else + + ASSERT(n == 1 || n == 3); + + ASSERT(parts[0] == db_pool_); + + if (n == 1) { // pool + + eckit::RadosPool pool{uri}; + + logVerbose << "destroy Rados pool: " << pool.name() << std::endl; + + if (doit) pool.ensureDestroyed(); + + } else { // object + + ASSERT(parts[1] == "default"); + + eckit::RadosObject obj{uri}; + + logVerbose << "destroy Rados object: " << obj.str() << std::endl; + + #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + if (doit) obj.ensureAllDestroyed(); + #else + if (doit) obj.ensureDestroyed(); + #endif -void RadosStore::closeDataHandles() { - for ( HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j ) { - eckit::DataHandle *dh = j->second; - dh->close(); - delete dh; } - handles_.clear(); + +#endif + } -eckit::DataHandle *RadosStore::createFileHandle(const eckit::PathName &path) { +void RadosStore::print(std::ostream& out) const { -// static size_t sizeBuffer = eckit::Resource("fdbBufferSize", 64 * 1024 * 1024); - eckit::Log::debug() << "Creating RadosWriteHandle to " << path -// << " with buffer of " << eckit::Bytes(sizeBuffer) - << std::endl; +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL - return new RadosWriteHandle(path, 0); -} + out << "RadosStore(" << pool_ << "/" << db_namespace_ << ")"; + +#else -eckit::DataHandle *RadosStore::createAsyncHandle(const eckit::PathName &path) { - NOTIMP; + out << "RadosStore(" << db_pool_ << "/" << namespace_ << ")"; -/* static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); - static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); +#endif - return new eckit::AIOHandle(path, nbBuffers, sizeBuffer);*/ } -eckit::DataHandle *RadosStore::createDataHandle(const eckit::PathName &path) { +/// @note: unique name generation copied from LocalPathName::unique. +static eckit::StaticMutex local_mutex; - static bool fdbWriteToNull = eckit::Resource("fdbWriteToNull;$FDB_WRITE_TO_NULL", false); - if(fdbWriteToNull) - return new eckit::EmptyHandle(); +eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { - static bool fdbAsyncWrite = eckit::Resource("fdbAsyncWrite;$FDB_ASYNC_WRITE", false); - if(fdbAsyncWrite) - return createAsyncHandle(path); + eckit::AutoLock lock(local_mutex); - return createFileHandle(path); -} + std::string hostname = eckit::Main::hostname(); + + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + + std::string name = os.str(); -eckit::DataHandle& RadosStore::getDataHandle( const eckit::PathName &path ) { - eckit::DataHandle *dh = getCachedHandle(path); - if ( !dh ) { - dh = createDataHandle(path); - ASSERT(dh); - handles_[path] = dh; - dh->openForWrite(0); + while (::access(name.c_str(), F_OK) == 0) { + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + name = os.str(); } - return *dh; -} -eckit::PathName RadosStore::generateDataPath(const Key &key) const { + eckit::MD5 md5(name); + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + + #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD + + return eckit::RadosObject{pool_, db_namespace_, md5.digest()}; + + #else + + return eckit::RadosObject{pool_, db_namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; + + #endif + +#else + + #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD + + return eckit::RadosObject{db_pool_, namespace_, md5.digest()}; + + #else + + return eckit::RadosObject{db_pool_, namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; + + #endif + +#endif - eckit::PathName dpath ( directory_ ); - dpath /= key.valuesToString(); - dpath = eckit::PathName::unique(dpath) + ".data"; - return dpath; } -eckit::PathName RadosStore::getDataPath(const Key &key) { - PathStore::const_iterator j = dataPaths_.find(key); - if ( j != dataPaths_.end() ) +#ifndef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD + +const eckit::RadosObject& RadosStore::getDataObject(const Key& key) const { + + ObjectStore::const_iterator j = dataObjects_.find(key); + + if ( j != dataObjects_.end() ) return j->second; - eckit::PathName dataPath = generateDataPath(key); + // eckit::RadosObject dataObject = generateDataObject(key); + + dataObjects_.insert(std::pair(key, generateDataObject(key))); + + return dataObjects_.find(key)->second; + +} + +eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosObject& name) { + + HandleStore::const_iterator j = handles_.find(key); + if ( j != handles_.end() ) + return *(j->second); + + #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART + + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + eckit::DataHandle *dh = name.persistentMultipartWriteHandle(maxObjectSize_, maxAioBuffSize_, maxPartHandleBuffSize_); + #else + eckit::DataHandle *dh = name.multipartWriteHandle(maxObjectSize_); + #endif + + #else + + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + eckit::DataHandle *dh = name.persistentDataHandle(false, maxAioBuffSize_); + #else + eckit::DataHandle *dh = name.dataHandle(); + #endif - dataPaths_[ key ] = dataPath; + #endif + + ASSERT(dh); + + handles_[ key ] = dh; + + dh->openForWrite(0); + + return *dh; + +} + +void RadosStore::closeDataHandles() { + + for ( HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j ) { + eckit::DataHandle *dh = j->second; + dh->close(); + delete dh; + } + + handles_.clear(); + dataObjects_.clear(); - return dataPath; } void RadosStore::flushDataHandles() { @@ -185,12 +525,54 @@ void RadosStore::flushDataHandles() { } } -void RadosStore::print(std::ostream &out) const { - out << "RadosStore(" << directory_ << ")"; -} +#endif -static StoreBuilder builder("rados"); +// eckit::DataHandle *RadosStore::createAsyncHandle(const eckit::PathName &path) { +// NOTIMP; + +// /* static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); +// static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); + +// return new eckit::AIOHandle(path, nbBuffers, sizeBuffer);*/ +// } + +// eckit::DataHandle *RadosStore::createDataHandle(const eckit::PathName &path) { + +// static bool fdbWriteToNull = eckit::Resource("fdbWriteToNull;$FDB_WRITE_TO_NULL", false); +// if(fdbWriteToNull) +// return new eckit::EmptyHandle(); + +// static bool fdbAsyncWrite = eckit::Resource("fdbAsyncWrite;$FDB_ASYNC_WRITE", false); +// if(fdbAsyncWrite) +// return createAsyncHandle(path); + +// return new RadosMultiObjWriteHandle(path, 0); +// } + +void RadosStore::parseConfig(const fdb5::Config& config) { + + eckit::LocalConfiguration rados{}, store_conf{}; + + if (config.has("rados")) { + rados = config.getSubConfiguration("rados"); + if (rados.has("store")) store_conf = rados.getSubConfiguration("store"); + } + +#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) && defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) + maxHandleBuffSize_ = store_conf.getInt("maxHandleBuffSize", 1024 * 1024); +#endif + +#if (!defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD)) && defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) + #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART + maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024); + maxPartHandleBuffSize_ = store_conf.getInt("maxPartHandleBuffSize", 1024); + #else + maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024 * 1024); + #endif +#endif + +} //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index b35189be6..3e6a21f60 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -8,25 +8,28 @@ * does it submit to any jurisdiction. */ -/// @file RadosStore.h /// @author Emanuele Danovaro -/// @date Jan 2020 +/// @author Nicolau Manubens +/// @date Feb 2024 -#ifndef fdb5_RadosStore_H -#define fdb5_RadosStore_H +#pragma once + +#include "eckit/io/rados/RadosObject.h" + +#include "fdb5/fdb5_config.h" -#include "fdb5/database/DB.h" -#include "fdb5/database/Index.h" #include "fdb5/database/Store.h" #include "fdb5/rules/Schema.h" +#include "fdb5/rados/RadosCommon.h" + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- /// Store that implements the FDB on CEPH object store -class RadosStore : public Store { +class RadosStore : public Store, public RadosCommon { public: // methods @@ -36,6 +39,10 @@ class RadosStore : public Store { ~RadosStore() override {} eckit::URI uri() const override; + bool uriBelongs(const eckit::URI&) const override; + bool uriExists(const eckit::URI&) const override; + std::vector storeUnitURIs() const override; + std::set asStoreUnitURIs(const std::vector&) const override; bool open() override { return true; } void flush() override; @@ -49,40 +56,53 @@ class RadosStore : public Store { bool exists() const override; - eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override; - FieldLocation* archive(const Key &key, const void *data, eckit::Length length) override; + eckit::DataHandle* retrieve(Field& field) const override; + std::unique_ptr archive(const Key& key, const void * data, eckit::Length length) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; - eckit::DataHandle *getCachedHandle( const eckit::PathName &path ) const; + void print(std::ostream &out) const override; + + void parseConfig(const fdb5::Config& config); + + eckit::RadosObject generateDataObject(const Key& key) const; + +#ifndef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD + const eckit::RadosObject& getDataObject(const Key& key) const; + eckit::DataHandle& getDataHandle(const Key& key, const eckit::RadosObject& name); void closeDataHandles(); - eckit::DataHandle *createFileHandle(const eckit::PathName &path); - eckit::DataHandle *createAsyncHandle(const eckit::PathName &path); - eckit::DataHandle *createDataHandle(const eckit::PathName &path); - eckit::DataHandle& getDataHandle( const eckit::PathName &path ); - eckit::PathName generateDataPath(const Key &key) const; - eckit::PathName getDataPath(const Key &key); void flushDataHandles(); - void print( std::ostream &out ) const override; - private: // types - typedef std::map< std::string, eckit::DataHandle * > HandleStore; - typedef std::map< Key, std::string > PathStore; + typedef std::map HandleStore; + typedef std::map ObjectStore; +#endif private: // members + + const Config& config_; + + // mutable bool dirty_; + +#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + std::vector handles_; + size_t maxHandleBuffSize_; + #endif +#else + HandleStore handles_; + mutable ObjectStore dataObjects_; + #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + size_t maxAioBuffSize_; + #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART + size_t maxPartHandleBuffSize_; + #endif + #endif +#endif - HandleStore handles_; ///< stores the DataHandles being used by the Session - - PathStore dataPaths_; - eckit::PathName directory_; - - mutable bool dirty_; }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 - -#endif //fdb5_RadosStore_H +} // namespace fdb5 \ No newline at end of file diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index 3f9748011..b6f1a2278 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -74,3 +74,4 @@ add_subdirectory( pmem ) add_subdirectory( api ) add_subdirectory( tools ) add_subdirectory( type ) +add_subdirectory( rados ) diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt new file mode 100644 index 000000000..4b311927b --- /dev/null +++ b/tests/fdb/rados/CMakeLists.txt @@ -0,0 +1,18 @@ +if (HAVE_RADOSFDB) + + list( APPEND rados_tests + rados_store + ) + + list( APPEND unit_test_libraries fdb5 ) + + foreach( _test ${rados_tests} ) + + ecbuild_add_test( TARGET test_fdb5_rados_${_test} + SOURCES test_${_test}.cc + LIBS "${unit_test_libraries}" + INCLUDES "${unit_test_include_dirs}" ) + + endforeach() + +endif() \ No newline at end of file diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc new file mode 100644 index 000000000..bba80fca4 --- /dev/null +++ b/tests/fdb/rados/test_rados_store.cc @@ -0,0 +1,675 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +// #include +// #include + +// #include "eckit/config/Resource.h" +#include "eckit/testing/Test.h" +// #include "eckit/filesystem/URI.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +// #include "eckit/filesystem/TmpDir.h" +// #include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/config/YAMLConfiguration.h" + +// #include "metkit/mars/MarsRequest.h" + +#include "fdb5/fdb5_config.h" +// #include "fdb5/config/Config.h" +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" + +#include "fdb5/toc/TocCatalogueWriter.h" +#include "fdb5/toc/TocCatalogueReader.h" + +// #include "eckit/io/s3/S3Client.h" +// #include "eckit/io/s3/S3Session.h" +// #include "eckit/io/s3/S3Credential.h" +#include "eckit/io/PartHandle.h" + +#include "fdb5/rados/RadosStore.h" +#include "fdb5/rados/RadosFieldLocation.h" +// #include "fdb5/daos/DaosException.h" + +using namespace eckit::testing; +using namespace eckit; + +namespace { + + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; + + // S3Config cfg("eu-central-1", "127.0.0.1", 8888); + + void ensureClean(const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listPools()) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosPool{name}.destroy(); + } + } + } + +} + +// temporary schema,spaces,root files common to all DAOS Store tests + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& spaces_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& roots_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& store_tests_tmp_root() { + static eckit::PathName sd("./rados_store_tests_fdb_root"); + return sd; +} + +namespace fdb { +namespace test { + +CASE( "Setup" ) { + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Store tests fail. + if (store_tests_tmp_root().exists()) deldir(store_tests_tmp_root()); + store_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", store_tests_tmp_root().path().c_str(), 1); + + // prepare schema for tests involving S3Store + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); + + // prepare scpaces + + std::string spaces_str{".* all Default"}; + + std::unique_ptr hsp(spaces_file().fileHandle()); + hsp->openForWrite(spaces_str.size()); + { + eckit::AutoClose closer(*hsp); + hsp->write(spaces_str.data(), spaces_str.size()); + } + + ::setenv("FDB_SPACES_FILE", spaces_file().path().c_str(), 1); + + // prepare roots + + std::string roots_str{store_tests_tmp_root().asString() + " all yes yes"}; + + std::unique_ptr hr(roots_file().fileHandle()); + hr->openForWrite(roots_str.size()); + { + eckit::AutoClose closer(*hr); + hr->write(roots_str.data(), roots_str.size()); + } + + ::setenv("FDB_ROOTS_FILE", roots_file().path().c_str(), 1); + +} + +CASE("RadosStore tests") { + + SECTION("archive and retrieve") { + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + std::string pool{"fdb-test1"}; + + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer + + std::string config_str{ + "rados:\n" + " store:\n" + " pool: " + pool + "\n" + }; +#else + std::string prefix{"fdb-test1"}; + + ensureClean(prefix); + + std::string config_str{ + "rados:\n" + " poolPrefix: " + prefix + "\n" + }; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Schema schema{schema_file()}; + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key db_key{"a=1,b=2"}; + fdb5::Key index_key{"c=3,d=4"}; + + char data[] = "test"; + + // archive + + fdb5::RadosStore rados_store{schema, db_key, config}; + fdb5::Store& store = rados_store; + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + rados_store.flush(); + + // retrieve + fdb5::Field field(std::move(loc), std::time(nullptr)); + std::cout << "Read location: " << field.location() << std::endl; + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + /// @todo: if multiparts is enabled, RadosMultiObjReadHandle + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // remove +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + eckit::RadosObject field_name{field.location().uri()}; + eckit::RadosNamespace store_name = field_name.nspace(); + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT(store_name.listObjects().size() == 0); +#else + eckit::RadosObject field_name{field.location().uri()}; + eckit::RadosPool store_name = field_name.nspace().pool(); + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT_NOT(store_name.exists()); +#endif + + } + + SECTION("with POSIX Catalogue") { + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + std::string pool{"fdb-test2"}; + + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer + + std::string config_str{ + "schema : " + schema_file().path() + "\n" + "rados:\n" + " store:\n" + " pool: " + pool + "\n" + }; +#else + std::string prefix{"fdb-test2"}; + + ensureClean(prefix); + + std::string config_str{ + "schema : " + schema_file().path() + "\n" + "rados:\n" + " poolPrefix: " + prefix + "\n" + }; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key db_key{"a=1,b=2"}; + fdb5::Key index_key{"c=3,d=4"}; + fdb5::Key field_key{"e=5,f=6"}; + + // store data + + char data[] = "test"; + + fdb5::RadosStore rados_store{schema, db_key, config}; + fdb5::Store& store = static_cast(rados_store); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + // index data + + { + /// @todo: could have a unique ptr here, might not need a static cast + fdb5::TocCatalogueWriter tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.deselectIndex(); + cat.selectIndex(index_key); + //const fdb5::Index& idx = tcat.currentIndex(); + static_cast(tcat).archive(field_key, std::move(loc)); + + /// flush store before flushing catalogue + rados_store.flush(); + } + + // find data + + fdb5::Field field; + { + fdb5::TocCatalogueReader tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.selectIndex(index_key); + static_cast(tcat).retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + /// @todo: if multiparts is enabled, RadosMultiObjReadHandle + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // remove data +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + eckit::RadosObject field_name{field.location().uri()}; + eckit::RadosNamespace store_name{field_name.nspace()}; + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT(store_name.listObjects().size() == 0); +#else + eckit::RadosObject field_name{field.location().uri()}; + eckit::RadosPool store_name = field_name.nspace().pool(); + eckit::URI store_uri(store_name.uri()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT_NOT(store_name.exists()); +#endif + + // deindex data + + { + fdb5::TocCatalogueWriter tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + metkit::mars::MarsRequest r = db_key.request("retrieve"); + std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + cat.visitEntries(*wv, store, false); + } + + } + + SECTION("VIA FDB API") { + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + std::string pool{"fdb-test3"}; + + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer +#else + std::string prefix{"fdb-test3"}; + + ensureClean(prefix); +#endif + + std::string config_str{ + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: toc\n" + "store: rados\n" + "rados:\n" + }; + +#ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + config_str += " poolPrefix: " + prefix + "\n"; +#endif + +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + config_str += " maxObjectSize: 16\n"; +#endif + + config_str += " store:\n"; + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + config_str += " pool: " + pool + "\n"; +#endif + +#if defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) + #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + config_str += " maxHandleBuffSize: 100\n"; + #else + #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART + config_str += " maxAioBuffSize: 10\n"; + config_str += " maxPartHandleBuffSize: 10\n"; + #else + config_str += " maxAioBuffSize: 100\n"; + #endif + #endif +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; + fdb5::Key db_key{"a=1,b=2"}; + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + + // initialise store + + fdb5::FDB fdb(config); + + // check store is empty + + size_t count; + fdb5::ListElement info; + + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // store data + + char data[] = "test123456"; + +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + /// @note: maxObjectSize is set to 16, and four 10-byte fields are archived, spanning 3 objects + for (int i = 0; i < 4; i++) { + std::cout << "Archive field " << i << std::endl; + fdb5::Key request_key_i{std::string("a=1,b=2,c=3,d=4,e=5,f=") + std::to_string(6 + i)}; + fdb.archive(request_key_i, data, sizeof(data)); + } +#else + fdb.archive(request_key, data, sizeof(data)); +#endif + + fdb.flush(); + + // retrieve data + +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + for (int i = 0; i < 4; i++) { + std::cout << "Retrieve field " << i << std::endl; + fdb5::Key request_key_i{std::string("a=1,b=2,c=3,d=4,e=5,f=") + std::to_string(6 + i)}; + metkit::mars::MarsRequest r_i = request_key_i.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r_i)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + } +#else + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); +#endif + + // wipe data + + fdb5::WipeElement elem; + + // dry run attempt to wipe with too specific request + + auto wipeObject = fdb.wipe(full_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + + // dry run wipe index and store unit + wipeObject = fdb.wipe(index_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // dry run wipe database + wipeObject = fdb.wipe(db_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // ensure field still exists + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // attempt to wipe with too specific request + wipeObject = fdb.wipe(full_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + /// @todo: really needed? + fdb.flush(); + + // wipe index and store unit (and DB pool or namespace as there is only one index) + wipeObject = fdb.wipe(index_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) count++; + EXPECT(count == 0); + + } + + /// @todo: if doing what's in this section at the end of the previous section reusing the same FDB object, + // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe + SECTION("FDB API RE-STORE AND WIPE DB") { + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + std::string pool{"fdb-test4"}; + + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer +#else + std::string prefix{"fdb-test4"}; + + ensureClean(prefix); +#endif + + std::string config_str{ + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: toc\n" + "store: rados\n" + "rados:\n" + }; + +#ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + config_str += " poolPrefix: " + prefix + "\n"; +#endif + +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + config_str += " maxObjectSize: 16\n"; +#endif + + config_str += " store:\n"; + +#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + config_str += " pool: " + pool + "\n"; +#endif + +#if defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) + #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + config_str += " maxHandleBuffSize: 100\n"; + #else + #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART + config_str += " maxAioBuffSize: 10\n"; + config_str += " maxPartHandleBuffSize: 10\n"; + #else + config_str += " maxAioBuffSize: 100\n"; + #endif + #endif +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; + fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; + fdb5::Key db_key{"a=1,b=2"}; + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + + // initialise store + + fdb5::FDB fdb(config); + + // store again + + char data[] = "test"; + + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + size_t count; + + // wipe all database + + fdb5::WipeElement elem; + auto wipeObject = fdb.wipe(db_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + + fdb5::ListElement info; + auto listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 0); + + } + +} + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + + int ret = -1; + + try { + ret = run_tests ( argc, argv ); + } catch(...) {} + + ensureClean("fdb-test"); + + return ret; +} \ No newline at end of file From 1f3f15a89d6e41ed88a75bebf0ec34e0fce7a612 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 23 Jun 2024 01:02:51 +0200 Subject: [PATCH 013/109] Fixes to enable DAOS backend unit tests by default. --- CMakeLists.txt | 2 +- cmake/FindUUID.cmake | 4 ++++ src/fdb5/daos/DaosStore.cc | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 341b5412f..69dad7799 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,7 +122,7 @@ ecbuild_add_option( FEATURE DAOSFDB DESCRIPTION "DAOS support for FDB Store" ) ecbuild_add_option( FEATURE DAOS_ADMIN - DEFAULT OFF + DEFAULT ${_default_dummy_daos} CONDITION HAVE_DAOSFDB AND DAOS_TESTS_FOUND DESCRIPTION "Add features for DAOS pool management. Removes need to manually create a pool for DAOS unit tests" ) diff --git a/cmake/FindUUID.cmake b/cmake/FindUUID.cmake index 0072ed0b0..e2b4bfbcd 100644 --- a/cmake/FindUUID.cmake +++ b/cmake/FindUUID.cmake @@ -33,6 +33,8 @@ find_path(UUID_INCLUDE_DIR NO_DEFAULT_PATH ) +find_path(UUID_INCLUDE_DIR NAMES uuid/uuid.h PATH_SUFFIXES include include/uuid) + find_library(UUID_LIBRARY NAMES uuid HINTS @@ -45,6 +47,8 @@ find_library(UUID_LIBRARY PATH_SUFFIXES lib lib64 ) +find_library(UUID_LIBRARY NAMES uuid PATH_SUFFIXES lib lib64) + find_package_handle_standard_args(UUID DEFAULT_MSG UUID_LIBRARY UUID_INCLUDE_DIR) mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY) diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc index 697d91571..51ddf7e28 100644 --- a/src/fdb5/daos/DaosStore.cc +++ b/src/fdb5/daos/DaosStore.cc @@ -51,7 +51,6 @@ bool DaosStore::uriExists(const eckit::URI& uri) const { ASSERT(n.hasContainerName()); ASSERT(n.poolName() == pool_); ASSERT(n.containerName() == db_str_); - ASSERT(n.hasOID()); return n.exists(); From dd5729cf5505118822b352ce3e9b3711d7273073 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 23 Jun 2024 01:04:14 +0200 Subject: [PATCH 014/109] Added RadosCatalogue with archive, retrieve and list. --- CMakeLists.txt | 10 +- src/fdb5/CMakeLists.txt | 12 + src/fdb5/fdb5_config.h.in | 6 +- src/fdb5/rados/README | 58 ++-- src/fdb5/rados/RadosCatalogue.cc | 187 ++++++++++++ src/fdb5/rados/RadosCatalogue.h | 78 +++++ src/fdb5/rados/RadosCatalogueReader.cc | 138 +++++++++ src/fdb5/rados/RadosCatalogueReader.h | 60 ++++ src/fdb5/rados/RadosCatalogueWriter.cc | 364 +++++++++++++++++++++++ src/fdb5/rados/RadosCatalogueWriter.h | 82 +++++ src/fdb5/rados/RadosCommon.cc | 148 ++++++--- src/fdb5/rados/RadosCommon.h | 28 +- src/fdb5/rados/RadosIndex.cc | 325 ++++++++++++++++++++ src/fdb5/rados/RadosIndex.h | 97 ++++++ src/fdb5/rados/RadosIndexLocation.cc | 31 ++ src/fdb5/rados/RadosIndexLocation.h | 67 +++++ src/fdb5/rados/RadosLazyFieldLocation.cc | 66 ++++ src/fdb5/rados/RadosLazyFieldLocation.h | 62 ++++ src/fdb5/rados/RadosStore.cc | 44 +-- src/fdb5/rados/RadosStore.h | 10 +- src/fdb5/toc/TocWipeVisitor.cc | 2 +- tests/fdb/rados/test_rados_store.cc | 56 ++-- 22 files changed, 1787 insertions(+), 144 deletions(-) create mode 100644 src/fdb5/rados/RadosCatalogue.cc create mode 100644 src/fdb5/rados/RadosCatalogue.h create mode 100644 src/fdb5/rados/RadosCatalogueReader.cc create mode 100644 src/fdb5/rados/RadosCatalogueReader.h create mode 100644 src/fdb5/rados/RadosCatalogueWriter.cc create mode 100644 src/fdb5/rados/RadosCatalogueWriter.h create mode 100644 src/fdb5/rados/RadosIndex.cc create mode 100644 src/fdb5/rados/RadosIndex.h create mode 100644 src/fdb5/rados/RadosIndexLocation.cc create mode 100644 src/fdb5/rados/RadosIndexLocation.h create mode 100644 src/fdb5/rados/RadosLazyFieldLocation.cc create mode 100644 src/fdb5/rados/RadosLazyFieldLocation.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 69dad7799..e94dcba1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,7 +49,7 @@ ecbuild_add_option( FEATURE RADOSFDB # option defined in fdb5_config.h DEFAULT OFF DESCRIPTION "Ceph/Rados support for FDB Store" ) -ecbuild_add_option( FEATURE RADOS_STORE_SINGLE_POOL +ecbuild_add_option( FEATURE RADOS_BACKENDS_SINGLE_POOL DEFAULT ON DESCRIPTION "Use a single Rados pool with a namespace per database (ON) or a pool per database (OFF)" ) @@ -61,14 +61,14 @@ ecbuild_add_option( FEATURE RADOS_STORE_MULTIPART DEFAULT ON DESCRIPTION "If RADOS_STORE_OBJ_PER_FIELD=OFF and the maximum object size is exceeded, use multiple Rados objects per collocation key (ON) or throw an exception (OFF)" ) -ecbuild_add_option( FEATURE RADOS_STORE_PERSIST_ON_FLUSH +ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_FLUSH DEFAULT OFF DESCRIPTION "Ensure writes are persisted in Rados storage on flush." ) -ecbuild_add_option( FEATURE RADOS_STORE_PERSIST_ON_WRITE - CONDITION fdb5_HAVE_STORE_OBJ_PER_FIELD AND NOT fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH +ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_WRITE + CONDITION NOT fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH DEFAULT OFF - DESCRIPTION "If RADOS_STORE_OBJ_PER_FIELD=ON, ensure every object write is persisted immediately." ) + DESCRIPTION "Ensure every object write or kv put is persisted immediately." ) ### FDB backend in indexed filesystem with table-of-contents, i.e. TOC diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 49f5242d6..8ccc6b39f 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -369,6 +369,18 @@ if( HAVE_RADOSFDB ) rados/RadosStore.h rados/RadosCommon.cc rados/RadosCommon.h + rados/RadosCatalogue.cc + rados/RadosCatalogue.h + rados/RadosCatalogueWriter.cc + rados/RadosCatalogueWriter.h + rados/RadosCatalogueReader.cc + rados/RadosCatalogueReader.h + rados/RadosIndex.cc + rados/RadosIndex.h + rados/RadosIndexLocation.cc + rados/RadosIndexLocation.h + rados/RadosLazyFieldLocation.cc + rados/RadosLazyFieldLocation.h ) endif() diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index e70e49c21..f82c954fc 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -10,11 +10,11 @@ #cmakedefine fdb5_HAVE_LUSTRE #cmakedefine fdb5_HAVE_PMEMFDB #cmakedefine fdb5_HAVE_RADOSFDB -#cmakedefine fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#cmakedefine fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #cmakedefine fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD #cmakedefine fdb5_HAVE_RADOS_STORE_MULTIPART -#cmakedefine fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH -#cmakedefine fdb5_HAVE_RADOS_STORE_PERSIST_ON_WRITE +#cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH +#cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE #cmakedefine fdb5_HAVE_TOCFDB #cmakedefine fdb5_HAVE_DUMMY_DAOS #cmakedefine fdb5_HAVE_DAOSFDB diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index 762bcd451..8b69052c1 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -4,6 +4,8 @@ Running RadosStore unit tests against Ceph on Docker on mac: git clone https://github.com/datenkollektiv/ceph-playground.git cd ceph-playground sed -i '' 's#volumes:#volumes:\n - < PATH TO YOUR LOCAL FDB BUNDLE SOURCE >:/root/git/fdb-bundle#g' docker-compose.yaml +sed -i '' 's#volumes:#volumes:\n - < PATH TO YOUR LOCAL CEPH SOURCE >/src/include/rados:/usr/include/rados#g' docker-compose.yaml +sed -i '' 's#5000:5000#7777:5000#g' docker-compose.yaml docker-compose down rm -rf docker/ceph/etc/* @@ -16,7 +18,7 @@ docker exec -it ceph-playground_ceph_1 /bin/bash sed -i -e "s|mirrorlist=|#mirrorlist=|g" -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Linux-* -yum install -y gcc gcc-c++ gcc-gfortran make cmake openssl openssl-devel git vim +yum install -y gcc gcc-c++ gcc-gfortran make cmake openssl openssl-devel git vim libuuid-devel yum update -y libarchive ln -s /usr/lib64/librados.so.2 /usr/lib64/librados.so @@ -48,63 +50,63 @@ cmake options: # single pool, multiple fields per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF # pool per db, multiple fields per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=OFF \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF # single pool, field per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF # pool per db, field per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=OFF \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF # single pool, multiple fields per obj, multipart cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF # single pool, multiple fields per obj, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=ON \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF # single pool, field per obj, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=ON \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF # single pool, field per obj, persist on write cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=ON + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=ON # single pool, multiple fields per obj, multipart, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_STORE_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ - -DENABLE_RADOS_STORE_PERSIST_ON_FLUSH=ON \ - -DENABLE_RADOS_STORE_PERSIST_ON_WRITE=OFF \ No newline at end of file + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \ + -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc new file mode 100644 index 000000000..e6a84a05a --- /dev/null +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -0,0 +1,187 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +// #include "eckit/config/Resource.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/io/rados/RadosException.h" + +// #include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/LibFdb5.h" +#include "fdb5/database/DatabaseNotFoundException.h" + +#include "fdb5/rados/RadosCatalogue.h" +// #include "fdb5/daos/DaosName.h" +// #include "fdb5/daos/DaosSession.h" +// #include "fdb5/daos/DaosIndex.h" +// #include "fdb5/daos/DaosWipeVisitor.h" + +// using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : + Catalogue(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) { + + // TODO: apply the mechanism in RootManager::directory, using + // FileSpaceTables to determine root_pool_name_ according to key + // and using DbPathNamerTables to determine db_cont_name_ according + // to key + +} + +RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) : + Catalogue(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string pool = pool_; + std::string nspace = db_namespace_; +#else + std::string pool = db_pool_; + std::string nspace = namespace_; +#endif + + // Read the real DB key into the DB base object + try { + + std::vector data; + eckit::MemoryStream ms = db_kv_->getMemoryStream(data, "key", "DB kv"); + dbKey_ = fdb5::Key(ms); + + } catch (eckit::RadosEntityNotFoundException& e) { + + throw fdb5::DatabaseNotFoundException( + std::string("RadosCatalogue database not found ") + + "(pool: '" + pool + "', namespace: '" + nspace + "')" + ); + + } + +} + +bool RadosCatalogue::exists() const { + + return db_kv_->exists(); + +} + +eckit::URI RadosCatalogue::uri() const { + + return db_kv_->nspace().uri(); + +} + +const Schema& RadosCatalogue::schema() const { + + return schema_; + +} + +void RadosCatalogue::loadSchema() { + + eckit::Timer timer("RadosCatalogue::loadSchema()", eckit::Log::debug()); + + /// @note: performed RPCs: + /// - daos_obj_generate_oid + /// - daos_kv_open + /// - daos_kv_get without a buffer + /// - daos_kv_get + std::vector data; + db_kv_->getMemoryStream(data, "schema", "DB Key-Value"); + + std::istringstream stream{std::string(data.begin(), data.end())}; + schema_.load(stream); + +} + +WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { + NOTIMP; + // return new RadosWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); +} + +std::vector RadosCatalogue::indexes(bool) const { + + NOTIMP; + +// /// @note: sorted is not implemented as is not necessary in this backend. + +// fdb5::DaosKeyValueName catalogue_kv_name{pool_, db_cont_, catalogue_kv_}; +// fdb5::DaosSession s{}; + +// /// @note: performed RPCs: +// /// - db kv open (daos_kv_open) +// /// - db kv list keys (daos_kv_list) +// fdb5::DaosKeyValue catalogue_kv{s, catalogue_kv_name}; /// @note: throws if not exists + +// std::vector res; + +// for (const auto& key : catalogue_kv.keys()) { + +// /// @todo: document these well. Single source these reserved values. +// /// Ensure where appropriate that user-provided keys do not collide. +// if (key == "schema" || key == "key") continue; + +// /// @note: performed RPCs: +// /// - db kv get index location size (daos_kv_get without a buffer) +// /// - db kv get index location (daos_kv_get) +// uint64_t size{catalogue_kv.size(key)}; +// std::vector v(size); +// catalogue_kv.get(key, v.data(), size); + +// fdb5::DaosKeyValueName index_kv_name{eckit::URI(std::string(v.begin(), v.end()))}; + +// /// @note: performed RPCs: +// /// - index kv open (daos_kv_open) +// /// - index kv get size (daos_kv_get without a buffer) +// /// - index kv get key (daos_kv_get) +// /// @note: the following three lines intend to check whether the index kv exists +// /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, +// /// so it is not useful on its own to check whether the index KV existed or not. +// /// Instead, presence of a "key" key in the KV is used to determine if the index +// /// KV existed. +// fdb5::DaosKeyValue index_kv{s, index_kv_name}; +// std::optional index_key; +// try { +// std::vector data; +// eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); +// index_key.emplace(ms); +// } catch (fdb5::DaosEntityNotFoundException& e) { +// continue; /// @note: the index_kv may not exist after a failed wipe +// /// @todo: the index_kv may exist even if it does not have the "key" key +// } + +// res.push_back(Index(new fdb5::DaosIndex(index_key.value(), index_kv_name, false))); + +// } + +// return res; + +} + +std::string RadosCatalogue::type() const { + + return RadosCatalogue::catalogueTypeName(); + +} + +// void RadosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, std::ostream& logVerbose, bool doit) { + +// ASSERT(n.hasContainerName()); + +// logVerbose << "Removing " << (n.hasOID() ? "KV" : "container") << ": "; +// logAlways << n.URI() << std::endl; +// if (doit) n.destroy(); + +// } + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h new file mode 100644 index 000000000..2ee68b01d --- /dev/null +++ b/src/fdb5/rados/RadosCatalogue.h @@ -0,0 +1,78 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/database/DB.h" +#include "fdb5/rules/Schema.h" +#include "fdb5/rados/RadosCommon.h" +// #include "fdb5/rados/RadosEngine.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on Rados + +class RadosCatalogue : public Catalogue, public RadosCommon { + +public: // methods + + RadosCatalogue(const Key& key, const fdb5::Config& config); + RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config); + + // static const char* catalogueTypeName() { return fdb5::RadosEngine::typeName(); } + static const char* catalogueTypeName() { return "rados"; } + + eckit::URI uri() const override; + const Key& indexKey() const override { return currentIndexKey_; } + + // static void remove(const eckit::RadosObject&, std::ostream& logAlways, std::ostream& logVerbose, bool doit); + + std::string type() const override; + + void checkUID() const override { NOTIMP; }; + bool exists() const override; + void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override { NOTIMP; }; + std::vector metadataPaths() const override { NOTIMP; }; + const Schema& schema() const override; + + StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; + PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }; + WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override; + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; }; + void maskIndexEntry(const Index& index) const override { NOTIMP; }; + + void loadSchema() override; + + std::vector indexes(bool sorted=false) const override; + + void allMasked(std::set>& metadata, + std::set& data) const override { NOTIMP; }; + + // Control access properties of the DB + void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override { NOTIMP; }; + +protected: // members + + Key currentIndexKey_; + +private: // members + + Schema schema_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc new file mode 100644 index 000000000..9f21a7b23 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -0,0 +1,138 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/LibFdb5.h" +#include "fdb5/rados/RadosIndex.h" +#include "fdb5/rados/RadosCatalogueReader.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// @note: as opposed to the TOC catalogue, the DAOS catalogue does not pre-load all indexes from storage. +/// Instead, it selects and loads only those indexes that are required to fulfil the request. + +RadosCatalogueReader::RadosCatalogueReader(const Key& key, const fdb5::Config& config) : + RadosCatalogue(key, config) { + + /// @todo: schema is being loaded at DaosCatalogueWriter creation for write, but being loaded + /// at DaosCatalogueReader::open for read. Is this OK? + +} + +RadosCatalogueReader::RadosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config) : + RadosCatalogue(uri, ControlIdentifiers{}, config) {} + +bool RadosCatalogueReader::selectIndex(const Key &key) { + + if (currentIndexKey_ == key) { + return true; + } + + /// @todo: shouldn't this be set only if found a matching index? + currentIndexKey_ = key; + + if (indexes_.find(key) == indexes_.end()) { + + /// @note: performed RPCs: + /// - generate catalogue kv oid (daos_obj_generate_oid) + /// - ensure catalogue kv exists (daos_kv_open) + + int idx_loc_max_len = 512; /// @todo: take from config + std::vector n((long) idx_loc_max_len); + long res; + + try { + + /// @note: performed RPCs: + /// - retrieve index kv location from catalogue kv (daos_kv_get) + res = db_kv_->get(key.valuesToString(), &n[0], idx_loc_max_len); + + } catch (eckit::RadosEntityNotFoundException& e) { + + /// @note: performed RPCs: + /// - close catalogue kv (daos_obj_close) + + return false; + + } + + eckit::URI uri{std::string{n.begin(), std::next(n.begin(), res)}}; +// #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE +// eckit::RadosPersistentKeyValue index_kv{uri, true}; +// #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH +// eckit::RadosPersistentKeyValue index_kv{uri}; +// #else + eckit::RadosKeyValue index_kv{uri}; +// #endif + + indexes_[key] = Index(new fdb5::RadosIndex(key, index_kv, true)); + + /// @note: performed RPCs: + /// - close catalogue kv (daos_obj_close) + + } + + current_ = indexes_[key]; + + return true; + +} + +void RadosCatalogueReader::deselectIndex() { + + NOTIMP; //< should not be called + +} + +bool RadosCatalogueReader::open() { + + /// @note: performed RPCs: + /// - daos_pool_connect + /// - daos_cont_open + /// - daos_obj_generate_oid + /// - daos_kv_open + if (!RadosCatalogue::exists()) { + return false; + } + + RadosCatalogue::loadSchema(); + return true; + +} + +bool RadosCatalogueReader::axis(const std::string &keyword, eckit::StringSet &s) const { + + bool found = false; + if (current_.axes().has(keyword)) { + found = true; + const eckit::DenseSet& a = current_.axes().values(keyword); + s.insert(a.begin(), a.end()); + } + return found; + +} + +bool RadosCatalogueReader::retrieve(const Key& key, Field& field) const { + + eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; + eckit::Log::debug() << "Scanning index " << current_.location() << std::endl; + + if (!current_.mayContain(key)) return false; + + return current_.get(key, fdb5::Key(), field); + +} + +static fdb5::CatalogueBuilder builder("rados.reader"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueReader.h b/src/fdb5/rados/RadosCatalogueReader.h new file mode 100644 index 000000000..7aa787275 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueReader.h @@ -0,0 +1,60 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/rados/RadosCatalogue.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on Rados + +class RadosCatalogueReader : public RadosCatalogue, public CatalogueReader { + +public: // methods + + RadosCatalogueReader(const Key& key, const fdb5::Config& config); + RadosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config); + + DbStats stats() const override { NOTIMP; } + + bool selectIndex(const Key &key) override; + void deselectIndex() override; + + bool open() override; + void flush() override {} + void clean() override {} + void close() override {} + + bool axis(const std::string &keyword, eckit::StringSet &s) const override; + + bool retrieve(const Key& key, Field& field) const override; + + void print( std::ostream &out ) const override { NOTIMP; } + +private: // types + + typedef std::map< Key, Index> IndexStore; + +private: // members + + IndexStore indexes_; + Index current_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc new file mode 100644 index 000000000..a59411781 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -0,0 +1,364 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +// #include + +#include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/serialisation/HandleStream.h" + +#include "fdb5/LibFdb5.h" + +// #include "fdb5/daos/DaosSession.h" +// #include "fdb5/daos/DaosName.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" + +#include "fdb5/rados/RadosIndex.h" +#include "fdb5/rados/RadosCatalogueWriter.h" + +// using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& config) : + RadosCatalogue(key, config), firstIndexWrite_(false) { + + /// @note: performed RPCs: + /// - daos_pool_connect + /// - root cont open (daos_cont_open) + /// - root cont create (daos_cont_create) +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string db_name = db_namespace_; + ASSERT(root_kv_->nspace().pool().exists()); +#else + std::string db_name = db_pool_; + root_kv_->nspace().pool().ensureCreated(); +#endif + + /// @note: the DaosKeyValue constructor checks if the kv exists, which results in creation if not exists + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + + /// @note: performed RPCs: + /// - check if main kv contains db key (daos_kv_get without a buffer) + if (!root_kv_->has(db_name)) { + + /// create catalogue kv + db_kv_->ensureCreated(); + + /// write schema under "schema" + eckit::Log::debug() << "Copy schema from " + << config_.schemaPath() + << " to " + << db_kv_->uri().asString() + << " at key 'schema'." + << std::endl; + + eckit::FileHandle in(config_.schemaPath()); + std::vector data; + data.resize(in.size()); + in.read(&data[0], in.size()); + db_kv_->put("schema", &data[0], data.size()); + + /// write dbKey under "key" + eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + hs << dbKey_; + } + + int db_key_max_len = 512; // @todo: take from config + if (hs.bytesWritten() > db_key_max_len) + throw eckit::Exception("Serialised db key exceeded configured maximum db key length."); + + db_kv_->put("key", h.data(), hs.bytesWritten()); + + /// index newly created catalogue kv in main kv + int db_loc_max_len = 512; // @todo: take from config + std::string nstr = db_kv_->uri().asString(); + if (nstr.length() > db_loc_max_len) + throw eckit::Exception("Serialised db location exceeded configured maximum db location length."); + + root_kv_->put(db_name, nstr.data(), nstr.length()); + + } + + /// @todo: record or read dbUID + + /// @note: performed RPCs: + /// - catalogue container open (daos_cont_open) + /// - get schema from catalogue kv (daos_kv_get) + RadosCatalogue::loadSchema(); + + /// @todo: TocCatalogue::checkUID(); + +} + +RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config) : + RadosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) { + + NOTIMP; + +} + +RadosCatalogueWriter::~RadosCatalogueWriter() { + + clean(); + close(); + +} + +bool RadosCatalogueWriter::selectIndex(const Key& key) { + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string pool = pool_; + std::string nspace = db_namespace_; +#else + std::string pool = db_pool_; + std::string nspace = namespace_; +#endif + + currentIndexKey_ = key; + + if (indexes_.find(key) == indexes_.end()) { + + /// @note: performed RPCs: + /// - generate catalogue kv oid (daos_obj_generate_oid) + /// - ensure catalogue kv exists (daos_kv_open) + + int idx_loc_max_len = 512; /// @todo: take from config + + try { + + std::vector n((long) idx_loc_max_len); + long res; + + /// @note: performed RPCs: + /// - get index location from catalogue kv (daos_kv_get) + res = db_kv_->get(key.valuesToString(), &n[0], idx_loc_max_len); + + indexes_[key] = Index( + new fdb5::RadosIndex( + key, +// #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE +// eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}, true}, +// #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH +// eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, +// #else + eckit::RadosKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, +// #endif + false + ) + ); + + } catch (eckit::RadosEntityNotFoundException& e) { + + firstIndexWrite_ = true; + + indexes_[key] = Index( + new fdb5::RadosIndex( + key, + eckit::RadosNamespace{pool, nspace} + ) + ); + + /// index index kv in catalogue kv + std::string nstr{indexes_[key].location().uri().asString()}; + if (nstr.length() > idx_loc_max_len) + throw eckit::Exception("Serialised index location exceeded configured maximum index location length."); + /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on first write to an index key): + /// - record index kv location into catalogue kv (daos_kv_put) -- always performed + db_kv_->put(key.valuesToString(), nstr.data(), nstr.length()); + + /// @note: performed RPCs: + /// - close index kv when destroyed (daos_obj_close) + + } + + /// @note: performed RPCs: + /// - close catalogue kv (daos_obj_close) + + } + + current_ = indexes_[key]; + + return true; + +} + +void RadosCatalogueWriter::deselectIndex() { + + current_ = Index(); + currentIndexKey_ = Key(); + firstIndexWrite_ = false; + +} + +void RadosCatalogueWriter::clean() { + + flush(); + + deselectIndex(); + +} + +void RadosCatalogueWriter::close() { + + closeIndexes(); + +} + +const Index& RadosCatalogueWriter::currentIndex() { + + if (current_.null()) { + ASSERT(!currentIndexKey_.empty()); + selectIndex(currentIndexKey_); + } + + return current_; + +} + +/// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these +/// new updates be retrieved and put into in-memory axes from time to time, e.g. every +/// time a value is put in an axis KeyValue? +void RadosCatalogueWriter::archive(const Key& key, std::unique_ptr fieldLocation) { + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string pool = pool_; + std::string nspace = db_namespace_; +#else + std::string pool = db_pool_; + std::string nspace = namespace_; +#endif + + if (current_.null()) { + ASSERT(!currentIndexKey_.empty()); + selectIndex(currentIndexKey_); + } + + /// @note: the current index timestamp is undefined at this point + Field field(std::move(fieldLocation), currentIndex().timestamp()); + + /// @todo: is sorting axes really necessary? + /// @note: sort in-memory axis values. Not triggering retrieval from DAOS axes. + const_cast(current_.axes()).sort(); + + /// before in-memory axes are updated as part of current_.put, we determine which + /// additions will need to be performed on axes in DAOS after the field gets indexed. + std::vector axesToExpand; + std::vector valuesToAdd; + std::string axisNames = ""; + std::string sep = ""; + + for (Key::const_iterator i = key.begin(); i != key.end(); ++i) { + + const std::string &keyword = i->first; + + std::string value = key.canonicalValue(keyword); + + if (value.length() == 0) continue; + + axisNames += sep + keyword; + sep = ","; + + /// @note: obtain in-memory axis values. Not triggering retrieval from DAOS axes. + /// @note: on first archive the in-memory axes will be empty and values() will return + /// empty sets. This is fine. + const auto& axis_set = current_.axes().values(keyword); + + //if (!axis_set.has_value() || !axis_set->get().contains(value)) { + if (!axis_set.contains(value)) { + + axesToExpand.push_back(keyword); + valuesToAdd.push_back(value); + + } + + } + + /// index the field and update in-memory axes + current_.put(key, field); + + /// persist axis names + if (firstIndexWrite_) { + + /// @note: performed RPCs: + /// - generate index kv oid (daos_obj_generate_oid) + /// - ensure index kv exists (daos_obj_open) + + int axis_names_max_len = 512; + if (axisNames.length() > axis_names_max_len) + throw eckit::Exception("Serialised axis names exceeded configured maximum axis names length."); + + /// @note: performed RPCs: + /// - record axis names into index kv (daos_kv_put) + /// - close index kv when destroyed (daos_obj_close) + dynamic_cast(current_.content())->putAxisNames(axisNames); + + firstIndexWrite_ = false; + + } + + /// @todo: axes are supposed to be sorted before persisting. How do we do this with the DAOS approach? + /// sort axes every time they are loaded in the read pathway? + + if (axesToExpand.empty()) return; + + /// expand axis info in DAOS + while (!axesToExpand.empty()) { + + /// @note: performed RPCs: + /// - generate axis kv oid (daos_obj_generate_oid) + /// - ensure axis kv exists (daos_obj_open) + + /// @note: performed RPCs: + /// - record axis value into axis kv (daos_kv_put) + /// - close axis kv when destroyed (daos_obj_close) + dynamic_cast(current_.content())->putAxisValue(axesToExpand.back(), valuesToAdd.back()); + + axesToExpand.pop_back(); + valuesToAdd.pop_back(); + + } + +} + +void RadosCatalogueWriter::flush() { + +#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH + for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j ) + j->second.flush(); +flush axis kvs if not done as part of RadosIndex::flush + db_kv_->flush(); + root_kv_->flush(); +#endif + + if (!current_.null()) current_ = Index(); + +} + +void RadosCatalogueWriter::closeIndexes() { + + indexes_.clear(); // all indexes instances destroyed + +} + +static fdb5::CatalogueBuilder builder("rados.writer"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h new file mode 100644 index 000000000..195cf11e6 --- /dev/null +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -0,0 +1,82 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "fdb5/rados/RadosCatalogue.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on Rados + +class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { + +public: // methods + + RadosCatalogueWriter(const Key &key, const fdb5::Config& config); + RadosCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config); + virtual ~RadosCatalogueWriter() override; + + void index(const Key &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; + + void reconsolidate() override { NOTIMP; } + + /// Mount an existing TocCatalogue, which has a different metadata key (within + /// constraints) to allow on-line rebadging of data + /// variableKeys: The keys that are allowed to differ between the two DBs + void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { NOTIMP; }; + +// // Hide the contents of the DB!!! +// void hideContents() override; + +// bool enabled(const ControlIdentifier& controlIdentifier) const override; + + const Index& currentIndex() override; + +protected: // methods + + virtual bool selectIndex(const Key &key) override; + virtual void deselectIndex() override; + + bool open() override { NOTIMP; } + void flush() override; + void clean() override; + void close() override; + + void archive(const Key& key, std::unique_ptr fieldLocation) override; + + virtual void print( std::ostream &out ) const override { NOTIMP; } + +private: // methods + + void closeIndexes(); + +private: // types + + typedef std::map< Key, Index> IndexStore; + +private: // members + + IndexStore indexes_; + + Index current_; + + bool firstIndexWrite_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 61be120f2..96c368e6d 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -10,17 +10,12 @@ #include -#include "eckit/io/rados/RadosObject.h" -// #include "eckit/io/s3/S3Bucket.h" -// #include "eckit/io/s3/S3Credential.h" -// #include "eckit/io/s3/S3Session.h" - -#include "fdb5/rados/RadosCommon.h" - #include "eckit/exception/Exceptions.h" -// #include "eckit/config/Resource.h" +#include "eckit/config/Resource.h" #include "eckit/utils/Tokenizer.h" +#include "fdb5/rados/RadosCommon.h" + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -30,37 +25,33 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen std::vector valid{"catalogue", "store"}; ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); - parseConfig(config, component); - - eckit::LocalConfiguration rados{}, comp_conf{}; - - if (config.has("rados")) { - rados = config.getSubConfiguration("rados"); - if (rados.has(component)) comp_conf = rados.getSubConfiguration(component); - } - -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - pool_ = rados.getString("pool", pool_); - pool_ = comp_conf.getString("pool", pool_); - - // std::string first_cap{component}; - // first_cap[0] = toupper(component[0]); - // std::string all_caps{component}; - // for (auto & c: all_caps) c = toupper(c); - // bucket_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); + db_namespace_ = key.valuesToString(); - ASSERT_MSG(pool_.length() > 0, "No pool configured for Rados " + component); + readConfig(config, component, true); - db_namespace_ = key.valuesToString(); + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + root_kv_.emplace(pool_, root_namespace_, "main_kv", true); + db_kv_.emplace(pool_, db_namespace_, "catalogue_kv", true); + #else + root_kv_.emplace(pool_, root_namespace_, "main_kv"); + db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); + #endif #else - prefix_ = rados.getString("poolPrefix", prefix_); - prefix_ = comp_conf.getString("poolPrefix", prefix_); - ASSERT_MSG(prefix_.find("_") == std::string::npos, "The configured poolPrefix must not contain underscores."); + readConfig(config, component, true); + db_pool_ = prefix_ + "_" + key.valuesToString(); - namespace_ = "default"; + + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + root_kv_.emplace(root_pool_, namespace_, "main_kv", true); + db_kv_.emplace(db_pool_, namespace_, "catalogue_kv", true); + #else + root_kv_.emplace(root_pool_, namespace_, "main_kv"); + db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); + #endif #endif @@ -71,40 +62,101 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen /// @note: validity of input URI is not checked here because this constructor is only triggered /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand - parseConfig(config, component); + eckit::RadosKeyValue db_name{uri}; -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - eckit::RadosObject o{uri}; - pool_ = o.nspace().pool().name(); - db_namespace_ = o.nspace().name(); + pool_ = db_name.nspace().pool().name(); + db_namespace_ = db_name.nspace().name(); + + readConfig(config, component, false); + + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + root_kv_.emplace(pool_, root_namespace_, "main_kv", true); + db_kv_.emplace(pool_, db_namespace_, "catalogue_kv", true); + #else + root_kv_.emplace(pool_, root_namespace_, "main_kv"); + db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); + #endif #else - eckit::RadosObject o{uri}; - db_pool_ = o.nspace().pool().name(); - namespace_ = o.nspace().name(); - if (namespace_ != "default") - throw eckit::SeriousBug("Unexpected namespace name '" + namespace_ + "'. Expected 'default'."); + db_pool_ = db_name.nspace().pool().name(); + namespace_ = db_name.nspace().name(); + + readConfig(config, component, false); + const auto parts = eckit::Tokenizer("_").tokenize(db_pool_); const auto n = parts.size(); ASSERT(n > 1); prefix_ = parts[0]; + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + root_kv_.emplace(root_pool_, namespace_, "main_kv", true); + db_kv_.emplace(db_pool_, namespace_, "catalogue_kv", true); + #else + root_kv_.emplace(root_pool_, namespace_, "main_kv"); + db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); + #endif + #endif } -void RadosCommon::parseConfig(const fdb5::Config& config, const std::string& component) { +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL +void RadosCommon::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) { +#else +void RadosCommon::readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) { +#endif + + eckit::LocalConfiguration c{}; + + if (config.has("rados")) c = config.getSubConfiguration("rados"); + + maxObjectSize_ = c.getInt("maxObjectSize", 0); + + std::string first_cap{component}; + first_cap[0] = toupper(component[0]); + + std::string all_caps{component}; + for (auto & c: all_caps) c = toupper(c); + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - eckit::LocalConfiguration rados{}, comp_conf{}; + if (readPool) pool_ = "default"; + root_namespace_ = "root"; - if (config.has("rados")) { - rados = config.getSubConfiguration("rados"); - if (rados.has(component)) comp_conf = rados.getSubConfiguration(component); + if (readPool) { + pool_ = c.getString("pool", pool_); + if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); } + if (c.has(component)) root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + + if (readPool) + pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); + root_namespace_ = eckit::Resource("fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + +#else + + if (readNamespace) namespace_ = "default"; + root_pool_ = "root"; + + if (readNamespace) + if (c.has(component)) namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + if (c.has(component)) root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); + + if (readNamespace) + namespace_ = eckit::Resource("fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); + root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); + + prefix_ = c.getString("pool_prefix", prefix_); + if (c.has(component)) prefix_ = c.getSubconfiguration(component).getString("pool_prefix", prefix_); + ASSERT_MSG(prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); + +#endif - maxObjectSize_ = rados.getInt("maxObjectSize", 0); + // if (c.has("client")) + // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); } diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index af46df65d..ef5c5a808 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -14,6 +14,9 @@ #pragma once #include "eckit/filesystem/URI.h" +#include "eckit/utils/Optional.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosPersistentKeyValue.h" #include "fdb5/fdb5_config.h" #include "fdb5/database/Key.h" @@ -30,20 +33,35 @@ class RadosCommon { private: // methods - void parseConfig(const fdb5::Config& config, const std::string& component); +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + void readConfig(const fdb5::Config& config, const std::string& component, bool readPool); +#else + void readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace); +#endif protected: // members -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool_; + std::string root_namespace_; std::string db_namespace_; #else + std::string root_pool_; std::string db_pool_; std::string namespace_; #endif + +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + eckit::Optional root_kv_; + eckit::Optional db_kv_; +#else + eckit::Optional root_kv_; + eckit::Optional db_kv_; +#endif + eckit::Length maxObjectSize_; -#ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL private: // members std::string prefix_; @@ -51,4 +69,6 @@ class RadosCommon { }; -} \ No newline at end of file +} + + diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc new file mode 100644 index 000000000..d8b8d2296 --- /dev/null +++ b/src/fdb5/rados/RadosIndex.cc @@ -0,0 +1,325 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include // for PATH_MAX + +#include + +#include "eckit/io/MemoryHandle.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/HandleStream.h" +#include "eckit/utils/Tokenizer.h" + +#include "fdb5/rados/RadosIndex.h" +#include "fdb5/rados/RadosLazyFieldLocation.h" + +// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) +// eckit::RadosPersistentKeyValue buildIndexKvName(const fdb5::Key& key, const eckit::RadosNamespace& name) { +// #else +// eckit::RadosKeyValue buildIndexKvName(const fdb5::Key& key, const eckit::RadosNamespace& name) { +// #endif + /// create index kv + /// @todo: pass oclass from config + /// @todo: hash string into lower oid bits + +// #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE +// return eckit::RadosPersistentKeyValue{name.poolName(), name.containerName(), key.valuesToString(), true}; +// #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH +// return eckit::RadosPersistentKeyValue{name.poolName(), name.containerName(), key.valuesToString()}; +// #else + // return eckit::RadosKeyValue{name.poolName(), name.containerName(), key.valuesToString()}; +// #endif + +// } + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : + IndexBase(key, "radosKeyValue"), + location_(eckit::RadosKeyValue{name.pool().name(), name.name(), key.valuesToString()}, 0), +#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + idx_kv_(location_.radosName().uri(), true) { +#else + idx_kv_(location_.radosName().uri()) { +#endif + + /// @note: performed RPCs: + /// - generate index kv oid (daos_obj_generate_oid) + /// - create/open index kv (daos_kv_open) + + /// write indexKey under "key" + eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + hs << key; + } + + int idx_key_max_len = 512; + + if (hs.bytesWritten() > idx_key_max_len) + throw eckit::Exception("Serialised index key exceeded configured maximum index key length."); + + /// @note: performed RPCs: + /// - record index key into index kv (daos_kv_put) + idx_kv_.put("key", h.data(), hs.bytesWritten()); + +} + +// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) +// RadosIndex::RadosIndex(const Key& key, const eckit::RadosPersistentKeyValue& name, bool readAxes) : +// #else +RadosIndex::RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes) : +// #endif + IndexBase(key, "radosKeyValue"), + location_(name, 0), +#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + idx_kv_(name.uri(), true) { +#else + idx_kv_(name.uri()) { +#endif + + if (readAxes) updateAxes(); + +} + +void RadosIndex::putAxisNames(const std::string& names) { + + idx_kv_.put("axes", names.data(), names.length()); + +} + +void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) { + + auto axis_kv = axis_kvs_.find(axis); + + if (axis_kv == axis_kvs_.end()) { + std::string kv_name = key().valuesToString() + std::string{"."} + axis; +#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + axis_kvs_.emplace( + std::piecewise_construct, + std::forward_as_tuple(axis), + std::forward_as_tuple( + location_.radosName().nspace().pool().name(), + location_.radosName().nspace().name(), + kv_name, + true + ) + ); +#else + axis_kvs_.emplace( + std::piecewise_construct, + std::forward_as_tuple(axis), + std::forward_as_tuple( + location_.radosName().nspace().pool().name(), + location_.radosName().nspace().name(), + kv_name + ) + ); +#endif + + axis_kv = axis_kvs_.find(axis); + } + + std::string v{"1"}; + axis_kv->second.put(value, v.data(), v.length()); + +} + +void RadosIndex::updateAxes() { + + /// @note: performed RPCs: + /// - ensure axis kv exists (daos_obj_open) + + int axis_names_max_len = 512; /// @todo: take from config + std::vector axes_data((long) axis_names_max_len); + + /// @note: performed RPCs: + /// - get axes key size and content (daos_kv_get without buffer + daos_kv_get) + long res = idx_kv_.get("axes", &axes_data[0], axis_names_max_len); + + std::vector axis_names; + eckit::Tokenizer parse(","); + parse(std::string(axes_data.begin(), std::next(axes_data.begin(), res)), axis_names); + std::string indexKey{key_.valuesToString()}; + for (const auto& name : axis_names) { + /// @note: performed RPCs: + /// - generate axis kv oid (daos_obj_generate_oid) + /// - ensure axis kv exists (daos_obj_open) + eckit::RadosKeyValue axis_kv{idx_kv_.nspace().pool().name(), idx_kv_.nspace().name(), indexKey + std::string{"."} + name}; + + /// @note: performed RPCs: + /// - one or more kv list (daos_kv_list) + axes_.insert(name, axis_kv.keys()); + } + + axes_.sort(); + +} + +bool RadosIndex::get(const Key &key, const Key &remapKey, Field &field) const { + + /// @note: performed RPCs: + /// - ensure index kv exists (daos_obj_open) + + std::string query{key.valuesToString()}; + + int field_loc_max_len = 512; /// @todo: read from config + std::vector loc_data((long) field_loc_max_len); + long res; + + try { + + /// @note: performed RPCs: + /// - retrieve field array location from index kv (daos_kv_get) + res = idx_kv_.get(query, &loc_data[0], (long) field_loc_max_len); + + } catch (eckit::RadosEntityNotFoundException& e) { + + /// @note: performed RPCs: + /// - close index kv (daos_obj_close) + + return false; + + } + + eckit::MemoryStream ms{&loc_data[0], (size_t) res}; + + /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. + time_t ts; + ms >> ts; + + fdb5::FieldLocation* loc = eckit::Reanimator::reanimate(ms); + field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); + + /// @note: performed RPCs: + /// - close index kv (daos_obj_close) + + return true; + +} + +void RadosIndex::add(const Key &key, const Field &field) { + + eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + /// @note: in the POSIX back-end, keeping a timestamp per index is necessary, to allow + /// determining which was the latest indexed field in cases where multiple processes + /// index a same field or in cases where multiple catalogues are combined with DistFDB. + /// In the DAOS back-end, however, determining the latest indexed field is straigthforward + /// as all parallel processes writing fields for a same index key will share a DAOS + /// key-value, and the last indexing will supersede the previous ones. + /// DistFDB will be obsoleted in favour of a centralised catalogue mechanism which can + /// index fields on multiple catalogues. + /// Therefore keeping timestamps in DAOS should not be necessary. + /// They are kept for now only for informational purposes. + takeTimestamp(); + hs << timestamp(); + hs << field.location(); + } + + int field_loc_max_len = 512; /// @todo: read from config + if (hs.bytesWritten() > field_loc_max_len) + throw eckit::Exception("Serialised field location exceeded configured maximum location length."); + + /// @note: performed RPCs: + /// - ensure index kv exists (daos_obj_open) + /// - record field key and location into index kv (daos_kv_put) + /// - close index kv when destroyed (daos_obj_close) + idx_kv_.put(key.valuesToString(), h.data(), hs.bytesWritten()); + +} + +void RadosIndex::entries(EntryVisitor &visitor) const { + + Index instantIndex(const_cast(this)); + + // Allow the visitor to selectively decline to visit the entries in this index + if (visitor.visitIndex(instantIndex)) { + + /// @note: performed RPCs: + /// - index kv open (daos_obj_open) + /// - index kv list keys (daos_kv_list) + + for (const auto& key : idx_kv_.keys()) { + + if (key == "axes" || key == "key") continue; + + /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each + /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field + /// keys present in an index -- without retrieving the actual location --, and + /// ListVisitor::visitDatum is called for each (see note at the top of DaosLazyFieldLocation.h). + /// When a field key is matched in visitDatum, DaosLazyFieldLocation::stableLocation is called, + /// which in turn calls this method here and triggers retrieval and deserialisation of the + /// indexed DaosFieldLocation, and returns it. Since the deserialised instance is of a + /// polymorphic class, it needs to be reanimated. + fdb5::FieldLocation* loc = new fdb5::RadosLazyFieldLocation(location_.radosName(), key); + fdb5::Field field(std::move(*loc), time_t(), fdb5::FieldDetails()); + visitor.visitDatum(field, key); + + } + + } +} + +const std::vector RadosIndex::dataURIs() const { + + /// @note: if daos index + daos store, this will return a uri to a DAOS array for each indexed field + /// @note: if daos index + posix store, this will return a vector of unique uris to all referenced posix files + /// in this index (one for each writer process that has written to the index) + /// @note: in the case where we have a daos store, the current implementation of dataURIs is unnecessarily inefficient. + /// This method is only called in DaosWipeVisitor, where the uris obtained from this method are processed to obtain + /// unique store container paths - will always result in just one container uri! Having a URI store for each index in + /// DAOS could make this process more efficient, but it would imply more KV operations and slow down field writes. + /// @note: in the case where we have a posix store there will be more than one unique store file paths. The current + /// implementation is still inefficient but preferred to maintaining a URI store in the DAOS catalogue + + std::set res; + + for (const auto& key : idx_kv_.keys()) { + + if (key == "axes" || key == "key") continue; + + std::vector data; + eckit::MemoryStream ms = idx_kv_.getMemoryStream(data, key, "index kv"); + + time_t ts; + ms >> ts; + + std::unique_ptr fl(eckit::Reanimator::reanimate(ms)); + res.insert(fl->uri()); + + } + + return std::vector(res.begin(), res.end()); + +} + +#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH +void RadosIndex::flush() { + + for (auto axis : axis_kvs_) { + axis->second.flush(); + } + + idx_kv_.flush(); + +} +#endif + +//----------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h new file mode 100644 index 000000000..e6aca1f08 --- /dev/null +++ b/src/fdb5/rados/RadosIndex.h @@ -0,0 +1,97 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosPersistentKeyValue.h" + +#include "fdb5/database/Index.h" +#include "fdb5/rados/RadosIndexLocation.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + + +class RadosIndex : public IndexBase { + +public: // methods + + /// @note: creates a new index in DAOS, in the container pointed to by 'name' + RadosIndex(const Key& key, const eckit::RadosNamespace& name); + /// @note: used to represent and operate with an index which already exists in DAOS +// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) +// RadosIndex(const Key& key, const eckit::RadosPersistentKeyValue& name, bool readAxes = true); +// #else + RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes = true); +// #endif + + void flock() const override { NOTIMP; } + void funlock() const override { NOTIMP; } + + /// @note: these methods are required for RadosCatalogueWriter to directly manipulate + /// idx_kv_ and axis_kvs_ within the RadosIndex. Upon flush, the index will flush all + /// operations performed on these kvs (if PERSIST_ON_FLUSH). + void putAxisNames(const std::string& names); + void putAxisValue(const std::string& axis, const std::string& value); + +private: // methods + + const IndexLocation& location() const override { return location_; } + const std::vector dataURIs() const override; + + bool dirty() const override { NOTIMP; } + + void open() override { NOTIMP; }; + void close() override { NOTIMP; } + void reopen() override { NOTIMP; } + + void visit(IndexLocationVisitor& visitor) const override { NOTIMP; } + + bool get( const Key &key, const Key &remapKey, Field &field ) const override; + void add( const Key &key, const Field &field ) override; +#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH + void flush() override; +#else + void flush() override { NOTIMP; } +#endif + void encode(eckit::Stream& s, const int version) const override { NOTIMP; } + void entries(EntryVisitor& visitor) const override; + + void print( std::ostream &out ) const override { NOTIMP; } + void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override { NOTIMP; } + + IndexStats statistics() const override { NOTIMP; } + + /// @note: reads complete axis info from DAOS. + void updateAxes(); + +private: // members + + fdb5::RadosIndexLocation location_; + +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + eckit::RadosPersistentKeyValue idx_kv_; + std::map axis_kvs_; +#else + eckit::RadosKeyValue idx_kv_; + std::map axis_kvs_; +#endif + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndexLocation.cc b/src/fdb5/rados/RadosIndexLocation.cc new file mode 100644 index 000000000..372d5d967 --- /dev/null +++ b/src/fdb5/rados/RadosIndexLocation.cc @@ -0,0 +1,31 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosIndexLocation.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) +// RadosIndexLocation::RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset) : name_(name), offset_(offset) {} +// #else +RadosIndexLocation::RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset) : name_(name), offset_(offset) {} +// #endif + +void RadosIndexLocation::print(std::ostream &out) const { + + out << "(" << name_.uri().asString() << ":" << offset_ << ")"; + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndexLocation.h b/src/fdb5/rados/RadosIndexLocation.h new file mode 100644 index 000000000..8058531e6 --- /dev/null +++ b/src/fdb5/rados/RadosIndexLocation.h @@ -0,0 +1,67 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "eckit/exception/Exceptions.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosPersistentKeyValue.h" + +#include "fdb5/database/IndexLocation.h" + + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class RadosIndexLocation : public IndexLocation { + +public: // methods + +// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) +// RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset); + +// const eckit::RadosPersistentKeyValue& radosName() const { return name_; }; +// #else + RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset); + + const eckit::RadosKeyValue& radosName() const { return name_; }; +// #endif + + eckit::URI uri() const override { return name_.uri(); } + + IndexLocation* clone() const override { NOTIMP; } + +protected: // For Streamable + + void encode(eckit::Stream&) const override { NOTIMP; } + +private: // methods + + void print(std::ostream &out) const override; + +private: // members + +// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) +// eckit::RadosPersistentKeyValue name_; +// #else + eckit::RadosKeyValue name_; +// #endif + + off_t offset_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosLazyFieldLocation.cc b/src/fdb5/rados/RadosLazyFieldLocation.cc new file mode 100644 index 000000000..fc3e2b509 --- /dev/null +++ b/src/fdb5/rados/RadosLazyFieldLocation.cc @@ -0,0 +1,66 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/rados/RadosLazyFieldLocation.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +RadosLazyFieldLocation::RadosLazyFieldLocation(const fdb5::RadosLazyFieldLocation& rhs) : + FieldLocation(), index_(rhs.index_), key_(rhs.key_) {} + +RadosLazyFieldLocation::RadosLazyFieldLocation(const eckit::RadosKeyValue& index, const std::string& key) : + FieldLocation(), index_(index), key_(key) {} + +std::shared_ptr RadosLazyFieldLocation::make_shared() const { + return std::make_shared(std::move(*this)); +} + +eckit::DataHandle* RadosLazyFieldLocation::dataHandle() const { + + return realise()->dataHandle(); + +} + +void RadosLazyFieldLocation::print(std::ostream &out) const { + out << *realise(); +} + +void RadosLazyFieldLocation::visit(FieldLocationVisitor& visitor) const { + realise()->visit(visitor); +} + +std::shared_ptr RadosLazyFieldLocation::stableLocation() const { + return realise()->make_shared(); +} + +std::unique_ptr& RadosLazyFieldLocation::realise() const { + + if (fl_) return fl_; + + /// @note: performed RPCs: + /// - index kv get (daos_kv_get) + std::vector data; + eckit::MemoryStream ms = index_.getMemoryStream(data, key_, "index kv"); + + /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. + time_t ts; + ms >> ts; + + fl_.reset(eckit::Reanimator::reanimate(ms)); + + return fl_; + +} + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosLazyFieldLocation.h b/src/fdb5/rados/RadosLazyFieldLocation.h new file mode 100644 index 000000000..3b18058df --- /dev/null +++ b/src/fdb5/rados/RadosLazyFieldLocation.h @@ -0,0 +1,62 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date June 2024 + +#pragma once + +#include "fdb5/database/FieldLocation.h" + +#include "eckit/io/rados/RadosKeyValue.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// @note: used in fdb-list index visiting, in DaosIndex::entries. During +/// visitation, DaosFieldLocations are built, which normally require +/// retrieving the location information from DAOS, inflicting RPCs. +/// This DaosLazyFieldLocation, instead, remains empty and the actual +/// information is only be retrieved from DAOS when stableLocation() +/// is called. This allows the visiting mechanism to discard unmatching +/// FieldLocations before any RPC is performed for them. +class RadosLazyFieldLocation : public FieldLocation { +public: + + RadosLazyFieldLocation(const fdb5::RadosLazyFieldLocation& rhs); + RadosLazyFieldLocation(const eckit::RadosKeyValue& index, const std::string& key); + + eckit::DataHandle* dataHandle() const override; + + virtual std::shared_ptr make_shared() const override; + + virtual void visit(FieldLocationVisitor& visitor) const override; + + virtual std::shared_ptr stableLocation() const override; + +private: // methods + + std::unique_ptr& realise() const; + + void print(std::ostream &out) const override; + +private: // members + + eckit::RadosKeyValue index_; + std::string key_; + mutable std::unique_ptr fl_; + +}; + + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index caa571e57..8ed8e4f57 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -51,7 +51,7 @@ RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& confi eckit::URI RadosStore::uri() const { -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL return eckit::RadosNamespace(pool_, db_namespace_).uri(); @@ -68,7 +68,7 @@ bool RadosStore::uriBelongs(const eckit::URI& uri) const { const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL ASSERT(n == 2 || n == 3); return ( @@ -97,7 +97,7 @@ bool RadosStore::uriExists(const eckit::URI& uri) const { ASSERT(uri.scheme() == type()); -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL ASSERT(n == 2 || n == 3); ASSERT(parts[0] == pool_); @@ -119,11 +119,11 @@ bool RadosStore::uriExists(const eckit::URI& uri) const { } -std::vector RadosStore::storeUnitURIs() const { +std::vector RadosStore::collocatedDataURIs() const { std::vector store_unit_uris; -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL eckit::RadosNamespace n{pool_, db_namespace_}; @@ -151,7 +151,7 @@ std::vector RadosStore::storeUnitURIs() const { } -std::set RadosStore::asStoreUnitURIs(const std::vector& uris) const { +std::set RadosStore::asCollocatedDataURIs(const std::vector& uris) const { std::set res; @@ -166,7 +166,7 @@ std::set RadosStore::asStoreUnitURIs(const std::vector& bool RadosStore::exists() const { -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL return eckit::RadosNamespace(pool_, db_namespace_).exists(); @@ -196,7 +196,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * /// @note: generate unique object name starting by indexkey_ eckit::RadosObject o = generateDataObject(key); - #ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL /// @todo: ensure pool if not yet seen by this process static std::set knownPools; @@ -208,11 +208,11 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * #endif - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH eckit::DataHandle* h = o.persistentDataHandle(); ASSERT(handles_.size() < maxHandleBuffSize_); handles_.push_back(h); - #elif fdb5_HAVE_RADOS_STORE_PERSIST_ON_WRITE + #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE std::unique_ptr h(o.persistentDataHandle(true)); #else std::unique_ptr h(o.dataHandle()); @@ -233,7 +233,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * /// @note: get or generate unique key name const eckit::RadosObject& o = getDataObject(key); - #ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL + #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL /// @todo: ensure pool if not yet seen by this process static std::set knownPools; @@ -263,7 +263,7 @@ void RadosStore::flush() { #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH for (const auto& h : handles_) h->flush(); #else // NOOP @@ -273,14 +273,14 @@ void RadosStore::flush() { #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH flushDataHandles(); #endif closeDataHandles(); #else - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH flushDataHandles(); #else // NOOP @@ -296,7 +296,7 @@ void RadosStore::close() { #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH for (const auto& h : handles_) h->close(); #else // NOOP @@ -315,7 +315,7 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL ASSERT(n == 2 || n == 3); @@ -380,7 +380,7 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost void RadosStore::print(std::ostream& out) const { -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL out << "RadosStore(" << pool_ << "/" << db_namespace_ << ")"; @@ -417,7 +417,7 @@ eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { eckit::MD5 md5(name); -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD @@ -470,7 +470,7 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH eckit::DataHandle *dh = name.persistentMultipartWriteHandle(maxObjectSize_, maxAioBuffSize_, maxPartHandleBuffSize_); #else eckit::DataHandle *dh = name.multipartWriteHandle(maxObjectSize_); @@ -478,7 +478,7 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO #else - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH eckit::DataHandle *dh = name.persistentDataHandle(false, maxAioBuffSize_); #else eckit::DataHandle *dh = name.dataHandle(); @@ -550,11 +550,11 @@ void RadosStore::parseConfig(const fdb5::Config& config) { if (rados.has("store")) store_conf = rados.getSubConfiguration("store"); } -#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) && defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) +#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) && defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) maxHandleBuffSize_ = store_conf.getInt("maxHandleBuffSize", 1024 * 1024); #endif -#if (!defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD)) && defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) +#if (!defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD)) && defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024); maxPartHandleBuffSize_ = store_conf.getInt("maxPartHandleBuffSize", 1024); diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 95111cff0..33989328f 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -40,8 +40,8 @@ class RadosStore : public Store, public RadosCommon { eckit::URI uri() const override; bool uriBelongs(const eckit::URI&) const override; bool uriExists(const eckit::URI&) const override; - std::vector storeUnitURIs() const override; - std::set asStoreUnitURIs(const std::vector&) const override; + std::vector collocatedDataURIs() const override; + std::set asCollocatedDataURIs(const std::vector&) const override; bool open() override { return true; } void flush() override; @@ -85,14 +85,14 @@ class RadosStore : public Store, public RadosCommon { // mutable bool dirty_; #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH std::vector handles_; size_t maxHandleBuffSize_; #endif #else HandleStore handles_; mutable ObjectStore dataObjects_; - #ifdef fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH + #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH size_t maxAioBuffSize_; #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART size_t maxPartHandleBuffSize_; @@ -104,4 +104,4 @@ class RadosStore : public Store, public RadosCommon { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index e9412379a..ff42e5ed1 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -161,7 +161,7 @@ bool TocWipeVisitor::visitIndex(const Index& index) { // Enumerate data files. - std::vector indexDataURIs(index.dataPaths()); + std::vector indexDataURIs(index.dataURIs()); for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataURIs)) { if (include) { if (!store_.uriBelongs(uri)) { diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index bba80fca4..eb5ddeff1 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -158,7 +158,7 @@ CASE("RadosStore tests") { SECTION("archive and retrieve") { -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool{"fdb-test1"}; eckit::RadosPool{pool}.ensureDestroyed(); @@ -184,9 +184,9 @@ CASE("RadosStore tests") { fdb5::Schema schema{schema_file()}; - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key db_key{"a=1,b=2"}; - fdb5::Key index_key{"c=3,d=4"}; + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}, schema.registry()); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); char data[] = "test"; @@ -211,7 +211,7 @@ CASE("RadosStore tests") { EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); // remove -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL eckit::RadosObject field_name{field.location().uri()}; eckit::RadosNamespace store_name = field_name.nspace(); eckit::URI store_uri(store_name.uri()); @@ -237,7 +237,7 @@ CASE("RadosStore tests") { SECTION("with POSIX Catalogue") { -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool{"fdb-test2"}; eckit::RadosPool{pool}.ensureDestroyed(); @@ -269,10 +269,10 @@ CASE("RadosStore tests") { // request - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key db_key{"a=1,b=2"}; - fdb5::Key index_key{"c=3,d=4"}; - fdb5::Key field_key{"e=5,f=6"}; + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}, schema.registry()); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}, schema.registry()); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}, schema.registry()); // store data @@ -320,7 +320,7 @@ CASE("RadosStore tests") { EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); // remove data -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL eckit::RadosObject field_name{field.location().uri()}; eckit::RadosNamespace store_name{field_name.nspace()}; eckit::URI store_uri(store_name.uri()); @@ -356,7 +356,7 @@ CASE("RadosStore tests") { SECTION("VIA FDB API") { -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool{"fdb-test3"}; eckit::RadosPool{pool}.ensureDestroyed(); @@ -375,7 +375,7 @@ CASE("RadosStore tests") { "rados:\n" }; -#ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL config_str += " poolPrefix: " + prefix + "\n"; #endif @@ -385,11 +385,11 @@ CASE("RadosStore tests") { config_str += " store:\n"; -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL config_str += " pool: " + pool + "\n"; #endif -#if defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxHandleBuffSize: 100\n"; #else @@ -406,9 +406,9 @@ CASE("RadosStore tests") { // request - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; - fdb5::Key db_key{"a=1,b=2"}; + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); fdb5::FDBToolRequest full_req{ request_key.request("retrieve"), @@ -453,7 +453,7 @@ CASE("RadosStore tests") { /// @note: maxObjectSize is set to 16, and four 10-byte fields are archived, spanning 3 objects for (int i = 0; i < 4; i++) { std::cout << "Archive field " << i << std::endl; - fdb5::Key request_key_i{std::string("a=1,b=2,c=3,d=4,e=5,f=") + std::to_string(6 + i)}; + fdb5::Key request_key_i({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); fdb.archive(request_key_i, data, sizeof(data)); } #else @@ -467,7 +467,7 @@ CASE("RadosStore tests") { #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) for (int i = 0; i < 4; i++) { std::cout << "Retrieve field " << i << std::endl; - fdb5::Key request_key_i{std::string("a=1,b=2,c=3,d=4,e=5,f=") + std::to_string(6 + i)}; + fdb5::Key request_key_i({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); metkit::mars::MarsRequest r_i = request_key_i.request("retrieve"); std::unique_ptr dh(fdb.retrieve(r_i)); @@ -547,7 +547,7 @@ CASE("RadosStore tests") { // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe SECTION("FDB API RE-STORE AND WIPE DB") { -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool{"fdb-test4"}; eckit::RadosPool{pool}.ensureDestroyed(); @@ -566,7 +566,7 @@ CASE("RadosStore tests") { "rados:\n" }; -#ifndef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL config_str += " poolPrefix: " + prefix + "\n"; #endif @@ -576,11 +576,11 @@ CASE("RadosStore tests") { config_str += " store:\n"; -#ifdef fdb5_HAVE_RADOS_STORE_SINGLE_POOL +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL config_str += " pool: " + pool + "\n"; #endif -#if defined(fdb5_HAVE_RADOS_STORE_PERSIST_ON_FLUSH) +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxHandleBuffSize: 100\n"; #else @@ -597,9 +597,9 @@ CASE("RadosStore tests") { // request - fdb5::Key request_key{"a=1,b=2,c=3,d=4,e=5,f=6"}; - fdb5::Key index_key{"a=1,b=2,c=3,d=4"}; - fdb5::Key db_key{"a=1,b=2"}; + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); fdb5::FDBToolRequest full_req{ request_key.request("retrieve"), @@ -672,4 +672,4 @@ int main(int argc, char **argv) ensureClean("fdb-test"); return ret; -} \ No newline at end of file +} From fe324d1ee445c638c9ecb5f46d221b25d8eefa76 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 23 Jun 2024 13:26:17 +0200 Subject: [PATCH 015/109] Rados wait_for_safe is deprecated. Only possible to do async IO and ensure persisted on flush, or wait for persist on every IO. --- CMakeLists.txt | 8 +-- src/fdb5/fdb5_config.h.in | 2 +- src/fdb5/rados/README | 33 +++------- src/fdb5/rados/RadosCatalogueWriter.cc | 1 - src/fdb5/rados/RadosCommon.cc | 22 +------ src/fdb5/rados/RadosCommon.h | 8 +-- src/fdb5/rados/RadosIndex.cc | 21 ------- src/fdb5/rados/RadosIndex.h | 8 +-- src/fdb5/rados/RadosIndexLocation.h | 2 +- src/fdb5/rados/RadosStore.cc | 8 +-- tests/fdb/rados/CMakeLists.txt | 3 +- tests/fdb/rados/test_rados_store.cc | 86 ++++++++++++++++++-------- 12 files changed, 88 insertions(+), 114 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e94dcba1e..de986f553 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,13 +63,11 @@ ecbuild_add_option( FEATURE RADOS_STORE_MULTIPART ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_FLUSH DEFAULT OFF - DESCRIPTION "Ensure writes are persisted in Rados storage on flush." ) + DESCRIPTION "Ensure writes/puts are persisted in Rados storage on flush rather than immediately." ) -ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_WRITE - CONDITION NOT fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH +ecbuild_add_option( FEATURE RADOS_ADMIN DEFAULT OFF - DESCRIPTION "Ensure every object write or kv put is persisted immediately." ) - + DESCRIPTION "Have unit tests create pools automatically rather than using an existing pool specified in the ECKIT_RADOS_TEST_POOL cmake variable." ) ### FDB backend in indexed filesystem with table-of-contents, i.e. TOC ### Supports Lustre parallel filesystem stripping control diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index f82c954fc..0b94ec012 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -14,7 +14,7 @@ #cmakedefine fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD #cmakedefine fdb5_HAVE_RADOS_STORE_MULTIPART #cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH -#cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE +#cmakedefine fdb5_HAVE_RADOS_ADMIN #cmakedefine fdb5_HAVE_TOCFDB #cmakedefine fdb5_HAVE_DUMMY_DAOS #cmakedefine fdb5_HAVE_DAOSFDB diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index 8b69052c1..a6c086bf8 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -52,61 +52,46 @@ cmake options: cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF # pool per db, multiple fields per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF # single pool, field per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF # pool per db, field per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF -# single pool, multiple fields per obj, multipart +# single pool, multiple fields per obj, multipart (default) cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF # single pool, multiple fields per obj, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON # single pool, field per obj, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF - -# single pool, field per obj, persist on write -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=ON + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON # single pool, multiple fields per obj, multipart, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF + -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index a59411781..b84f434db 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -342,7 +342,6 @@ void RadosCatalogueWriter::flush() { #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j ) j->second.flush(); -flush axis kvs if not done as part of RadosIndex::flush db_kv_->flush(); root_kv_->flush(); #endif diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 96c368e6d..bf69ad6f7 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -31,13 +31,8 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen readConfig(config, component, true); - #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - root_kv_.emplace(pool_, root_namespace_, "main_kv", true); - db_kv_.emplace(pool_, db_namespace_, "catalogue_kv", true); - #else root_kv_.emplace(pool_, root_namespace_, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); - #endif #else @@ -45,13 +40,8 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen db_pool_ = prefix_ + "_" + key.valuesToString(); - #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - root_kv_.emplace(root_pool_, namespace_, "main_kv", true); - db_kv_.emplace(db_pool_, namespace_, "catalogue_kv", true); - #else root_kv_.emplace(root_pool_, namespace_, "main_kv"); db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); - #endif #endif @@ -71,13 +61,8 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen readConfig(config, component, false); - #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - root_kv_.emplace(pool_, root_namespace_, "main_kv", true); - db_kv_.emplace(pool_, db_namespace_, "catalogue_kv", true); - #else root_kv_.emplace(pool_, root_namespace_, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); - #endif #else @@ -91,13 +76,8 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen ASSERT(n > 1); prefix_ = parts[0]; - #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - root_kv_.emplace(root_pool_, namespace_, "main_kv", true); - db_kv_.emplace(db_pool_, namespace_, "catalogue_kv", true); - #else root_kv_.emplace(root_pool_, namespace_, "main_kv"); db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); - #endif #endif @@ -150,7 +130,7 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); prefix_ = c.getString("pool_prefix", prefix_); - if (c.has(component)) prefix_ = c.getSubconfiguration(component).getString("pool_prefix", prefix_); + if (c.has(component)) prefix_ = c.getSubConfiguration(component).getString("pool_prefix", prefix_); ASSERT_MSG(prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); #endif diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index ef5c5a808..43ab61a1e 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -16,7 +16,7 @@ #include "eckit/filesystem/URI.h" #include "eckit/utils/Optional.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/io/rados/RadosPersistentKeyValue.h" +#include "eckit/io/rados/RadosAsyncKeyValue.h" #include "fdb5/fdb5_config.h" #include "fdb5/database/Key.h" @@ -51,9 +51,9 @@ class RadosCommon { std::string namespace_; #endif -#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - eckit::Optional root_kv_; - eckit::Optional db_kv_; +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + eckit::Optional root_kv_; + eckit::Optional db_kv_; #else eckit::Optional root_kv_; eckit::Optional db_kv_; diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index d8b8d2296..3a15d610f 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -46,11 +46,7 @@ namespace fdb5 { RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : IndexBase(key, "radosKeyValue"), location_(eckit::RadosKeyValue{name.pool().name(), name.name(), key.valuesToString()}, 0), -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - idx_kv_(location_.radosName().uri(), true) { -#else idx_kv_(location_.radosName().uri()) { -#endif /// @note: performed RPCs: /// - generate index kv oid (daos_obj_generate_oid) @@ -83,11 +79,7 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool re // #endif IndexBase(key, "radosKeyValue"), location_(name, 0), -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - idx_kv_(name.uri(), true) { -#else idx_kv_(name.uri()) { -#endif if (readAxes) updateAxes(); @@ -105,18 +97,6 @@ void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) if (axis_kv == axis_kvs_.end()) { std::string kv_name = key().valuesToString() + std::string{"."} + axis; -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - axis_kvs_.emplace( - std::piecewise_construct, - std::forward_as_tuple(axis), - std::forward_as_tuple( - location_.radosName().nspace().pool().name(), - location_.radosName().nspace().name(), - kv_name, - true - ) - ); -#else axis_kvs_.emplace( std::piecewise_construct, std::forward_as_tuple(axis), @@ -126,7 +106,6 @@ void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) kv_name ) ); -#endif axis_kv = axis_kvs_.find(axis); } diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index e6aca1f08..73302309f 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -15,7 +15,7 @@ #include "eckit/io/rados/RadosNamespace.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/io/rados/RadosPersistentKeyValue.h" +#include "eckit/io/rados/RadosAsyncKeyValue.h" #include "fdb5/database/Index.h" #include "fdb5/rados/RadosIndexLocation.h" @@ -82,9 +82,9 @@ class RadosIndex : public IndexBase { fdb5::RadosIndexLocation location_; -#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - eckit::RadosPersistentKeyValue idx_kv_; - std::map axis_kvs_; +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + eckit::RadosAsyncKeyValue idx_kv_; + std::map axis_kvs_; #else eckit::RadosKeyValue idx_kv_; std::map axis_kvs_; diff --git a/src/fdb5/rados/RadosIndexLocation.h b/src/fdb5/rados/RadosIndexLocation.h index 8058531e6..1546990cc 100644 --- a/src/fdb5/rados/RadosIndexLocation.h +++ b/src/fdb5/rados/RadosIndexLocation.h @@ -15,7 +15,7 @@ #include "eckit/exception/Exceptions.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/io/rados/RadosPersistentKeyValue.h" +#include "eckit/io/rados/RadosAsyncKeyValue.h" #include "fdb5/database/IndexLocation.h" diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 8ed8e4f57..6fbcae491 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -209,11 +209,9 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * #endif #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle* h = o.persistentDataHandle(); + eckit::DataHandle* h = o.asyncDataHandle(); ASSERT(handles_.size() < maxHandleBuffSize_); handles_.push_back(h); - #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - std::unique_ptr h(o.persistentDataHandle(true)); #else std::unique_ptr h(o.dataHandle()); #endif @@ -471,7 +469,7 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle *dh = name.persistentMultipartWriteHandle(maxObjectSize_, maxAioBuffSize_, maxPartHandleBuffSize_); + eckit::DataHandle *dh = name.asyncMultipartWriteHandle(maxObjectSize_, maxAioBuffSize_, maxPartHandleBuffSize_); #else eckit::DataHandle *dh = name.multipartWriteHandle(maxObjectSize_); #endif @@ -479,7 +477,7 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO #else #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle *dh = name.persistentDataHandle(false, maxAioBuffSize_); + eckit::DataHandle *dh = name.asyncDataHandle(maxAioBuffSize_); #else eckit::DataHandle *dh = name.dataHandle(); #endif diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt index 4b311927b..de2661407 100644 --- a/tests/fdb/rados/CMakeLists.txt +++ b/tests/fdb/rados/CMakeLists.txt @@ -11,7 +11,8 @@ if (HAVE_RADOSFDB) ecbuild_add_test( TARGET test_fdb5_rados_${_test} SOURCES test_${_test}.cc LIBS "${unit_test_libraries}" - INCLUDES "${unit_test_include_dirs}" ) + INCLUDES "${unit_test_include_dirs}" + ENVIRONMENT FDB_RADOS_TEST_POOL=${FDB_RADOS_TEST_POOL} ) endforeach() diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index eb5ddeff1..0b7f3eda1 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -11,7 +11,7 @@ // #include // #include -// #include "eckit/config/Resource.h" +#include "eckit/config/Resource.h" #include "eckit/testing/Test.h" // #include "eckit/filesystem/URI.h" #include "eckit/filesystem/PathName.h" @@ -66,6 +66,7 @@ namespace { // S3Config cfg("eu-central-1", "127.0.0.1", 8888); +#ifdef fdb5_HAVE_RADOS_ADMIN void ensureClean(const std::string& prefix) { ASSERT(prefix.length() > 3); for (const std::string& name : eckit::RadosCluster::instance().listPools()) { @@ -74,6 +75,7 @@ namespace { } } } +#endif } @@ -104,6 +106,12 @@ namespace test { CASE( "Setup" ) { +#if !defined(fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL) && !defined(fdb5_HAVE_RADOS_ADMIN) + throw eckit::Exception( + "RadosStore unit tests require Rados admin permissions to create pools if " + "RADOS_BACKENDS_SINGLE_POOL=OFF, and require enabling RADOS_ADMIN=ON."); +#endif + // ensure fdb root directory exists. If not, then that root is // registered as non existing and Store tests fail. if (store_tests_tmp_root().exists()) deldir(store_tests_tmp_root()); @@ -159,24 +167,30 @@ CASE("RadosStore tests") { SECTION("archive and retrieve") { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool{"fdb-test1"}; - + #ifdef eckit_HAVE_RADOS_ADMIN + std::string pool = "test-store1"; eckit::RadosPool{pool}.ensureDestroyed(); - eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer - + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer + #else + std::string pool; + pool = eckit::Resource( + "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool + ); + EXPECT(pool.length() > 0); + #endif std::string config_str{ "rados:\n" " store:\n" " pool: " + pool + "\n" }; #else - std::string prefix{"fdb-test1"}; + std::string prefix{"test-store1"}; ensureClean(prefix); std::string config_str{ "rados:\n" - " poolPrefix: " + prefix + "\n" + " pool_prefix: " + prefix + "\n" }; #endif @@ -238,11 +252,17 @@ CASE("RadosStore tests") { SECTION("with POSIX Catalogue") { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool{"fdb-test2"}; - + #ifdef eckit_HAVE_RADOS_ADMIN + std::string pool = "test-store2"; eckit::RadosPool{pool}.ensureDestroyed(); - eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer - + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer + #else + std::string pool; + pool = eckit::Resource( + "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool + ); + EXPECT(pool.length() > 0); + #endif std::string config_str{ "schema : " + schema_file().path() + "\n" "rados:\n" @@ -250,14 +270,14 @@ CASE("RadosStore tests") { " pool: " + pool + "\n" }; #else - std::string prefix{"fdb-test2"}; + std::string prefix{"test-store2"}; ensureClean(prefix); std::string config_str{ "schema : " + schema_file().path() + "\n" "rados:\n" - " poolPrefix: " + prefix + "\n" + " pool_prefix: " + prefix + "\n" }; #endif @@ -357,13 +377,19 @@ CASE("RadosStore tests") { SECTION("VIA FDB API") { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool{"fdb-test3"}; - + #ifdef eckit_HAVE_RADOS_ADMIN + std::string pool = "test-store3"; eckit::RadosPool{pool}.ensureDestroyed(); - eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer + #else + std::string pool; + pool = eckit::Resource( + "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool + ); + EXPECT(pool.length() > 0); + #endif #else - std::string prefix{"fdb-test3"}; - + std::string prefix{"test-store3"}; ensureClean(prefix); #endif @@ -376,7 +402,7 @@ CASE("RadosStore tests") { }; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " poolPrefix: " + prefix + "\n"; + config_str += " pool_prefix: " + prefix + "\n"; #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) @@ -548,13 +574,19 @@ CASE("RadosStore tests") { SECTION("FDB API RE-STORE AND WIPE DB") { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool{"fdb-test4"}; - + #ifdef eckit_HAVE_RADOS_ADMIN + std::string pool = "test-store4"; eckit::RadosPool{pool}.ensureDestroyed(); - eckit::RadosPool{pool}.create(); /// @todo: auto pool destroyer + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer + #else + std::string pool; + pool = eckit::Resource( + "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool + ); + EXPECT(pool.length() > 0); + #endif #else - std::string prefix{"fdb-test4"}; - + std::string prefix{"test-store4"}; ensureClean(prefix); #endif @@ -567,7 +599,7 @@ CASE("RadosStore tests") { }; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " poolPrefix: " + prefix + "\n"; + config_str += " pool_prefix: " + prefix + "\n"; #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) @@ -669,7 +701,9 @@ int main(int argc, char **argv) ret = run_tests ( argc, argv ); } catch(...) {} - ensureClean("fdb-test"); +#ifdef fdb5_HAVE_RADOS_ADMIN + ensureClean("test-store"); +#endif return ret; } From 006bea57508d806cd7db4bc9e16a0e29b4a526a9 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Tue, 25 Jun 2024 18:05:35 +0200 Subject: [PATCH 016/109] Added RadosCatalogue unit tests and several fixes. --- src/fdb5/CMakeLists.txt | 2 + src/fdb5/rados/RadosCatalogue.cc | 101 ++- src/fdb5/rados/RadosCatalogueWriter.cc | 10 +- src/fdb5/rados/RadosCommon.cc | 21 +- src/fdb5/rados/RadosCommon.h | 6 +- src/fdb5/rados/RadosEngine.cc | 291 ++++++++ src/fdb5/rados/RadosEngine.h | 97 +++ src/fdb5/rados/RadosFieldLocation.cc | 46 -- src/fdb5/rados/RadosIndex.cc | 4 +- src/fdb5/rados/RadosIndex.h | 1 + src/fdb5/rados/RadosStore.cc | 4 +- src/fdb5/rados/RadosStore.h | 2 - tests/fdb/rados/CMakeLists.txt | 1 + tests/fdb/rados/test_rados_catalogue.cc | 870 ++++++++++++++++++++++++ tests/fdb/rados/test_rados_store.cc | 115 ++-- 15 files changed, 1402 insertions(+), 169 deletions(-) create mode 100644 src/fdb5/rados/RadosEngine.cc create mode 100644 src/fdb5/rados/RadosEngine.h create mode 100644 tests/fdb/rados/test_rados_catalogue.cc diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8ccc6b39f..9f5841794 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -381,6 +381,8 @@ if( HAVE_RADOSFDB ) rados/RadosIndexLocation.h rados/RadosLazyFieldLocation.cc rados/RadosLazyFieldLocation.h + rados/RadosEngine.cc + rados/RadosEngine.h ) endif() diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index e6a84a05a..d45c57172 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -19,7 +19,7 @@ #include "fdb5/rados/RadosCatalogue.h" // #include "fdb5/daos/DaosName.h" // #include "fdb5/daos/DaosSession.h" -// #include "fdb5/daos/DaosIndex.h" +#include "fdb5/rados/RadosIndex.h" // #include "fdb5/daos/DaosWipeVisitor.h" // using namespace eckit; @@ -109,60 +109,53 @@ WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars: std::vector RadosCatalogue::indexes(bool) const { - NOTIMP; + /// @note: sorted is not implemented as is not necessary in this backend. + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db kv list keys (daos_kv_list) + + std::vector res; + + for (const auto& key : db_kv_->keys()) { + + /// @todo: document these well. Single source these reserved values. + /// Ensure where appropriate that user-provided keys do not collide. + if (key == "schema" || key == "key") continue; + + /// @note: performed RPCs: + /// - db kv get index location size (daos_kv_get without a buffer) + /// - db kv get index location (daos_kv_get) + std::vector v; + auto m = db_kv_->getMemoryStream(v, key, "DB kv"); + + eckit::URI uri(std::string(v.begin(), v.end())); + + /// @note: performed RPCs: + /// - index kv open (daos_kv_open) + /// - index kv get size (daos_kv_get without a buffer) + /// - index kv get key (daos_kv_get) + /// @note: the following three lines intend to check whether the index kv exists + /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, + /// so it is not useful on its own to check whether the index KV existed or not. + /// Instead, presence of a "key" key in the KV is used to determine if the index + /// KV existed. + eckit::RadosKeyValue index_kv{uri}; + std::optional index_key; + try { + std::vector data; + eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); + index_key.emplace(ms); + } catch (eckit::RadosEntityNotFoundException& e) { + continue; /// @note: the index_kv may not exist after a failed wipe + /// @todo: the index_kv may exist even if it does not have the "key" key + } + + res.push_back(Index(new fdb5::RadosIndex(index_key.value(), index_kv, false))); + + } -// /// @note: sorted is not implemented as is not necessary in this backend. - -// fdb5::DaosKeyValueName catalogue_kv_name{pool_, db_cont_, catalogue_kv_}; -// fdb5::DaosSession s{}; - -// /// @note: performed RPCs: -// /// - db kv open (daos_kv_open) -// /// - db kv list keys (daos_kv_list) -// fdb5::DaosKeyValue catalogue_kv{s, catalogue_kv_name}; /// @note: throws if not exists - -// std::vector res; - -// for (const auto& key : catalogue_kv.keys()) { - -// /// @todo: document these well. Single source these reserved values. -// /// Ensure where appropriate that user-provided keys do not collide. -// if (key == "schema" || key == "key") continue; - -// /// @note: performed RPCs: -// /// - db kv get index location size (daos_kv_get without a buffer) -// /// - db kv get index location (daos_kv_get) -// uint64_t size{catalogue_kv.size(key)}; -// std::vector v(size); -// catalogue_kv.get(key, v.data(), size); - -// fdb5::DaosKeyValueName index_kv_name{eckit::URI(std::string(v.begin(), v.end()))}; - -// /// @note: performed RPCs: -// /// - index kv open (daos_kv_open) -// /// - index kv get size (daos_kv_get without a buffer) -// /// - index kv get key (daos_kv_get) -// /// @note: the following three lines intend to check whether the index kv exists -// /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, -// /// so it is not useful on its own to check whether the index KV existed or not. -// /// Instead, presence of a "key" key in the KV is used to determine if the index -// /// KV existed. -// fdb5::DaosKeyValue index_kv{s, index_kv_name}; -// std::optional index_key; -// try { -// std::vector data; -// eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); -// index_key.emplace(ms); -// } catch (fdb5::DaosEntityNotFoundException& e) { -// continue; /// @note: the index_kv may not exist after a failed wipe -// /// @todo: the index_kv may exist even if it does not have the "key" key -// } - -// res.push_back(Index(new fdb5::DaosIndex(index_key.value(), index_kv_name, false))); - -// } - -// return res; + return res; } diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index b84f434db..eb0a2471a 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -52,9 +52,13 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c /// @note: performed RPCs: /// - check if main kv contains db key (daos_kv_get without a buffer) + root_kv_->ensureCreated(); if (!root_kv_->has(db_name)) { /// create catalogue kv +#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + db_kv_->nspace().pool().ensureCreated(); +#endif db_kv_->ensureCreated(); /// write schema under "schema" @@ -68,7 +72,11 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c eckit::FileHandle in(config_.schemaPath()); std::vector data; data.resize(in.size()); - in.read(&data[0], in.size()); + { + eckit::AutoClose ac{in}; + in.openForRead(); + in.read(&data[0], in.size()); + } db_kv_->put("schema", &data[0], data.size()); /// write dbKey under "key" diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index bf69ad6f7..778122712 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -27,10 +27,10 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - db_namespace_ = key.valuesToString(); - readConfig(config, component, true); + db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); + root_kv_.emplace(pool_, root_namespace_, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); @@ -38,7 +38,7 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen readConfig(config, component, true); - db_pool_ = prefix_ + "_" + key.valuesToString(); + db_pool_ = pool_prefix_ + "_" + key.valuesToString(); root_kv_.emplace(root_pool_, namespace_, "main_kv"); db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); @@ -74,7 +74,7 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen const auto parts = eckit::Tokenizer("_").tokenize(db_pool_); const auto n = parts.size(); ASSERT(n > 1); - prefix_ = parts[0]; + pool_prefix_ = parts[0]; root_kv_.emplace(root_pool_, namespace_, "main_kv"); db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); @@ -110,28 +110,35 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp pool_ = c.getString("pool", pool_); if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); } + root_namespace_ = c.getString("root_namespace", root_namespace_); if (c.has(component)) root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); if (readPool) pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); root_namespace_ = eckit::Resource("fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); + if (c.has(component)) nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); + ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); + #else if (readNamespace) namespace_ = "default"; root_pool_ = "root"; if (readNamespace) + namespace_ = c.getString("namespace", namespace_); if (c.has(component)) namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + root_pool_ = c.getString("root_pool", root_pool_); if (c.has(component)) root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); if (readNamespace) namespace_ = eckit::Resource("fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); - prefix_ = c.getString("pool_prefix", prefix_); - if (c.has(component)) prefix_ = c.getSubConfiguration(component).getString("pool_prefix", prefix_); - ASSERT_MSG(prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); + pool_prefix_ = c.getString("pool_prefix", pool_prefix_); + if (c.has(component)) pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); #endif diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 43ab61a1e..24b94bf51 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -61,10 +61,12 @@ class RadosCommon { eckit::Length maxObjectSize_; -#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL private: // members - std::string prefix_; +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string nspace_prefix_; +#else + std::string pool_prefix_; #endif }; diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc new file mode 100644 index 000000000..72238d852 --- /dev/null +++ b/src/fdb5/rados/RadosEngine.cc @@ -0,0 +1,291 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/config/Resource.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/rados/RadosEngine.h" + +using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +std::string RadosEngine::name() const { + return RadosEngine::typeName(); +} + +// bool DaosEngine::canHandle(const eckit::URI& uri, const Config& config) const { + +// configureDaos(config); + +// if (uri.scheme() != "daos") +// return false; + +// fdb5::DaosName n{uri}; + +// if (!n.hasOID()) return false; + +// /// @todo: check containerName is not root_cont_. root_cont_ should be populated in +// /// configureDaos as done in DaosCommon +// // bool is_root_name = (n.containerName().find(root_cont_) != std::string::npos); +// bool is_root_name = false; +// bool is_store_name = (n.containerName().find("_") != std::string::npos); + +// /// @note: performed RPCs: +// /// - generate oids (daos_obj_generate_oid) +// /// - db kv open (daos_kv_open) + +// fdb5::DaosName n2{n.poolName(), n.containerName(), catalogue_kv_}; +// bool is_catalogue_kv = (!is_root_name && !is_store_name && (n.OID() == n2.OID())); + +// return is_catalogue_kv && n.exists(); + +// } + +std::vector RadosEngine::visitableLocations(const Key& key, const Config& config) const +{ + + /// @note: code mostly copied from DaosCommon + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// DaosCommon are not accessible from here + + std::string component = "catalogue"; + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + readConfig(config, component, true); + + // db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(pool_, root_namespace_, "main_kv"); + // db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); + +#else + + readConfig(config, component, true); + + // db_pool_ = pool_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(root_pool_, namespace_, "main_kv"); + // db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); + +#endif + + /// --- + + std::vector res{}; + + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + + if (!root_kv_->exists()) return res; + + /// @note: performed RPCs: + /// - main kv list keys (daos_kv_list) + for (const auto& k : root_kv_->keys()) { + + try { + + /// @note: performed RPCs: + /// - main kv get db location size (daos_kv_get without a buffer) + /// - main kv get db location (daos_kv_get) + std::vector v; + auto m = root_kv_->getMemoryStream(v, k, "root kv"); + + eckit::URI uri(std::string(v.begin(), v.end())); + ASSERT(uri.scheme() == typeName()); + + /// @todo: this exact deserialisation is performed twice. Once here and once + /// in DaosCatalogue::(uri, ...). Try to avoid one. + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db key get size (daos_kv_get without a buffer) + /// - db key get (daos_kv_get) + eckit::RadosKeyValue db_kv{uri}; /// @note: includes exist check + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + fdb5::Key db_key(ms); + + if (db_key.match(key)) { + + Log::debug() << " found match with " << root_kv_->uri() << " at key " << k << std::endl; + res.push_back(uri); + + } + + } catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; + eckit::Log::error() << e.what() << std::endl; + } + + } + + return res; + +} + +std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const +{ + + /// @note: code mostly copied from DaosCommon + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// DaosCommon are not accessible from here + + std::string component = "catalogue"; + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + readConfig(config, component, true); + + // db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(pool_, root_namespace_, "main_kv"); + // db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); + +#else + + readConfig(config, component, true); + + // db_pool_ = pool_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(root_pool_, namespace_, "main_kv"); + // db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); + +#endif + + /// --- + + std::vector res{}; + + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + + if (!root_kv_->exists()) return res; + + /// @note: performed RPCs: + /// - main kv list keys (daos_kv_list) + for (const auto& k : root_kv_->keys()) { + + try { + + /// @note: performed RPCs: + /// - main kv get db location size (daos_kv_get without a buffer) + /// - main kv get db location (daos_kv_get) + std::vector v; + auto m = root_kv_->getMemoryStream(v, k, "root kv"); + + eckit::URI uri(std::string(v.begin(), v.end())); + ASSERT(uri.scheme() == typeName()); + + /// @todo: this exact deserialisation is performed twice. Once here and once + /// in DaosCatalogue::(uri, ...). Try to avoid one. + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db key get size (daos_kv_get without a buffer) + /// - db key get (daos_kv_get) + eckit::RadosKeyValue db_kv{uri}; /// @note: includes exist check + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + fdb5::Key db_key(ms); + + if (db_key.partialMatch(request)) { + + Log::debug() << " found match with " << root_kv_->uri() << " at key " << k << std::endl; + res.push_back(uri); + + } + + } catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; + eckit::Log::error() << e.what() << std::endl; + } + + } + + return res; + +} + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL +void RadosEngine::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const { +#else +void RadosEngine::readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) const { +#endif + + eckit::LocalConfiguration c{}; + + if (config.has("rados")) c = config.getSubConfiguration("rados"); + + // maxObjectSize_ = c.getInt("maxObjectSize", 0); + + std::string first_cap{component}; + first_cap[0] = toupper(component[0]); + + std::string all_caps{component}; + for (auto & c: all_caps) c = toupper(c); + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + if (readPool) pool_ = "default"; + root_namespace_ = "root"; + + if (readPool) { + pool_ = c.getString("pool", pool_); + if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); + } + root_namespace_ = c.getString("root_namespace", root_namespace_); + if (c.has(component)) root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + + if (readPool) + pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); + root_namespace_ = eckit::Resource("fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + + nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); + if (c.has(component)) nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); + ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); + +#else + + if (readNamespace) namespace_ = "default"; + root_pool_ = "root"; + + if (readNamespace) + namespace_ = c.getString("namespace", namespace_); + if (c.has(component)) namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + root_pool_ = c.getString("root_pool", root_pool_); + if (c.has(component)) root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); + + if (readNamespace) + namespace_ = eckit::Resource("fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); + root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); + + pool_prefix_ = c.getString("pool_prefix", pool_prefix_); + if (c.has(component)) pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); + +#endif + +} + +static EngineBuilder rados_builder; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h new file mode 100644 index 000000000..b48f7dc2c --- /dev/null +++ b/src/fdb5/rados/RadosEngine.h @@ -0,0 +1,97 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "eckit/utils/Optional.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosAsyncKeyValue.h" + +#include "fdb5/database/Engine.h" +#include "fdb5/fdb5_config.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class RadosEngine : public fdb5::Engine { + +public: // methods + + RadosEngine() {}; + + static const char* typeName() { return "rados"; } + +protected: // methods + + virtual std::string name() const override; + + virtual std::string dbType() const override { NOTIMP; }; + + virtual eckit::URI location(const Key &key, const Config& config) const override { NOTIMP; }; + + virtual bool canHandle(const eckit::URI&, const Config&) const override { NOTIMP; }; + + virtual std::vector allLocations(const Key& key, const Config& config) const override { NOTIMP; }; + + virtual std::vector visitableLocations(const Key& key, const Config& config) const override; + virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; + + virtual std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; }; + + virtual void print( std::ostream &out ) const override { NOTIMP; }; + +private: // methods + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + void readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const; +#else + void readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) const; +#endif + +protected: // members + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + mutable std::string pool_; + mutable std::string root_namespace_; + // std::string db_namespace_; +#else + mutable std::string root_pool_; + // std::string db_pool_; + mutable std::string namespace_; +#endif + +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + mutable eckit::Optional root_kv_; + // eckit::Optional db_kv_; +#else + mutable eckit::Optional root_kv_; + // eckit::Optional db_kv_; +#endif + + // eckit::Length maxObjectSize_; + +private: // members + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + mutable std::string nspace_prefix_; +#else + mutable std::string pool_prefix_; +#endif + +}; + +//---------------------------------------------------------------------------------------------------------------------- + + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index bafbbb99f..919d11f49 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -79,50 +79,4 @@ void RadosFieldLocation::visit(FieldLocationVisitor& visitor) const { //---------------------------------------------------------------------------------------------------------------------- -class RadosURIManager : public eckit::URIManager { - virtual bool query() override { return true; } - virtual bool fragment() override { return true; } - - // virtual eckit::PathName path(const eckit::URI& f) const override { return f.name(); } - - virtual bool exists(const eckit::URI& f) override { - - return eckit::RadosObject(f).exists(); - - } - - virtual eckit::DataHandle* newWriteHandle(const eckit::URI& f) override { - - return eckit::RadosObject(f).dataHandle(); - - } - - virtual eckit::DataHandle* newReadHandle(const eckit::URI& f) override { - - return eckit::RadosObject(f).dataHandle(); - - } - - virtual eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { - - NOTIMP; - - } - - virtual std::string asString(const eckit::URI& uri) const override { - std::string q = uri.query(); - if (!q.empty()) - q = "?" + q; - std::string f = uri.fragment(); - if (!f.empty()) - f = "#" + f; - - return uri.scheme() + ":" + uri.name() + q + f; - } -public: - RadosURIManager(const std::string& name) : eckit::URIManager(name) {} -}; - -static RadosURIManager rados_uri_manager("rados"); - } // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 3a15d610f..9cce9feb4 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -290,8 +290,8 @@ const std::vector RadosIndex::dataURIs() const { #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH void RadosIndex::flush() { - for (auto axis : axis_kvs_) { - axis->second.flush(); + for (auto& axis : axis_kvs_) { + axis.second.flush(); } idx_kv_.flush(); diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index 73302309f..4762b4d5a 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -17,6 +17,7 @@ #include "eckit/io/rados/RadosKeyValue.h" #include "eckit/io/rados/RadosAsyncKeyValue.h" +#include "fdb5/fdb5_config.h" #include "fdb5/database/Index.h" #include "fdb5/rados/RadosIndexLocation.h" diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 6fbcae491..437f14fed 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -224,7 +224,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * h->write(data, length); - return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key())); + return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key(nullptr, true))); #else @@ -251,7 +251,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * ASSERT(len == length); - return std::unique_ptr(new RadosFieldLocation(o.uri(), offset, length, fdb5::Key())); + return std::unique_ptr(new RadosFieldLocation(o.uri(), offset, length, fdb5::Key(nullptr, true))); #endif diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 33989328f..144f98d53 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -16,8 +16,6 @@ #include "eckit/io/rados/RadosObject.h" -#include "fdb5/fdb5_config.h" - #include "fdb5/database/Store.h" #include "fdb5/rules/Schema.h" diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt index de2661407..30d4caf3b 100644 --- a/tests/fdb/rados/CMakeLists.txt +++ b/tests/fdb/rados/CMakeLists.txt @@ -2,6 +2,7 @@ if (HAVE_RADOSFDB) list( APPEND rados_tests rados_store + rados_catalogue ) list( APPEND unit_test_libraries fdb5 ) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc new file mode 100644 index 000000000..fab033e13 --- /dev/null +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -0,0 +1,870 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +// #include +// #include + +#include "eckit/config/Resource.h" +#include "eckit/testing/Test.h" +// #include "eckit/filesystem/URI.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +// #include "eckit/filesystem/TmpDir.h" +// #include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/io/PartHandle.h" + +// #include "metkit/mars/MarsRequest.h" + +#include "fdb5/fdb5_config.h" +// #include "fdb5/config/Config.h" +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" + +#include "fdb5/toc/TocStore.h" + +// #include "fdb5/daos/DaosSession.h" +// #include "fdb5/daos/DaosPool.h" +// #include "fdb5/daos/DaosArrayPartHandle.h" + +#include "fdb5/rados/RadosStore.h" +#include "fdb5/rados/RadosFieldLocation.h" +#include "fdb5/rados/RadosCatalogueWriter.h" +#include "fdb5/rados/RadosCatalogueReader.h" + +using namespace eckit::testing; +using namespace eckit; + +namespace { + + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; + + void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); + } + } + } + +#ifdef fdb5_HAVE_RADOS_ADMIN + void ensureClean(const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listPools()) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosPool{name}.destroy(); + } + } + } +#endif + +} + +// temporary schema,spaces,root files common to all DAOS Catalogue tests + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& opt_schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& catalogue_tests_tmp_root() { + static eckit::PathName cd("./rados_catalogue_tests_fdb_root"); + return cd; +} + +namespace fdb { +namespace test { + +CASE( "Setup" ) { + +#if !defined(fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL) && !defined(fdb5_HAVE_RADOS_ADMIN) + throw eckit::Exception( + "RadosStore unit tests require Rados admin permissions to create pools if " + "RADOS_BACKENDS_SINGLE_POOL=OFF, and require enabling RADOS_ADMIN=ON."); +#endif + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Catalogue/Store tests fail. + if (catalogue_tests_tmp_root().exists()) deldir(catalogue_tests_tmp_root()); + catalogue_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); + + // prepare schema for tests involving DaosCatalogue + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + std::string opt_schema_str{"[ a, b [ c?, d [ e?, f ]]]"}; + + std::unique_ptr hs_opt(opt_schema_file().fileHandle()); + hs_opt->openForWrite(opt_schema_str.size()); + { + eckit::AutoClose closer(*hs_opt); + hs_opt->write(opt_schema_str.data(), opt_schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); + +} + +CASE("RadosCatalogue tests") { + + std::string test_id = "test-catalogue"; +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + #ifdef eckit_HAVE_RADOS_ADMIN + std::string pool = test_id; + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer + #else + std::string pool; + pool = eckit::Resource( + "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool + ); + EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); + #endif +#else + std::string prefix = test_id; + ensureClean(prefix); +#endif + + SECTION("DaosCatalogue archive (index) and retrieve without a Store") { + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" + }; +#else + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " catalogue:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n" + }; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Schema schema{schema_file()}; + + /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using a=1,b=2 + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}, schema.registry()); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}, schema.registry()); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}, schema.registry()); + + // archive + + std::unique_ptr loc(new fdb5::RadosFieldLocation( + eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key(nullptr, true) + )); + + { + fdb5::RadosCatalogueWriter dcatw{db_key, config}; + + // fdb5::DaosName db_cont{pool_name, db_key.valuesToString()}; + // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + // EXPECT(db_cont.exists()); + // EXPECT(cat_kv.exists()); + + fdb5::Catalogue& cat = dcatw; + cat.selectIndex(index_key); + // fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from config + // fdb5::DaosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; + // EXPECT(index_kv.exists()); + // EXPECT(cat_kv.has(index_key.valuesToString())); + + fdb5::CatalogueWriter& catw = dcatw; + catw.archive(field_key, std::move(loc)); + cat.flush(); + // EXPECT(index_kv.has(field_key.valuesToString())); + // fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; + // fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; + // EXPECT(e_axis_kv.exists()); + // EXPECT(e_axis_kv.has("5")); + // fdb5::DaosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; + // fdb5::DaosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; + // EXPECT(f_axis_kv.exists()); + // EXPECT(f_axis_kv.has("6")); + } + + // retrieve + + { + fdb5::RadosCatalogueReader dcatr{db_key, config}; + + fdb5::Catalogue& cat = dcatr; + cat.selectIndex(index_key); + + fdb5::Field f; + fdb5::CatalogueReader& catr = dcatr; + catr.retrieve(field_key, f); + EXPECT(f.location().uri().name() == eckit::URI("rados", "test_uri").name()); + EXPECT(f.location().offset() == eckit::Offset(0)); + EXPECT(f.location().length() == eckit::Length(1)); + } + + // // remove (manual deindex) + + // { + // fdb5::DaosCatalogueWriter dcatw{db_key, config}; + // fdb5::DaosName db_cont{dcatw.uri()}; + // std::ostream out(std::cout.rdbuf()); + + // fdb5::DaosCatalogue::remove(db_cont, out, out, true); + + // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + // EXPECT_NOT(cat_kv.exists()); + // EXPECT_NOT(db_cont.exists()); + // } + + } + + SECTION("RadosCatalogue archive (index) and retrieve with a RadosStore") { + + // FDB configuration + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" + }; +#else + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n" + }; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // store data + + char data[] = "test"; + + fdb5::RadosStore rstore{schema, db_key, config}; + fdb5::Store& store = static_cast(rstore); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + // index data + + { + fdb5::RadosCatalogueWriter rcatw{db_key, config}; + fdb5::Catalogue& cat = rcatw; + cat.deselectIndex(); + cat.selectIndex(index_key); + fdb5::CatalogueWriter& catw = rcatw; + catw.archive(field_key, std::move(loc)); + + /// flush store before flushing catalogue + rstore.flush(); // not necessary if using a DAOS store + } + + // find data + + fdb5::Field field; + { + fdb5::RadosCatalogueReader rcatr{db_key, config}; + fdb5::Catalogue& cat = rcatr; + cat.selectIndex(index_key); + fdb5::CatalogueReader& catr = rcatr; + catr.retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // // deindex data + + // { + // fdb5::DaosCatalogueWriter dcat{db_key, config}; + // fdb5::Catalogue& cat = static_cast(dcat); + // std::ostream out(std::cout.rdbuf()); + // metkit::mars::MarsRequest r = db_key.request("retrieve"); + // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + // cat.visitEntries(*wv, store, false); + // } + + } + + // SECTION("DaosCatalogue archive (index) and retrieve with a TocStore") { + + // // FDB configuration + + // std::string config_str{ + // "spaces:\n" + // "- roots:\n" + // " - path: " + catalogue_tests_tmp_root().asString() + "\n" + // "schema : " + schema_file().path() + "\n" + // "daos:\n" + // " catalogue:\n" + // " pool: " + pool_name + "\n" + // " root_cont: " + root_cont_name + "\n" + // " client:\n" + // " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + // }; + + // fdb5::Config config{YAMLConfiguration(config_str)}; + + // // schema + + // fdb5::Schema schema{schema_file()}; + + // // request + + // fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + // fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + // fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + // fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // // store data + + // char data[] = "test"; + + // fdb5::TocStore tstore{schema, db_key, config}; + // fdb5::Store& store = static_cast(tstore); + // std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + // /// @todo: there are two cont create with label here + // /// @todo: again, daos_fini happening before cont and pool close + + // // index data + + // { + // fdb5::DaosCatalogueWriter dcatw{db_key, config}; + // fdb5::Catalogue& cat = dcatw; + // cat.deselectIndex(); + // cat.selectIndex(index_key); + // fdb5::CatalogueWriter& catw = dcatw; + // catw.archive(field_key, std::move(loc)); + + // /// flush store before flushing catalogue + // tstore.flush(); + // } + + // // find data + + // fdb5::Field field; + // { + // fdb5::DaosCatalogueReader dcatr{db_key, config}; + // fdb5::Catalogue& cat = dcatr; + // cat.selectIndex(index_key); + // fdb5::CatalogueReader& catr = dcatr; + // catr.retrieve(field_key, field); + // } + // std::cout << "Read location: " << field.location() << std::endl; + + // // retrieve data + + // std::unique_ptr dh(store.retrieve(field)); + + // std::vector test(dh->size()); + // dh->openForRead(); + // { + // eckit::AutoClose closer(*dh); + // dh->read(&test[0], test.size() - 3); + // } + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data))); + // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // // remove data + + // /// @todo: should DaosStore::remove accept full URIs to field arrays and remove the store container? + // eckit::PathName store_path{field.location().uri().path()}; + // std::ostream out(std::cout.rdbuf()); + // store.remove(field.location().uri(), out, out, false); + // EXPECT(store_path.exists()); + // store.remove(field.location().uri(), out, out, true); + // EXPECT_NOT(store_path.exists()); + + // // deindex data + + // { + // fdb5::DaosCatalogueWriter dcat{db_key, config}; + // fdb5::Catalogue& cat = static_cast(dcat); + // std::ostream out(std::cout.rdbuf()); + // metkit::mars::MarsRequest r = db_key.request("retrieve"); + // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + // cat.visitEntries(*wv, store, false); + // } + + // /// @todo: again, daos_fini happening before + + // } + + SECTION("Via FDB API with a Rados catalogue and store") { + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: rados\n" + "store: rados\n" + "rados:\n" + }; + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + config_str += " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n"; +#else + config_str += " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n"; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest all_req{ + metkit::mars::MarsRequest{}, + true, + std::vector{} + }; + + // initialise FDB + + fdb5::FDB fdb(config); + + // check FDB is empty + + size_t count; + fdb5::ListElement info; + + /// @todo: here, DaosManager is being configured with DAOS client config passed to FDB instance constructor. + // It happens in EntryVisitMechanism::visit when calling DB::open. Is this OK, or should this configuring + // rather happen as part of transforming a FieldLocation into a DataHandle? It is probably OK. One thing + // is to configure the DAOS client and the other thing is to initialise it. + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // archive data + + char data[] = "test"; + + /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times below. + // Should this be avoided? + fdb.archive(request_key, data, sizeof(data)); + fdb.flush(); + + // retrieve data + + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // list all + + listObject = fdb.list(all_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // // wipe data + + // fdb5::WipeElement elem; + + // // dry run attempt to wipe with too specific request + + // auto wipeObject = fdb.wipe(full_req); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count == 0); + + // // dry run wipe index and store unit + // wipeObject = fdb.wipe(index_req); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + + // // dry run wipe database + // wipeObject = fdb.wipe(db_req); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + + // // ensure field still exists + // listObject = fdb.list(full_req); + // count = 0; + // while (listObject.next(info)) { + // // info.print(std::cout, true, true); + // // std::cout << std::endl; + // count++; + // } + // EXPECT(count == 1); + + // // attempt to wipe with too specific request + // wipeObject = fdb.wipe(full_req, true); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count == 0); + // /// @todo: really needed? + // fdb.flush(); + + // // wipe index and store unit + // wipeObject = fdb.wipe(index_req, true); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + // /// @todo: really needed? + // fdb.flush(); + + // // ensure field does not exist + // listObject = fdb.list(full_req); + // count = 0; + // while (listObject.next(info)) count++; + // EXPECT(count == 0); + + // /// @todo: ensure index and corresponding container do not exist + // /// @todo: ensure DB still exists + // /// @todo: list db or index and expect count = 0? + + // // re-archive data + + // /// @note: FDB holds a LocalFDB which holds an Archiver which holds open DBs (DaosCatalogueWriters). + // /// If a whole DB is wiped, the top-level structures for that DB (main and catalogue KVs in this case) + // /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the + // /// top-level structures are only generated as part of the DaosCatalogueWriter constructor. There is + // /// no way currently to destroy the open DBs held by FDB other than entirely destroying FDB. + // /// Alternatively, a separate FDB instance can be created. + // fdb5::FDB fdb2(config); + + // fdb2.archive(request_key, data, sizeof(data)); + + // fdb2.flush(); + + // listObject = fdb2.list(full_req); + // count = 0; + // while (listObject.next(info)) { + // // info.print(std::cout, true, true); + // // std::cout << std::endl; + // count++; + // } + // EXPECT(count == 1); + + // // wipe full database + + // wipeObject = fdb2.wipe(db_req, true); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + // /// @todo: really needed? + // fdb2.flush(); + + // // ensure field does not exist + + // listObject = fdb2.list(full_req); + // count = 0; + // while (listObject.next(info)) { + // // info.print(std::cout, true, true); + // // std::cout << std::endl; + // count++; + // } + // EXPECT(count == 0); + + // /// @todo: ensure DB and corresponding pool do not exist + + // /// @todo: ensure new DaosSession has updated daos client config + + } + + // SECTION("OPTIONAL SCHEMA KEYS") { + + // // FDB configuration + + // ::setenv("FDB_SCHEMA_FILE", opt_schema_file().path().c_str(), 1); + + // std::string config_str{ + // "spaces:\n" + // "- roots:\n" + // " - path: " + catalogue_tests_tmp_root().asString() + "\n" + // "type: local\n" + // "schema : " + opt_schema_file().path() + "\n" + // "engine: daos\n" + // "store: daos\n" + // "daos:\n" + // " catalogue:\n" + // " pool: " + pool_name + "\n" + // " root_cont: " + root_cont_name + "\n" + // " store:\n" + // " pool: " + pool_name + "\n" + // " client:\n" + // " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + // }; + + // fdb5::Config config{YAMLConfiguration(config_str)}; + + // // request + + // fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"f", "6"}}); + // fdb5::Key request_key2({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + // fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + // fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"d", "4"}}); + + // fdb5::FDBToolRequest full_req{ + // request_key.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest full_req2{ + // request_key2.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest index_req{ + // index_key.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest db_req{ + // db_key.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest all_req{ + // metkit::mars::MarsRequest{}, + // true, + // std::vector{} + // }; + + // // initialise FDB + + // fdb5::FDB fdb(config); + + // // check FDB is empty + + // size_t count; + // fdb5::ListElement info; + + // auto listObject = fdb.list(db_req); + + // count = 0; + // while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + // ++count; + // } + // EXPECT(count == 0); + + // // archive data with incomplete key + + // char data[] = "test"; + + // fdb.archive(request_key, data, sizeof(data)); + + // fdb.flush(); + + // // list data + + // listObject = fdb.list(db_req); + + // count = 0; + // while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + // ++count; + // } + // EXPECT(count == 1); + + // // retrieve data + + // { + // metkit::mars::MarsRequest r = request_key.request("retrieve"); + // std::unique_ptr dh(fdb.retrieve(r)); + + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data))); + // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + // } + + // // archive data with complete key + + // char data2[] = "abcd"; + + // fdb.archive(request_key2, data2, sizeof(data)); + + // fdb.flush(); + + // // list data + + // listObject = fdb.list(db_req); + + // count = 0; + // while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + // ++count; + // } + // EXPECT(count == 2); + + // // retrieve data + + // { + // metkit::mars::MarsRequest r = request_key.request("retrieve"); + // std::unique_ptr dh(fdb.retrieve(r)); + + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data))); + // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + // } + + // { + // metkit::mars::MarsRequest r = request_key2.request("retrieve"); + // std::unique_ptr dh(fdb.retrieve(r)); + + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data2))); + // EXPECT(::memcmp(mh.data(), data2, sizeof(data2)) == 0); + // } + + // } + + // teardown rados + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + #ifdef eckit_HAVE_RADOS_ADMIN + eckit::RadosPool{pool}.ensureDestroyed(); + #else + ensureCleanNamespaces(pool, test_id); + #endif +#else + ensureClean(prefix); +#endif + +} + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + return run_tests ( argc, argv ); +} diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 0b7f3eda1..020ea15f5 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -64,7 +64,14 @@ namespace { p.rmdir(); }; - // S3Config cfg("eu-central-1", "127.0.0.1", 8888); + void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); + } + } + } #ifdef fdb5_HAVE_RADOS_ADMIN void ensureClean(const std::string& prefix) { @@ -86,16 +93,6 @@ eckit::TmpFile& schema_file() { return f; } -eckit::TmpFile& spaces_file() { - static eckit::TmpFile f{}; - return f; -} - -eckit::TmpFile& roots_file() { - static eckit::TmpFile f{}; - return f; -} - eckit::PathName& store_tests_tmp_root() { static eckit::PathName sd("./rados_store_tests_fdb_root"); return sd; @@ -134,41 +131,16 @@ CASE( "Setup" ) { // due to no specified schema file (e.g. in Key::registry()) ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); - // prepare scpaces - - std::string spaces_str{".* all Default"}; - - std::unique_ptr hsp(spaces_file().fileHandle()); - hsp->openForWrite(spaces_str.size()); - { - eckit::AutoClose closer(*hsp); - hsp->write(spaces_str.data(), spaces_str.size()); - } - - ::setenv("FDB_SPACES_FILE", spaces_file().path().c_str(), 1); - - // prepare roots - - std::string roots_str{store_tests_tmp_root().asString() + " all yes yes"}; - - std::unique_ptr hr(roots_file().fileHandle()); - hr->openForWrite(roots_str.size()); - { - eckit::AutoClose closer(*hr); - hr->write(roots_str.data(), roots_str.size()); - } - - ::setenv("FDB_ROOTS_FILE", roots_file().path().c_str(), 1); - } CASE("RadosStore tests") { SECTION("archive and retrieve") { + std::string test_id = "test-store1"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store1"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -177,19 +149,28 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "rados:\n" " store:\n" " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" }; #else - std::string prefix{"test-store1"}; - + std::string prefix = test_id; ensureClean(prefix); - std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "rados:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" " pool_prefix: " + prefix + "\n" }; #endif @@ -251,9 +232,10 @@ CASE("RadosStore tests") { SECTION("with POSIX Catalogue") { + std::string test_id = "test-store2"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store2"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -262,21 +244,30 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "schema : " + schema_file().path() + "\n" "rados:\n" " store:\n" " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" }; #else - std::string prefix{"test-store2"}; - + std::string prefix = test_id; ensureClean(prefix); - std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "schema : " + schema_file().path() + "\n" "rados:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" " pool_prefix: " + prefix + "\n" }; #endif @@ -376,9 +367,10 @@ CASE("RadosStore tests") { SECTION("VIA FDB API") { + std::string test_id = "test-store3"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store3"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -387,13 +379,17 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif #else - std::string prefix{"test-store3"}; + std::string prefix = test_id; ensureClean(prefix); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "type: local\n" "schema : " + schema_file().path() + "\n" "engine: toc\n" @@ -402,7 +398,9 @@ CASE("RadosStore tests") { }; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool_prefix: " + prefix + "\n"; + config_str += " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n"; #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) @@ -412,7 +410,9 @@ CASE("RadosStore tests") { config_str += " store:\n"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool: " + pool + "\n"; + config_str += " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n"; #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) @@ -573,9 +573,10 @@ CASE("RadosStore tests") { // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe SECTION("FDB API RE-STORE AND WIPE DB") { + std::string test_id = "test-store4"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store4"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -584,13 +585,17 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif #else - std::string prefix{"test-store4"}; + std::string prefix = test_id; ensureClean(prefix); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "type: local\n" "schema : " + schema_file().path() + "\n" "engine: toc\n" @@ -599,7 +604,9 @@ CASE("RadosStore tests") { }; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool_prefix: " + prefix + "\n"; + config_str += " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n"; #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) @@ -609,7 +616,9 @@ CASE("RadosStore tests") { config_str += " store:\n"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool: " + pool + "\n"; + config_str += " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n"; #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) From 6af5bbab7e17cb9b2120e6c7c60463d673272e59 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 30 Jun 2024 12:50:33 +0200 Subject: [PATCH 017/109] Improved name of the Rados backend configuration item to limit part size in multipart mode. --- src/fdb5/rados/RadosCommon.cc | 2 +- src/fdb5/rados/RadosCommon.h | 2 +- src/fdb5/rados/RadosEngine.cc | 2 +- src/fdb5/rados/RadosEngine.h | 2 +- src/fdb5/rados/RadosStore.cc | 4 ++-- tests/fdb/rados/test_rados_store.cc | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 778122712..6e290c745 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -93,7 +93,7 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp if (config.has("rados")) c = config.getSubConfiguration("rados"); - maxObjectSize_ = c.getInt("maxObjectSize", 0); + maxPartSize_ = c.getInt("maxPartSize", 0); std::string first_cap{component}; first_cap[0] = toupper(component[0]); diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 24b94bf51..bd981b17f 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -59,7 +59,7 @@ class RadosCommon { eckit::Optional db_kv_; #endif - eckit::Length maxObjectSize_; + eckit::Length maxPartSize_; private: // members diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index 72238d852..2138dc5a7 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -233,7 +233,7 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp if (config.has("rados")) c = config.getSubConfiguration("rados"); - // maxObjectSize_ = c.getInt("maxObjectSize", 0); + // maxPartSize_ = c.getInt("maxPartSize", 0); std::string first_cap{component}; first_cap[0] = toupper(component[0]); diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index b48f7dc2c..75d6ce3d7 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -79,7 +79,7 @@ class RadosEngine : public fdb5::Engine { // eckit::Optional db_kv_; #endif - // eckit::Length maxObjectSize_; + // eckit::Length maxPartSize_; private: // members diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 437f14fed..458d11edb 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -469,9 +469,9 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle *dh = name.asyncMultipartWriteHandle(maxObjectSize_, maxAioBuffSize_, maxPartHandleBuffSize_); + eckit::DataHandle *dh = name.asyncMultipartWriteHandle(maxPartSize_, maxAioBuffSize_, maxPartHandleBuffSize_); #else - eckit::DataHandle *dh = name.multipartWriteHandle(maxObjectSize_); + eckit::DataHandle *dh = name.multipartWriteHandle(maxPartSize_); #endif #else diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 020ea15f5..a14f70ed5 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -404,7 +404,7 @@ CASE("RadosStore tests") { #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - config_str += " maxObjectSize: 16\n"; + config_str += " maxPartSize: 16\n"; #endif config_str += " store:\n"; @@ -476,7 +476,7 @@ CASE("RadosStore tests") { char data[] = "test123456"; #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - /// @note: maxObjectSize is set to 16, and four 10-byte fields are archived, spanning 3 objects + /// @note: maxPartSize is set to 16, and four 10-byte fields are archived, spanning 3 objects for (int i = 0; i < 4; i++) { std::cout << "Archive field " << i << std::endl; fdb5::Key request_key_i({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); @@ -610,7 +610,7 @@ CASE("RadosStore tests") { #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - config_str += " maxObjectSize: 16\n"; + config_str += " maxPartSize: 16\n"; #endif config_str += " store:\n"; From 0720e7d8f7b9c21b22a24561399cbd4426026f37 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 30 Jun 2024 19:52:56 +0200 Subject: [PATCH 018/109] Not closing Rados multipart handles on flush. --- src/fdb5/rados/RadosStore.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 458d11edb..82acdb3b0 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -273,8 +273,9 @@ void RadosStore::flush() { #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH flushDataHandles(); + #else + // NOOP #endif - closeDataHandles(); #else From 08b12637a8ba9cab8b68a504904b8d04a76fcfec Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 30 Jun 2024 22:00:56 +0200 Subject: [PATCH 019/109] Fix in RadosStore in multipart mode and persist on write mode. --- src/fdb5/rados/RadosStore.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 82acdb3b0..8f86085a3 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -271,11 +271,10 @@ void RadosStore::flush() { #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH + /// @note: needs to be called even if PERSIST_ON_FLUSH=OFF, as the + /// multipart handles need to persist the multipart attributes which + /// is performed in the multihandle flush. flushDataHandles(); - #else - // NOOP - #endif #else From 020d69f932861303c2c9c5597d56ba0b25719d07 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Thu, 5 Jun 2025 15:24:20 +0200 Subject: [PATCH 020/109] Improve Ceph backends cmake parameters. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de986f553..de902abeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,11 +55,11 @@ ecbuild_add_option( FEATURE RADOS_BACKENDS_SINGLE_POOL ecbuild_add_option( FEATURE RADOS_STORE_OBJ_PER_FIELD DEFAULT OFF - DESCRIPTION "Use a Rados object per archived field (ON) or per collocation key (OFF)" ) + DESCRIPTION "Use a Rados object per archived field (ON) or per process and collocation key (OFF)" ) ecbuild_add_option( FEATURE RADOS_STORE_MULTIPART DEFAULT ON - DESCRIPTION "If RADOS_STORE_OBJ_PER_FIELD=OFF and the maximum object size is exceeded, use multiple Rados objects per collocation key (ON) or throw an exception (OFF)" ) + DESCRIPTION "If RADOS_STORE_OBJ_PER_FIELD=OFF and the maximum object size is exceeded, use multiple Rados objects per process and collocation key (ON) or throw an exception (OFF)" ) ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_FLUSH DEFAULT OFF From a7afcdb1c7d6c10d97cb20ee5edaa54255362321 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Sun, 8 Jun 2025 11:02:25 +0000 Subject: [PATCH 021/109] Fix Rados backend tests after adding RadosPartHandle. --- tests/fdb/rados/test_rados_catalogue.cc | 4 ++-- tests/fdb/rados/test_rados_store.cc | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index fab033e13..3b732ad5f 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -20,7 +20,7 @@ // #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/config/YAMLConfiguration.h" -#include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosPartHandle.h" // #include "metkit/mars/MarsRequest.h" @@ -352,7 +352,7 @@ CASE("RadosCatalogue tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); - EXPECT(dynamic_cast(dh.get())); + EXPECT(dynamic_cast(dh.get())); eckit::MemoryHandle mh; dh->copyTo(mh); diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index a14f70ed5..d11cf6254 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -34,7 +34,7 @@ // #include "eckit/io/s3/S3Client.h" // #include "eckit/io/s3/S3Session.h" // #include "eckit/io/s3/S3Credential.h" -#include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosPartHandle.h" #include "fdb5/rados/RadosStore.h" #include "fdb5/rados/RadosFieldLocation.h" @@ -197,7 +197,7 @@ CASE("RadosStore tests") { fdb5::Field field(std::move(loc), std::time(nullptr)); std::cout << "Read location: " << field.location() << std::endl; std::unique_ptr dh(store.retrieve(field)); - EXPECT(dynamic_cast(dh.get())); + EXPECT(dynamic_cast(dh.get())); /// @todo: if multiparts is enabled, RadosMultiObjReadHandle eckit::MemoryHandle mh; @@ -322,7 +322,7 @@ CASE("RadosStore tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); - EXPECT(dynamic_cast(dh.get())); + EXPECT(dynamic_cast(dh.get())); /// @todo: if multiparts is enabled, RadosMultiObjReadHandle eckit::MemoryHandle mh; From b6b52163859e0809b3e3b038143770a69b44aff9 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 30 Jun 2026 18:21:27 +0200 Subject: [PATCH 022/109] fix daos --- src/fdb5/daos/DaosStore.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc index 656b6c643..57c96e39b 100644 --- a/src/fdb5/daos/DaosStore.cc +++ b/src/fdb5/daos/DaosStore.cc @@ -62,7 +62,7 @@ bool DaosStore::uriExists(const eckit::URI& uri) const { fdb5::DaosName n(uri); ASSERT(n.hasContainerName()); ASSERT(n.poolName() == pool_); - ASSERT(n.containerName() == db_str_); + ASSERT(n.containerName() == db_cont_); return n.exists(); } From f7ac0de1b1292eb3a1f873d2bd82894d142a6456 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 3 Jul 2026 15:53:22 +0200 Subject: [PATCH 023/109] fix rados commong --- src/fdb5/rados/RadosCommon.cc | 70 +++++++++++++++++++++++------------ src/fdb5/rados/RadosCommon.h | 30 +++++++-------- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 6e290c745..4583c299b 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -10,9 +10,9 @@ #include -#include "eckit/exception/Exceptions.h" #include "eckit/config/Resource.h" -#include "eckit/utils/Tokenizer.h" +#include "eckit/exception/Exceptions.h" +// #include "eckit/utils/Tokenizer.h" #include "fdb5/rados/RadosCommon.h" @@ -44,7 +44,6 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); #endif - } RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { @@ -80,7 +79,6 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); #endif - } #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL @@ -91,7 +89,9 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp eckit::LocalConfiguration c{}; - if (config.has("rados")) c = config.getSubConfiguration("rados"); + if (config.has("rados")) { + c = config.getSubConfiguration("rados"); + } maxPartSize_ = c.getInt("maxPartSize", 0); @@ -99,54 +99,78 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp first_cap[0] = toupper(component[0]); std::string all_caps{component}; - for (auto & c: all_caps) c = toupper(c); + for (auto& c : all_caps) { + c = toupper(c); + } #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - if (readPool) pool_ = "default"; + if (readPool) { + pool_ = "default"; + } root_namespace_ = "root"; if (readPool) { pool_ = c.getString("pool", pool_); - if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); + if (c.has(component)) { + pool_ = c.getSubConfiguration(component).getString("pool", pool_); + } } root_namespace_ = c.getString("root_namespace", root_namespace_); - if (c.has(component)) root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + if (c.has(component)) { + root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + } - if (readPool) + if (readPool) { pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); - root_namespace_ = eckit::Resource("fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + } + root_namespace_ = eckit::Resource( + "fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); - if (c.has(component)) nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); - ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); + if (c.has(component)) { + nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); + } + ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, + "The configured namespace prefix must not contain underscores."); #else - if (readNamespace) namespace_ = "default"; + if (readNamespace) { + namespace_ = "default"; + } root_pool_ = "root"; - if (readNamespace) + if (readNamespace) { namespace_ = c.getString("namespace", namespace_); - if (c.has(component)) namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + } + if (c.has(component)) { + namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + } root_pool_ = c.getString("root_pool", root_pool_); - if (c.has(component)) root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); + if (c.has(component)) { + root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); + } - if (readNamespace) - namespace_ = eckit::Resource("fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); - root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); + if (readNamespace) { + namespace_ = eckit::Resource( + "fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); + } + root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", + root_pool_); pool_prefix_ = c.getString("pool_prefix", pool_prefix_); - if (c.has(component)) pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + if (c.has(component)) { + pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + } ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); #endif // if (c.has("client")) // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index bd981b17f..e6f8b6bfc 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -13,25 +13,26 @@ #pragma once +#include #include "eckit/filesystem/URI.h" -#include "eckit/utils/Optional.h" +// #include "eckit/utils/Optional.h" +// #include "eckit/io/rados/RadosAsyncKeyValue.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/io/rados/RadosAsyncKeyValue.h" -#include "fdb5/fdb5_config.h" -#include "fdb5/database/Key.h" #include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" +#include "fdb5/fdb5_config.h" namespace fdb5 { class RadosCommon { -public: // methods +public: // methods RadosCommon(const fdb5::Config&, const std::string& component, const fdb5::Key&); RadosCommon(const fdb5::Config&, const std::string& component, const eckit::URI&); -private: // methods +private: // methods #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL void readConfig(const fdb5::Config& config, const std::string& component, bool readPool); @@ -39,7 +40,7 @@ class RadosCommon { void readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace); #endif -protected: // members +protected: // members #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool_; @@ -52,25 +53,22 @@ class RadosCommon { #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - eckit::Optional root_kv_; - eckit::Optional db_kv_; + std::optional root_kv_; + std::optional db_kv_; #else - eckit::Optional root_kv_; - eckit::Optional db_kv_; + std::optional root_kv_; + std::optional db_kv_; #endif eckit::Length maxPartSize_; -private: // members +private: // members #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string nspace_prefix_; #else std::string pool_prefix_; #endif - }; -} - - +} // namespace fdb5 From 6c795dc1a130f7855e2643cb453284c1c79a70bf Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 3 Jul 2026 15:53:32 +0200 Subject: [PATCH 024/109] fix rados cat --- src/fdb5/rados/RadosCatalogue.cc | 52 ++++++++++++++------------------ src/fdb5/rados/RadosCatalogue.h | 30 ++++++++++-------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index d45c57172..f6959c9bf 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -9,12 +9,12 @@ */ // #include "eckit/config/Resource.h" -#include "eckit/serialisation/MemoryStream.h" -#include "eckit/io/rados/RadosException.h" +// #include "eckit/serialisation/MemoryStream.h" +// #include "eckit/io/rados/RadosException.h" // #include "fdb5/api/helpers/ControlIterator.h" -#include "fdb5/LibFdb5.h" -#include "fdb5/database/DatabaseNotFoundException.h" +// #include "fdb5/LibFdb5.h" +// #include "fdb5/database/DatabaseNotFoundException.h" #include "fdb5/rados/RadosCatalogue.h" // #include "fdb5/daos/DaosName.h" @@ -35,10 +35,10 @@ RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : // FileSpaceTables to determine root_pool_name_ according to key // and using DbPathNamerTables to determine db_cont_name_ according // to key - } -RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) : +RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, + const fdb5::Config& config) : Catalogue(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL @@ -55,34 +55,27 @@ RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& std::vector data; eckit::MemoryStream ms = db_kv_->getMemoryStream(data, "key", "DB kv"); dbKey_ = fdb5::Key(ms); - - } catch (eckit::RadosEntityNotFoundException& e) { - - throw fdb5::DatabaseNotFoundException( - std::string("RadosCatalogue database not found ") + - "(pool: '" + pool + "', namespace: '" + nspace + "')" - ); - } + catch (eckit::RadosEntityNotFoundException& e) { + throw fdb5::DatabaseNotFoundException(std::string("RadosCatalogue database not found ") + "(pool: '" + pool + + "', namespace: '" + nspace + "')"); + } } bool RadosCatalogue::exists() const { return db_kv_->exists(); - } eckit::URI RadosCatalogue::uri() const { return db_kv_->nspace().uri(); - } const Schema& RadosCatalogue::schema() const { return schema_; - } void RadosCatalogue::loadSchema() { @@ -99,10 +92,10 @@ void RadosCatalogue::loadSchema() { std::istringstream stream{std::string(data.begin(), data.end())}; schema_.load(stream); - } -WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { +WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, + std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { NOTIMP; // return new RadosWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); } @@ -121,7 +114,9 @@ std::vector RadosCatalogue::indexes(bool) const { /// @todo: document these well. Single source these reserved values. /// Ensure where appropriate that user-provided keys do not collide. - if (key == "schema" || key == "key") continue; + if (key == "schema" || key == "key") { + continue; + } /// @note: performed RPCs: /// - db kv get index location size (daos_kv_get without a buffer) @@ -135,10 +130,10 @@ std::vector RadosCatalogue::indexes(bool) const { /// - index kv open (daos_kv_open) /// - index kv get size (daos_kv_get without a buffer) /// - index kv get key (daos_kv_get) - /// @note: the following three lines intend to check whether the index kv exists + /// @note: the following three lines intend to check whether the index kv exists /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, /// so it is not useful on its own to check whether the index KV existed or not. - /// Instead, presence of a "key" key in the KV is used to determine if the index + /// Instead, presence of a "key" key in the KV is used to determine if the index /// KV existed. eckit::RadosKeyValue index_kv{uri}; std::optional index_key; @@ -146,26 +141,25 @@ std::vector RadosCatalogue::indexes(bool) const { std::vector data; eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); index_key.emplace(ms); - } catch (eckit::RadosEntityNotFoundException& e) { - continue; /// @note: the index_kv may not exist after a failed wipe + } + catch (eckit::RadosEntityNotFoundException& e) { + continue; /// @note: the index_kv may not exist after a failed wipe /// @todo: the index_kv may exist even if it does not have the "key" key } res.push_back(Index(new fdb5::RadosIndex(index_key.value(), index_kv, false))); - } return res; - } std::string RadosCatalogue::type() const { return RadosCatalogue::catalogueTypeName(); - } -// void RadosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, std::ostream& logVerbose, bool doit) { +// void RadosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, std::ostream& logVerbose, bool +// doit) { // ASSERT(n.hasContainerName()); @@ -177,4 +171,4 @@ std::string RadosCatalogue::type() const { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h index 2ee68b01d..c936a1ef5 100644 --- a/src/fdb5/rados/RadosCatalogue.h +++ b/src/fdb5/rados/RadosCatalogue.h @@ -13,9 +13,10 @@ #pragma once -#include "fdb5/database/DB.h" -#include "fdb5/rules/Schema.h" +// #include "fdb5/database/DB.h" +#include "fdb5/database/Catalogue.h" #include "fdb5/rados/RadosCommon.h" +#include "fdb5/rules/Schema.h" // #include "fdb5/rados/RadosEngine.h" namespace fdb5 { @@ -26,14 +27,14 @@ namespace fdb5 { class RadosCatalogue : public Catalogue, public RadosCommon { -public: // methods +public: // methods RadosCatalogue(const Key& key, const fdb5::Config& config); RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config); // static const char* catalogueTypeName() { return fdb5::RadosEngine::typeName(); } static const char* catalogueTypeName() { return "rados"; } - + eckit::URI uri() const override; const Key& indexKey() const override { return currentIndexKey_; } @@ -49,30 +50,35 @@ class RadosCatalogue : public Catalogue, public RadosCommon { StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }; - WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override; - MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; }; + WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, + bool porcelain, bool unsafeWipeAll) const override; + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, + eckit::Queue& queue) const override { + NOTIMP; + }; void maskIndexEntry(const Index& index) const override { NOTIMP; }; void loadSchema() override; - std::vector indexes(bool sorted=false) const override; + std::vector indexes(bool sorted = false) const override; void allMasked(std::set>& metadata, - std::set& data) const override { NOTIMP; }; + std::set& data) const override { + NOTIMP; + }; // Control access properties of the DB void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override { NOTIMP; }; -protected: // members +protected: // members Key currentIndexKey_; -private: // members +private: // members Schema schema_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 From 70b6fc09bb4eb6870cbaa52598277170ab448043 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 3 Jul 2026 15:53:41 +0200 Subject: [PATCH 025/109] fix rados engine --- src/fdb5/rados/RadosEngine.h | 24 ++++++++++++------------ src/fdb5/rados/RadosFieldLocation.h | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 75d6ce3d7..916473cc5 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -13,9 +13,9 @@ #pragma once -#include "eckit/utils/Optional.h" +// #include "eckit/utils/Optional.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/io/rados/RadosAsyncKeyValue.h" +// #include "eckit/io/rados/RadosAsyncKeyValue.h" #include "fdb5/database/Engine.h" #include "fdb5/fdb5_config.h" @@ -26,32 +26,33 @@ namespace fdb5 { class RadosEngine : public fdb5::Engine { -public: // methods +public: // methods RadosEngine() {}; static const char* typeName() { return "rados"; } -protected: // methods +protected: // methods virtual std::string name() const override; virtual std::string dbType() const override { NOTIMP; }; - virtual eckit::URI location(const Key &key, const Config& config) const override { NOTIMP; }; + virtual eckit::URI location(const Key& key, const Config& config) const override { NOTIMP; }; virtual bool canHandle(const eckit::URI&, const Config&) const override { NOTIMP; }; virtual std::vector allLocations(const Key& key, const Config& config) const override { NOTIMP; }; virtual std::vector visitableLocations(const Key& key, const Config& config) const override; - virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; + virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, + const Config& config) const override; virtual std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; }; - virtual void print( std::ostream &out ) const override { NOTIMP; }; + virtual void print(std::ostream& out) const override { NOTIMP; }; -private: // methods +private: // methods #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL void readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const; @@ -59,7 +60,7 @@ class RadosEngine : public fdb5::Engine { void readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) const; #endif -protected: // members +protected: // members #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL mutable std::string pool_; @@ -81,17 +82,16 @@ class RadosEngine : public fdb5::Engine { // eckit::Length maxPartSize_; -private: // members +private: // members #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL mutable std::string nspace_prefix_; #else mutable std::string pool_prefix_; #endif - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h index a9f04c92e..36c31105a 100644 --- a/src/fdb5/rados/RadosFieldLocation.h +++ b/src/fdb5/rados/RadosFieldLocation.h @@ -40,7 +40,7 @@ class RadosFieldLocation : public FieldLocation { eckit::DataHandle* dataHandle() const override; // eckit::DataHandle* dataHandle(const Key& remapKey) const override; - virtual std::shared_ptr make_shared() const override; + virtual std::shared_ptr make_shared() const override; virtual void visit(FieldLocationVisitor& visitor) const override; From d72f9ed3f95c25c1d3bcf2e4502fb00e57ad3330 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 3 Jul 2026 15:53:49 +0200 Subject: [PATCH 026/109] fix rados store --- tests/fdb/rados/test_rados_store.cc | 369 +++++++++++++++------------- 1 file changed, 198 insertions(+), 171 deletions(-) diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index d11cf6254..58435807c 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -18,8 +18,8 @@ #include "eckit/filesystem/TmpFile.h" // #include "eckit/filesystem/TmpDir.h" // #include "eckit/io/FileHandle.h" -#include "eckit/io/MemoryHandle.h" #include "eckit/config/YAMLConfiguration.h" +#include "eckit/io/MemoryHandle.h" // #include "metkit/mars/MarsRequest.h" @@ -28,16 +28,16 @@ #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" -#include "fdb5/toc/TocCatalogueWriter.h" #include "fdb5/toc/TocCatalogueReader.h" +#include "fdb5/toc/TocCatalogueWriter.h" // #include "eckit/io/s3/S3Client.h" // #include "eckit/io/s3/S3Session.h" // #include "eckit/io/s3/S3Credential.h" #include "eckit/io/rados/RadosPartHandle.h" -#include "fdb5/rados/RadosStore.h" #include "fdb5/rados/RadosFieldLocation.h" +#include "fdb5/rados/RadosStore.h" // #include "fdb5/daos/DaosException.h" using namespace eckit::testing; @@ -45,46 +45,46 @@ using namespace eckit; namespace { - void deldir(eckit::PathName& p) { - if (!p.exists()) { - return; - } +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } - std::vector files; - std::vector dirs; - p.children(files, dirs); + std::vector files; + std::vector dirs; + p.children(files, dirs); - for (auto& f : files) { - f.unlink(); - } - for (auto& d : dirs) { - deldir(d); - } + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } - p.rmdir(); - }; + p.rmdir(); +}; - void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { - ASSERT(prefix.length() > 3); - for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { - if (name.rfind(prefix, 0) == 0) { - eckit::RadosNamespace{pool, name}.destroy(); - } +void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); } } +} #ifdef fdb5_HAVE_RADOS_ADMIN - void ensureClean(const std::string& prefix) { - ASSERT(prefix.length() > 3); - for (const std::string& name : eckit::RadosCluster::instance().listPools()) { - if (name.rfind(prefix, 0) == 0) { - eckit::RadosPool{name}.destroy(); - } +void ensureClean(const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listPools()) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosPool{name}.destroy(); } } +} #endif -} +} // namespace // temporary schema,spaces,root files common to all DAOS Store tests @@ -101,7 +101,7 @@ eckit::PathName& store_tests_tmp_root() { namespace fdb { namespace test { -CASE( "Setup" ) { +CASE("Setup") { #if !defined(fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL) && !defined(fdb5_HAVE_RADOS_ADMIN) throw eckit::Exception( @@ -109,9 +109,11 @@ CASE( "Setup" ) { "RADOS_BACKENDS_SINGLE_POOL=OFF, and require enabling RADOS_ADMIN=ON."); #endif - // ensure fdb root directory exists. If not, then that root is + // ensure fdb root directory exists. If not, then that root is // registered as non existing and Store tests fail. - if (store_tests_tmp_root().exists()) deldir(store_tests_tmp_root()); + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } store_tests_tmp_root().mkdir(); ::setenv("FDB_ROOT_DIRECTORY", store_tests_tmp_root().path().c_str(), 1); @@ -130,7 +132,6 @@ CASE( "Setup" ) { // LibFdb5::instance().defaultConfig().schema() is called // due to no specified schema file (e.g. in Key::registry()) ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); - } CASE("RadosStore tests") { @@ -139,40 +140,48 @@ CASE("RadosStore tests") { std::string test_id = "test-store1"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - #ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer - #else +#else std::string pool; - pool = eckit::Resource( - "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool - ); + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); - #endif +#endif std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" "rados:\n" " store:\n" - " pool: " + pool + "\n" - " root_namespace: " + test_id + "_root\n" - " namespace_prefix: " + test_id + "\n" - }; + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; #else std::string prefix = test_id; ensureClean(prefix); std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" "rados:\n" " namespace: default\n" - " root_pool: " + prefix + "_root\n" - " pool_prefix: " + prefix + "\n" - }; + " root_pool: " + + prefix + + "_root\n" + " pool_prefix: " + + prefix + "\n"}; #endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -199,7 +208,7 @@ CASE("RadosStore tests") { std::unique_ptr dh(store.retrieve(field)); EXPECT(dynamic_cast(dh.get())); /// @todo: if multiparts is enabled, RadosMultiObjReadHandle - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -227,49 +236,60 @@ CASE("RadosStore tests") { EXPECT_NOT(field_name.exists()); EXPECT_NOT(store_name.exists()); #endif - } SECTION("with POSIX Catalogue") { std::string test_id = "test-store2"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - #ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer - #else +#else std::string pool; - pool = eckit::Resource( - "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool - ); + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); - #endif +#endif std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "rados:\n" " store:\n" - " pool: " + pool + "\n" - " root_namespace: " + test_id + "_root\n" - " namespace_prefix: " + test_id + "\n" - }; + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; #else std::string prefix = test_id; ensureClean(prefix); std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "rados:\n" " namespace: default\n" - " root_pool: " + prefix + "_root\n" - " pool_prefix: " + prefix + "\n" - }; + " root_pool: " + + prefix + + "_root\n" + " pool_prefix: " + + prefix + "\n"}; #endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -301,8 +321,8 @@ CASE("RadosStore tests") { fdb5::Catalogue& cat = static_cast(tcat); cat.deselectIndex(); cat.selectIndex(index_key); - //const fdb5::Index& idx = tcat.currentIndex(); - static_cast(tcat).archive(field_key, std::move(loc)); + // const fdb5::Index& idx = tcat.currentIndex(); + static_cast(tcat).archive(index_key, field_key, std::move(loc)); /// flush store before flushing catalogue rados_store.flush(); @@ -324,7 +344,7 @@ CASE("RadosStore tests") { std::unique_ptr dh(store.retrieve(field)); EXPECT(dynamic_cast(dh.get())); /// @todo: if multiparts is enabled, RadosMultiObjReadHandle - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -362,25 +382,22 @@ CASE("RadosStore tests") { std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); cat.visitEntries(*wv, store, false); } - } SECTION("VIA FDB API") { std::string test_id = "test-store3"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - #ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer - #else +#else std::string pool; - pool = eckit::Resource( - "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool - ); + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); - #endif +#endif #else std::string prefix = test_id; ensureClean(prefix); @@ -389,43 +406,54 @@ CASE("RadosStore tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" "type: local\n" - "schema : " + schema_file().path() + "\n" + "schema : " + + schema_file().path() + + "\n" "engine: toc\n" "store: rados\n" - "rados:\n" - }; + "rados:\n"}; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " namespace: default\n" - " root_pool: " + prefix + "_root\n" - " pool_prefix: " + prefix + "\n"; + config_str += + " namespace: default\n" + " root_pool: " + + prefix + + "_root\n" + " pool_prefix: " + + prefix + "\n"; #endif -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxPartSize: 16\n"; #endif config_str += " store:\n"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool: " + pool + "\n" - " root_namespace: " + test_id + "_root\n" - " namespace_prefix: " + test_id + "\n"; + config_str += " pool: " + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"; #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxHandleBuffSize: 100\n"; - #else - #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART +#else +#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART config_str += " maxAioBuffSize: 10\n"; config_str += " maxPartHandleBuffSize: 10\n"; - #else +#else config_str += " maxAioBuffSize: 100\n"; - #endif - #endif +#endif +#endif #endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -436,21 +464,9 @@ CASE("RadosStore tests") { fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; // initialise store @@ -475,11 +491,12 @@ CASE("RadosStore tests") { char data[] = "test123456"; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) /// @note: maxPartSize is set to 16, and four 10-byte fields are archived, spanning 3 objects for (int i = 0; i < 4; i++) { std::cout << "Archive field " << i << std::endl; - fdb5::Key request_key_i({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); + fdb5::Key request_key_i( + {{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); fdb.archive(request_key_i, data, sizeof(data)); } #else @@ -490,13 +507,14 @@ CASE("RadosStore tests") { // retrieve data -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) for (int i = 0; i < 4; i++) { std::cout << "Retrieve field " << i << std::endl; - fdb5::Key request_key_i({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); + fdb5::Key request_key_i( + {{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); metkit::mars::MarsRequest r_i = request_key_i.request("retrieve"); std::unique_ptr dh(fdb.retrieve(r_i)); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -505,7 +523,7 @@ CASE("RadosStore tests") { #else metkit::mars::MarsRequest r = request_key.request("retrieve"); std::unique_ptr dh(fdb.retrieve(r)); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -520,19 +538,25 @@ CASE("RadosStore tests") { auto wipeObject = fdb.wipe(full_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) { + count++; + } EXPECT(count == 0); // dry run wipe index and store unit wipeObject = fdb.wipe(index_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) { + count++; + } EXPECT(count > 0); // dry run wipe database wipeObject = fdb.wipe(db_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) { + count++; + } EXPECT(count > 0); // ensure field still exists @@ -548,7 +572,9 @@ CASE("RadosStore tests") { // attempt to wipe with too specific request wipeObject = fdb.wipe(full_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) { + count++; + } EXPECT(count == 0); /// @todo: really needed? fdb.flush(); @@ -556,7 +582,9 @@ CASE("RadosStore tests") { // wipe index and store unit (and DB pool or namespace as there is only one index) wipeObject = fdb.wipe(index_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) { + count++; + } EXPECT(count > 0); /// @todo: really needed? fdb.flush(); @@ -564,9 +592,10 @@ CASE("RadosStore tests") { // ensure field does not exist listObject = fdb.list(full_req); count = 0; - while (listObject.next(info)) count++; + while (listObject.next(info)) { + count++; + } EXPECT(count == 0); - } /// @todo: if doing what's in this section at the end of the previous section reusing the same FDB object, @@ -575,18 +604,16 @@ CASE("RadosStore tests") { std::string test_id = "test-store4"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - #ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer - #else +#else std::string pool; - pool = eckit::Resource( - "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool - ); + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); - #endif +#endif #else std::string prefix = test_id; ensureClean(prefix); @@ -595,43 +622,54 @@ CASE("RadosStore tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" "type: local\n" - "schema : " + schema_file().path() + "\n" + "schema : " + + schema_file().path() + + "\n" "engine: toc\n" "store: rados\n" - "rados:\n" - }; + "rados:\n"}; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " namespace: default\n" - " root_pool: " + prefix + "_root\n" - " pool_prefix: " + prefix + "\n"; + config_str += + " namespace: default\n" + " root_pool: " + + prefix + + "_root\n" + " pool_prefix: " + + prefix + "\n"; #endif -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxPartSize: 16\n"; #endif config_str += " store:\n"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool: " + pool + "\n" - " root_namespace: " + test_id + "_root\n" - " namespace_prefix: " + test_id + "\n"; + config_str += " pool: " + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"; #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxHandleBuffSize: 100\n"; - #else - #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART +#else +#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART config_str += " maxAioBuffSize: 10\n"; config_str += " maxPartHandleBuffSize: 10\n"; - #else +#else config_str += " maxAioBuffSize: 100\n"; - #endif - #endif +#endif +#endif #endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -642,21 +680,9 @@ CASE("RadosStore tests") { fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; // initialise store @@ -667,17 +693,19 @@ CASE("RadosStore tests") { char data[] = "test"; fdb.archive(request_key, data, sizeof(data)); - + fdb.flush(); size_t count; - + // wipe all database fdb5::WipeElement elem; auto wipeObject = fdb.wipe(db_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) { + count++; + } EXPECT(count > 0); /// @todo: really needed? fdb.flush(); @@ -693,22 +721,21 @@ CASE("RadosStore tests") { count++; } EXPECT(count == 0); - } - } } // namespace test } // namespace fdb -int main(int argc, char **argv) -{ +int main(int argc, char** argv) { int ret = -1; try { - ret = run_tests ( argc, argv ); - } catch(...) {} + ret = run_tests(argc, argv); + } + catch (...) { + } #ifdef fdb5_HAVE_RADOS_ADMIN ensureClean("test-store"); From d41d808db33137d0aa27606ce62f748e7d1414d4 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 3 Jul 2026 15:54:06 +0200 Subject: [PATCH 027/109] update rados cmake --- cmake/FindRADOS.cmake | 83 +++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/cmake/FindRADOS.cmake b/cmake/FindRADOS.cmake index 54eb0fc54..1a861823c 100644 --- a/cmake/FindRADOS.cmake +++ b/cmake/FindRADOS.cmake @@ -1,49 +1,54 @@ -# (C) Copyright 2011- ECMWF. +# (C) Copyright 2026- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. - -# - Try to find Rados -# Once done this will define -# -# RADOS_FOUND - system has Armadillo -# RADOS_INCLUDE_DIRS - the Armadillo include directory -# RADOS_LIBRARIES - the Armadillo library -# RADOS_VERSION - This is set to $major.$minor.$patch (eg. 0.9.8) # -# The following paths will be searched with priority if set in CMake or env +# This module defines the following variables: +# RADOS_INCLUDE_DIRS - Where to find rados/librados.h +# RADOS_LIBRARIES - The libraries needed to use Rados +# RADOS_FOUND - True if Rados was found # -# RADOS_PATH - prefix path of the Armadillo installation -# RADOS_ROOT - Set this variable to the root installation - -# Search with priority for RADOS_PATH if given as CMake or env var - -find_path(RADOS_INCLUDE_DIR rados/librados.hpp - HINTS $ENV{RADOS_ROOT} ${RADOS_ROOT} - PATHS ${RADOS_PATH} ENV RADOS_PATH - PATH_SUFFIXES include NO_DEFAULT_PATH) - -find_path(RADOS_INCLUDE_DIR rados/librados.hpp PATH_SUFFIXES include ) - -# Search with priority for RADOS_PATH if given as CMake or env var -find_library(RADOS_LIBRARY rados - HINTS $ENV{RADOS_ROOT} ${RADOS_ROOT} - PATHS ${RADOS_PATH} ENV RADOS_PATH - PATH_SUFFIXES lib64 lib NO_DEFAULT_PATH) - -find_library( RADOS_LIBRARY rados PATH_SUFFIXES lib64 lib ) - -set( RADOS_LIBRARIES ${RADOS_LIBRARY} ) -set( RADOS_INCLUDE_DIRS ${RADOS_INCLUDE_DIR} ) - +# This module also defines the following IMPORTED target: +# Ceph::rados + +# Find the header path by looking for the subdirectory file +find_path(RADOS_INCLUDE_DIR + NAMES rados/librados.h + DOC "Path to Rados include directory" +) + +# Find the library +find_library(RADOS_LIBRARY + NAMES rados + DOC "Path to Rados library" +) + +# Handle the QUIETLY and REQUIRED arguments and set RADOS_FOUND to TRUE if +# all listed variables are TRUE. include(FindPackageHandleStandardArgs) - -# handle the QUIET and REQUIRED arguments and set RADOS_FOUND to TRUE -# if all listed variables are TRUE -# Note: capitalisation of the package name must be the same as in the file name -find_package_handle_standard_args(RADOS DEFAULT_MSG RADOS_LIBRARY RADOS_INCLUDE_DIR) - +find_package_handle_standard_args(RADOS + REQUIRED_VARS RADOS_LIBRARY RADOS_INCLUDE_DIR +) + +message(STATUS "DEBUG: RADOS_INCLUDE_DIR = ${RADOS_INCLUDE_DIR}") +message(STATUS "DEBUG: RADOS_LIBRARY = ${RADOS_LIBRARY}") + +if(RADOS_FOUND) + set(RADOS_LIBRARIES ${RADOS_LIBRARY}) + set(RADOS_INCLUDE_DIRS ${RADOS_INCLUDE_DIR}) + + # Create an modern generic imported target + if(NOT TARGET Ceph::RADOS) + add_library(Ceph::RADOS UNKNOWN IMPORTED) + set_target_properties(Ceph::RADOS PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${RADOS_INCLUDE_DIRS}" + IMPORTED_LOCATION "${RADOS_LIBRARY}" + ) + endif() +endif() + +# Hide these variables from the GUI cache view mark_as_advanced(RADOS_INCLUDE_DIR RADOS_LIBRARY) From ebab5972c01712dbe49cf05d0a1aa729b50bed1e Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 6 Jul 2026 10:24:40 +0200 Subject: [PATCH 028/109] merge: undo --- src/fdb5/toc/FieldRef.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fdb5/toc/FieldRef.cc b/src/fdb5/toc/FieldRef.cc index adfd66788..8ed227dcb 100644 --- a/src/fdb5/toc/FieldRef.cc +++ b/src/fdb5/toc/FieldRef.cc @@ -18,6 +18,7 @@ #include "fdb5/database/UriStore.h" #include "fdb5/fdb5_config.h" + namespace fdb5 { From 4c7f92c2000129214ff3df71ccde30c4a28a5daf Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 6 Jul 2026 10:26:41 +0200 Subject: [PATCH 029/109] merge: undo --- src/fdb5/toc/TocWipeVisitor.cc | 568 --------------------------------- src/fdb5/toc/TocWipeVisitor.h | 84 ----- 2 files changed, 652 deletions(-) delete mode 100644 src/fdb5/toc/TocWipeVisitor.cc delete mode 100644 src/fdb5/toc/TocWipeVisitor.h diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc deleted file mode 100644 index ff42e5ed1..000000000 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ /dev/null @@ -1,568 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - - -#include - -#include "eckit/os/Stat.h" - -#include "fdb5/api/helpers/ControlIterator.h" -#include "fdb5/database/DB.h" -#include "fdb5/toc/TocCatalogue.h" -#include "fdb5/toc/TocWipeVisitor.h" - -#include -#include -#include -#include -#include - -using namespace eckit; - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -namespace { -class StdDir { - - eckit::PathName path_; - DIR *d_; - -public: - - StdDir(const eckit::PathName& p) : - path_(p), - d_(opendir(p.localPath())) { - - if (!d_) { - std::stringstream ss; - ss << "Failed to open directory " << p << " (" << errno << "): " << strerror(errno); - throw eckit::SeriousBug(ss.str(), Here()); - } - } - - ~StdDir() { if (d_) closedir(d_); } - - void children(std::vector& paths) { - - // Implemented here as PathName::match() does not return hidden files starting with '.' - - struct dirent* e; - - while ((e = readdir(d_)) != nullptr) { - - if (e->d_name[0] == '.') { - if (e->d_name[1] == '\0' || (e->d_name[1] == '.' && e->d_name[2] == '\0')) continue; - } - - eckit::PathName p(path_ / e->d_name); - - eckit::Stat::Struct info; - SYSCALL(eckit::Stat::lstat(p.localPath(), &info)); - - if (S_ISDIR(info.st_mode)) { - StdDir d(p); - d.children(paths); - } - - // n.b. added after all children - paths.push_back(p); - } - } -}; -} - -//---------------------------------------------------------------------------------------------------------------------- - -// TODO: Warnings and errors form inside here back to the user. - -TocWipeVisitor::TocWipeVisitor(const TocCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, - bool unsafeWipeAll) : - WipeVisitor(request, out, doit, porcelain, unsafeWipeAll), - catalogue_(catalogue), - store_(store), - tocPath_(""), - schemaPath_("") {} - -TocWipeVisitor::~TocWipeVisitor() {} - - -bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) { - - // Overall checks - - ASSERT(&catalogue_ == &catalogue); -// ASSERT(&store_ == &store); - ASSERT(catalogue.enabled(ControlIdentifier::Wipe)); - WipeVisitor::visitDatabase(catalogue, store); - - // Check that we are in a clean state (i.e. we only visit one DB). - - ASSERT(subtocPaths_.empty()); - ASSERT(lockfilePaths_.empty()); - ASSERT(indexPaths_.empty()); - ASSERT(dataURIs_.empty()); - ASSERT(safePaths_.empty()); - ASSERT(indexesToMask_.empty()); - - ASSERT(!tocPath_.asString().size()); - ASSERT(!schemaPath_.asString().size()); - - // Having selected a DB, construct the residual request. This is the request that is used for - // matching Index(es) -- which is relevant if there is subselection of the DB. - - indexRequest_ = request_; - for (const auto& kv : catalogue.key()) { - indexRequest_.unsetValues(kv.first); - } - - return true; // Explore contained indexes -} - -bool TocWipeVisitor::visitIndex(const Index& index) { - - eckit::PathName location(index.location().uri().path()); - const auto& basePath(catalogue_.basePath()); - - // Is this index matched by the supplied request? - // n.b. If the request is over-specified (i.e. below the index level), nothing will be removed - - bool include = index.key().match(indexRequest_); - - // If we have cross fdb-mounted another DB, ensure we can't delete another DBs data. - if (!location.dirName().sameAs(basePath)) { - include = false; - } - ASSERT(location.dirName().sameAs(basePath) || !include); - - // Add the index paths to be removed. - - if (include) { - indexesToMask_.push_back(index); - indexPaths_.insert(location); - } else { - // This will ensure that if only some indexes are to be removed from a file, then - // they will be masked out but the file not deleted. - safePaths_.insert(location); - } - - // Enumerate data files. - - std::vector indexDataURIs(index.dataURIs()); - for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataURIs)) { - if (include) { - if (!store_.uriBelongs(uri)) { - Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." << std::endl; - Log::error() << "Configured Store URI: " << store_.uri().asString() << std::endl; - Log::error() << "Pointed Store unit URI: " << uri.asString() << std::endl; - Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." << std::endl; - NOTIMP; - } - dataURIs_.insert(uri); - } else { - safeURIs_.insert(uri); - } - } - - return true; // Explore contained entries -} - -void TocWipeVisitor::addMaskedPaths() { - - //ASSERT(indexRequest_.empty()); - - std::set> metadata; - std::set data; - catalogue_.allMasked(metadata, data); - for (const auto& entry : metadata) { - eckit::PathName path = entry.first.path(); - if (path.dirName().sameAs(catalogue_.basePath())) { - if (path.baseName().asString().substr(0, 4) == "toc.") { - subtocPaths_.insert(path); - } else { - indexPaths_.insert(path); - } - } - } - for (const auto& uri : data) { - if (store_.uriBelongs(uri)) dataURIs_.insert(uri); - } -} - -void TocWipeVisitor::addMetadataPaths() { - - // toc, schema - - schemaPath_ = catalogue_.schemaPath(); - tocPath_ = catalogue_.tocPath(); - - // subtocs - - const auto&& subtocs(catalogue_.subTocPaths()); - subtocPaths_.insert(subtocs.begin(), subtocs.end()); - - // lockfiles - - const auto&& lockfiles(catalogue_.lockfilePaths()); - lockfilePaths_.insert(lockfiles.begin(), lockfiles.end()); -} - -void TocWipeVisitor::ensureSafePaths() { - - // Very explicitly ensure that we cannot delete anything marked as safe - - if (safePaths_.find(tocPath_) != safePaths_.end()) tocPath_ = ""; - if (safePaths_.find(schemaPath_) != safePaths_.end()) schemaPath_ = ""; - - for (const auto& p : safePaths_) { - for (std::set* s : {&subtocPaths_, &lockfilePaths_, &indexPaths_}) { - s->erase(p); - } - } - for (const auto& p : safeURIs_) { - for (std::set* s : {&dataURIs_}) { - s->erase(p); - } - } -} - -void TocWipeVisitor::calculateResidualPaths() { - - // Remove paths to non-existant files. This is reasonable as we may be recovering from a - // previous failed, partial wipe. As such, referenced files may not exist any more. - - for (std::set* fileset : {&subtocPaths_, &lockfilePaths_, &indexPaths_}) { - for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { - - if (it->exists()) { - ++it; - } else { - fileset->erase(it++); - } - - } - } - - for (std::set* uriset : {&dataURIs_}) { - for (std::set::iterator it = uriset->begin(); it != uriset->end(); ) { - - if (store_.uriExists(*it)) { - ++it; - } else { - uriset->erase(it++); - } - - } - } - - if (tocPath_.asString().size() && !tocPath_.exists()) tocPath_ = ""; - - if (schemaPath_.asString().size() && !schemaPath_.exists()) - schemaPath_ = ""; - - // Consider the total sets of paths - - std::set deletePaths; - std::set deleteURIs; - deletePaths.insert(subtocPaths_.begin(), subtocPaths_.end()); - deletePaths.insert(lockfilePaths_.begin(), lockfilePaths_.end()); - deletePaths.insert(indexPaths_.begin(), indexPaths_.end()); - if (store_.type() == "file") - for (auto u : dataURIs_) - deletePaths.insert(eckit::PathName{u.name()}); - if (tocPath_.asString().size()) deletePaths.insert(tocPath_); - if (schemaPath_.asString().size()) - deletePaths.insert(schemaPath_); - - std::vector allPathsVector; - StdDir(catalogue_.basePath()).children(allPathsVector); - std::set allPaths(allPathsVector.begin(), allPathsVector.end()); - - ASSERT(residualPaths_.empty()); - - if (!(deletePaths == allPaths)) { - - // First we check if there are paths marked to delete that don't exist. This is an error - - std::set paths; - std::set_difference(deletePaths.begin(), deletePaths.end(), - allPaths.begin(), allPaths.end(), - std::inserter(paths, paths.begin())); - - if (!paths.empty()) { - Log::error() << "Paths not in existing paths set:" << std::endl; - for (const auto& p : paths) { - Log::error() << " - " << p << std::endl; - } - throw SeriousBug("Path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); - } - - std::set_difference(allPaths.begin(), allPaths.end(), - deletePaths.begin(), deletePaths.end(), - std::inserter(residualPaths_, residualPaths_.begin())); - } - - // if the store uses a backend other than POSIX (file), repeat the algorithm specialized - // for its store units - - if (store_.type() == "file") return; - - std::vector allCollocatedDataURIs(store_.collocatedDataURIs()); - - std::set allDataURIs(allCollocatedDataURIs.begin(), allCollocatedDataURIs.end()); - - ASSERT(residualDataURIs_.empty()); - - if (!(dataURIs_ == allDataURIs)) { - - // First we check if there are paths marked to delete that don't exist. This is an error - - std::set uris; - std::set_difference(dataURIs_.begin(), dataURIs_.end(), - allDataURIs.begin(), allDataURIs.end(), - std::inserter(uris, uris.begin())); - - if (!uris.empty()) { - Log::error() << "Store unit uris not in existing uris set:" << std::endl; - for (const auto& u : uris) { - Log::error() << " - " << u << std::endl; - } - throw SeriousBug("Store unit uri to delete should be in existing uri set. Are multiple wipe commands running simultaneously?", Here()); - } - - std::set_difference(allDataURIs.begin(), allDataURIs.end(), - dataURIs_.begin(), dataURIs_.end(), - std::inserter(residualDataURIs_, residualDataURIs_.begin())); - } - -} - -bool TocWipeVisitor::anythingToWipe() const { - return (!subtocPaths_.empty() || !lockfilePaths_.empty() || !indexPaths_.empty() || - !dataURIs_.empty() || !indexesToMask_.empty() || - tocPath_.asString().size() || schemaPath_.asString().size()); -} - -void TocWipeVisitor::report(bool wipeAll) { - - ASSERT(anythingToWipe()); - - out_ << "FDB owner: " << catalogue_.owner() << std::endl - << std::endl; - - out_ << "Toc files to delete:" << std::endl; - if (!tocPath_.asString().size() && subtocPaths_.empty()) out_ << " - NONE -" << std::endl; - if (tocPath_.asString().size()) out_ << " " << tocPath_ << std::endl; - for (const auto& f : subtocPaths_) { - out_ << " " << f << std::endl; - } - out_ << std::endl; - - out_ << "Control files to delete:" << std::endl; - if (!schemaPath_.asString().size() && lockfilePaths_.empty()) out_ << " - NONE -" << std::endl; - if (schemaPath_.asString().size()) out_ << " " << schemaPath_ << std::endl; - for (const auto& f : lockfilePaths_) { - out_ << " " << f << std::endl; - } - out_ << std::endl; - - out_ << "Index files to delete: " << std::endl; - if (indexPaths_.empty()) out_ << " - NONE -" << std::endl; - for (const auto& f : indexPaths_) { - out_ << " " << f << std::endl; - } - out_ << std::endl; - - out_ << "Data URIs to delete: " << std::endl; - if (dataURIs_.empty()) out_ << " - NONE -" << std::endl; - for (const auto& f : dataURIs_) { - out_ << " " << f << std::endl; - } - out_ << std::endl; - - if (store_.type() != "file") { - out_ << "Store URI to delete:" << std::endl; - if (wipeAll) { - out_ << " " << store_.uri() << std::endl; - } else { - out_ << " - NONE -" << std::endl; - } - out_ << std::endl; - } - - out_ << "Protected files (explicitly untouched):" << std::endl; - if (safePaths_.empty()) out_ << " - NONE - " << std::endl; - for (const auto& f : safePaths_) { - out_ << " " << f << std::endl; - } - out_ << std::endl; - - out_ << "Protected URIs (explicitly untouched):" << std::endl; - if (safeURIs_.empty()) out_ << " - NONE - " << std::endl; - for (const auto& u : safeURIs_) { - out_ << " " << u << std::endl; - } - out_ << std::endl; - - if (!safePaths_.empty()) { - out_ << "Indexes to mask:" << std::endl; - if (indexesToMask_.empty()) out_ << " - NONE - " << std::endl; - for (const auto& i : indexesToMask_) { - out_ << " " << i.location() << std::endl; - } - } -} - -void TocWipeVisitor::wipe(bool wipeAll) { - - ASSERT(anythingToWipe()); - - std::ostream& logAlways(out_); - std::ostream& logVerbose(porcelain_ ? Log::debug() : out_); - - // Sanity checks... - - catalogue_.checkUID(); - - // If we are wiping the metadata files, then we need to lock the DB to ensure we don't get - // into a state we don't like. - - if (wipeAll && doit_) { - catalogue_.control(ControlAction::Disable, ControlIdentifier::List | - ControlIdentifier::Retrieve | - ControlIdentifier::Archive); - - ASSERT(!catalogue_.enabled(ControlIdentifier::List)); - ASSERT(!catalogue_.enabled(ControlIdentifier::Retrieve)); - ASSERT(!catalogue_.enabled(ControlIdentifier::Archive)); - - // The lock will have occurred after the visitation phase, so add the lockfiles. - const auto&& lockfiles(catalogue_.lockfilePaths()); - lockfilePaths_.insert(lockfiles.begin(), lockfiles.end()); - } - - // If we are wiping only a subset, and as a result have indexes to mask out; do that first. - // This results in a failure mode merely being data becoming invisible (which has the correct - // effect for the user), to be wiped at a later date. - - if (!indexesToMask_.empty() && !wipeAll) { - for (const auto& index : indexesToMask_) { - logVerbose << "Index to mask: "; - logAlways << index << std::endl; - if (doit_) catalogue_.maskIndexEntry(index); - } - } - - // Now we want to do the actual deletion - // n.b. We delete carefully in a order such that we can always access the DB by what is left - - /// @todo: are all these exist checks necessary? - - for (const URI& uri : residualDataURIs_) { - if (store_.uriExists(uri)) { - store_.remove(uri, logAlways, logVerbose, doit_); - } - } - for (const PathName& path : residualPaths_) { - if (path.exists()) { - catalogue_.remove(path, logAlways, logVerbose, doit_); - } - } - - for (const URI& uri : dataURIs_) { - if (store_.uriExists(uri)) { - store_.remove(uri, logAlways, logVerbose, doit_); - } - } - - /// @todo: do not remove store uri if backend is S3 and uses a single bucket for all DBs - if (wipeAll && store_.type() != "file") - /// @todo: if the store is holding catalogue information (e.g. daos KVs) it - /// should not be removed - if (store_.uriExists(store_.uri())) - store_.remove(store_.uri(), logAlways, logVerbose, doit_); - - for (const std::set& pathset : {indexPaths_, - std::set{schemaPath_}, subtocPaths_, - std::set{tocPath_}, lockfilePaths_, - (wipeAll ? std::set{catalogue_.basePath()} : std::set{})}) { - - for (const PathName& path : pathset) { - if (path.exists()) { - catalogue_.remove(path, logAlways, logVerbose, doit_); - } - } - } -} - - -void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { - WipeVisitor::catalogueComplete(catalogue); - - // We wipe everything if there is nothingn within safePaths - i.e. there is - // no data that wasn't matched by the request - - bool wipeAll = safePaths_.empty() && safeURIs_.empty(); - - if (wipeAll) { - addMaskedPaths(); - addMetadataPaths(); - } else { - // Ensure we _really_ don't delete these if not wiping everything - subtocPaths_.clear(); - lockfilePaths_.clear(); - tocPath_ = ""; - schemaPath_ = ""; - } - - ensureSafePaths(); - - if (anythingToWipe()) { - if (wipeAll) calculateResidualPaths(); - - if (!porcelain_) report(wipeAll); - - // This is here as it needs to run whatever combination of doit/porcelain/... - if (wipeAll && !residualPaths_.empty()) { - - out_ << "Unexpected files present in directory: " << std::endl; - for (const auto& p : residualPaths_) out_ << " " << p << std::endl; - out_ << std::endl; - - } - if (wipeAll && !residualDataURIs_.empty()) { - - out_ << "Unexpected store units present in store: " << std::endl; - for (const auto& u : residualDataURIs_) out_ << " " << u << std::endl; - out_ << std::endl; - - } - if (wipeAll && (!residualPaths_.empty() || !residualDataURIs_.empty())) { - if (!unsafeWipeAll_) { - out_ << "Full wipe will not proceed without --unsafe-wipe-all" << std::endl; - if (doit_) - throw Exception("Cannot fully wipe unclean TocDB", Here()); - } - } - - if (doit_ || porcelain_) wipe(wipeAll); - } -} - - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 diff --git a/src/fdb5/toc/TocWipeVisitor.h b/src/fdb5/toc/TocWipeVisitor.h deleted file mode 100644 index fb5d32752..000000000 --- a/src/fdb5/toc/TocWipeVisitor.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/// @author Simon Smart -/// @date August 2019 - -#ifndef fdb5_TocWipeVisitor_H -#define fdb5_TocWipeVisitor_H - - -#include "fdb5/database/WipeVisitor.h" -#include "fdb5/toc/TocCatalogue.h" - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -class TocWipeVisitor : public WipeVisitor { - -public: - - TocWipeVisitor(const TocCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, - bool unsafeWipeAll); - ~TocWipeVisitor() override; - -private: // methods - - bool visitDatabase(const Catalogue& catalogue, const Store& store) override; - bool visitIndex(const Index& index) override; - void catalogueComplete(const Catalogue& catalogue) override; - - void addMaskedPaths(); - void addMetadataPaths(); - void ensureSafePaths(); - void calculateResidualPaths(); - - bool anythingToWipe() const; - - void report(bool wipeAll); - void wipe(bool wipeAll); - -private: // members - - // What are the parameters of the wipe operation - const TocCatalogue& catalogue_; - const Store& store_; - - metkit::mars::MarsRequest indexRequest_; - - std::string owner_; - - eckit::PathName tocPath_; - eckit::PathName schemaPath_; - - std::set subtocPaths_; - std::set lockfilePaths_; - std::set indexPaths_; - std::set dataURIs_; - - std::set safePaths_; - std::set safeURIs_; - std::set residualPaths_; - std::set residualDataURIs_; - - std::vector indexesToMask_; -}; - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 - -#endif From 2bc9a1959c813684c6fbf2655b4541deceaa09d5 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 6 Jul 2026 10:28:44 +0200 Subject: [PATCH 030/109] merge: undo --- src/fdb5/database/FieldLocation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/database/FieldLocation.cc b/src/fdb5/database/FieldLocation.cc index b9ecb67d1..208afcefc 100644 --- a/src/fdb5/database/FieldLocation.cc +++ b/src/fdb5/database/FieldLocation.cc @@ -115,7 +115,7 @@ FieldLocationBuilderBase::~FieldLocationBuilderBase() { //---------------------------------------------------------------------------------------------------------------------- -FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri.scheme() + ":" + uri.name()) { +FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri) { try { offset_ = eckit::Offset(std::stoll(uri.fragment())); } From cbfdf16e2ee5754305be474b76a661bec5625a89 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 6 Jul 2026 10:31:48 +0200 Subject: [PATCH 031/109] fix(rados): test name --- tests/fdb/rados/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt index 30d4caf3b..2dfa061d4 100644 --- a/tests/fdb/rados/CMakeLists.txt +++ b/tests/fdb/rados/CMakeLists.txt @@ -9,12 +9,13 @@ if (HAVE_RADOSFDB) foreach( _test ${rados_tests} ) - ecbuild_add_test( TARGET test_fdb5_rados_${_test} + ecbuild_add_test( TARGET fdb_test_${_test} SOURCES test_${_test}.cc + LABELS rados LIBS "${unit_test_libraries}" INCLUDES "${unit_test_include_dirs}" ENVIRONMENT FDB_RADOS_TEST_POOL=${FDB_RADOS_TEST_POOL} ) endforeach() -endif() \ No newline at end of file +endif() From eb6586134f98c739d505fd501a60bff199bfac85 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:51:52 +0200 Subject: [PATCH 032/109] fix(rados): catalogue --- src/fdb5/rados/RadosCatalogue.cc | 104 ++++++++++++++++++++++++++----- src/fdb5/rados/RadosCatalogue.h | 51 ++++++++++++--- 2 files changed, 133 insertions(+), 22 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index f6959c9bf..bd8df5e3c 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -17,19 +17,38 @@ // #include "fdb5/database/DatabaseNotFoundException.h" #include "fdb5/rados/RadosCatalogue.h" -// #include "fdb5/daos/DaosName.h" -// #include "fdb5/daos/DaosSession.h" + +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/Timer.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Tokenizer.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/DatabaseNotFoundException.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/database/WipeState.h" +#include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosIndex.h" -// #include "fdb5/daos/DaosWipeVisitor.h" +#include "fdb5/rules/Rule.h" +#include "fdb5/rules/Schema.h" -// using namespace eckit; +#include +#include +#include +#include namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : - Catalogue(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) { + CatalogueImpl(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) { // TODO: apply the mechanism in RootManager::directory, using // FileSpaceTables to determine root_pool_name_ according to key @@ -39,22 +58,21 @@ RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) : - Catalogue(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { + CatalogueImpl(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool = pool_; + std::string pool = pool_; std::string nspace = db_namespace_; #else - std::string pool = db_pool_; + std::string pool = db_pool_; std::string nspace = namespace_; #endif // Read the real DB key into the DB base object try { - std::vector data; eckit::MemoryStream ms = db_kv_->getMemoryStream(data, "key", "DB kv"); - dbKey_ = fdb5::Key(ms); + dbKey_ = fdb5::Key(ms); } catch (eckit::RadosEntityNotFoundException& e) { @@ -78,6 +96,12 @@ const Schema& RadosCatalogue::schema() const { return schema_; } +const Rule& RadosCatalogue::rule() const { + + ASSERT(rule_); + return *rule_; +} + void RadosCatalogue::loadSchema() { eckit::Timer timer("RadosCatalogue::loadSchema()", eckit::Log::debug()); @@ -92,14 +116,16 @@ void RadosCatalogue::loadSchema() { std::istringstream stream{std::string(data.begin(), data.end())}; schema_.load(stream); -} -WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, - std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { - NOTIMP; - // return new RadosWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); + rule_ = &schema_.matchingRule(dbKey_); } +// WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, +// std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { +// NOTIMP; +// // return new RadosWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); +// } + std::vector RadosCatalogue::indexes(bool) const { /// @note: sorted is not implemented as is not necessary in this backend. @@ -158,6 +184,54 @@ std::string RadosCatalogue::type() const { return RadosCatalogue::catalogueTypeName(); } +bool RadosCatalogue::uriBelongs(const eckit::URI& uri) const { + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + return (uri.scheme() == type()) && (n >= 2) && (parts[0] == pool_) && (parts[1] == db_namespace_); + +#else + + return (uri.scheme() == type()) && (n >= 2) && (parts[0] == db_pool_) && (parts[1] == namespace_); + +#endif +} + +//---------------------------------------------------------------------------------------------------------------------- + +/// Wipe-related methods are not implemented for the Rados backend. + +CatalogueWipeState RadosCatalogue::wipeInit() const { + NOTIMP; +} + +bool RadosCatalogue::markIndexForWipe(const Index&, bool, CatalogueWipeState&) const { + NOTIMP; +} + +void RadosCatalogue::finaliseWipeState(CatalogueWipeState&) const { + NOTIMP; +} + +bool RadosCatalogue::doWipeUnknowns(const std::set&) const { + NOTIMP; +} + +bool RadosCatalogue::doWipeURIs(const CatalogueWipeState&) const { + NOTIMP; +} + +void RadosCatalogue::doWipeEmptyDatabase() const { + NOTIMP; +} + +bool RadosCatalogue::doUnsafeFullWipe() const { + NOTIMP; +} + // void RadosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, std::ostream& logVerbose, bool // doit) { diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h index c936a1ef5..abe6259d0 100644 --- a/src/fdb5/rados/RadosCatalogue.h +++ b/src/fdb5/rados/RadosCatalogue.h @@ -13,19 +13,40 @@ #pragma once -// #include "fdb5/database/DB.h" +#include "eckit/config/Configuration.h" +#include "eckit/container/Queue.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Offset.h" + +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/api/helpers/MoveIterator.h" +#include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/MoveVisitor.h" +#include "fdb5/database/PurgeVisitor.h" +#include "fdb5/database/StatsReportVisitor.h" #include "fdb5/rados/RadosCommon.h" #include "fdb5/rules/Schema.h" -// #include "fdb5/rados/RadosEngine.h" + +#include +#include +#include +#include +#include namespace fdb5 { +class Rule; +class RuleDatabase; +class CatalogueWipeState; + //---------------------------------------------------------------------------------------------------------------------- /// DB that implements the FDB on Rados -class RadosCatalogue : public Catalogue, public RadosCommon { +class RadosCatalogue : public CatalogueImpl, public RadosCommon { public: // methods @@ -45,18 +66,18 @@ class RadosCatalogue : public Catalogue, public RadosCommon { void checkUID() const override { NOTIMP; }; bool exists() const override; void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override { NOTIMP; }; - std::vector metadataPaths() const override { NOTIMP; }; const Schema& schema() const override; StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }; - WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, - bool porcelain, bool unsafeWipeAll) const override; + // WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool + // doit, + // bool porcelain, bool unsafeWipeAll) const override; MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; }; - void maskIndexEntry(const Index& index) const override { NOTIMP; }; + // void maskIndexEntry(const Index& index) const override { NOTIMP; }; void loadSchema() override; @@ -70,6 +91,21 @@ class RadosCatalogue : public Catalogue, public RadosCommon { // Control access properties of the DB void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override { NOTIMP; }; + const Rule& rule() const override; + + bool uriBelongs(const eckit::URI& uri) const override; + + void maskIndexEntries(const std::set& indexes) const override { NOTIMP; } + + /// Wipe-related methods (not implemented for the Rados backend) + CatalogueWipeState wipeInit() const override; + bool markIndexForWipe(const Index& index, bool include, CatalogueWipeState& wipeState) const override; + void finaliseWipeState(CatalogueWipeState& wipeState) const override; + bool doWipeUnknowns(const std::set& unknownURIs) const override; + bool doWipeURIs(const CatalogueWipeState& wipeState) const override; + void doWipeEmptyDatabase() const override; + bool doUnsafeFullWipe() const override; + protected: // members Key currentIndexKey_; @@ -77,6 +113,7 @@ class RadosCatalogue : public Catalogue, public RadosCommon { private: // members Schema schema_; + const RuleDatabase* rule_{nullptr}; }; //---------------------------------------------------------------------------------------------------------------------- From edacdf2e14c3242c7c2ec2503ecf7c7f1cf87c2e Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:52:16 +0200 Subject: [PATCH 033/109] fix(rados): catalogue read --- src/fdb5/rados/RadosCatalogueReader.cc | 60 +++++++++++++------------- src/fdb5/rados/RadosCatalogueReader.h | 38 +++++++++++----- 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index 9f21a7b23..7d6f2e2b1 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -8,9 +8,10 @@ * does it submit to any jurisdiction. */ +#include "fdb5/rados/RadosCatalogueReader.h" + #include "fdb5/LibFdb5.h" #include "fdb5/rados/RadosIndex.h" -#include "fdb5/rados/RadosCatalogueReader.h" namespace fdb5 { @@ -19,18 +20,16 @@ namespace fdb5 { /// @note: as opposed to the TOC catalogue, the DAOS catalogue does not pre-load all indexes from storage. /// Instead, it selects and loads only those indexes that are required to fulfil the request. -RadosCatalogueReader::RadosCatalogueReader(const Key& key, const fdb5::Config& config) : - RadosCatalogue(key, config) { +RadosCatalogueReader::RadosCatalogueReader(const Key& key, const Config& config) : RadosCatalogue(key, config) { /// @todo: schema is being loaded at DaosCatalogueWriter creation for write, but being loaded /// at DaosCatalogueReader::open for read. Is this OK? - } -RadosCatalogueReader::RadosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config) : +RadosCatalogueReader::RadosCatalogueReader(const eckit::URI& uri, const Config& config) : RadosCatalogue(uri, ControlIdentifiers{}, config) {} -bool RadosCatalogueReader::selectIndex(const Key &key) { +bool RadosCatalogueReader::selectIndex(const Key& key) { if (currentIndexKey_ == key) { return true; @@ -46,7 +45,7 @@ bool RadosCatalogueReader::selectIndex(const Key &key) { /// - ensure catalogue kv exists (daos_kv_open) int idx_loc_max_len = 512; /// @todo: take from config - std::vector n((long) idx_loc_max_len); + std::vector n((long)idx_loc_max_len); long res; try { @@ -54,42 +53,38 @@ bool RadosCatalogueReader::selectIndex(const Key &key) { /// @note: performed RPCs: /// - retrieve index kv location from catalogue kv (daos_kv_get) res = db_kv_->get(key.valuesToString(), &n[0], idx_loc_max_len); - - } catch (eckit::RadosEntityNotFoundException& e) { + } + catch (eckit::RadosEntityNotFoundException& e) { /// @note: performed RPCs: /// - close catalogue kv (daos_obj_close) return false; - } eckit::URI uri{std::string{n.begin(), std::next(n.begin(), res)}}; -// #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE -// eckit::RadosPersistentKeyValue index_kv{uri, true}; -// #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH -// eckit::RadosPersistentKeyValue index_kv{uri}; -// #else + // #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + // eckit::RadosPersistentKeyValue index_kv{uri, true}; + // #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH + // eckit::RadosPersistentKeyValue index_kv{uri}; + // #else eckit::RadosKeyValue index_kv{uri}; -// #endif + // #endif - indexes_[key] = Index(new fdb5::RadosIndex(key, index_kv, true)); + indexes_[key] = Index(new RadosIndex(key, index_kv, true)); /// @note: performed RPCs: /// - close catalogue kv (daos_obj_close) - } current_ = indexes_[key]; return true; - } void RadosCatalogueReader::deselectIndex() { - NOTIMP; //< should not be called - + NOTIMP; //< should not be called } bool RadosCatalogueReader::open() { @@ -105,19 +100,22 @@ bool RadosCatalogueReader::open() { RadosCatalogue::loadSchema(); return true; - } -bool RadosCatalogueReader::axis(const std::string &keyword, eckit::StringSet &s) const { +std::optional RadosCatalogueReader::computeAxis(const std::string& keyword) const { + + Axis s; bool found = false; if (current_.axes().has(keyword)) { found = true; - const eckit::DenseSet& a = current_.axes().values(keyword); - s.insert(a.begin(), a.end()); + s.merge(current_.axes().values(keyword)); } - return found; + if (found) { + return s; + } + return std::nullopt; } bool RadosCatalogueReader::retrieve(const Key& key, Field& field) const { @@ -125,14 +123,14 @@ bool RadosCatalogueReader::retrieve(const Key& key, Field& field) const { eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; eckit::Log::debug() << "Scanning index " << current_.location() << std::endl; - if (!current_.mayContain(key)) return false; - - return current_.get(key, fdb5::Key(), field); + if (!current_.mayContain(key)) + return false; + return current_.get(key, Key(), field); } -static fdb5::CatalogueBuilder builder("rados.reader"); +static CatalogueReaderBuilder builder("rados"); //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueReader.h b/src/fdb5/rados/RadosCatalogueReader.h index 7aa787275..84d840f95 100644 --- a/src/fdb5/rados/RadosCatalogueReader.h +++ b/src/fdb5/rados/RadosCatalogueReader.h @@ -13,8 +13,23 @@ #pragma once +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/types/Types.h" + +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/DbStats.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" #include "fdb5/rados/RadosCatalogue.h" +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -23,38 +38,39 @@ namespace fdb5 { class RadosCatalogueReader : public RadosCatalogue, public CatalogueReader { -public: // methods +public: // methods RadosCatalogueReader(const Key& key, const fdb5::Config& config); RadosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config); DbStats stats() const override { NOTIMP; } - bool selectIndex(const Key &key) override; + bool selectIndex(const Key& key) override; void deselectIndex() override; bool open() override; - void flush() override {} + void flush(size_t archivedFields) override {} void clean() override {} void close() override {} - - bool axis(const std::string &keyword, eckit::StringSet &s) const override; bool retrieve(const Key& key, Field& field) const override; - void print( std::ostream &out ) const override { NOTIMP; } + void print(std::ostream& out) const override { NOTIMP; } + +private: // methods -private: // types + std::optional computeAxis(const std::string& keyword) const override; - typedef std::map< Key, Index> IndexStore; +private: // types -private: // members + typedef std::map IndexStore; + +private: // members IndexStore indexes_; Index current_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 From 3c6a5131a923c10de4aef86469147c31fbef83fd Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:52:21 +0200 Subject: [PATCH 034/109] fix(rados): catalogue write --- src/fdb5/rados/RadosCatalogueWriter.cc | 135 +++++++++++-------------- src/fdb5/rados/RadosCatalogueWriter.h | 42 ++++---- 2 files changed, 80 insertions(+), 97 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index eb0a2471a..b054ef331 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -22,8 +22,8 @@ #include "eckit/io/rados/RadosException.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "fdb5/rados/RadosIndex.h" #include "fdb5/rados/RadosCatalogueWriter.h" +#include "fdb5/rados/RadosIndex.h" // using namespace eckit; @@ -31,7 +31,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& config) : +RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& config) : RadosCatalogue(key, config), firstIndexWrite_(false) { /// @note: performed RPCs: @@ -62,12 +62,8 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c db_kv_->ensureCreated(); /// write schema under "schema" - eckit::Log::debug() << "Copy schema from " - << config_.schemaPath() - << " to " - << db_kv_->uri().asString() - << " at key 'schema'." - << std::endl; + eckit::Log::debug() << "Copy schema from " << config_.schemaPath() << " to " + << db_kv_->uri().asString() << " at key 'schema'." << std::endl; eckit::FileHandle in(config_.schemaPath()); std::vector data; @@ -80,7 +76,7 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c db_kv_->put("schema", &data[0], data.size()); /// write dbKey under "key" - eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); { @@ -91,19 +87,18 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c int db_key_max_len = 512; // @todo: take from config if (hs.bytesWritten() > db_key_max_len) throw eckit::Exception("Serialised db key exceeded configured maximum db key length."); - - db_kv_->put("key", h.data(), hs.bytesWritten()); + + db_kv_->put("key", h.data(), hs.bytesWritten()); /// index newly created catalogue kv in main kv int db_loc_max_len = 512; // @todo: take from config - std::string nstr = db_kv_->uri().asString(); - if (nstr.length() > db_loc_max_len) + std::string nstr = db_kv_->uri().asString(); + if (nstr.length() > db_loc_max_len) throw eckit::Exception("Serialised db location exceeded configured maximum db location length."); root_kv_->put(db_name, nstr.data(), nstr.length()); - } - + /// @todo: record or read dbUID /// @note: performed RPCs: @@ -112,30 +107,31 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c RadosCatalogue::loadSchema(); /// @todo: TocCatalogue::checkUID(); - } -RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config) : +RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config) : RadosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) { NOTIMP; - } RadosCatalogueWriter::~RadosCatalogueWriter() { clean(); close(); +} +bool RadosCatalogueWriter::createIndex(const Key& /* idxKey */, size_t /* datumKeySize */) { + return true; } bool RadosCatalogueWriter::selectIndex(const Key& key) { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool = pool_; + std::string pool = pool_; std::string nspace = db_namespace_; #else - std::string pool = db_pool_; + std::string pool = db_pool_; std::string nspace = namespace_; #endif @@ -151,82 +147,71 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { try { - std::vector n((long) idx_loc_max_len); + std::vector n((long)idx_loc_max_len); long res; /// @note: performed RPCs: /// - get index location from catalogue kv (daos_kv_get) res = db_kv_->get(key.valuesToString(), &n[0], idx_loc_max_len); - indexes_[key] = Index( - new fdb5::RadosIndex( - key, -// #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE -// eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}, true}, -// #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH -// eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, -// #else - eckit::RadosKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, -// #endif - false - ) - ); - - } catch (eckit::RadosEntityNotFoundException& e) { + indexes_[key] = Index(new fdb5::RadosIndex( + key, + // #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE + // eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), + // std::next(n.begin(), res)}}, true}, + // #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH + // eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), + // std::next(n.begin(), res)}}}, + // #else + eckit::RadosKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, + // #endif + false)); + } + catch (eckit::RadosEntityNotFoundException& e) { firstIndexWrite_ = true; - - indexes_[key] = Index( - new fdb5::RadosIndex( - key, - eckit::RadosNamespace{pool, nspace} - ) - ); + + indexes_[key] = Index(new fdb5::RadosIndex(key, eckit::RadosNamespace{pool, nspace})); /// index index kv in catalogue kv std::string nstr{indexes_[key].location().uri().asString()}; if (nstr.length() > idx_loc_max_len) throw eckit::Exception("Serialised index location exceeded configured maximum index location length."); - /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on first write to an index key): + /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on + /// first write to an index key): /// - record index kv location into catalogue kv (daos_kv_put) -- always performed db_kv_->put(key.valuesToString(), nstr.data(), nstr.length()); /// @note: performed RPCs: /// - close index kv when destroyed (daos_obj_close) - } /// @note: performed RPCs: /// - close catalogue kv (daos_obj_close) - } current_ = indexes_[key]; return true; - } void RadosCatalogueWriter::deselectIndex() { - current_ = Index(); + current_ = Index(); currentIndexKey_ = Key(); firstIndexWrite_ = false; - } void RadosCatalogueWriter::clean() { - flush(); + flush(0); deselectIndex(); - } void RadosCatalogueWriter::close() { closeIndexes(); - } const Index& RadosCatalogueWriter::currentIndex() { @@ -237,19 +222,19 @@ const Index& RadosCatalogueWriter::currentIndex() { } return current_; - } /// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these /// new updates be retrieved and put into in-memory axes from time to time, e.g. every /// time a value is put in an axis KeyValue? -void RadosCatalogueWriter::archive(const Key& key, std::unique_ptr fieldLocation) { +void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, + std::shared_ptr fieldLocation) { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool = pool_; + std::string pool = pool_; std::string nspace = db_namespace_; #else - std::string pool = db_pool_; + std::string pool = db_pool_; std::string nspace = namespace_; #endif @@ -270,15 +255,16 @@ void RadosCatalogueWriter::archive(const Key& key, std::unique_ptr axesToExpand; std::vector valuesToAdd; std::string axisNames = ""; - std::string sep = ""; + std::string sep = ""; - for (Key::const_iterator i = key.begin(); i != key.end(); ++i) { + for (Key::const_iterator i = datumKey.begin(); i != datumKey.end(); ++i) { - const std::string &keyword = i->first; + const std::string& keyword = i->first; - std::string value = key.canonicalValue(keyword); + const std::string& value = i->second; - if (value.length() == 0) continue; + if (value.length() == 0) + continue; axisNames += sep + keyword; sep = ","; @@ -288,18 +274,16 @@ void RadosCatalogueWriter::archive(const Key& key, std::unique_ptrget().contains(value)) { + // if (!axis_set.has_value() || !axis_set->get().contains(value)) { if (!axis_set.contains(value)) { axesToExpand.push_back(keyword); valuesToAdd.push_back(value); - } - } /// index the field and update in-memory axes - current_.put(key, field); + current_.put(datumKey, field); /// persist axis names if (firstIndexWrite_) { @@ -318,13 +302,13 @@ void RadosCatalogueWriter::archive(const Key& key, std::unique_ptr(current_.content())->putAxisNames(axisNames); firstIndexWrite_ = false; - } /// @todo: axes are supposed to be sorted before persisting. How do we do this with the DAOS approach? /// sort axes every time they are loaded in the read pathway? - if (axesToExpand.empty()) return; + if (axesToExpand.empty()) + return; /// expand axis info in DAOS while (!axesToExpand.empty()) { @@ -340,32 +324,29 @@ void RadosCatalogueWriter::archive(const Key& key, std::unique_ptrsecond.flush(); db_kv_->flush(); root_kv_->flush(); #endif - if (!current_.null()) current_ = Index(); - + if (!current_.null()) + current_ = Index(); } void RadosCatalogueWriter::closeIndexes() { - indexes_.clear(); // all indexes instances destroyed - + indexes_.clear(); // all indexes instances destroyed } -static fdb5::CatalogueBuilder builder("rados.writer"); +static fdb5::CatalogueWriterBuilder builder("rados"); //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h index 195cf11e6..eb3eceb42 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.h +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -23,60 +23,62 @@ namespace fdb5 { class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { -public: // methods +public: // methods - RadosCatalogueWriter(const Key &key, const fdb5::Config& config); - RadosCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config); + RadosCatalogueWriter(const Key& key, const fdb5::Config& config); + RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config); virtual ~RadosCatalogueWriter() override; - void index(const Key &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; + void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; void reconsolidate() override { NOTIMP; } /// Mount an existing TocCatalogue, which has a different metadata key (within /// constraints) to allow on-line rebadging of data /// variableKeys: The keys that are allowed to differ between the two DBs - void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { NOTIMP; }; + void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { + NOTIMP; + }; -// // Hide the contents of the DB!!! -// void hideContents() override; + // // Hide the contents of the DB!!! + // void hideContents() override; -// bool enabled(const ControlIdentifier& controlIdentifier) const override; + // bool enabled(const ControlIdentifier& controlIdentifier) const override; const Index& currentIndex() override; -protected: // methods +protected: // methods - virtual bool selectIndex(const Key &key) override; + virtual bool selectIndex(const Key& key) override; + bool createIndex(const Key& idxKey, size_t datumKeySize) override; virtual void deselectIndex() override; bool open() override { NOTIMP; } - void flush() override; + void flush(size_t archivedFields) override; void clean() override; void close() override; - void archive(const Key& key, std::unique_ptr fieldLocation) override; + void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; - virtual void print( std::ostream &out ) const override { NOTIMP; } + virtual void print(std::ostream& out) const override { NOTIMP; } -private: // methods +private: // methods void closeIndexes(); -private: // types +private: // types - typedef std::map< Key, Index> IndexStore; + typedef std::map IndexStore; -private: // members +private: // members - IndexStore indexes_; + IndexStore indexes_; Index current_; bool firstIndexWrite_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 From f712fc680bb3bde8a12c2e786577700069b75b2e Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:52:28 +0200 Subject: [PATCH 035/109] fix(rados): common --- src/fdb5/rados/RadosCommon.cc | 30 +++++++++++++++++++----------- src/fdb5/rados/RadosCommon.h | 15 ++++++++------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 4583c299b..6c50d62f9 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -8,19 +8,27 @@ * does it submit to any jurisdiction. */ -#include +#include "fdb5/rados/RadosCommon.h" +#include "eckit/config/LocalConfiguration.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" -// #include "eckit/utils/Tokenizer.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" -#include "fdb5/rados/RadosCommon.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" + +#include +#include +#include +#include namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& component, const fdb5::Key& key) { +RadosCommon::RadosCommon(const Config& config, const std::string& component, const Key& key) { std::vector valid{"catalogue", "store"}; ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); @@ -46,7 +54,7 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen #endif } -RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { +RadosCommon::RadosCommon(const Config& config, const std::string& component, const eckit::URI& uri) { /// @note: validity of input URI is not checked here because this constructor is only triggered /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand @@ -55,7 +63,7 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - pool_ = db_name.nspace().pool().name(); + pool_ = db_name.nspace().pool().name(); db_namespace_ = db_name.nspace().name(); readConfig(config, component, false); @@ -65,13 +73,13 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen #else - db_pool_ = db_name.nspace().pool().name(); + db_pool_ = db_name.nspace().pool().name(); namespace_ = db_name.nspace().name(); readConfig(config, component, false); const auto parts = eckit::Tokenizer("_").tokenize(db_pool_); - const auto n = parts.size(); + const auto n = parts.size(); ASSERT(n > 1); pool_prefix_ = parts[0]; @@ -82,9 +90,9 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen } #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL -void RadosCommon::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) { +void RadosCommon::readConfig(const Config& config, const std::string& component, bool readPool) { #else -void RadosCommon::readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) { +void RadosCommon::readConfig(const Config& config, const std::string& component, bool readNamespace) { #endif eckit::LocalConfiguration c{}; @@ -168,7 +176,7 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp #endif // if (c.has("client")) - // fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); + // DaosManager::instance().configure(c.getSubConfiguration("client")); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index e6f8b6bfc..25e4604ea 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -13,31 +13,32 @@ #pragma once -#include #include "eckit/filesystem/URI.h" -// #include "eckit/utils/Optional.h" -// #include "eckit/io/rados/RadosAsyncKeyValue.h" +#include "eckit/io/Length.h" #include "eckit/io/rados/RadosKeyValue.h" #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" #include "fdb5/fdb5_config.h" +#include +#include + namespace fdb5 { class RadosCommon { public: // methods - RadosCommon(const fdb5::Config&, const std::string& component, const fdb5::Key&); - RadosCommon(const fdb5::Config&, const std::string& component, const eckit::URI&); + RadosCommon(const Config&, const std::string& component, const Key&); + RadosCommon(const Config&, const std::string& component, const eckit::URI&); private: // methods #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - void readConfig(const fdb5::Config& config, const std::string& component, bool readPool); + void readConfig(const Config& config, const std::string& component, bool readPool); #else - void readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace); + void readConfig(const Config& config, const std::string& component, bool readNamespace); #endif protected: // members From fc632fd5761d11c3164f189010efa2bc08076bad Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:52:36 +0200 Subject: [PATCH 036/109] fix(rados): engine --- src/fdb5/rados/RadosEngine.h | 39 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 916473cc5..ccb46fb75 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -13,13 +13,20 @@ #pragma once -// #include "eckit/utils/Optional.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosKeyValue.h" -// #include "eckit/io/rados/RadosAsyncKeyValue.h" + +#include "metkit/mars/MarsRequest.h" #include "fdb5/database/Engine.h" #include "fdb5/fdb5_config.h" +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -34,23 +41,23 @@ class RadosEngine : public fdb5::Engine { protected: // methods - virtual std::string name() const override; + std::string name() const override; - virtual std::string dbType() const override { NOTIMP; }; + std::string dbType() const override { NOTIMP; }; - virtual eckit::URI location(const Key& key, const Config& config) const override { NOTIMP; }; + eckit::URI location(const Key& key, const Config& config) const override { NOTIMP; }; - virtual bool canHandle(const eckit::URI&, const Config&) const override { NOTIMP; }; + bool canHandle(const eckit::URI&, const Config&) const override { NOTIMP; }; - virtual std::vector allLocations(const Key& key, const Config& config) const override { NOTIMP; }; + // std::vector allLocations(const Key& key, const Config& config) const override { NOTIMP; }; - virtual std::vector visitableLocations(const Key& key, const Config& config) const override; - virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, - const Config& config) const override; + std::vector visitableLocations(const Key& key, const Config& config) const override; + std::vector visitableLocations(const metkit::mars::MarsRequest& rq, + const Config& config) const override; - virtual std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; }; + // std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; }; - virtual void print(std::ostream& out) const override { NOTIMP; }; + void print(std::ostream& out) const override { NOTIMP; }; private: // methods @@ -73,11 +80,11 @@ class RadosEngine : public fdb5::Engine { #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - mutable eckit::Optional root_kv_; - // eckit::Optional db_kv_; + mutable std::optional root_kv_; + // std::optional db_kv_; #else - mutable eckit::Optional root_kv_; - // eckit::Optional db_kv_; + mutable std::optional root_kv_; + // std::optional db_kv_; #endif // eckit::Length maxPartSize_; From 52eafdf881ceb32ef544376df0cba94d094059df Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:52:43 +0200 Subject: [PATCH 037/109] fix(rados): index --- src/fdb5/rados/RadosIndex.cc | 152 ++++++++++++++++++----------------- src/fdb5/rados/RadosIndex.h | 46 ++++++----- 2 files changed, 108 insertions(+), 90 deletions(-) diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 9cce9feb4..e4e5fc392 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -8,33 +8,55 @@ * does it submit to any jurisdiction. */ -#include // for PATH_MAX - -#include +#include "fdb5/rados/RadosIndex.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" +#include "eckit/io/Length.h" #include "eckit/io/MemoryHandle.h" -#include "eckit/serialisation/MemoryStream.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" #include "eckit/serialisation/HandleStream.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/Reanimator.h" #include "eckit/utils/Tokenizer.h" -#include "fdb5/rados/RadosIndex.h" +#include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldDetails.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" #include "fdb5/rados/RadosLazyFieldLocation.h" +#include // for PATH_MAX +#include +#include +#include +#include +#include +#include +#include +#include +#include + // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) // eckit::RadosPersistentKeyValue buildIndexKvName(const fdb5::Key& key, const eckit::RadosNamespace& name) { // #else // eckit::RadosKeyValue buildIndexKvName(const fdb5::Key& key, const eckit::RadosNamespace& name) { // #endif - /// create index kv - /// @todo: pass oclass from config - /// @todo: hash string into lower oid bits +/// create index kv +/// @todo: pass oclass from config +/// @todo: hash string into lower oid bits // #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE // return eckit::RadosPersistentKeyValue{name.poolName(), name.containerName(), key.valuesToString(), true}; // #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH // return eckit::RadosPersistentKeyValue{name.poolName(), name.containerName(), key.valuesToString()}; // #else - // return eckit::RadosKeyValue{name.poolName(), name.containerName(), key.valuesToString()}; +// return eckit::RadosKeyValue{name.poolName(), name.containerName(), key.valuesToString()}; // #endif // } @@ -43,8 +65,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : - IndexBase(key, "radosKeyValue"), +RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : + IndexBase(key, "radosKeyValue"), location_(eckit::RadosKeyValue{name.pool().name(), name.name(), key.valuesToString()}, 0), idx_kv_(location_.radosName().uri()) { @@ -53,7 +75,7 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : /// - create/open index kv (daos_kv_open) /// write indexKey under "key" - eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); { @@ -69,26 +91,23 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : /// @note: performed RPCs: /// - record index key into index kv (daos_kv_put) idx_kv_.put("key", h.data(), hs.bytesWritten()); - } // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) // RadosIndex::RadosIndex(const Key& key, const eckit::RadosPersistentKeyValue& name, bool readAxes) : // #else RadosIndex::RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes) : -// #endif - IndexBase(key, "radosKeyValue"), - location_(name, 0), - idx_kv_(name.uri()) { - - if (readAxes) updateAxes(); + // #endif + IndexBase(key, "radosKeyValue"), location_(name, 0), idx_kv_(name.uri()) { + if (readAxes) { + updateAxes(); + } } void RadosIndex::putAxisNames(const std::string& names) { idx_kv_.put("axes", names.data(), names.length()); - } void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) { @@ -97,22 +116,15 @@ void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) if (axis_kv == axis_kvs_.end()) { std::string kv_name = key().valuesToString() + std::string{"."} + axis; - axis_kvs_.emplace( - std::piecewise_construct, - std::forward_as_tuple(axis), - std::forward_as_tuple( - location_.radosName().nspace().pool().name(), - location_.radosName().nspace().name(), - kv_name - ) - ); + axis_kvs_.emplace(std::piecewise_construct, std::forward_as_tuple(axis), + std::forward_as_tuple(location_.radosName().nspace().pool().name(), + location_.radosName().nspace().name(), kv_name)); axis_kv = axis_kvs_.find(axis); } std::string v{"1"}; axis_kv->second.put(value, v.data(), v.length()); - } void RadosIndex::updateAxes() { @@ -121,7 +133,7 @@ void RadosIndex::updateAxes() { /// - ensure axis kv exists (daos_obj_open) int axis_names_max_len = 512; /// @todo: take from config - std::vector axes_data((long) axis_names_max_len); + std::vector axes_data((long)axis_names_max_len); /// @note: performed RPCs: /// - get axes key size and content (daos_kv_get without buffer + daos_kv_get) @@ -135,7 +147,8 @@ void RadosIndex::updateAxes() { /// @note: performed RPCs: /// - generate axis kv oid (daos_obj_generate_oid) /// - ensure axis kv exists (daos_obj_open) - eckit::RadosKeyValue axis_kv{idx_kv_.nspace().pool().name(), idx_kv_.nspace().name(), indexKey + std::string{"."} + name}; + eckit::RadosKeyValue axis_kv{idx_kv_.nspace().pool().name(), idx_kv_.nspace().name(), + indexKey + std::string{"."} + name}; /// @note: performed RPCs: /// - one or more kv list (daos_kv_list) @@ -143,10 +156,9 @@ void RadosIndex::updateAxes() { } axes_.sort(); - } -bool RadosIndex::get(const Key &key, const Key &remapKey, Field &field) const { +bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { /// @note: performed RPCs: /// - ensure index kv exists (daos_obj_open) @@ -154,54 +166,52 @@ bool RadosIndex::get(const Key &key, const Key &remapKey, Field &field) const { std::string query{key.valuesToString()}; int field_loc_max_len = 512; /// @todo: read from config - std::vector loc_data((long) field_loc_max_len); + std::vector loc_data((long)field_loc_max_len); long res; try { /// @note: performed RPCs: /// - retrieve field array location from index kv (daos_kv_get) - res = idx_kv_.get(query, &loc_data[0], (long) field_loc_max_len); - - } catch (eckit::RadosEntityNotFoundException& e) { + res = idx_kv_.get(query, &loc_data[0], (long)field_loc_max_len); + } + catch (eckit::RadosEntityNotFoundException& e) { /// @note: performed RPCs: /// - close index kv (daos_obj_close) return false; - } - eckit::MemoryStream ms{&loc_data[0], (size_t) res}; + eckit::MemoryStream ms{&loc_data[0], (size_t)res}; /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. time_t ts; ms >> ts; fdb5::FieldLocation* loc = eckit::Reanimator::reanimate(ms); - field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); + field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); /// @note: performed RPCs: /// - close index kv (daos_obj_close) return true; - } -void RadosIndex::add(const Key &key, const Field &field) { +void RadosIndex::add(const Key& key, const Field& field) { - eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); { eckit::AutoClose closer(h); /// @note: in the POSIX back-end, keeping a timestamp per index is necessary, to allow - /// determining which was the latest indexed field in cases where multiple processes + /// determining which was the latest indexed field in cases where multiple processes /// index a same field or in cases where multiple catalogues are combined with DistFDB. /// In the DAOS back-end, however, determining the latest indexed field is straigthforward - /// as all parallel processes writing fields for a same index key will share a DAOS + /// as all parallel processes writing fields for a same index key will share a DAOS /// key-value, and the last indexing will supersede the previous ones. - /// DistFDB will be obsoleted in favour of a centralised catalogue mechanism which can + /// DistFDB will be obsoleted in favour of a centralised catalogue mechanism which can /// index fields on multiple catalogues. /// Therefore keeping timestamps in DAOS should not be necessary. /// They are kept for now only for informational purposes. @@ -218,12 +228,11 @@ void RadosIndex::add(const Key &key, const Field &field) { /// - ensure index kv exists (daos_obj_open) /// - record field key and location into index kv (daos_kv_put) /// - close index kv when destroyed (daos_obj_close) - idx_kv_.put(key.valuesToString(), h.data(), hs.bytesWritten()); - + idx_kv_.put(key.valuesToString(), h.data(), hs.bytesWritten()); } -void RadosIndex::entries(EntryVisitor &visitor) const { - +void RadosIndex::entries(EntryVisitor& visitor) const { + Index instantIndex(const_cast(this)); // Allow the visitor to selectively decline to visit the entries in this index @@ -235,56 +244,56 @@ void RadosIndex::entries(EntryVisitor &visitor) const { for (const auto& key : idx_kv_.keys()) { - if (key == "axes" || key == "key") continue; + if (key == "axes" || key == "key") + continue; - /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each - /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field - /// keys present in an index -- without retrieving the actual location --, and + /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each + /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field + /// keys present in an index -- without retrieving the actual location --, and /// ListVisitor::visitDatum is called for each (see note at the top of DaosLazyFieldLocation.h). - /// When a field key is matched in visitDatum, DaosLazyFieldLocation::stableLocation is called, - /// which in turn calls this method here and triggers retrieval and deserialisation of the - /// indexed DaosFieldLocation, and returns it. Since the deserialised instance is of a + /// When a field key is matched in visitDatum, DaosLazyFieldLocation::stableLocation is called, + /// which in turn calls this method here and triggers retrieval and deserialisation of the + /// indexed DaosFieldLocation, and returns it. Since the deserialised instance is of a /// polymorphic class, it needs to be reanimated. fdb5::FieldLocation* loc = new fdb5::RadosLazyFieldLocation(location_.radosName(), key); fdb5::Field field(std::move(*loc), time_t(), fdb5::FieldDetails()); visitor.visitDatum(field, key); - } - } } -const std::vector RadosIndex::dataURIs() const { +std::vector RadosIndex::dataURIs() const { /// @note: if daos index + daos store, this will return a uri to a DAOS array for each indexed field /// @note: if daos index + posix store, this will return a vector of unique uris to all referenced posix files /// in this index (one for each writer process that has written to the index) - /// @note: in the case where we have a daos store, the current implementation of dataURIs is unnecessarily inefficient. - /// This method is only called in DaosWipeVisitor, where the uris obtained from this method are processed to obtain - /// unique store container paths - will always result in just one container uri! Having a URI store for each index in - /// DAOS could make this process more efficient, but it would imply more KV operations and slow down field writes. - /// @note: in the case where we have a posix store there will be more than one unique store file paths. The current + /// @note: in the case where we have a daos store, the current implementation of dataURIs is unnecessarily + /// inefficient. + /// This method is only called in DaosWipeVisitor, where the uris obtained from this method are processed to + /// obtain unique store container paths - will always result in just one container uri! Having a URI store for + /// each index in DAOS could make this process more efficient, but it would imply more KV operations and slow down + /// field writes. + /// @note: in the case where we have a posix store there will be more than one unique store file paths. The current /// implementation is still inefficient but preferred to maintaining a URI store in the DAOS catalogue std::set res; for (const auto& key : idx_kv_.keys()) { - if (key == "axes" || key == "key") continue; + if (key == "axes" || key == "key") + continue; std::vector data; eckit::MemoryStream ms = idx_kv_.getMemoryStream(data, key, "index kv"); - + time_t ts; ms >> ts; std::unique_ptr fl(eckit::Reanimator::reanimate(ms)); res.insert(fl->uri()); - } return std::vector(res.begin(), res.end()); - } #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH @@ -295,10 +304,9 @@ void RadosIndex::flush() { } idx_kv_.flush(); - } #endif //----------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index 4762b4d5a..b9092e56c 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -13,14 +13,23 @@ #pragma once -#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/io/rados/RadosAsyncKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" -#include "fdb5/fdb5_config.h" +#include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/Field.h" #include "fdb5/database/Index.h" +#include "fdb5/database/IndexStats.h" +#include "fdb5/database/Key.h" #include "fdb5/rados/RadosIndexLocation.h" +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -28,30 +37,30 @@ namespace fdb5 { class RadosIndex : public IndexBase { -public: // methods +public: // methods /// @note: creates a new index in DAOS, in the container pointed to by 'name' RadosIndex(const Key& key, const eckit::RadosNamespace& name); /// @note: used to represent and operate with an index which already exists in DAOS -// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -// RadosIndex(const Key& key, const eckit::RadosPersistentKeyValue& name, bool readAxes = true); -// #else + // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + // RadosIndex(const Key& key, const eckit::RadosPersistentKeyValue& name, bool readAxes = true); + // #else RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes = true); -// #endif + // #endif void flock() const override { NOTIMP; } void funlock() const override { NOTIMP; } - /// @note: these methods are required for RadosCatalogueWriter to directly manipulate + /// @note: these methods are required for RadosCatalogueWriter to directly manipulate /// idx_kv_ and axis_kvs_ within the RadosIndex. Upon flush, the index will flush all /// operations performed on these kvs (if PERSIST_ON_FLUSH). void putAxisNames(const std::string& names); void putAxisValue(const std::string& axis, const std::string& value); -private: // methods +private: // methods const IndexLocation& location() const override { return location_; } - const std::vector dataURIs() const override; + std::vector dataURIs() const override; bool dirty() const override { NOTIMP; } @@ -61,8 +70,8 @@ class RadosIndex : public IndexBase { void visit(IndexLocationVisitor& visitor) const override { NOTIMP; } - bool get( const Key &key, const Key &remapKey, Field &field ) const override; - void add( const Key &key, const Field &field ) override; + bool get(const Key& key, const Key& remapKey, Field& field) const override; + void add(const Key& key, const Field& field) override; #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH void flush() override; #else @@ -71,15 +80,17 @@ class RadosIndex : public IndexBase { void encode(eckit::Stream& s, const int version) const override { NOTIMP; } void entries(EntryVisitor& visitor) const override; - void print( std::ostream &out ) const override { NOTIMP; } - void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override { NOTIMP; } + void print(std::ostream& out) const override { NOTIMP; } + void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override { + NOTIMP; + } IndexStats statistics() const override { NOTIMP; } /// @note: reads complete axis info from DAOS. void updateAxes(); -private: // members +private: // members fdb5::RadosIndexLocation location_; @@ -90,9 +101,8 @@ class RadosIndex : public IndexBase { eckit::RadosKeyValue idx_kv_; std::map axis_kvs_; #endif - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 From fd63eb210c0f186cea30b9254b2571d53c409d02 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:52:51 +0200 Subject: [PATCH 038/109] fix(rados): field location --- src/fdb5/rados/RadosLazyFieldLocation.cc | 28 +++++++++++++++++------- src/fdb5/rados/RadosLazyFieldLocation.h | 22 +++++++++---------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/fdb5/rados/RadosLazyFieldLocation.cc b/src/fdb5/rados/RadosLazyFieldLocation.cc index fc3e2b509..59343f863 100644 --- a/src/fdb5/rados/RadosLazyFieldLocation.cc +++ b/src/fdb5/rados/RadosLazyFieldLocation.cc @@ -8,9 +8,21 @@ * does it submit to any jurisdiction. */ +#include "fdb5/rados/RadosLazyFieldLocation.h" + +#include "eckit/filesystem/PathName.h" +#include "eckit/io/rados/RadosKeyValue.h" #include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/Reanimator.h" -#include "fdb5/rados/RadosLazyFieldLocation.h" +#include "fdb5/database/FieldLocation.h" + +#include +#include +#include +#include +#include +#include namespace fdb5 { @@ -22,17 +34,16 @@ RadosLazyFieldLocation::RadosLazyFieldLocation(const fdb5::RadosLazyFieldLocatio RadosLazyFieldLocation::RadosLazyFieldLocation(const eckit::RadosKeyValue& index, const std::string& key) : FieldLocation(), index_(index), key_(key) {} -std::shared_ptr RadosLazyFieldLocation::make_shared() const { +std::shared_ptr RadosLazyFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); } eckit::DataHandle* RadosLazyFieldLocation::dataHandle() const { return realise()->dataHandle(); - } -void RadosLazyFieldLocation::print(std::ostream &out) const { +void RadosLazyFieldLocation::print(std::ostream& out) const { out << *realise(); } @@ -40,13 +51,15 @@ void RadosLazyFieldLocation::visit(FieldLocationVisitor& visitor) const { realise()->visit(visitor); } -std::shared_ptr RadosLazyFieldLocation::stableLocation() const { +std::shared_ptr RadosLazyFieldLocation::stableLocation() const { return realise()->make_shared(); } std::unique_ptr& RadosLazyFieldLocation::realise() const { - if (fl_) return fl_; + if (fl_) { + return fl_; + } /// @note: performed RPCs: /// - index kv get (daos_kv_get) @@ -60,7 +73,6 @@ std::unique_ptr& RadosLazyFieldLocation::realise() const { fl_.reset(eckit::Reanimator::reanimate(ms)); return fl_; - } -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosLazyFieldLocation.h b/src/fdb5/rados/RadosLazyFieldLocation.h index 3b18058df..3bf75e1d3 100644 --- a/src/fdb5/rados/RadosLazyFieldLocation.h +++ b/src/fdb5/rados/RadosLazyFieldLocation.h @@ -13,6 +13,7 @@ #pragma once +#include #include "fdb5/database/FieldLocation.h" #include "eckit/io/rados/RadosKeyValue.h" @@ -21,13 +22,13 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// @note: used in fdb-list index visiting, in DaosIndex::entries. During -/// visitation, DaosFieldLocations are built, which normally require +/// @note: used in fdb-list index visiting, in DaosIndex::entries. During +/// visitation, DaosFieldLocations are built, which normally require /// retrieving the location information from DAOS, inflicting RPCs. -/// This DaosLazyFieldLocation, instead, remains empty and the actual +/// This DaosLazyFieldLocation, instead, remains empty and the actual /// information is only be retrieved from DAOS when stableLocation() /// is called. This allows the visiting mechanism to discard unmatching -/// FieldLocations before any RPC is performed for them. +/// FieldLocations before any RPC is performed for them. class RadosLazyFieldLocation : public FieldLocation { public: @@ -36,27 +37,26 @@ class RadosLazyFieldLocation : public FieldLocation { eckit::DataHandle* dataHandle() const override; - virtual std::shared_ptr make_shared() const override; + virtual std::shared_ptr make_shared() const override; virtual void visit(FieldLocationVisitor& visitor) const override; - virtual std::shared_ptr stableLocation() const override; + virtual std::shared_ptr stableLocation() const override; -private: // methods +private: // methods std::unique_ptr& realise() const; - void print(std::ostream &out) const override; + void print(std::ostream& out) const override; -private: // members +private: // members eckit::RadosKeyValue index_; std::string key_; mutable std::unique_ptr fl_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 From d1548a7e67afc7040c9de4f51d544cb505aac2bc Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:52:57 +0200 Subject: [PATCH 039/109] fix(rados): store --- src/fdb5/rados/RadosStore.cc | 479 ++++++++++++++--------------------- src/fdb5/rados/RadosStore.h | 25 +- 2 files changed, 206 insertions(+), 298 deletions(-) diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index b7e615058..7d0dc0371 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -8,15 +8,15 @@ * does it submit to any jurisdiction. */ -#include "eckit/log/Bytes.h" -#include "eckit/log/Timer.h" +#include "fdb5/rados/RadosStore.h" #include "eckit/config/Resource.h" #include "eckit/io/EmptyHandle.h" -#include "eckit/io/rados/RadosWriteHandle.h" -// #include - +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosPool.h" +#include "eckit/log/Bytes.h" #include "eckit/log/TimeStamp.h" +#include "eckit/log/Timer.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" @@ -25,28 +25,10 @@ #include "fdb5/LibFdb5.h" #include "fdb5/database/FieldLocation.h" -#include "fdb5/io/FDBFileHandle.h" -// // #include "eckit/config/Resource.h" - -#include "eckit/io/rados/RadosNamespace.h" -#include "eckit/io/rados/RadosPool.h" - #include "fdb5/rados/RadosFieldLocation.h" -#include "fdb5/rados/RadosStore.h" #include "fdb5/rules/Rule.h" -using namespace eckit; - -// #include "eckit/log/Timer.h" -// #include "eckit/log/Bytes.h" - -// #include "eckit/io/EmptyHandle.h" -// #include "eckit/io/rados/RadosMultiObjWriteHandle.h" - -// #include "fdb5/LibFdb5.h" -// #include "fdb5/rules/Rule.h" -// #include "fdb5/database/FieldLocation.h" -// #include "fdb5/io/FDBFileHandle.h" +#include namespace fdb5 { @@ -54,8 +36,16 @@ namespace fdb5 { static StoreBuilder builder("rados"); -RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : - Store(schema), RadosCommon(config, "store", key), config_(config) { +RadosStore::RadosStore(const Key& key, const Config& config) : + Store(), RadosCommon(config, "store", key), config_(config), archivedFields_(0) { + + parseConfig(config_); +} + +RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : RadosStore(key, config) {} + +RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : + Store(), RadosCommon(config, "store", uri), config_(config), archivedFields_(0) { parseConfig(config_); } @@ -73,10 +63,25 @@ eckit::URI RadosStore::uri() const { #endif } +eckit::URI RadosStore::uri(const eckit::URI& dataURI) { + + eckit::RadosObject o{dataURI}; + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + return o.nspace().uri(); + +#else + + return o.nspace().pool().uri(); + +#endif +} + bool RadosStore::uriBelongs(const eckit::URI& uri) const { const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); + const auto n = parts.size(); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL @@ -96,7 +101,7 @@ bool RadosStore::uriExists(const eckit::URI& uri) const { /// @todo: revisit the name of this method const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); + const auto n = parts.size(); ASSERT(uri.scheme() == type()); @@ -127,9 +132,9 @@ bool RadosStore::uriExists(const eckit::URI& uri) const { return eckit::RadosObject(uri).exists(); } -std::vector RadosStore::collocatedDataURIs() const { +std::set RadosStore::collocatedDataURIs() const { - std::vector store_unit_uris; + std::set store_unit_uris; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL @@ -155,13 +160,13 @@ std::vector RadosStore::collocatedDataURIs() const { } #endif - store_unit_uris.push_back(obj.uri()); + store_unit_uris.insert(obj.uri()); } return store_unit_uris; } -std::set RadosStore::asCollocatedDataURIs(const std::vector& uris) const { +std::set RadosStore::asCollocatedDataURIs(const std::set& uris) const { std::set res; @@ -175,11 +180,6 @@ std::set RadosStore::asCollocatedDataURIs(const std::vector RadosStore::archive(const uint32_t, const Key& key, const void* data, - eckit::Length length) { +std::unique_ptr RadosStore::archive(const Key& key, const void* data, eckit::Length length) { + archivedFields_++; #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD @@ -238,8 +234,7 @@ std::unique_ptr RadosStore::archive(const uint32_t, const K h->write(data, length); - - return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key(nullptr, true))); + return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key{})); #else @@ -262,18 +257,17 @@ std::unique_ptr RadosStore::archive(const uint32_t, const K eckit::Offset offset{h.position()}; - long len = dh.write(data, length); long len = h.write(data, length); ASSERT(len == length); - return std::unique_ptr( - new RadosFieldLocation(o.uri(), offset, length, fdb5::Key(nullptr, true))); + return std::unique_ptr(new RadosFieldLocation(o.uri(), offset, length, fdb5::Key{})); #endif } -void RadosStore::flush() { +size_t RadosStore::flush() { + if (archivedFields_ == 0) { return 0; } @@ -302,14 +296,14 @@ void RadosStore::flush() { #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH flushDataHandles(); #else -// NOOP + // NOOP #endif #endif #endif - size_t out = archivedFields_; + size_t out = archivedFields_; archivedFields_ = 0; return out; } @@ -334,155 +328,155 @@ void RadosStore::close() { } void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { - ASSERT(uri.scheme() == type()); - eckit::PathName path = uri.path(); - if (path.isDir()) { - logVerbose << "rmdir: "; - logAlways << path << std::endl; - if (doit) { - path.rmdir(false); - } - } - else { - logVerbose << "Unlinking: "; - logAlways << path << std::endl; - if (doit) { - path.unlink(false); - } + ASSERT(uri.scheme() == type()); - const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + const auto n = parts.size(); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - ASSERT(n == 2 || n == 3); + ASSERT(n == 2 || n == 3); - ASSERT(parts[0] == pool_); - ASSERT(parts[1] == db_namespace_); + ASSERT(parts[0] == pool_); + ASSERT(parts[1] == db_namespace_); - if (n == 2) { // namespace + if (n == 2) { // namespace - eckit::RadosNamespace ns{uri}; + eckit::RadosNamespace ns{uri}; - logVerbose << "destroy Rados namespace: " << ns.str() << std::endl; + logVerbose << "destroy Rados namespace: "; + logAlways << ns.str() << std::endl; - if (doit) { - ns.destroy(); /// @todo: ensureDestroyed? - } + if (doit) { + ns.destroy(); /// @todo: ensureDestroyed? } - else { // object + } + else { // object - eckit::RadosObject obj{uri}; + eckit::RadosObject obj{uri}; - logVerbose << "destroy Rados object: " << obj.str() << std::endl; + logVerbose << "destroy Rados object: "; + logAlways << obj.str() << std::endl; #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - if (doit) { - obj.ensureAllDestroyed(); - } + if (doit) { + obj.ensureAllDestroyed(); + } #else - if (doit) { - obj.ensureDestroyed(); - } -#endif + if (doit) { + obj.ensureDestroyed(); } +#endif + } #else - ASSERT(n == 1 || n == 3); + ASSERT(n == 1 || n == 3); - ASSERT(parts[0] == db_pool_); + ASSERT(parts[0] == db_pool_); - if (n == 1) { // pool + if (n == 1) { // pool - eckit::RadosPool pool{uri}; + eckit::RadosPool pool{uri}; - logVerbose << "destroy Rados pool: " << pool.name() << std::endl; + logVerbose << "destroy Rados pool: "; + logAlways << pool.name() << std::endl; - if (doit) { - pool.ensureDestroyed(); - } + if (doit) { + pool.ensureDestroyed(); } - else { // object + } + else { // object - ASSERT(parts[1] == "default"); + ASSERT(parts[1] == namespace_); - eckit::RadosObject obj{uri}; + eckit::RadosObject obj{uri}; - logVerbose << "destroy Rados object: " << obj.str() << std::endl; + logVerbose << "destroy Rados object: "; + logAlways << obj.str() << std::endl; #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - if (doit) { - obj.ensureAllDestroyed(); - } + if (doit) { + obj.ensureAllDestroyed(); + } #else - if (doit) { - obj.ensureDestroyed(); - } -#endif + if (doit) { + obj.ensureDestroyed(); } - #endif } - void RadosStore::print(std::ostream & out) const { +#endif +} + +void RadosStore::print(std::ostream& out) const { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - out << "RadosStore(" << pool_ << "/" << db_namespace_ << ")"; + out << "RadosStore(" << pool_ << "/" << db_namespace_ << ")"; #else - out << "RadosStore(" << db_pool_ << "/" << namespace_ << ")"; + out << "RadosStore(" << db_pool_ << "/" << namespace_ << ")"; #endif - } +} - /// @note: unique name generation copied from LocalPathName::unique. - static eckit::StaticMutex local_mutex; +//---------------------------------------------------------------------------------------------------------------------- - eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { +/// Wipe-related methods are not implemented for the Rados backend. - eckit::AutoLock lock(local_mutex); +void RadosStore::finaliseWipeState(StoreWipeState&, bool, bool) { + NOTIMP; +} - std::string hostname = eckit::Main::hostname(); +bool RadosStore::doWipeUnknowns(const std::set&) const { + NOTIMP; +} - static unsigned long long n = (((unsigned long long)::getpid()) << 32); +bool RadosStore::doWipeURIs(const StoreWipeState&) const { + NOTIMP; +} - static std::string format = "%Y%m%d.%H%M%S"; - std::ostringstream os; - os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; +void RadosStore::doWipeEmptyDatabase() const { + NOTIMP; +} - std::string name = os.str(); +bool RadosStore::doUnsafeFullWipe() const { + NOTIMP; +} - while (::access(name.c_str(), F_OK) == 0) { - std::ostringstream os; - os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; - name = os.str(); - } - } +//---------------------------------------------------------------------------------------------------------------------- - eckit::DataHandle* RadosStore::getCachedHandle(const eckit::PathName& path) const { - HandleStore::const_iterator j = handles_.find(path); - if (j != handles_.end()) { - return j->second; - } - else { - return nullptr; - } +/// @note: unique name generation copied from LocalPathName::unique. +static eckit::StaticMutex local_mutex; + +eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { + + eckit::AutoLock lock(local_mutex); + + std::string hostname = eckit::Main::hostname(); - eckit::MD5 md5(name); + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + + std::string name = os.str(); + + eckit::MD5 md5(name); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - return eckit::RadosObject{pool_, db_namespace_, md5.digest()}; + return eckit::RadosObject{pool_, db_namespace_, md5.digest()}; #else - return eckit::RadosObject{pool_, db_namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; + return eckit::RadosObject{pool_, db_namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; #endif @@ -490,212 +484,113 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - return eckit::RadosObject{db_pool_, namespace_, md5.digest()}; + return eckit::RadosObject{db_pool_, namespace_, md5.digest()}; #else - return eckit::RadosObject{db_pool_, namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; + return eckit::RadosObject{db_pool_, namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; #endif #endif - } +} #ifndef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - const eckit::RadosObject& RadosStore::getDataObject(const Key& key) const { - - ObjectStore::const_iterator j = dataObjects_.find(key); +const eckit::RadosObject& RadosStore::getDataObject(const Key& key) const { - if (j != dataObjects_.end()) { - return j->second; - } + ObjectStore::const_iterator j = dataObjects_.find(key); - // eckit::RadosObject dataObject = generateDataObject(key); + if (j != dataObjects_.end()) { + return j->second; + } - dataObjects_.insert(std::pair(key, generateDataObject(key))); + dataObjects_.insert(std::pair(key, generateDataObject(key))); - return dataObjects_.find(key)->second; - } + return dataObjects_.find(key)->second; +} - eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosObject& name) { +eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosObject& name) { - HandleStore::const_iterator j = handles_.find(key); - if (j != handles_.end()) { - return *(j->second); - } + HandleStore::const_iterator j = handles_.find(key); + if (j != handles_.end()) { + return *(j->second); + } #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle* dh = name.asyncMultipartWriteHandle(maxPartSize_, maxAioBuffSize_, maxPartHandleBuffSize_); + eckit::DataHandle* dh = name.asyncMultipartWriteHandle(maxPartSize_, maxAioBuffSize_, maxPartHandleBuffSize_); #else - eckit::DataHandle* dh = name.multipartWriteHandle(maxPartSize_); + eckit::DataHandle* dh = name.multipartWriteHandle(maxPartSize_); #endif #else #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle* dh = name.asyncDataHandle(maxAioBuffSize_); + eckit::DataHandle* dh = name.asyncDataHandle(maxAioBuffSize_); #else - eckit::DataHandle* dh = name.dataHandle(); + eckit::DataHandle* dh = name.dataHandle(); #endif #endif - ASSERT(dh); - - handles_[key] = dh; - - dh->openForWrite(0); - - return *dh; - } - - void RadosStore::closeDataHandles() { - for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle* dh = j->second; - - for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle* dh = j->second; - dh->close(); - delete dh; - } - - handles_.clear(); - } - - eckit::DataHandle* RadosStore::createFileHandle(const eckit::PathName& path) { - - // static size_t sizeBuffer = eckit::Resource("fdbBufferSize", 64 * 1024 * 1024); + ASSERT(dh); - LOG_DEBUG_LIB(LibFdb5) << "Creating RadosWriteHandle to " - << path - // << " with buffer of " << eckit::Bytes(sizeBuffer) - << std::endl; + handles_[key] = dh; - return new RadosWriteHandle(path, 0); - } - - eckit::DataHandle* RadosStore::createAsyncHandle(const eckit::PathName& path) { - NOTIMP; - - /* static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); - static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); - - return new eckit::AIOHandle(path, nbBuffers, sizeBuffer);*/ - } - - eckit::DataHandle* RadosStore::createDataHandle(const eckit::PathName& path) { - - static bool fdbWriteToNull = eckit::Resource("fdbWriteToNull;$FDB_WRITE_TO_NULL", false); - if (fdbWriteToNull) { - return new eckit::EmptyHandle(); - } - - static bool fdbAsyncWrite = eckit::Resource("fdbAsyncWrite;$FDB_ASYNC_WRITE", false); - if (fdbAsyncWrite) { - return createAsyncHandle(path); - } - - return createFileHandle(path); - } - - eckit::DataHandle& RadosStore::getDataHandle(const eckit::PathName& path) { - eckit::DataHandle* dh = getCachedHandle(path); - if (!dh) { - dh = createDataHandle(path); - ASSERT(dh); - handles_[path] = dh; - dh->openForWrite(0); - } - return *dh; - } + dh->openForWrite(0); - eckit::PathName RadosStore::generateDataPath(const Key& key) const { - - eckit::PathName dpath(directory_); - dpath /= key.valuesToString(); - dpath = eckit::PathName::unique(dpath) + ".data"; - return dpath; - } - - eckit::PathName RadosStore::getDataPath(const Key& key) { - PathStore::const_iterator j = dataPaths_.find(key); - if (j != dataPaths_.end()) { - return j->second; - } - - eckit::PathName dataPath = generateDataPath(key); + return *dh; +} - dataPaths_[key] = dataPath; +void RadosStore::closeDataHandles() { - return dataPath; - dataObjects_.clear(); - } + for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { + eckit::DataHandle* dh = j->second; + dh->close(); + delete dh; + } - void RadosStore::flushDataHandles() { + handles_.clear(); + dataObjects_.clear(); +} - for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle* dh = j->second; - dh->flush(); - } - } +void RadosStore::flushDataHandles() { - void RadosStore::print(std::ostream & out) const { - out << "RadosStore(" << directory_ << ")"; - } + for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { + eckit::DataHandle* dh = j->second; + dh->flush(); + } +} - static StoreBuilder builder("rados"); #endif - // eckit::DataHandle *RadosStore::createAsyncHandle(const eckit::PathName &path) { - // NOTIMP; - - // /* static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); - // static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); - - // return new eckit::AIOHandle(path, nbBuffers, sizeBuffer);*/ - // } - - // eckit::DataHandle *RadosStore::createDataHandle(const eckit::PathName &path) { +void RadosStore::parseConfig(const fdb5::Config& config) { - // static bool fdbWriteToNull = eckit::Resource("fdbWriteToNull;$FDB_WRITE_TO_NULL", false); - // if(fdbWriteToNull) - // return new eckit::EmptyHandle(); + eckit::LocalConfiguration rados{}, store_conf{}; - // static bool fdbAsyncWrite = eckit::Resource("fdbAsyncWrite;$FDB_ASYNC_WRITE", false); - // if(fdbAsyncWrite) - // return createAsyncHandle(path); - - // return new RadosMultiObjWriteHandle(path, 0); - // } - - void RadosStore::parseConfig(const fdb5::Config& config) { - - eckit::LocalConfiguration rados{}, store_conf{}; - - if (config.has("rados")) { - rados = config.getSubConfiguration("rados"); - if (rados.has("store")) { - store_conf = rados.getSubConfiguration("store"); - } - } + if (config.has("rados")) { + rados = config.getSubConfiguration("rados"); + if (rados.has("store")) { + store_conf = rados.getSubConfiguration("store"); + } + } #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) && defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - maxHandleBuffSize_ = store_conf.getInt("maxHandleBuffSize", 1024 * 1024); + maxHandleBuffSize_ = store_conf.getInt("maxHandleBuffSize", 1024 * 1024); #endif #if (!defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD)) && defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024); - maxPartHandleBuffSize_ = store_conf.getInt("maxPartHandleBuffSize", 1024); + maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024); + maxPartHandleBuffSize_ = store_conf.getInt("maxPartHandleBuffSize", 1024); #else - maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024 * 1024); + maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024 * 1024); #endif #endif - } +} - //---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- - } // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 27ba83d26..db57a69c1 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -31,16 +31,18 @@ class RadosStore : public Store, public RadosCommon { public: // methods + RadosStore(const Key& key, const Config& config); RadosStore(const Schema& schema, const Key& key, const Config& config); - RadosStore(const eckit::URI& uri); + RadosStore(const eckit::URI& uri, const Config& config); ~RadosStore() override {} eckit::URI uri() const override; + static eckit::URI uri(const eckit::URI& dataURI); bool uriBelongs(const eckit::URI&) const override; bool uriExists(const eckit::URI&) const override; - std::vector collocatedDataURIs() const override; - std::set asCollocatedDataURIs(const std::vector&) const override; + std::set collocatedDataURIs() const override; + std::set asCollocatedDataURIs(const std::set&) const override; bool open() override { return true; } size_t flush() override; @@ -48,14 +50,25 @@ class RadosStore : public Store, public RadosCommon { void checkUID() const override { /* nothing to do */ } + /// Wipe-related methods (not implemented for the Rados backend) + void finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) override; + bool doWipeUnknowns(const std::set& unknownURIs) const override; + bool doWipeURIs(const StoreWipeState& wipeState) const override; + void doWipeEmptyDatabase() const override; + bool doUnsafeFullWipe() const override; + + // Rados store does not currently support auxiliary objects + std::vector getAuxiliaryURIs(const eckit::URI&, bool onlyExisting = false) const override { + return {}; + } + protected: // methods std::string type() const override { return "rados"; } bool exists() const override; eckit::DataHandle* retrieve(Field& field) const override; - std::unique_ptr archive(const uint32_t, const Key& key, const void* data, - eckit::Length length) override; + std::unique_ptr archive(const Key& key, const void* data, eckit::Length length) override; using Store::remove; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; @@ -83,7 +96,7 @@ class RadosStore : public Store, public RadosCommon { const Config& config_; // mutable bool dirty_; - size_t archivedFields_; + size_t archivedFields_{0}; #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH From 17b9f4c60ef19e8d55598b41a2d7e5f209abcde9 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 08:53:03 +0200 Subject: [PATCH 040/109] fix(rados): tests --- tests/fdb/rados/CMakeLists.txt | 19 +- tests/fdb/rados/test_rados_catalogue.cc | 336 +++++++++++++----------- tests/fdb/rados/test_rados_store.cc | 47 ++-- 3 files changed, 217 insertions(+), 185 deletions(-) diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt index 2dfa061d4..87abd3718 100644 --- a/tests/fdb/rados/CMakeLists.txt +++ b/tests/fdb/rados/CMakeLists.txt @@ -7,6 +7,23 @@ if (HAVE_RADOSFDB) list( APPEND unit_test_libraries fdb5 ) + # The Rados unit tests need a pool to run against (with RADOS_ADMIN=OFF the pool + # must already exist, e.g. one created in the Ceph service). The pool name can be + # provided at configure time via -DFDB_RADOS_TEST_POOL=, or inherited from + # the FDB_RADOS_TEST_POOL environment variable (e.g. exported in the dev container + # that talks to the Ceph service). + if( NOT FDB_RADOS_TEST_POOL AND DEFINED ENV{FDB_RADOS_TEST_POOL} ) + set( FDB_RADOS_TEST_POOL "$ENV{FDB_RADOS_TEST_POOL}" ) + endif() + + # Only inject the variable into the test environment when we have a value: + # passing an empty "FDB_RADOS_TEST_POOL=" would clobber any value already present + # in the runtime environment when the test executes. + unset( _rados_test_environment ) + if( FDB_RADOS_TEST_POOL ) + set( _rados_test_environment ENVIRONMENT FDB_RADOS_TEST_POOL=${FDB_RADOS_TEST_POOL} ) + endif() + foreach( _test ${rados_tests} ) ecbuild_add_test( TARGET fdb_test_${_test} @@ -14,7 +31,7 @@ if (HAVE_RADOSFDB) LABELS rados LIBS "${unit_test_libraries}" INCLUDES "${unit_test_include_dirs}" - ENVIRONMENT FDB_RADOS_TEST_POOL=${FDB_RADOS_TEST_POOL} ) + ${_rados_test_environment} ) endforeach() diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 3b732ad5f..7d5352399 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -18,8 +18,8 @@ #include "eckit/filesystem/TmpFile.h" // #include "eckit/filesystem/TmpDir.h" // #include "eckit/io/FileHandle.h" -#include "eckit/io/MemoryHandle.h" #include "eckit/config/YAMLConfiguration.h" +#include "eckit/io/MemoryHandle.h" #include "eckit/io/rados/RadosPartHandle.h" // #include "metkit/mars/MarsRequest.h" @@ -28,63 +28,62 @@ // #include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" - #include "fdb5/toc/TocStore.h" // #include "fdb5/daos/DaosSession.h" // #include "fdb5/daos/DaosPool.h" // #include "fdb5/daos/DaosArrayPartHandle.h" -#include "fdb5/rados/RadosStore.h" -#include "fdb5/rados/RadosFieldLocation.h" -#include "fdb5/rados/RadosCatalogueWriter.h" #include "fdb5/rados/RadosCatalogueReader.h" +#include "fdb5/rados/RadosCatalogueWriter.h" +#include "fdb5/rados/RadosFieldLocation.h" +#include "fdb5/rados/RadosStore.h" using namespace eckit::testing; using namespace eckit; namespace { - void deldir(eckit::PathName& p) { - if (!p.exists()) { - return; - } +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } - std::vector files; - std::vector dirs; - p.children(files, dirs); + std::vector files; + std::vector dirs; + p.children(files, dirs); - for (auto& f : files) { - f.unlink(); - } - for (auto& d : dirs) { - deldir(d); - } + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } - p.rmdir(); - }; + p.rmdir(); +}; - void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { - ASSERT(prefix.length() > 3); - for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { - if (name.rfind(prefix, 0) == 0) { - eckit::RadosNamespace{pool, name}.destroy(); - } +void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); } } +} #ifdef fdb5_HAVE_RADOS_ADMIN - void ensureClean(const std::string& prefix) { - ASSERT(prefix.length() > 3); - for (const std::string& name : eckit::RadosCluster::instance().listPools()) { - if (name.rfind(prefix, 0) == 0) { - eckit::RadosPool{name}.destroy(); - } +void ensureClean(const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listPools()) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosPool{name}.destroy(); } } +} #endif -} +} // namespace // temporary schema,spaces,root files common to all DAOS Catalogue tests @@ -106,7 +105,7 @@ eckit::PathName& catalogue_tests_tmp_root() { namespace fdb { namespace test { -CASE( "Setup" ) { +CASE("Setup") { #if !defined(fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL) && !defined(fdb5_HAVE_RADOS_ADMIN) throw eckit::Exception( @@ -114,9 +113,10 @@ CASE( "Setup" ) { "RADOS_BACKENDS_SINGLE_POOL=OFF, and require enabling RADOS_ADMIN=ON."); #endif - // ensure fdb root directory exists. If not, then that root is + // ensure fdb root directory exists. If not, then that root is // registered as non existing and Catalogue/Store tests fail. - if (catalogue_tests_tmp_root().exists()) deldir(catalogue_tests_tmp_root()); + if (catalogue_tests_tmp_root().exists()) + deldir(catalogue_tests_tmp_root()); catalogue_tests_tmp_root().mkdir(); ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); @@ -144,25 +144,22 @@ CASE( "Setup" ) { // LibFdb5::instance().defaultConfig().schema() is called // due to no specified schema file (e.g. in Key::registry()) ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); - } CASE("RadosCatalogue tests") { std::string test_id = "test-catalogue"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - #ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer - #else +#else std::string pool; - pool = eckit::Resource( - "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool - ); + pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool); EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); - #endif +#endif #else std::string prefix = test_id; ensureClean(prefix); @@ -174,71 +171,85 @@ CASE("RadosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "rados:\n" " catalogue:\n" - " pool: " + pool + "\n" - " root_namespace: " + test_id + "_root\n" - " namespace_prefix: " + test_id + "\n" - }; + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; #else std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "rados:\n" " catalogue:\n" " namespace: default\n" - " root_pool: " + prefix + "_root\n" - " pool_prefix: " + prefix + "\n" - }; + " root_pool: " + + prefix + + "_root\n" + " pool_prefix: " + + prefix + "\n"}; #endif fdb5::Config config{YAMLConfiguration(config_str)}; fdb5::Schema schema{schema_file()}; - /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using a=1,b=2 + /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using + /// a=1,b=2 fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); - fdb5::Key db_key({{"a", "11"}, {"b", "22"}}, schema.registry()); - fdb5::Key index_key({{"c", "3"}, {"d", "4"}}, schema.registry()); - fdb5::Key field_key({{"e", "5"}, {"f", "6"}}, schema.registry()); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); // archive std::unique_ptr loc(new fdb5::RadosFieldLocation( - eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key(nullptr, true) - )); + eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key{})); { fdb5::RadosCatalogueWriter dcatw{db_key, config}; - // fdb5::DaosName db_cont{pool_name, db_key.valuesToString()}; - // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config - // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; - // EXPECT(db_cont.exists()); - // EXPECT(cat_kv.exists()); + // fdb5::DaosName db_cont{pool_name, db_key.valuesToString()}; + // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + // EXPECT(db_cont.exists()); + // EXPECT(cat_kv.exists()); fdb5::Catalogue& cat = dcatw; cat.selectIndex(index_key); - // fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from config - // fdb5::DaosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; - // EXPECT(index_kv.exists()); - // EXPECT(cat_kv.has(index_key.valuesToString())); + // fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from + // config fdb5::DaosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; + // EXPECT(index_kv.exists()); + // EXPECT(cat_kv.has(index_key.valuesToString())); fdb5::CatalogueWriter& catw = dcatw; - catw.archive(field_key, std::move(loc)); - cat.flush(); - // EXPECT(index_kv.has(field_key.valuesToString())); - // fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; - // fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; - // EXPECT(e_axis_kv.exists()); - // EXPECT(e_axis_kv.has("5")); - // fdb5::DaosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; - // fdb5::DaosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; - // EXPECT(f_axis_kv.exists()); - // EXPECT(f_axis_kv.has("6")); + catw.archive(index_key, field_key, std::move(loc)); + cat.flush(0); + // EXPECT(index_kv.has(field_key.valuesToString())); + // fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; + // fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; + // EXPECT(e_axis_kv.exists()); + // EXPECT(e_axis_kv.has("5")); + // fdb5::DaosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; + // fdb5::DaosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; + // EXPECT(f_axis_kv.exists()); + // EXPECT(f_axis_kv.has("6")); } // retrieve @@ -271,7 +282,6 @@ CASE("RadosCatalogue tests") { // EXPECT_NOT(cat_kv.exists()); // EXPECT_NOT(db_cont.exists()); // } - } SECTION("RadosCatalogue archive (index) and retrieve with a RadosStore") { @@ -282,24 +292,38 @@ CASE("RadosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "rados:\n" - " pool: " + pool + "\n" - " root_namespace: " + test_id + "_root\n" - " namespace_prefix: " + test_id + "\n" - }; + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; #else std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "rados:\n" " namespace: default\n" - " root_pool: " + prefix + "_root\n" - " pool_prefix: " + prefix + "\n" - }; + " root_pool: " + + prefix + + "_root\n" + " pool_prefix: " + + prefix + "\n"}; #endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -321,7 +345,7 @@ CASE("RadosCatalogue tests") { fdb5::RadosStore rstore{schema, db_key, config}; fdb5::Store& store = static_cast(rstore); - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); // index data @@ -331,7 +355,7 @@ CASE("RadosCatalogue tests") { cat.deselectIndex(); cat.selectIndex(index_key); fdb5::CatalogueWriter& catw = rcatw; - catw.archive(field_key, std::move(loc)); + catw.archive(index_key, field_key, std::move(loc)); /// flush store before flushing catalogue rstore.flush(); // not necessary if using a DAOS store @@ -353,23 +377,22 @@ CASE("RadosCatalogue tests") { std::unique_ptr dh(store.retrieve(field)); EXPECT(dynamic_cast(dh.get())); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - // // deindex data - - // { - // fdb5::DaosCatalogueWriter dcat{db_key, config}; - // fdb5::Catalogue& cat = static_cast(dcat); - // std::ostream out(std::cout.rdbuf()); - // metkit::mars::MarsRequest r = db_key.request("retrieve"); - // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); - // cat.visitEntries(*wv, store, false); - // } + // // deindex data + // { + // fdb5::DaosCatalogueWriter dcat{db_key, config}; + // fdb5::Catalogue& cat = static_cast(dcat); + // std::ostream out(std::cout.rdbuf()); + // metkit::mars::MarsRequest r = db_key.request("retrieve"); + // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + // cat.visitEntries(*wv, store, false); + // } } // SECTION("DaosCatalogue archive (index) and retrieve with a TocStore") { @@ -441,7 +464,7 @@ CASE("RadosCatalogue tests") { // // retrieve data // std::unique_ptr dh(store.retrieve(field)); - + // std::vector test(dh->size()); // dh->openForRead(); // { @@ -485,22 +508,33 @@ CASE("RadosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" "type: local\n" - "schema : " + schema_file().path() + "\n" + "schema : " + + schema_file().path() + + "\n" "engine: rados\n" "store: rados\n" - "rados:\n" - }; + "rados:\n"}; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool: " + pool + "\n" - " root_namespace: " + test_id + "_root\n" - " namespace_prefix: " + test_id + "\n"; + config_str += " pool: " + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"; #else - config_str += " namespace: default\n" - " root_pool: " + prefix + "_root\n" - " pool_prefix: " + prefix + "\n"; + config_str += + " namespace: default\n" + " root_pool: " + + prefix + + "_root\n" + " pool_prefix: " + + prefix + "\n"; #endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -511,26 +545,10 @@ CASE("RadosCatalogue tests") { fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}}); - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest all_req{ - metkit::mars::MarsRequest{}, - true, - std::vector{} - }; + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest all_req{metkit::mars::MarsRequest{}, true, std::vector{}}; // initialise FDB @@ -549,7 +567,7 @@ CASE("RadosCatalogue tests") { count = 0; while (listObject.next(info)) { - info.print(std::cout, true, true); + info.print(std::cout, true, true, false, " "); std::cout << std::endl; ++count; } @@ -559,7 +577,8 @@ CASE("RadosCatalogue tests") { char data[] = "test"; - /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times below. + /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times + /// below. // Should this be avoided? fdb.archive(request_key, data, sizeof(data)); fdb.flush(); @@ -568,7 +587,7 @@ CASE("RadosCatalogue tests") { metkit::mars::MarsRequest r = request_key.request("retrieve"); std::unique_ptr dh(fdb.retrieve(r)); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -577,7 +596,7 @@ CASE("RadosCatalogue tests") { // list all listObject = fdb.list(all_req); - count = 0; + count = 0; while (listObject.next(info)) { // info.print(std::cout, true, true); // std::cout << std::endl; @@ -648,7 +667,7 @@ CASE("RadosCatalogue tests") { // /// @note: FDB holds a LocalFDB which holds an Archiver which holds open DBs (DaosCatalogueWriters). // /// If a whole DB is wiped, the top-level structures for that DB (main and catalogue KVs in this case) - // /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the + // /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the // /// top-level structures are only generated as part of the DaosCatalogueWriter constructor. There is // /// no way currently to destroy the open DBs held by FDB other than entirely destroying FDB. // /// Alternatively, a separate FDB instance can be created. @@ -666,7 +685,7 @@ CASE("RadosCatalogue tests") { // count++; // } // EXPECT(count == 1); - + // // wipe full database // wipeObject = fdb2.wipe(db_req, true); @@ -690,7 +709,6 @@ CASE("RadosCatalogue tests") { // /// @todo: ensure DB and corresponding pool do not exist // /// @todo: ensure new DaosSession has updated daos client config - } // SECTION("OPTIONAL SCHEMA KEYS") { @@ -727,28 +745,28 @@ CASE("RadosCatalogue tests") { // fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"d", "4"}}); // fdb5::FDBToolRequest full_req{ - // request_key.request("retrieve"), - // false, + // request_key.request("retrieve"), + // false, // std::vector{"a", "b"} // }; // fdb5::FDBToolRequest full_req2{ - // request_key2.request("retrieve"), - // false, + // request_key2.request("retrieve"), + // false, // std::vector{"a", "b"} // }; // fdb5::FDBToolRequest index_req{ - // index_key.request("retrieve"), - // false, + // index_key.request("retrieve"), + // false, // std::vector{"a", "b"} // }; // fdb5::FDBToolRequest db_req{ - // db_key.request("retrieve"), - // false, + // db_key.request("retrieve"), + // false, // std::vector{"a", "b"} // }; // fdb5::FDBToolRequest all_req{ - // metkit::mars::MarsRequest{}, - // true, + // metkit::mars::MarsRequest{}, + // true, // std::vector{} // }; @@ -850,21 +868,19 @@ CASE("RadosCatalogue tests") { // teardown rados #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - #ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_ADMIN eckit::RadosPool{pool}.ensureDestroyed(); - #else +#else ensureCleanNamespaces(pool, test_id); - #endif +#endif #else ensureClean(prefix); #endif - } } // namespace test } // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 58435807c..1fd92c3a9 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -27,7 +27,6 @@ // #include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" - #include "fdb5/toc/TocCatalogueReader.h" #include "fdb5/toc/TocCatalogueWriter.h" @@ -189,7 +188,7 @@ CASE("RadosStore tests") { fdb5::Schema schema{schema_file()}; fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); - fdb5::Key db_key({{"a", "1"}, {"b", "2"}}, schema.registry()); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); char data[] = "test"; @@ -198,7 +197,7 @@ CASE("RadosStore tests") { fdb5::RadosStore rados_store{schema, db_key, config}; fdb5::Store& store = rados_store; - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); rados_store.flush(); @@ -301,9 +300,9 @@ CASE("RadosStore tests") { // request fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); - fdb5::Key db_key({{"a", "1"}, {"b", "2"}}, schema.registry()); - fdb5::Key index_key({{"c", "3"}, {"d", "4"}}, schema.registry()); - fdb5::Key field_key({{"e", "5"}, {"f", "6"}}, schema.registry()); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); // store data @@ -311,7 +310,7 @@ CASE("RadosStore tests") { fdb5::RadosStore rados_store{schema, db_key, config}; fdb5::Store& store = static_cast(rados_store); - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); // index data @@ -375,13 +374,13 @@ CASE("RadosStore tests") { // deindex data - { - fdb5::TocCatalogueWriter tcat{db_key, config}; - fdb5::Catalogue& cat = static_cast(tcat); - metkit::mars::MarsRequest r = db_key.request("retrieve"); - std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); - cat.visitEntries(*wv, store, false); - } + // { + // fdb5::TocCatalogueWriter tcat{db_key, config}; + // fdb5::Catalogue& cat = static_cast(tcat); + // metkit::mars::MarsRequest r = db_key.request("retrieve"); + // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + // cat.visitEntries(*wv, store, false); + // } } SECTION("VIA FDB API") { @@ -481,7 +480,7 @@ CASE("RadosStore tests") { count = 0; while (listObject.next(info)) { - info.print(std::cout, true, true); + info.print(std::cout, true, true, false, " "); std::cout << std::endl; ++count; } @@ -537,7 +536,7 @@ CASE("RadosStore tests") { // dry run attempt to wipe with too specific request auto wipeObject = fdb.wipe(full_req); - count = 0; + count = 0; while (wipeObject.next(elem)) { count++; } @@ -545,7 +544,7 @@ CASE("RadosStore tests") { // dry run wipe index and store unit wipeObject = fdb.wipe(index_req); - count = 0; + count = 0; while (wipeObject.next(elem)) { count++; } @@ -553,7 +552,7 @@ CASE("RadosStore tests") { // dry run wipe database wipeObject = fdb.wipe(db_req); - count = 0; + count = 0; while (wipeObject.next(elem)) { count++; } @@ -561,7 +560,7 @@ CASE("RadosStore tests") { // ensure field still exists listObject = fdb.list(full_req); - count = 0; + count = 0; while (listObject.next(info)) { // info.print(std::cout, true, true); // std::cout << std::endl; @@ -571,7 +570,7 @@ CASE("RadosStore tests") { // attempt to wipe with too specific request wipeObject = fdb.wipe(full_req, true); - count = 0; + count = 0; while (wipeObject.next(elem)) { count++; } @@ -581,7 +580,7 @@ CASE("RadosStore tests") { // wipe index and store unit (and DB pool or namespace as there is only one index) wipeObject = fdb.wipe(index_req, true); - count = 0; + count = 0; while (wipeObject.next(elem)) { count++; } @@ -591,7 +590,7 @@ CASE("RadosStore tests") { // ensure field does not exist listObject = fdb.list(full_req); - count = 0; + count = 0; while (listObject.next(info)) { count++; } @@ -702,7 +701,7 @@ CASE("RadosStore tests") { fdb5::WipeElement elem; auto wipeObject = fdb.wipe(db_req, true); - count = 0; + count = 0; while (wipeObject.next(elem)) { count++; } @@ -714,7 +713,7 @@ CASE("RadosStore tests") { fdb5::ListElement info; auto listObject = fdb.list(full_req); - count = 0; + count = 0; while (listObject.next(info)) { // info.print(std::cout, true, true); // std::cout << std::endl; From 0aa9a98e1d11c5a4ab2d198c01fba322023e4a6c Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 16:00:00 +0200 Subject: [PATCH 041/109] fix(rados): index --- src/fdb5/rados/RadosIndex.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index b9092e56c..f54c61a69 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -65,7 +65,9 @@ class RadosIndex : public IndexBase { bool dirty() const override { NOTIMP; } void open() override { NOTIMP; }; - void close() override { NOTIMP; } + /// @note: the Rados KV index holds no open file/handle state, so closing is a no-op. + /// This must not throw: it is invoked during normal read/list flows via eckit::AutoCloser. + void close() override {} void reopen() override { NOTIMP; } void visit(IndexLocationVisitor& visitor) const override { NOTIMP; } From c3f24365bb6dd0b4add5a2bd93ae31a5a6e7aa93 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 16:00:04 +0200 Subject: [PATCH 042/109] fix(rados): store --- src/fdb5/rados/RadosStore.cc | 110 ++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 7d0dc0371..f5699ae22 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -25,6 +25,7 @@ #include "fdb5/LibFdb5.h" #include "fdb5/database/FieldLocation.h" +#include "fdb5/database/WipeState.h" #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rules/Rule.h" @@ -425,26 +426,117 @@ void RadosStore::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- -/// Wipe-related methods are not implemented for the Rados backend. +/// @note: for SINGLE_POOL the database maps to a Rados namespace, otherwise to a Rados pool. +/// Only the namespace/pool holding this database's objects is ever touched here. -void RadosStore::finaliseWipeState(StoreWipeState&, bool, bool) { - NOTIMP; +void RadosStore::finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) { + /// @note: doit and unsafeWipeAll do not affect the preparation of a Rados store wipe. + + const std::set& dataURIs = storeState.includedDataURIs(); // included according to cat + const std::set& safeURIs = storeState.safeURIs(); // excluded according to cat + + // Objects included by the catalogue may no longer exist (e.g. due to a prior incomplete wipe). + std::set nonExistingURIs; + for (const auto& uri : dataURIs) { + if (!eckit::RadosObject{uri}.exists()) { + nonExistingURIs.insert(uri); + } + } + for (const auto& uri : nonExistingURIs) { + storeState.markAsMissing(uri); + } + + const bool all = safeURIs.empty(); + if (!all) { + return; + } + + // Full wipe: scan the database namespace/pool for any objects unaccounted for by the catalogue. +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + eckit::RadosNamespace db{pool_, db_namespace_}; +#else + eckit::RadosNamespace db{db_pool_, namespace_}; +#endif + + if (!db.exists()) { + return; + } + + for (const auto& obj : db.listObjects()) { + +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + // Parts belong to a main object and are removed together with it. + if (obj.name().find(";part-") != std::string::npos) { + continue; + } +#endif + + const eckit::URI uri = obj.uri(); + if (dataURIs.find(uri) == dataURIs.end() && safeURIs.find(uri) == safeURIs.end()) { + storeState.insertUnrecognised(uri); + } + } } -bool RadosStore::doWipeUnknowns(const std::set&) const { - NOTIMP; +bool RadosStore::doWipeUnknowns(const std::set& unknownURIs) const { + for (const auto& uri : unknownURIs) { + if (eckit::RadosObject{uri}.exists()) { + remove(uri, std::cout, std::cout, true); + } + } + return true; } -bool RadosStore::doWipeURIs(const StoreWipeState&) const { - NOTIMP; +bool RadosStore::doWipeURIs(const StoreWipeState& wipeState) const { + const bool wipeAll = wipeState.safeURIs().empty(); + + for (const auto& uri : wipeState.includedDataURIs()) { + remove(uri, std::cout, std::cout, true); + } + + if (wipeAll) { + cleanupEmptyDatabase_ = true; + } + + return true; } void RadosStore::doWipeEmptyDatabase() const { - NOTIMP; + + if (!cleanupEmptyDatabase_) { + return; + } + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + eckit::RadosNamespace db{pool_, db_namespace_}; +#else + eckit::RadosNamespace db{db_pool_, namespace_}; +#endif + + if (db.exists()) { + remove(db.uri(), std::cout, std::cout, true); + } } bool RadosStore::doUnsafeFullWipe() const { - NOTIMP; + + /// @note: if the database namespace/pool also holds a catalogue, the wiping is skipped as the + /// catalogue is in charge. The presence of a "key" entry in the database key-value is used to + /// determine whether a catalogue exists here. + if (db_kv_ && (!db_kv_->exists() || !db_kv_->has("key"))) { + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + eckit::RadosNamespace db{pool_, db_namespace_}; +#else + eckit::RadosNamespace db{db_pool_, namespace_}; +#endif + + if (db.exists()) { + remove(db.uri(), std::cout, std::cout, true); + } + } + + return true; } //---------------------------------------------------------------------------------------------------------------------- From dd7b3102b3e01000295bf7d51aad9fc67bb2efe9 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 16:00:10 +0200 Subject: [PATCH 043/109] fix(rados): common --- src/fdb5/rados/RadosCommon.cc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 6c50d62f9..a7474503c 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -15,6 +15,7 @@ #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/utils/Tokenizer.h" #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" @@ -56,15 +57,18 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con RadosCommon::RadosCommon(const Config& config, const std::string& component, const eckit::URI& uri) { - /// @note: validity of input URI is not checked here because this constructor is only triggered - /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand + /// @note: this constructor is triggered both by DB::buildReader in EntryVisitMechanism (with a + /// catalogue key-value URI, i.e. pool/namespace/oid) and by StoreFactory during wipe (with a + /// store namespace URI, i.e. pool/namespace). Only the pool and namespace are needed here, so + /// parse them directly and accept both the 2-token and 3-token forms. - eckit::RadosKeyValue db_name{uri}; + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + ASSERT(parts.size() == 2 || parts.size() == 3); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - pool_ = db_name.nspace().pool().name(); - db_namespace_ = db_name.nspace().name(); + pool_ = parts[0]; + db_namespace_ = parts[1]; readConfig(config, component, false); @@ -73,8 +77,8 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con #else - db_pool_ = db_name.nspace().pool().name(); - namespace_ = db_name.nspace().name(); + db_pool_ = parts[0]; + namespace_ = parts[1]; readConfig(config, component, false); From b80969150b48c96a21f15f5a56f7978974650faf Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 16:00:14 +0200 Subject: [PATCH 044/109] fix(rados): tests --- tests/fdb/rados/test_rados_catalogue.cc | 20 ++++++ tests/fdb/rados/test_rados_store.cc | 86 +++++++++++++++---------- 2 files changed, 71 insertions(+), 35 deletions(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 7d5352399..129903123 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -20,6 +20,7 @@ // #include "eckit/io/FileHandle.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/io/MemoryHandle.h" +#include "eckit/io/PartHandle.h" #include "eckit/io/rados/RadosPartHandle.h" // #include "metkit/mars/MarsRequest.h" @@ -376,7 +377,13 @@ CASE("RadosCatalogue tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + /// @note: with multipart enabled, the field spans potentially several objects and is + /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. + EXPECT(dynamic_cast(dh.get())); +#else EXPECT(dynamic_cast(dh.get())); +#endif eckit::MemoryHandle mh; dh->copyTo(mh); @@ -503,6 +510,19 @@ CASE("RadosCatalogue tests") { SECTION("Via FDB API with a Rados catalogue and store") { + /// @note: earlier sections share the same catalogue namespaces/pool; reset them so this + /// section starts from a clean, empty catalogue (it asserts the FDB is initially empty). +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL +#ifdef eckit_HAVE_RADOS_ADMIN + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); +#else + ensureCleanNamespaces(pool, test_id); +#endif +#else + ensureClean(prefix); +#endif + // FDB configuration std::string config_str{ diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 1fd92c3a9..0a2ea0ca6 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -27,12 +27,14 @@ // #include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/WipeIterator.h" #include "fdb5/toc/TocCatalogueReader.h" #include "fdb5/toc/TocCatalogueWriter.h" // #include "eckit/io/s3/S3Client.h" // #include "eckit/io/s3/S3Session.h" // #include "eckit/io/s3/S3Credential.h" +#include "eckit/io/PartHandle.h" #include "eckit/io/rados/RadosPartHandle.h" #include "fdb5/rados/RadosFieldLocation.h" @@ -97,6 +99,23 @@ eckit::PathName& store_tests_tmp_root() { return sd; } +/// @note: counts only the URIs that would actually be deleted, filtering out purely +/// informational wipe elements (safe/info/error) so a too-specific request yields 0. +size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { + size_t count = 0; + fdb5::WipeElement elem; + while (wipeObject.next(elem)) { + if (print) { + std::cout << elem << std::endl; + } + if (elem.type() != fdb5::WipeElementType::ERROR && elem.type() != fdb5::WipeElementType::CATALOGUE_INFO && + elem.type() != fdb5::WipeElementType::CATALOGUE_SAFE && elem.type() != fdb5::WipeElementType::STORE_SAFE) { + count += elem.uris().size(); + } + } + return count; +} + namespace fdb { namespace test { @@ -205,8 +224,13 @@ CASE("RadosStore tests") { fdb5::Field field(std::move(loc), std::time(nullptr)); std::cout << "Read location: " << field.location() << std::endl; std::unique_ptr dh(store.retrieve(field)); +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + /// @note: with multipart enabled, the field spans potentially several objects and is + /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. + EXPECT(dynamic_cast(dh.get())); +#else EXPECT(dynamic_cast(dh.get())); - /// @todo: if multiparts is enabled, RadosMultiObjReadHandle +#endif eckit::MemoryHandle mh; dh->copyTo(mh); @@ -341,8 +365,13 @@ CASE("RadosStore tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) + /// @note: with multipart enabled, the field spans potentially several objects and is + /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. + EXPECT(dynamic_cast(dh.get())); +#else EXPECT(dynamic_cast(dh.get())); - /// @todo: if multiparts is enabled, RadosMultiObjReadHandle +#endif eckit::MemoryHandle mh; dh->copyTo(mh); @@ -386,6 +415,13 @@ CASE("RadosStore tests") { SECTION("VIA FDB API") { std::string test_id = "test-store3"; + + /// @note: the POSIX toc catalogue root is shared across sections; reset it so this + /// section is not polluted by entries left behind by previous sections. + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } + store_tests_tmp_root().mkdir(); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; @@ -531,32 +567,18 @@ CASE("RadosStore tests") { // wipe data - fdb5::WipeElement elem; - // dry run attempt to wipe with too specific request auto wipeObject = fdb.wipe(full_req); - count = 0; - while (wipeObject.next(elem)) { - count++; - } - EXPECT(count == 0); + EXPECT(countWipeable(wipeObject) == 0); // dry run wipe index and store unit wipeObject = fdb.wipe(index_req); - count = 0; - while (wipeObject.next(elem)) { - count++; - } - EXPECT(count > 0); + EXPECT(countWipeable(wipeObject) > 0); // dry run wipe database wipeObject = fdb.wipe(db_req); - count = 0; - while (wipeObject.next(elem)) { - count++; - } - EXPECT(count > 0); + EXPECT(countWipeable(wipeObject) > 0); // ensure field still exists listObject = fdb.list(full_req); @@ -570,21 +592,13 @@ CASE("RadosStore tests") { // attempt to wipe with too specific request wipeObject = fdb.wipe(full_req, true); - count = 0; - while (wipeObject.next(elem)) { - count++; - } - EXPECT(count == 0); + EXPECT(countWipeable(wipeObject) == 0); /// @todo: really needed? fdb.flush(); // wipe index and store unit (and DB pool or namespace as there is only one index) wipeObject = fdb.wipe(index_req, true); - count = 0; - while (wipeObject.next(elem)) { - count++; - } - EXPECT(count > 0); + EXPECT(countWipeable(wipeObject) > 0); /// @todo: really needed? fdb.flush(); @@ -602,6 +616,13 @@ CASE("RadosStore tests") { SECTION("FDB API RE-STORE AND WIPE DB") { std::string test_id = "test-store4"; + + /// @note: the POSIX toc catalogue root is shared across sections; reset it so this + /// section is not polluted by entries left behind by previous sections. + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } + store_tests_tmp_root().mkdir(); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; @@ -699,13 +720,8 @@ CASE("RadosStore tests") { // wipe all database - fdb5::WipeElement elem; auto wipeObject = fdb.wipe(db_req, true); - count = 0; - while (wipeObject.next(elem)) { - count++; - } - EXPECT(count > 0); + EXPECT(countWipeable(wipeObject) > 0); /// @todo: really needed? fdb.flush(); From 1f815bd0c44cbef9aa028391ca21925a2b601a3f Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 7 Jul 2026 16:50:08 +0200 Subject: [PATCH 045/109] fix(ceph): common --- src/fdb5/rados/RadosCatalogue.cc | 8 +- src/fdb5/rados/RadosCatalogueReader.cc | 5 +- src/fdb5/rados/RadosCatalogueWriter.cc | 46 +++-- src/fdb5/rados/RadosCommon.cc | 12 +- src/fdb5/rados/RadosCommon.h | 5 + src/fdb5/rados/RadosEngine.cc | 245 +++++++----------------- src/fdb5/rados/RadosEngine.h | 6 + src/fdb5/rados/RadosIndex.cc | 24 ++- src/fdb5/rados/RadosIndexLocation.cc | 9 +- src/fdb5/rados/RadosIndexLocation.h | 33 ++-- src/fdb5/rados/RadosStore.cc | 30 ++- src/fdb5/rados/RadosStore.h | 6 +- tests/fdb/rados/test_rados_catalogue.cc | 5 +- tests/fdb/rados/test_rados_store.cc | 6 +- 14 files changed, 172 insertions(+), 268 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index bd8df5e3c..b6595f76e 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -61,10 +61,10 @@ RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& CatalogueImpl(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool = pool_; + std::string pool = pool_; std::string nspace = db_namespace_; #else - std::string pool = db_pool_; + std::string pool = db_pool_; std::string nspace = namespace_; #endif @@ -72,7 +72,7 @@ RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& try { std::vector data; eckit::MemoryStream ms = db_kv_->getMemoryStream(data, "key", "DB kv"); - dbKey_ = fdb5::Key(ms); + dbKey_ = fdb5::Key(ms); } catch (eckit::RadosEntityNotFoundException& e) { @@ -187,7 +187,7 @@ std::string RadosCatalogue::type() const { bool RadosCatalogue::uriBelongs(const eckit::URI& uri) const { const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); + const auto n = parts.size(); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index 7d6f2e2b1..96a87216f 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -44,7 +44,7 @@ bool RadosCatalogueReader::selectIndex(const Key& key) { /// - generate catalogue kv oid (daos_obj_generate_oid) /// - ensure catalogue kv exists (daos_kv_open) - int idx_loc_max_len = 512; /// @todo: take from config + int idx_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: take from config std::vector n((long)idx_loc_max_len); long res; @@ -123,8 +123,9 @@ bool RadosCatalogueReader::retrieve(const Key& key, Field& field) const { eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; eckit::Log::debug() << "Scanning index " << current_.location() << std::endl; - if (!current_.mayContain(key)) + if (!current_.mayContain(key)) { return false; + } return current_.get(key, Key(), field); } diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index b054ef331..e06c329b6 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -84,17 +84,19 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c hs << dbKey_; } - int db_key_max_len = 512; // @todo: take from config - if (hs.bytesWritten() > db_key_max_len) + int db_key_max_len = RADOS_MAX_SERIALISED_LEN; // @todo: take from config + if (hs.bytesWritten() > db_key_max_len) { throw eckit::Exception("Serialised db key exceeded configured maximum db key length."); + } db_kv_->put("key", h.data(), hs.bytesWritten()); /// index newly created catalogue kv in main kv - int db_loc_max_len = 512; // @todo: take from config - std::string nstr = db_kv_->uri().asString(); - if (nstr.length() > db_loc_max_len) + int db_loc_max_len = RADOS_MAX_SERIALISED_LEN; // @todo: take from config + std::string nstr = db_kv_->uri().asString(); + if (nstr.length() > db_loc_max_len) { throw eckit::Exception("Serialised db location exceeded configured maximum db location length."); + } root_kv_->put(db_name, nstr.data(), nstr.length()); } @@ -128,10 +130,10 @@ bool RadosCatalogueWriter::createIndex(const Key& /* idxKey */, size_t /* datumK bool RadosCatalogueWriter::selectIndex(const Key& key) { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool = pool_; + std::string pool = pool_; std::string nspace = db_namespace_; #else - std::string pool = db_pool_; + std::string pool = db_pool_; std::string nspace = namespace_; #endif @@ -143,7 +145,7 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { /// - generate catalogue kv oid (daos_obj_generate_oid) /// - ensure catalogue kv exists (daos_kv_open) - int idx_loc_max_len = 512; /// @todo: take from config + int idx_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: take from config try { @@ -175,8 +177,9 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { /// index index kv in catalogue kv std::string nstr{indexes_[key].location().uri().asString()}; - if (nstr.length() > idx_loc_max_len) + if (nstr.length() > idx_loc_max_len) { throw eckit::Exception("Serialised index location exceeded configured maximum index location length."); + } /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on /// first write to an index key): /// - record index kv location into catalogue kv (daos_kv_put) -- always performed @@ -197,7 +200,7 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { void RadosCatalogueWriter::deselectIndex() { - current_ = Index(); + current_ = Index(); currentIndexKey_ = Key(); firstIndexWrite_ = false; } @@ -231,10 +234,10 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) { #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - std::string pool = pool_; + std::string pool = pool_; std::string nspace = db_namespace_; #else - std::string pool = db_pool_; + std::string pool = db_pool_; std::string nspace = namespace_; #endif @@ -255,7 +258,7 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::vector axesToExpand; std::vector valuesToAdd; std::string axisNames = ""; - std::string sep = ""; + std::string sep = ""; for (Key::const_iterator i = datumKey.begin(); i != datumKey.end(); ++i) { @@ -263,8 +266,9 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, const std::string& value = i->second; - if (value.length() == 0) + if (value.length() == 0) { continue; + } axisNames += sep + keyword; sep = ","; @@ -292,9 +296,10 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, /// - generate index kv oid (daos_obj_generate_oid) /// - ensure index kv exists (daos_obj_open) - int axis_names_max_len = 512; - if (axisNames.length() > axis_names_max_len) + int axis_names_max_len = RADOS_MAX_SERIALISED_LEN; + if (axisNames.length() > axis_names_max_len) { throw eckit::Exception("Serialised axis names exceeded configured maximum axis names length."); + } /// @note: performed RPCs: /// - record axis names into index kv (daos_kv_put) @@ -307,8 +312,9 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, /// @todo: axes are supposed to be sorted before persisting. How do we do this with the DAOS approach? /// sort axes every time they are loaded in the read pathway? - if (axesToExpand.empty()) + if (axesToExpand.empty()) { return; + } /// expand axis info in DAOS while (!axesToExpand.empty()) { @@ -330,14 +336,16 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, void RadosCatalogueWriter::flush(size_t archivedFields) { #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j) + for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j) { j->second.flush(); + } db_kv_->flush(); root_kv_->flush(); #endif - if (!current_.null()) + if (!current_.null()) { current_ = Index(); + } } void RadosCatalogueWriter::closeIndexes() { diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index a7474503c..f8828f2c4 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -14,7 +14,6 @@ #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" -#include "eckit/io/rados/RadosKeyValue.h" #include "eckit/utils/Tokenizer.h" #include "fdb5/config/Config.h" @@ -67,7 +66,7 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - pool_ = parts[0]; + pool_ = parts[0]; db_namespace_ = parts[1]; readConfig(config, component, false); @@ -77,15 +76,14 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con #else - db_pool_ = parts[0]; + db_pool_ = parts[0]; namespace_ = parts[1]; readConfig(config, component, false); - const auto parts = eckit::Tokenizer("_").tokenize(db_pool_); - const auto n = parts.size(); - ASSERT(n > 1); - pool_prefix_ = parts[0]; + const auto poolParts = eckit::Tokenizer("_").tokenize(db_pool_); + ASSERT(poolParts.size() > 1); + pool_prefix_ = poolParts[0]; root_kv_.emplace(root_pool_, namespace_, "main_kv"); db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 25e4604ea..7ead50faf 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -26,6 +26,11 @@ namespace fdb5 { +/// @note: maximum length (in bytes) of the serialised blobs exchanged with Rados key-values +/// (index/field/db locations, serialised keys, axis names). +/// @todo: make configurable (the call sites currently carry a "take from config" note). +constexpr long RADOS_MAX_SERIALISED_LEN = 512; + class RadosCommon { public: // methods diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index 2138dc5a7..ca85bed2c 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -9,8 +9,8 @@ */ -#include "eckit/serialisation/MemoryStream.h" #include "eckit/config/Resource.h" +#include "eckit/serialisation/MemoryStream.h" #include "fdb5/LibFdb5.h" #include "fdb5/rados/RadosEngine.h" @@ -25,202 +25,65 @@ std::string RadosEngine::name() const { return RadosEngine::typeName(); } -// bool DaosEngine::canHandle(const eckit::URI& uri, const Config& config) const { - -// configureDaos(config); - -// if (uri.scheme() != "daos") -// return false; - -// fdb5::DaosName n{uri}; - -// if (!n.hasOID()) return false; - -// /// @todo: check containerName is not root_cont_. root_cont_ should be populated in -// /// configureDaos as done in DaosCommon -// // bool is_root_name = (n.containerName().find(root_cont_) != std::string::npos); -// bool is_root_name = false; -// bool is_store_name = (n.containerName().find("_") != std::string::npos); - -// /// @note: performed RPCs: -// /// - generate oids (daos_obj_generate_oid) -// /// - db kv open (daos_kv_open) - -// fdb5::DaosName n2{n.poolName(), n.containerName(), catalogue_kv_}; -// bool is_catalogue_kv = (!is_root_name && !is_store_name && (n.OID() == n2.OID())); - -// return is_catalogue_kv && n.exists(); - -// } - -std::vector RadosEngine::visitableLocations(const Key& key, const Config& config) const -{ +std::vector RadosEngine::visitableLocations(const std::function& matches, + const Config& config) const { - /// @note: code mostly copied from DaosCommon - /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is - /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily - /// initialised. If owning a private instance of DaosCommon here, then the private members of - /// DaosCommon are not accessible from here + /// @note: cannot inherit from RadosCommon here, as the Engine is always instantiated even when + /// Rados is not used; it would then initialise RadosCommon unnecessarily. So the root key-value + /// naming is resolved locally via readConfig. - std::string component = "catalogue"; - -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + const std::string component = "catalogue"; readConfig(config, component, true); - // db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); - +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL root_kv_.emplace(pool_, root_namespace_, "main_kv"); - // db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); - #else - - readConfig(config, component, true); - - // db_pool_ = pool_prefix_ + "_" + key.valuesToString(); - root_kv_.emplace(root_pool_, namespace_, "main_kv"); - // db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); - #endif - /// --- - std::vector res{}; - /// @note: performed RPCs: - /// - main kv open (daos_kv_open) - - if (!root_kv_->exists()) return res; + if (!root_kv_->exists()) { + return res; + } - /// @note: performed RPCs: - /// - main kv list keys (daos_kv_list) for (const auto& k : root_kv_->keys()) { try { - /// @note: performed RPCs: - /// - main kv get db location size (daos_kv_get without a buffer) - /// - main kv get db location (daos_kv_get) std::vector v; - auto m = root_kv_->getMemoryStream(v, k, "root kv"); + root_kv_->getMemoryStream(v, k, "root kv"); eckit::URI uri(std::string(v.begin(), v.end())); ASSERT(uri.scheme() == typeName()); - /// @todo: this exact deserialisation is performed twice. Once here and once - /// in DaosCatalogue::(uri, ...). Try to avoid one. - - /// @note: performed RPCs: - /// - db kv open (daos_kv_open) - /// - db key get size (daos_kv_get without a buffer) - /// - db key get (daos_kv_get) + /// @todo: this deserialisation is also performed in RadosCatalogue(uri, ...). Try to avoid one. eckit::RadosKeyValue db_kv{uri}; /// @note: includes exist check std::vector data; eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); fdb5::Key db_key(ms); - if (db_key.match(key)) { - + if (matches(db_key)) { Log::debug() << " found match with " << root_kv_->uri() << " at key " << k << std::endl; res.push_back(uri); - } - - } catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; + } + catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; eckit::Log::error() << e.what() << std::endl; } - } return res; - } -std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const -{ - - /// @note: code mostly copied from DaosCommon - /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is - /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily - /// initialised. If owning a private instance of DaosCommon here, then the private members of - /// DaosCommon are not accessible from here - - std::string component = "catalogue"; - -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - - readConfig(config, component, true); - - // db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); - - root_kv_.emplace(pool_, root_namespace_, "main_kv"); - // db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); - -#else - - readConfig(config, component, true); - - // db_pool_ = pool_prefix_ + "_" + key.valuesToString(); - - root_kv_.emplace(root_pool_, namespace_, "main_kv"); - // db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); - -#endif - - /// --- - - std::vector res{}; - - /// @note: performed RPCs: - /// - main kv open (daos_kv_open) - - if (!root_kv_->exists()) return res; - - /// @note: performed RPCs: - /// - main kv list keys (daos_kv_list) - for (const auto& k : root_kv_->keys()) { - - try { - - /// @note: performed RPCs: - /// - main kv get db location size (daos_kv_get without a buffer) - /// - main kv get db location (daos_kv_get) - std::vector v; - auto m = root_kv_->getMemoryStream(v, k, "root kv"); - - eckit::URI uri(std::string(v.begin(), v.end())); - ASSERT(uri.scheme() == typeName()); - - /// @todo: this exact deserialisation is performed twice. Once here and once - /// in DaosCatalogue::(uri, ...). Try to avoid one. - - /// @note: performed RPCs: - /// - db kv open (daos_kv_open) - /// - db key get size (daos_kv_get without a buffer) - /// - db key get (daos_kv_get) - eckit::RadosKeyValue db_kv{uri}; /// @note: includes exist check - std::vector data; - eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); - fdb5::Key db_key(ms); - - if (db_key.partialMatch(request)) { - - Log::debug() << " found match with " << root_kv_->uri() << " at key " << k << std::endl; - res.push_back(uri); - - } - - } catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; - eckit::Log::error() << e.what() << std::endl; - } - - } - - return res; +std::vector RadosEngine::visitableLocations(const Key& key, const Config& config) const { + return visitableLocations([&key](const fdb5::Key& dbKey) { return dbKey.match(key); }, config); +} +std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const { + return visitableLocations([&request](const fdb5::Key& dbKey) { return dbKey.partialMatch(request); }, config); } #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL @@ -231,7 +94,9 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp eckit::LocalConfiguration c{}; - if (config.has("rados")) c = config.getSubConfiguration("rados"); + if (config.has("rados")) { + c = config.getSubConfiguration("rados"); + } // maxPartSize_ = c.getInt("maxPartSize", 0); @@ -239,53 +104,77 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp first_cap[0] = toupper(component[0]); std::string all_caps{component}; - for (auto & c: all_caps) c = toupper(c); + for (auto& c : all_caps) { + c = toupper(c); + } #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - if (readPool) pool_ = "default"; + if (readPool) { + pool_ = "default"; + } root_namespace_ = "root"; if (readPool) { pool_ = c.getString("pool", pool_); - if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); + if (c.has(component)) { + pool_ = c.getSubConfiguration(component).getString("pool", pool_); + } } root_namespace_ = c.getString("root_namespace", root_namespace_); - if (c.has(component)) root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + if (c.has(component)) { + root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + } - if (readPool) + if (readPool) { pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); - root_namespace_ = eckit::Resource("fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + } + root_namespace_ = eckit::Resource( + "fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); - if (c.has(component)) nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); - ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); + if (c.has(component)) { + nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); + } + ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, + "The configured namespace prefix must not contain underscores."); #else - if (readNamespace) namespace_ = "default"; + if (readNamespace) { + namespace_ = "default"; + } root_pool_ = "root"; - if (readNamespace) + if (readNamespace) { namespace_ = c.getString("namespace", namespace_); - if (c.has(component)) namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + } + if (c.has(component)) { + namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + } root_pool_ = c.getString("root_pool", root_pool_); - if (c.has(component)) root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); + if (c.has(component)) { + root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); + } - if (readNamespace) - namespace_ = eckit::Resource("fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); - root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); + if (readNamespace) { + namespace_ = eckit::Resource( + "fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); + } + root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", + root_pool_); pool_prefix_ = c.getString("pool_prefix", pool_prefix_); - if (c.has(component)) pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + if (c.has(component)) { + pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + } ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); #endif - } static EngineBuilder rados_builder; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index ccb46fb75..1a4409903 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -22,6 +22,7 @@ #include "fdb5/database/Engine.h" #include "fdb5/fdb5_config.h" +#include #include #include #include @@ -61,6 +62,11 @@ class RadosEngine : public fdb5::Engine { private: // methods + /// @note: shared implementation of the two visitableLocations overloads; lists all databases + /// registered in the root key-value and returns those whose key satisfies the predicate. + std::vector visitableLocations(const std::function& matches, + const Config& config) const; + #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL void readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const; #else diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index e4e5fc392..8f1b85321 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -23,6 +23,8 @@ #include "eckit/serialisation/Reanimator.h" #include "eckit/utils/Tokenizer.h" +#include "fdb5/rados/RadosCommon.h" + #include "fdb5/database/EntryVisitMechanism.h" #include "fdb5/database/Field.h" #include "fdb5/database/FieldDetails.h" @@ -83,10 +85,11 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : hs << key; } - int idx_key_max_len = 512; + int idx_key_max_len = RADOS_MAX_SERIALISED_LEN; - if (hs.bytesWritten() > idx_key_max_len) + if (hs.bytesWritten() > idx_key_max_len) { throw eckit::Exception("Serialised index key exceeded configured maximum index key length."); + } /// @note: performed RPCs: /// - record index key into index kv (daos_kv_put) @@ -132,7 +135,7 @@ void RadosIndex::updateAxes() { /// @note: performed RPCs: /// - ensure axis kv exists (daos_obj_open) - int axis_names_max_len = 512; /// @todo: take from config + int axis_names_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: take from config std::vector axes_data((long)axis_names_max_len); /// @note: performed RPCs: @@ -165,7 +168,7 @@ bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { std::string query{key.valuesToString()}; - int field_loc_max_len = 512; /// @todo: read from config + int field_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: read from config std::vector loc_data((long)field_loc_max_len); long res; @@ -190,7 +193,7 @@ bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { ms >> ts; fdb5::FieldLocation* loc = eckit::Reanimator::reanimate(ms); - field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); + field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); /// @note: performed RPCs: /// - close index kv (daos_obj_close) @@ -220,9 +223,10 @@ void RadosIndex::add(const Key& key, const Field& field) { hs << field.location(); } - int field_loc_max_len = 512; /// @todo: read from config - if (hs.bytesWritten() > field_loc_max_len) + int field_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: read from config + if (hs.bytesWritten() > field_loc_max_len) { throw eckit::Exception("Serialised field location exceeded configured maximum location length."); + } /// @note: performed RPCs: /// - ensure index kv exists (daos_obj_open) @@ -244,8 +248,9 @@ void RadosIndex::entries(EntryVisitor& visitor) const { for (const auto& key : idx_kv_.keys()) { - if (key == "axes" || key == "key") + if (key == "axes" || key == "key") { continue; + } /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field @@ -280,8 +285,9 @@ std::vector RadosIndex::dataURIs() const { for (const auto& key : idx_kv_.keys()) { - if (key == "axes" || key == "key") + if (key == "axes" || key == "key") { continue; + } std::vector data; eckit::MemoryStream ms = idx_kv_.getMemoryStream(data, key, "index kv"); diff --git a/src/fdb5/rados/RadosIndexLocation.cc b/src/fdb5/rados/RadosIndexLocation.cc index 372d5d967..49329c756 100644 --- a/src/fdb5/rados/RadosIndexLocation.cc +++ b/src/fdb5/rados/RadosIndexLocation.cc @@ -15,17 +15,16 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -// RadosIndexLocation::RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset) : name_(name), offset_(offset) {} -// #else +// RadosIndexLocation::RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset) : name_(name), +// offset_(offset) {} #else RadosIndexLocation::RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset) : name_(name), offset_(offset) {} // #endif -void RadosIndexLocation::print(std::ostream &out) const { +void RadosIndexLocation::print(std::ostream& out) const { out << "(" << name_.uri().asString() << ":" << offset_ << ")"; - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndexLocation.h b/src/fdb5/rados/RadosIndexLocation.h index 1546990cc..65393a2d1 100644 --- a/src/fdb5/rados/RadosIndexLocation.h +++ b/src/fdb5/rados/RadosIndexLocation.h @@ -14,8 +14,8 @@ #pragma once #include "eckit/exception/Exceptions.h" -#include "eckit/io/rados/RadosKeyValue.h" #include "eckit/io/rados/RadosAsyncKeyValue.h" +#include "eckit/io/rados/RadosKeyValue.h" #include "fdb5/database/IndexLocation.h" @@ -26,42 +26,41 @@ namespace fdb5 { class RadosIndexLocation : public IndexLocation { -public: // methods +public: // methods -// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -// RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset); + // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + // RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset); -// const eckit::RadosPersistentKeyValue& radosName() const { return name_; }; -// #else + // const eckit::RadosPersistentKeyValue& radosName() const { return name_; }; + // #else RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset); const eckit::RadosKeyValue& radosName() const { return name_; }; -// #endif + // #endif eckit::URI uri() const override { return name_.uri(); } IndexLocation* clone() const override { NOTIMP; } -protected: // For Streamable +protected: // For Streamable void encode(eckit::Stream&) const override { NOTIMP; } -private: // methods +private: // methods - void print(std::ostream &out) const override; + void print(std::ostream& out) const override; -private: // members +private: // members -// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -// eckit::RadosPersistentKeyValue name_; -// #else + // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + // eckit::RadosPersistentKeyValue name_; + // #else eckit::RadosKeyValue name_; -// #endif + // #endif off_t offset_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index f5699ae22..f17d71575 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -38,17 +38,17 @@ namespace fdb5 { static StoreBuilder builder("rados"); RadosStore::RadosStore(const Key& key, const Config& config) : - Store(), RadosCommon(config, "store", key), config_(config), archivedFields_(0) { + Store(), RadosCommon(config, "store", key), archivedFields_(0) { - parseConfig(config_); + parseConfig(config); } RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : RadosStore(key, config) {} RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : - Store(), RadosCommon(config, "store", uri), config_(config), archivedFields_(0) { + Store(), RadosCommon(config, "store", uri), archivedFields_(0) { - parseConfig(config_); + parseConfig(config); } eckit::URI RadosStore::uri() const { @@ -82,7 +82,7 @@ eckit::URI RadosStore::uri(const eckit::URI& dataURI) { bool RadosStore::uriBelongs(const eckit::URI& uri) const { const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); + const auto n = parts.size(); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL @@ -102,7 +102,7 @@ bool RadosStore::uriExists(const eckit::URI& uri) const { /// @todo: revisit the name of this method const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); + const auto n = parts.size(); ASSERT(uri.scheme() == type()); @@ -304,7 +304,7 @@ size_t RadosStore::flush() { #endif - size_t out = archivedFields_; + size_t out = archivedFields_; archivedFields_ = 0; return out; } @@ -333,7 +333,7 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost ASSERT(uri.scheme() == type()); const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); - const auto n = parts.size(); + const auto n = parts.size(); #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL @@ -591,15 +591,11 @@ eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { const eckit::RadosObject& RadosStore::getDataObject(const Key& key) const { - ObjectStore::const_iterator j = dataObjects_.find(key); - - if (j != dataObjects_.end()) { - return j->second; + auto it = dataObjects_.find(key); + if (it == dataObjects_.end()) { + it = dataObjects_.emplace(key, generateDataObject(key)).first; } - - dataObjects_.insert(std::pair(key, generateDataObject(key))); - - return dataObjects_.find(key)->second; + return it->second; } eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosObject& name) { @@ -675,7 +671,7 @@ void RadosStore::parseConfig(const fdb5::Config& config) { #if (!defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD)) && defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024); + maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024); maxPartHandleBuffSize_ = store_conf.getInt("maxPartHandleBuffSize", 1024); #else maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024 * 1024); diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index db57a69c1..e71b99431 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -58,9 +58,7 @@ class RadosStore : public Store, public RadosCommon { bool doUnsafeFullWipe() const override; // Rados store does not currently support auxiliary objects - std::vector getAuxiliaryURIs(const eckit::URI&, bool onlyExisting = false) const override { - return {}; - } + std::vector getAuxiliaryURIs(const eckit::URI&, bool onlyExisting = false) const override { return {}; } protected: // methods @@ -93,8 +91,6 @@ class RadosStore : public Store, public RadosCommon { private: // members - const Config& config_; - // mutable bool dirty_; size_t archivedFields_{0}; diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 129903123..eaf21db4a 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -116,8 +116,9 @@ CASE("Setup") { // ensure fdb root directory exists. If not, then that root is // registered as non existing and Catalogue/Store tests fail. - if (catalogue_tests_tmp_root().exists()) + if (catalogue_tests_tmp_root().exists()) { deldir(catalogue_tests_tmp_root()); + } catalogue_tests_tmp_root().mkdir(); ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); @@ -616,7 +617,7 @@ CASE("RadosCatalogue tests") { // list all listObject = fdb.list(all_req); - count = 0; + count = 0; while (listObject.next(info)) { // info.print(std::cout, true, true); // std::cout << std::endl; diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 0a2ea0ca6..6610bf6a8 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -582,7 +582,7 @@ CASE("RadosStore tests") { // ensure field still exists listObject = fdb.list(full_req); - count = 0; + count = 0; while (listObject.next(info)) { // info.print(std::cout, true, true); // std::cout << std::endl; @@ -604,7 +604,7 @@ CASE("RadosStore tests") { // ensure field does not exist listObject = fdb.list(full_req); - count = 0; + count = 0; while (listObject.next(info)) { count++; } @@ -729,7 +729,7 @@ CASE("RadosStore tests") { fdb5::ListElement info; auto listObject = fdb.list(full_req); - count = 0; + count = 0; while (listObject.next(info)) { // info.print(std::cout, true, true); // std::cout << std::endl; From 6205215c0816e9b775784db1cf8591a457526dd3 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 8 Jul 2026 10:35:49 +0200 Subject: [PATCH 046/109] fix(ceph): find rados --- cmake/FindRADOS.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmake/FindRADOS.cmake b/cmake/FindRADOS.cmake index 1a861823c..8fd21afa0 100644 --- a/cmake/FindRADOS.cmake +++ b/cmake/FindRADOS.cmake @@ -33,9 +33,6 @@ find_package_handle_standard_args(RADOS REQUIRED_VARS RADOS_LIBRARY RADOS_INCLUDE_DIR ) -message(STATUS "DEBUG: RADOS_INCLUDE_DIR = ${RADOS_INCLUDE_DIR}") -message(STATUS "DEBUG: RADOS_LIBRARY = ${RADOS_LIBRARY}") - if(RADOS_FOUND) set(RADOS_LIBRARIES ${RADOS_LIBRARY}) set(RADOS_INCLUDE_DIRS ${RADOS_INCLUDE_DIR}) From 30ba31e2cb4fc7607ccbdf9ea1eef6ce1afd01ce Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 8 Jul 2026 10:10:08 +0200 Subject: [PATCH 047/109] fix(ci): undo me --- .github/ci-config.yml | 2 +- .github/ci-hpc-config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ci-config.yml b/.github/ci-config.yml index ff76733dc..e91d60d49 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -2,7 +2,7 @@ dependencies: | ecmwf/ecbuild MathisRosenhauer/libaec@refs/tags/v1.1.3 ecmwf/eccodes - ecmwf/eckit + ecmwf/eckit@feature/ECKIT-684-ceph-backend ecmwf/metkit dependency_branch: develop cmake_options: -DENABLE_DUMMY_DAOS=ON diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index c63727590..ab64d7c52 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -4,7 +4,7 @@ build: dependencies: - ecmwf/ecbuild@develop - ecmwf/eccodes@develop - - ecmwf/eckit@develop + - ecmwf/eckit@feature/ECKIT-684-ceph-backend - ecmwf/metkit@develop cmake_options: - -DENABLE_LUSTRE=OFF From a5a4c5e3d1956ab62f87d5a9ddc91fc30650d8ee Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 8 Jul 2026 10:45:26 +0200 Subject: [PATCH 048/109] fix(ceph): restore --- src/fdb5/daos/DaosStore.cc | 1 + src/fdb5/toc/TocStore.cc | 9 --------- tests/fdb/CMakeLists.txt | 1 - 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc index 57c96e39b..5412f227d 100644 --- a/src/fdb5/daos/DaosStore.cc +++ b/src/fdb5/daos/DaosStore.cc @@ -63,6 +63,7 @@ bool DaosStore::uriExists(const eckit::URI& uri) const { ASSERT(n.hasContainerName()); ASSERT(n.poolName() == pool_); ASSERT(n.containerName() == db_cont_); + ASSERT(n.hasOID()); return n.exists(); } diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc index 7d70ab52f..52d72af39 100644 --- a/src/fdb5/toc/TocStore.cc +++ b/src/fdb5/toc/TocStore.cc @@ -276,15 +276,6 @@ eckit::PathName TocStore::generateDataPath(const Key& key) const { eckit::PathName dpath(directory_); dpath /= key.valuesToString(); - /// @todo: in cases where a catalogue other than POSIX is used and a - /// POSIX store is used, the DB directory for the store is first - /// created within PathName::unique(). - /// DB directory creation should maybe be removed from there and - /// performed here, or as part of FDB/LustreFileHandle::openForAppend - /// if not exists. - /// If doing it in openForAppend, it should be ensured that the - /// existence of the database directory is not checked an excessive - /// amount of times. dpath = eckit::PathName::unique(dpath) + ".data"; return dpath; } diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index e79973f17..7570df833 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -72,7 +72,6 @@ add_subdirectory( type ) add_subdirectory( rados ) add_subdirectory( daos ) - if (HAVE_FDB_BUILD_TOOLS) add_subdirectory( timespan ) add_subdirectory( tools ) From 50fe570497963cd6750236a93ce1458a5399fc03 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 9 Jul 2026 14:19:01 +0200 Subject: [PATCH 049/109] fix(ceph): add ci --- .github/ci-rados.yml | 143 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 .github/ci-rados.yml diff --git a/.github/ci-rados.yml b/.github/ci-rados.yml new file mode 100644 index 000000000..7bdbe96dd --- /dev/null +++ b/.github/ci-rados.yml @@ -0,0 +1,143 @@ +name: ci-rados + +on: + push: + branches: + - "master" + - "develop" + tags-ignore: + - "**" + + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger on public pull request approval + pull_request_target: + types: [labeled] + + # Trigger the workflow manually + workflow_dispatch: ~ + +jobs: + rados: + name: Test Ceph/RADOS Backend + runs-on: ubuntu-latest + + env: + CEPH_ETC: /etc/ceph + FDB_RADOS_TEST_POOL: fdb_test + FDB_RADOS_CLUSTER_NAME: ceph + FDB_RADOS_CLUSTER_USER: client.admin + # Local dirs used to cache apt archives and the Ceph docker image. + APT_CACHE: ${{ github.workspace }}/.apt-cache + CEPH_IMAGE: quay.io/ceph/demo:latest-squid + CEPH_IMAGE_CACHE: ${{ github.workspace }}/.docker-ceph + + steps: + - name: Checkout ecbuild + uses: actions/checkout@v4 + with: + repository: ecmwf/ecbuild + path: ecbuild + + - name: Checkout eckit + uses: actions/checkout@v4 + with: + path: eckit + + - name: Cache apt packages + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.apt-cache + # Bump the suffix to invalidate when the package list changes. + key: apt-rados-${{ runner.os }}-v1 + + - name: Install build dependencies + run: | + mkdir -p "${APT_CACHE}/archives/partial" + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + -o Dir::Cache::archives="${APT_CACHE}/archives" \ + -o APT::Keep-Downloaded-Packages=true \ + build-essential \ + cmake \ + ninja-build \ + libssl-dev \ + uuid-dev \ + libbz2-dev \ + libcurl4-openssl-dev \ + libaec-dev \ + librados-dev + # Make the cached .deb archives readable/writable by the runner user. + sudo chown -R "$(id -u):$(id -g)" "${APT_CACHE}" + + - name: Cache Ceph image + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.docker-ceph + # Bump the suffix to refresh the pinned image snapshot. + key: ceph-image-latest-squid-v1 + + - name: Load or pull Ceph image + run: | + if [ -f "${CEPH_IMAGE_CACHE}/ceph-demo.tar" ]; then + docker load -i "${CEPH_IMAGE_CACHE}/ceph-demo.tar" + else + docker pull "${CEPH_IMAGE}" + mkdir -p "${CEPH_IMAGE_CACHE}" + docker save "${CEPH_IMAGE}" -o "${CEPH_IMAGE_CACHE}/ceph-demo.tar" + fi + + - name: Start Ceph cluster + run: | + sudo mkdir -p "${CEPH_ETC}" + docker run -d --name ceph-demo \ + --network host \ + -e MON_IP=127.0.0.1 \ + -e CEPH_PUBLIC_NETWORK=0.0.0.0/0 \ + -e CEPH_DEMO_UID=ci \ + -v "${CEPH_ETC}:/etc/ceph" \ + "${CEPH_IMAGE}" demo + + - name: Wait for Ceph and create test pool + run: | + echo "Waiting for Ceph to become ready..." + for i in $(seq 1 60); do + if docker exec ceph-demo ceph osd pool ls >/dev/null 2>&1; then + ready=1 + break + fi + sleep 5 + done + if [ "${ready:-0}" != "1" ]; then + echo "Ceph did not become ready in time" >&2 + docker logs ceph-demo || true + exit 1 + fi + docker exec ceph-demo ceph osd pool create "${FDB_RADOS_TEST_POOL}" 8 8 + docker exec ceph-demo ceph osd pool application enable "${FDB_RADOS_TEST_POOL}" rados + docker exec ceph-demo ceph osd pool ls + sudo chmod 0644 "${CEPH_ETC}/ceph.client.admin.keyring" + + - name: Configure + run: | + # eckit locates ecbuild via find_package(ecbuild ... HINTS .../../ecbuild), + # which resolves to the sibling ecbuild checkout above. + cmake -S eckit -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_RADOS=ON \ + -DENABLE_RADOS_ADMIN=OFF \ + -DFDB_RADOS_TEST_POOL="${FDB_RADOS_TEST_POOL}" + + - name: Build + run: cmake --build build --parallel + + - name: Run RADOS tests + env: + FDB_RADOS_CLUSTER_CONF: /etc/ceph/ceph.conf + run: | + ctest --test-dir build -L FDB_rados --output-on-failure + + - name: Dump Ceph logs on failure + if: failure() + run: docker logs ceph-demo || true From 382cedc4881393cd321814017348671d7cee2c70 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 9 Jul 2026 14:34:09 +0200 Subject: [PATCH 050/109] fix(ceph): undo me ci --- .github/{ => workflows}/ci-rados.yml | 1 + 1 file changed, 1 insertion(+) rename .github/{ => workflows}/ci-rados.yml (99%) diff --git a/.github/ci-rados.yml b/.github/workflows/ci-rados.yml similarity index 99% rename from .github/ci-rados.yml rename to .github/workflows/ci-rados.yml index 7bdbe96dd..32c5361c1 100644 --- a/.github/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -5,6 +5,7 @@ on: branches: - "master" - "develop" + - "feature/FDB-683-ceph-backend" tags-ignore: - "**" From cbdc802f7f81ee94dc935b941b2f05a2b569d0ca Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 9 Jul 2026 15:49:55 +0200 Subject: [PATCH 051/109] fix(ceph): ci --- .github/workflows/ci-rados.yml | 62 ++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index 32c5361c1..c7c761a57 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -29,6 +29,8 @@ jobs: FDB_RADOS_TEST_POOL: fdb_test FDB_RADOS_CLUSTER_NAME: ceph FDB_RADOS_CLUSTER_USER: client.admin + # Install prefix shared by all bundle projects (eckit, eccodes, metkit, fdb). + INSTALL_PREFIX: ${{ github.workspace }}/install # Local dirs used to cache apt archives and the Ceph docker image. APT_CACHE: ${{ github.workspace }}/.apt-cache CEPH_IMAGE: quay.io/ceph/demo:latest-squid @@ -44,8 +46,26 @@ jobs: - name: Checkout eckit uses: actions/checkout@v4 with: + repository: ecmwf/eckit path: eckit + - name: Checkout eccodes + uses: actions/checkout@v4 + with: + repository: ecmwf/eccodes + path: eccodes + + - name: Checkout metkit + uses: actions/checkout@v4 + with: + repository: ecmwf/metkit + path: metkit + + - name: Checkout fdb + uses: actions/checkout@v4 + with: + path: fdb + - name: Cache apt packages uses: actions/cache@v4 with: @@ -120,24 +140,52 @@ jobs: docker exec ceph-demo ceph osd pool ls sudo chmod 0644 "${CEPH_ETC}/ceph.client.admin.keyring" - - name: Configure + - name: Build eckit run: | # eckit locates ecbuild via find_package(ecbuild ... HINTS .../../ecbuild), # which resolves to the sibling ecbuild checkout above. - cmake -S eckit -B build -G Ninja \ + cmake -S eckit -B build/eckit -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ -DENABLE_RADOS=ON \ - -DENABLE_RADOS_ADMIN=OFF \ - -DFDB_RADOS_TEST_POOL="${FDB_RADOS_TEST_POOL}" + -DENABLE_RADOS_ADMIN=OFF + cmake --build build/eckit --parallel + cmake --install build/eckit - - name: Build - run: cmake --build build --parallel + - name: Build eccodes + run: | + cmake -S eccodes -B build/eccodes -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ + -DENABLE_MEMFS=ON + cmake --build build/eccodes --parallel + cmake --install build/eccodes + + - name: Build metkit + run: | + cmake -S metkit -B build/metkit -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" + cmake --build build/metkit --parallel + cmake --install build/metkit + + - name: Build fdb + run: | + cmake -S fdb -B build/fdb -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ + -DENABLE_RADOSFDB=ON \ + -DFDB_RADOS_TEST_POOL="${FDB_RADOS_TEST_POOL}" + cmake --build build/fdb --parallel - name: Run RADOS tests env: FDB_RADOS_CLUSTER_CONF: /etc/ceph/ceph.conf run: | - ctest --test-dir build -L FDB_rados --output-on-failure + ctest --test-dir build/fdb -L rados --output-on-failure - name: Dump Ceph logs on failure if: failure() From 12ee3c7dbc57a8369614797e0a614ff6e5ee48fd Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 9 Jul 2026 16:08:51 +0200 Subject: [PATCH 052/109] fix(ceph): ci flags --- .github/workflows/ci-rados.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index c7c761a57..5cf42c461 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -35,6 +35,7 @@ jobs: APT_CACHE: ${{ github.workspace }}/.apt-cache CEPH_IMAGE: quay.io/ceph/demo:latest-squid CEPH_IMAGE_CACHE: ${{ github.workspace }}/.docker-ceph + CMAKE_FLAGS: -DENABLE_AEC=ON -DENABLE_EXAMPLES=OFF -DENABLE_EXPERIMENTAL=OFF -DENABLE_NETCDF=OFF steps: - name: Checkout ecbuild @@ -83,6 +84,8 @@ jobs: build-essential \ cmake \ ninja-build \ + gfortran \ + libopenmpi-dev \ libssl-dev \ uuid-dev \ libbz2-dev \ @@ -147,6 +150,8 @@ jobs: cmake -S eckit -B build/eckit -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ + ${CMAKE_FLAGS} \ + -DENABLE_MPI=ON \ -DENABLE_RADOS=ON \ -DENABLE_RADOS_ADMIN=OFF cmake --build build/eckit --parallel @@ -158,7 +163,11 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ - -DENABLE_MEMFS=ON + ${CMAKE_FLAGS} \ + -DENABLE_JPG=OFF \ + -DENABLE_FORTRAN=ON \ + -DENABLE_MEMFS=ON \ + -DENABLE_ECCODES_THREADS=ON cmake --build build/eccodes --parallel cmake --install build/eccodes @@ -167,7 +176,8 @@ jobs: cmake -S metkit -B build/metkit -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ - -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" + -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ + ${CMAKE_FLAGS} cmake --build build/metkit --parallel cmake --install build/metkit @@ -177,11 +187,13 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ + ${CMAKE_FLAGS} \ -DENABLE_RADOSFDB=ON \ + -DENABLE_RADOS_ADMIN=OFF \ -DFDB_RADOS_TEST_POOL="${FDB_RADOS_TEST_POOL}" cmake --build build/fdb --parallel - - name: Run RADOS tests + - name: Run RADOS FDB tests env: FDB_RADOS_CLUSTER_CONF: /etc/ceph/ceph.conf run: | From 5ac112c7862503682a6e8e8a4c825c5371b04673 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 9 Jul 2026 16:12:23 +0200 Subject: [PATCH 053/109] fix(ceph): ci avoid duplicates --- .github/workflows/ci-rados.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index 5cf42c461..b9cdadbdb 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -5,7 +5,6 @@ on: branches: - "master" - "develop" - - "feature/FDB-683-ceph-backend" tags-ignore: - "**" @@ -19,6 +18,11 @@ on: # Trigger the workflow manually workflow_dispatch: ~ +# avoid duplicate runs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: rados: name: Test Ceph/RADOS Backend From bc1defc8e369cf277195f3a44d2506f0af5e827a Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 9 Jul 2026 16:37:47 +0200 Subject: [PATCH 054/109] fix(ceph): ci eckit branch --- .github/workflows/ci-rados.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index b9cdadbdb..9816e2564 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -39,7 +39,7 @@ jobs: APT_CACHE: ${{ github.workspace }}/.apt-cache CEPH_IMAGE: quay.io/ceph/demo:latest-squid CEPH_IMAGE_CACHE: ${{ github.workspace }}/.docker-ceph - CMAKE_FLAGS: -DENABLE_AEC=ON -DENABLE_EXAMPLES=OFF -DENABLE_EXPERIMENTAL=OFF -DENABLE_NETCDF=OFF + CMAKE_FLAGS: -DENABLE_AEC=OFF -DENABLE_EXAMPLES=OFF -DENABLE_EXPERIMENTAL=OFF -DENABLE_NETCDF=OFF steps: - name: Checkout ecbuild @@ -52,6 +52,7 @@ jobs: uses: actions/checkout@v4 with: repository: ecmwf/eckit + ref: feature/ECKIT-684-ceph-backend path: eckit - name: Checkout eccodes From 1344fea339470faad83567728438b308bda1db10 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 10 Jul 2026 09:57:59 +0200 Subject: [PATCH 055/109] fix(ceph): ci --- .github/workflows/ci-rados.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index 9816e2564..10b5441bc 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -30,9 +30,9 @@ jobs: env: CEPH_ETC: /etc/ceph + ECKIT_RADOS_CLUSTER_NAME: ceph + ECKIT_RADOS_CLUSTER_USER: client.admin FDB_RADOS_TEST_POOL: fdb_test - FDB_RADOS_CLUSTER_NAME: ceph - FDB_RADOS_CLUSTER_USER: client.admin # Install prefix shared by all bundle projects (eckit, eccodes, metkit, fdb). INSTALL_PREFIX: ${{ github.workspace }}/install # Local dirs used to cache apt archives and the Ceph docker image. @@ -200,7 +200,7 @@ jobs: - name: Run RADOS FDB tests env: - FDB_RADOS_CLUSTER_CONF: /etc/ceph/ceph.conf + ECKIT_RADOS_CLUSTER_CONF: /etc/ceph/ceph.conf run: | ctest --test-dir build/fdb -L rados --output-on-failure From 67e134feeaf53aedc3bd9ffa3c40f5b7f9433dfb Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 13 Jul 2026 09:04:44 +0200 Subject: [PATCH 056/109] fix(ceph): header include --- src/fdb5/rados/RadosCatalogueReader.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index 96a87216f..bc04ffc82 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -10,7 +10,25 @@ #include "fdb5/rados/RadosCatalogueReader.h" +#include +#include +#include +#include +#include + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/Log.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCatalogue.h" +#include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosIndex.h" namespace fdb5 { From e302cc3d50f9887491f983628d6c9673be76ddee Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 17 Jul 2026 16:46:51 +0200 Subject: [PATCH 057/109] fix(uuid): findlibuuid.cmake --- cmake/FindLibUUID.cmake | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/cmake/FindLibUUID.cmake b/cmake/FindLibUUID.cmake index b2c83cf6c..38fca40fb 100644 --- a/cmake/FindLibUUID.cmake +++ b/cmake/FindLibUUID.cmake @@ -64,7 +64,27 @@ may also be set to help find libuuid library. #]=======================================================================] -find_path(LIB_UUID_INCLUDE_DIR uuid.h +set(_libuuid_extra_hints) +if(APPLE) + find_program(_libuuid_brew_executable brew) + if(_libuuid_brew_executable) + execute_process( + COMMAND ${_libuuid_brew_executable} --prefix util-linux + OUTPUT_VARIABLE _libuuid_util_linux_prefix + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + if(_libuuid_util_linux_prefix) + list(APPEND _libuuid_extra_hints "${_libuuid_util_linux_prefix}") + endif() + endif() + list(APPEND _libuuid_extra_hints + /opt/homebrew/opt/util-linux + /usr/local/opt/util-linux) + set(_libuuid_saved_find_framework "${CMAKE_FIND_FRAMEWORK}") + set(CMAKE_FIND_FRAMEWORK NEVER) +endif() + +find_path(LIB_UUID_INCLUDE_DIR uuid/uuid.h HINTS ${LIB_UUID_ROOT} ${LIB_UUID_DIR} @@ -72,7 +92,8 @@ find_path(LIB_UUID_INCLUDE_DIR uuid.h ENV LIB_UUID_ROOT ENV LIB_UUID_DIR ENV LIB_UUID_PATH - PATH_SUFFIXES uuid + ${_libuuid_extra_hints} + PATH_SUFFIXES include ) find_library(LIB_UUID_LIBRARY @@ -84,9 +105,15 @@ find_library(LIB_UUID_LIBRARY ENV LIB_UUID_ROOT ENV LIB_UUID_DIR ENV LIB_UUID_PATH + ${_libuuid_extra_hints} PATH_SUFFIXES lib lib64 ) +if(APPLE) + set(CMAKE_FIND_FRAMEWORK "${_libuuid_saved_find_framework}") + unset(_libuuid_saved_find_framework) +endif() + include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LibUUID REQUIRED_VARS From 337c6ee7c054c5e41c409ea8afa6d30ff3542515 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 21 Jul 2026 09:34:14 +0200 Subject: [PATCH 058/109] feat: add copilot instructions --- .github/copilot-instructions.md | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..a6de2d74e --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,46 @@ +--- +name: "FDB Development" +description: "Use when modifying FDB sources, CMake configuration, FDB tests, Python or Rust bindings, or validating FDB changes in the ecmwf bundle. Covers linux (and macOS) builds, CTest, clang-format, and clang-tidy." +applyTo: "src/**, tests/**, python/**, rust/**" +--- + +# Development + +## Project Boundaries + +- The `fdb` project provides the FDB library, command-line tools, and optional Python and Rust bindings. +- The project depends on other libraries, such as `eckit`, `eccodes`, and `metkit`. +- Keep an FDB change within `src/` unless the dependency contract or bundle integration genuinely needs a corresponding change elsewhere. + +## Configure And Build + +- Use an out-of-source CMake build. Preserve an existing build tree's generator and preset; do not configure over it with a different generator. +- Enable optional FDB interfaces during configuration only when the change requires them: `-DENABLE_PYTHON_FDB_INTERFACE=ON` and `-DENABLE_PYTHON_ZARR_INTERFACE=ON`. Zarr enables the PyFDB interface as a dependency. + +## Tests + +- FDB tests are registered through `src/tests/`: core and tool tests are under `src/tests/fdb/`, regressions under `src/tests/regressions/`, and optional Python/Zarr coverage under `src/tests/pyfdb/`, `src/tests/z3fdb/`, and `src/tests/pychunked_data_view/`. +- Start with the most specific affected test. Use CTest against the configured bundle, for example: + + ```sh + ctest --test-dir ./build/bundle -R 'fdb_test_api' --output-on-failure + ctest --test-dir ./build/bundle -L remotefdb --output-on-failure + ``` + +- Use `ctest --test-dir ./build/bundle -N` to find test names and `ctest --test-dir ./build/bundle --output-on-failure` for the broader suite. Some tests are deliberately absent when GRIB, tools, or Python interfaces are disabled; distinguish that configuration from a test failure. + +## Validation + +- Run the focused CTest selection after the affected target builds. Broaden to the relevant regression, label, or full suite when the change crosses an API, storage backend, or bundle dependency boundary. +- Format C and C++ changes with `src/.clang-format`. The FDB CI applies clang-format and ignores `third_party/`. +- The root presets generate `compile_commands.json`. After a successful build, run `src/run-clang-tidy ./build/bundle` for C++ changes when `clang-tidy`, `jq`, and GNU Parallel are available. + +# Code Reviews + +- Respond to review comments promptly. If you disagree with a comment, explain your reasoning and provide an alternative solution. If you accept a comment, make the change and mark it as resolved. + +- When performing a code review: + - Check that the code adheres to the project's coding standards and guidelines. + - Check for potential bugs, memory safety issues, security vulnerabilities, and performance issues. + - Check that the code has clear comments and explanations where necessary. + - Check that the code is tested, and that tests cover edge cases and failure scenarios. From 019db0d7b7256689f32d9c5d2f3b19f356e0b451 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 21 Jul 2026 09:36:18 +0200 Subject: [PATCH 059/109] fix(cmake): remove uuid cmake --- cmake/FindUUID.cmake | 64 -------------------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 cmake/FindUUID.cmake diff --git a/cmake/FindUUID.cmake b/cmake/FindUUID.cmake deleted file mode 100644 index e2b4bfbcd..000000000 --- a/cmake/FindUUID.cmake +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation nor -# does it submit to any jurisdiction. - -include(FindPackageHandleStandardArgs) - -# uuid - -find_path(UUID_INCLUDE_DIR - NAMES uuid/uuid.h - HINTS - ${UUID_ROOT} - ${UUID_DIR} - ${UUID_PATH} - ENV UUID_ROOT - ENV UUID_DIR - ENV UUID_PATH - PATH_SUFFIXES include include/uuid - NO_DEFAULT_PATH -) - -find_path(UUID_INCLUDE_DIR NAMES uuid/uuid.h PATH_SUFFIXES include include/uuid) - -find_library(UUID_LIBRARY - NAMES uuid - HINTS - ${UUID_ROOT} - ${UUID_DIR} - ${UUID_PATH} - ENV UUID_ROOT - ENV UUID_DIR - ENV UUID_PATH - PATH_SUFFIXES lib lib64 -) - -find_library(UUID_LIBRARY NAMES uuid PATH_SUFFIXES lib lib64) - -find_package_handle_standard_args(UUID DEFAULT_MSG UUID_LIBRARY UUID_INCLUDE_DIR) - -mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY) - -if(UUID_FOUND) - add_library(uuid UNKNOWN IMPORTED GLOBAL) - set_target_properties(uuid PROPERTIES - IMPORTED_LOCATION ${UUID_LIBRARY} - INTERFACE_INCLUDE_DIRECTORIES ${UUID_INCLUDE_DIR} - ) - set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR}) - set(UUID_LIBRARIES uuid) -endif() From 0e61584910ba9d7474106a1f247a494cce927a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 09:47:00 +0200 Subject: [PATCH 060/109] fix(ceph): typos --- CMakeLists.txt | 2 +- cmake/FindRADOS.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 31e365698..e4ba5365d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,7 @@ ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_FLUSH ecbuild_add_option( FEATURE RADOS_ADMIN DEFAULT OFF - DESCRIPTION "Have unit tests create pools automatically rather than using an existing pool specified in the ECKIT_RADOS_TEST_POOL cmake variable." ) + DESCRIPTION "Have unit tests create pools automatically rather than using an existing pool specified in the FDB_RADOS_TEST_POOL cmake variable." ) ### FDB backend in indexed filesystem with table-of-contents, i.e. TOC ### Supports Lustre parallel filesystem stripping control diff --git a/cmake/FindRADOS.cmake b/cmake/FindRADOS.cmake index 8fd21afa0..9c1783e9a 100644 --- a/cmake/FindRADOS.cmake +++ b/cmake/FindRADOS.cmake @@ -12,7 +12,7 @@ # RADOS_FOUND - True if Rados was found # # This module also defines the following IMPORTED target: -# Ceph::rados +# Ceph::RADOS # Find the header path by looking for the subdirectory file find_path(RADOS_INCLUDE_DIR From d9f6ea30930b323bd56c7286d9c2afdca00caa21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 09:48:27 +0200 Subject: [PATCH 061/109] fix(ceph): engine --- src/fdb5/rados/RadosEngine.cc | 47 ++++++++++++++++++++++++++++++++--- src/fdb5/rados/RadosEngine.h | 24 ++++++++---------- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index ca85bed2c..f19998ccf 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -9,11 +9,14 @@ */ -#include "eckit/config/Resource.h" -#include "eckit/serialisation/MemoryStream.h" +#include "fdb5/rados/RadosEngine.h" #include "fdb5/LibFdb5.h" -#include "fdb5/rados/RadosEngine.h" + +#include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Tokenizer.h" using namespace eckit; @@ -25,6 +28,44 @@ std::string RadosEngine::name() const { return RadosEngine::typeName(); } +eckit::URI RadosEngine::location(const Key& key, const Config& config) const { + + /// @note: cannot inherit from RadosCommon here, as the Engine is always instantiated even when + /// Rados is not used; it would then initialise RadosCommon unnecessarily. So the db key-value + /// naming is resolved locally via readConfig, mirroring RadosCommon's key-based constructor. + + readConfig(config, "catalogue", true); + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + const std::string db_namespace = nspace_prefix_ + "_" + key.valuesToString(); + return eckit::RadosKeyValue{pool_, db_namespace, "catalogue_kv"}.uri(); +#else + const std::string db_pool = pool_prefix_ + "_" + key.valuesToString(); + return eckit::RadosKeyValue{db_pool, namespace_, "catalogue_kv"}.uri(); +#endif +} + +bool RadosEngine::canHandle(const eckit::URI& uri, const Config&) const { + + if (uri.scheme() != typeName()) { + return false; + } + + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + if (parts.size() != 2 && parts.size() != 3) { + return false; + } + + try { + return eckit::RadosKeyValue{parts[0], parts[1], "catalogue_kv"}.exists(); + } + catch (const eckit::Exception& e) { + Log::debug() << "RadosEngine::canHandle: exception checking URI " << uri << ": " << e.what() + << std::endl; + return false; + } +} + std::vector RadosEngine::visitableLocations(const std::function& matches, const Config& config) const { diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 1a4409903..635bac871 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -13,14 +13,14 @@ #pragma once -#include "eckit/exception/Exceptions.h" -#include "eckit/filesystem/URI.h" -#include "eckit/io/rados/RadosKeyValue.h" +#include "fdb5/database/Engine.h" +#include "fdb5/fdb5_config.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/database/Engine.h" -#include "fdb5/fdb5_config.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" #include #include @@ -32,11 +32,11 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class RadosEngine : public fdb5::Engine { +class RadosEngine : public Engine { public: // methods - RadosEngine() {}; + RadosEngine() = default; static const char* typeName() { return "rados"; } @@ -44,20 +44,16 @@ class RadosEngine : public fdb5::Engine { std::string name() const override; - std::string dbType() const override { NOTIMP; }; - - eckit::URI location(const Key& key, const Config& config) const override { NOTIMP; }; + std::string dbType() const override { return typeName(); }; - bool canHandle(const eckit::URI&, const Config&) const override { NOTIMP; }; + eckit::URI location(const Key& key, const Config& config) const override; - // std::vector allLocations(const Key& key, const Config& config) const override { NOTIMP; }; + bool canHandle(const eckit::URI& uri, const Config& config) const override; std::vector visitableLocations(const Key& key, const Config& config) const override; std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; - // std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; }; - void print(std::ostream& out) const override { NOTIMP; }; private: // methods From 17a606359bae79021d9fe3464ad81bbeaaa4a9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 11:05:01 +0200 Subject: [PATCH 062/109] fix(ceph): remove single pool option --- src/fdb5/fdb5_config.h.in | 8 +- src/fdb5/rados/README | 36 ++--- src/fdb5/rados/RadosCatalogue.cc | 13 -- src/fdb5/rados/RadosCatalogueWriter.cc | 35 +--- src/fdb5/rados/RadosCommon.cc | 70 -------- src/fdb5/rados/RadosCommon.h | 14 -- src/fdb5/rados/RadosEngine.cc | 48 ------ src/fdb5/rados/RadosEngine.h | 14 -- src/fdb5/rados/RadosStore.cc | 202 ++++-------------------- tests/fdb/rados/test_rados_catalogue.cc | 79 --------- tests/fdb/rados/test_rados_store.cc | 103 ------------ 11 files changed, 48 insertions(+), 574 deletions(-) diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index d896550bc..82e6a19e0 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -1,15 +1,13 @@ #ifndef fdb5_fdb5_config_h #define fdb5_fdb5_config_h -#include "fdb5_ecbuild_config.h" // generated by ecbuild_generate_config_headers() - -#include "fdb5_version.h" // generated by ecbuild_generate_config_headers() +#include "fdb5_ecbuild_config.h" // generated by ecbuild_generate_config_headers() +#include "fdb5_version.h" // generated by ecbuild_generate_config_headers() // features #cmakedefine fdb5_HAVE_LUSTRE #cmakedefine fdb5_HAVE_RADOSFDB -#cmakedefine fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #cmakedefine fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD #cmakedefine fdb5_HAVE_RADOS_STORE_MULTIPART #cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH @@ -20,4 +18,4 @@ #cmakedefine fdb5_HAVE_DAOS_ADMIN #cmakedefine01 fdb5_HAVE_GRIB -#endif // fdb5_fdb5_config_h +#endif // fdb5_fdb5_config_h diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index a6c086bf8..d14f5c1d9 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -48,50 +48,38 @@ ctest -R rados_store cmake options: ============== -# single pool, multiple fields per obj +# multiple fields per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF -# pool per db, multiple fields per obj +# field per obj cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF - -# single pool, field per obj -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF - -# pool per db, field per obj -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \ + -DENABLE_RADOSFDB=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF -# single pool, multiple fields per obj, multipart (default) +# multiple fields per obj, multipart (default) cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF -# single pool, multiple fields per obj, persist on flush +# multiple fields per obj, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON -# single pool, field per obj, persist on flush +# field per obj, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON -# single pool, multiple fields per obj, multipart, persist on flush +# multiple fields per obj, multipart, persist on flush cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \ + -DENABLE_RADOSFDB=ON \ -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index b6595f76e..27f60df1b 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -60,13 +60,8 @@ RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& const fdb5::Config& config) : CatalogueImpl(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool = pool_; std::string nspace = db_namespace_; -#else - std::string pool = db_pool_; - std::string nspace = namespace_; -#endif // Read the real DB key into the DB base object try { @@ -189,15 +184,7 @@ bool RadosCatalogue::uriBelongs(const eckit::URI& uri) const { const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - return (uri.scheme() == type()) && (n >= 2) && (parts[0] == pool_) && (parts[1] == db_namespace_); - -#else - - return (uri.scheme() == type()) && (n >= 2) && (parts[0] == db_pool_) && (parts[1] == namespace_); - -#endif } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index e06c329b6..82fd01146 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -8,24 +8,17 @@ * does it submit to any jurisdiction. */ -#include -// #include -#include "eckit/io/FileHandle.h" -#include "eckit/io/MemoryHandle.h" -#include "eckit/serialisation/HandleStream.h" +#include "fdb5/rados/RadosCatalogueWriter.h" #include "fdb5/LibFdb5.h" +#include "fdb5/rados/RadosIndex.h" -// #include "fdb5/daos/DaosSession.h" -// #include "fdb5/daos/DaosName.h" +#include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" #include "eckit/io/rados/RadosException.h" #include "eckit/io/rados/RadosKeyValue.h" - -#include "fdb5/rados/RadosCatalogueWriter.h" -#include "fdb5/rados/RadosIndex.h" - -// using namespace eckit; +#include "eckit/serialisation/HandleStream.h" namespace fdb5 { @@ -38,13 +31,8 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c /// - daos_pool_connect /// - root cont open (daos_cont_open) /// - root cont create (daos_cont_create) -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string db_name = db_namespace_; ASSERT(root_kv_->nspace().pool().exists()); -#else - std::string db_name = db_pool_; - root_kv_->nspace().pool().ensureCreated(); -#endif /// @note: the DaosKeyValue constructor checks if the kv exists, which results in creation if not exists /// @note: performed RPCs: @@ -56,9 +44,6 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c if (!root_kv_->has(db_name)) { /// create catalogue kv -#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - db_kv_->nspace().pool().ensureCreated(); -#endif db_kv_->ensureCreated(); /// write schema under "schema" @@ -129,13 +114,8 @@ bool RadosCatalogueWriter::createIndex(const Key& /* idxKey */, size_t /* datumK bool RadosCatalogueWriter::selectIndex(const Key& key) { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool = pool_; std::string nspace = db_namespace_; -#else - std::string pool = db_pool_; - std::string nspace = namespace_; -#endif currentIndexKey_ = key; @@ -233,13 +213,8 @@ const Index& RadosCatalogueWriter::currentIndex() { void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool = pool_; std::string nspace = db_namespace_; -#else - std::string pool = db_pool_; - std::string nspace = namespace_; -#endif if (current_.null()) { ASSERT(!currentIndexKey_.empty()); diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index f8828f2c4..ef355a2ca 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -33,25 +33,12 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con std::vector valid{"catalogue", "store"}; ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - readConfig(config, component, true); db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); root_kv_.emplace(pool_, root_namespace_, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); - -#else - - readConfig(config, component, true); - - db_pool_ = pool_prefix_ + "_" + key.valuesToString(); - - root_kv_.emplace(root_pool_, namespace_, "main_kv"); - db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); - -#endif } RadosCommon::RadosCommon(const Config& config, const std::string& component, const eckit::URI& uri) { @@ -64,8 +51,6 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); ASSERT(parts.size() == 2 || parts.size() == 3); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - pool_ = parts[0]; db_namespace_ = parts[1]; @@ -73,29 +58,9 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con root_kv_.emplace(pool_, root_namespace_, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); - -#else - - db_pool_ = parts[0]; - namespace_ = parts[1]; - - readConfig(config, component, false); - - const auto poolParts = eckit::Tokenizer("_").tokenize(db_pool_); - ASSERT(poolParts.size() > 1); - pool_prefix_ = poolParts[0]; - - root_kv_.emplace(root_pool_, namespace_, "main_kv"); - db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); - -#endif } -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL void RadosCommon::readConfig(const Config& config, const std::string& component, bool readPool) { -#else -void RadosCommon::readConfig(const Config& config, const std::string& component, bool readNamespace) { -#endif eckit::LocalConfiguration c{}; @@ -113,8 +78,6 @@ void RadosCommon::readConfig(const Config& config, const std::string& component, c = toupper(c); } -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - if (readPool) { pool_ = "default"; } @@ -144,39 +107,6 @@ void RadosCommon::readConfig(const Config& config, const std::string& component, ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); -#else - - if (readNamespace) { - namespace_ = "default"; - } - root_pool_ = "root"; - - if (readNamespace) { - namespace_ = c.getString("namespace", namespace_); - } - if (c.has(component)) { - namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); - } - root_pool_ = c.getString("root_pool", root_pool_); - if (c.has(component)) { - root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); - } - - if (readNamespace) { - namespace_ = eckit::Resource( - "fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); - } - root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", - root_pool_); - - pool_prefix_ = c.getString("pool_prefix", pool_prefix_); - if (c.has(component)) { - pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); - } - ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); - -#endif - // if (c.has("client")) // DaosManager::instance().configure(c.getSubConfiguration("client")); } diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 7ead50faf..4d07e56b3 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -40,23 +40,13 @@ class RadosCommon { private: // methods -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL void readConfig(const Config& config, const std::string& component, bool readPool); -#else - void readConfig(const Config& config, const std::string& component, bool readNamespace); -#endif protected: // members -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string pool_; std::string root_namespace_; std::string db_namespace_; -#else - std::string root_pool_; - std::string db_pool_; - std::string namespace_; -#endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) std::optional root_kv_; @@ -70,11 +60,7 @@ class RadosCommon { private: // members -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string nspace_prefix_; -#else - std::string pool_prefix_; -#endif }; } // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index f19998ccf..9e2157303 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -36,13 +36,8 @@ eckit::URI RadosEngine::location(const Key& key, const Config& config) const { readConfig(config, "catalogue", true); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL const std::string db_namespace = nspace_prefix_ + "_" + key.valuesToString(); return eckit::RadosKeyValue{pool_, db_namespace, "catalogue_kv"}.uri(); -#else - const std::string db_pool = pool_prefix_ + "_" + key.valuesToString(); - return eckit::RadosKeyValue{db_pool, namespace_, "catalogue_kv"}.uri(); -#endif } bool RadosEngine::canHandle(const eckit::URI& uri, const Config&) const { @@ -77,11 +72,7 @@ std::vector RadosEngine::visitableLocations(const std::function res{}; @@ -127,11 +118,7 @@ std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest return visitableLocations([&request](const fdb5::Key& dbKey) { return dbKey.partialMatch(request); }, config); } -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL void RadosEngine::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const { -#else -void RadosEngine::readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) const { -#endif eckit::LocalConfiguration c{}; @@ -149,8 +136,6 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp c = toupper(c); } -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - if (readPool) { pool_ = "default"; } @@ -179,39 +164,6 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp } ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); - -#else - - if (readNamespace) { - namespace_ = "default"; - } - root_pool_ = "root"; - - if (readNamespace) { - namespace_ = c.getString("namespace", namespace_); - } - if (c.has(component)) { - namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); - } - root_pool_ = c.getString("root_pool", root_pool_); - if (c.has(component)) { - root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); - } - - if (readNamespace) { - namespace_ = eckit::Resource( - "fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); - } - root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", - root_pool_); - - pool_prefix_ = c.getString("pool_prefix", pool_prefix_); - if (c.has(component)) { - pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); - } - ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); - -#endif } static EngineBuilder rados_builder; diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 635bac871..c98b62d5d 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -63,23 +63,13 @@ class RadosEngine : public Engine { std::vector visitableLocations(const std::function& matches, const Config& config) const; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL void readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const; -#else - void readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) const; -#endif protected: // members -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL mutable std::string pool_; mutable std::string root_namespace_; // std::string db_namespace_; -#else - mutable std::string root_pool_; - // std::string db_pool_; - mutable std::string namespace_; -#endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) mutable std::optional root_kv_; @@ -93,11 +83,7 @@ class RadosEngine : public Engine { private: // members -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL mutable std::string nspace_prefix_; -#else - mutable std::string pool_prefix_; -#endif }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index f17d71575..3ab116f5c 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -10,27 +10,40 @@ #include "fdb5/rados/RadosStore.h" -#include "eckit/config/Resource.h" -#include "eckit/io/EmptyHandle.h" +#include "fdb5/LibFdb5.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Store.h" +#include "fdb5/database/WipeState.h" +#include "fdb5/rados/RadosCommon.h" +#include "fdb5/rados/RadosFieldLocation.h" +#include "fdb5/rules/Rule.h" + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" #include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosObject.h" #include "eckit/io/rados/RadosPool.h" -#include "eckit/log/Bytes.h" #include "eckit/log/TimeStamp.h" -#include "eckit/log/Timer.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" #include "eckit/utils/MD5.h" #include "eckit/utils/Tokenizer.h" -#include "fdb5/LibFdb5.h" -#include "fdb5/database/FieldLocation.h" -#include "fdb5/database/WipeState.h" -#include "fdb5/rados/RadosFieldLocation.h" -#include "fdb5/rules/Rule.h" - #include +#include +#include +#include +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -53,30 +66,14 @@ RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : eckit::URI RadosStore::uri() const { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - return eckit::RadosNamespace(pool_, db_namespace_).uri(); - -#else - - return eckit::RadosPool(db_pool_).uri(); - -#endif } eckit::URI RadosStore::uri(const eckit::URI& dataURI) { eckit::RadosObject o{dataURI}; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - return o.nspace().uri(); - -#else - - return o.nspace().pool().uri(); - -#endif } bool RadosStore::uriBelongs(const eckit::URI& uri) const { @@ -84,17 +81,8 @@ bool RadosStore::uriBelongs(const eckit::URI& uri) const { const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - ASSERT(n == 2 || n == 3); return ((uri.scheme() == type()) && (parts[0] == pool_) && (parts[1] == db_namespace_)); - -#else - - ASSERT(n == 2 || n == 3); - return ((uri.scheme() == type()) && (parts[0] == db_pool_) && (parts[1] == namespace_)); - -#endif } bool RadosStore::uriExists(const eckit::URI& uri) const { @@ -106,8 +94,6 @@ bool RadosStore::uriExists(const eckit::URI& uri) const { ASSERT(uri.scheme() == type()); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - ASSERT(n == 2 || n == 3); ASSERT(parts[0] == pool_); ASSERT(parts[1] == db_namespace_); @@ -116,20 +102,6 @@ bool RadosStore::uriExists(const eckit::URI& uri) const { return eckit::RadosNamespace(uri).exists(); } -#else - - ASSERT(n == 1 || n == 3); - ASSERT(parts[0] == db_pool_); - if (n > 1) { - ASSERT(parts[1] == namespace_); - } - - if (n == 1) { - return eckit::RadosPool(uri).exists(); - } - -#endif - return eckit::RadosObject(uri).exists(); } @@ -137,16 +109,8 @@ std::set RadosStore::collocatedDataURIs() const { std::set store_unit_uris; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - eckit::RadosNamespace n{pool_, db_namespace_}; -#else - - eckit::RadosNamespace n{db_pool_, namespace_}; - -#endif - if (!n.exists()) { return store_unit_uris; } @@ -173,7 +137,7 @@ std::set RadosStore::asCollocatedDataURIs(const std::set /// @note: this is only uniquefying the input uris (coming from an index) /// in case theres any duplicate. - for (auto& uri : uris) { + for (const auto& uri : uris) { res.insert(uri); } @@ -182,15 +146,7 @@ std::set RadosStore::asCollocatedDataURIs(const std::set bool RadosStore::exists() const { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - return eckit::RadosNamespace(pool_, db_namespace_).exists(); - -#else - - return eckit::RadosNamespace(db_pool_, namespace_).exists(); - -#endif } /// @todo: never used in actual fdb-read? @@ -208,18 +164,6 @@ std::unique_ptr RadosStore::archive(const Key& key, const v /// @note: generate unique object name starting by indexkey_ eckit::RadosObject o = generateDataObject(key); -#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - - /// @todo: ensure pool if not yet seen by this process - static std::set knownPools; - const eckit::RadosPool& p = o.nspace().pool(); - if (knownPools.find(p.name()) == knownPools.end()) { - p.ensureCreated(); - knownPools.insert(p.name()); - } - -#endif - #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH eckit::DataHandle* h = o.asyncDataHandle(); ASSERT(handles_.size() < maxHandleBuffSize_); @@ -239,21 +183,8 @@ std::unique_ptr RadosStore::archive(const Key& key, const v #else - /// @note: get or generate unique key name const eckit::RadosObject& o = getDataObject(key); -#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - - /// @todo: ensure pool if not yet seen by this process - static std::set knownPools; - const eckit::RadosPool& p = o.nspace().pool(); - if (knownPools.find(p.name()) == knownPools.end()) { - p.ensureCreated(); - knownPools.insert(p.name()); - } - -#endif - eckit::DataHandle& h = getDataHandle(key, o); eckit::Offset offset{h.position()}; @@ -262,7 +193,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const v ASSERT(len == length); - return std::unique_ptr(new RadosFieldLocation(o.uri(), offset, length, fdb5::Key{})); + return std::make_unique(o.uri(), offset, length, fdb5::Key{}); #endif } @@ -335,8 +266,6 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - ASSERT(n == 2 || n == 3); ASSERT(parts[0] == pool_); @@ -370,64 +299,17 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost } #endif } - -#else - - ASSERT(n == 1 || n == 3); - - ASSERT(parts[0] == db_pool_); - - if (n == 1) { // pool - - eckit::RadosPool pool{uri}; - - logVerbose << "destroy Rados pool: "; - logAlways << pool.name() << std::endl; - - if (doit) { - pool.ensureDestroyed(); - } - } - else { // object - - ASSERT(parts[1] == namespace_); - - eckit::RadosObject obj{uri}; - - logVerbose << "destroy Rados object: "; - logAlways << obj.str() << std::endl; - -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - if (doit) { - obj.ensureAllDestroyed(); - } -#else - if (doit) { - obj.ensureDestroyed(); - } -#endif - } - -#endif } void RadosStore::print(std::ostream& out) const { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - out << "RadosStore(" << pool_ << "/" << db_namespace_ << ")"; - -#else - - out << "RadosStore(" << db_pool_ << "/" << namespace_ << ")"; - -#endif } //---------------------------------------------------------------------------------------------------------------------- -/// @note: for SINGLE_POOL the database maps to a Rados namespace, otherwise to a Rados pool. -/// Only the namespace/pool holding this database's objects is ever touched here. +/// @note: the database maps to a Rados namespace. Only the namespace holding this database's +/// objects is ever touched here. void RadosStore::finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) { /// @note: doit and unsafeWipeAll do not affect the preparation of a Rados store wipe. @@ -451,12 +333,8 @@ void RadosStore::finaliseWipeState(StoreWipeState& storeState, bool doit, bool u return; } - // Full wipe: scan the database namespace/pool for any objects unaccounted for by the catalogue. -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + // Full wipe: scan the database namespace for any objects unaccounted for by the catalogue. eckit::RadosNamespace db{pool_, db_namespace_}; -#else - eckit::RadosNamespace db{db_pool_, namespace_}; -#endif if (!db.exists()) { return; @@ -507,11 +385,7 @@ void RadosStore::doWipeEmptyDatabase() const { return; } -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL eckit::RadosNamespace db{pool_, db_namespace_}; -#else - eckit::RadosNamespace db{db_pool_, namespace_}; -#endif if (db.exists()) { remove(db.uri(), std::cout, std::cout, true); @@ -525,11 +399,7 @@ bool RadosStore::doUnsafeFullWipe() const { /// determine whether a catalogue exists here. if (db_kv_ && (!db_kv_->exists() || !db_kv_->has("key"))) { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL eckit::RadosNamespace db{pool_, db_namespace_}; -#else - eckit::RadosNamespace db{db_pool_, namespace_}; -#endif if (db.exists()) { remove(db.uri(), std::cout, std::cout, true); @@ -560,8 +430,6 @@ eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { eckit::MD5 md5(name); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - #ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD return eckit::RadosObject{pool_, db_namespace_, md5.digest()}; @@ -570,20 +438,6 @@ eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { return eckit::RadosObject{pool_, db_namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; -#endif - -#else - -#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - - return eckit::RadosObject{db_pool_, namespace_, md5.digest()}; - -#else - - return eckit::RadosObject{db_pool_, namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; - -#endif - #endif } diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index eaf21db4a..bc4f4551b 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -73,17 +73,6 @@ void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { } } -#ifdef fdb5_HAVE_RADOS_ADMIN -void ensureClean(const std::string& prefix) { - ASSERT(prefix.length() > 3); - for (const std::string& name : eckit::RadosCluster::instance().listPools()) { - if (name.rfind(prefix, 0) == 0) { - eckit::RadosPool{name}.destroy(); - } - } -} -#endif - } // namespace // temporary schema,spaces,root files common to all DAOS Catalogue tests @@ -108,12 +97,6 @@ namespace test { CASE("Setup") { -#if !defined(fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL) && !defined(fdb5_HAVE_RADOS_ADMIN) - throw eckit::Exception( - "RadosStore unit tests require Rados admin permissions to create pools if " - "RADOS_BACKENDS_SINGLE_POOL=OFF, and require enabling RADOS_ADMIN=ON."); -#endif - // ensure fdb root directory exists. If not, then that root is // registered as non existing and Catalogue/Store tests fail. if (catalogue_tests_tmp_root().exists()) { @@ -151,7 +134,6 @@ CASE("Setup") { CASE("RadosCatalogue tests") { std::string test_id = "test-catalogue"; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); @@ -162,14 +144,9 @@ CASE("RadosCatalogue tests") { EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); #endif -#else - std::string prefix = test_id; - ensureClean(prefix); -#endif SECTION("DaosCatalogue archive (index) and retrieve without a Store") { -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string config_str{ "spaces:\n" "- roots:\n" @@ -189,25 +166,6 @@ CASE("RadosCatalogue tests") { "_root\n" " namespace_prefix: " + test_id + "\n"}; -#else - std::string config_str{ - "spaces:\n" - "- roots:\n" - " - path: " + - catalogue_tests_tmp_root().asString() + - "\n" - "schema : " + - schema_file().path() + - "\n" - "rados:\n" - " catalogue:\n" - " namespace: default\n" - " root_pool: " + - prefix + - "_root\n" - " pool_prefix: " + - prefix + "\n"}; -#endif fdb5::Config config{YAMLConfiguration(config_str)}; fdb5::Schema schema{schema_file()}; @@ -290,7 +248,6 @@ CASE("RadosCatalogue tests") { // FDB configuration -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL std::string config_str{ "spaces:\n" "- roots:\n" @@ -309,24 +266,6 @@ CASE("RadosCatalogue tests") { "_root\n" " namespace_prefix: " + test_id + "\n"}; -#else - std::string config_str{ - "spaces:\n" - "- roots:\n" - " - path: " + - catalogue_tests_tmp_root().asString() + - "\n" - "schema : " + - schema_file().path() + - "\n" - "rados:\n" - " namespace: default\n" - " root_pool: " + - prefix + - "_root\n" - " pool_prefix: " + - prefix + "\n"}; -#endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -513,16 +452,12 @@ CASE("RadosCatalogue tests") { /// @note: earlier sections share the same catalogue namespaces/pool; reset them so this /// section starts from a clean, empty catalogue (it asserts the FDB is initially empty). -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); #else ensureCleanNamespaces(pool, test_id); #endif -#else - ensureClean(prefix); -#endif // FDB configuration @@ -540,7 +475,6 @@ CASE("RadosCatalogue tests") { "store: rados\n" "rados:\n"}; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL config_str += " pool: " + pool + "\n" " root_namespace: " + @@ -548,15 +482,6 @@ CASE("RadosCatalogue tests") { "_root\n" " namespace_prefix: " + test_id + "\n"; -#else - config_str += - " namespace: default\n" - " root_pool: " + - prefix + - "_root\n" - " pool_prefix: " + - prefix + "\n"; -#endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -888,15 +813,11 @@ CASE("RadosCatalogue tests") { // teardown rados -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN eckit::RadosPool{pool}.ensureDestroyed(); #else ensureCleanNamespaces(pool, test_id); #endif -#else - ensureClean(prefix); -#endif } } // namespace test diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 6610bf6a8..430bf423d 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -121,12 +121,6 @@ namespace test { CASE("Setup") { -#if !defined(fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL) && !defined(fdb5_HAVE_RADOS_ADMIN) - throw eckit::Exception( - "RadosStore unit tests require Rados admin permissions to create pools if " - "RADOS_BACKENDS_SINGLE_POOL=OFF, and require enabling RADOS_ADMIN=ON."); -#endif - // ensure fdb root directory exists. If not, then that root is // registered as non existing and Store tests fail. if (store_tests_tmp_root().exists()) { @@ -157,7 +151,6 @@ CASE("RadosStore tests") { SECTION("archive and retrieve") { std::string test_id = "test-store1"; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); @@ -184,23 +177,6 @@ CASE("RadosStore tests") { "_root\n" " namespace_prefix: " + test_id + "\n"}; -#else - std::string prefix = test_id; - ensureClean(prefix); - std::string config_str{ - "spaces:\n" - "- roots:\n" - " - path: " + - store_tests_tmp_root().asString() + - "\n" - "rados:\n" - " namespace: default\n" - " root_pool: " + - prefix + - "_root\n" - " pool_prefix: " + - prefix + "\n"}; -#endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -238,7 +214,6 @@ CASE("RadosStore tests") { EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); // remove -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL eckit::RadosObject field_name{field.location().uri()}; eckit::RadosNamespace store_name = field_name.nspace(); eckit::URI store_uri(store_name.uri()); @@ -248,23 +223,11 @@ CASE("RadosStore tests") { store.remove(store_uri, out, out, true); EXPECT_NOT(field_name.exists()); EXPECT(store_name.listObjects().size() == 0); -#else - eckit::RadosObject field_name{field.location().uri()}; - eckit::RadosPool store_name = field_name.nspace().pool(); - eckit::URI store_uri(store_name.uri()); - std::ostream out(std::cout.rdbuf()); - store.remove(store_uri, out, out, false); - EXPECT(field_name.exists()); - store.remove(store_uri, out, out, true); - EXPECT_NOT(field_name.exists()); - EXPECT_NOT(store_name.exists()); -#endif } SECTION("with POSIX Catalogue") { std::string test_id = "test-store2"; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); @@ -294,26 +257,6 @@ CASE("RadosStore tests") { "_root\n" " namespace_prefix: " + test_id + "\n"}; -#else - std::string prefix = test_id; - ensureClean(prefix); - std::string config_str{ - "spaces:\n" - "- roots:\n" - " - path: " + - store_tests_tmp_root().asString() + - "\n" - "schema : " + - schema_file().path() + - "\n" - "rados:\n" - " namespace: default\n" - " root_pool: " + - prefix + - "_root\n" - " pool_prefix: " + - prefix + "\n"}; -#endif fdb5::Config config{YAMLConfiguration(config_str)}; @@ -379,7 +322,6 @@ CASE("RadosStore tests") { EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); // remove data -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL eckit::RadosObject field_name{field.location().uri()}; eckit::RadosNamespace store_name{field_name.nspace()}; eckit::URI store_uri(store_name.uri()); @@ -389,17 +331,6 @@ CASE("RadosStore tests") { store.remove(store_uri, out, out, true); EXPECT_NOT(field_name.exists()); EXPECT(store_name.listObjects().size() == 0); -#else - eckit::RadosObject field_name{field.location().uri()}; - eckit::RadosPool store_name = field_name.nspace().pool(); - eckit::URI store_uri(store_name.uri()); - std::ostream out(std::cout.rdbuf()); - store.remove(store_uri, out, out, false); - EXPECT(field_name.exists()); - store.remove(store_uri, out, out, true); - EXPECT_NOT(field_name.exists()); - EXPECT_NOT(store_name.exists()); -#endif // deindex data @@ -422,7 +353,6 @@ CASE("RadosStore tests") { deldir(store_tests_tmp_root()); } store_tests_tmp_root().mkdir(); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); @@ -433,10 +363,6 @@ CASE("RadosStore tests") { EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); #endif -#else - std::string prefix = test_id; - ensureClean(prefix); -#endif std::string config_str{ "spaces:\n" @@ -452,23 +378,12 @@ CASE("RadosStore tests") { "store: rados\n" "rados:\n"}; -#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += - " namespace: default\n" - " root_pool: " + - prefix + - "_root\n" - " pool_prefix: " + - prefix + "\n"; -#endif - #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxPartSize: 16\n"; #endif config_str += " store:\n"; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL config_str += " pool: " + pool + "\n" " root_namespace: " + @@ -476,7 +391,6 @@ CASE("RadosStore tests") { "_root\n" " namespace_prefix: " + test_id + "\n"; -#endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) @@ -623,7 +537,6 @@ CASE("RadosStore tests") { deldir(store_tests_tmp_root()); } store_tests_tmp_root().mkdir(); -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); @@ -634,10 +547,6 @@ CASE("RadosStore tests") { EXPECT(pool.length() > 0); ensureCleanNamespaces(pool, test_id); #endif -#else - std::string prefix = test_id; - ensureClean(prefix); -#endif std::string config_str{ "spaces:\n" @@ -653,23 +562,12 @@ CASE("RadosStore tests") { "store: rados\n" "rados:\n"}; -#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += - " namespace: default\n" - " root_pool: " + - prefix + - "_root\n" - " pool_prefix: " + - prefix + "\n"; -#endif - #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) config_str += " maxPartSize: 16\n"; #endif config_str += " store:\n"; -#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL config_str += " pool: " + pool + "\n" " root_namespace: " + @@ -677,7 +575,6 @@ CASE("RadosStore tests") { "_root\n" " namespace_prefix: " + test_id + "\n"; -#endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) #if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) From e85eb1ecb121f9b0b5e330d1fe027d2d6dbbfb23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 12:08:13 +0200 Subject: [PATCH 063/109] fix(ceph): remove option obj per field --- src/fdb5/fdb5_config.h.in | 2 - src/fdb5/rados/README | 34 +------ src/fdb5/rados/RadosCatalogueReader.cc | 29 +++--- src/fdb5/rados/RadosCatalogueWriter.cc | 22 +---- src/fdb5/rados/RadosCommon.h | 13 +-- src/fdb5/rados/RadosEngine.h | 5 - src/fdb5/rados/RadosFieldLocation.cc | 2 +- src/fdb5/rados/RadosIndex.cc | 53 ++-------- src/fdb5/rados/RadosIndex.h | 26 ++--- src/fdb5/rados/RadosIndexLocation.cc | 4 - src/fdb5/rados/RadosIndexLocation.h | 15 +-- src/fdb5/rados/RadosStore.cc | 126 ++---------------------- src/fdb5/rados/RadosStore.h | 22 +---- tests/fdb/rados/test_rados_catalogue.cc | 2 +- tests/fdb/rados/test_rados_store.cc | 44 ++------- 15 files changed, 58 insertions(+), 341 deletions(-) diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index 82e6a19e0..fb54218cf 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -8,9 +8,7 @@ #cmakedefine fdb5_HAVE_LUSTRE #cmakedefine fdb5_HAVE_RADOSFDB -#cmakedefine fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD #cmakedefine fdb5_HAVE_RADOS_STORE_MULTIPART -#cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH #cmakedefine fdb5_HAVE_RADOS_ADMIN #cmakedefine fdb5_HAVE_TOCFDB #cmakedefine fdb5_HAVE_DUMMY_DAOS diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index d14f5c1d9..e4b037c2b 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -48,38 +48,12 @@ ctest -R rados_store cmake options: ============== -# multiple fields per obj +# multipart disabled cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF + -DENABLE_RADOS_STORE_MULTIPART=OFF -# field per obj +# multipart enabled (default) cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF - -# multiple fields per obj, multipart (default) -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF - -# multiple fields per obj, persist on flush -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON - -# field per obj, persist on flush -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON - -# multiple fields per obj, multipart, persist on flush -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \ - -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON + -DENABLE_RADOS_STORE_MULTIPART=ON diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index bc04ffc82..441095b76 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -10,17 +10,6 @@ #include "fdb5/rados/RadosCatalogueReader.h" -#include -#include -#include -#include -#include - -#include "eckit/exception/Exceptions.h" -#include "eckit/filesystem/URI.h" -#include "eckit/io/rados/RadosException.h" -#include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/log/Log.h" #include "fdb5/LibFdb5.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/database/Catalogue.h" @@ -31,6 +20,18 @@ #include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosIndex.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/Log.h" + +#include +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -81,13 +82,7 @@ bool RadosCatalogueReader::selectIndex(const Key& key) { } eckit::URI uri{std::string{n.begin(), std::next(n.begin(), res)}}; - // #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - // eckit::RadosPersistentKeyValue index_kv{uri, true}; - // #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - // eckit::RadosPersistentKeyValue index_kv{uri}; - // #else eckit::RadosKeyValue index_kv{uri}; - // #endif indexes_[key] = Index(new RadosIndex(key, index_kv, true)); diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index 82fd01146..cc6c39b3c 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -137,17 +137,7 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { res = db_kv_->get(key.valuesToString(), &n[0], idx_loc_max_len); indexes_[key] = Index(new fdb5::RadosIndex( - key, - // #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE - // eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), - // std::next(n.begin(), res)}}, true}, - // #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - // eckit::RadosPersistentKeyValue{eckit::URI{std::string{n.begin(), - // std::next(n.begin(), res)}}}, - // #else - eckit::RadosKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, - // #endif - false)); + key, eckit::RadosKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, false)); } catch (eckit::RadosEntityNotFoundException& e) { @@ -308,15 +298,7 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, } } -void RadosCatalogueWriter::flush(size_t archivedFields) { - -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j) { - j->second.flush(); - } - db_kv_->flush(); - root_kv_->flush(); -#endif +void RadosCatalogueWriter::flush(size_t /* archivedFields */) { if (!current_.null()) { current_ = Index(); diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 4d07e56b3..11aaa7985 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -13,14 +13,14 @@ #pragma once -#include "eckit/filesystem/URI.h" -#include "eckit/io/Length.h" -#include "eckit/io/rados/RadosKeyValue.h" - #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" #include "fdb5/fdb5_config.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/rados/RadosKeyValue.h" + #include #include @@ -48,13 +48,8 @@ class RadosCommon { std::string root_namespace_; std::string db_namespace_; -#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - std::optional root_kv_; - std::optional db_kv_; -#else std::optional root_kv_; std::optional db_kv_; -#endif eckit::Length maxPartSize_; diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index c98b62d5d..8a384c8ca 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -71,13 +71,8 @@ class RadosEngine : public Engine { mutable std::string root_namespace_; // std::string db_namespace_; -#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - mutable std::optional root_kv_; - // std::optional db_kv_; -#else mutable std::optional root_kv_; // std::optional db_kv_; -#endif // eckit::Length maxPartSize_; diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index d4549304b..8797cfd50 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -53,7 +53,7 @@ std::shared_ptr RadosFieldLocation::make_shared() const { eckit::DataHandle* RadosFieldLocation::dataHandle() const { -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) return eckit::RadosObject(uri_).multipartRangeReadHandle(offset(), length()); diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 8f1b85321..7fd94031d 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -10,6 +10,15 @@ #include "fdb5/rados/RadosIndex.h" +#include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldDetails.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCommon.h" +#include "fdb5/rados/RadosLazyFieldLocation.h" + #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/DataHandle.h" @@ -23,16 +32,6 @@ #include "eckit/serialisation/Reanimator.h" #include "eckit/utils/Tokenizer.h" -#include "fdb5/rados/RadosCommon.h" - -#include "fdb5/database/EntryVisitMechanism.h" -#include "fdb5/database/Field.h" -#include "fdb5/database/FieldDetails.h" -#include "fdb5/database/FieldLocation.h" -#include "fdb5/database/Index.h" -#include "fdb5/database/Key.h" -#include "fdb5/rados/RadosLazyFieldLocation.h" - #include // for PATH_MAX #include #include @@ -44,25 +43,6 @@ #include #include -// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -// eckit::RadosPersistentKeyValue buildIndexKvName(const fdb5::Key& key, const eckit::RadosNamespace& name) { -// #else -// eckit::RadosKeyValue buildIndexKvName(const fdb5::Key& key, const eckit::RadosNamespace& name) { -// #endif -/// create index kv -/// @todo: pass oclass from config -/// @todo: hash string into lower oid bits - -// #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE -// return eckit::RadosPersistentKeyValue{name.poolName(), name.containerName(), key.valuesToString(), true}; -// #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH -// return eckit::RadosPersistentKeyValue{name.poolName(), name.containerName(), key.valuesToString()}; -// #else -// return eckit::RadosKeyValue{name.poolName(), name.containerName(), key.valuesToString()}; -// #endif - -// } - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -96,11 +76,7 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : idx_kv_.put("key", h.data(), hs.bytesWritten()); } -// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -// RadosIndex::RadosIndex(const Key& key, const eckit::RadosPersistentKeyValue& name, bool readAxes) : -// #else RadosIndex::RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes) : - // #endif IndexBase(key, "radosKeyValue"), location_(name, 0), idx_kv_(name.uri()) { if (readAxes) { @@ -302,17 +278,6 @@ std::vector RadosIndex::dataURIs() const { return std::vector(res.begin(), res.end()); } -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH -void RadosIndex::flush() { - - for (auto& axis : axis_kvs_) { - axis.second.flush(); - } - - idx_kv_.flush(); -} -#endif - //----------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index f54c61a69..2299134e2 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -13,11 +13,6 @@ #pragma once -#include "eckit/exception/Exceptions.h" -#include "eckit/filesystem/URI.h" -#include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/io/rados/RadosNamespace.h" - #include "fdb5/database/EntryVisitMechanism.h" #include "fdb5/database/Field.h" #include "fdb5/database/Index.h" @@ -25,6 +20,11 @@ #include "fdb5/database/Key.h" #include "fdb5/rados/RadosIndexLocation.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" + #include #include #include @@ -42,18 +42,13 @@ class RadosIndex : public IndexBase { /// @note: creates a new index in DAOS, in the container pointed to by 'name' RadosIndex(const Key& key, const eckit::RadosNamespace& name); /// @note: used to represent and operate with an index which already exists in DAOS - // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - // RadosIndex(const Key& key, const eckit::RadosPersistentKeyValue& name, bool readAxes = true); - // #else RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes = true); - // #endif void flock() const override { NOTIMP; } void funlock() const override { NOTIMP; } /// @note: these methods are required for RadosCatalogueWriter to directly manipulate - /// idx_kv_ and axis_kvs_ within the RadosIndex. Upon flush, the index will flush all - /// operations performed on these kvs (if PERSIST_ON_FLUSH). + /// idx_kv_ and axis_kvs_ within the RadosIndex. void putAxisNames(const std::string& names); void putAxisValue(const std::string& axis, const std::string& value); @@ -74,11 +69,7 @@ class RadosIndex : public IndexBase { bool get(const Key& key, const Key& remapKey, Field& field) const override; void add(const Key& key, const Field& field) override; -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - void flush() override; -#else void flush() override { NOTIMP; } -#endif void encode(eckit::Stream& s, const int version) const override { NOTIMP; } void entries(EntryVisitor& visitor) const override; @@ -96,13 +87,8 @@ class RadosIndex : public IndexBase { fdb5::RadosIndexLocation location_; -#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - eckit::RadosAsyncKeyValue idx_kv_; - std::map axis_kvs_; -#else eckit::RadosKeyValue idx_kv_; std::map axis_kvs_; -#endif }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosIndexLocation.cc b/src/fdb5/rados/RadosIndexLocation.cc index 49329c756..932944ac3 100644 --- a/src/fdb5/rados/RadosIndexLocation.cc +++ b/src/fdb5/rados/RadosIndexLocation.cc @@ -14,11 +14,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -// #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -// RadosIndexLocation::RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset) : name_(name), -// offset_(offset) {} #else RadosIndexLocation::RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset) : name_(name), offset_(offset) {} -// #endif void RadosIndexLocation::print(std::ostream& out) const { diff --git a/src/fdb5/rados/RadosIndexLocation.h b/src/fdb5/rados/RadosIndexLocation.h index 65393a2d1..142ed9ace 100644 --- a/src/fdb5/rados/RadosIndexLocation.h +++ b/src/fdb5/rados/RadosIndexLocation.h @@ -13,12 +13,11 @@ #pragma once +#include "fdb5/database/IndexLocation.h" + #include "eckit/exception/Exceptions.h" -#include "eckit/io/rados/RadosAsyncKeyValue.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "fdb5/database/IndexLocation.h" - namespace fdb5 { @@ -28,15 +27,9 @@ class RadosIndexLocation : public IndexLocation { public: // methods - // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - // RadosIndexLocation(const eckit::RadosPersistentKeyValue& name, off_t offset); - - // const eckit::RadosPersistentKeyValue& radosName() const { return name_; }; - // #else RadosIndexLocation(const eckit::RadosKeyValue& name, off_t offset); const eckit::RadosKeyValue& radosName() const { return name_; }; - // #endif eckit::URI uri() const override { return name_.uri(); } @@ -52,11 +45,7 @@ class RadosIndexLocation : public IndexLocation { private: // members - // #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - // eckit::RadosPersistentKeyValue name_; - // #else eckit::RadosKeyValue name_; - // #endif off_t offset_; }; diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 3ab116f5c..b8162652a 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -51,18 +51,12 @@ namespace fdb5 { static StoreBuilder builder("rados"); RadosStore::RadosStore(const Key& key, const Config& config) : - Store(), RadosCommon(config, "store", key), archivedFields_(0) { - - parseConfig(config); -} + Store(), RadosCommon(config, "store", key), archivedFields_(0) {} RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : RadosStore(key, config) {} RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : - Store(), RadosCommon(config, "store", uri), archivedFields_(0) { - - parseConfig(config); -} + Store(), RadosCommon(config, "store", uri), archivedFields_(0) {} eckit::URI RadosStore::uri() const { @@ -159,30 +153,6 @@ std::unique_ptr RadosStore::archive(const Key& key, const v archivedFields_++; -#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - - /// @note: generate unique object name starting by indexkey_ - eckit::RadosObject o = generateDataObject(key); - -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle* h = o.asyncDataHandle(); - ASSERT(handles_.size() < maxHandleBuffSize_); - handles_.push_back(h); -#else - std::unique_ptr h(o.dataHandle()); -#endif - - /// @todo: should throw here if object already exists - - h->openForWrite(length); - eckit::AutoClose closer(*h); - - h->write(data, length); - - return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key{})); - -#else - const eckit::RadosObject& o = getDataObject(key); eckit::DataHandle& h = getDataHandle(key, o); @@ -194,8 +164,6 @@ std::unique_ptr RadosStore::archive(const Key& key, const v ASSERT(len == length); return std::make_unique(o.uri(), offset, length, fdb5::Key{}); - -#endif } size_t RadosStore::flush() { @@ -204,34 +172,15 @@ size_t RadosStore::flush() { return 0; } -#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - for (const auto& h : handles_) { - h->flush(); - } -#else - // NOOP -#endif - -#else - #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - /// @note: needs to be called even if PERSIST_ON_FLUSH=OFF, as the - /// multipart handles need to persist the multipart attributes which - /// is performed in the multihandle flush. + /// @note: the multipart handles need to persist the multipart attributes which is + /// performed in the multihandle flush. flushDataHandles(); #else -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - flushDataHandles(); -#else // NOOP -#endif - -#endif #endif @@ -242,21 +191,7 @@ size_t RadosStore::flush() { void RadosStore::close() { -#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - for (const auto& h : handles_) { - h->close(); - } -#else - // NOOP -#endif - -#else - closeDataHandles(); - -#endif } void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { @@ -289,7 +224,7 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost logVerbose << "destroy Rados object: "; logAlways << obj.str() << std::endl; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) if (doit) { obj.ensureAllDestroyed(); } @@ -342,7 +277,7 @@ void RadosStore::finaliseWipeState(StoreWipeState& storeState, bool doit, bool u for (const auto& obj : db.listObjects()) { -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) // Parts belong to a main object and are removed together with it. if (obj.name().find(";part-") != std::string::npos) { continue; @@ -430,19 +365,9 @@ eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { eckit::MD5 md5(name); -#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - - return eckit::RadosObject{pool_, db_namespace_, md5.digest()}; - -#else - return eckit::RadosObject{pool_, db_namespace_, key.valuesToString() + "." + md5.digest() + ".data"}; - -#endif } -#ifndef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD - const eckit::RadosObject& RadosStore::getDataObject(const Key& key) const { auto it = dataObjects_.find(key); @@ -460,21 +385,9 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO } #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle* dh = name.asyncMultipartWriteHandle(maxPartSize_, maxAioBuffSize_, maxPartHandleBuffSize_); -#else eckit::DataHandle* dh = name.multipartWriteHandle(maxPartSize_); -#endif - -#else - -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - eckit::DataHandle* dh = name.asyncDataHandle(maxAioBuffSize_); #else eckit::DataHandle* dh = name.dataHandle(); -#endif - #endif ASSERT(dh); @@ -506,33 +419,6 @@ void RadosStore::flushDataHandles() { } } -#endif - -void RadosStore::parseConfig(const fdb5::Config& config) { - - eckit::LocalConfiguration rados{}, store_conf{}; - - if (config.has("rados")) { - rados = config.getSubConfiguration("rados"); - if (rados.has("store")) { - store_conf = rados.getSubConfiguration("store"); - } - } - -#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) && defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) - maxHandleBuffSize_ = store_conf.getInt("maxHandleBuffSize", 1024 * 1024); -#endif - -#if (!defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD)) && defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024); - maxPartHandleBuffSize_ = store_conf.getInt("maxPartHandleBuffSize", 1024); -#else - maxAioBuffSize_ = store_conf.getInt("maxAioBuffSize", 1024 * 1024); -#endif -#endif -} - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index e71b99431..62312cff4 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -14,12 +14,11 @@ #pragma once -#include "eckit/io/rados/RadosObject.h" - #include "fdb5/database/Store.h" +#include "fdb5/rados/RadosCommon.h" #include "fdb5/rules/Schema.h" -#include "fdb5/rados/RadosCommon.h" +#include "eckit/io/rados/RadosObject.h" namespace fdb5 { @@ -73,11 +72,8 @@ class RadosStore : public Store, public RadosCommon { void print(std::ostream& out) const override; - void parseConfig(const fdb5::Config& config); - eckit::RadosObject generateDataObject(const Key& key) const; -#ifndef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD const eckit::RadosObject& getDataObject(const Key& key) const; eckit::DataHandle& getDataHandle(const Key& key, const eckit::RadosObject& name); void closeDataHandles(); @@ -87,28 +83,14 @@ class RadosStore : public Store, public RadosCommon { typedef std::map HandleStore; typedef std::map ObjectStore; -#endif private: // members // mutable bool dirty_; size_t archivedFields_{0}; -#ifdef fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - std::vector handles_; - size_t maxHandleBuffSize_; -#endif -#else HandleStore handles_; mutable ObjectStore dataObjects_; -#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH - size_t maxAioBuffSize_; -#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - size_t maxPartHandleBuffSize_; -#endif -#endif -#endif }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index bc4f4551b..a03ceed18 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -317,7 +317,7 @@ CASE("RadosCatalogue tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) /// @note: with multipart enabled, the field spans potentially several objects and is /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. EXPECT(dynamic_cast(dh.get())); diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 430bf423d..16a30ca3c 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -34,11 +34,11 @@ // #include "eckit/io/s3/S3Client.h" // #include "eckit/io/s3/S3Session.h" // #include "eckit/io/s3/S3Credential.h" -#include "eckit/io/PartHandle.h" -#include "eckit/io/rados/RadosPartHandle.h" - #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rados/RadosStore.h" + +#include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosPartHandle.h" // #include "fdb5/daos/DaosException.h" using namespace eckit::testing; @@ -200,7 +200,7 @@ CASE("RadosStore tests") { fdb5::Field field(std::move(loc), std::time(nullptr)); std::cout << "Read location: " << field.location() << std::endl; std::unique_ptr dh(store.retrieve(field)); -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) /// @note: with multipart enabled, the field spans potentially several objects and is /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. EXPECT(dynamic_cast(dh.get())); @@ -308,7 +308,7 @@ CASE("RadosStore tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) /// @note: with multipart enabled, the field spans potentially several objects and is /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. EXPECT(dynamic_cast(dh.get())); @@ -378,7 +378,7 @@ CASE("RadosStore tests") { "store: rados\n" "rados:\n"}; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) config_str += " maxPartSize: 16\n"; #endif @@ -392,19 +392,6 @@ CASE("RadosStore tests") { " namespace_prefix: " + test_id + "\n"; -#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - config_str += " maxHandleBuffSize: 100\n"; -#else -#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - config_str += " maxAioBuffSize: 10\n"; - config_str += " maxPartHandleBuffSize: 10\n"; -#else - config_str += " maxAioBuffSize: 100\n"; -#endif -#endif -#endif - fdb5::Config config{YAMLConfiguration(config_str)}; // request @@ -440,7 +427,7 @@ CASE("RadosStore tests") { char data[] = "test123456"; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) /// @note: maxPartSize is set to 16, and four 10-byte fields are archived, spanning 3 objects for (int i = 0; i < 4; i++) { std::cout << "Archive field " << i << std::endl; @@ -456,7 +443,7 @@ CASE("RadosStore tests") { // retrieve data -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) for (int i = 0; i < 4; i++) { std::cout << "Retrieve field " << i << std::endl; fdb5::Key request_key_i( @@ -562,7 +549,7 @@ CASE("RadosStore tests") { "store: rados\n" "rados:\n"}; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && !defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) +#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) config_str += " maxPartSize: 16\n"; #endif @@ -576,19 +563,6 @@ CASE("RadosStore tests") { " namespace_prefix: " + test_id + "\n"; -#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) -#if defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) - config_str += " maxHandleBuffSize: 100\n"; -#else -#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - config_str += " maxAioBuffSize: 10\n"; - config_str += " maxPartHandleBuffSize: 10\n"; -#else - config_str += " maxAioBuffSize: 100\n"; -#endif -#endif -#endif - fdb5::Config config{YAMLConfiguration(config_str)}; // request From b9dbb8a9b4e5b0606aca16449c7e528af79fe908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 13:30:14 +0200 Subject: [PATCH 064/109] fix(ceph): default option multipart --- CMakeLists.txt | 16 ----------- src/fdb5/fdb5_config.h.in | 1 - src/fdb5/rados/README | 9 +------ src/fdb5/rados/RadosFieldLocation.cc | 8 ------ src/fdb5/rados/RadosStore.cc | 22 ---------------- tests/fdb/rados/test_rados_catalogue.cc | 9 ++----- tests/fdb/rados/test_rados_store.cc | 35 +++---------------------- 7 files changed, 7 insertions(+), 93 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e4ba5365d..3babc5728 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,22 +55,6 @@ ecbuild_add_option( FEATURE RADOSFDB # option defined in fdb5_config.h DEFAULT OFF DESCRIPTION "Ceph/Rados support for FDB Store" ) -ecbuild_add_option( FEATURE RADOS_BACKENDS_SINGLE_POOL - DEFAULT ON - DESCRIPTION "Use a single Rados pool with a namespace per database (ON) or a pool per database (OFF)" ) - -ecbuild_add_option( FEATURE RADOS_STORE_OBJ_PER_FIELD - DEFAULT OFF - DESCRIPTION "Use a Rados object per archived field (ON) or per process and collocation key (OFF)" ) - -ecbuild_add_option( FEATURE RADOS_STORE_MULTIPART - DEFAULT ON - DESCRIPTION "If RADOS_STORE_OBJ_PER_FIELD=OFF and the maximum object size is exceeded, use multiple Rados objects per process and collocation key (ON) or throw an exception (OFF)" ) - -ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_FLUSH - DEFAULT OFF - DESCRIPTION "Ensure writes/puts are persisted in Rados storage on flush rather than immediately." ) - ecbuild_add_option( FEATURE RADOS_ADMIN DEFAULT OFF DESCRIPTION "Have unit tests create pools automatically rather than using an existing pool specified in the FDB_RADOS_TEST_POOL cmake variable." ) diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index fb54218cf..c573351dd 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -8,7 +8,6 @@ #cmakedefine fdb5_HAVE_LUSTRE #cmakedefine fdb5_HAVE_RADOSFDB -#cmakedefine fdb5_HAVE_RADOS_STORE_MULTIPART #cmakedefine fdb5_HAVE_RADOS_ADMIN #cmakedefine fdb5_HAVE_TOCFDB #cmakedefine fdb5_HAVE_DUMMY_DAOS diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index e4b037c2b..3eeeaa7f8 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -48,12 +48,5 @@ ctest -R rados_store cmake options: ============== -# multipart disabled cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_MULTIPART=OFF - -# multipart enabled (default) -cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \ - -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_STORE_MULTIPART=ON + -DENABLE_RADOSFDB=ON diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index 8797cfd50..d0b6ff80d 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -53,15 +53,7 @@ std::shared_ptr RadosFieldLocation::make_shared() const { eckit::DataHandle* RadosFieldLocation::dataHandle() const { -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) - return eckit::RadosObject(uri_).multipartRangeReadHandle(offset(), length()); - -#else - - return eckit::RadosObject(uri_).rangeReadHandle(offset(), length()); - -#endif } // eckit::DataHandle *RadosFieldLocation::dataHandle(const Key& remapKey) const { diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index b8162652a..160dae454 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -113,11 +113,9 @@ std::set RadosStore::collocatedDataURIs() const { /// be done here to discriminate store objects from catalogue objects for (const auto& obj : n.listObjects()) { -#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART if (obj.name().find(";part-") != std::string::npos) { continue; } -#endif store_unit_uris.insert(obj.uri()); } @@ -172,18 +170,10 @@ size_t RadosStore::flush() { return 0; } -#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART - /// @note: the multipart handles need to persist the multipart attributes which is /// performed in the multihandle flush. flushDataHandles(); -#else - - // NOOP - -#endif - size_t out = archivedFields_; archivedFields_ = 0; return out; @@ -224,15 +214,9 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost logVerbose << "destroy Rados object: "; logAlways << obj.str() << std::endl; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) if (doit) { obj.ensureAllDestroyed(); } -#else - if (doit) { - obj.ensureDestroyed(); - } -#endif } } @@ -277,12 +261,10 @@ void RadosStore::finaliseWipeState(StoreWipeState& storeState, bool doit, bool u for (const auto& obj : db.listObjects()) { -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) // Parts belong to a main object and are removed together with it. if (obj.name().find(";part-") != std::string::npos) { continue; } -#endif const eckit::URI uri = obj.uri(); if (dataURIs.find(uri) == dataURIs.end() && safeURIs.find(uri) == safeURIs.end()) { @@ -384,11 +366,7 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO return *(j->second); } -#ifdef fdb5_HAVE_RADOS_STORE_MULTIPART eckit::DataHandle* dh = name.multipartWriteHandle(maxPartSize_); -#else - eckit::DataHandle* dh = name.dataHandle(); -#endif ASSERT(dh); diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index a03ceed18..dbe01e26c 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -21,7 +21,6 @@ #include "eckit/config/YAMLConfiguration.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/PartHandle.h" -#include "eckit/io/rados/RadosPartHandle.h" // #include "metkit/mars/MarsRequest.h" @@ -317,13 +316,9 @@ CASE("RadosCatalogue tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) - /// @note: with multipart enabled, the field spans potentially several objects and is - /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. + /// @note: the field spans potentially several objects and is returned as an + /// eckit::PartHandle wrapping a RadosMultiObjReadHandle. EXPECT(dynamic_cast(dh.get())); -#else - EXPECT(dynamic_cast(dh.get())); -#endif eckit::MemoryHandle mh; dh->copyTo(mh); diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 16a30ca3c..0b4b66072 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -38,7 +38,6 @@ #include "fdb5/rados/RadosStore.h" #include "eckit/io/PartHandle.h" -#include "eckit/io/rados/RadosPartHandle.h" // #include "fdb5/daos/DaosException.h" using namespace eckit::testing; @@ -200,13 +199,9 @@ CASE("RadosStore tests") { fdb5::Field field(std::move(loc), std::time(nullptr)); std::cout << "Read location: " << field.location() << std::endl; std::unique_ptr dh(store.retrieve(field)); -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) - /// @note: with multipart enabled, the field spans potentially several objects and is - /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. + /// @note: the field spans potentially several objects and is returned as an + /// eckit::PartHandle wrapping a RadosMultiObjReadHandle. EXPECT(dynamic_cast(dh.get())); -#else - EXPECT(dynamic_cast(dh.get())); -#endif eckit::MemoryHandle mh; dh->copyTo(mh); @@ -308,13 +303,9 @@ CASE("RadosStore tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) - /// @note: with multipart enabled, the field spans potentially several objects and is - /// returned as an eckit::PartHandle wrapping a RadosMultiObjReadHandle. + /// @note: the field spans potentially several objects and is returned as an + /// eckit::PartHandle wrapping a RadosMultiObjReadHandle. EXPECT(dynamic_cast(dh.get())); -#else - EXPECT(dynamic_cast(dh.get())); -#endif eckit::MemoryHandle mh; dh->copyTo(mh); @@ -378,9 +369,7 @@ CASE("RadosStore tests") { "store: rados\n" "rados:\n"}; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) config_str += " maxPartSize: 16\n"; -#endif config_str += " store:\n"; @@ -427,7 +416,6 @@ CASE("RadosStore tests") { char data[] = "test123456"; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) /// @note: maxPartSize is set to 16, and four 10-byte fields are archived, spanning 3 objects for (int i = 0; i < 4; i++) { std::cout << "Archive field " << i << std::endl; @@ -435,15 +423,11 @@ CASE("RadosStore tests") { {{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", std::to_string(6 + i)}}); fdb.archive(request_key_i, data, sizeof(data)); } -#else - fdb.archive(request_key, data, sizeof(data)); -#endif fdb.flush(); // retrieve data -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) for (int i = 0; i < 4; i++) { std::cout << "Retrieve field " << i << std::endl; fdb5::Key request_key_i( @@ -456,15 +440,6 @@ CASE("RadosStore tests") { EXPECT(mh.size() == eckit::Length(sizeof(data))); EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); } -#else - metkit::mars::MarsRequest r = request_key.request("retrieve"); - std::unique_ptr dh(fdb.retrieve(r)); - - eckit::MemoryHandle mh; - dh->copyTo(mh); - EXPECT(mh.size() == eckit::Length(sizeof(data))); - EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); -#endif // wipe data @@ -549,9 +524,7 @@ CASE("RadosStore tests") { "store: rados\n" "rados:\n"}; -#if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) config_str += " maxPartSize: 16\n"; -#endif config_str += " store:\n"; From b05d2e50f3253a26a63e4d54c2577609d38a666c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 14:53:57 +0200 Subject: [PATCH 065/109] refactor(ceph): rados_admin --- .github/workflows/ci-rados.yml | 4 ++-- CMakeLists.txt | 2 +- src/fdb5/fdb5_config.h.in | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index 10b5441bc..dd500d9d0 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -158,7 +158,7 @@ jobs: ${CMAKE_FLAGS} \ -DENABLE_MPI=ON \ -DENABLE_RADOS=ON \ - -DENABLE_RADOS_ADMIN=OFF + -DENABLE_RADOS_TESTS_MANAGE_POOLS=OFF cmake --build build/eckit --parallel cmake --install build/eckit @@ -194,7 +194,7 @@ jobs: -DCMAKE_PREFIX_PATH="${INSTALL_PREFIX}" \ ${CMAKE_FLAGS} \ -DENABLE_RADOSFDB=ON \ - -DENABLE_RADOS_ADMIN=OFF \ + -DENABLE_RADOS_TESTS_MANAGE_POOLS=OFF \ -DFDB_RADOS_TEST_POOL="${FDB_RADOS_TEST_POOL}" cmake --build build/fdb --parallel diff --git a/CMakeLists.txt b/CMakeLists.txt index 3babc5728..5f305de3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,7 @@ ecbuild_add_option( FEATURE RADOSFDB # option defined in fdb5_config.h DEFAULT OFF DESCRIPTION "Ceph/Rados support for FDB Store" ) -ecbuild_add_option( FEATURE RADOS_ADMIN +ecbuild_add_option( FEATURE RADOS_TESTS_MANAGE_POOLS DEFAULT OFF DESCRIPTION "Have unit tests create pools automatically rather than using an existing pool specified in the FDB_RADOS_TEST_POOL cmake variable." ) diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index c573351dd..123daa43f 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -8,7 +8,7 @@ #cmakedefine fdb5_HAVE_LUSTRE #cmakedefine fdb5_HAVE_RADOSFDB -#cmakedefine fdb5_HAVE_RADOS_ADMIN +#cmakedefine fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS #cmakedefine fdb5_HAVE_TOCFDB #cmakedefine fdb5_HAVE_DUMMY_DAOS #cmakedefine fdb5_HAVE_DAOSFDB From 1b2c16d0a15e38c50a1e7f396a9e76b49e8dbbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 14:54:49 +0200 Subject: [PATCH 066/109] tests(ceph): clean pools --- tests/fdb/rados/CMakeLists.txt | 2 +- tests/fdb/rados/test_rados_catalogue.cc | 6 +++--- tests/fdb/rados/test_rados_store.cc | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt index 87abd3718..374374c6d 100644 --- a/tests/fdb/rados/CMakeLists.txt +++ b/tests/fdb/rados/CMakeLists.txt @@ -7,7 +7,7 @@ if (HAVE_RADOSFDB) list( APPEND unit_test_libraries fdb5 ) - # The Rados unit tests need a pool to run against (with RADOS_ADMIN=OFF the pool + # The Rados unit tests need a pool to run against (with RADOS_TESTS_MANAGE_POOLS=OFF the pool # must already exist, e.g. one created in the Ceph service). The pool name can be # provided at configure time via -DFDB_RADOS_TEST_POOL=, or inherited from # the FDB_RADOS_TEST_POOL environment variable (e.g. exported in the dev container diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index dbe01e26c..f1a720edd 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -133,7 +133,7 @@ CASE("Setup") { CASE("RadosCatalogue tests") { std::string test_id = "test-catalogue"; -#ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -447,7 +447,7 @@ CASE("RadosCatalogue tests") { /// @note: earlier sections share the same catalogue namespaces/pool; reset them so this /// section starts from a clean, empty catalogue (it asserts the FDB is initially empty). -#ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); #else @@ -808,7 +808,7 @@ CASE("RadosCatalogue tests") { // teardown rados -#ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS eckit::RadosPool{pool}.ensureDestroyed(); #else ensureCleanNamespaces(pool, test_id); diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 0b4b66072..a8c6b8062 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -73,8 +73,8 @@ void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { } } -#ifdef fdb5_HAVE_RADOS_ADMIN -void ensureClean(const std::string& prefix) { +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS +void ensureCleanPools(const std::string& prefix) { ASSERT(prefix.length() > 3); for (const std::string& name : eckit::RadosCluster::instance().listPools()) { if (name.rfind(prefix, 0) == 0) { @@ -150,7 +150,7 @@ CASE("RadosStore tests") { SECTION("archive and retrieve") { std::string test_id = "test-store1"; -#ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -223,7 +223,7 @@ CASE("RadosStore tests") { SECTION("with POSIX Catalogue") { std::string test_id = "test-store2"; -#ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -344,7 +344,7 @@ CASE("RadosStore tests") { deldir(store_tests_tmp_root()); } store_tests_tmp_root().mkdir(); -#ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -499,7 +499,7 @@ CASE("RadosStore tests") { deldir(store_tests_tmp_root()); } store_tests_tmp_root().mkdir(); -#ifdef eckit_HAVE_RADOS_ADMIN +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -596,8 +596,8 @@ int main(int argc, char** argv) { catch (...) { } -#ifdef fdb5_HAVE_RADOS_ADMIN - ensureClean("test-store"); +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + ensureCleanPools("test-store"); #endif return ret; From 30429b8e875d7a5b2c1ba7b8eef20ccef9249662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 14:55:00 +0200 Subject: [PATCH 067/109] fix(ceph): headers --- src/fdb5/rados/RadosCatalogueWriter.cc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index cc6c39b3c..136d4e1c7 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -12,14 +12,37 @@ #include "fdb5/rados/RadosCatalogueWriter.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/IndexAxis.h" +#include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCatalogue.h" +#include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosIndex.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" #include "eckit/io/FileHandle.h" +#include "eckit/io/Length.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/rados/RadosException.h" #include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/log/Log.h" #include "eckit/serialisation/HandleStream.h" +#include +#include +#include +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- From 17efbc2cfab7229e198e48da48716b3bf3ea01f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 5 Aug 2026 17:41:33 +0200 Subject: [PATCH 068/109] fix(ceph): loop --- src/fdb5/rados/RadosCatalogueWriter.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index 136d4e1c7..1aa2b3f76 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -35,6 +35,7 @@ #include "eckit/log/Log.h" #include "eckit/serialisation/HandleStream.h" +#include #include #include #include @@ -248,11 +249,7 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::string axisNames = ""; std::string sep = ""; - for (Key::const_iterator i = datumKey.begin(); i != datumKey.end(); ++i) { - - const std::string& keyword = i->first; - - const std::string& value = i->second; + for (const auto& [keyword, value] : datumKey) { if (value.length() == 0) { continue; From 5452f3d4f9af1f254020238c1cf5714ef87d3fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 13 Aug 2026 11:44:25 +0200 Subject: [PATCH 069/109] fix(rados): field location --- src/fdb5/rados/RadosFieldLocation.cc | 32 +++++++++--------------- src/fdb5/rados/RadosFieldLocation.h | 23 ++++++++--------- src/fdb5/rados/RadosLazyFieldLocation.cc | 17 ++++++------- src/fdb5/rados/RadosLazyFieldLocation.h | 5 +++- 4 files changed, 34 insertions(+), 43 deletions(-) diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index d0b6ff80d..34002ef84 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -8,13 +8,20 @@ * does it submit to any jurisdiction. */ +#include "fdb5/rados/RadosFieldLocation.h" + +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" + #include "eckit/filesystem/URIManager.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" #include "eckit/io/rados/RadosObject.h" -// #include "eckit/io/rados/RadosMultiObjReadHandle.h" +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" -#include "fdb5/rados/RadosFieldLocation.h" -// #include "fdb5/LibFdb5.h" -// #include "fdb5/io/SingleGribMungePartFileHandle.h" +#include +#include namespace fdb5 { @@ -28,9 +35,6 @@ ::eckit::Reanimator RadosFieldLocation::reanimator_; static FieldLocationBuilder builder("rados"); -// RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length ) : -// FieldLocation(eckit::URI("rados", path), offset, length) {} - RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} @@ -41,14 +45,10 @@ RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offs const Key& remapKey) : FieldLocation(uri, offset, length, remapKey) {} -// RadosFieldLocation::RadosFieldLocation(const FileStore &store, const FieldRef &ref) : -// FieldLocation(store.get(ref.pathId()), ref.offset(), ref.length()) {} - RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) : FieldLocation(s) {} - std::shared_ptr RadosFieldLocation::make_shared() const { - return std::make_shared(std::move(*this)); + return std::make_shared(*this); } eckit::DataHandle* RadosFieldLocation::dataHandle() const { @@ -56,10 +56,6 @@ eckit::DataHandle* RadosFieldLocation::dataHandle() const { return eckit::RadosObject(uri_).multipartRangeReadHandle(offset(), length()); } -// eckit::DataHandle *RadosFieldLocation::dataHandle(const Key& remapKey) const { -// return new SingleGribMungePartFileHandle(path(), offset(), length(), remapKey); -// } - void RadosFieldLocation::print(std::ostream& out) const { out << "RadosFieldLocation[uri=" << uri_ << "]"; } @@ -68,10 +64,6 @@ void RadosFieldLocation::visit(FieldLocationVisitor& visitor) const { visitor(*this); } -// eckit::URI RadosFieldLocation::uri(const eckit::PathName &path) { -// return eckit::URI("rados", path); -// } - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h index 36c31105a..8526bc6f1 100644 --- a/src/fdb5/rados/RadosFieldLocation.h +++ b/src/fdb5/rados/RadosFieldLocation.h @@ -14,14 +14,16 @@ #pragma once -// #include "eckit/filesystem/PathName.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" + +#include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" +#include "eckit/serialisation/Reanimator.h" -#include "fdb5/database/FieldLocation.h" -#include "fdb5/fdb5_config.h" -// #include "fdb5/database/FileStore.h" -// #include "fdb5/toc/FieldRef.h" +#include +#include namespace fdb5 { @@ -31,18 +33,15 @@ class RadosFieldLocation : public FieldLocation { public: RadosFieldLocation(const RadosFieldLocation& rhs); - // RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length); RadosFieldLocation(const eckit::URI& uri); RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); - // RadosFieldLocation(const FileStore& store, const FieldRef& ref); RadosFieldLocation(eckit::Stream&); eckit::DataHandle* dataHandle() const override; - // eckit::DataHandle* dataHandle(const Key& remapKey) const override; - virtual std::shared_ptr make_shared() const override; + std::shared_ptr make_shared() const override; - virtual void visit(FieldLocationVisitor& visitor) const override; + void visit(FieldLocationVisitor& visitor) const override; public: // For Streamable @@ -50,7 +49,7 @@ class RadosFieldLocation : public FieldLocation { protected: // For Streamable - virtual const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; @@ -58,8 +57,6 @@ class RadosFieldLocation : public FieldLocation { private: // methods void print(std::ostream& out) const override; - - // eckit::URI uri(const eckit::PathName &path); }; diff --git a/src/fdb5/rados/RadosLazyFieldLocation.cc b/src/fdb5/rados/RadosLazyFieldLocation.cc index 59343f863..4581e5bc1 100644 --- a/src/fdb5/rados/RadosLazyFieldLocation.cc +++ b/src/fdb5/rados/RadosLazyFieldLocation.cc @@ -10,18 +10,17 @@ #include "fdb5/rados/RadosLazyFieldLocation.h" +#include "fdb5/database/FieldLocation.h" + #include "eckit/filesystem/PathName.h" #include "eckit/io/rados/RadosKeyValue.h" #include "eckit/serialisation/MemoryStream.h" #include "eckit/serialisation/Reanimator.h" -#include "fdb5/database/FieldLocation.h" - #include #include #include #include -#include #include namespace fdb5 { @@ -29,13 +28,13 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- RadosLazyFieldLocation::RadosLazyFieldLocation(const fdb5::RadosLazyFieldLocation& rhs) : - FieldLocation(), index_(rhs.index_), key_(rhs.key_) {} + index_(rhs.index_), key_(rhs.key_) {} RadosLazyFieldLocation::RadosLazyFieldLocation(const eckit::RadosKeyValue& index, const std::string& key) : - FieldLocation(), index_(index), key_(key) {} + index_(index), key_(key) {} std::shared_ptr RadosLazyFieldLocation::make_shared() const { - return std::make_shared(std::move(*this)); + return std::make_shared(*this); } eckit::DataHandle* RadosLazyFieldLocation::dataHandle() const { @@ -61,12 +60,10 @@ std::unique_ptr& RadosLazyFieldLocation::realise() const { return fl_; } - /// @note: performed RPCs: - /// - index kv get (daos_kv_get) std::vector data; eckit::MemoryStream ms = index_.getMemoryStream(data, key_, "index kv"); - /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. + /// @note: timestamp read for informational purposes. See note in DaosIndex::add. time_t ts; ms >> ts; @@ -75,4 +72,6 @@ std::unique_ptr& RadosLazyFieldLocation::realise() const { return fl_; } +//---------------------------------------------------------------------------------------------------------------------- + } // namespace fdb5 diff --git a/src/fdb5/rados/RadosLazyFieldLocation.h b/src/fdb5/rados/RadosLazyFieldLocation.h index 3bf75e1d3..396d3e1ba 100644 --- a/src/fdb5/rados/RadosLazyFieldLocation.h +++ b/src/fdb5/rados/RadosLazyFieldLocation.h @@ -13,11 +13,14 @@ #pragma once -#include #include "fdb5/database/FieldLocation.h" #include "eckit/io/rados/RadosKeyValue.h" +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- From dab2487a204a177954ce32772f5f47786bc91f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 13 Aug 2026 12:06:41 +0200 Subject: [PATCH 070/109] fix(rados): store --- src/fdb5/rados/RadosStore.cc | 36 ++++++++++++++++-------------------- src/fdb5/rados/RadosStore.h | 23 ++++++++++++++++------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 160dae454..9b7b7a985 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -19,7 +19,6 @@ #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rules/Rule.h" -#include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" @@ -43,6 +42,7 @@ #include #include #include +#include namespace fdb5 { @@ -50,23 +50,19 @@ namespace fdb5 { static StoreBuilder builder("rados"); -RadosStore::RadosStore(const Key& key, const Config& config) : - Store(), RadosCommon(config, "store", key), archivedFields_(0) {} +RadosStore::RadosStore(const Key& key, const Config& config) : RadosCommon(config, "store", key), archivedFields_(0) {} RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : RadosStore(key, config) {} RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : - Store(), RadosCommon(config, "store", uri), archivedFields_(0) {} + RadosCommon(config, "store", uri), archivedFields_(0) {} eckit::URI RadosStore::uri() const { - return eckit::RadosNamespace(pool_, db_namespace_).uri(); } eckit::URI RadosStore::uri(const eckit::URI& dataURI) { - eckit::RadosObject o{dataURI}; - return o.nspace().uri(); } @@ -81,8 +77,6 @@ bool RadosStore::uriBelongs(const eckit::URI& uri) const { bool RadosStore::uriExists(const eckit::URI& uri) const { - /// @todo: revisit the name of this method - const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); const auto n = parts.size(); @@ -137,13 +131,11 @@ std::set RadosStore::asCollocatedDataURIs(const std::set } bool RadosStore::exists() const { - return eckit::RadosNamespace(pool_, db_namespace_).exists(); } /// @todo: never used in actual fdb-read? eckit::DataHandle* RadosStore::retrieve(Field& field) const { - return field.dataHandle(); } @@ -180,7 +172,6 @@ size_t RadosStore::flush() { } void RadosStore::close() { - closeDataHandles(); } @@ -326,6 +317,11 @@ bool RadosStore::doUnsafeFullWipe() const { return true; } +std::vector RadosStore::getAuxiliaryURIs(const eckit::URI& /*uri*/, bool /*onlyExisting*/) const { + return {}; +} + + //---------------------------------------------------------------------------------------------------------------------- /// @note: unique name generation copied from LocalPathName::unique. @@ -361,12 +357,12 @@ const eckit::RadosObject& RadosStore::getDataObject(const Key& key) const { eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosObject& name) { - HandleStore::const_iterator j = handles_.find(key); - if (j != handles_.end()) { - return *(j->second); + auto iter = handles_.find(key); + if (iter != handles_.end()) { + return *(iter->second); } - eckit::DataHandle* dh = name.multipartWriteHandle(maxPartSize_); + auto* dh = name.multipartWriteHandle(maxPartSize_); ASSERT(dh); @@ -379,8 +375,8 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO void RadosStore::closeDataHandles() { - for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle* dh = j->second; + for (auto& handle : handles_) { + auto* dh = handle.second; dh->close(); delete dh; } @@ -391,8 +387,8 @@ void RadosStore::closeDataHandles() { void RadosStore::flushDataHandles() { - for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle* dh = j->second; + for (auto& handle : handles_) { + auto* dh = handle.second; dh->flush(); } } diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 62312cff4..86c511d11 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -14,12 +14,25 @@ #pragma once +#include "fdb5/config/Config.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" #include "fdb5/database/Store.h" #include "fdb5/rados/RadosCommon.h" #include "fdb5/rules/Schema.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" #include "eckit/io/rados/RadosObject.h" +#include +#include +#include +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -34,8 +47,6 @@ class RadosStore : public Store, public RadosCommon { RadosStore(const Schema& schema, const Key& key, const Config& config); RadosStore(const eckit::URI& uri, const Config& config); - ~RadosStore() override {} - eckit::URI uri() const override; static eckit::URI uri(const eckit::URI& dataURI); bool uriBelongs(const eckit::URI&) const override; @@ -56,8 +67,7 @@ class RadosStore : public Store, public RadosCommon { void doWipeEmptyDatabase() const override; bool doUnsafeFullWipe() const override; - // Rados store does not currently support auxiliary objects - std::vector getAuxiliaryURIs(const eckit::URI&, bool onlyExisting = false) const override { return {}; } + std::vector getAuxiliaryURIs(const eckit::URI& uri, bool onlyExisting) const override; protected: // methods @@ -81,12 +91,11 @@ class RadosStore : public Store, public RadosCommon { private: // types - typedef std::map HandleStore; - typedef std::map ObjectStore; + using HandleStore = std::map; + using ObjectStore = std::map; private: // members - // mutable bool dirty_; size_t archivedFields_{0}; HandleStore handles_; From bab8deb7d08555b659d4f54c9b82b4b0ce1ac1a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:55:41 +0200 Subject: [PATCH 071/109] fix(rados): ci --- .github/workflows/ci-rados.yml | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index dd500d9d0..36e5be69d 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -88,6 +88,7 @@ jobs: -o APT::Keep-Downloaded-Packages=true \ build-essential \ cmake \ + jq \ ninja-build \ gfortran \ libopenmpi-dev \ @@ -145,7 +146,21 @@ jobs: fi docker exec ceph-demo ceph osd pool create "${FDB_RADOS_TEST_POOL}" 8 8 docker exec ceph-demo ceph osd pool application enable "${FDB_RADOS_TEST_POOL}" rados - docker exec ceph-demo ceph osd pool ls + echo "Waiting for ${FDB_RADOS_TEST_POOL} placement groups to become active+clean..." + for i in $(seq 1 60); do + if docker exec ceph-demo ceph pg ls-by-pool "${FDB_RADOS_TEST_POOL}" --format json | \ + jq -e 'length > 0 and all(.[]; .state == "active+clean")' >/dev/null 2>&1; then + pool_ready=1 + break + fi + sleep 5 + done + if [ "${pool_ready:-0}" != "1" ]; then + echo "Ceph test pool did not become active+clean in time" >&2 + docker exec ceph-demo ceph status || true + docker exec ceph-demo ceph pg stat || true + exit 1 + fi sudo chmod 0644 "${CEPH_ETC}/ceph.client.admin.keyring" - name: Build eckit @@ -204,6 +219,16 @@ jobs: run: | ctest --test-dir build/fdb -L rados --output-on-failure - - name: Dump Ceph logs on failure + - name: Dump Ceph diagnostics on failure if: failure() - run: docker logs ceph-demo || true + run: | + echo "=== Ceph status ===" + docker exec ceph-demo ceph status || true + echo "=== Ceph capacity ===" + docker exec ceph-demo ceph df || true + echo "=== Test-pool objects ===" + docker exec ceph-demo rados -p "${FDB_RADOS_TEST_POOL}" ls || true + echo "=== CTest temporary files ===" + ls -la build/fdb/Testing/Temporary/ || true + echo "=== Ceph container logs ===" + docker logs ceph-demo || true From 71453261728dbbb90f02068abb2d1e1715ee52fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:56:07 +0200 Subject: [PATCH 072/109] fix(rados): readme --- src/fdb5/rados/README | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index 3eeeaa7f8..83ba1114b 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -1,6 +1,14 @@ Running RadosStore unit tests against Ceph on Docker on mac: ============================================================ +Supported RADOS backend scope: +============================== + +The backend supports archive, retrieve, and reopening catalogues by URI. Archive calls on a single RadosStore must be +serialised by the caller. Catalogue-side wipe, purge, statistics, move, and control operations are not implemented. + +RADOS tests require eckit to be built with RADOS support. + git clone https://github.com/datenkollektiv/ceph-playground.git cd ceph-playground sed -i '' 's#volumes:#volumes:\n - < PATH TO YOUR LOCAL FDB BUNDLE SOURCE >:/root/git/fdb-bundle#g' docker-compose.yaml From c97ce97c3da666aeb4b5e1b58dc8f55bd9d80ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:56:27 +0200 Subject: [PATCH 073/109] fix(rados): cat deselect index --- src/fdb5/rados/RadosCatalogueReader.cc | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index 441095b76..28519158d 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -54,24 +54,22 @@ bool RadosCatalogueReader::selectIndex(const Key& key) { return true; } - /// @todo: shouldn't this be set only if found a matching index? - currentIndexKey_ = key; - if (indexes_.find(key) == indexes_.end()) { /// @note: performed RPCs: /// - generate catalogue kv oid (daos_obj_generate_oid) /// - ensure catalogue kv exists (daos_kv_open) - int idx_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: take from config - std::vector n((long)idx_loc_max_len); - long res; - try { /// @note: performed RPCs: /// - retrieve index kv location from catalogue kv (daos_kv_get) - res = db_kv_->get(key.valuesToString(), &n[0], idx_loc_max_len); + std::vector data; + db_kv_->getMemoryStream(data, key.valuesToString(), "DB kv"); + eckit::URI uri{std::string{data.begin(), data.end()}}; + eckit::RadosKeyValue index_kv{uri}; + + indexes_[key] = Index(new RadosIndex(key, index_kv, true)); } catch (eckit::RadosEntityNotFoundException& e) { @@ -81,15 +79,11 @@ bool RadosCatalogueReader::selectIndex(const Key& key) { return false; } - eckit::URI uri{std::string{n.begin(), std::next(n.begin(), res)}}; - eckit::RadosKeyValue index_kv{uri}; - - indexes_[key] = Index(new RadosIndex(key, index_kv, true)); - /// @note: performed RPCs: /// - close catalogue kv (daos_obj_close) } + currentIndexKey_ = key; current_ = indexes_[key]; return true; @@ -97,7 +91,8 @@ bool RadosCatalogueReader::selectIndex(const Key& key) { void RadosCatalogueReader::deselectIndex() { - NOTIMP; //< should not be called + current_ = Index(); + currentIndexKey_ = Key(); } bool RadosCatalogueReader::open() { From 83c68e03bb48c61e32af69a3dcfa2919720752a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:57:22 +0200 Subject: [PATCH 074/109] fix(rados): cat writer URI-based --- src/fdb5/rados/RadosCatalogueWriter.cc | 33 ++++---------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index 1aa2b3f76..8d46488de 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -93,20 +93,10 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c hs << dbKey_; } - int db_key_max_len = RADOS_MAX_SERIALISED_LEN; // @todo: take from config - if (hs.bytesWritten() > db_key_max_len) { - throw eckit::Exception("Serialised db key exceeded configured maximum db key length."); - } - db_kv_->put("key", h.data(), hs.bytesWritten()); /// index newly created catalogue kv in main kv - int db_loc_max_len = RADOS_MAX_SERIALISED_LEN; // @todo: take from config std::string nstr = db_kv_->uri().asString(); - if (nstr.length() > db_loc_max_len) { - throw eckit::Exception("Serialised db location exceeded configured maximum db location length."); - } - root_kv_->put(db_name, nstr.data(), nstr.length()); } @@ -121,10 +111,7 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c } RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config) : - RadosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) { - - NOTIMP; -} + RadosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) {} RadosCatalogueWriter::~RadosCatalogueWriter() { @@ -149,19 +136,15 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { /// - generate catalogue kv oid (daos_obj_generate_oid) /// - ensure catalogue kv exists (daos_kv_open) - int idx_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: take from config - try { - std::vector n((long)idx_loc_max_len); - long res; - /// @note: performed RPCs: /// - get index location from catalogue kv (daos_kv_get) - res = db_kv_->get(key.valuesToString(), &n[0], idx_loc_max_len); + std::vector data; + db_kv_->getMemoryStream(data, key.valuesToString(), "DB kv"); indexes_[key] = Index(new fdb5::RadosIndex( - key, eckit::RadosKeyValue{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, false)); + key, eckit::RadosKeyValue{eckit::URI{std::string{data.begin(), data.end()}}}, false)); } catch (eckit::RadosEntityNotFoundException& e) { @@ -171,9 +154,6 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { /// index index kv in catalogue kv std::string nstr{indexes_[key].location().uri().asString()}; - if (nstr.length() > idx_loc_max_len) { - throw eckit::Exception("Serialised index location exceeded configured maximum index location length."); - } /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on /// first write to an index key): /// - record index kv location into catalogue kv (daos_kv_put) -- always performed @@ -281,11 +261,6 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, /// - generate index kv oid (daos_obj_generate_oid) /// - ensure index kv exists (daos_obj_open) - int axis_names_max_len = RADOS_MAX_SERIALISED_LEN; - if (axisNames.length() > axis_names_max_len) { - throw eckit::Exception("Serialised axis names exceeded configured maximum axis names length."); - } - /// @note: performed RPCs: /// - record axis names into index kv (daos_kv_put) /// - close index kv when destroyed (daos_obj_close) From 5596bb5f4b0724a47224e026392d6ec33ee0ffcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:57:54 +0200 Subject: [PATCH 075/109] fix(rados): common --- src/fdb5/rados/RadosCommon.cc | 5 +++-- src/fdb5/rados/RadosCommon.h | 5 ----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index ef355a2ca..4cb5bcd39 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -104,8 +104,9 @@ void RadosCommon::readConfig(const Config& config, const std::string& component, if (c.has(component)) { nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); } - ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, - "The configured namespace prefix must not contain underscores."); + if (nspace_prefix_.find('_') != std::string::npos) { + throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + nspace_prefix_ + "'", Here()); + } // if (c.has("client")) // DaosManager::instance().configure(c.getSubConfiguration("client")); diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 11aaa7985..8b938280f 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -26,11 +26,6 @@ namespace fdb5 { -/// @note: maximum length (in bytes) of the serialised blobs exchanged with Rados key-values -/// (index/field/db locations, serialised keys, axis names). -/// @todo: make configurable (the call sites currently carry a "take from config" note). -constexpr long RADOS_MAX_SERIALISED_LEN = 512; - class RadosCommon { public: // methods From e5ae3bebb344f1087b9ac8f4623d0e3d6e12f596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:58:18 +0200 Subject: [PATCH 076/109] fix(rados): engine --- src/fdb5/rados/RadosEngine.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index 9e2157303..17d905cfd 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -162,8 +162,9 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp if (c.has(component)) { nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); } - ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, - "The configured namespace prefix must not contain underscores."); + if (nspace_prefix_.find('_') != std::string::npos) { + throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + nspace_prefix_ + "'", Here()); + } } static EngineBuilder rados_builder; From 0d0a3933c627bd136050217ff4e93f34bd713764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:59:37 +0200 Subject: [PATCH 077/109] fix(rados): store close data handles --- src/fdb5/rados/RadosStore.cc | 29 +++++++++++++++++------------ src/fdb5/rados/RadosStore.h | 10 ++++++++-- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 9b7b7a985..c98baf6f0 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -156,6 +156,15 @@ std::unique_ptr RadosStore::archive(const Key& key, const v return std::make_unique(o.uri(), offset, length, fdb5::Key{}); } +RadosStore::~RadosStore() { + + try { + closeDataHandles(); + } + catch (...) { + } +} + size_t RadosStore::flush() { if (archivedFields_ == 0) { @@ -362,23 +371,20 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO return *(iter->second); } - auto* dh = name.multipartWriteHandle(maxPartSize_); - - ASSERT(dh); - - handles_[key] = dh; + auto handle = std::unique_ptr{name.multipartWriteHandle(maxPartSize_)}; + ASSERT(handle); - dh->openForWrite(0); + handle->openForWrite(0); + auto [inserted, success] = handles_.emplace(key, std::move(handle)); + ASSERT(success); - return *dh; + return *inserted->second; } void RadosStore::closeDataHandles() { for (auto& handle : handles_) { - auto* dh = handle.second; - dh->close(); - delete dh; + handle.second->close(); } handles_.clear(); @@ -388,8 +394,7 @@ void RadosStore::closeDataHandles() { void RadosStore::flushDataHandles() { for (auto& handle : handles_) { - auto* dh = handle.second; - dh->flush(); + handle.second->flush(); } } diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 86c511d11..ac3c453db 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -46,6 +46,12 @@ class RadosStore : public Store, public RadosCommon { RadosStore(const Key& key, const Config& config); RadosStore(const Schema& schema, const Key& key, const Config& config); RadosStore(const eckit::URI& uri, const Config& config); + ~RadosStore() override; + + RadosStore(const RadosStore&) = delete; + RadosStore& operator=(const RadosStore&) = delete; + RadosStore(RadosStore&&) = delete; + RadosStore& operator=(RadosStore&&) = delete; eckit::URI uri() const override; static eckit::URI uri(const eckit::URI& dataURI); @@ -60,7 +66,7 @@ class RadosStore : public Store, public RadosCommon { void checkUID() const override { /* nothing to do */ } - /// Wipe-related methods (not implemented for the Rados backend) + /// Wipe-related methods void finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) override; bool doWipeUnknowns(const std::set& unknownURIs) const override; bool doWipeURIs(const StoreWipeState& wipeState) const override; @@ -91,7 +97,7 @@ class RadosStore : public Store, public RadosCommon { private: // types - using HandleStore = std::map; + using HandleStore = std::map>; using ObjectStore = std::map; private: // members From 87563dbc1d7159e0228ac8654e52916d49801290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 13:59:56 +0200 Subject: [PATCH 078/109] fix(rados): index len --- src/fdb5/rados/RadosIndex.cc | 43 ++++++++++-------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 7fd94031d..1901232eb 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -16,7 +16,6 @@ #include "fdb5/database/FieldLocation.h" #include "fdb5/database/Index.h" #include "fdb5/database/Key.h" -#include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosLazyFieldLocation.h" #include "eckit/exception/Exceptions.h" @@ -65,12 +64,6 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : hs << key; } - int idx_key_max_len = RADOS_MAX_SERIALISED_LEN; - - if (hs.bytesWritten() > idx_key_max_len) { - throw eckit::Exception("Serialised index key exceeded configured maximum index key length."); - } - /// @note: performed RPCs: /// - record index key into index kv (daos_kv_put) idx_kv_.put("key", h.data(), hs.bytesWritten()); @@ -111,16 +104,14 @@ void RadosIndex::updateAxes() { /// @note: performed RPCs: /// - ensure axis kv exists (daos_obj_open) - int axis_names_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: take from config - std::vector axes_data((long)axis_names_max_len); - /// @note: performed RPCs: /// - get axes key size and content (daos_kv_get without buffer + daos_kv_get) - long res = idx_kv_.get("axes", &axes_data[0], axis_names_max_len); + std::vector axes_data; + idx_kv_.getMemoryStream(axes_data, "axes", "index kv"); std::vector axis_names; eckit::Tokenizer parse(","); - parse(std::string(axes_data.begin(), std::next(axes_data.begin(), res)), axis_names); + parse(std::string(axes_data.begin(), axes_data.end()), axis_names); std::string indexKey{key_.valuesToString()}; for (const auto& name : axis_names) { /// @note: performed RPCs: @@ -144,15 +135,19 @@ bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { std::string query{key.valuesToString()}; - int field_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: read from config - std::vector loc_data((long)field_loc_max_len); - long res; - try { /// @note: performed RPCs: /// - retrieve field array location from index kv (daos_kv_get) - res = idx_kv_.get(query, &loc_data[0], (long)field_loc_max_len); + std::vector loc_data; + eckit::MemoryStream ms = idx_kv_.getMemoryStream(loc_data, query, "index kv"); + + /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. + time_t ts; + ms >> ts; + + fdb5::FieldLocation* loc = eckit::Reanimator::reanimate(ms); + field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); } catch (eckit::RadosEntityNotFoundException& e) { @@ -162,15 +157,6 @@ bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { return false; } - eckit::MemoryStream ms{&loc_data[0], (size_t)res}; - - /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. - time_t ts; - ms >> ts; - - fdb5::FieldLocation* loc = eckit::Reanimator::reanimate(ms); - field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); - /// @note: performed RPCs: /// - close index kv (daos_obj_close) @@ -199,11 +185,6 @@ void RadosIndex::add(const Key& key, const Field& field) { hs << field.location(); } - int field_loc_max_len = RADOS_MAX_SERIALISED_LEN; /// @todo: read from config - if (hs.bytesWritten() > field_loc_max_len) { - throw eckit::Exception("Serialised field location exceeded configured maximum location length."); - } - /// @note: performed RPCs: /// - ensure index kv exists (daos_obj_open) /// - record field key and location into index kv (daos_kv_put) From 5e0fcf25c54a34fb29db8fb66bfa969444a15b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 14:00:26 +0200 Subject: [PATCH 079/109] test(rados): add regressions close --- tests/fdb/rados/test_rados_catalogue.cc | 74 ++++++++++++++++++++++++- tests/fdb/rados/test_rados_store.cc | 46 +++++++++++++-- 2 files changed, 114 insertions(+), 6 deletions(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index f1a720edd..f187dafa0 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -28,6 +28,7 @@ // #include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/database/Catalogue.h" #include "fdb5/toc/TocStore.h" // #include "fdb5/daos/DaosSession.h" @@ -181,6 +182,7 @@ CASE("RadosCatalogue tests") { std::unique_ptr loc(new fdb5::RadosFieldLocation( eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key{})); + eckit::URI catalogue_uri; { fdb5::RadosCatalogueWriter dcatw{db_key, config}; @@ -200,6 +202,7 @@ CASE("RadosCatalogue tests") { fdb5::CatalogueWriter& catw = dcatw; catw.archive(index_key, field_key, std::move(loc)); cat.flush(0); + catalogue_uri = cat.uri(); // EXPECT(index_kv.has(field_key.valuesToString())); // fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; // fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; @@ -211,13 +214,26 @@ CASE("RadosCatalogue tests") { // EXPECT(f_axis_kv.has("6")); } + { + auto reopened = fdb5::CatalogueWriterFactory::instance().build(catalogue_uri, config); + EXPECT(reopened->key() == db_key); + } + // retrieve { fdb5::RadosCatalogueReader dcatr{db_key, config}; fdb5::Catalogue& cat = dcatr; - cat.selectIndex(index_key); + EXPECT(cat.selectIndex(index_key)); + + fdb5::Key missing_index_key({{"c", "missing"}, {"d", "missing"}}); + EXPECT_NOT(cat.selectIndex(missing_index_key)); + EXPECT_NOT(cat.selectIndex(missing_index_key)); + + EXPECT(cat.selectIndex(index_key)); + cat.deselectIndex(); + EXPECT(cat.selectIndex(index_key)); fdb5::Field f; fdb5::CatalogueReader& catr = dcatr; @@ -337,6 +353,54 @@ CASE("RadosCatalogue tests") { // } } + SECTION("RadosCatalogue supports large serialised field locations") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Key db_key({{"a", "large"}, {"b", "large"}}); + fdb5::Key index_key({{"c", "large"}, {"d", "large"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + auto location = std::make_unique( + eckit::URI{"rados", std::string(600, 'x')}, eckit::Offset(0), eckit::Length(1), fdb5::Key{}); + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& catalogue = writer; + EXPECT(catalogue.selectIndex(index_key)); + static_cast(writer).archive(index_key, field_key, std::move(location)); + catalogue.flush(0); + } + + { + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::Catalogue& catalogue = reader; + EXPECT(catalogue.selectIndex(index_key)); + + fdb5::Field field; + EXPECT(static_cast(reader).retrieve(field_key, field)); + EXPECT(field.location().uri().name() == std::string(600, 'x')); + } + } + // SECTION("DaosCatalogue archive (index) and retrieve with a TocStore") { // // FDB configuration @@ -534,6 +598,14 @@ CASE("RadosCatalogue tests") { EXPECT(mh.size() == eckit::Length(sizeof(data))); EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + fdb5::FDB reopened(config); + std::unique_ptr reopened_handle(reopened.retrieve(r)); + + eckit::MemoryHandle reopened_data; + reopened_handle->copyTo(reopened_data); + EXPECT(reopened_data.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(reopened_data.data(), data, sizeof(data)) == 0); + // list all listObject = fdb.list(all_req); diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index a8c6b8062..5852ae60b 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -12,6 +12,7 @@ // #include #include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" #include "eckit/testing/Test.h" // #include "eckit/filesystem/URI.h" #include "eckit/filesystem/PathName.h" @@ -28,6 +29,7 @@ #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/database/Engine.h" #include "fdb5/toc/TocCatalogueReader.h" #include "fdb5/toc/TocCatalogueWriter.h" @@ -167,6 +169,7 @@ CASE("RadosStore tests") { store_tests_tmp_root().asString() + "\n" "rados:\n" + " maxPartSize: 16\n" " store:\n" " pool: " + pool + @@ -185,15 +188,15 @@ CASE("RadosStore tests") { fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); - char data[] = "test"; + const std::string data{"0123456789abcdef0123456789abcdef"}; // archive fdb5::RadosStore rados_store{schema, db_key, config}; fdb5::Store& store = rados_store; - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::unique_ptr loc(store.archive(index_key, data.data(), data.size())); - rados_store.flush(); + rados_store.close(); // retrieve fdb5::Field field(std::move(loc), std::time(nullptr)); @@ -205,8 +208,8 @@ CASE("RadosStore tests") { eckit::MemoryHandle mh; dh->copyTo(mh); - EXPECT(mh.size() == eckit::Length(sizeof(data))); - EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + EXPECT(mh.size() == eckit::Length(data.size())); + EXPECT(::memcmp(mh.data(), data.data(), data.size()) == 0); // remove eckit::RadosObject field_name{field.location().uri()}; @@ -218,6 +221,39 @@ CASE("RadosStore tests") { store.remove(store_uri, out, out, true); EXPECT_NOT(field_name.exists()); EXPECT(store_name.listObjects().size() == 0); + + std::unique_ptr expiring_location; + { + fdb5::RadosStore expiring_store{schema, db_key, config}; + fdb5::Store& store = expiring_store; + expiring_location = store.archive(index_key, data.data(), data.size()); + } + + fdb5::Field expiring_field(std::move(expiring_location), std::time(nullptr)); + std::unique_ptr expiring_handle(expiring_field.dataHandle()); + eckit::MemoryHandle expiring_data; + expiring_handle->copyTo(expiring_data); + EXPECT(expiring_data.size() == eckit::Length(data.size())); + EXPECT(::memcmp(expiring_data.data(), data.data(), data.size()) == 0); + + eckit::RadosObject{expiring_field.location().uri()}.nspace().destroy(); + } + + SECTION("rejects namespace prefixes containing underscores") { + + fdb5::Schema schema{schema_file()}; + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + + std::string config_str{ + "rados:\n" + " pool: " + + std::string{"unused"} + + "\n" + " namespace_prefix: invalid_prefix\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + EXPECT_THROWS_AS((fdb5::RadosStore{schema, db_key, config}), eckit::UserError); + EXPECT_THROWS_AS((fdb5::Engine::backend("rados").location(db_key, config)), eckit::UserError); } SECTION("with POSIX Catalogue") { From 9f7346d5de6b25b92b7df288907f747b1bb98c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 14:12:41 +0200 Subject: [PATCH 080/109] chore(rados): include --- src/fdb5/rados/RadosCommon.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 4cb5bcd39..d1fe895e7 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -10,15 +10,15 @@ #include "fdb5/rados/RadosCommon.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" + #include "eckit/config/LocalConfiguration.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/utils/Tokenizer.h" -#include "fdb5/config/Config.h" -#include "fdb5/database/Key.h" - #include #include #include From 502bbecf52219490f30dffc9a0aa86c004d8d582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 18 Aug 2026 15:42:09 +0200 Subject: [PATCH 081/109] test(rados): fix cat --- tests/fdb/rados/test_rados_catalogue.cc | 76 ++++++++++++------------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index f187dafa0..dbfb93937 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -8,37 +8,41 @@ * does it submit to any jurisdiction. */ -// #include -// #include - -#include "eckit/config/Resource.h" -#include "eckit/testing/Test.h" -// #include "eckit/filesystem/URI.h" -#include "eckit/filesystem/PathName.h" -#include "eckit/filesystem/TmpFile.h" -// #include "eckit/filesystem/TmpDir.h" -// #include "eckit/io/FileHandle.h" -#include "eckit/config/YAMLConfiguration.h" -#include "eckit/io/MemoryHandle.h" -#include "eckit/io/PartHandle.h" - -// #include "metkit/mars/MarsRequest.h" - -#include "fdb5/fdb5_config.h" -// #include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" +#include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" -#include "fdb5/toc/TocStore.h" - -// #include "fdb5/daos/DaosSession.h" -// #include "fdb5/daos/DaosPool.h" -// #include "fdb5/daos/DaosArrayPartHandle.h" - +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" #include "fdb5/rados/RadosCatalogueReader.h" #include "fdb5/rados/RadosCatalogueWriter.h" #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rados/RadosStore.h" +#include "fdb5/rules/Schema.h" + +#include "metkit/mars/MarsRequest.h" + +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/io/Offset.h" +#include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosPool.h" +#include "eckit/testing/Test.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include using namespace eckit::testing; using namespace eckit; @@ -64,17 +68,6 @@ void deldir(eckit::PathName& p) { p.rmdir(); }; -void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { - ASSERT(prefix.length() > 3); - for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { - if (name.rfind(prefix, 0) == 0) { - eckit::RadosNamespace{pool, name}.destroy(); - } - } -} - -} // namespace - // temporary schema,spaces,root files common to all DAOS Catalogue tests eckit::TmpFile& schema_file() { @@ -92,8 +85,9 @@ eckit::PathName& catalogue_tests_tmp_root() { return cd; } -namespace fdb { -namespace test { +} // namespace + +namespace fdb::test { CASE("Setup") { @@ -379,8 +373,8 @@ CASE("RadosCatalogue tests") { fdb5::Key index_key({{"c", "large"}, {"d", "large"}}); fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); - auto location = std::make_unique( - eckit::URI{"rados", std::string(600, 'x')}, eckit::Offset(0), eckit::Length(1), fdb5::Key{}); + auto location = std::make_unique(eckit::URI{"rados", std::string(600, 'x')}, + eckit::Offset(0), eckit::Length(1), fdb5::Key{}); { fdb5::RadosCatalogueWriter writer{db_key, config}; @@ -887,8 +881,8 @@ CASE("RadosCatalogue tests") { #endif } -} // namespace test -} // namespace fdb +} // namespace fdb::test + int main(int argc, char** argv) { return run_tests(argc, argv); From ae62eb52195ad7b3e4891eb949e6a1a23190701f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 10:45:35 +0200 Subject: [PATCH 082/109] ci(rados): fix ceph cluster --- .github/workflows/ci-rados.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index 36e5be69d..857c4a9a8 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -145,6 +145,9 @@ jobs: exit 1 fi docker exec ceph-demo ceph osd pool create "${FDB_RADOS_TEST_POOL}" 8 8 + # The demo cluster has one OSD; its test pool must use a single replica to become active+clean. + docker exec ceph-demo ceph osd pool set "${FDB_RADOS_TEST_POOL}" size 1 --yes-i-really-mean-it + docker exec ceph-demo ceph osd pool set "${FDB_RADOS_TEST_POOL}" min_size 1 docker exec ceph-demo ceph osd pool application enable "${FDB_RADOS_TEST_POOL}" rados echo "Waiting for ${FDB_RADOS_TEST_POOL} placement groups to become active+clean..." for i in $(seq 1 60); do From e63868661400c668aa3c43b5f424f81a899f7de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 11:16:05 +0200 Subject: [PATCH 083/109] ci(rados): pin ceph docker image --- .github/workflows/ci-rados.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index 857c4a9a8..784d98ef1 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -37,7 +37,7 @@ jobs: INSTALL_PREFIX: ${{ github.workspace }}/install # Local dirs used to cache apt archives and the Ceph docker image. APT_CACHE: ${{ github.workspace }}/.apt-cache - CEPH_IMAGE: quay.io/ceph/demo:latest-squid + CEPH_IMAGE: quay.io/ceph/demo@sha256:522483cf07cfce6386b8e18a3edfa88a1b32c688dee231d0a6962b1513557723 CEPH_IMAGE_CACHE: ${{ github.workspace }}/.docker-ceph CMAKE_FLAGS: -DENABLE_AEC=OFF -DENABLE_EXAMPLES=OFF -DENABLE_EXPERIMENTAL=OFF -DENABLE_NETCDF=OFF @@ -106,7 +106,7 @@ jobs: with: path: ${{ github.workspace }}/.docker-ceph # Bump the suffix to refresh the pinned image snapshot. - key: ceph-image-latest-squid-v1 + key: ceph-image-522483cf07cf-v1 - name: Load or pull Ceph image run: | From ec2da2f62a57f5188eb4a581fcef8aa118820ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 12:29:51 +0200 Subject: [PATCH 084/109] test(rados): add wipe --- tests/fdb/rados/test_rados_catalogue.cc | 168 +++++++++++------------- 1 file changed, 75 insertions(+), 93 deletions(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index dbfb93937..e5b908395 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -11,6 +11,7 @@ #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/api/helpers/ListElement.h" +#include "fdb5/api/helpers/WipeIterator.h" #include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/Field.h" @@ -68,6 +69,23 @@ void deldir(eckit::PathName& p) { p.rmdir(); }; +// Count only URIs that would actually be deleted, filtering out safe/info/error records so that +// too-specific requests yield zero. +size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { + size_t count = 0; + fdb5::WipeElement elem; + while (wipeObject.next(elem)) { + if (print) { + std::cout << elem << std::endl; + } + if (elem.type() != fdb5::WipeElementType::ERROR && elem.type() != fdb5::WipeElementType::CATALOGUE_INFO && + elem.type() != fdb5::WipeElementType::CATALOGUE_SAFE && elem.type() != fdb5::WipeElementType::STORE_SAFE) { + count += elem.uris().size(); + } + } + return count; +} + // temporary schema,spaces,root files common to all DAOS Catalogue tests eckit::TmpFile& schema_file() { @@ -611,111 +629,75 @@ CASE("RadosCatalogue tests") { } EXPECT(count == 1); - // // wipe data + // wipe data + + // dry run attempt to wipe with too specific request - // fdb5::WipeElement elem; + auto wipeObject = fdb.wipe(full_req); + EXPECT(countWipeable(wipeObject) == 0); - // // dry run attempt to wipe with too specific request + // dry run wipe index and store unit + wipeObject = fdb.wipe(index_req); + EXPECT(countWipeable(wipeObject) > 0); - // auto wipeObject = fdb.wipe(full_req); - // count = 0; - // while (wipeObject.next(elem)) count++; - // EXPECT(count == 0); + // dry run wipe database + wipeObject = fdb.wipe(db_req); + EXPECT(countWipeable(wipeObject) > 0); + + // ensure field still exists + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 1); - // // dry run wipe index and store unit - // wipeObject = fdb.wipe(index_req); - // count = 0; - // while (wipeObject.next(elem)) count++; - // EXPECT(count > 0); + // attempt to wipe with too specific request + wipeObject = fdb.wipe(full_req, true); + EXPECT(countWipeable(wipeObject) == 0); + fdb.flush(); - // // dry run wipe database - // wipeObject = fdb.wipe(db_req); - // count = 0; - // while (wipeObject.next(elem)) count++; - // EXPECT(count > 0); + // wipe index and store unit + wipeObject = fdb.wipe(index_req, true); + EXPECT(countWipeable(wipeObject) > 0); + fdb.flush(); - // // ensure field still exists - // listObject = fdb.list(full_req); - // count = 0; - // while (listObject.next(info)) { - // // info.print(std::cout, true, true); - // // std::cout << std::endl; - // count++; - // } - // EXPECT(count == 1); - - // // attempt to wipe with too specific request - // wipeObject = fdb.wipe(full_req, true); - // count = 0; - // while (wipeObject.next(elem)) count++; - // EXPECT(count == 0); - // /// @todo: really needed? - // fdb.flush(); - - // // wipe index and store unit - // wipeObject = fdb.wipe(index_req, true); - // count = 0; - // while (wipeObject.next(elem)) count++; - // EXPECT(count > 0); - // /// @todo: really needed? - // fdb.flush(); - - // // ensure field does not exist - // listObject = fdb.list(full_req); - // count = 0; - // while (listObject.next(info)) count++; - // EXPECT(count == 0); - - // /// @todo: ensure index and corresponding container do not exist - // /// @todo: ensure DB still exists - // /// @todo: list db or index and expect count = 0? - - // // re-archive data - - // /// @note: FDB holds a LocalFDB which holds an Archiver which holds open DBs (DaosCatalogueWriters). - // /// If a whole DB is wiped, the top-level structures for that DB (main and catalogue KVs in this case) - // /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the - // /// top-level structures are only generated as part of the DaosCatalogueWriter constructor. There is - // /// no way currently to destroy the open DBs held by FDB other than entirely destroying FDB. - // /// Alternatively, a separate FDB instance can be created. - // fdb5::FDB fdb2(config); - - // fdb2.archive(request_key, data, sizeof(data)); - - // fdb2.flush(); - - // listObject = fdb2.list(full_req); - // count = 0; - // while (listObject.next(info)) { - // // info.print(std::cout, true, true); - // // std::cout << std::endl; - // count++; - // } - // EXPECT(count == 1); + // ensure field does not exist + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 0); - // // wipe full database + // re-archive data - // wipeObject = fdb2.wipe(db_req, true); - // count = 0; - // while (wipeObject.next(elem)) count++; - // EXPECT(count > 0); - // /// @todo: really needed? - // fdb2.flush(); + // FDB caches open DBs. Once a full DB is wiped, a fresh FDB instance is needed + // to re-create the top-level catalogue KV. + fdb5::FDB fdb2(config); - // // ensure field does not exist + fdb2.archive(request_key, data, sizeof(data)); + fdb2.flush(); - // listObject = fdb2.list(full_req); - // count = 0; - // while (listObject.next(info)) { - // // info.print(std::cout, true, true); - // // std::cout << std::endl; - // count++; - // } - // EXPECT(count == 0); + listObject = fdb2.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 1); - // /// @todo: ensure DB and corresponding pool do not exist + // wipe full database + wipeObject = fdb2.wipe(db_req, true); + EXPECT(countWipeable(wipeObject) > 0); + fdb2.flush(); - // /// @todo: ensure new DaosSession has updated daos client config + // ensure field does not exist + listObject = fdb2.list(full_req); + count = 0; + while (listObject.next(info)) { + count++; + } + EXPECT(count == 0); } // SECTION("OPTIONAL SCHEMA KEYS") { From 18f36e8b27385a3b88a5a3d856cfaed268a5570c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 12:30:11 +0200 Subject: [PATCH 085/109] feat(rados): add wipe --- src/fdb5/rados/RadosCatalogue.cc | 197 ++++++++++++++++++++++++++----- src/fdb5/rados/RadosCatalogue.h | 22 ++-- 2 files changed, 177 insertions(+), 42 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index 27f60df1b..280d1d153 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -18,15 +18,9 @@ #include "fdb5/rados/RadosCatalogue.h" -#include "eckit/filesystem/URI.h" -#include "eckit/io/rados/RadosException.h" -#include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/log/Timer.h" -#include "eckit/serialisation/MemoryStream.h" -#include "eckit/utils/Tokenizer.h" - #include "fdb5/LibFdb5.h" #include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/api/helpers/WipeIterator.h" #include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/DatabaseNotFoundException.h" @@ -38,7 +32,21 @@ #include "fdb5/rules/Rule.h" #include "fdb5/rules/Schema.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosException.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosObject.h" +#include "eckit/log/Log.h" +#include "eckit/log/Timer.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Tokenizer.h" + +#include #include +#include +#include #include #include #include @@ -50,10 +58,10 @@ namespace fdb5 { RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : CatalogueImpl(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) { - // TODO: apply the mechanism in RootManager::directory, using - // FileSpaceTables to determine root_pool_name_ according to key - // and using DbPathNamerTables to determine db_cont_name_ according - // to key + /// TODO: apply the mechanism in RootManager::directory, using + /// FileSpaceTables to determine root_pool_name_ according to key + /// and using DbPathNamerTables to determine db_cont_name_ according + /// to key } RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, @@ -189,46 +197,171 @@ bool RadosCatalogue::uriBelongs(const eckit::URI& uri) const { //---------------------------------------------------------------------------------------------------------------------- -/// Wipe-related methods are not implemented for the Rados backend. +/// @note: Catalogue and store share the same Rados namespace, so wipe reports every non-safe object +/// found there as unrecognised. The `WipeCoordinator` then cross-checks store and catalogue +/// `uriBelongs()` to attribute each unknown to the correct owner. CatalogueWipeState RadosCatalogue::wipeInit() const { - NOTIMP; + return CatalogueWipeState{dbKey_, config()}; +} + +void RadosCatalogue::maskIndexEntries(const std::set& indexes) const { + + for (const auto& index : indexes) { + std::string key = index.key().valuesToString(); + if (db_kv_->has(key)) { + db_kv_->remove(key); + } + } } -bool RadosCatalogue::markIndexForWipe(const Index&, bool, CatalogueWipeState&) const { - NOTIMP; +bool RadosCatalogue::markIndexForWipe(const Index& index, bool include, CatalogueWipeState& wipeState) const { + + eckit::RadosKeyValue index_kv{index.location().uri()}; + + // A cross fdb-mount must never delete another DB's index/axis KVs. + if (index_kv.nspace().pool().name() != pool_ || index_kv.nspace().name() != db_namespace_) { + include = false; + } + + std::vector axis_uris; + try { + std::vector axes_data; + index_kv.getMemoryStream(axes_data, "axes", "index kv"); + std::vector axis_names; + eckit::Tokenizer parse(","); + parse(std::string(axes_data.begin(), axes_data.end()), axis_names); + const std::string idx_key = index.key().valuesToString(); + for (const auto& axis : axis_names) { + axis_uris.push_back( + eckit::RadosKeyValue{index_kv.nspace().pool().name(), index_kv.nspace().name(), idx_key + "." + axis} + .uri()); + } + } + catch (const eckit::RadosEntityNotFoundException&) { + // Index KV or its axes list may already be gone (e.g. after an incomplete wipe). + } + + const eckit::URI index_uri = index.location().uri(); + + if (include) { + wipeState.markForMasking(index); + wipeState.markForDeletion(WipeElementType::CATALOGUE_INDEX, index_uri); + for (const auto& uri : axis_uris) { + wipeState.markForDeletion(WipeElementType::CATALOGUE_INDEX, uri); + } + } + else { + wipeState.markAsSafe({index_uri}); + for (const auto& uri : axis_uris) { + wipeState.markAsSafe({uri}); + } + } + + return include; +} + +void RadosCatalogue::finaliseWipeState(CatalogueWipeState& wipeState) const { + + const eckit::URI db_kv_uri = db_kv_->uri(); + + const bool wipeAll = wipeState.safeURIs().empty(); + if (wipeAll) { + wipeState.markForDeletion(WipeElementType::CATALOGUE, db_kv_uri); + } + else { + wipeState.markAsSafe({db_kv_uri}); + return; + } + + eckit::RadosNamespace db{pool_, db_namespace_}; + if (!db.exists()) { + return; + } + + for (const auto& obj : db.listObjects()) { + if (obj.name().find(";part-") != std::string::npos) { + continue; + } + const eckit::URI uri = obj.uri(); + if (!wipeState.isMarkedForDeletion(uri)) { + wipeState.insertUnrecognised(uri); + } + } } -void RadosCatalogue::finaliseWipeState(CatalogueWipeState&) const { - NOTIMP; +namespace { + +void remove_catalogue_uri(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) { + + eckit::RadosObject obj{uri}; + logVerbose << "destroy Rados object: "; + logAlways << obj.str() << std::endl; + if (doit) { + obj.ensureAllDestroyed(); + } } -bool RadosCatalogue::doWipeUnknowns(const std::set&) const { - NOTIMP; +} // namespace + +bool RadosCatalogue::doWipeUnknowns(const std::set& unknownURIs) const { + + for (const auto& uri : unknownURIs) { + if (eckit::RadosObject{uri}.exists()) { + remove_catalogue_uri(uri, std::cout, std::cout, true); + } + } + return true; } -bool RadosCatalogue::doWipeURIs(const CatalogueWipeState&) const { - NOTIMP; +bool RadosCatalogue::doWipeURIs(const CatalogueWipeState& wipeState) const { + + const bool wipeAll = wipeState.safeURIs().empty(); + + for (const auto& [type, uris] : wipeState.deleteMap()) { + for (const auto& uri : uris) { + remove_catalogue_uri(uri, std::cout, std::cout, true); + } + } + + if (wipeAll) { + cleanupEmptyDatabase_ = true; + } + + return true; } void RadosCatalogue::doWipeEmptyDatabase() const { - NOTIMP; -} -bool RadosCatalogue::doUnsafeFullWipe() const { - NOTIMP; + if (!cleanupEmptyDatabase_) { + return; + } + + eckit::RadosNamespace db{pool_, db_namespace_}; + if (db.exists()) { + db.destroy(); + } + + if (root_kv_ && root_kv_->exists() && root_kv_->has(db_namespace_)) { + root_kv_->remove(db_namespace_); + } + + cleanupEmptyDatabase_ = false; } -// void RadosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, std::ostream& logVerbose, bool -// doit) { +bool RadosCatalogue::doUnsafeFullWipe() const { -// ASSERT(n.hasContainerName()); + eckit::RadosNamespace db{pool_, db_namespace_}; + if (db.exists()) { + db.destroy(); + } -// logVerbose << "Removing " << (n.hasOID() ? "KV" : "container") << ": "; -// logAlways << n.URI() << std::endl; -// if (doit) n.destroy(); + if (root_kv_ && root_kv_->exists() && root_kv_->has(db_namespace_)) { + root_kv_->remove(db_namespace_); + } -// } + return true; +} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h index abe6259d0..41de3cf24 100644 --- a/src/fdb5/rados/RadosCatalogue.h +++ b/src/fdb5/rados/RadosCatalogue.h @@ -13,12 +13,6 @@ #pragma once -#include "eckit/config/Configuration.h" -#include "eckit/container/Queue.h" -#include "eckit/exception/Exceptions.h" -#include "eckit/filesystem/URI.h" -#include "eckit/io/Offset.h" - #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/api/helpers/MoveIterator.h" #include "fdb5/config/Config.h" @@ -30,6 +24,12 @@ #include "fdb5/rados/RadosCommon.h" #include "fdb5/rules/Schema.h" +#include "eckit/config/Configuration.h" +#include "eckit/container/Queue.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Offset.h" + #include #include #include @@ -63,7 +63,7 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { std::string type() const override; - void checkUID() const override { NOTIMP; }; + void checkUID() const override { /* nothing to do */ } bool exists() const override; void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override { NOTIMP; }; const Schema& schema() const override; @@ -89,15 +89,17 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { }; // Control access properties of the DB - void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override { NOTIMP; }; + /// @todo: control identifiers are not persisted for RADOS yet; wipe/coordinator invocations rely on default-enabled + /// semantics. + void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override {} const Rule& rule() const override; bool uriBelongs(const eckit::URI& uri) const override; - void maskIndexEntries(const std::set& indexes) const override { NOTIMP; } + void maskIndexEntries(const std::set& indexes) const override; - /// Wipe-related methods (not implemented for the Rados backend) + /// Wipe-related methods CatalogueWipeState wipeInit() const override; bool markIndexForWipe(const Index& index, bool include, CatalogueWipeState& wipeState) const override; void finaliseWipeState(CatalogueWipeState& wipeState) const override; From c67e591ade85f580f2fd6921d8312e38425fc022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 12:42:46 +0200 Subject: [PATCH 086/109] feat(rados): cat writer --- src/fdb5/rados/RadosCatalogue.cc | 5 ++-- src/fdb5/rados/RadosCatalogue.h | 10 ++++--- src/fdb5/rados/RadosCatalogueReader.cc | 36 +------------------------ src/fdb5/rados/RadosCatalogueReader.h | 11 ++++---- src/fdb5/rados/RadosCatalogueWriter.h | 2 +- src/fdb5/rados/RadosEngine.h | 2 +- src/fdb5/rados/RadosStore.cc | 15 +++++++---- tests/fdb/rados/test_rados_catalogue.cc | 13 +++++++++ 8 files changed, 40 insertions(+), 54 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index 280d1d153..9a8097234 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -238,8 +238,9 @@ bool RadosCatalogue::markIndexForWipe(const Index& index, bool include, Catalogu .uri()); } } - catch (const eckit::RadosEntityNotFoundException&) { - // Index KV or its axes list may already be gone (e.g. after an incomplete wipe). + catch (const eckit::RadosEntityNotFoundException& e) { + LOG_DEBUG_LIB(LibFdb5) << "RadosCatalogue::markIndexForWipe: axes lookup missing for index " << index.key() + << " (assuming stale index kv): " << e.what() << std::endl; } const eckit::URI index_uri = index.location().uri(); diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h index 41de3cf24..360f3552f 100644 --- a/src/fdb5/rados/RadosCatalogue.h +++ b/src/fdb5/rados/RadosCatalogue.h @@ -65,7 +65,9 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { void checkUID() const override { /* nothing to do */ } bool exists() const override; - void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override { NOTIMP; }; + void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override { + out << "RadosCatalogue(" << type() << ":" << dbKey_ << ")"; + } const Schema& schema() const override; StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; @@ -83,10 +85,10 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { std::vector indexes(bool sorted = false) const override; + // No masking metadata is persisted for this backend; wipe removes entries directly, so there is + // nothing to enumerate here. void allMasked(std::set>& metadata, - std::set& data) const override { - NOTIMP; - }; + std::set& data) const override {} // Control access properties of the DB /// @todo: control identifiers are not persisted for RADOS yet; wipe/coordinator invocations rely on default-enabled diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index 28519158d..0745b988e 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -17,16 +17,13 @@ #include "fdb5/database/Index.h" #include "fdb5/database/Key.h" #include "fdb5/rados/RadosCatalogue.h" -#include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosIndex.h" -#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosException.h" #include "eckit/io/rados/RadosKeyValue.h" #include "eckit/log/Log.h" -#include #include #include #include @@ -36,14 +33,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// @note: as opposed to the TOC catalogue, the DAOS catalogue does not pre-load all indexes from storage. -/// Instead, it selects and loads only those indexes that are required to fulfil the request. - -RadosCatalogueReader::RadosCatalogueReader(const Key& key, const Config& config) : RadosCatalogue(key, config) { - - /// @todo: schema is being loaded at DaosCatalogueWriter creation for write, but being loaded - /// at DaosCatalogueReader::open for read. Is this OK? -} +RadosCatalogueReader::RadosCatalogueReader(const Key& key, const Config& config) : RadosCatalogue(key, config) {} RadosCatalogueReader::RadosCatalogueReader(const eckit::URI& uri, const Config& config) : RadosCatalogue(uri, ControlIdentifiers{}, config) {} @@ -55,32 +45,16 @@ bool RadosCatalogueReader::selectIndex(const Key& key) { } if (indexes_.find(key) == indexes_.end()) { - - /// @note: performed RPCs: - /// - generate catalogue kv oid (daos_obj_generate_oid) - /// - ensure catalogue kv exists (daos_kv_open) - try { - - /// @note: performed RPCs: - /// - retrieve index kv location from catalogue kv (daos_kv_get) std::vector data; db_kv_->getMemoryStream(data, key.valuesToString(), "DB kv"); eckit::URI uri{std::string{data.begin(), data.end()}}; eckit::RadosKeyValue index_kv{uri}; - indexes_[key] = Index(new RadosIndex(key, index_kv, true)); } catch (eckit::RadosEntityNotFoundException& e) { - - /// @note: performed RPCs: - /// - close catalogue kv (daos_obj_close) - return false; } - - /// @note: performed RPCs: - /// - close catalogue kv (daos_obj_close) } currentIndexKey_ = key; @@ -90,22 +64,14 @@ bool RadosCatalogueReader::selectIndex(const Key& key) { } void RadosCatalogueReader::deselectIndex() { - current_ = Index(); currentIndexKey_ = Key(); } bool RadosCatalogueReader::open() { - - /// @note: performed RPCs: - /// - daos_pool_connect - /// - daos_cont_open - /// - daos_obj_generate_oid - /// - daos_kv_open if (!RadosCatalogue::exists()) { return false; } - RadosCatalogue::loadSchema(); return true; } diff --git a/src/fdb5/rados/RadosCatalogueReader.h b/src/fdb5/rados/RadosCatalogueReader.h index 84d840f95..d7122c324 100644 --- a/src/fdb5/rados/RadosCatalogueReader.h +++ b/src/fdb5/rados/RadosCatalogueReader.h @@ -13,10 +13,6 @@ #pragma once -#include "eckit/exception/Exceptions.h" -#include "eckit/filesystem/URI.h" -#include "eckit/types/Types.h" - #include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/DbStats.h" @@ -25,6 +21,9 @@ #include "fdb5/database/Key.h" #include "fdb5/rados/RadosCatalogue.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" + #include #include #include @@ -55,7 +54,7 @@ class RadosCatalogueReader : public RadosCatalogue, public CatalogueReader { bool retrieve(const Key& key, Field& field) const override; - void print(std::ostream& out) const override { NOTIMP; } + void print(std::ostream& out) const override { out << "RadosCatalogueReader(" << uri() << ")"; } private: // methods @@ -63,7 +62,7 @@ class RadosCatalogueReader : public RadosCatalogue, public CatalogueReader { private: // types - typedef std::map IndexStore; + using IndexStore = std::map; private: // members diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h index eb3eceb42..4a4f51eef 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.h +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -60,7 +60,7 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; - virtual void print(std::ostream& out) const override { NOTIMP; } + void print(std::ostream& out) const override { out << "RadosCatalogueWriter(" << uri() << ")"; } private: // methods diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 8a384c8ca..1137dd9a3 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -54,7 +54,7 @@ class RadosEngine : public Engine { std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; - void print(std::ostream& out) const override { NOTIMP; }; + void print(std::ostream& out) const override { out << "RadosEngine()"; } private: // methods diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index c98baf6f0..3bf497e8c 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -26,6 +26,7 @@ #include "eckit/io/rados/RadosNamespace.h" #include "eckit/io/rados/RadosObject.h" #include "eckit/io/rados/RadosPool.h" +#include "eckit/log/Log.h" #include "eckit/log/TimeStamp.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" @@ -36,12 +37,14 @@ #include #include +#include #include #include #include #include #include #include +#include #include namespace fdb5 { @@ -50,12 +53,11 @@ namespace fdb5 { static StoreBuilder builder("rados"); -RadosStore::RadosStore(const Key& key, const Config& config) : RadosCommon(config, "store", key), archivedFields_(0) {} +RadosStore::RadosStore(const Key& key, const Config& config) : RadosCommon(config, "store", key) {} -RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : RadosStore(key, config) {} +RadosStore::RadosStore(const Schema& /*schema*/, const Key& key, const Config& config) : RadosStore(key, config) {} -RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : - RadosCommon(config, "store", uri), archivedFields_(0) {} +RadosStore::RadosStore(const eckit::URI& uri, const Config& config) : RadosCommon(config, "store", uri) {} eckit::URI RadosStore::uri() const { return eckit::RadosNamespace(pool_, db_namespace_).uri(); @@ -157,11 +159,14 @@ std::unique_ptr RadosStore::archive(const Key& key, const v } RadosStore::~RadosStore() { - try { closeDataHandles(); } + catch (const std::exception& e) { + eckit::Log::error() << "~RadosStore: closeDataHandles failed: " << e.what() << std::endl; + } catch (...) { + eckit::Log::error() << "~RadosStore: closeDataHandles failed with unknown exception" << std::endl; } } diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index e5b908395..66003db49 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -24,6 +24,7 @@ #include "metkit/mars/MarsRequest.h" +#include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/TmpFile.h" @@ -32,6 +33,8 @@ #include "eckit/io/MemoryHandle.h" #include "eckit/io/Offset.h" #include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosCluster.h" +#include "eckit/io/rados/RadosNamespace.h" #include "eckit/io/rados/RadosPool.h" #include "eckit/testing/Test.h" @@ -69,6 +72,16 @@ void deldir(eckit::PathName& p) { p.rmdir(); }; +// Guard against clobbering unrelated namespaces in a shared CI pool. +void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); + } + } +} + // Count only URIs that would actually be deleted, filtering out safe/info/error records so that // too-specific requests yield zero. size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { From d7230b5c7244d6523808e12fbd1b05d5c718dd64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 12:44:35 +0200 Subject: [PATCH 087/109] chore(rados): cleanup --- src/fdb5/rados/RadosCatalogue.cc | 14 ------- src/fdb5/rados/RadosCatalogue.h | 7 ---- src/fdb5/rados/RadosCatalogueWriter.cc | 1 - src/fdb5/rados/RadosCatalogueWriter.h | 5 --- src/fdb5/rados/RadosCommon.cc | 3 -- src/fdb5/rados/RadosEngine.cc | 56 ++++++++++++++++---------- src/fdb5/rados/RadosEngine.h | 14 ++----- src/fdb5/rados/RadosStore.cc | 12 ------ 8 files changed, 37 insertions(+), 75 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index 9a8097234..6770df851 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -8,14 +8,6 @@ * does it submit to any jurisdiction. */ -// #include "eckit/config/Resource.h" -// #include "eckit/serialisation/MemoryStream.h" -// #include "eckit/io/rados/RadosException.h" - -// #include "fdb5/api/helpers/ControlIterator.h" -// #include "fdb5/LibFdb5.h" -// #include "fdb5/database/DatabaseNotFoundException.h" - #include "fdb5/rados/RadosCatalogue.h" #include "fdb5/LibFdb5.h" @@ -123,12 +115,6 @@ void RadosCatalogue::loadSchema() { rule_ = &schema_.matchingRule(dbKey_); } -// WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, -// std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { -// NOTIMP; -// // return new RadosWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); -// } - std::vector RadosCatalogue::indexes(bool) const { /// @note: sorted is not implemented as is not necessary in this backend. diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h index 360f3552f..7ca7eed1a 100644 --- a/src/fdb5/rados/RadosCatalogue.h +++ b/src/fdb5/rados/RadosCatalogue.h @@ -53,14 +53,11 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { RadosCatalogue(const Key& key, const fdb5::Config& config); RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config); - // static const char* catalogueTypeName() { return fdb5::RadosEngine::typeName(); } static const char* catalogueTypeName() { return "rados"; } eckit::URI uri() const override; const Key& indexKey() const override { return currentIndexKey_; } - // static void remove(const eckit::RadosObject&, std::ostream& logAlways, std::ostream& logVerbose, bool doit); - std::string type() const override; void checkUID() const override { /* nothing to do */ } @@ -72,14 +69,10 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }; - // WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool - // doit, - // bool porcelain, bool unsafeWipeAll) const override; MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; }; - // void maskIndexEntry(const Index& index) const override { NOTIMP; }; void loadSchema() override; diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index 8d46488de..82ae908ac 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -243,7 +243,6 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, /// empty sets. This is fine. const auto& axis_set = current_.axes().values(keyword); - // if (!axis_set.has_value() || !axis_set->get().contains(value)) { if (!axis_set.contains(value)) { axesToExpand.push_back(keyword); diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h index 4a4f51eef..ece1b8f5b 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.h +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -40,11 +40,6 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { NOTIMP; }; - // // Hide the contents of the DB!!! - // void hideContents() override; - - // bool enabled(const ControlIdentifier& controlIdentifier) const override; - const Index& currentIndex() override; protected: // methods diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index d1fe895e7..98be6066f 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -107,9 +107,6 @@ void RadosCommon::readConfig(const Config& config, const std::string& component, if (nspace_prefix_.find('_') != std::string::npos) { throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + nspace_prefix_ + "'", Here()); } - - // if (c.has("client")) - // DaosManager::instance().configure(c.getSubConfiguration("client")); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index 17d905cfd..6d4154276 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -12,13 +12,26 @@ #include "fdb5/rados/RadosEngine.h" #include "fdb5/LibFdb5.h" +#include "fdb5/database/Engine.h" +#include "fdb5/database/Key.h" +#include "metkit/mars/MarsRequest.h" + +#include "eckit/config/LocalConfiguration.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" #include "eckit/serialisation/MemoryStream.h" #include "eckit/utils/Tokenizer.h" -using namespace eckit; +#include +#include +#include +#include +#include namespace fdb5 { @@ -36,7 +49,7 @@ eckit::URI RadosEngine::location(const Key& key, const Config& config) const { readConfig(config, "catalogue", true); - const std::string db_namespace = nspace_prefix_ + "_" + key.valuesToString(); + const std::string db_namespace = nspacePrefix_ + "_" + key.valuesToString(); return eckit::RadosKeyValue{pool_, db_namespace, "catalogue_kv"}.uri(); } @@ -55,8 +68,8 @@ bool RadosEngine::canHandle(const eckit::URI& uri, const Config&) const { return eckit::RadosKeyValue{parts[0], parts[1], "catalogue_kv"}.exists(); } catch (const eckit::Exception& e) { - Log::debug() << "RadosEngine::canHandle: exception checking URI " << uri << ": " << e.what() - << std::endl; + eckit::Log::debug() << "RadosEngine::canHandle: exception checking URI " << uri << ": " << e.what() + << std::endl; return false; } } @@ -72,20 +85,20 @@ std::vector RadosEngine::visitableLocations(const std::function res{}; - if (!root_kv_->exists()) { + if (!rootKv_->exists()) { return res; } - for (const auto& k : root_kv_->keys()) { + for (const auto& k : rootKv_->keys()) { try { std::vector v; - root_kv_->getMemoryStream(v, k, "root kv"); + rootKv_->getMemoryStream(v, k, "root kv"); eckit::URI uri(std::string(v.begin(), v.end())); ASSERT(uri.scheme() == typeName()); @@ -97,12 +110,12 @@ std::vector RadosEngine::visitableLocations(const std::function() << " found match with " << root_kv_->uri() << " at key " << k << std::endl; + eckit::Log::debug() << " found match with " << rootKv_->uri() << " at key " << k << std::endl; res.push_back(uri); } } catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; + eckit::Log::error() << "Error loading FDB database " << k << " from " << rootKv_->uri() << std::endl; eckit::Log::error() << e.what() << std::endl; } } @@ -114,7 +127,8 @@ std::vector RadosEngine::visitableLocations(const Key& key, const Co return visitableLocations([&key](const fdb5::Key& dbKey) { return dbKey.match(key); }, config); } -std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const { +std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest& request, + const Config& config) const { return visitableLocations([&request](const fdb5::Key& dbKey) { return dbKey.partialMatch(request); }, config); } @@ -126,8 +140,6 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp c = config.getSubConfiguration("rados"); } - // maxPartSize_ = c.getInt("maxPartSize", 0); - std::string first_cap{component}; first_cap[0] = toupper(component[0]); @@ -139,7 +151,7 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp if (readPool) { pool_ = "default"; } - root_namespace_ = "root"; + rootNamespace_ = "root"; if (readPool) { pool_ = c.getString("pool", pool_); @@ -147,23 +159,23 @@ void RadosEngine::readConfig(const fdb5::Config& config, const std::string& comp pool_ = c.getSubConfiguration(component).getString("pool", pool_); } } - root_namespace_ = c.getString("root_namespace", root_namespace_); + rootNamespace_ = c.getString("root_namespace", rootNamespace_); if (c.has(component)) { - root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + rootNamespace_ = c.getSubConfiguration(component).getString("root_namespace", rootNamespace_); } if (readPool) { pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); } - root_namespace_ = eckit::Resource( - "fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + rootNamespace_ = eckit::Resource( + "fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", rootNamespace_); - nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); + nspacePrefix_ = c.getString("namespace_prefix", nspacePrefix_); if (c.has(component)) { - nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); + nspacePrefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspacePrefix_); } - if (nspace_prefix_.find('_') != std::string::npos) { - throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + nspace_prefix_ + "'", Here()); + if (nspacePrefix_.find('_') != std::string::npos) { + throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + nspacePrefix_ + "'", Here()); } } diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 1137dd9a3..5c7fe360c 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -14,11 +14,9 @@ #pragma once #include "fdb5/database/Engine.h" -#include "fdb5/fdb5_config.h" #include "metkit/mars/MarsRequest.h" -#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosKeyValue.h" @@ -58,8 +56,6 @@ class RadosEngine : public Engine { private: // methods - /// @note: shared implementation of the two visitableLocations overloads; lists all databases - /// registered in the root key-value and returns those whose key satisfies the predicate. std::vector visitableLocations(const std::function& matches, const Config& config) const; @@ -68,17 +64,13 @@ class RadosEngine : public Engine { protected: // members mutable std::string pool_; - mutable std::string root_namespace_; - // std::string db_namespace_; + mutable std::string rootNamespace_; - mutable std::optional root_kv_; - // std::optional db_kv_; - - // eckit::Length maxPartSize_; + mutable std::optional rootKv_; private: // members - mutable std::string nspace_prefix_; + mutable std::string nspacePrefix_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 3bf497e8c..511e1905a 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -105,14 +105,10 @@ std::set RadosStore::collocatedDataURIs() const { return store_unit_uris; } - /// @note if a RadosCatalogue is implemented, some filtering will need to - /// be done here to discriminate store objects from catalogue objects for (const auto& obj : n.listObjects()) { - if (obj.name().find(";part-") != std::string::npos) { continue; } - store_unit_uris.insert(obj.uri()); } @@ -120,15 +116,10 @@ std::set RadosStore::collocatedDataURIs() const { } std::set RadosStore::asCollocatedDataURIs(const std::set& uris) const { - std::set res; - - /// @note: this is only uniquefying the input uris (coming from an index) - /// in case theres any duplicate. for (const auto& uri : uris) { res.insert(uri); } - return res; } @@ -136,7 +127,6 @@ bool RadosStore::exists() const { return eckit::RadosNamespace(pool_, db_namespace_).exists(); } -/// @todo: never used in actual fdb-read? eckit::DataHandle* RadosStore::retrieve(Field& field) const { return field.dataHandle(); } @@ -176,8 +166,6 @@ size_t RadosStore::flush() { return 0; } - /// @note: the multipart handles need to persist the multipart attributes which is - /// performed in the multihandle flush. flushDataHandles(); size_t out = archivedFields_; From 0fb4618324460ea4dda0a58abc9d9920e00b8f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 12:53:37 +0200 Subject: [PATCH 088/109] fix(rados): URI reopen --- src/fdb5/rados/RadosCatalogueWriter.cc | 4 +++- tests/fdb/rados/test_rados_catalogue.cc | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index 82ae908ac..9c8e0cccd 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -111,7 +111,9 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c } RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config) : - RadosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) {} + RadosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) { + RadosCatalogue::loadSchema(); +} RadosCatalogueWriter::~RadosCatalogueWriter() { diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 66003db49..686b95ebc 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -242,6 +242,7 @@ CASE("RadosCatalogue tests") { { auto reopened = fdb5::CatalogueWriterFactory::instance().build(catalogue_uri, config); EXPECT(reopened->key() == db_key); + EXPECT_NOT(reopened->schema().empty()); } // retrieve From ce241a84e8305f90d427c16928bc81adf2096ec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 13:06:05 +0200 Subject: [PATCH 089/109] test(rados): report missing db --- tests/fdb/rados/test_rados_catalogue.cc | 44 +++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 686b95ebc..a73674e19 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -14,6 +14,7 @@ #include "fdb5/api/helpers/WipeIterator.h" #include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" +#include "fdb5/database/DatabaseNotFoundException.h" #include "fdb5/database/Field.h" #include "fdb5/database/FieldLocation.h" #include "fdb5/rados/RadosCatalogueReader.h" @@ -379,6 +380,49 @@ CASE("RadosCatalogue tests") { // } } + SECTION("RadosCatalogue reports missing databases via factory paths") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // A key-based reader over a DB that was never written must fail to open. + fdb5::Key missing_db_key({{"a", "99"}, {"b", "99"}}); + { + fdb5::RadosCatalogueReader reader{missing_db_key, config}; + fdb5::CatalogueReader& cr = reader; + EXPECT_NOT(cr.open()); + } + + // A URI-based reader/writer over a DB whose namespace has no catalogue KV must throw + // DatabaseNotFoundException at construction so the caller does not proceed on empty state. + const std::string missing_ns = test_id + "_" + missing_db_key.valuesToString(); + const eckit::URI missing_uri = eckit::RadosKeyValue{pool, missing_ns, "catalogue_kv"}.uri(); + + EXPECT_THROWS_AS(fdb5::CatalogueReaderFactory::instance().build(missing_uri, config), + fdb5::DatabaseNotFoundException); + EXPECT_THROWS_AS(fdb5::CatalogueWriterFactory::instance().build(missing_uri, config), + fdb5::DatabaseNotFoundException); + } + SECTION("RadosCatalogue supports large serialised field locations") { std::string config_str{ From 37d7a9906e3be6f17b16fa7a1fd96ece286c5c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 13:33:51 +0200 Subject: [PATCH 090/109] feat(rados): safe cleanup and close --- src/fdb5/CMakeLists.txt | 1 + src/fdb5/rados/RadosCatalogueWriter.cc | 94 +++----------------------- src/fdb5/rados/RadosCatalogueWriter.h | 26 +++++-- src/fdb5/rados/RadosCleanup.h | 44 ++++++++++++ src/fdb5/rados/RadosEngine.cc | 11 --- src/fdb5/rados/RadosEngine.h | 2 +- src/fdb5/rados/RadosStore.cc | 36 +++++----- 7 files changed, 94 insertions(+), 120 deletions(-) create mode 100644 src/fdb5/rados/RadosCleanup.h diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 5c2aeb066..20b082a6c 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -353,6 +353,7 @@ if( HAVE_RADOSFDB ) rados/RadosStore.h rados/RadosCommon.cc rados/RadosCommon.h + rados/RadosCleanup.h rados/RadosCatalogue.cc rados/RadosCatalogue.h rados/RadosCatalogueWriter.cc diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index 9c8e0cccd..d8b7a95fa 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -20,7 +20,7 @@ #include "fdb5/database/IndexAxis.h" #include "fdb5/database/Key.h" #include "fdb5/rados/RadosCatalogue.h" -#include "fdb5/rados/RadosCommon.h" +#include "fdb5/rados/RadosCleanup.h" #include "fdb5/rados/RadosIndex.h" #include "eckit/exception/Exceptions.h" @@ -37,7 +37,7 @@ #include #include -#include +#include #include #include #include @@ -51,26 +51,14 @@ namespace fdb5 { RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& config) : RadosCatalogue(key, config), firstIndexWrite_(false) { - /// @note: performed RPCs: - /// - daos_pool_connect - /// - root cont open (daos_cont_open) - /// - root cont create (daos_cont_create) std::string db_name = db_namespace_; ASSERT(root_kv_->nspace().pool().exists()); - /// @note: the DaosKeyValue constructor checks if the kv exists, which results in creation if not exists - /// @note: performed RPCs: - /// - main kv open (daos_kv_open) - - /// @note: performed RPCs: - /// - check if main kv contains db key (daos_kv_get without a buffer) root_kv_->ensureCreated(); if (!root_kv_->has(db_name)) { - /// create catalogue kv db_kv_->ensureCreated(); - /// write schema under "schema" eckit::Log::debug() << "Copy schema from " << config_.schemaPath() << " to " << db_kv_->uri().asString() << " at key 'schema'." << std::endl; @@ -84,7 +72,6 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c } db_kv_->put("schema", &data[0], data.size()); - /// write dbKey under "key" eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); @@ -95,16 +82,10 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c db_kv_->put("key", h.data(), hs.bytesWritten()); - /// index newly created catalogue kv in main kv std::string nstr = db_kv_->uri().asString(); root_kv_->put(db_name, nstr.data(), nstr.length()); } - /// @todo: record or read dbUID - - /// @note: performed RPCs: - /// - catalogue container open (daos_cont_open) - /// - get schema from catalogue kv (daos_kv_get) RadosCatalogue::loadSchema(); /// @todo: TocCatalogue::checkUID(); @@ -116,9 +97,9 @@ RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Co } RadosCatalogueWriter::~RadosCatalogueWriter() { - - clean(); - close(); + std::exception_ptr ignored; + best_effort(ignored, "~RadosCatalogueWriter::clean", [&] { clean(); }); + best_effort(ignored, "~RadosCatalogueWriter::close", [&] { close(); }); } bool RadosCatalogueWriter::createIndex(const Key& /* idxKey */, size_t /* datumKeySize */) { @@ -134,14 +115,7 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { if (indexes_.find(key) == indexes_.end()) { - /// @note: performed RPCs: - /// - generate catalogue kv oid (daos_obj_generate_oid) - /// - ensure catalogue kv exists (daos_kv_open) - try { - - /// @note: performed RPCs: - /// - get index location from catalogue kv (daos_kv_get) std::vector data; db_kv_->getMemoryStream(data, key.valuesToString(), "DB kv"); @@ -156,17 +130,8 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { /// index index kv in catalogue kv std::string nstr{indexes_[key].location().uri().asString()}; - /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on - /// first write to an index key): - /// - record index kv location into catalogue kv (daos_kv_put) -- always performed db_kv_->put(key.valuesToString(), nstr.data(), nstr.length()); - - /// @note: performed RPCs: - /// - close index kv when destroyed (daos_obj_close) } - - /// @note: performed RPCs: - /// - close catalogue kv (daos_obj_close) } current_ = indexes_[key]; @@ -175,37 +140,28 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { } void RadosCatalogueWriter::deselectIndex() { - current_ = Index(); currentIndexKey_ = Key(); firstIndexWrite_ = false; } void RadosCatalogueWriter::clean() { - flush(0); - deselectIndex(); } void RadosCatalogueWriter::close() { - closeIndexes(); } const Index& RadosCatalogueWriter::currentIndex() { - if (current_.null()) { ASSERT(!currentIndexKey_.empty()); selectIndex(currentIndexKey_); } - return current_; } -/// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these -/// new updates be retrieved and put into in-memory axes from time to time, e.g. every -/// time a value is put in an axis KeyValue? void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) { @@ -217,19 +173,14 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, selectIndex(currentIndexKey_); } - /// @note: the current index timestamp is undefined at this point Field field(std::move(fieldLocation), currentIndex().timestamp()); - /// @todo: is sorting axes really necessary? - /// @note: sort in-memory axis values. Not triggering retrieval from DAOS axes. const_cast(current_.axes()).sort(); - /// before in-memory axes are updated as part of current_.put, we determine which - /// additions will need to be performed on axes in DAOS after the field gets indexed. std::vector axesToExpand; std::vector valuesToAdd; - std::string axisNames = ""; - std::string sep = ""; + std::string axisNames; + std::string sep; for (const auto& [keyword, value] : datumKey) { @@ -240,9 +191,6 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, axisNames += sep + keyword; sep = ","; - /// @note: obtain in-memory axis values. Not triggering retrieval from DAOS axes. - /// @note: on first archive the in-memory axes will be empty and values() will return - /// empty sets. This is fine. const auto& axis_set = current_.axes().values(keyword); if (!axis_set.contains(value)) { @@ -252,58 +200,32 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, } } - /// index the field and update in-memory axes current_.put(datumKey, field); - /// persist axis names if (firstIndexWrite_) { - - /// @note: performed RPCs: - /// - generate index kv oid (daos_obj_generate_oid) - /// - ensure index kv exists (daos_obj_open) - - /// @note: performed RPCs: - /// - record axis names into index kv (daos_kv_put) - /// - close index kv when destroyed (daos_obj_close) dynamic_cast(current_.content())->putAxisNames(axisNames); - firstIndexWrite_ = false; } - /// @todo: axes are supposed to be sorted before persisting. How do we do this with the DAOS approach? - /// sort axes every time they are loaded in the read pathway? - if (axesToExpand.empty()) { return; } - /// expand axis info in DAOS while (!axesToExpand.empty()) { - - /// @note: performed RPCs: - /// - generate axis kv oid (daos_obj_generate_oid) - /// - ensure axis kv exists (daos_obj_open) - - /// @note: performed RPCs: - /// - record axis value into axis kv (daos_kv_put) - /// - close axis kv when destroyed (daos_obj_close) dynamic_cast(current_.content())->putAxisValue(axesToExpand.back(), valuesToAdd.back()); - axesToExpand.pop_back(); valuesToAdd.pop_back(); } } void RadosCatalogueWriter::flush(size_t /* archivedFields */) { - if (!current_.null()) { current_ = Index(); } } void RadosCatalogueWriter::closeIndexes() { - - indexes_.clear(); // all indexes instances destroyed + indexes_.clear(); } static fdb5::CatalogueWriterBuilder builder("rados"); diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h index ece1b8f5b..7d900d48d 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.h +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -13,8 +13,25 @@ #pragma once +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Index.h" +#include "fdb5/database/Key.h" #include "fdb5/rados/RadosCatalogue.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" + +#include +#include +#include +#include +#include +#include + namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -33,9 +50,6 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { void reconsolidate() override { NOTIMP; } - /// Mount an existing TocCatalogue, which has a different metadata key (within - /// constraints) to allow on-line rebadging of data - /// variableKeys: The keys that are allowed to differ between the two DBs void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { NOTIMP; }; @@ -44,9 +58,9 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { protected: // methods - virtual bool selectIndex(const Key& key) override; + bool selectIndex(const Key& key) override; bool createIndex(const Key& idxKey, size_t datumKeySize) override; - virtual void deselectIndex() override; + void deselectIndex() override; bool open() override { NOTIMP; } void flush(size_t archivedFields) override; @@ -63,7 +77,7 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { private: // types - typedef std::map IndexStore; + using IndexStore = std::map; private: // members diff --git a/src/fdb5/rados/RadosCleanup.h b/src/fdb5/rados/RadosCleanup.h new file mode 100644 index 000000000..41da86548 --- /dev/null +++ b/src/fdb5/rados/RadosCleanup.h @@ -0,0 +1,44 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#include "eckit/log/Log.h" + +#include +#include +#include + +namespace fdb5 { + +// Runs `op`; on failure logs via Log::error and records the first exception into `first`. +// Intended for destructor-safe cleanup paths where all steps must be attempted. +template +void best_effort(std::exception_ptr& first, const char* context, Op&& op) { + try { + std::forward(op)(); + } + catch (...) { + if (!first) { + first = std::current_exception(); + } + try { + throw; + } + catch (const std::exception& e) { + eckit::Log::error() << context << ": " << e.what() << std::endl; + } + catch (...) { + eckit::Log::error() << context << ": unknown exception" << std::endl; + } + } +} + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index 6d4154276..6c81c6386 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -42,13 +42,7 @@ std::string RadosEngine::name() const { } eckit::URI RadosEngine::location(const Key& key, const Config& config) const { - - /// @note: cannot inherit from RadosCommon here, as the Engine is always instantiated even when - /// Rados is not used; it would then initialise RadosCommon unnecessarily. So the db key-value - /// naming is resolved locally via readConfig, mirroring RadosCommon's key-based constructor. - readConfig(config, "catalogue", true); - const std::string db_namespace = nspacePrefix_ + "_" + key.valuesToString(); return eckit::RadosKeyValue{pool_, db_namespace, "catalogue_kv"}.uri(); } @@ -77,10 +71,6 @@ bool RadosEngine::canHandle(const eckit::URI& uri, const Config&) const { std::vector RadosEngine::visitableLocations(const std::function& matches, const Config& config) const { - /// @note: cannot inherit from RadosCommon here, as the Engine is always instantiated even when - /// Rados is not used; it would then initialise RadosCommon unnecessarily. So the root key-value - /// naming is resolved locally via readConfig. - const std::string component = "catalogue"; readConfig(config, component, true); @@ -94,7 +84,6 @@ std::vector RadosEngine::visitableLocations(const std::functionkeys()) { - try { std::vector v; diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 5c7fe360c..5adb98ee0 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -52,7 +52,7 @@ class RadosEngine : public Engine { std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; - void print(std::ostream& out) const override { out << "RadosEngine()"; } + void print(std::ostream& out) const override { out << "RadosEngine(" << pool_ << ", " << rootNamespace_ << ")"; } private: // methods diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 511e1905a..e35690a0c 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -15,6 +15,7 @@ #include "fdb5/database/FieldLocation.h" #include "fdb5/database/Store.h" #include "fdb5/database/WipeState.h" +#include "fdb5/rados/RadosCleanup.h" #include "fdb5/rados/RadosCommon.h" #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rules/Rule.h" @@ -149,15 +150,8 @@ std::unique_ptr RadosStore::archive(const Key& key, const v } RadosStore::~RadosStore() { - try { - closeDataHandles(); - } - catch (const std::exception& e) { - eckit::Log::error() << "~RadosStore: closeDataHandles failed: " << e.what() << std::endl; - } - catch (...) { - eckit::Log::error() << "~RadosStore: closeDataHandles failed with unknown exception" << std::endl; - } + std::exception_ptr ignored; + best_effort(ignored, "~RadosStore::closeDataHandles", [&] { closeDataHandles(); }); } size_t RadosStore::flush() { @@ -376,18 +370,28 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO void RadosStore::closeDataHandles() { - for (auto& handle : handles_) { - handle.second->close(); - } - - handles_.clear(); + // Detach the map first so partial failures never leave the destructor retrying the same handle. + HandleStore handles; + handles.swap(handles_); dataObjects_.clear(); + + std::exception_ptr first; + for (auto& [key, handle] : handles) { + best_effort(first, "RadosStore::closeDataHandles", [&] { handle->close(); }); + } + if (first) { + std::rethrow_exception(first); + } } void RadosStore::flushDataHandles() { - for (auto& handle : handles_) { - handle.second->flush(); + std::exception_ptr first; + for (auto& [key, handle] : handles_) { + best_effort(first, "RadosStore::flushDataHandles", [&] { handle->flush(); }); + } + if (first) { + std::rethrow_exception(first); } } From 5687a6376c5966ac7d95c577dd79cd724f175e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 13:40:39 +0200 Subject: [PATCH 091/109] chore(rados): cleanup --- src/fdb5/rados/RadosCatalogue.cc | 7 ++----- src/fdb5/rados/RadosCatalogueWriter.cc | 21 +++++++-------------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index 6770df851..84f44616d 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -60,9 +60,6 @@ RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& const fdb5::Config& config) : CatalogueImpl(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { - std::string pool = pool_; - std::string nspace = db_namespace_; - // Read the real DB key into the DB base object try { std::vector data; @@ -71,8 +68,8 @@ RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& } catch (eckit::RadosEntityNotFoundException& e) { - throw fdb5::DatabaseNotFoundException(std::string("RadosCatalogue database not found ") + "(pool: '" + pool + - "', namespace: '" + nspace + "')"); + throw fdb5::DatabaseNotFoundException(std::string("RadosCatalogue database not found ") + "(pool: '" + pool_ + + "', namespace: '" + db_namespace_ + "')"); } } diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index d8b7a95fa..72fc646c1 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -108,9 +108,6 @@ bool RadosCatalogueWriter::createIndex(const Key& /* idxKey */, size_t /* datumK bool RadosCatalogueWriter::selectIndex(const Key& key) { - std::string pool = pool_; - std::string nspace = db_namespace_; - currentIndexKey_ = key; if (indexes_.find(key) == indexes_.end()) { @@ -126,7 +123,7 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { firstIndexWrite_ = true; - indexes_[key] = Index(new fdb5::RadosIndex(key, eckit::RadosNamespace{pool, nspace})); + indexes_[key] = Index(new fdb5::RadosIndex(key, eckit::RadosNamespace{pool_, db_namespace_})); /// index index kv in catalogue kv std::string nstr{indexes_[key].location().uri().asString()}; @@ -162,12 +159,9 @@ const Index& RadosCatalogueWriter::currentIndex() { return current_; } -void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, +void RadosCatalogueWriter::archive(const Key& /* idxKey */, const Key& datumKey, std::shared_ptr fieldLocation) { - std::string pool = pool_; - std::string nspace = db_namespace_; - if (current_.null()) { ASSERT(!currentIndexKey_.empty()); selectIndex(currentIndexKey_); @@ -202,17 +196,16 @@ void RadosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, current_.put(datumKey, field); + auto* radosIndex = dynamic_cast(current_.content()); + ASSERT(radosIndex); + if (firstIndexWrite_) { - dynamic_cast(current_.content())->putAxisNames(axisNames); + radosIndex->putAxisNames(axisNames); firstIndexWrite_ = false; } - if (axesToExpand.empty()) { - return; - } - while (!axesToExpand.empty()) { - dynamic_cast(current_.content())->putAxisValue(axesToExpand.back(), valuesToAdd.back()); + radosIndex->putAxisValue(axesToExpand.back(), valuesToAdd.back()); axesToExpand.pop_back(); valuesToAdd.pop_back(); } From 9570a09e28eb5529d8d58f733f2571adbc93fcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 13:47:46 +0200 Subject: [PATCH 092/109] docs(rados): cleanup --- src/fdb5/rados/RadosCatalogue.cc | 33 ++-------- src/fdb5/rados/RadosIndex.cc | 80 +++--------------------- src/fdb5/rados/RadosIndex.h | 13 ++-- src/fdb5/rados/RadosLazyFieldLocation.cc | 2 +- src/fdb5/rados/RadosLazyFieldLocation.h | 10 +-- 5 files changed, 26 insertions(+), 112 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index 84f44616d..a91237929 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -98,11 +98,6 @@ void RadosCatalogue::loadSchema() { eckit::Timer timer("RadosCatalogue::loadSchema()", eckit::Log::debug()); - /// @note: performed RPCs: - /// - daos_obj_generate_oid - /// - daos_kv_open - /// - daos_kv_get without a buffer - /// - daos_kv_get std::vector data; db_kv_->getMemoryStream(data, "schema", "DB Key-Value"); @@ -114,39 +109,23 @@ void RadosCatalogue::loadSchema() { std::vector RadosCatalogue::indexes(bool) const { - /// @note: sorted is not implemented as is not necessary in this backend. - - /// @note: performed RPCs: - /// - db kv open (daos_kv_open) - /// - db kv list keys (daos_kv_list) - + // `sorted` is intentionally ignored; the RADOS backend does not need ordered enumeration. std::vector res; for (const auto& key : db_kv_->keys()) { - /// @todo: document these well. Single source these reserved values. - /// Ensure where appropriate that user-provided keys do not collide. + // "schema" and "key" are reserved DB-KV entries and never index locations. if (key == "schema" || key == "key") { continue; } - /// @note: performed RPCs: - /// - db kv get index location size (daos_kv_get without a buffer) - /// - db kv get index location (daos_kv_get) std::vector v; auto m = db_kv_->getMemoryStream(v, key, "DB kv"); eckit::URI uri(std::string(v.begin(), v.end())); - /// @note: performed RPCs: - /// - index kv open (daos_kv_open) - /// - index kv get size (daos_kv_get without a buffer) - /// - index kv get key (daos_kv_get) - /// @note: the following three lines intend to check whether the index kv exists - /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, - /// so it is not useful on its own to check whether the index KV existed or not. - /// Instead, presence of a "key" key in the KV is used to determine if the index - /// KV existed. + // The RadosKeyValue constructor does not itself verify the object exists; presence of a + // "key" entry is used as the existence signal for the index KV. eckit::RadosKeyValue index_kv{uri}; std::optional index_key; try { @@ -155,8 +134,8 @@ std::vector RadosCatalogue::indexes(bool) const { index_key.emplace(ms); } catch (eckit::RadosEntityNotFoundException& e) { - continue; /// @note: the index_kv may not exist after a failed wipe - /// @todo: the index_kv may exist even if it does not have the "key" key + // Stale index_kv left behind by a failed wipe; skip. + continue; } res.push_back(Index(new fdb5::RadosIndex(index_key.value(), index_kv, false))); diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 1901232eb..8da847526 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -51,11 +51,7 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : location_(eckit::RadosKeyValue{name.pool().name(), name.name(), key.valuesToString()}, 0), idx_kv_(location_.radosName().uri()) { - /// @note: performed RPCs: - /// - generate index kv oid (daos_obj_generate_oid) - /// - create/open index kv (daos_kv_open) - - /// write indexKey under "key" + // Persist indexKey under "key" so the index KV can later be identified when reopened. eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); @@ -64,8 +60,6 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : hs << key; } - /// @note: performed RPCs: - /// - record index key into index kv (daos_kv_put) idx_kv_.put("key", h.data(), hs.bytesWritten()); } @@ -101,11 +95,6 @@ void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) void RadosIndex::updateAxes() { - /// @note: performed RPCs: - /// - ensure axis kv exists (daos_obj_open) - - /// @note: performed RPCs: - /// - get axes key size and content (daos_kv_get without buffer + daos_kv_get) std::vector axes_data; idx_kv_.getMemoryStream(axes_data, "axes", "index kv"); @@ -114,14 +103,9 @@ void RadosIndex::updateAxes() { parse(std::string(axes_data.begin(), axes_data.end()), axis_names); std::string indexKey{key_.valuesToString()}; for (const auto& name : axis_names) { - /// @note: performed RPCs: - /// - generate axis kv oid (daos_obj_generate_oid) - /// - ensure axis kv exists (daos_obj_open) eckit::RadosKeyValue axis_kv{idx_kv_.nspace().pool().name(), idx_kv_.nspace().name(), indexKey + std::string{"."} + name}; - /// @note: performed RPCs: - /// - one or more kv list (daos_kv_list) axes_.insert(name, axis_kv.keys()); } @@ -130,19 +114,13 @@ void RadosIndex::updateAxes() { bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { - /// @note: performed RPCs: - /// - ensure index kv exists (daos_obj_open) - std::string query{key.valuesToString()}; try { - - /// @note: performed RPCs: - /// - retrieve field array location from index kv (daos_kv_get) std::vector loc_data; eckit::MemoryStream ms = idx_kv_.getMemoryStream(loc_data, query, "index kv"); - /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. + // Timestamp is read for informational purposes only; see note in RadosIndex::add. time_t ts; ms >> ts; @@ -150,16 +128,9 @@ bool RadosIndex::get(const Key& key, const Key& remapKey, Field& field) const { field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); } catch (eckit::RadosEntityNotFoundException& e) { - - /// @note: performed RPCs: - /// - close index kv (daos_obj_close) - return false; } - /// @note: performed RPCs: - /// - close index kv (daos_obj_close) - return true; } @@ -170,25 +141,13 @@ void RadosIndex::add(const Key& key, const Field& field) { h.openForWrite(eckit::Length(0)); { eckit::AutoClose closer(h); - /// @note: in the POSIX back-end, keeping a timestamp per index is necessary, to allow - /// determining which was the latest indexed field in cases where multiple processes - /// index a same field or in cases where multiple catalogues are combined with DistFDB. - /// In the DAOS back-end, however, determining the latest indexed field is straigthforward - /// as all parallel processes writing fields for a same index key will share a DAOS - /// key-value, and the last indexing will supersede the previous ones. - /// DistFDB will be obsoleted in favour of a centralised catalogue mechanism which can - /// index fields on multiple catalogues. - /// Therefore keeping timestamps in DAOS should not be necessary. - /// They are kept for now only for informational purposes. + // Timestamp kept per-entry for informational purposes; correctness does not depend on it because + // parallel writers targeting the same index key share this KV and last-write-wins. takeTimestamp(); hs << timestamp(); hs << field.location(); } - /// @note: performed RPCs: - /// - ensure index kv exists (daos_obj_open) - /// - record field key and location into index kv (daos_kv_put) - /// - close index kv when destroyed (daos_obj_close) idx_kv_.put(key.valuesToString(), h.data(), hs.bytesWritten()); } @@ -199,26 +158,16 @@ void RadosIndex::entries(EntryVisitor& visitor) const { // Allow the visitor to selectively decline to visit the entries in this index if (visitor.visitIndex(instantIndex)) { - /// @note: performed RPCs: - /// - index kv open (daos_obj_open) - /// - index kv list keys (daos_kv_list) - for (const auto& key : idx_kv_.keys()) { if (key == "axes" || key == "key") { continue; } - /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each - /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field - /// keys present in an index -- without retrieving the actual location --, and - /// ListVisitor::visitDatum is called for each (see note at the top of DaosLazyFieldLocation.h). - /// When a field key is matched in visitDatum, DaosLazyFieldLocation::stableLocation is called, - /// which in turn calls this method here and triggers retrieval and deserialisation of the - /// indexed DaosFieldLocation, and returns it. Since the deserialised instance is of a - /// polymorphic class, it needs to be reanimated. - fdb5::FieldLocation* loc = new fdb5::RadosLazyFieldLocation(location_.radosName(), key); - fdb5::Field field(std::move(*loc), time_t(), fdb5::FieldDetails()); + // Build a lazy location so ListVisitor::visitDatum can filter without triggering a KV read. + // The real FieldLocation is retrieved and reanimated only when stableLocation() is called. + auto loc = std::make_shared(location_.radosName(), key); + fdb5::Field field(loc, time_t(), fdb5::FieldDetails()); visitor.visitDatum(field, key); } } @@ -226,17 +175,8 @@ void RadosIndex::entries(EntryVisitor& visitor) const { std::vector RadosIndex::dataURIs() const { - /// @note: if daos index + daos store, this will return a uri to a DAOS array for each indexed field - /// @note: if daos index + posix store, this will return a vector of unique uris to all referenced posix files - /// in this index (one for each writer process that has written to the index) - /// @note: in the case where we have a daos store, the current implementation of dataURIs is unnecessarily - /// inefficient. - /// This method is only called in DaosWipeVisitor, where the uris obtained from this method are processed to - /// obtain unique store container paths - will always result in just one container uri! Having a URI store for - /// each index in DAOS could make this process more efficient, but it would imply more KV operations and slow down - /// field writes. - /// @note: in the case where we have a posix store there will be more than one unique store file paths. The current - /// implementation is still inefficient but preferred to maintaining a URI store in the DAOS catalogue + // Iterates the index KV; each entry is a serialised RadosFieldLocation. Duplicate URIs + // are collapsed via std::set. Callers (e.g. wipe) only require the set of referenced objects. std::set res; diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index 2299134e2..68b4ee0d6 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -39,16 +39,15 @@ class RadosIndex : public IndexBase { public: // methods - /// @note: creates a new index in DAOS, in the container pointed to by 'name' + // Creates a new index KV under `name`. RadosIndex(const Key& key, const eckit::RadosNamespace& name); - /// @note: used to represent and operate with an index which already exists in DAOS + // Wraps an already-existing index KV. RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool readAxes = true); void flock() const override { NOTIMP; } void funlock() const override { NOTIMP; } - /// @note: these methods are required for RadosCatalogueWriter to directly manipulate - /// idx_kv_ and axis_kvs_ within the RadosIndex. + // Exposed so RadosCatalogueWriter can persist axis metadata into idx_kv_ / axis_kvs_. void putAxisNames(const std::string& names); void putAxisValue(const std::string& axis, const std::string& value); @@ -60,8 +59,8 @@ class RadosIndex : public IndexBase { bool dirty() const override { NOTIMP; } void open() override { NOTIMP; }; - /// @note: the Rados KV index holds no open file/handle state, so closing is a no-op. - /// This must not throw: it is invoked during normal read/list flows via eckit::AutoCloser. + // The RADOS KV index holds no open file/handle state, so closing is a no-op. + // Must not throw: invoked during normal read/list flows via eckit::AutoCloser. void close() override {} void reopen() override { NOTIMP; } @@ -80,7 +79,7 @@ class RadosIndex : public IndexBase { IndexStats statistics() const override { NOTIMP; } - /// @note: reads complete axis info from DAOS. + // Rehydrates the complete axis info from RADOS. void updateAxes(); private: // members diff --git a/src/fdb5/rados/RadosLazyFieldLocation.cc b/src/fdb5/rados/RadosLazyFieldLocation.cc index 4581e5bc1..cd58342c2 100644 --- a/src/fdb5/rados/RadosLazyFieldLocation.cc +++ b/src/fdb5/rados/RadosLazyFieldLocation.cc @@ -63,7 +63,7 @@ std::unique_ptr& RadosLazyFieldLocation::realise() const { std::vector data; eckit::MemoryStream ms = index_.getMemoryStream(data, key_, "index kv"); - /// @note: timestamp read for informational purposes. See note in DaosIndex::add. + // Timestamp is read for informational purposes only; see note in RadosIndex::add. time_t ts; ms >> ts; diff --git a/src/fdb5/rados/RadosLazyFieldLocation.h b/src/fdb5/rados/RadosLazyFieldLocation.h index 396d3e1ba..73f3d5fdc 100644 --- a/src/fdb5/rados/RadosLazyFieldLocation.h +++ b/src/fdb5/rados/RadosLazyFieldLocation.h @@ -25,13 +25,9 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// @note: used in fdb-list index visiting, in DaosIndex::entries. During -/// visitation, DaosFieldLocations are built, which normally require -/// retrieving the location information from DAOS, inflicting RPCs. -/// This DaosLazyFieldLocation, instead, remains empty and the actual -/// information is only be retrieved from DAOS when stableLocation() -/// is called. This allows the visiting mechanism to discard unmatching -/// FieldLocations before any RPC is performed for them. +// Used by fdb-list index visiting in RadosIndex::entries. Instances remain empty until the +// visitor accepts the enclosing key; only then does stableLocation() trigger the RADOS read +// and reanimate the concrete RadosFieldLocation. This avoids RPCs for unmatched keys. class RadosLazyFieldLocation : public FieldLocation { public: From a0161b506f868fee1385f7cb867fcf2d53166de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 14:01:05 +0200 Subject: [PATCH 093/109] docs(rados): cleanup --- src/fdb5/rados/RadosCatalogue.cc | 12 +++++------- src/fdb5/rados/RadosCatalogue.h | 6 +++--- src/fdb5/rados/RadosCatalogueWriter.cc | 1 - src/fdb5/rados/RadosCommon.cc | 8 +++----- src/fdb5/rados/RadosIndex.cc | 5 +---- src/fdb5/rados/RadosStore.cc | 14 +++++++------- src/fdb5/rados/RadosStore.h | 2 +- 7 files changed, 20 insertions(+), 28 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index a91237929..359da7396 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -50,10 +50,8 @@ namespace fdb5 { RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : CatalogueImpl(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) { - /// TODO: apply the mechanism in RootManager::directory, using - /// FileSpaceTables to determine root_pool_name_ according to key - /// and using DbPathNamerTables to determine db_cont_name_ according - /// to key + /// @todo: derive pool_ and db_namespace_ from the key via RootManager (FileSpaceTables, + /// DbPathNamerTables) instead of the current fixed prefix + values-string scheme. } RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, @@ -159,9 +157,9 @@ bool RadosCatalogue::uriBelongs(const eckit::URI& uri) const { //---------------------------------------------------------------------------------------------------------------------- -/// @note: Catalogue and store share the same Rados namespace, so wipe reports every non-safe object -/// found there as unrecognised. The `WipeCoordinator` then cross-checks store and catalogue -/// `uriBelongs()` to attribute each unknown to the correct owner. +// Catalogue and store share the same RADOS namespace: wipe reports every non-safe object found +// there as unrecognised, and the WipeCoordinator cross-checks store and catalogue `uriBelongs()` +// to attribute each unknown to the correct owner. CatalogueWipeState RadosCatalogue::wipeInit() const { return CatalogueWipeState{dbKey_, config()}; diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h index 7ca7eed1a..31f361eda 100644 --- a/src/fdb5/rados/RadosCatalogue.h +++ b/src/fdb5/rados/RadosCatalogue.h @@ -84,8 +84,8 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { std::set& data) const override {} // Control access properties of the DB - /// @todo: control identifiers are not persisted for RADOS yet; wipe/coordinator invocations rely on default-enabled - /// semantics. + // @todo: control identifiers are not persisted for RADOS yet; wipe/coordinator invocations rely + // on default-enabled semantics. void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override {} const Rule& rule() const override; @@ -94,7 +94,7 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { void maskIndexEntries(const std::set& indexes) const override; - /// Wipe-related methods + // Wipe-related methods CatalogueWipeState wipeInit() const override; bool markIndexForWipe(const Index& index, bool include, CatalogueWipeState& wipeState) const override; void finaliseWipeState(CatalogueWipeState& wipeState) const override; diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index 72fc646c1..fb77a3e7d 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -125,7 +125,6 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { indexes_[key] = Index(new fdb5::RadosIndex(key, eckit::RadosNamespace{pool_, db_namespace_})); - /// index index kv in catalogue kv std::string nstr{indexes_[key].location().uri().asString()}; db_kv_->put(key.valuesToString(), nstr.data(), nstr.length()); } diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index 98be6066f..e700a2b2f 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -43,11 +43,9 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con RadosCommon::RadosCommon(const Config& config, const std::string& component, const eckit::URI& uri) { - /// @note: this constructor is triggered both by DB::buildReader in EntryVisitMechanism (with a - /// catalogue key-value URI, i.e. pool/namespace/oid) and by StoreFactory during wipe (with a - /// store namespace URI, i.e. pool/namespace). Only the pool and namespace are needed here, so - /// parse them directly and accept both the 2-token and 3-token forms. - + // Accepts URIs from two callers: DB::buildReader in EntryVisitMechanism supplies a catalogue KV + // URI (pool/namespace/oid); StoreFactory during wipe supplies a store namespace URI + // (pool/namespace). Only pool and namespace are needed here. const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); ASSERT(parts.size() == 2 || parts.size() == 3); diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 8da847526..3d148d12f 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -18,7 +18,6 @@ #include "fdb5/database/Key.h" #include "fdb5/rados/RadosLazyFieldLocation.h" -#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/DataHandle.h" #include "eckit/io/Length.h" @@ -34,7 +33,6 @@ #include // for PATH_MAX #include #include -#include #include #include #include @@ -175,8 +173,7 @@ void RadosIndex::entries(EntryVisitor& visitor) const { std::vector RadosIndex::dataURIs() const { - // Iterates the index KV; each entry is a serialised RadosFieldLocation. Duplicate URIs - // are collapsed via std::set. Callers (e.g. wipe) only require the set of referenced objects. + // Iterates the index KV; each entry is a serialised RadosFieldLocation. std::set res; diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index e35690a0c..5fb1c7828 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -214,11 +214,12 @@ void RadosStore::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- -/// @note: the database maps to a Rados namespace. Only the namespace holding this database's -/// objects is ever touched here. +// The database maps to a RADOS namespace; only the namespace holding this database's objects is +// ever touched by the wipe-related methods below. void RadosStore::finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) { - /// @note: doit and unsafeWipeAll do not affect the preparation of a Rados store wipe. + + // `doit` and `unsafeWipeAll` do not affect the preparation of a RADOS store wipe. const std::set& dataURIs = storeState.includedDataURIs(); // included according to cat const std::set& safeURIs = storeState.safeURIs(); // excluded according to cat @@ -298,9 +299,8 @@ void RadosStore::doWipeEmptyDatabase() const { bool RadosStore::doUnsafeFullWipe() const { - /// @note: if the database namespace/pool also holds a catalogue, the wiping is skipped as the - /// catalogue is in charge. The presence of a "key" entry in the database key-value is used to - /// determine whether a catalogue exists here. + // If the database namespace also holds a catalogue, skip: the catalogue-driven wipe owns the + // namespace. Presence of a "key" entry in the DB KV is used as the catalogue-exists signal. if (db_kv_ && (!db_kv_->exists() || !db_kv_->has("key"))) { eckit::RadosNamespace db{pool_, db_namespace_}; @@ -320,7 +320,7 @@ std::vector RadosStore::getAuxiliaryURIs(const eckit::URI& /*uri*/, //---------------------------------------------------------------------------------------------------------------------- -/// @note: unique name generation copied from LocalPathName::unique. +// Unique name generation copied from eckit::LocalPathName::unique. static eckit::StaticMutex local_mutex; eckit::RadosObject RadosStore::generateDataObject(const Key& key) const { diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index ac3c453db..2f8b5bf6f 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -66,7 +66,7 @@ class RadosStore : public Store, public RadosCommon { void checkUID() const override { /* nothing to do */ } - /// Wipe-related methods + // Wipe-related methods void finaliseWipeState(StoreWipeState& storeState, bool doit, bool unsafeWipeAll) override; bool doWipeUnknowns(const std::set& unknownURIs) const override; bool doWipeURIs(const StoreWipeState& wipeState) const override; From d2801b6fd63d3d92d9ca7cd488322cb043d57f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 14:22:43 +0200 Subject: [PATCH 094/109] feat(rados): wipe -> wipe --- src/fdb5/rados/README | 5 +++-- src/fdb5/rados/RadosCatalogueWriter.h | 3 ++- src/fdb5/rados/RadosIndex.cc | 2 +- src/fdb5/rados/RadosStore.cc | 4 ++-- src/fdb5/rados/RadosStore.h | 3 ++- tests/fdb/rados/test_rados_catalogue.cc | 5 +++++ 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index 83ba1114b..b01d5c8d6 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -4,8 +4,9 @@ Running RadosStore unit tests against Ceph on Docker on mac: Supported RADOS backend scope: ============================== -The backend supports archive, retrieve, and reopening catalogues by URI. Archive calls on a single RadosStore must be -serialised by the caller. Catalogue-side wipe, purge, statistics, move, and control operations are not implemented. +The backend supports archive, retrieve, list, wipe, and reopening catalogues by URI. Archive calls +on a single RadosStore must be serialised by the caller. Catalogue-side purge, statistics, move, +control (hide/mount), and overlay operations are not implemented. RADOS tests require eckit to be built with RADOS support. diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h index 7d900d48d..9b78a3938 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.h +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -36,7 +36,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// DB that implements the FDB on Rados +/// DB writer that implements the FDB on Rados. +/// Not thread-safe: archive/flush/close calls on a single instance must be serialised by the caller. class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 3d148d12f..bd1330625 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -193,7 +193,7 @@ std::vector RadosIndex::dataURIs() const { res.insert(fl->uri()); } - return std::vector(res.begin(), res.end()); + return {res.begin(), res.end()}; } //----------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 5fb1c7828..332e9dd52 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -190,8 +190,8 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost logVerbose << "destroy Rados namespace: "; logAlways << ns.str() << std::endl; - if (doit) { - ns.destroy(); /// @todo: ensureDestroyed? + if (doit && ns.exists()) { + ns.destroy(); } } else { // object diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 2f8b5bf6f..3dd401352 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -37,7 +37,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// Store that implements the FDB on CEPH object store +/// Store that implements the FDB on CEPH object store. +/// Not thread-safe: archive/flush/close calls on a single instance must be serialised by the caller. class RadosStore : public Store, public RadosCommon { diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index a73674e19..791af81be 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -756,6 +756,11 @@ CASE("RadosCatalogue tests") { count++; } EXPECT(count == 0); + + // Wipe an already-wiped DB. The store-side namespace destroy path must be idempotent so + // recovering from a partial wipe does not raise. + EXPECT_NO_THROW(fdb2.wipe(db_req, true)); + fdb2.flush(); } // SECTION("OPTIONAL SCHEMA KEYS") { From a6ddaf75e5d96d0e519099de4778bbac3e112f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 14:30:30 +0200 Subject: [PATCH 095/109] feat(rados): removed remapkey, dedup read_db_key --- src/fdb5/rados/RadosCatalogue.cc | 27 +++---------------------- src/fdb5/rados/RadosCommon.cc | 9 +++++++++ src/fdb5/rados/RadosCommon.h | 5 ++++- src/fdb5/rados/RadosEngine.cc | 8 +++----- src/fdb5/rados/RadosFieldLocation.cc | 10 ++++++--- src/fdb5/rados/RadosFieldLocation.h | 2 ++ src/fdb5/rados/RadosStore.cc | 2 +- tests/fdb/rados/test_rados_catalogue.cc | 6 +++--- tests/fdb/rados/test_rados_store.cc | 8 ++++++++ 9 files changed, 40 insertions(+), 37 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index 359da7396..cab59439e 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -48,46 +48,33 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- RadosCatalogue::RadosCatalogue(const Key& key, const fdb5::Config& config) : - CatalogueImpl(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) { - - /// @todo: derive pool_ and db_namespace_ from the key via RootManager (FileSpaceTables, - /// DbPathNamerTables) instead of the current fixed prefix + values-string scheme. -} + CatalogueImpl(key, ControlIdentifiers{}, config), RadosCommon(config, "catalogue", key) {} RadosCatalogue::RadosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) : CatalogueImpl(Key(), controlIdentifiers, config), RadosCommon(config, "catalogue", uri) { - - // Read the real DB key into the DB base object try { - std::vector data; - eckit::MemoryStream ms = db_kv_->getMemoryStream(data, "key", "DB kv"); - dbKey_ = fdb5::Key(ms); + dbKey_ = read_db_key(*db_kv_); } catch (eckit::RadosEntityNotFoundException& e) { - throw fdb5::DatabaseNotFoundException(std::string("RadosCatalogue database not found ") + "(pool: '" + pool_ + "', namespace: '" + db_namespace_ + "')"); } } bool RadosCatalogue::exists() const { - return db_kv_->exists(); } eckit::URI RadosCatalogue::uri() const { - return db_kv_->nspace().uri(); } const Schema& RadosCatalogue::schema() const { - return schema_; } const Rule& RadosCatalogue::rule() const { - ASSERT(rule_); return *rule_; } @@ -112,7 +99,6 @@ std::vector RadosCatalogue::indexes(bool) const { for (const auto& key : db_kv_->keys()) { - // "schema" and "key" are reserved DB-KV entries and never index locations. if (key == "schema" || key == "key") { continue; } @@ -122,8 +108,6 @@ std::vector RadosCatalogue::indexes(bool) const { eckit::URI uri(std::string(v.begin(), v.end())); - // The RadosKeyValue constructor does not itself verify the object exists; presence of a - // "key" entry is used as the existence signal for the index KV. eckit::RadosKeyValue index_kv{uri}; std::optional index_key; try { @@ -132,7 +116,6 @@ std::vector RadosCatalogue::indexes(bool) const { index_key.emplace(ms); } catch (eckit::RadosEntityNotFoundException& e) { - // Stale index_kv left behind by a failed wipe; skip. continue; } @@ -157,12 +140,8 @@ bool RadosCatalogue::uriBelongs(const eckit::URI& uri) const { //---------------------------------------------------------------------------------------------------------------------- -// Catalogue and store share the same RADOS namespace: wipe reports every non-safe object found -// there as unrecognised, and the WipeCoordinator cross-checks store and catalogue `uriBelongs()` -// to attribute each unknown to the correct owner. - CatalogueWipeState RadosCatalogue::wipeInit() const { - return CatalogueWipeState{dbKey_, config()}; + return {dbKey_, config()}; } void RadosCatalogue::maskIndexEntries(const std::set& indexes) const { diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index e700a2b2f..ee374e319 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -17,6 +17,7 @@ #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" +#include "eckit/serialisation/MemoryStream.h" #include "eckit/utils/Tokenizer.h" #include @@ -28,6 +29,14 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +fdb5::Key read_db_key(const eckit::RadosKeyValue& db_kv) { + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + return fdb5::Key(ms); +} + +//---------------------------------------------------------------------------------------------------------------------- + RadosCommon::RadosCommon(const Config& config, const std::string& component, const Key& key) { std::vector valid{"catalogue", "store"}; diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 8b938280f..5a393bf4e 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -15,7 +15,6 @@ #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" -#include "fdb5/fdb5_config.h" #include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" @@ -26,6 +25,10 @@ namespace fdb5 { +// Reads the persisted `key` entry from a RADOS DB KV and deserialises it into an fdb5::Key. +// Throws eckit::RadosEntityNotFoundException if the DB KV or the `key` entry is missing. +fdb5::Key read_db_key(const eckit::RadosKeyValue& db_kv); + class RadosCommon { public: // methods diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index 6c81c6386..9adef59fd 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -14,6 +14,7 @@ #include "fdb5/LibFdb5.h" #include "fdb5/database/Engine.h" #include "fdb5/database/Key.h" +#include "fdb5/rados/RadosCommon.h" #include "metkit/mars/MarsRequest.h" @@ -92,11 +93,8 @@ std::vector RadosEngine::visitableLocations(const std::function data; - eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); - fdb5::Key db_key(ms); + eckit::RadosKeyValue db_kv{uri}; + fdb5::Key db_key = read_db_key(db_kv); if (matches(db_key)) { eckit::Log::debug() << " found match with " << rootKv_->uri() << " at key " << k << std::endl; diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index 34002ef84..d9e23fadf 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -40,10 +40,14 @@ RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri) : FieldLocation(uri) {} -/// @todo: remove remapKey from signature and always pass empty Key to FieldLocation +RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length) : + FieldLocation(uri, offset, length, Key{}) {} + +// Kept for FieldLocationBuilder factory compatibility; `remapKey` is unused because the RADOS +// backend does not support key remapping. RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, - const Key& remapKey) : - FieldLocation(uri, offset, length, remapKey) {} + const Key& /* remapKey */) : + RadosFieldLocation(uri, offset, length) {} RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) : FieldLocation(s) {} diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h index 8526bc6f1..5e18dd7f3 100644 --- a/src/fdb5/rados/RadosFieldLocation.h +++ b/src/fdb5/rados/RadosFieldLocation.h @@ -34,6 +34,8 @@ class RadosFieldLocation : public FieldLocation { RadosFieldLocation(const RadosFieldLocation& rhs); RadosFieldLocation(const eckit::URI& uri); + RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length); + // Factory-only overload; `remapKey` is ignored (see RadosFieldLocation.cc). RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); RadosFieldLocation(eckit::Stream&); diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 332e9dd52..7c4f8a27d 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -146,7 +146,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const v ASSERT(len == length); - return std::make_unique(o.uri(), offset, length, fdb5::Key{}); + return std::make_unique(o.uri(), offset, length); } RadosStore::~RadosStore() { diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 791af81be..6b6f67a74 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -205,8 +205,8 @@ CASE("RadosCatalogue tests") { // archive - std::unique_ptr loc(new fdb5::RadosFieldLocation( - eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key{})); + std::unique_ptr loc( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1))); eckit::URI catalogue_uri; { @@ -450,7 +450,7 @@ CASE("RadosCatalogue tests") { fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); auto location = std::make_unique(eckit::URI{"rados", std::string(600, 'x')}, - eckit::Offset(0), eckit::Length(1), fdb5::Key{}); + eckit::Offset(0), eckit::Length(1)); { fdb5::RadosCatalogueWriter writer{db_key, config}; diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 5852ae60b..3853d1258 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -256,6 +256,14 @@ CASE("RadosStore tests") { EXPECT_THROWS_AS((fdb5::Engine::backend("rados").location(db_key, config)), eckit::UserError); } + SECTION("RadosFieldLocation three-argument constructor forwards an empty remapKey") { + fdb5::RadosFieldLocation loc{eckit::URI{"rados", "pool/ns/obj"}, eckit::Offset(0), eckit::Length(1)}; + EXPECT(loc.remapKey().empty()); + EXPECT(loc.uri().name() == "pool/ns/obj"); + EXPECT(loc.offset() == eckit::Offset(0)); + EXPECT(loc.length() == eckit::Length(1)); + } + SECTION("with POSIX Catalogue") { std::string test_id = "test-store2"; From bfe48dfc9598ee32b6ecf6981a7d24ba5955f65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 14:36:02 +0200 Subject: [PATCH 096/109] feat(rados): add stats --- src/fdb5/CMakeLists.txt | 2 + src/fdb5/rados/RadosCatalogueReader.cc | 21 ++++++++ src/fdb5/rados/RadosCatalogueReader.h | 2 +- src/fdb5/rados/RadosCatalogueWriter.h | 2 +- src/fdb5/rados/RadosIndex.h | 3 ++ src/fdb5/rados/RadosStats.cc | 65 ++++++++++++++++++++++++ src/fdb5/rados/RadosStats.h | 59 ++++++++++++++++++++++ tests/fdb/rados/test_rados_catalogue.cc | 67 +++++++++++++++++++++++-- 8 files changed, 216 insertions(+), 5 deletions(-) create mode 100644 src/fdb5/rados/RadosStats.cc create mode 100644 src/fdb5/rados/RadosStats.h diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 20b082a6c..7b69c239d 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -368,6 +368,8 @@ if( HAVE_RADOSFDB ) rados/RadosLazyFieldLocation.h rados/RadosEngine.cc rados/RadosEngine.h + rados/RadosStats.cc + rados/RadosStats.h ) endif() diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index 0745b988e..4fd6e5994 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -18,6 +18,7 @@ #include "fdb5/database/Key.h" #include "fdb5/rados/RadosCatalogue.h" #include "fdb5/rados/RadosIndex.h" +#include "fdb5/rados/RadosStats.h" #include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosException.h" @@ -76,6 +77,26 @@ bool RadosCatalogueReader::open() { return true; } +DbStats RadosCatalogueReader::stats() const { + + auto* content = new RadosDbStats(); + content->dbCount_ = 1; + + for (const auto& indexEntry : indexes(false)) { + content->indexCount_++; + const auto* radosIndex = dynamic_cast(indexEntry.content()); + ASSERT(radosIndex); + for (const auto& key : radosIndex->idx_kv().keys()) { + if (key == "axes" || key == "key") { + continue; + } + content->fieldCount_++; + } + } + + return DbStats(content); +} + std::optional RadosCatalogueReader::computeAxis(const std::string& keyword) const { Axis s; diff --git a/src/fdb5/rados/RadosCatalogueReader.h b/src/fdb5/rados/RadosCatalogueReader.h index d7122c324..dcac4f97a 100644 --- a/src/fdb5/rados/RadosCatalogueReader.h +++ b/src/fdb5/rados/RadosCatalogueReader.h @@ -42,7 +42,7 @@ class RadosCatalogueReader : public RadosCatalogue, public CatalogueReader { RadosCatalogueReader(const Key& key, const fdb5::Config& config); RadosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config); - DbStats stats() const override { NOTIMP; } + DbStats stats() const override; bool selectIndex(const Key& key) override; void deselectIndex() override; diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h index 9b78a3938..109717270 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.h +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -45,7 +45,7 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { RadosCatalogueWriter(const Key& key, const fdb5::Config& config); RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config); - virtual ~RadosCatalogueWriter() override; + ~RadosCatalogueWriter() override; void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index 68b4ee0d6..114cd8fc6 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -51,6 +51,9 @@ class RadosIndex : public IndexBase { void putAxisNames(const std::string& names); void putAxisValue(const std::string& axis, const std::string& value); + // Exposed so RadosCatalogueReader can enumerate field entries directly for stats. + const eckit::RadosKeyValue& idx_kv() const { return idx_kv_; } + private: // methods const IndexLocation& location() const override { return location_; } diff --git a/src/fdb5/rados/RadosStats.cc b/src/fdb5/rados/RadosStats.cc new file mode 100644 index 000000000..1d943268f --- /dev/null +++ b/src/fdb5/rados/RadosStats.cc @@ -0,0 +1,65 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/rados/RadosStats.h" + +#include "fdb5/database/DbStats.h" + +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" + +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +::eckit::ClassSpec RadosDbStats::classSpec_ = { + &DbStatsContent::classSpec(), + "RadosDbStats", +}; +::eckit::Reanimator RadosDbStats::reanimator_; + +//---------------------------------------------------------------------------------------------------------------------- + +RadosDbStats::RadosDbStats() : dbCount_(0), indexCount_(0), fieldCount_(0) {} + +RadosDbStats::RadosDbStats(eckit::Stream& out) { + out >> dbCount_; + out >> indexCount_; + out >> fieldCount_; +} + +RadosDbStats& RadosDbStats::operator+=(const RadosDbStats& rhs) { + dbCount_ += rhs.dbCount_; + indexCount_ += rhs.indexCount_; + fieldCount_ += rhs.fieldCount_; + return *this; +} + +void RadosDbStats::add(const DbStatsContent& rhs) { + *this += dynamic_cast(rhs); +} + +void RadosDbStats::report(std::ostream& out, const char* indent) const { + reportCount(out, "Databases", dbCount_, indent); + reportCount(out, "Indexes", indexCount_, indent); + reportCount(out, "Fields", fieldCount_, indent); +} + +void RadosDbStats::encode(eckit::Stream& out) const { + out << dbCount_; + out << indexCount_; + out << fieldCount_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosStats.h b/src/fdb5/rados/RadosStats.h new file mode 100644 index 000000000..bccd6d10b --- /dev/null +++ b/src/fdb5/rados/RadosStats.h @@ -0,0 +1,59 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#include "fdb5/database/DbStats.h" + +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" + +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +// Minimal DB-level statistics for the RADOS backend: databases, indexes and fields visited. +// Byte totals are intentionally omitted; adding them would require per-object HEAD reads. +class RadosDbStats : public DbStatsContent { +public: + + RadosDbStats(); + RadosDbStats(eckit::Stream&); + + static DbStats make() { return DbStats(new RadosDbStats()); } + + size_t dbCount_; + size_t indexCount_; + size_t fieldCount_; + + RadosDbStats& operator+=(const RadosDbStats& rhs); + + void add(const DbStatsContent&) override; + void report(std::ostream& out, const char* indent) const override; + +public: // For Streamable + + static const eckit::ClassSpec& classSpec() { return classSpec_; } + +protected: // For Streamable + + void encode(eckit::Stream&) const override; + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 6b6f67a74..8b300f5f7 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -15,6 +15,7 @@ #include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/DatabaseNotFoundException.h" +#include "fdb5/database/DbStats.h" #include "fdb5/database/Field.h" #include "fdb5/database/FieldLocation.h" #include "fdb5/rados/RadosCatalogueReader.h" @@ -25,8 +26,8 @@ #include "metkit/mars/MarsRequest.h" -#include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" +#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/TmpFile.h" #include "eckit/filesystem/URI.h" @@ -45,13 +46,15 @@ #include #include #include +#include #include #include #include -using namespace eckit::testing; using namespace eckit; +//---------------------------------------------------------------------------------------------------------------------- + namespace { void deldir(eckit::PathName& p) { @@ -423,6 +426,63 @@ CASE("RadosCatalogue tests") { fdb5::DatabaseNotFoundException); } + SECTION("RadosCatalogueReader::stats reports index and field counts") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Key db_key({{"a", "77"}, {"b", "77"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key_1({{"e", "5"}, {"f", "6"}}); + fdb5::Key field_key_2({{"e", "5"}, {"f", "7"}}); + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& cat = writer; + cat.selectIndex(index_key); + std::unique_ptr loc1( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, eckit::Offset(0), eckit::Length(1))); + std::unique_ptr loc2( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, eckit::Offset(1), eckit::Length(1))); + static_cast(writer).archive(index_key, field_key_1, std::move(loc1)); + static_cast(writer).archive(index_key, field_key_2, std::move(loc2)); + cat.flush(0); + } + + { + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::CatalogueReader& cr = reader; + EXPECT(cr.open()); + fdb5::DbStats stats = cr.stats(); + std::ostringstream oss; + stats.report(oss); + const std::string report = oss.str(); + // Must expose non-empty output rather than throwing NOTIMP. + EXPECT(!report.empty()); + EXPECT(report.find("Indexes") != std::string::npos); + EXPECT(report.find("Fields") != std::string::npos); + } + } + SECTION("RadosCatalogue supports large serialised field locations") { std::string config_str{ @@ -928,7 +988,8 @@ CASE("RadosCatalogue tests") { } // namespace fdb::test +//---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { - return run_tests(argc, argv); + return eckit::testing::run_tests(argc, argv); } From 82038142bbc23caf46f8a894dc678ef04126c37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 14:52:18 +0200 Subject: [PATCH 097/109] feat(rados): concurrent write --- src/fdb5/rados/RadosCatalogue.cc | 59 ++++++- src/fdb5/rados/RadosCatalogue.h | 8 +- src/fdb5/rados/RadosCatalogueReader.cc | 6 +- src/fdb5/rados/RadosCatalogueWriter.cc | 24 +-- src/fdb5/rados/RadosCatalogueWriter.h | 6 +- src/fdb5/rados/RadosCleanup.h | 3 + src/fdb5/rados/RadosIndex.cc | 34 ++-- src/fdb5/rados/RadosIndex.h | 1 - src/fdb5/rados/RadosStats.h | 9 +- tests/fdb/rados/test_rados_catalogue.cc | 213 +++++++++++++++++++----- tests/fdb/rados/test_rados_store.cc | 53 +++--- 11 files changed, 296 insertions(+), 120 deletions(-) diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index cab59439e..f0d81b4a6 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -99,7 +99,7 @@ std::vector RadosCatalogue::indexes(bool) const { for (const auto& key : db_kv_->keys()) { - if (key == "schema" || key == "key") { + if (key == "schema" || key == "key" || key.rfind("control.", 0) == 0) { continue; } @@ -305,4 +305,61 @@ bool RadosCatalogue::doUnsafeFullWipe() const { //---------------------------------------------------------------------------------------------------------------------- +namespace { + +// Reserved KV entry name for a given control identifier; must be filtered out of index enumeration. +std::string control_kv_key(ControlIdentifier id) { + switch (id) { + case ControlIdentifier::List: + return "control.list"; + case ControlIdentifier::Retrieve: + return "control.retrieve"; + case ControlIdentifier::Archive: + return "control.archive"; + case ControlIdentifier::Wipe: + return "control.wipe"; + case ControlIdentifier::UniqueRoot: + return "control.unique_root"; + default: + return ""; + } +} + +} // namespace + +void RadosCatalogue::control(const ControlAction& action, const ControlIdentifiers& identifiers) const { + + for (ControlIdentifier id : identifiers) { + const std::string key = control_kv_key(id); + if (key.empty()) { + continue; + } + switch (action) { + case ControlAction::Disable: { + const char flag = '1'; + db_kv_->put(key, &flag, 1); + break; + } + case ControlAction::Enable: + if (db_kv_->has(key)) { + db_kv_->remove(key); + } + break; + default: + eckit::Log::warning() << "RadosCatalogue::control: unexpected action " << static_cast(action) + << std::endl; + } + } +} + +bool RadosCatalogue::enabled(const ControlIdentifier& controlIdentifier) const { + const std::string key = control_kv_key(controlIdentifier); + if (key.empty()) { + return true; + } + return !db_kv_->has(key); +} + +//---------------------------------------------------------------------------------------------------------------------- + } // namespace fdb5 diff --git a/src/fdb5/rados/RadosCatalogue.h b/src/fdb5/rados/RadosCatalogue.h index 31f361eda..87d34faf5 100644 --- a/src/fdb5/rados/RadosCatalogue.h +++ b/src/fdb5/rados/RadosCatalogue.h @@ -83,10 +83,10 @@ class RadosCatalogue : public CatalogueImpl, public RadosCommon { void allMasked(std::set>& metadata, std::set& data) const override {} - // Control access properties of the DB - // @todo: control identifiers are not persisted for RADOS yet; wipe/coordinator invocations rely - // on default-enabled semantics. - void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override {} + // Control access properties of the DB. Persisted as per-identifier reserved KV entries (`control.list`, etc.) + // in the catalogue KV; absence of the entry means the identifier is enabled. + void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override; + bool enabled(const ControlIdentifier& controlIdentifier) const override; const Rule& rule() const override; diff --git a/src/fdb5/rados/RadosCatalogueReader.cc b/src/fdb5/rados/RadosCatalogueReader.cc index 4fd6e5994..4805631e7 100644 --- a/src/fdb5/rados/RadosCatalogueReader.cc +++ b/src/fdb5/rados/RadosCatalogueReader.cc @@ -13,6 +13,7 @@ #include "fdb5/LibFdb5.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/database/Catalogue.h" +#include "fdb5/database/DbStats.h" #include "fdb5/database/Field.h" #include "fdb5/database/Index.h" #include "fdb5/database/Key.h" @@ -20,6 +21,7 @@ #include "fdb5/rados/RadosIndex.h" #include "fdb5/rados/RadosStats.h" +#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosException.h" #include "eckit/io/rados/RadosKeyValue.h" @@ -87,14 +89,14 @@ DbStats RadosCatalogueReader::stats() const { const auto* radosIndex = dynamic_cast(indexEntry.content()); ASSERT(radosIndex); for (const auto& key : radosIndex->idx_kv().keys()) { - if (key == "axes" || key == "key") { + if (key == "axes" || key == "key" || key.rfind("axis.", 0) == 0) { continue; } content->fieldCount_++; } } - return DbStats(content); + return {content}; } std::optional RadosCatalogueReader::computeAxis(const std::string& keyword) const { diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index fb77a3e7d..129c71370 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -48,8 +48,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& config) : - RadosCatalogue(key, config), firstIndexWrite_(false) { +RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& config) : RadosCatalogue(key, config) { std::string db_name = db_namespace_; ASSERT(root_kv_->nspace().pool().exists()); @@ -87,12 +86,10 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key& key, const fdb5::Config& c } RadosCatalogue::loadSchema(); - - /// @todo: TocCatalogue::checkUID(); } RadosCatalogueWriter::RadosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config) : - RadosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) { + RadosCatalogue(uri, ControlIdentifiers{}, config) { RadosCatalogue::loadSchema(); } @@ -106,6 +103,10 @@ bool RadosCatalogueWriter::createIndex(const Key& /* idxKey */, size_t /* datumK return true; } +void RadosCatalogueWriter::hideContents() { + control(ControlAction::Disable, ControlIdentifier::List | ControlIdentifier::Retrieve); +} + bool RadosCatalogueWriter::selectIndex(const Key& key) { currentIndexKey_ = key; @@ -121,8 +122,6 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { } catch (eckit::RadosEntityNotFoundException& e) { - firstIndexWrite_ = true; - indexes_[key] = Index(new fdb5::RadosIndex(key, eckit::RadosNamespace{pool_, db_namespace_})); std::string nstr{indexes_[key].location().uri().asString()}; @@ -138,7 +137,6 @@ bool RadosCatalogueWriter::selectIndex(const Key& key) { void RadosCatalogueWriter::deselectIndex() { current_ = Index(); currentIndexKey_ = Key(); - firstIndexWrite_ = false; } void RadosCatalogueWriter::clean() { @@ -172,8 +170,6 @@ void RadosCatalogueWriter::archive(const Key& /* idxKey */, const Key& datumKey, std::vector axesToExpand; std::vector valuesToAdd; - std::string axisNames; - std::string sep; for (const auto& [keyword, value] : datumKey) { @@ -181,9 +177,6 @@ void RadosCatalogueWriter::archive(const Key& /* idxKey */, const Key& datumKey, continue; } - axisNames += sep + keyword; - sep = ","; - const auto& axis_set = current_.axes().values(keyword); if (!axis_set.contains(value)) { @@ -198,11 +191,6 @@ void RadosCatalogueWriter::archive(const Key& /* idxKey */, const Key& datumKey, auto* radosIndex = dynamic_cast(current_.content()); ASSERT(radosIndex); - if (firstIndexWrite_) { - radosIndex->putAxisNames(axisNames); - firstIndexWrite_ = false; - } - while (!axesToExpand.empty()) { radosIndex->putAxisValue(axesToExpand.back(), valuesToAdd.back()); axesToExpand.pop_back(); diff --git a/src/fdb5/rados/RadosCatalogueWriter.h b/src/fdb5/rados/RadosCatalogueWriter.h index 109717270..304e1124e 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.h +++ b/src/fdb5/rados/RadosCatalogueWriter.h @@ -37,7 +37,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- /// DB writer that implements the FDB on Rados. -/// Not thread-safe: archive/flush/close calls on a single instance must be serialised by the caller. +/// Not thread-safe per instance; separate writer instances may archive to the same database. class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { @@ -55,6 +55,8 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { NOTIMP; }; + void hideContents() override; + const Index& currentIndex() override; protected: // methods @@ -85,8 +87,6 @@ class RadosCatalogueWriter : public RadosCatalogue, public CatalogueWriter { IndexStore indexes_; Index current_; - - bool firstIndexWrite_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosCleanup.h b/src/fdb5/rados/RadosCleanup.h index 41da86548..7e1fcc5e6 100644 --- a/src/fdb5/rados/RadosCleanup.h +++ b/src/fdb5/rados/RadosCleanup.h @@ -8,6 +8,9 @@ * does it submit to any jurisdiction. */ +/// @author Metin Cakircali +/// @date Aug 2026 + #pragma once #include "eckit/log/Log.h" diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index bd1330625..b9451f33c 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -69,13 +69,12 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool re } } -void RadosIndex::putAxisNames(const std::string& names) { - - idx_kv_.put("axes", names.data(), names.length()); -} - void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) { + const std::string axis_marker = "axis." + axis; + const char marker = '1'; + idx_kv_.put(axis_marker, &marker, 1); + auto axis_kv = axis_kvs_.find(axis); if (axis_kv == axis_kvs_.end()) { @@ -93,12 +92,23 @@ void RadosIndex::putAxisValue(const std::string& axis, const std::string& value) void RadosIndex::updateAxes() { - std::vector axes_data; - idx_kv_.getMemoryStream(axes_data, "axes", "index kv"); + std::set axis_names; + for (const auto& key : idx_kv_.keys()) { + if (key.rfind("axis.", 0) == 0) { + axis_names.insert(key.substr(5)); + } + } + + // Compatibility with catalogues written before per-axis markers were introduced. + if (axis_names.empty() && idx_kv_.has("axes")) { + std::vector axes_data; + idx_kv_.getMemoryStream(axes_data, "axes", "index kv"); + std::vector legacy_axis_names; + eckit::Tokenizer parse(","); + parse(std::string(axes_data.begin(), axes_data.end()), legacy_axis_names); + axis_names.insert(legacy_axis_names.begin(), legacy_axis_names.end()); + } - std::vector axis_names; - eckit::Tokenizer parse(","); - parse(std::string(axes_data.begin(), axes_data.end()), axis_names); std::string indexKey{key_.valuesToString()}; for (const auto& name : axis_names) { eckit::RadosKeyValue axis_kv{idx_kv_.nspace().pool().name(), idx_kv_.nspace().name(), @@ -158,7 +168,7 @@ void RadosIndex::entries(EntryVisitor& visitor) const { for (const auto& key : idx_kv_.keys()) { - if (key == "axes" || key == "key") { + if (key == "axes" || key == "key" || key.rfind("axis.", 0) == 0) { continue; } @@ -179,7 +189,7 @@ std::vector RadosIndex::dataURIs() const { for (const auto& key : idx_kv_.keys()) { - if (key == "axes" || key == "key") { + if (key == "axes" || key == "key" || key.rfind("axis.", 0) == 0) { continue; } diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index 114cd8fc6..43724d008 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -48,7 +48,6 @@ class RadosIndex : public IndexBase { void funlock() const override { NOTIMP; } // Exposed so RadosCatalogueWriter can persist axis metadata into idx_kv_ / axis_kvs_. - void putAxisNames(const std::string& names); void putAxisValue(const std::string& axis, const std::string& value); // Exposed so RadosCatalogueReader can enumerate field entries directly for stats. diff --git a/src/fdb5/rados/RadosStats.h b/src/fdb5/rados/RadosStats.h index bccd6d10b..080683f47 100644 --- a/src/fdb5/rados/RadosStats.h +++ b/src/fdb5/rados/RadosStats.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 1996- ECMWF. + * (C) Copyright 2026- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -8,6 +8,9 @@ * does it submit to any jurisdiction. */ +/// @author Metin Cakircali +/// @date Aug 2026 + #pragma once #include "fdb5/database/DbStats.h" @@ -22,7 +25,6 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -// Minimal DB-level statistics for the RADOS backend: databases, indexes and fields visited. // Byte totals are intentionally omitted; adding them would require per-object HEAD reads. class RadosDbStats : public DbStatsContent { public: @@ -38,7 +40,8 @@ class RadosDbStats : public DbStatsContent { RadosDbStats& operator+=(const RadosDbStats& rhs); - void add(const DbStatsContent&) override; + void add(const DbStatsContent& rhs) override; + void report(std::ostream& out, const char* indent) const override; public: // For Streamable diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 8b300f5f7..bae718069 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -43,11 +43,15 @@ #include #include #include +#include +#include #include #include +#include #include #include #include +#include #include #include @@ -103,7 +107,7 @@ size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { return count; } -// temporary schema,spaces,root files common to all DAOS Catalogue tests +// temporary schema,spaces,root files common to all RADOS Catalogue tests eckit::TmpFile& schema_file() { static eckit::TmpFile f{}; @@ -134,8 +138,6 @@ CASE("Setup") { catalogue_tests_tmp_root().mkdir(); ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); - // prepare schema for tests involving DaosCatalogue - std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; std::unique_ptr hs(schema_file().fileHandle()); @@ -174,7 +176,7 @@ CASE("RadosCatalogue tests") { ensureCleanNamespaces(pool, test_id); #endif - SECTION("DaosCatalogue archive (index) and retrieve without a Store") { + SECTION("RadosCatalogue archive (index) and retrieve without a Store") { std::string config_str{ "spaces:\n" @@ -215,16 +217,16 @@ CASE("RadosCatalogue tests") { { fdb5::RadosCatalogueWriter dcatw{db_key, config}; - // fdb5::DaosName db_cont{pool_name, db_key.valuesToString()}; - // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config - // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + // fdb5::RadosName db_cont{pool_name, db_key.valuesToString()}; + // fdb5::RadosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + // fdb5::RadosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; // EXPECT(db_cont.exists()); // EXPECT(cat_kv.exists()); fdb5::Catalogue& cat = dcatw; cat.selectIndex(index_key); - // fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from - // config fdb5::DaosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; + // fdb5::RadosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from + // config fdb5::RadosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; // EXPECT(index_kv.exists()); // EXPECT(cat_kv.has(index_key.valuesToString())); @@ -233,12 +235,12 @@ CASE("RadosCatalogue tests") { cat.flush(0); catalogue_uri = cat.uri(); // EXPECT(index_kv.has(field_key.valuesToString())); - // fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; - // fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; + // fdb5::RadosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; + // fdb5::RadosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; // EXPECT(e_axis_kv.exists()); // EXPECT(e_axis_kv.has("5")); - // fdb5::DaosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; - // fdb5::DaosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; + // fdb5::RadosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; + // fdb5::RadosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; // EXPECT(f_axis_kv.exists()); // EXPECT(f_axis_kv.has("6")); } @@ -276,14 +278,14 @@ CASE("RadosCatalogue tests") { // // remove (manual deindex) // { - // fdb5::DaosCatalogueWriter dcatw{db_key, config}; - // fdb5::DaosName db_cont{dcatw.uri()}; + // fdb5::RadosCatalogueWriter dcatw{db_key, config}; + // fdb5::RadosName db_cont{dcatw.uri()}; // std::ostream out(std::cout.rdbuf()); - // fdb5::DaosCatalogue::remove(db_cont, out, out, true); + // fdb5::RadosCatalogue::remove(db_cont, out, out, true); - // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config - // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + // fdb5::RadosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + // fdb5::RadosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; // EXPECT_NOT(cat_kv.exists()); // EXPECT_NOT(db_cont.exists()); // } @@ -344,7 +346,7 @@ CASE("RadosCatalogue tests") { catw.archive(index_key, field_key, std::move(loc)); /// flush store before flushing catalogue - rstore.flush(); // not necessary if using a DAOS store + rstore.flush(); // not necessary if using a RADOS store } // find data @@ -374,7 +376,7 @@ CASE("RadosCatalogue tests") { // // deindex data // { - // fdb5::DaosCatalogueWriter dcat{db_key, config}; + // fdb5::RadosCatalogueWriter dcat{db_key, config}; // fdb5::Catalogue& cat = static_cast(dcat); // std::ostream out(std::cout.rdbuf()); // metkit::mars::MarsRequest r = db_key.request("retrieve"); @@ -483,6 +485,144 @@ CASE("RadosCatalogue tests") { } } + SECTION("RadosCatalogue persists ControlIdentifiers across processes") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Key db_key({{"a", "88"}, {"b", "88"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& cat = writer; + cat.selectIndex(index_key); + std::unique_ptr loc( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, eckit::Offset(0), eckit::Length(1))); + static_cast(writer).archive(index_key, field_key, std::move(loc)); + cat.flush(0); + + // Default: everything enabled. + EXPECT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::List)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); + + // hideContents disables List and Retrieve, leaves Archive. + cat.hideContents(); + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::List)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); + } + + { + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::Catalogue& cat = reader; + EXPECT(static_cast(reader).open()); + // State survives process boundary. + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::List)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); + } + + { + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& cat = writer; + fdb5::ControlIdentifiers ids = fdb5::ControlIdentifier::List | fdb5::ControlIdentifier::Retrieve; + cat.control(fdb5::ControlAction::Enable, ids); + EXPECT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT(cat.enabled(fdb5::ControlIdentifier::List)); + } + } + + SECTION("RadosCatalogueWriter supports concurrent writers on the same database") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + "\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Key db_key({{"a", "66"}, {"b", "66"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key first_field_key({{"e", "5"}}); + fdb5::Key second_field_key({{"g", "6"}}); + + std::promise start; + const std::shared_future ready = start.get_future().share(); + std::mutex error_mutex; + std::exception_ptr error; + const auto archive = [&](const fdb5::Key& field_key, eckit::Offset offset) { + try { + ready.wait(); + fdb5::RadosCatalogueWriter writer{db_key, config}; + fdb5::Catalogue& catalogue = writer; + catalogue.selectIndex(index_key); + std::unique_ptr location( + new fdb5::RadosFieldLocation(eckit::URI{"rados", "unused"}, offset, eckit::Length(1))); + static_cast(writer).archive(index_key, field_key, std::move(location)); + catalogue.flush(0); + } + catch (...) { + std::lock_guard lock{error_mutex}; + if (!error) { + error = std::current_exception(); + } + } + }; + + std::thread first{archive, std::cref(first_field_key), eckit::Offset(0)}; + std::thread second{archive, std::cref(second_field_key), eckit::Offset(1)}; + start.set_value(); + first.join(); + second.join(); + EXPECT(!error); + + fdb5::RadosCatalogueReader reader{db_key, config}; + fdb5::Catalogue& catalogue = reader; + EXPECT(static_cast(reader).open()); + EXPECT(catalogue.selectIndex(index_key)); + const auto e_axis = static_cast(reader).axis("e"); + const auto g_axis = static_cast(reader).axis("g"); + EXPECT(e_axis && e_axis->get().contains("5")); + EXPECT(g_axis && g_axis->get().contains("6")); + } + SECTION("RadosCatalogue supports large serialised field locations") { std::string config_str{ @@ -531,7 +671,7 @@ CASE("RadosCatalogue tests") { } } - // SECTION("DaosCatalogue archive (index) and retrieve with a TocStore") { + // SECTION("RadosCatalogue archive (index) and retrieve with a TocStore") { // // FDB configuration @@ -540,7 +680,7 @@ CASE("RadosCatalogue tests") { // "- roots:\n" // " - path: " + catalogue_tests_tmp_root().asString() + "\n" // "schema : " + schema_file().path() + "\n" - // "daos:\n" + // "Rados:\n" // " catalogue:\n" // " pool: " + pool_name + "\n" // " root_cont: " + root_cont_name + "\n" @@ -569,12 +709,12 @@ CASE("RadosCatalogue tests") { // fdb5::Store& store = static_cast(tstore); // std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); // /// @todo: there are two cont create with label here - // /// @todo: again, daos_fini happening before cont and pool close + // /// @todo: again, Rados_fini happening before cont and pool close // // index data // { - // fdb5::DaosCatalogueWriter dcatw{db_key, config}; + // fdb5::RadosCatalogueWriter dcatw{db_key, config}; // fdb5::Catalogue& cat = dcatw; // cat.deselectIndex(); // cat.selectIndex(index_key); @@ -589,7 +729,7 @@ CASE("RadosCatalogue tests") { // fdb5::Field field; // { - // fdb5::DaosCatalogueReader dcatr{db_key, config}; + // fdb5::RadosCatalogueReader dcatr{db_key, config}; // fdb5::Catalogue& cat = dcatr; // cat.selectIndex(index_key); // fdb5::CatalogueReader& catr = dcatr; @@ -614,7 +754,7 @@ CASE("RadosCatalogue tests") { // // remove data - // /// @todo: should DaosStore::remove accept full URIs to field arrays and remove the store container? + // /// @todo: should RadosStore::remove accept full URIs to field arrays and remove the store container? // eckit::PathName store_path{field.location().uri().path()}; // std::ostream out(std::cout.rdbuf()); // store.remove(field.location().uri(), out, out, false); @@ -625,7 +765,7 @@ CASE("RadosCatalogue tests") { // // deindex data // { - // fdb5::DaosCatalogueWriter dcat{db_key, config}; + // fdb5::RadosCatalogueWriter dcat{db_key, config}; // fdb5::Catalogue& cat = static_cast(dcat); // std::ostream out(std::cout.rdbuf()); // metkit::mars::MarsRequest r = db_key.request("retrieve"); @@ -633,14 +773,12 @@ CASE("RadosCatalogue tests") { // cat.visitEntries(*wv, store, false); // } - // /// @todo: again, daos_fini happening before + // /// @todo: again, Rados_fini happening before // } SECTION("Via FDB API with a Rados catalogue and store") { - /// @note: earlier sections share the same catalogue namespaces/pool; reset them so this - /// section starts from a clean, empty catalogue (it asserts the FDB is initially empty). #ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); @@ -694,10 +832,6 @@ CASE("RadosCatalogue tests") { size_t count; fdb5::ListElement info; - /// @todo: here, DaosManager is being configured with DAOS client config passed to FDB instance constructor. - // It happens in EntryVisitMechanism::visit when calling DB::open. Is this OK, or should this configuring - // rather happen as part of transforming a FieldLocation into a DataHandle? It is probably OK. One thing - // is to configure the DAOS client and the other thing is to initialise it. auto listObject = fdb.list(db_req); count = 0; @@ -712,9 +846,6 @@ CASE("RadosCatalogue tests") { char data[] = "test"; - /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times - /// below. - // Should this be avoided? fdb.archive(request_key, data, sizeof(data)); fdb.flush(); @@ -741,8 +872,6 @@ CASE("RadosCatalogue tests") { listObject = fdb.list(all_req); count = 0; while (listObject.next(info)) { - // info.print(std::cout, true, true); - // std::cout << std::endl; count++; } EXPECT(count == 1); @@ -835,9 +964,9 @@ CASE("RadosCatalogue tests") { // " - path: " + catalogue_tests_tmp_root().asString() + "\n" // "type: local\n" // "schema : " + opt_schema_file().path() + "\n" - // "engine: daos\n" - // "store: daos\n" - // "daos:\n" + // "engine: Rados\n" + // "store: Rados\n" + // "Rados:\n" // " catalogue:\n" // " pool: " + pool_name + "\n" // " root_cont: " + root_cont_name + "\n" diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 3853d1258..b7154424f 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -8,43 +8,28 @@ * does it submit to any jurisdiction. */ -// #include -// #include - -#include "eckit/config/Resource.h" -#include "eckit/exception/Exceptions.h" -#include "eckit/testing/Test.h" -// #include "eckit/filesystem/URI.h" -#include "eckit/filesystem/PathName.h" -#include "eckit/filesystem/TmpFile.h" -// #include "eckit/filesystem/TmpDir.h" -// #include "eckit/io/FileHandle.h" -#include "eckit/config/YAMLConfiguration.h" -#include "eckit/io/MemoryHandle.h" - -// #include "metkit/mars/MarsRequest.h" - -#include "fdb5/fdb5_config.h" -// #include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/api/helpers/WipeIterator.h" #include "fdb5/database/Engine.h" -#include "fdb5/toc/TocCatalogueReader.h" -#include "fdb5/toc/TocCatalogueWriter.h" - -// #include "eckit/io/s3/S3Client.h" -// #include "eckit/io/s3/S3Session.h" -// #include "eckit/io/s3/S3Credential.h" +#include "fdb5/fdb5_config.h" #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rados/RadosStore.h" +#include "fdb5/toc/TocCatalogueReader.h" +#include "fdb5/toc/TocCatalogueWriter.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/io/MemoryHandle.h" #include "eckit/io/PartHandle.h" -// #include "fdb5/daos/DaosException.h" +#include "eckit/testing/Test.h" -using namespace eckit::testing; using namespace eckit; +//---------------------------------------------------------------------------------------------------------------------- + namespace { void deldir(eckit::PathName& p) { @@ -88,8 +73,6 @@ void ensureCleanPools(const std::string& prefix) { } // namespace -// temporary schema,spaces,root files common to all DAOS Store tests - eckit::TmpFile& schema_file() { static eckit::TmpFile f{}; return f; @@ -117,8 +100,9 @@ size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { return count; } -namespace fdb { -namespace test { +//---------------------------------------------------------------------------------------------------------------------- + +namespace fdb::test { CASE("Setup") { @@ -627,15 +611,16 @@ CASE("RadosStore tests") { } } -} // namespace test -} // namespace fdb +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb::test + int main(int argc, char** argv) { int ret = -1; - try { - ret = run_tests(argc, argv); + ret = eckit::testing::run_tests(argc, argv); } catch (...) { } From fae75fb68089c3da15804e95be24d2b0537b58f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 15:12:39 +0200 Subject: [PATCH 098/109] test(rados): uri and wipe hidden --- tests/fdb/rados/test_rados_catalogue.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index bae718069..82f6ed3c4 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -512,6 +512,7 @@ CASE("RadosCatalogue tests") { fdb5::Key db_key({{"a", "88"}, {"b", "88"}}); fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + eckit::URI catalogue_uri; { fdb5::RadosCatalogueWriter writer{db_key, config}; @@ -532,6 +533,7 @@ CASE("RadosCatalogue tests") { EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::Retrieve)); EXPECT_NOT(cat.enabled(fdb5::ControlIdentifier::List)); EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); + catalogue_uri = cat.uri(); } { @@ -544,6 +546,13 @@ CASE("RadosCatalogue tests") { EXPECT(cat.enabled(fdb5::ControlIdentifier::Archive)); } + { + auto reader = fdb5::CatalogueReaderFactory::instance().build(catalogue_uri, config); + EXPECT_NOT(reader->enabled(fdb5::ControlIdentifier::Retrieve)); + EXPECT_NOT(reader->enabled(fdb5::ControlIdentifier::List)); + EXPECT(reader->enabled(fdb5::ControlIdentifier::Archive)); + } + { fdb5::RadosCatalogueWriter writer{db_key, config}; fdb5::Catalogue& cat = writer; @@ -587,6 +596,8 @@ CASE("RadosCatalogue tests") { const std::shared_future ready = start.get_future().share(); std::mutex error_mutex; std::exception_ptr error; + + // const auto archive = [&](const fdb5::Key& field_key, eckit::Offset offset) { try { ready.wait(); @@ -933,11 +944,16 @@ CASE("RadosCatalogue tests") { } EXPECT(count == 1); - // wipe full database + // Wipe remains enabled after hideContents disables only List and Retrieve. + fdb2.control(db_req, fdb5::ControlAction::Disable, + fdb5::ControlIdentifier::List | fdb5::ControlIdentifier::Retrieve); wipeObject = fdb2.wipe(db_req, true); EXPECT(countWipeable(wipeObject) > 0); fdb2.flush(); + fdb5::RadosCatalogueReader hidden_reader{db_key, config}; + EXPECT_NOT(static_cast(hidden_reader).open()); + // ensure field does not exist listObject = fdb2.list(full_req); count = 0; From a865cc0a093f448b36f557ac19af17e3a70250e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 15:58:43 +0200 Subject: [PATCH 099/109] feat(rados): add spaces --- src/fdb5/rados/README | 30 +- src/fdb5/rados/RadosCommon.cc | 134 ++++--- src/fdb5/rados/RadosCommon.h | 21 +- src/fdb5/rados/RadosEngine.cc | 103 ++---- src/fdb5/rados/RadosEngine.h | 15 +- tests/fdb/rados/test_rados_catalogue.cc | 446 ++++++++---------------- tests/fdb/rados/test_rados_store.cc | 47 ++- 7 files changed, 334 insertions(+), 462 deletions(-) diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README index b01d5c8d6..f7ed7b9b1 100644 --- a/src/fdb5/rados/README +++ b/src/fdb5/rados/README @@ -4,9 +4,33 @@ Running RadosStore unit tests against Ceph on Docker on mac: Supported RADOS backend scope: ============================== -The backend supports archive, retrieve, list, wipe, and reopening catalogues by URI. Archive calls -on a single RadosStore must be serialised by the caller. Catalogue-side purge, statistics, move, -control (hide/mount), and overlay operations are not implemented. +The backend supports archive, retrieve, list, wipe, statistics, hide, and reopening catalogues by +URI. Operations on a single RadosStore instance must be serialised by the caller; separate writer +instances can archive concurrently to the same database. Catalogue-side purge, move, mount, and +overlay operations are not implemented. + +RADOS placement is configured on `spaces[].roots[]`; legacy `rados.pool`, `rados.root_namespace`, +and `rados.namespace_prefix` values are not used. The first matching space must have exactly one +root with all of these required attributes: + +```yaml +spaces: +- regex: ".*" + roots: + - path: /local/fdb-root + pool: fdb-rados + root_namespace: fdb-root + namespace_prefix: fdb +``` + +- `pool` is the Ceph pool for the selected space. +- `root_namespace` holds the space's `main_kv` registry, which maps database namespaces to catalogue URIs. +- `namespace_prefix` forms each database namespace: a DB key with values `11:22` is stored in + `fdb_11:22`. It must not contain `_`, the prefix/key separator. + +The catalogue KV is `rados:fdb-rados/fdb_11:22/catalogue_kv`; index and data objects share the +`fdb_11:22` namespace. Multiple spaces may share a pool when their root namespaces and namespace +prefixes are distinct. RADOS tests require eckit to be built with RADOS support. diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index ee374e319..1d979a5f1 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -14,10 +14,12 @@ #include "fdb5/database/Key.h" #include "eckit/config/LocalConfiguration.h" -#include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/log/CodeLocation.h" #include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Regex.h" #include "eckit/utils/Tokenizer.h" #include @@ -29,12 +31,84 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- +namespace { + +RadosSpace space_from_root(const eckit::LocalConfiguration& root) { + RadosSpace space{root.getString("pool"), root.getString("root_namespace"), root.getString("namespace_prefix")}; + if (space.namespacePrefix.find('_') != std::string::npos) { + throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + space.namespacePrefix + "'", + Here()); + } + return space; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + fdb5::Key read_db_key(const eckit::RadosKeyValue& db_kv) { std::vector data; eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); return fdb5::Key(ms); } +std::string RadosSpace::databaseNamespace(const Key& key) const { + return namespacePrefix + "_" + key.valuesToString(); +} + +std::vector rados_spaces(const Config& config) { + if (!config.has("spaces")) { + throw eckit::UserError("RADOS placement requires at least one spaces[] entry", Here()); + } + + std::vector spaces; + for (const auto& space : config.getSubConfigurations("spaces")) { + if (!space.has("roots")) { + throw eckit::UserError("RADOS placement requires roots[] in every spaces[] entry", Here()); + } + for (const auto& root : space.getSubConfigurations("roots")) { + spaces.emplace_back(space_from_root(root)); + } + } + return spaces; +} + +RadosSpace rados_space(const Config& config, const Key& key) { + if (!config.has("spaces")) { + throw eckit::UserError("RADOS placement requires at least one spaces[] entry", Here()); + } + + const std::string keyString = key.valuesToString(); + for (const auto& space : config.getSubConfigurations("spaces")) { + if (!eckit::Regex{space.getString("regex", ".*")}.match(keyString)) { + continue; + } + if (!space.has("roots")) { + throw eckit::UserError("RADOS placement requires roots[] in matching spaces[] entry", Here()); + } + const auto roots = space.getSubConfigurations("roots"); + if (roots.size() != 1) { + throw eckit::UserError("RADOS placement requires exactly one root in matching spaces[] entry", Here()); + } + return space_from_root(roots.front()); + } + + throw eckit::UserError("No RADOS placement matches database key " + keyString, Here()); +} + +RadosSpace rados_space(const Config& config, const eckit::URI& uri) { + const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); + ASSERT(parts.size() == 2 || parts.size() == 3); + + for (const auto& space : rados_spaces(config)) { + if (space.pool == parts[0] && parts[1].rfind(space.namespacePrefix + "_", 0) == 0) { + return space; + } + } + + throw eckit::UserError("No RADOS placement matches URI " + uri.asString(), Here()); +} + //---------------------------------------------------------------------------------------------------------------------- RadosCommon::RadosCommon(const Config& config, const std::string& component, const Key& key) { @@ -42,32 +116,30 @@ RadosCommon::RadosCommon(const Config& config, const std::string& component, con std::vector valid{"catalogue", "store"}; ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); - readConfig(config, component, true); - - db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); + const RadosSpace space = rados_space(config, key); + pool_ = space.pool; + db_namespace_ = space.databaseNamespace(key); + readConfig(config, component); - root_kv_.emplace(pool_, root_namespace_, "main_kv"); + root_kv_.emplace(pool_, space.rootNamespace, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); } RadosCommon::RadosCommon(const Config& config, const std::string& component, const eckit::URI& uri) { - // Accepts URIs from two callers: DB::buildReader in EntryVisitMechanism supplies a catalogue KV - // URI (pool/namespace/oid); StoreFactory during wipe supplies a store namespace URI - // (pool/namespace). Only pool and namespace are needed here. const auto parts = eckit::Tokenizer("/").tokenize(uri.name()); ASSERT(parts.size() == 2 || parts.size() == 3); + const RadosSpace space = rados_space(config, uri); pool_ = parts[0]; db_namespace_ = parts[1]; + readConfig(config, component); - readConfig(config, component, false); - - root_kv_.emplace(pool_, root_namespace_, "main_kv"); + root_kv_.emplace(pool_, space.rootNamespace, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); } -void RadosCommon::readConfig(const Config& config, const std::string& component, bool readPool) { +void RadosCommon::readConfig(const Config& config, const std::string& component) { eckit::LocalConfiguration c{}; @@ -76,44 +148,6 @@ void RadosCommon::readConfig(const Config& config, const std::string& component, } maxPartSize_ = c.getInt("maxPartSize", 0); - - std::string first_cap{component}; - first_cap[0] = toupper(component[0]); - - std::string all_caps{component}; - for (auto& c : all_caps) { - c = toupper(c); - } - - if (readPool) { - pool_ = "default"; - } - root_namespace_ = "root"; - - if (readPool) { - pool_ = c.getString("pool", pool_); - if (c.has(component)) { - pool_ = c.getSubConfiguration(component).getString("pool", pool_); - } - } - root_namespace_ = c.getString("root_namespace", root_namespace_); - if (c.has(component)) { - root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); - } - - if (readPool) { - pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); - } - root_namespace_ = eckit::Resource( - "fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); - - nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); - if (c.has(component)) { - nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); - } - if (nspace_prefix_.find('_') != std::string::npos) { - throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + nspace_prefix_ + "'", Here()); - } } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 5a393bf4e..8b756f557 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -22,13 +22,27 @@ #include #include +#include namespace fdb5 { +struct RadosSpace { + std::string pool; + std::string rootNamespace; + std::string namespacePrefix; + + std::string databaseNamespace(const Key& key) const; +}; + // Reads the persisted `key` entry from a RADOS DB KV and deserialises it into an fdb5::Key. // Throws eckit::RadosEntityNotFoundException if the DB KV or the `key` entry is missing. fdb5::Key read_db_key(const eckit::RadosKeyValue& db_kv); +// RADOS space is selected from the sole root of the first matching `spaces[]` entry. +RadosSpace rados_space(const Config&, const Key&); +RadosSpace rados_space(const Config&, const eckit::URI&); +std::vector rados_spaces(const Config&); + class RadosCommon { public: // methods @@ -38,22 +52,17 @@ class RadosCommon { private: // methods - void readConfig(const Config& config, const std::string& component, bool readPool); + void readConfig(const Config& config, const std::string& component); protected: // members std::string pool_; - std::string root_namespace_; std::string db_namespace_; std::optional root_kv_; std::optional db_kv_; eckit::Length maxPartSize_; - -private: // members - - std::string nspace_prefix_; }; } // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc index 9adef59fd..0d1c8e743 100644 --- a/src/fdb5/rados/RadosEngine.cc +++ b/src/fdb5/rados/RadosEngine.cc @@ -18,14 +18,10 @@ #include "metkit/mars/MarsRequest.h" -#include "eckit/config/LocalConfiguration.h" -#include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/rados/RadosKeyValue.h" -#include "eckit/log/CodeLocation.h" #include "eckit/log/Log.h" -#include "eckit/serialisation/MemoryStream.h" #include "eckit/utils/Tokenizer.h" #include @@ -43,9 +39,8 @@ std::string RadosEngine::name() const { } eckit::URI RadosEngine::location(const Key& key, const Config& config) const { - readConfig(config, "catalogue", true); - const std::string db_namespace = nspacePrefix_ + "_" + key.valuesToString(); - return eckit::RadosKeyValue{pool_, db_namespace, "catalogue_kv"}.uri(); + const RadosSpace space = rados_space(config, key); + return eckit::RadosKeyValue{space.pool, space.databaseNamespace(key), "catalogue_kv"}.uri(); } bool RadosEngine::canHandle(const eckit::URI& uri, const Config&) const { @@ -72,38 +67,35 @@ bool RadosEngine::canHandle(const eckit::URI& uri, const Config&) const { std::vector RadosEngine::visitableLocations(const std::function& matches, const Config& config) const { - const std::string component = "catalogue"; - - readConfig(config, component, true); - - rootKv_.emplace(pool_, rootNamespace_, "main_kv"); - std::vector res{}; - if (!rootKv_->exists()) { - return res; - } - - for (const auto& k : rootKv_->keys()) { - try { + for (const auto& space : rados_spaces(config)) { + eckit::RadosKeyValue rootKv{space.pool, space.rootNamespace, "main_kv"}; + if (!rootKv.exists()) { + continue; + } + for (const auto& key : rootKv.keys()) { + try { - std::vector v; - rootKv_->getMemoryStream(v, k, "root kv"); + std::vector val; + rootKv.getMemoryStream(val, key, "root kv"); - eckit::URI uri(std::string(v.begin(), v.end())); - ASSERT(uri.scheme() == typeName()); + eckit::URI uri(std::string(val.begin(), val.end())); + ASSERT(uri.scheme() == typeName()); - eckit::RadosKeyValue db_kv{uri}; - fdb5::Key db_key = read_db_key(db_kv); + eckit::RadosKeyValue db_kv{uri}; + fdb5::Key db_key = read_db_key(db_kv); - if (matches(db_key)) { - eckit::Log::debug() << " found match with " << rootKv_->uri() << " at key " << k << std::endl; - res.push_back(uri); + if (matches(db_key)) { + eckit::Log::debug() + << " found match with " << rootKv.uri() << " at key " << key << std::endl; + res.push_back(uri); + } + } + catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << key << " from " << rootKv.uri() << std::endl; + eckit::Log::error() << e.what() << std::endl; } - } - catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database " << k << " from " << rootKv_->uri() << std::endl; - eckit::Log::error() << e.what() << std::endl; } } @@ -119,53 +111,6 @@ std::vector RadosEngine::visitableLocations(const metkit::mars::Mars return visitableLocations([&request](const fdb5::Key& dbKey) { return dbKey.partialMatch(request); }, config); } -void RadosEngine::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const { - - eckit::LocalConfiguration c{}; - - if (config.has("rados")) { - c = config.getSubConfiguration("rados"); - } - - std::string first_cap{component}; - first_cap[0] = toupper(component[0]); - - std::string all_caps{component}; - for (auto& c : all_caps) { - c = toupper(c); - } - - if (readPool) { - pool_ = "default"; - } - rootNamespace_ = "root"; - - if (readPool) { - pool_ = c.getString("pool", pool_); - if (c.has(component)) { - pool_ = c.getSubConfiguration(component).getString("pool", pool_); - } - } - rootNamespace_ = c.getString("root_namespace", rootNamespace_); - if (c.has(component)) { - rootNamespace_ = c.getSubConfiguration(component).getString("root_namespace", rootNamespace_); - } - - if (readPool) { - pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); - } - rootNamespace_ = eckit::Resource( - "fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", rootNamespace_); - - nspacePrefix_ = c.getString("namespace_prefix", nspacePrefix_); - if (c.has(component)) { - nspacePrefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspacePrefix_); - } - if (nspacePrefix_.find('_') != std::string::npos) { - throw eckit::UserError("RADOS namespace_prefix must not contain underscores: '" + nspacePrefix_ + "'", Here()); - } -} - static EngineBuilder rados_builder; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h index 5adb98ee0..0c70af155 100644 --- a/src/fdb5/rados/RadosEngine.h +++ b/src/fdb5/rados/RadosEngine.h @@ -52,25 +52,12 @@ class RadosEngine : public Engine { std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; - void print(std::ostream& out) const override { out << "RadosEngine(" << pool_ << ", " << rootNamespace_ << ")"; } + void print(std::ostream& out) const override { out << "RadosEngine"; } private: // methods std::vector visitableLocations(const std::function& matches, const Config& config) const; - - void readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const; - -protected: // members - - mutable std::string pool_; - mutable std::string rootNamespace_; - - mutable std::optional rootKv_; - -private: // members - - mutable std::string nspacePrefix_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 82f6ed3c4..ca080379d 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -16,6 +16,7 @@ #include "fdb5/database/Catalogue.h" #include "fdb5/database/DatabaseNotFoundException.h" #include "fdb5/database/DbStats.h" +#include "fdb5/database/Engine.h" #include "fdb5/database/Field.h" #include "fdb5/database/FieldLocation.h" #include "fdb5/rados/RadosCatalogueReader.h" @@ -184,6 +185,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -217,32 +227,13 @@ CASE("RadosCatalogue tests") { { fdb5::RadosCatalogueWriter dcatw{db_key, config}; - // fdb5::RadosName db_cont{pool_name, db_key.valuesToString()}; - // fdb5::RadosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config - // fdb5::RadosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; - // EXPECT(db_cont.exists()); - // EXPECT(cat_kv.exists()); - fdb5::Catalogue& cat = dcatw; cat.selectIndex(index_key); - // fdb5::RadosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from - // config fdb5::RadosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; - // EXPECT(index_kv.exists()); - // EXPECT(cat_kv.has(index_key.valuesToString())); fdb5::CatalogueWriter& catw = dcatw; catw.archive(index_key, field_key, std::move(loc)); cat.flush(0); catalogue_uri = cat.uri(); - // EXPECT(index_kv.has(field_key.valuesToString())); - // fdb5::RadosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; - // fdb5::RadosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; - // EXPECT(e_axis_kv.exists()); - // EXPECT(e_axis_kv.has("5")); - // fdb5::RadosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; - // fdb5::RadosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; - // EXPECT(f_axis_kv.exists()); - // EXPECT(f_axis_kv.has("6")); } { @@ -274,21 +265,6 @@ CASE("RadosCatalogue tests") { EXPECT(f.location().offset() == eckit::Offset(0)); EXPECT(f.location().length() == eckit::Length(1)); } - - // // remove (manual deindex) - - // { - // fdb5::RadosCatalogueWriter dcatw{db_key, config}; - // fdb5::RadosName db_cont{dcatw.uri()}; - // std::ostream out(std::cout.rdbuf()); - - // fdb5::RadosCatalogue::remove(db_cont, out, out, true); - - // fdb5::RadosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config - // fdb5::RadosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; - // EXPECT_NOT(cat_kv.exists()); - // EXPECT_NOT(db_cont.exists()); - // } } SECTION("RadosCatalogue archive (index) and retrieve with a RadosStore") { @@ -301,6 +277,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -372,17 +357,6 @@ CASE("RadosCatalogue tests") { dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - - // // deindex data - - // { - // fdb5::RadosCatalogueWriter dcat{db_key, config}; - // fdb5::Catalogue& cat = static_cast(dcat); - // std::ostream out(std::cout.rdbuf()); - // metkit::mars::MarsRequest r = db_key.request("retrieve"); - // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); - // cat.visitEntries(*wv, store, false); - // } } SECTION("RadosCatalogue reports missing databases via factory paths") { @@ -393,6 +367,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -436,6 +419,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -493,6 +485,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -571,6 +572,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -634,6 +644,74 @@ CASE("RadosCatalogue tests") { EXPECT(g_axis && g_axis->get().contains("6")); } + SECTION("Rados placements are selected from matching space roots") { + + const std::string alpha_prefix = test_id + "alpha"; + const std::string beta_prefix = test_id + "beta"; + std::string config_str{ + "spaces:\n" + "- regex: 11:11\n" + " roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_alpha_root\n" + " namespace_prefix: " + + alpha_prefix + + "\n" + "- regex: 22:22\n" + " roots:\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_beta_root\n" + " namespace_prefix: " + + beta_prefix + + "\n" + "schema : " + + schema_file().path() + + "\n" + "rados:\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_legacy_root\n" + " namespace_prefix: legacy\n"}; + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Key alpha_key({{"a", "11"}, {"b", "11"}}); + fdb5::Key beta_key({{"a", "22"}, {"b", "22"}}); + + fdb5::RadosCatalogueWriter alpha{alpha_key, config}; + fdb5::RadosCatalogueWriter beta{beta_key, config}; + + const std::string alpha_namespace = pool + "/" + alpha_prefix + "_" + alpha_key.valuesToString(); + const std::string beta_namespace = pool + "/" + beta_prefix + "_" + beta_key.valuesToString(); + EXPECT(alpha.uri().name() == alpha_namespace); + EXPECT(beta.uri().name() == beta_namespace); + EXPECT(fdb5::Engine::backend("rados").location(alpha_key, config).name() == alpha_namespace + "/catalogue_kv"); + EXPECT(fdb5::Engine::backend("rados").location(beta_key, config).name() == beta_namespace + "/catalogue_kv"); + + const auto alpha_locations = fdb5::Engine::backend("rados").visitableLocations(alpha_key, config); + const auto beta_locations = fdb5::Engine::backend("rados").visitableLocations(beta_key, config); + EXPECT(alpha_locations.size() == 1); + EXPECT(beta_locations.size() == 1); + EXPECT(alpha_locations.front().name() == alpha_namespace + "/catalogue_kv"); + EXPECT(beta_locations.front().name() == beta_namespace + "/catalogue_kv"); + } + SECTION("RadosCatalogue supports large serialised field locations") { std::string config_str{ @@ -642,6 +720,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -682,112 +769,6 @@ CASE("RadosCatalogue tests") { } } - // SECTION("RadosCatalogue archive (index) and retrieve with a TocStore") { - - // // FDB configuration - - // std::string config_str{ - // "spaces:\n" - // "- roots:\n" - // " - path: " + catalogue_tests_tmp_root().asString() + "\n" - // "schema : " + schema_file().path() + "\n" - // "Rados:\n" - // " catalogue:\n" - // " pool: " + pool_name + "\n" - // " root_cont: " + root_cont_name + "\n" - // " client:\n" - // " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - // }; - - // fdb5::Config config{YAMLConfiguration(config_str)}; - - // // schema - - // fdb5::Schema schema{schema_file()}; - - // // request - - // fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); - // fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); - // fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); - // fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); - - // // store data - - // char data[] = "test"; - - // fdb5::TocStore tstore{schema, db_key, config}; - // fdb5::Store& store = static_cast(tstore); - // std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); - // /// @todo: there are two cont create with label here - // /// @todo: again, Rados_fini happening before cont and pool close - - // // index data - - // { - // fdb5::RadosCatalogueWriter dcatw{db_key, config}; - // fdb5::Catalogue& cat = dcatw; - // cat.deselectIndex(); - // cat.selectIndex(index_key); - // fdb5::CatalogueWriter& catw = dcatw; - // catw.archive(field_key, std::move(loc)); - - // /// flush store before flushing catalogue - // tstore.flush(); - // } - - // // find data - - // fdb5::Field field; - // { - // fdb5::RadosCatalogueReader dcatr{db_key, config}; - // fdb5::Catalogue& cat = dcatr; - // cat.selectIndex(index_key); - // fdb5::CatalogueReader& catr = dcatr; - // catr.retrieve(field_key, field); - // } - // std::cout << "Read location: " << field.location() << std::endl; - - // // retrieve data - - // std::unique_ptr dh(store.retrieve(field)); - - // std::vector test(dh->size()); - // dh->openForRead(); - // { - // eckit::AutoClose closer(*dh); - // dh->read(&test[0], test.size() - 3); - // } - // eckit::MemoryHandle mh; - // dh->copyTo(mh); - // EXPECT(mh.size() == eckit::Length(sizeof(data))); - // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - - // // remove data - - // /// @todo: should RadosStore::remove accept full URIs to field arrays and remove the store container? - // eckit::PathName store_path{field.location().uri().path()}; - // std::ostream out(std::cout.rdbuf()); - // store.remove(field.location().uri(), out, out, false); - // EXPECT(store_path.exists()); - // store.remove(field.location().uri(), out, out, true); - // EXPECT_NOT(store_path.exists()); - - // // deindex data - - // { - // fdb5::RadosCatalogueWriter dcat{db_key, config}; - // fdb5::Catalogue& cat = static_cast(dcat); - // std::ostream out(std::cout.rdbuf()); - // metkit::mars::MarsRequest r = db_key.request("retrieve"); - // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); - // cat.visitEntries(*wv, store, false); - // } - - // /// @todo: again, Rados_fini happening before - - // } - SECTION("Via FDB API with a Rados catalogue and store") { #ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS @@ -805,6 +786,15 @@ CASE("RadosCatalogue tests") { " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "type: local\n" "schema : " + schema_file().path() + @@ -968,160 +958,6 @@ CASE("RadosCatalogue tests") { fdb2.flush(); } - // SECTION("OPTIONAL SCHEMA KEYS") { - - // // FDB configuration - - // ::setenv("FDB_SCHEMA_FILE", opt_schema_file().path().c_str(), 1); - - // std::string config_str{ - // "spaces:\n" - // "- roots:\n" - // " - path: " + catalogue_tests_tmp_root().asString() + "\n" - // "type: local\n" - // "schema : " + opt_schema_file().path() + "\n" - // "engine: Rados\n" - // "store: Rados\n" - // "Rados:\n" - // " catalogue:\n" - // " pool: " + pool_name + "\n" - // " root_cont: " + root_cont_name + "\n" - // " store:\n" - // " pool: " + pool_name + "\n" - // " client:\n" - // " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - // }; - - // fdb5::Config config{YAMLConfiguration(config_str)}; - - // // request - - // fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"f", "6"}}); - // fdb5::Key request_key2({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); - // fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); - // fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"d", "4"}}); - - // fdb5::FDBToolRequest full_req{ - // request_key.request("retrieve"), - // false, - // std::vector{"a", "b"} - // }; - // fdb5::FDBToolRequest full_req2{ - // request_key2.request("retrieve"), - // false, - // std::vector{"a", "b"} - // }; - // fdb5::FDBToolRequest index_req{ - // index_key.request("retrieve"), - // false, - // std::vector{"a", "b"} - // }; - // fdb5::FDBToolRequest db_req{ - // db_key.request("retrieve"), - // false, - // std::vector{"a", "b"} - // }; - // fdb5::FDBToolRequest all_req{ - // metkit::mars::MarsRequest{}, - // true, - // std::vector{} - // }; - - // // initialise FDB - - // fdb5::FDB fdb(config); - - // // check FDB is empty - - // size_t count; - // fdb5::ListElement info; - - // auto listObject = fdb.list(db_req); - - // count = 0; - // while (listObject.next(info)) { - // info.print(std::cout, true, true); - // std::cout << std::endl; - // ++count; - // } - // EXPECT(count == 0); - - // // archive data with incomplete key - - // char data[] = "test"; - - // fdb.archive(request_key, data, sizeof(data)); - - // fdb.flush(); - - // // list data - - // listObject = fdb.list(db_req); - - // count = 0; - // while (listObject.next(info)) { - // info.print(std::cout, true, true); - // std::cout << std::endl; - // ++count; - // } - // EXPECT(count == 1); - - // // retrieve data - - // { - // metkit::mars::MarsRequest r = request_key.request("retrieve"); - // std::unique_ptr dh(fdb.retrieve(r)); - - // eckit::MemoryHandle mh; - // dh->copyTo(mh); - // EXPECT(mh.size() == eckit::Length(sizeof(data))); - // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - // } - - // // archive data with complete key - - // char data2[] = "abcd"; - - // fdb.archive(request_key2, data2, sizeof(data)); - - // fdb.flush(); - - // // list data - - // listObject = fdb.list(db_req); - - // count = 0; - // while (listObject.next(info)) { - // info.print(std::cout, true, true); - // std::cout << std::endl; - // ++count; - // } - // EXPECT(count == 2); - - // // retrieve data - - // { - // metkit::mars::MarsRequest r = request_key.request("retrieve"); - // std::unique_ptr dh(fdb.retrieve(r)); - - // eckit::MemoryHandle mh; - // dh->copyTo(mh); - // EXPECT(mh.size() == eckit::Length(sizeof(data))); - // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); - // } - - // { - // metkit::mars::MarsRequest r = request_key2.request("retrieve"); - // std::unique_ptr dh(fdb.retrieve(r)); - - // eckit::MemoryHandle mh; - // dh->copyTo(mh); - // EXPECT(mh.size() == eckit::Length(sizeof(data2))); - // EXPECT(::memcmp(mh.data(), data2, sizeof(data2)) == 0); - // } - - // } - // teardown rados #ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index b7154424f..f147d22bc 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -152,6 +152,15 @@ CASE("RadosStore tests") { " - path: " + store_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "rados:\n" " maxPartSize: 16\n" " store:\n" @@ -229,11 +238,12 @@ CASE("RadosStore tests") { fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); std::string config_str{ - "rados:\n" - " pool: " + - std::string{"unused"} + - "\n" - " namespace_prefix: invalid_prefix\n"}; + "spaces:\n" + "- roots:\n" + " - path: unused\n" + " pool: unused\n" + " root_namespace: unused\n" + " namespace_prefix: invalid_prefix\n"}; fdb5::Config config{YAMLConfiguration(config_str)}; EXPECT_THROWS_AS((fdb5::RadosStore{schema, db_key, config}), eckit::UserError); @@ -267,6 +277,15 @@ CASE("RadosStore tests") { " - path: " + store_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "schema : " + schema_file().path() + "\n" @@ -389,6 +408,15 @@ CASE("RadosStore tests") { " - path: " + store_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "type: local\n" "schema : " + schema_file().path() + @@ -544,6 +572,15 @@ CASE("RadosStore tests") { " - path: " + store_tests_tmp_root().asString() + "\n" + " pool: " + + pool + + "\n" + " root_namespace: " + + test_id + + "_root\n" + " namespace_prefix: " + + test_id + + "\n" "type: local\n" "schema : " + schema_file().path() + From d0180fc960c6c67d69cfdd523e54cdd4fe562287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Wed, 19 Aug 2026 16:16:52 +0200 Subject: [PATCH 100/109] ci(rados): fix size 1 --- .github/workflows/ci-rados.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index 784d98ef1..d8e818508 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -144,6 +144,7 @@ jobs: docker logs ceph-demo || true exit 1 fi + docker exec ceph-demo ceph config set mon mon_allow_pool_size_one true docker exec ceph-demo ceph osd pool create "${FDB_RADOS_TEST_POOL}" 8 8 # The demo cluster has one OSD; its test pool must use a single replica to become active+clean. docker exec ceph-demo ceph osd pool set "${FDB_RADOS_TEST_POOL}" size 1 --yes-i-really-mean-it From e5a66fbac9bcb66ae7c3d6b773318ea538faffb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 20 Aug 2026 09:29:53 +0200 Subject: [PATCH 101/109] ci(rados): bump version --- .github/workflows/ci-rados.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index d8e818508..e8719c340 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -77,7 +77,7 @@ jobs: with: path: ${{ github.workspace }}/.apt-cache # Bump the suffix to invalidate when the package list changes. - key: apt-rados-${{ runner.os }}-v1 + key: apt-rados-${{ runner.os }}-v2 - name: Install build dependencies run: | @@ -106,7 +106,7 @@ jobs: with: path: ${{ github.workspace }}/.docker-ceph # Bump the suffix to refresh the pinned image snapshot. - key: ceph-image-522483cf07cf-v1 + key: ceph-image-522483cf07cf-v2 - name: Load or pull Ceph image run: | From 9228d5de7220553d10158ddbd100e1ed2f1216ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 20 Aug 2026 09:45:45 +0200 Subject: [PATCH 102/109] ci(rados): fix polling --- .github/workflows/ci-rados.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index e8719c340..a5e12bfe9 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -152,8 +152,8 @@ jobs: docker exec ceph-demo ceph osd pool application enable "${FDB_RADOS_TEST_POOL}" rados echo "Waiting for ${FDB_RADOS_TEST_POOL} placement groups to become active+clean..." for i in $(seq 1 60); do - if docker exec ceph-demo ceph pg ls-by-pool "${FDB_RADOS_TEST_POOL}" --format json | \ - jq -e 'length > 0 and all(.[]; .state == "active+clean")' >/dev/null 2>&1; then + if docker exec ceph-demo ceph pg ls-by-pool "${FDB_RADOS_TEST_POOL}" --format json 2>/dev/null | \ + jq -e 'length > 0 and all(.[]; (.state | contains("active+clean")))' >/dev/null 2>&1; then pool_ready=1 break fi From 890d9dc84fb87fdbbfc9e11dfca9e833668abec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 20 Aug 2026 10:25:44 +0200 Subject: [PATCH 103/109] ci(rados): add health --- .github/workflows/ci-rados.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-rados.yml b/.github/workflows/ci-rados.yml index a5e12bfe9..ce02bbcbd 100644 --- a/.github/workflows/ci-rados.yml +++ b/.github/workflows/ci-rados.yml @@ -126,6 +126,7 @@ jobs: -e MON_IP=127.0.0.1 \ -e CEPH_PUBLIC_NETWORK=0.0.0.0/0 \ -e CEPH_DEMO_UID=ci \ + -e DEMO_DAEMONS=mon,mgr,osd \ -v "${CEPH_ETC}:/etc/ceph" \ "${CEPH_IMAGE}" demo @@ -144,6 +145,20 @@ jobs: docker logs ceph-demo || true exit 1 fi + echo "Waiting for Ceph cluster health..." + for i in $(seq 1 60); do + if docker exec ceph-demo ceph health 2>/dev/null | grep -q '^HEALTH_OK'; then + health_ready=1 + break + fi + sleep 5 + done + if [ "${health_ready:-0}" != "1" ]; then + echo "Ceph cluster did not become healthy in time" >&2 + docker exec ceph-demo ceph status || true + docker logs ceph-demo || true + exit 1 + fi docker exec ceph-demo ceph config set mon mon_allow_pool_size_one true docker exec ceph-demo ceph osd pool create "${FDB_RADOS_TEST_POOL}" 8 8 # The demo cluster has one OSD; its test pool must use a single replica to become active+clean. @@ -152,8 +167,8 @@ jobs: docker exec ceph-demo ceph osd pool application enable "${FDB_RADOS_TEST_POOL}" rados echo "Waiting for ${FDB_RADOS_TEST_POOL} placement groups to become active+clean..." for i in $(seq 1 60); do - if docker exec ceph-demo ceph pg ls-by-pool "${FDB_RADOS_TEST_POOL}" --format json 2>/dev/null | \ - jq -e 'length > 0 and all(.[]; (.state | contains("active+clean")))' >/dev/null 2>&1; then + if docker exec ceph-demo ceph pg stat 2>/dev/null | \ + awk '$2 == "pgs:" { gsub("[,;]", "", $4); exit !($1 == $3 && $4 == "active+clean") }'; then pool_ready=1 break fi From 755e65cf5bbcea29445bb9352d3a438cfff0642a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 20 Aug 2026 14:13:56 +0200 Subject: [PATCH 104/109] fix include --- tests/fdb/rados/test_rados_catalogue.cc | 1 + tests/fdb/rados/test_rados_store.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index ca080379d..7ace3acd9 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -27,6 +27,7 @@ #include "metkit/mars/MarsRequest.h" +#include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index f147d22bc..47c4ede91 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -18,6 +18,7 @@ #include "fdb5/toc/TocCatalogueReader.h" #include "fdb5/toc/TocCatalogueWriter.h" +#include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" From 7800d0b88b00033e9a04ceb8bf0f30b83afceb1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 20 Aug 2026 14:39:39 +0200 Subject: [PATCH 105/109] fix include --- tests/fdb/rados/test_rados_store.cc | 35 ++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 47c4ede91..5f4a36315 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -10,23 +10,48 @@ #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/database/Catalogue.h" #include "fdb5/database/Engine.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Store.h" #include "fdb5/fdb5_config.h" #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rados/RadosStore.h" +#include "fdb5/rules/Schema.h" #include "fdb5/toc/TocCatalogueReader.h" #include "fdb5/toc/TocCatalogueWriter.h" +#include "metkit/mars/MarsRequest.h" + #include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" #include "eckit/io/MemoryHandle.h" +#include "eckit/io/Offset.h" #include "eckit/io/PartHandle.h" +#include "eckit/io/rados/RadosCluster.h" +#include "eckit/io/rados/RadosNamespace.h" +#include "eckit/io/rados/RadosObject.h" +#include "eckit/io/rados/RadosPool.h" #include "eckit/testing/Test.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + using namespace eckit; //---------------------------------------------------------------------------------------------------------------------- @@ -370,16 +395,6 @@ CASE("RadosStore tests") { store.remove(store_uri, out, out, true); EXPECT_NOT(field_name.exists()); EXPECT(store_name.listObjects().size() == 0); - - // deindex data - - // { - // fdb5::TocCatalogueWriter tcat{db_key, config}; - // fdb5::Catalogue& cat = static_cast(tcat); - // metkit::mars::MarsRequest r = db_key.request("retrieve"); - // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); - // cat.visitEntries(*wv, store, false); - // } } SECTION("VIA FDB API") { From f782557670413a2e996b734b35fb7dfc146f1c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 20 Aug 2026 14:59:47 +0200 Subject: [PATCH 106/109] test(rados): cleanup at end --- tests/fdb/rados/test_rados_catalogue.cc | 29 ++++++++++++++++++++++++- tests/fdb/rados/test_rados_store.cc | 25 ++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 7ace3acd9..52bf8bcbe 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -40,6 +40,7 @@ #include "eckit/io/rados/RadosCluster.h" #include "eckit/io/rados/RadosNamespace.h" #include "eckit/io/rados/RadosPool.h" +#include "eckit/log/Log.h" #include "eckit/testing/Test.h" #include @@ -126,6 +127,23 @@ eckit::PathName& catalogue_tests_tmp_root() { return cd; } +void cleanupRados() noexcept { + try { +#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS + eckit::RadosPool{"test-catalogue"}.ensureDestroyed(); +#else + ensureCleanNamespaces(eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", ""), + "test-catalogue"); +#endif + if (catalogue_tests_tmp_root().exists()) { + deldir(catalogue_tests_tmp_root()); + } + } + catch (...) { + eckit::Log::error() << "FDB RADOS catalogue cleanup failed" << std::endl; + } +} + } // namespace namespace fdb::test { @@ -973,5 +991,14 @@ CASE("RadosCatalogue tests") { //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { - return eckit::testing::run_tests(argc, argv); + int ret = -1; + try { + ret = eckit::testing::run_tests(argc, argv); + } + catch (...) { + eckit::Log::error() << "FDB RADOS catalogue tests terminated with an exception" << std::endl; + } + + cleanupRados(); + return ret; } diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 5f4a36315..8318549f0 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -40,6 +40,7 @@ #include "eckit/io/rados/RadosNamespace.h" #include "eckit/io/rados/RadosObject.h" #include "eckit/io/rados/RadosPool.h" +#include "eckit/log/Log.h" #include "eckit/testing/Test.h" #include @@ -109,6 +110,25 @@ eckit::PathName& store_tests_tmp_root() { return sd; } +void cleanupRados() noexcept { + try { +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS + ensureCleanPools("test-store"); +#else + const std::string pool = eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", ""); + for (const std::string& prefix : {"test-store1", "test-store2", "test-store3", "test-store4"}) { + ensureCleanNamespaces(pool, prefix); + } +#endif + if (store_tests_tmp_root().exists()) { + deldir(store_tests_tmp_root()); + } + } + catch (...) { + eckit::Log::error() << "FDB RADOS store cleanup failed" << std::endl; + } +} + /// @note: counts only the URIs that would actually be deleted, filtering out purely /// informational wipe elements (safe/info/error) so a too-specific request yields 0. size_t countWipeable(fdb5::WipeIterator& wipeObject, bool print = true) { @@ -676,11 +696,10 @@ int main(int argc, char** argv) { ret = eckit::testing::run_tests(argc, argv); } catch (...) { + eckit::Log::error() << "FDB RADOS store tests terminated with an exception" << std::endl; } -#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS - ensureCleanPools("test-store"); -#endif + cleanupRados(); return ret; } From cb57f756485ca9a57bf2555142b7a05b2c3e771a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Thu, 20 Aug 2026 15:19:05 +0200 Subject: [PATCH 107/109] test(rados): fdb5 manage pools --- tests/fdb/rados/test_rados_catalogue.cc | 8 ++++---- tests/fdb/rados/test_rados_store.cc | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc index 52bf8bcbe..48f93b36b 100644 --- a/tests/fdb/rados/test_rados_catalogue.cc +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -129,7 +129,7 @@ eckit::PathName& catalogue_tests_tmp_root() { void cleanupRados() noexcept { try { -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS eckit::RadosPool{"test-catalogue"}.ensureDestroyed(); #else ensureCleanNamespaces(eckit::Resource("fdbRadosTestPool;$FDB_RADOS_TEST_POOL", ""), @@ -185,7 +185,7 @@ CASE("Setup") { CASE("RadosCatalogue tests") { std::string test_id = "test-catalogue"; -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -790,7 +790,7 @@ CASE("RadosCatalogue tests") { SECTION("Via FDB API with a Rados catalogue and store") { -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); #else @@ -979,7 +979,7 @@ CASE("RadosCatalogue tests") { // teardown rados -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS eckit::RadosPool{pool}.ensureDestroyed(); #else ensureCleanNamespaces(pool, test_id); diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 8318549f0..3d3ad1ea7 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -182,7 +182,7 @@ CASE("RadosStore tests") { SECTION("archive and retrieve") { std::string test_id = "test-store1"; -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -307,7 +307,7 @@ CASE("RadosStore tests") { SECTION("with POSIX Catalogue") { std::string test_id = "test-store2"; -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -427,7 +427,7 @@ CASE("RadosStore tests") { deldir(store_tests_tmp_root()); } store_tests_tmp_root().mkdir(); -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer @@ -591,7 +591,7 @@ CASE("RadosStore tests") { deldir(store_tests_tmp_root()); } store_tests_tmp_root().mkdir(); -#ifdef eckit_HAVE_RADOS_TESTS_MANAGE_POOLS +#ifdef fdb5_HAVE_RADOS_TESTS_MANAGE_POOLS std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer From 63cd125ab513392c1a93cdf0efb6c4d9b772bfef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 25 Aug 2026 13:18:31 +0200 Subject: [PATCH 108/109] docs(rados): add rados backend --- docs/fdb/content/rados-backend.rst | 346 +++++++++++++++++++++++++++++ docs/fdb/index.rst | 1 + 2 files changed, 347 insertions(+) create mode 100644 docs/fdb/content/rados-backend.rst diff --git a/docs/fdb/content/rados-backend.rst b/docs/fdb/content/rados-backend.rst new file mode 100644 index 000000000..895a25dfd --- /dev/null +++ b/docs/fdb/content/rados-backend.rst @@ -0,0 +1,346 @@ +=================== +FDB RADOS backend +=================== + +Overview +======== + +The RADOS backend stores an FDB database in a Ceph object store through +eckit's RADOS API. It separates the database into two kinds of persistent +state: + +* **Catalogue metadata**: database identity, schema, index references, index + entries, and axis values. +* **Field data**: the encoded field payloads archived by FDB. + +The backend is selected with the ``rados`` store and catalogue type. A +database key is mapped to one Ceph pool and one RADOS namespace. The namespace +contains both the catalogue metadata and the field objects for that database. + +.. mermaid:: + + flowchart TD + FDB[FDB API] --> C[Catalogue writer/reader] + FDB --> S[Store writer/reader] + C --> KV[Catalogue RADOS KV] + C --> IKV[Index and axis RADOS KVs] + S --> OBJ[Field RADOS objects] + KV --> NS[Ceph pool and database namespace] + IKV --> NS + OBJ --> NS + +Build-time enablement +===================== + +RADOS support is optional. FDB enables it when both eckit RADOS support and +the Ceph RADOS development library are available: + +.. code-block:: console + + cmake \ + -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON + +``HAVE_RADOSFDB`` controls whether the RADOS source files and tests are added +to the build. The generated ``fdb5_config.h`` exposes the corresponding +``fdb5_HAVE_RADOSFDB`` feature macro. RADOS tests are compiled only when the +backend is enabled. + +The separate ``RADOS_TESTS_MANAGE_POOLS`` option affects test setup only. It +allows tests to create and destroy their own pool; it does not change the +production backend. + +Placement and configuration +=========================== + +RADOS placement is configured through ``spaces[].roots[]``. The first +``spaces`` entry whose ``regex`` matches the database key is selected. The +current implementation requires exactly one root in the matching space. + +.. code-block:: yaml + + spaces: + - regex: ".*" + roots: + - pool: fdb-rados + root_namespace: fdb-root + namespace_prefix: fdb + +The root attributes have these meanings: + +* ``pool``: Ceph pool used for the database. +* ``root_namespace``: the **registry namespace**, a shared RADOS namespace + containing the ``main_kv`` registry for the space. +* ``namespace_prefix``: prefix used to derive the **database namespace**, the + RADOS namespace containing one database's catalogue and field objects. + +Unlike filesystem-backed FDB engines, the RADOS backend does not use a root +filesystem path. The generic FDB configuration model permits ``path`` in +``spaces[].roots[]``, but it is not needed for RADOS placement and is not read +by the RADOS engine. + +For a database key whose values serialize as ``11:22``, the database namespace +is ``fdb_11:22``. The namespace prefix must not contain ``_``, because the +underscore separates the prefix from the serialized database key. The +``root_namespace`` and the derived database namespace are different: the +registry namespace contains ``main_kv``, while the database namespace contains +``catalogue_kv``, index KVs, and field objects. + +The optional ``rados`` block currently provides the maximum multipart object part size: + +.. code-block:: yaml + + rados: + maxPartSize: 67108864 + +The value is expressed in bytes. A value of zero uses eckit's default behavior +for the multipart write handle. + +RADOS layout +============ + +For the example above, the catalogue is represented by:: + + rados:fdb-rados/fdb_11:22/catalogue_kv + +The ``main_kv`` object in the registry namespace ``fdb-root`` maps the database +namespace to this catalogue URI. This registry allows a database to be found +again when a catalogue is opened by its FDB key. + +The ``catalogue_kv`` object contains: + +* ``key``: serialized FDB database key. +* ``schema``: serialized schema used by the database. +* One entry per index key, whose value is the URI of the index RADOS KV. +* ``control.*`` entries for persisted control state, such as list and + retrieve visibility. + +Each index is a RADOS key/value object in the database namespace. Its omap +entries contain: + +* ``key``: serialized index key. +* Datum keys mapped to serialized timestamps and ``FieldLocation`` values. +* ``axis.`` markers and per-axis key/value objects used for axis + enumeration. + +Field payloads are RADOS objects in the same database namespace. Their names +are generated from the field key and a unique timestamp/host/process value +hashed with MD5, for example:: + + ..data + +Multipart object writes +---------------------- + +The RADOS multipart handle used by FDB is an eckit abstraction over several +ordinary RADOS objects. It is not the multipart-upload protocol of an S3 +gateway. It allows one logical FDB field object to be split into independently +stored RADOS objects when the payload is larger than the configured part size. + +For a logical object named ````, eckit uses this naming convention: + +* the first part is ````; +* subsequent parts are ``;part-1``, ``;part-2``, and so on. + +The writer keeps the current part open until it reaches ``maxPartSize``. A +write that crosses a part boundary is divided between the current part and +the next one. FDB supplies ``rados.maxPartSize`` to the writer; the value is +in bytes. When it is zero, eckit uses the Ceph cluster's maximum object size. + +On flush, eckit stores attributes on the base object describing the logical +object, including its total ``length``, number of ``parts``, and ``maxsize``. +The field location recorded by FDB refers to the logical base-object URI and +contains a byte offset and length. It does not expose the individual part +names to the catalogue. + +On read, eckit reads those attributes, opens the base object followed by its +``;part-N`` objects, and presents them as one contiguous, seekable stream. +This means a field can be retrieved normally even when its bytes span several +RADOS objects. The stored offset and length still allow FDB to retrieve only +the field range within a collocated logical object. + +The base object and all of its parts must be managed together. FDB therefore +uses eckit's ``ensureAllDestroyed()`` operation when removing a field and +ignores names containing ``;part-`` during object enumeration and full-wipe +discovery. A part must not be deleted independently, or the logical object +will be incomplete. + +Catalogue operation +=================== + +Creation and reopening +---------------------- + +When a ``RadosCatalogueWriter`` is created from an FDB key: + +#. The matching RADOS space is selected. +#. The root namespace and database namespace are opened. +#. ``main_kv`` is created if necessary. +#. A new ``catalogue_kv`` is created when the database does not yet exist. +#. The configured schema and serialized database key are stored in the + catalogue KV. +#. The catalogue URI is registered in ``main_kv`` under the database + namespace. + +When opened from a ``rados:`` URI, the database key and schema are read from +the catalogue KV. A missing ``key`` entry is reported as a database-not-found +error. + +Indexing and archiving metadata +------------------------------- + +Selecting an index creates or reopens the corresponding index KV. Archiving a +datum stores its serialized field location under the datum key and updates +axis values for newly observed values. Index enumeration reads the index +references from the catalogue KV and reconstructs the RADOS indexes. + +The backend intentionally does not require sorted index enumeration; the +``sorted`` argument to ``indexes()`` is ignored because RADOS key enumeration +is used directly. + +Catalogue features +------------------ + +Implemented catalogue behavior includes: + +* schema loading and persistence; +* index creation, selection, lookup, and enumeration; +* axis value persistence; +* hiding contents through persisted control entries; +* URI ownership and existence checks; +* catalogue-driven wipe and cleanup. + +The catalogue's purge, move, mount, and overlay operations are not +implemented. Statistics visitors and catalogue purge/move visitors are also +unavailable for this backend. + +Store operation +=============== + +Writing fields +-------------- + +``RadosStore::archive()`` obtains one generated RADOS object per FDB key and +reuses it for subsequent writes for that key during the store lifetime. It +obtains an eckit multipart write handle, writes the field bytes, and returns +an ``RadosFieldLocation`` containing the object URI, byte offset, and length. + +``flush()`` flushes all open data handles. ``close()`` closes them. The +catalogue subsequently stores the returned field locations in its index KVs. + +The generated field objects allow multiple writer instances to archive +concurrently to the same database. A single ``RadosStore`` instance is not +thread-safe: calls to ``archive``, ``flush``, and ``close`` must be serialized +by the caller. + +Reading fields +-------------- + +A field location points directly to a RADOS object and byte range. The store's +retrieve path returns the field's data handle, allowing FDB to read the stored +payload using the location recorded in the index. + +Store URIs use the form:: + + rados:/ + +Field object URIs add the object name as a third component. The backend checks +the URI scheme, pool, and database namespace before treating a URI as +belonging to a store. + +Store features +-------------- + +Implemented store behavior includes: + +* archive and retrieve; +* flush and close; +* object and namespace existence checks; +* listing collocated field objects; +* removing individual objects or a database namespace; +* catalogue-aware and full wipes; +* detection and removal of unrecognised objects during a full wipe; +* statistics through the normal FDB store interfaces where supported by the + caller. + +The store does not expose auxiliary URIs; ``getAuxiliaryURIs()`` returns an +empty set of results. + +Use of RADOS features +===================== + +The backend relies on the following Ceph/eckit RADOS features: + +* **Pools** provide the physical Ceph storage boundary selected by FDB space + placement. +* **Namespaces** isolate each FDB database within a pool. Multiple FDB + spaces may share a pool if their root namespaces and namespace prefixes are + distinct. +* **RADOS objects** hold field payloads and provide object existence, deletion, + enumeration, and URI addressing. +* **Object-map key/value entries** provide compact catalogue and index metadata + without creating a separate object for every metadata property. +* **Multipart writes** allow large field payloads to be written in parts, + controlled by ``rados.maxPartSize``. +* **RADOS object listing** supports collocated-data discovery and detection of + unrecognised objects during wipes. +* **URI addressing** permits stores and catalogues to be reopened from + persisted RADOS locations. + +The eckit RADOS API also provides asynchronous handles and range-read handles, +but the current FDB implementation uses synchronous data-handle operations for +retrieval and multipart write handles for archival. It does not currently use +the asynchronous or range-read APIs directly. + +Wipe and cleanup safety +======================= + +A catalogue-driven wipe first determines which index and data URIs are +included and which are safe. It removes only the selected catalogue entries, +index/axis KVs, and field objects. When the complete database is selected, the +database namespace and its root registry entry are removed after the contents +have been removed. + +A full store wipe scans only the database namespace. Objects named as +multipart parts are handled with their main object and are not independently +treated as data records. If the namespace also contains a catalogue, the +catalogue owns the namespace cleanup and the store avoids deleting it during +an unsafe full wipe. + +Limitations and operational requirements +========================================= + +* Ceph and eckit RADOS support must be present at configure time. +* The target pool must exist and the configured Ceph identity must have + permissions to access the pool and namespaces. +* RADOS placement requires at least one matching ``spaces[]`` entry and + exactly one root in that entry. +* Operations on one ``RadosStore`` instance must be serialized by the caller. +* Separate writer instances can archive concurrently, but concurrent writers + targeting the same index entry use last-write-wins semantics for that entry. +* Catalogue-side purge, move, mount, and overlay operations are not + implemented. +* The RADOS backend does not persist masking metadata; wipe removes entries + directly. +* The ``sorted`` index enumeration request is ignored. +* Auxiliary store URIs are not provided. +* Runtime tests require a reachable Ceph cluster and an existing test pool + unless ``RADOS_TESTS_MANAGE_POOLS`` is enabled. + +Minimal test setup +================== + +With an existing Ceph pool: + +.. code-block:: console + + cmake \ + -DENABLE_RADOS=ON \ + -DENABLE_RADOSFDB=ON \ + -DFDB_RADOS_TEST_POOL=fdb_test + + ctest -R 'fdb_test_rados_(store|catalogue)' + +The RADOS test environment must also provide the Ceph configuration and +credentials expected by eckit, for example through the standard Ceph +configuration directory and the configured RADOS cluster/user settings. diff --git a/docs/fdb/index.rst b/docs/fdb/index.rst index d18c5e0cc..3631353d7 100644 --- a/docs/fdb/index.rst +++ b/docs/fdb/index.rst @@ -22,6 +22,7 @@ the MARS Archive. content/mars content/config-schema content/environment-variables + content/rados-backend cli_tools/index content/api content/license From 0e73a27f7fc0a9258b6b130cbcf833afac6bf99e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Metin=20=C3=87ak=C4=B1rcal=C4=B1?= Date: Tue, 25 Aug 2026 14:37:17 +0200 Subject: [PATCH 109/109] ignore doc-build --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 48937c521..efa7b4f29 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ __pycache__/ # Rust rust/target/ rust/Cargo.lock +doc-build