diff --git a/.github/ci-config.yml b/.github/ci-config.yml index e368cd714..644ab2a97 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -1,7 +1,9 @@ +dependency_cmake_options: | + ecmwf/eccodes: "-DENABLE_ECKIT_GEO=ON" dependencies: | ecmwf/ecbuild Deutsches-Klimarechenzentrum/libaec@refs/tags/v1.1.6 - ecmwf/eccodes ecmwf/eckit + ecmwf/eccodes dependency_branch: develop parallelism_factor: 8 diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index 19d38d1f1..bd8854219 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -3,8 +3,10 @@ build: - ninja dependencies: - ecmwf/ecbuild@develop - - ecmwf/eccodes@develop - ecmwf/eckit@develop + - ecmwf/eccodes@develop + dependency_cmake_options: + - "ecmwf/eccodes: '-DENABLE_ECKIT_GEO=ON'" parallel: 64 env: - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples diff --git a/.github/workflows/pymetkit.yml b/.github/workflows/pymetkit.yml index 56a250791..2f6a455b2 100644 --- a/.github/workflows/pymetkit.yml +++ b/.github/workflows/pymetkit.yml @@ -42,6 +42,23 @@ jobs: run: | mkdir stack-dependencies-build stack-dependencies-src/build.sh --build-path stack-dependencies-build --install-path dependencies --with-deps libaec,pybind11 + - name: Get eckit + uses: actions/checkout@v5 + with: + repository: ecmwf/eckit + ref: develop + path: eckit-src + - name: Install eckit + run: | + mkdir eckit-build + cmake \ + -B eckit-build \ + -S eckit-src \ + -GNinja \ + -DCMAKE_INSTALL_PREFIX=dependencies \ + -DCMAKE_PREFIX_PATH=dependencies \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo + cmake --build eckit-build -j -t install - name: Get eccodes uses: actions/checkout@v5 with: @@ -59,25 +76,9 @@ jobs: -DCMAKE_PREFIX_PATH=dependencies \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DENABLE_MEMFS=ON \ - -DENABLE_AEC=ON + -DENABLE_AEC=ON \ + -DENABLE_ECKIT_GEO=ON cmake --build eccodes-build -j -t install - - name: Get eckit - uses: actions/checkout@v5 - with: - repository: ecmwf/eckit - ref: develop - path: eckit-src - - name: Install eckit - run: | - mkdir eckit-build - cmake \ - -B eckit-build \ - -S eckit-src \ - -GNinja \ - -DCMAKE_INSTALL_PREFIX=dependencies \ - -DCMAKE_PREFIX_PATH=dependencies \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo - cmake --build eckit-build -j -t install - name: Archive with permissions preserved run: tar --zstd -cpf files.tar.zst dependencies/ ecbuild/ - name: Upload dependencies diff --git a/CMakeLists.txt b/CMakeLists.txt index 7dd86c0aa..d70b37124 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,7 @@ ecbuild_add_option( FEATURE ODB ecbuild_add_option( FEATURE GRIB2MARS DEFAULT ON - CONDITION HAVE_GRIB + CONDITION HAVE_GRIB AND eckit_HAVE_ECKIT_GEO AND eccodes_HAVE_ECKIT_GEO DESCRIPTION "Build the GRIB2MARS converter" ) ecbuild_add_option( FEATURE MARS2MARS @@ -84,7 +84,7 @@ ecbuild_add_option( FEATURE MARS2MARS ecbuild_add_option( FEATURE MARS2GRIB DEFAULT ON - CONDITION HAVE_GRIB AND eckit_HAVE_ECKIT_GEO + CONDITION HAVE_GRIB AND eckit_HAVE_ECKIT_GEO AND eccodes_HAVE_ECKIT_GEO DESCRIPTION "Build the MARS2GRIB encoder" ) # Pythonic pybind11 interface (pymetkit) diff --git a/src/metkit/grib2mars/api/Grib2Mars.cc b/src/metkit/grib2mars/api/Grib2Mars.cc index e35e5c0b6..775dc881f 100644 --- a/src/metkit/grib2mars/api/Grib2Mars.cc +++ b/src/metkit/grib2mars/api/Grib2Mars.cc @@ -20,6 +20,8 @@ #include "Grib2Mars.h" +#include + #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" #include "metkit/grib2mars/mappings/mappings.h" @@ -38,18 +40,42 @@ namespace metkit::grib2mars { +namespace { + +/// @brief Apply environment side effects implied by a set of options. +/// +/// When `skipSection3` is enabled the encoder delegates geometry handling to +/// gridSpec/ecCodes, which requires ecCodes to be configured with eckit_geo +/// support enabled. This is controlled by the `ECCODES_ECKIT_GEO` environment +/// variable, so force it to "1" whenever `skipSection3` is requested. +inline void applyOptionSideEffects(const Options& opts) { + if (opts.skipSection3) { + ::setenv("ECCODES_ECKIT_GEO", "1", 1); + } +} + +} // namespace + // ----------------------------------------------------------------------------- // Grib2Mars construction // ----------------------------------------------------------------------------- /// @brief Default construct a Grib2Mars converter. -Grib2Mars::Grib2Mars() : opts_{} {} +Grib2Mars::Grib2Mars() : opts_{} { + applyOptionSideEffects(opts_); +} -Grib2Mars::Grib2Mars(const Options& opts) : opts_{opts} {} +Grib2Mars::Grib2Mars(const Options& opts) : opts_{opts} { + applyOptionSideEffects(opts_); +} -Grib2Mars::Grib2Mars(const eckit::LocalConfiguration& opts) : opts_{detail::readOptions(opts)} {} +Grib2Mars::Grib2Mars(const eckit::LocalConfiguration& opts) : opts_{detail::readOptions(opts)} { + applyOptionSideEffects(opts_); +} -Grib2Mars::Grib2Mars(OptionList opts) : opts_{detail::readOptions(opts)} {} +Grib2Mars::Grib2Mars(OptionList opts) : opts_{detail::readOptions(opts)} { + applyOptionSideEffects(opts_); +} /// @brief Convert an `eckit::LocalConfiguration` request. template <> diff --git a/src/metkit/grib2mars/mappings/rules/grid.h b/src/metkit/grib2mars/mappings/rules/grid.h index e13d26e3b..c52cc4880 100644 --- a/src/metkit/grib2mars/mappings/rules/grid.h +++ b/src/metkit/grib2mars/mappings/rules/grid.h @@ -12,12 +12,27 @@ namespace metkit::grib2mars::rules::impl { template void extractGrid(const std::string& keyword, const metkit::codes::CodesHandle& grib, MarsDict& mars, MiscDict& misc, const OptDict_t& opts) { + using metkit::grib2mars::utils::dict_traits::get_or_throw; using metkit::grib2mars::utils::dict_traits::set_or_throw; using metkit::grib2mars::utils::exceptions::Grib2MarsGenericException; try { - (void)opts; - (void)misc; + if (get_or_throw(opts, "skipSection3")) { + if (!grib.has("gridSpec")) { + throw Grib2MarsGenericException("Missing GRIB key `gridSpec` required to extract MARS keyword `grid`", + Here()); + } + + const auto gridSpec = grib.getString("gridSpec"); + set_or_throw(mars, "grid", gridSpec); + + if (grib.has("shapeOfTheEarth")) { + const long shapeOfTheEarth = grib.getLong("shapeOfTheEarth"); + misc.set("shapeOfTheEarth", shapeOfTheEarth); + } + + return; + } if (!grib.has("gridType")) { throw Grib2MarsGenericException( diff --git a/src/metkit/grib2mars/mappings/rules/truncation.h b/src/metkit/grib2mars/mappings/rules/truncation.h index e1ef98394..f6c5d72af 100644 --- a/src/metkit/grib2mars/mappings/rules/truncation.h +++ b/src/metkit/grib2mars/mappings/rules/truncation.h @@ -12,11 +12,50 @@ namespace metkit::grib2mars::rules::impl { template void extractTruncation(const std::string& keyword, const metkit::codes::CodesHandle& grib, MarsDict& mars, MiscDict& misc, const OptDict_t& opts) { + using metkit::grib2mars::utils::dict_traits::get_or_throw; using metkit::grib2mars::utils::dict_traits::set_or_throw; using metkit::grib2mars::utils::exceptions::Grib2MarsGenericException; try { - (void)opts; + if (get_or_throw(opts, "skipSection3")) { + if (!grib.has("gridSpec")) { + throw Grib2MarsGenericException("Missing GRIB key `gridSpec` required to extract MARS keyword `grid`", + Here()); + } + + const auto gridSpec = grib.getString("gridSpec"); + set_or_throw(mars, "grid", gridSpec); + + const long j = grib.getLong("J"); + const long k = grib.getLong("K"); + const long m = grib.getLong("M"); + + if (j != k || j != m) { + throw Grib2MarsGenericException("Grib keys `J/K/M` must be equal! J=" + std::to_string(j) + + ", K=" + std::to_string(k) + ", M=" + std::to_string(m), + Here()); + } + + if (grib.has("JS")) { + const long js = grib.getLong("JS"); + const long ks = grib.getLong("KS"); + const long ms = grib.getLong("MS"); + + if (js != ks || js != ms) { + throw Grib2MarsGenericException("Grib keys `JS/KS/MS` must be equal! JS=" + std::to_string(js) + + ", KS=" + std::to_string(ks) + ", MS=" + std::to_string(ms), + Here()); + } + + if (js > 0 && js <= j) { + // Set subSetTruncation based on JS, which is equal to KS and MS + set_or_throw(misc, "subSetTruncation", js); + } + } + + return; + } + if (!grib.has("gridType")) { throw Grib2MarsGenericException( "Missing GRIB key `gridType` required to extract MARS keyword `" + keyword + "`", Here()); diff --git a/src/metkit/mars2grib/api/Mars2Grib.cc b/src/metkit/mars2grib/api/Mars2Grib.cc index 77476f58f..91d4b1aa1 100644 --- a/src/metkit/mars2grib/api/Mars2Grib.cc +++ b/src/metkit/mars2grib/api/Mars2Grib.cc @@ -60,6 +60,8 @@ /// #include "Mars2Grib.h" +#include + // other libraries #include "eckit/exception/Exceptions.h" @@ -397,13 +399,37 @@ std::unique_ptr Mars2Grib::finaliseEncoding(const Ca // Mars2Grib construction // ----------------------------------------------------------------------------- -Mars2Grib::Mars2Grib() : opts_{} {} +namespace { + +/// @brief Apply environment side effects implied by a set of options. +/// +/// When `skipSection3` is enabled the encoder delegates geometry handling to +/// gridSpec/ecCodes, which requires ecCodes to be configured with eckit_geo +/// support enabled. This is controlled by the `ECCODES_ECKIT_GEO` environment +/// variable, so force it to "1" whenever `skipSection3` is requested. +inline void applyOptionSideEffects(const Options& opts) { + if (opts.skipSection3) { + ::setenv("ECCODES_ECKIT_GEO", "1", 1); + } +} + +} // namespace -Mars2Grib::Mars2Grib(const Options& opts) : opts_{opts} {} +Mars2Grib::Mars2Grib() : opts_{} { + applyOptionSideEffects(opts_); +} -Mars2Grib::Mars2Grib(const eckit::LocalConfiguration& opts) : opts_{detail::readOptions(opts)} {} +Mars2Grib::Mars2Grib(const Options& opts) : opts_{opts} { + applyOptionSideEffects(opts_); +} -Mars2Grib::Mars2Grib(OptionList opts) : opts_{detail::readOptions(opts)} {} +Mars2Grib::Mars2Grib(const eckit::LocalConfiguration& opts) : opts_{detail::readOptions(opts)} { + applyOptionSideEffects(opts_); +} + +Mars2Grib::Mars2Grib(OptionList opts) : opts_{detail::readOptions(opts)} { + applyOptionSideEffects(opts_); +} // ----------------------------------------------------------------------------- diff --git a/src/metkit/mars2grib/backend/concepts/representation/representationEncoding.h b/src/metkit/mars2grib/backend/concepts/representation/representationEncoding.h index 2942eb5ab..b681c5d59 100644 --- a/src/metkit/mars2grib/backend/concepts/representation/representationEncoding.h +++ b/src/metkit/mars2grib/backend/concepts/representation/representationEncoding.h @@ -213,7 +213,7 @@ template constexpr bool representationApplicable() { // Disable the concept for the Dummy variant - if constexpr (Variant == RepresentationType::Dummy) { + if constexpr (Variant == RepresentationType::Dummy || Variant == RepresentationType::DummySH) { return false; } diff --git a/src/metkit/mars2grib/backend/concepts/representation/representationEnum.h b/src/metkit/mars2grib/backend/concepts/representation/representationEnum.h index 5300609e5..79044829d 100644 --- a/src/metkit/mars2grib/backend/concepts/representation/representationEnum.h +++ b/src/metkit/mars2grib/backend/concepts/representation/representationEnum.h @@ -93,7 +93,8 @@ enum class RepresentationType : std::size_t { Healpix, Orca, Fesom, - Dummy, // Used to skip section3 resolution and delegate to gridSpec + Dummy, // Used to skip section3 resolution and delegate to gridSpec + DummySH, // Special case for spherical harmonics Default }; @@ -114,7 +115,7 @@ using RepresentationList = ValueList; + RepresentationType::Dummy, RepresentationType::DummySH, RepresentationType::Default>; /// @@ -153,6 +154,7 @@ DEF(RepresentationType::Healpix, "healpix"); DEF(RepresentationType::Orca, "orca"); DEF(RepresentationType::Fesom, "fesom"); DEF(RepresentationType::Dummy, "dummy"); +DEF(RepresentationType::DummySH, "dummySH"); DEF(RepresentationType::Default, "default"); #undef DEF diff --git a/src/metkit/mars2grib/backend/concepts/representation/representationMatcher.h b/src/metkit/mars2grib/backend/concepts/representation/representationMatcher.h index 813b02c9d..ab7cf645b 100644 --- a/src/metkit/mars2grib/backend/concepts/representation/representationMatcher.h +++ b/src/metkit/mars2grib/backend/concepts/representation/representationMatcher.h @@ -67,6 +67,13 @@ std::size_t representationMatcher(const MarsDict_t& mars, const OptDict_t& opt) // This is used to fully delegate section3 setting to gridSpec if (get_or_throw(opt, "skipSection3")) { + const auto marsGrid = get_or_throw(mars, "grid"); + const auto gridType = + std::unique_ptr(eckit::geo::GridFactory::make_from_string(marsGrid))->type(); + + if (gridType == "sh") { + return static_cast(RepresentationType::DummySH); + } return static_cast(RepresentationType::Dummy); } diff --git a/src/metkit/mars2grib/backend/deductions/subSetTrunc.h b/src/metkit/mars2grib/backend/deductions/subSetTrunc.h index 24388f999..8ba21afd9 100644 --- a/src/metkit/mars2grib/backend/deductions/subSetTrunc.h +++ b/src/metkit/mars2grib/backend/deductions/subSetTrunc.h @@ -44,9 +44,10 @@ // System includes #include -#include #include +#include "eckit/geo/Grid.h" + // Core deduction includes #include "metkit/config/LibMetkit.h" #include "metkit/mars2grib/utils/generalUtils.h" @@ -92,8 +93,6 @@ long resolve_SubSetTruncation_or_throw(const MarsDict_t& mars, const ParDict_t& using metkit::mars2grib::utils::dict_traits::has; using metkit::mars2grib::utils::exceptions::Mars2GribDeductionException; - static const std::regex tGridRegex("^T(0|[1-9][0-9]*)$"); - try { // subSetTruncation must not be larger than any pentagonalResolutionParameter @@ -102,13 +101,9 @@ long resolve_SubSetTruncation_or_throw(const MarsDict_t& mars, const ParDict_t& const long truncation = [&]() { if (get_or_throw(opt, "skipSection3")) { - const std::string grid = get_or_throw(mars, "grid"); - std::smatch tGridMatch; - if (!std::regex_match(grid, tGridMatch, tGridRegex) || tGridMatch.size() != 2) { - throw Mars2GribDeductionException("Cannot extract truncation from MARS grid '" + grid + "'", - Here()); - } - return std::stol(tGridMatch[1].str()); + const std::string grid = get_or_throw(mars, "grid"); + const eckit::geo::Grid* gridSpec = eckit::geo::GridFactory::make_from_string(grid); + return static_cast(gridSpec->truncation()); } return get_or_throw(mars, "truncation"); }(); diff --git a/src/metkit/mars2grib/backend/encodeValues.h b/src/metkit/mars2grib/backend/encodeValues.h index ffded3c8f..772823253 100644 --- a/src/metkit/mars2grib/backend/encodeValues.h +++ b/src/metkit/mars2grib/backend/encodeValues.h @@ -130,13 +130,21 @@ void encodeValues(Span values, const MiscDict_t& misc, const OptDic } // 3. Set operation(s) to be applied on the values + const bool isSH = get_or_throw(handle, "gridType") == "sh"; + if (has(misc, "scaleValuesBy")) { const double scaleValuesBy = get_or_throw(misc, "scaleValuesBy"); + if (scaleValuesBy != 1.0 && isSH) { + throw Mars2GribGenericException("Value scaling is not implemented for spherical harmonics!", Here()); + } set_or_throw(handle, "scaleValuesBy", scaleValuesBy); } if (has(misc, "offsetValuesBy")) { const double offsetValuesBy = get_or_throw(misc, "offsetValuesBy"); + if (offsetValuesBy != 0.0 && isSH) { + throw Mars2GribGenericException("Value offsetting is not implemented for spherical harmonics!", Here()); + } set_or_throw(handle, "offsetValuesBy", offsetValuesBy); } } @@ -238,13 +246,21 @@ void encodeValuesGridSpec(Span values, const MarsDict_t& mars, cons } // 3. Set operation(s) to be applied on the values + const bool isSH = get_or_throw(handle, "gridType") == "sh"; + if (has(misc, "scaleValuesBy")) { const double scaleValuesBy = get_or_throw(misc, "scaleValuesBy"); + if (scaleValuesBy != 1.0 && isSH) { + throw Mars2GribGenericException("Value scaling is not implemented for spherical harmonics!", Here()); + } set_or_throw(handle, "scaleValuesBy", scaleValuesBy); } if (has(misc, "offsetValuesBy")) { const double offsetValuesBy = get_or_throw(misc, "offsetValuesBy"); + if (offsetValuesBy != 0.0 && isSH) { + throw Mars2GribGenericException("Value offsetting is not implemented for spherical harmonics!", Here()); + } set_or_throw(handle, "offsetValuesBy", offsetValuesBy); } } diff --git a/src/metkit/mars2grib/backend/sections/initializers/sectionInitializer3.h b/src/metkit/mars2grib/backend/sections/initializers/sectionInitializer3.h index 7d3573438..5c03c64c2 100644 --- a/src/metkit/mars2grib/backend/sections/initializers/sectionInitializer3.h +++ b/src/metkit/mars2grib/backend/sections/initializers/sectionInitializer3.h @@ -85,7 +85,7 @@ void allocateTemplateNumber3(const MarsDict_t& mars, const ParDict_t& par, const try { // Special handling for spectral grids (Template 50) - if constexpr (TemplateNumber == 50) { + if constexpr (TemplateNumber == 50 || TemplateNumber == 1001) { // Precondition GRIB handle for spectral representation set_or_throw(out, "numberOfDataPoints", 6L); set_or_throw(out, "numberOfValues", 6L); @@ -105,7 +105,7 @@ void allocateTemplateNumber3(const MarsDict_t& mars, const ParDict_t& par, const set_or_throw(out, "dataRepresentationTemplateNumber", 51L); } - if constexpr (TemplateNumber != 50 && TemplateNumber != 1000) { + if constexpr (TemplateNumber != 50 && TemplateNumber != 1000 && TemplateNumber != 1001) { // Standard grid definition template long drt = static_cast(TemplateNumber); set_or_throw(out, "gridDefinitionTemplateNumber", drt); diff --git a/src/metkit/mars2grib/backend/sections/initializers/sectionRegistry.h b/src/metkit/mars2grib/backend/sections/initializers/sectionRegistry.h index c6970cd17..1e6945677 100644 --- a/src/metkit/mars2grib/backend/sections/initializers/sectionRegistry.h +++ b/src/metkit/mars2grib/backend/sections/initializers/sectionRegistry.h @@ -101,7 +101,8 @@ inline constexpr Entry Sec3Reg[] = {50, &allocateTemplateNumber3<3, 50, MarsDict_t, ParDict_t, OptDict_t, OutDict_t>}, {101, &allocateTemplateNumber3<3, 101, MarsDict_t, ParDict_t, OptDict_t, OutDict_t>}, {150, &allocateTemplateNumber3<3, 150, MarsDict_t, ParDict_t, OptDict_t, OutDict_t>}, - {1000, &allocateTemplateNumber3<3, 1000, MarsDict_t, ParDict_t, OptDict_t, OutDict_t>}}; + {1000, &allocateTemplateNumber3<3, 1000, MarsDict_t, ParDict_t, OptDict_t, OutDict_t>}, + {1001, &allocateTemplateNumber3<3, 1001, MarsDict_t, ParDict_t, OptDict_t, OutDict_t>}}; /// /// @brief Registry for GRIB Section 4 (Product Definition Section) initializers. diff --git a/src/metkit/mars2grib/frontend/resolution/section-recipes/impl/section3Recipes.h b/src/metkit/mars2grib/frontend/resolution/section-recipes/impl/section3Recipes.h index 65431438a..10401a974 100644 --- a/src/metkit/mars2grib/frontend/resolution/section-recipes/impl/section3Recipes.h +++ b/src/metkit/mars2grib/frontend/resolution/section-recipes/impl/section3Recipes.h @@ -75,6 +75,15 @@ inline const Recipe S3_R1000 = RepresentationType::Dummy> >(); +// Same as above, but with prepping of the sample for spherical harmonics still enabled +inline const Recipe S3_R1001 = + make_recipe<1001, + Select, + Select + >(); + //------------------------------------------------------------------------------ // Section 3 – Aggregated Recipes //------------------------------------------------------------------------------ @@ -87,7 +96,8 @@ inline const Recipes Section3Recipes{ 3, &S3_R101, &S3_R101_ORCA, &S3_R150, - &S3_R1000 + &S3_R1000, + &S3_R1001 } }; diff --git a/src/metkit/mars2mars/api/Mars2Mars.cc b/src/metkit/mars2mars/api/Mars2Mars.cc index 58dbf4264..a3895d459 100644 --- a/src/metkit/mars2mars/api/Mars2Mars.cc +++ b/src/metkit/mars2mars/api/Mars2Mars.cc @@ -20,6 +20,8 @@ #include "Mars2Mars.h" +#include + #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" #include "metkit/mars/MarsRequest.h" @@ -38,41 +40,115 @@ namespace metkit::mars2mars { +namespace { + +/// @brief Apply environment side effects implied by a set of options. +/// +/// When `skipSection3` is enabled the encoder delegates geometry handling to +/// gridSpec/ecCodes, which requires ecCodes to be configured with eckit_geo +/// support enabled. This is controlled by the `ECCODES_ECKIT_GEO` environment +/// variable, so force it to "1" whenever `skipSection3` is requested. +inline void applyOptionSideEffects(const Options& opts) { + if (opts.skipSection3) { + ::setenv("ECCODES_ECKIT_GEO", "1", 1); + } +} + +eckit::LocalConfiguration mergeLocalConfigs(const eckit::LocalConfiguration& base, + const eckit::LocalConfiguration& overwrite) { + eckit::LocalConfiguration result{base}; + for (const auto& key : overwrite.keys()) { + if (overwrite.isString(key)) { + result.set(key, overwrite.getString(key)); + } + else if (overwrite.isIntegral(key)) { + result.set(key, overwrite.getLong(key)); + } + else if (overwrite.isFloatingPoint(key)) { + result.set(key, overwrite.getDouble(key)); + } + else if (overwrite.isBoolean(key)) { + result.set(key, overwrite.getBool(key)); + } + else if (overwrite.isFloatingPointList(key)) { + result.set(key, overwrite.getDoubleVector(key)); + } + else { + throw eckit::NotImplemented("Unexpected type for '" + key + "'", Here()); + } + } + return result; +} + +} // namespace + // ----------------------------------------------------------------------------- // Mars2Mars construction // ----------------------------------------------------------------------------- /// @brief Default construct a Mars2Mars converter. -Mars2Mars::Mars2Mars() : opts_{} {} +Mars2Mars::Mars2Mars() : opts_{} { + applyOptionSideEffects(opts_); +} -Mars2Mars::Mars2Mars(const Options& opts) : opts_{opts} {} +Mars2Mars::Mars2Mars(const Options& opts) : opts_{opts} { + applyOptionSideEffects(opts_); +} -Mars2Mars::Mars2Mars(const eckit::LocalConfiguration& opts) : opts_{detail::readOptions(opts)} {} +Mars2Mars::Mars2Mars(const eckit::LocalConfiguration& opts) : opts_{detail::readOptions(opts)} { + applyOptionSideEffects(opts_); +} -Mars2Mars::Mars2Mars(OptionList opts) : opts_{detail::readOptions(opts)} {} +Mars2Mars::Mars2Mars(OptionList opts) : opts_{detail::readOptions(opts)} { + applyOptionSideEffects(opts_); +} /// @brief Convert an `eckit::LocalConfiguration` request. -template <> -Mars2MarsResult Mars2Mars::convert( - const eckit::LocalConfiguration& mars) { - +Mars2MarsResult Mars2Mars::convert(const eckit::LocalConfiguration& mars) { using metkit::mars2mars::utils::exceptions::withMars2MarsApiErrorHandling; return withMars2MarsApiErrorHandling>( - "Mars2Mars::convert", opts_, + "Mars2Mars::convert", opts_, [&]() { return rules::convertAll(mars, opts_); }, Here()); } -/// @brief Convert a `metkit::mars::MarsRequest` request. -template <> -Mars2MarsResult Mars2Mars::convert( - const metkit::mars::MarsRequest& mars) { +/// @brief Convert an `eckit::LocalConfiguration` request. +Mars2MarsResult Mars2Mars::convert(const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc) { + using metkit::mars2mars::utils::exceptions::withMars2MarsApiErrorHandling; + return withMars2MarsApiErrorHandling>( + "Mars2Mars::convert", opts_, + [&]() { + auto result = rules::convertAll(mars, opts_); + result.misc = mergeLocalConfigs(result.misc, misc); + return result; + }, + Here()); +} + +/// @brief Convert a `metkit::mars::MarsRequest` request. +Mars2MarsResult Mars2Mars::convert(const metkit::mars::MarsRequest& mars) { using metkit::mars2mars::utils::exceptions::withMars2MarsApiErrorHandling; return withMars2MarsApiErrorHandling>( - "Mars2Mars::convert", opts_, + "Mars2Mars::convert", opts_, [&]() { return rules::convertAll(mars, opts_); }, Here()); } +/// @brief Convert a `metkit::mars::MarsRequest` request. +Mars2MarsResult Mars2Mars::convert(const metkit::mars::MarsRequest& mars, + const eckit::LocalConfiguration& misc) { + using metkit::mars2mars::utils::exceptions::withMars2MarsApiErrorHandling; + + return withMars2MarsApiErrorHandling>( + "Mars2Mars::convert", opts_, + [&]() { + auto result = rules::convertAll(mars, opts_); + result.misc = mergeLocalConfigs(result.misc, misc); + return result; + }, + Here()); +} + } // namespace metkit::mars2mars diff --git a/src/metkit/mars2mars/api/Mars2Mars.h b/src/metkit/mars2mars/api/Mars2Mars.h index a95495154..8b0649a29 100644 --- a/src/metkit/mars2mars/api/Mars2Mars.h +++ b/src/metkit/mars2mars/api/Mars2Mars.h @@ -108,40 +108,53 @@ class Mars2Mars { ~Mars2Mars() = default; - /// @brief Convert a supported dictionary type. + /// @brief Convert a MARS description /// - /// The primary template is deleted intentionally. Only explicit - /// specializations are available through the public API. + /// @param[in] mars + /// Input MARS dictionary. + /// + /// @return + /// A converted MARS + misc result + Mars2MarsResult convert(const eckit::LocalConfiguration& mars); + + /// @brief Convert a MARS description + /// + /// @param[in] mars + /// Input MARS dictionary. + /// + /// @param[in] misc + /// Input misc dictionary. + /// + /// @return + /// A converted MARS + misc result + Mars2MarsResult convert(const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc); + + /// @brief Convert a MARS description + /// + /// @param[in] mars + /// Input MARS dictionary. /// - /// @tparam Dict_t - /// Input dictionary type. + /// @return + /// A converted MARS + misc result + Mars2MarsResult convert(const metkit::mars::MarsRequest& mars); + + /// @brief Convert a MARS description /// /// @param[in] mars /// Input MARS dictionary. /// + /// @param[in] misc + /// Input misc dictionary. + /// /// @return - /// A converted result for the supported dictionary type. - template - Mars2MarsResult convert(const Dict_t& mars) = delete; + /// A converted MARS + misc result + Mars2MarsResult convert(const metkit::mars::MarsRequest& mars, + const eckit::LocalConfiguration& misc); private: const Options opts_; }; - -// ----------------------------------------------------------------------------- -// Supported API specializations -// ----------------------------------------------------------------------------- - -/// @brief Convert an `eckit::LocalConfiguration` request. -template <> -Mars2MarsResult Mars2Mars::convert( - const eckit::LocalConfiguration& mars); - -/// @brief Convert a `metkit::mars::MarsRequest` request. -template <> -Mars2MarsResult Mars2Mars::convert( - const metkit::mars::MarsRequest& mars); - } // namespace metkit::mars2mars diff --git a/src/pymetkit/experimental/mars2mars/_internal/mars2mars_bindings.cc b/src/pymetkit/experimental/mars2mars/_internal/mars2mars_bindings.cc index 7fbb0292e..2a3f19d8e 100644 --- a/src/pymetkit/experimental/mars2mars/_internal/mars2mars_bindings.cc +++ b/src/pymetkit/experimental/mars2mars/_internal/mars2mars_bindings.cc @@ -160,20 +160,20 @@ PYBIND11_MODULE(mars2mars_bindings, m) { // Compile-time mars2mars version m.attr("__mars2mars_build_version__") = metkit_VERSION_STR; - auto mars2mars = - py::class_(m, "Mars2Mars") - .def(py::init<>()) - .def(py::init( - [](py::dict dict) { return std::make_unique(dictToLocalConfig(dict)); })) - .def("convert", - [](Mars2Mars& mars2mars, py::dict& dict) { - const auto& result = mars2mars.convert(dictToLocalConfig(dict)); - return std::make_pair(to_dict(result.mars), to_dict(result.misc)); - }) - .def("convert", [](Mars2Mars& mars2mars, const metkit::mars::MarsRequest& mars_request) { - const metkit::mars2mars::Mars2MarsResult result = - mars2mars.convert(mars_request); - - return std::pair(to_dict(result.mars), to_dict(result.misc)); - }); + auto mars2mars = py::class_(m, "Mars2Mars") + .def(py::init<>()) + .def(py::init([](py::dict dict) { + return std::make_unique(dictToLocalConfig(dict)); + })) + .def("convert", + [](Mars2Mars& mars2mars, py::dict& dict) { + const auto& result = mars2mars.convert(dictToLocalConfig(dict)); + return std::make_pair(to_dict(result.mars), to_dict(result.misc)); + }) + .def("convert", [](Mars2Mars& mars2mars, const metkit::mars::MarsRequest& mars_request) { + const metkit::mars2mars::Mars2MarsResult result = + mars2mars.convert(mars_request); + + return std::pair(to_dict(result.mars), to_dict(result.misc)); + }); } diff --git a/src/tools/grib-to-mtg2.cc b/src/tools/grib-to-mtg2.cc index 0a709d2af..eb566e168 100644 --- a/src/tools/grib-to-mtg2.cc +++ b/src/tools/grib-to-mtg2.cc @@ -120,32 +120,6 @@ std::unique_ptr readCodesHandle(eckit::message::Mess return metkit::codes::codesHandleFromMessageCopy(metkit::codes::Span(data, size)); } -eckit::LocalConfiguration mergeLocalConfigs(const eckit::LocalConfiguration& base, - const eckit::LocalConfiguration& overwrite) { - eckit::LocalConfiguration result{base}; - for (const auto& key : overwrite.keys()) { - if (overwrite.isString(key)) { - result.set(key, overwrite.getString(key)); - } - else if (overwrite.isIntegral(key)) { - result.set(key, overwrite.getLong(key)); - } - else if (overwrite.isFloatingPoint(key)) { - result.set(key, overwrite.getDouble(key)); - } - else if (overwrite.isBoolean(key)) { - result.set(key, overwrite.getBool(key)); - } - else if (overwrite.isFloatingPointList(key)) { - result.set(key, overwrite.getDoubleVector(key)); - } - else { - throw eckit::NotImplemented("Unexpected type for '" + key + "'", Here()); - } - } - return result; -} - //---------------------------------------------------------------------------------------------------------------------- bool isDiscipline192(const long param, const bool ignoreIfMappable = true) { @@ -416,9 +390,9 @@ bool skipStepZero(const long param) { void Grib1ToGrib2Tool::execute(const CmdArgs& args) { // Handles to conversion libraries - metkit::grib2mars::Grib2Mars grib2mars; - metkit::mars2mars::Mars2Mars mars2mars; - metkit::mars2grib::Mars2Grib mars2grib; + metkit::grib2mars::Grib2Mars grib2mars{{"skipSection3", true}}; + metkit::mars2mars::Mars2Mars mars2mars{{"skipSection3", true}}; + metkit::mars2grib::Mars2Grib mars2grib{{"skipSection3", true}}; auto inputPath = eckit::PathName(args(0)); auto outputPath = eckit::PathName(args(1)); @@ -457,10 +431,10 @@ void Grib1ToGrib2Tool::execute(const CmdArgs& args) { const std::vector values = codesHandle->getDoubleArray("values"); // Apply mappings to convert pre-MTG2 MARS/Misc to post-MTG2 MARS/Misc - const auto mappedMarsMisc = mars2mars.convert(originalMarsMisc.mars); + const auto mappedMarsMisc = mars2mars.convert(originalMarsMisc.mars, originalMarsMisc.misc); auto mars = mappedMarsMisc.mars; - auto misc = mergeLocalConfigs(mappedMarsMisc.misc, originalMarsMisc.misc); + auto misc = mappedMarsMisc.misc; // Override values if specified by the user in the arguments if (expver_) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ae9882896..1dcfc6d95 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -108,10 +108,18 @@ ecbuild_add_test( TARGET metkit_test_c_compiled add_subdirectory(regressions) add_subdirectory(marsgen) -add_subdirectory(mars2grib) -add_subdirectory(mars2mars) add_subdirectory(tools) +if(HAVE_MARS2GRIB) + add_subdirectory(mars2grib) +endif() +if(HAVE_MARS2MARS) + add_subdirectory(mars2mars) +endif() +if(HAVE_GRIB2MARS AND HAVE_MARS2GRIB) + add_subdirectory(grib2grib) +endif() + if( HAVE_PYTHON_METKIT_INTERFACE ) add_subdirectory(pymetkit) endif() diff --git a/tests/grib2grib/CMakeLists.txt b/tests/grib2grib/CMakeLists.txt new file mode 100644 index 000000000..b321cfef3 --- /dev/null +++ b/tests/grib2grib/CMakeLists.txt @@ -0,0 +1,30 @@ +ecbuild_add_executable( + TARGET roundtrip + SOURCES roundtrip.cc + LIBS metkit eccodes + NOINSTALL +) + +macro(add_roundtrip_test arg1) + string(JOIN "_" _extra ${ARGN}) + string(MAKE_C_IDENTIFIER "metkit_test_grib2grib_${arg1}_${_extra}" _target) + string(REGEX REPLACE "__+" "_" _target "${_target}") + string(REGEX REPLACE "_$" "" _target "${_target}") + + ecbuild_add_test( + TARGET "${_target}" + COMMAND roundtrip + ARGS "${arg1}" ${ARGN} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ENVIRONMENT ECCODES_ECKIT_GEO=1 + ) +endmacro() + +add_roundtrip_test("mtg2-sfc.grib2") +add_roundtrip_test("mtg2-sfc.grib2" "--grid=O1280") +add_roundtrip_test("mtg2-o2d.grib2") +add_roundtrip_test("mtg2-o2d.grib2" "--grid=eORCA025_T") +add_roundtrip_test("mtg2-t.grib2") +add_roundtrip_test("mtg2-t.grib2" "--grid=T20") +add_roundtrip_test("mtg2-vod.grib2") +add_roundtrip_test("mtg2-vod.grib2" "--grid=T20") diff --git a/tests/grib2grib/mtg2-o2d.grib2 b/tests/grib2grib/mtg2-o2d.grib2 new file mode 100644 index 000000000..b223288bb Binary files /dev/null and b/tests/grib2grib/mtg2-o2d.grib2 differ diff --git a/tests/grib2grib/mtg2-sfc.grib2 b/tests/grib2grib/mtg2-sfc.grib2 new file mode 100644 index 000000000..af4adb7c4 Binary files /dev/null and b/tests/grib2grib/mtg2-sfc.grib2 differ diff --git a/tests/grib2grib/mtg2-t.grib2 b/tests/grib2grib/mtg2-t.grib2 new file mode 100644 index 000000000..16e3bddef Binary files /dev/null and b/tests/grib2grib/mtg2-t.grib2 differ diff --git a/tests/grib2grib/mtg2-vod.grib2 b/tests/grib2grib/mtg2-vod.grib2 new file mode 100644 index 000000000..6171d6875 Binary files /dev/null and b/tests/grib2grib/mtg2-vod.grib2 differ diff --git a/tests/grib2grib/roundtrip.cc b/tests/grib2grib/roundtrip.cc new file mode 100644 index 000000000..5ea90252b --- /dev/null +++ b/tests/grib2grib/roundtrip.cc @@ -0,0 +1,214 @@ +/* + * (C) Copyright 2026- ECMWF and individual contributors. + * + * This software is licensed under the terms of the Apache Licence cVersion 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/log/Log.h" +#include "eckit/option/CmdArgs.h" +#include "eckit/option/SimpleOption.h" +#include "eckit/runtime/Main.h" +#include "eckit/runtime/Tool.h" + +#include "eccodes.h" +#include "metkit/grib2mars/api/Grib2Mars.h" +#include "metkit/mars2grib/api/Mars2Grib.h" + + +// Note: not in eccodes.h +extern "C" int codes_compare_key(codes_handle*, codes_handle*, const char* key, int compare_flags); + + +void get_vector_double(codes_handle* h, const std::string& k, std::vector& v) { + size_t size1 = 0; + ASSERT(CODES_SUCCESS == codes_get_size(h, k.c_str(), &size1)); + + v.resize(size1); + size_t size2 = size1; + + ASSERT(CODES_SUCCESS == codes_get_double_array(h, k.c_str(), v.data(), &size2)); + ASSERT(size2 == size1); +} + + +auto print_key_value = [](const codes_handle* h, const char* key, eckit::Channel& out) { + int type = CODES_TYPE_UNDEFINED; + ASSERT(codes_get_native_type(h, key, &type) == CODES_SUCCESS); + + out << "'" << key << "': "; + if (type == CODES_TYPE_UNDEFINED) { + out << "undefined" << std::endl; + } + else if (type == CODES_TYPE_LONG) { + long value = 0; + codes_get_long(h, key, &value); + if (value == CODES_MISSING_LONG) { + out << "MISSING" << std::endl; + } + else { + out << "long=" << value << std::endl; + } + } + else if (type == CODES_TYPE_DOUBLE) { + double value = 0; + codes_get_double(h, key, &value); + if (value == CODES_MISSING_DOUBLE) { + out << "MISSING" << std::endl; + } + else { + out << "double=" << value << std::endl; + } + } + else if (type == CODES_TYPE_STRING) { + std::string str(512, '\0'); + size_t size = str.size(); + codes_get_string(h, key, str.data(), &size); + str.resize(std::strlen(str.c_str())); // 'size' may include the trailing '\0' + out << "string='" << str << "'" << std::endl; + } + else if (type == CODES_TYPE_BYTES) { + std::vector bytes(512, '\0'); + size_t size = bytes.size(); + codes_get_bytes(h, key, bytes.data(), &size); + bytes.resize(size); + out << std::hex; + out << "bytes=["; + for (size_t i = 0; i < size; ++i) { + out << bytes[i]; + } + out << std::dec << "]" << std::endl; + } + else if (type == CODES_TYPE_SECTION) { + out << "section" << std::endl; + } + else if (type == CODES_TYPE_LABEL) { + out << "label" << std::endl; + } + else if (type == CODES_TYPE_MISSING) { + out << "missing" << std::endl; + } +}; + + +template +using Option = eckit::option::SimpleOption; + + +class Roundtrip : public eckit::Tool { + using Tool::Tool; + + static void usage(const std::string& name) { + eckit::Log::info() << name << " file [grid]" << std::endl; + exit(1); + } + + void run() override { + + std::vector options; + options.push_back(new Option("grid", "MARS grid")); + options.push_back(new Option("valid", "Check isMessageValid (deafult true)")); + options.push_back(new Option("cmp", "Check message bytes (cmp-like) (deafult false)")); + options.push_back( + new Option("keys", "Check grib key values (grib_compare-like, in-memory) (deafult false)")); + + eckit::option::CmdArgs args(usage, options, 1, 1); + ASSERT(args.count() == 1); + + auto* f = std::fopen(args(0).c_str(), "rb"); + ASSERT(f != nullptr); + + int err = 0; + auto* g = codes_handle_new_from_file(0, f, PRODUCT_GRIB, &err); + ASSERT(g != nullptr && err == 0); + + std::vector values; + get_vector_double(g, "values", values); + + + { + eckit::LocalConfiguration cfg; + cfg.set("skipSection3", true); + + metkit::grib2mars::Grib2Mars grib2mars(cfg); + metkit::mars2grib::Mars2Grib mars2grib(cfg); + + auto ch = metkit::codes::codesHandleFromGRIBHandle(g); + ASSERT(ch); + + auto [mars, misc] = grib2mars.convert(*ch); + + if (args.has("grid")) { + mars.remove("area"); + mars.remove("rotation"); + mars.set("grid", args.getString("grid")); + } + + const auto h = mars2grib.encode(values, mars, misc); + ASSERT(h); + + if (args.getBool("valid", true)) { + ASSERT(h->getLong("isMessageValid") != 0); + } + + if (args.getBool("cmp", false)) { + size_t size = 0; + ASSERT(CODES_SUCCESS == codes_get_message_size(g, &size)); + + eckit::Log::info() << "Original message size: " << size + << ", encoded message size: " << h->messageSize() << std::endl; + ASSERT(size == h->messageSize()); + + const void* input_message = nullptr; + ASSERT(CODES_SUCCESS == codes_get_message(g, &input_message, &size)); + + auto encoded = h->messageData(); + ASSERT(encoded.size() == size); + ASSERT(std::memcmp(input_message, encoded.data(), size) == 0); + } + + if (args.getBool("keys", false)) { + auto& out = eckit::Log::error(); + + // release() is the last use of h, so ownership transfer is safe + auto* h2 = reinterpret_cast(h->release()); + auto* kiter = codes_keys_iterator_new(g, CODES_KEYS_ITERATOR_SKIP_READ_ONLY, nullptr); + + auto diff = false; + while (codes_keys_iterator_next(kiter) != 0) { + const auto* key = codes_keys_iterator_get_name(kiter); + auto err = codes_compare_key(g, h2, key, 0); + if (err != CODES_SUCCESS && err != CODES_NOT_IMPLEMENTED /*key doesn't support comparison*/) { + out << "Key differs: '" << key << "' (" << codes_get_error_message(err) << ")" << std::endl; + print_key_value(g, key, out); + print_key_value(h2, key, out); + diff = true; + } + } + + codes_keys_iterator_delete(kiter); + codes_handle_delete(h2); + + ASSERT(!diff); + } + } + + codes_handle_delete(g); + ASSERT(std::fclose(f) == 0); + } +}; + + +int main(int argc, char** argv) { + return Roundtrip(argc, argv).start(); +} diff --git a/tests/mars2grib/CMakeLists.txt b/tests/mars2grib/CMakeLists.txt index 26072c994..b72793500 100644 --- a/tests/mars2grib/CMakeLists.txt +++ b/tests/mars2grib/CMakeLists.txt @@ -1,7 +1,3 @@ -if( HAVE_MARS2GRIB ) - add_subdirectory( api ) add_subdirectory( utils ) add_subdirectory( backend ) - -endif() diff --git a/tests/mars2mars/CMakeLists.txt b/tests/mars2mars/CMakeLists.txt index 1b168b7ec..4f9b9933a 100644 --- a/tests/mars2mars/CMakeLists.txt +++ b/tests/mars2mars/CMakeLists.txt @@ -1,5 +1,3 @@ -if( HAVE_MARS2MARS ) - configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/mars2mars-test-requests.mars ${CMAKE_CURRENT_BINARY_DIR}/mars2mars-test-requests.mars @@ -33,5 +31,3 @@ ecbuild_add_test( eckit metkit ) - -endif()