diff --git a/src/core/uritemplate/helpers.h b/src/core/uritemplate/helpers.h index 0d21976d6..74c3adc31 100644 --- a/src/core/uritemplate/helpers.h +++ b/src/core/uritemplate/helpers.h @@ -89,11 +89,26 @@ inline auto append_percent_encoded(std::string &output, const char character) output += HEX_DIGITS[byte & 0x0F]; } -inline auto percent_encode(std::string &output, const std::string_view input) - -> void { +// In the IRI mode, the bytes of internationalized characters pass through +// rather than becoming percent encoded triplets. This is an extension beyond +// RFC 6570, whose result always takes the URI syntax (Section 1.1: "Although +// the URI syntax is used for the result, the template string is allowed to +// contain the broader set of characters that can be found in +// Internationalized Resource Identifier (IRI) references"). Nothing is ever +// decoded, and whether the passed through codepoints are valid in an IRI is +// left to the caller +inline auto passes_through_unencoded(const char character, + const URITemplateExpansionMode mode) + -> bool { + return mode == URITemplateExpansionMode::IRI && + static_cast(character) >= 0x80; +} + +inline auto percent_encode(std::string &output, const std::string_view input, + const URITemplateExpansionMode mode) -> void { output.reserve(output.size() + input.size() * 3); for (const char character : input) { - if (is_unreserved(character)) { + if (is_unreserved(character) || passes_through_unencoded(character, mode)) { output += character; } else { append_percent_encoded(output, character); @@ -102,11 +117,14 @@ inline auto percent_encode(std::string &output, const std::string_view input) } inline auto percent_encode_reserved(std::string &output, - const std::string_view input) -> void { + const std::string_view input, + const URITemplateExpansionMode mode) + -> void { output.reserve(output.size() + input.size() * 3); for (std::size_t index = 0; index < input.size(); ++index) { const char character = input[index]; if (is_unreserved(character) || is_reserved(character) || + passes_through_unencoded(character, mode) || (character == '%' && index + 2 < input.size() && is_hex_digit(input[index + 1]) && is_hex_digit(input[index + 2]))) { output += character; @@ -117,11 +135,12 @@ inline auto percent_encode_reserved(std::string &output, } template -inline auto encode(std::string &output, const std::string_view input) -> void { +inline auto encode(std::string &output, const std::string_view input, + const URITemplateExpansionMode mode) -> void { if constexpr (T::allow_reserved) { - percent_encode_reserved(output, input); + percent_encode_reserved(output, input, mode); } else { - percent_encode(output, input); + percent_encode(output, input, mode); } } @@ -399,7 +418,8 @@ template auto expand_expression( std::string &result, const std::vector &variables, - const std::function &callback) -> void { + const std::function &callback, + const URITemplateExpansionMode mode) -> void { bool first_var = true; for (const auto &variable : variables) { @@ -435,14 +455,14 @@ auto expand_expression( } if (object_key.has_value()) { - encode(result, object_key.value()); + encode(result, object_key.value(), mode); if (actual_value.empty()) { if constexpr (has_empty_suffix::value) { result += T::empty_suffix; } } else { result += '='; - encode(result, actual_value); + encode(result, actual_value, mode); } } else if constexpr (T::named) { result += variable.name; @@ -452,10 +472,10 @@ auto expand_expression( } } else { result += '='; - encode(result, actual_value); + encode(result, actual_value, mode); } } else { - encode(result, actual_value); + encode(result, actual_value, mode); } } else { // An associative-array pair always contributes its key, so it is never @@ -476,10 +496,10 @@ auto expand_expression( if (!first_value || !value_empty || has_more) { if (object_key.has_value()) { - encode(result, object_key.value()); + encode(result, object_key.value(), mode); result += ','; } - encode(result, actual_value); + encode(result, actual_value, mode); } } diff --git a/src/core/uritemplate/include/sourcemeta/core/uritemplate.h b/src/core/uritemplate/include/sourcemeta/core/uritemplate.h index 29f5a49ff..a0843e908 100644 --- a/src/core/uritemplate/include/sourcemeta/core/uritemplate.h +++ b/src/core/uritemplate/include/sourcemeta/core/uritemplate.h @@ -37,6 +37,22 @@ namespace sourcemeta::core { using URITemplateValue = std::optional< std::tuple, bool>>; +/// @ingroup uritemplate +/// The alphabet that expansion encodes variable values against +enum class URITemplateExpansionMode : std::uint8_t { + /// Encode against the URI alphabet as defined by RFC 6570 + URI, + /// Encode against the IRI alphabet, passing internationalized characters + /// through unencoded rather than percent encoding them. RFC 6570 Section + /// 1.1 acknowledges that "a URI Template is also an IRI template", but + /// only by expanding to a URI and converting the result per RFC 3987 + /// Section 3.2, a conversion that also decodes percent encoded triplets + /// already present in variable values, whereas this mode never decodes + /// anything. Passed through characters are not validated against the + /// RFC 3987 grammar, which is left to the caller + IRI +}; + /// @ingroup uritemplate /// The result of parsing a token: the token and how many characters were /// consumed @@ -95,14 +111,18 @@ class SOURCEMETA_CORE_URITEMPLATE_EXPORT URITemplate { /// Expand the template by looking up variable values via a callback. /// The callback is called repeatedly for composite values - [[nodiscard]] auto expand( - const std::function &callback) const + [[nodiscard]] auto + expand(const std::function &callback, + URITemplateExpansionMode mode = URITemplateExpansionMode::URI) const -> std::string; /// Expand the template using an associative container (string values only) template > - [[nodiscard]] auto expand(const Container &variables) const -> std::string { + [[nodiscard]] auto expand( + const Container &variables, + const URITemplateExpansionMode mode = URITemplateExpansionMode::URI) const + -> std::string { return this->expand( [&variables](const std::string_view name) -> URITemplateValue { const auto iterator{find_variable(variables, name)}; @@ -112,7 +132,8 @@ class SOURCEMETA_CORE_URITEMPLATE_EXPORT URITemplate { return std::make_tuple(std::string_view{iterator->second}, std::nullopt, false); } - }); + }, + mode); } private: diff --git a/src/core/uritemplate/uritemplate.cc b/src/core/uritemplate/uritemplate.cc index 1605e5f27..17bf4163e 100644 --- a/src/core/uritemplate/uritemplate.cc +++ b/src/core/uritemplate/uritemplate.cc @@ -109,18 +109,18 @@ auto URITemplate::end() const noexcept } auto URITemplate::expand( - const std::function &callback) - const -> std::string { + const std::function &callback, + const URITemplateExpansionMode mode) const -> std::string { std::string result; for (const auto &token : this->tokens_) { std::visit( - [&result, &callback](const auto &expansion) -> void { + [&result, &callback, mode](const auto &expansion) -> void { using T = std::decay_t; if constexpr (std::is_same_v) { result += expansion.value; } else { - expand_expression(result, expansion.variables, callback); + expand_expression(result, expansion.variables, callback, mode); } }, token); diff --git a/test/uritemplate/CMakeLists.txt b/test/uritemplate/CMakeLists.txt index 2d3b377c2..4efdb6cf4 100644 --- a/test/uritemplate/CMakeLists.txt +++ b/test/uritemplate/CMakeLists.txt @@ -4,7 +4,8 @@ sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME uritemplate uritemplate_test.cc uritemplate_parse_test.cc uritemplate_parse_error_test.cc - uritemplate_expand_test.cc + uritemplate_expand_uri_test.cc + uritemplate_expand_iri_test.cc uritemplate_router_test.cc uritemplate_router_view_test.cc) diff --git a/test/uritemplate/uritemplate_expand_iri_test.cc b/test/uritemplate/uritemplate_expand_iri_test.cc new file mode 100644 index 000000000..fe0f9e16f --- /dev/null +++ b/test/uritemplate/uritemplate_expand_iri_test.cc @@ -0,0 +1,426 @@ +#include +#include + +#include // std::size_t +#include // std::map +#include // std::optional, std::nullopt +#include // std::string +#include // std::string_view +#include // std::make_tuple +#include // std::unordered_map + +static auto single_value(const std::string_view value) { + return + [value]( + const std::string_view name) -> sourcemeta::core::URITemplateValue { + if (name == "var") { + return std::make_tuple(value, std::nullopt, false); + } else { + return std::nullopt; + } + }; +} + +TEST(simple_two_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "café"); +} + +TEST(simple_two_byte_character_default_mode_still_encodes) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand(single_value("café")); + + EXPECT_EQ(result, "caf%C3%A9"); +} + +TEST(simple_two_byte_character_explicit_uri_mode_matches_default) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::URI); + + EXPECT_EQ(result, "caf%C3%A9"); +} + +TEST(reserved_full_identifier_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = + uri_template.expand(single_value("https://example.com/café"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "https://example.com/café"); +} + +TEST(reserved_full_identifier_default_mode_still_encodes) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = + uri_template.expand(single_value("https://example.com/café")); + + EXPECT_EQ(result, "https://example.com/caf%C3%A9"); +} + +TEST(fragment_two_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{#var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "#café"); +} + +TEST(label_two_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{.var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, ".café"); +} + +TEST(path_two_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{/var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "/café"); +} + +TEST(path_style_two_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{;var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, ";var=café"); +} + +TEST(query_two_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{?var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "?var=café"); +} + +TEST(query_continuation_two_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{&var}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "&var=café"); +} + +TEST(simple_three_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand( + single_value("a→b"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "a→b"); +} + +TEST(simple_four_byte_character_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand( + single_value("😀"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "😀"); +} + +TEST(simple_han_characters_pass_through) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand( + single_value("東京"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "東京"); +} + +TEST(reserved_mixed_widths_pass_through) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = + uri_template.expand(single_value("café/→/東京/😀"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "café/→/東京/😀"); +} + +TEST(ascii_only_value_matches_uri_mode_simple) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto uri_result = uri_template.expand(single_value("plain-value")); + const auto iri_result = + uri_template.expand(single_value("plain-value"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(uri_result, "plain-value"); + EXPECT_EQ(iri_result, "plain-value"); +} + +TEST(ascii_only_value_matches_uri_mode_reserved) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto uri_result = + uri_template.expand(single_value("https://example.com/a?b=c#d")); + const auto iri_result = + uri_template.expand(single_value("https://example.com/a?b=c#d"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(uri_result, "https://example.com/a?b=c#d"); + EXPECT_EQ(iri_result, "https://example.com/a?b=c#d"); +} + +TEST(reserved_existing_triplets_are_not_decoded) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = + uri_template.expand(single_value("caf%C3%A9"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "caf%C3%A9"); +} + +TEST(reserved_mixed_raw_and_encoded_keeps_both_spellings) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = + uri_template.expand(single_value("café%20menu"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "café%20menu"); +} + +TEST(simple_still_encodes_percent_sign) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = + uri_template.expand(single_value("caf%C3%A9"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "caf%25C3%25A9"); +} + +TEST(simple_still_encodes_space) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand( + single_value("a b"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "a%20b"); +} + +TEST(reserved_still_encodes_space) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = uri_template.expand( + single_value("a b"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "a%20b"); +} + +TEST(reserved_still_encodes_control_character) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = uri_template.expand( + single_value("a\nb"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "a%0Ab"); +} + +TEST(reserved_still_encodes_angle_brackets_and_quote) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = + uri_template.expand(single_value("\"b\""), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "%3Ca%3E%22b%22"); +} + +TEST(simple_still_encodes_reserved_characters) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + const auto result = uri_template.expand( + single_value("a/b?c#d"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "a%2Fb%3Fc%23d"); +} + +TEST(reserved_still_passes_reserved_characters) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const auto result = uri_template.expand( + single_value("a/b?c#d"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "a/b?c#d"); +} + +TEST(literal_and_variable_both_stay_raw) { + const sourcemeta::core::URITemplate uri_template{ + "https://example.com/café/{var}"}; + const auto result = uri_template.expand( + single_value("menü"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "https://example.com/café/menü"); +} + +TEST(literal_stays_raw_while_default_mode_encodes_the_variable) { + const sourcemeta::core::URITemplate uri_template{ + "https://example.com/café/{var}"}; + const auto result = uri_template.expand(single_value("menü")); + + EXPECT_EQ(result, "https://example.com/café/men%C3%BC"); +} + +TEST(prefix_modifier_counts_characters_and_passes_them_through) { + const sourcemeta::core::URITemplate uri_template{"{var:3}"}; + const auto result = uri_template.expand( + single_value("café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "caf"); +} + +TEST(prefix_modifier_covering_multi_byte_character_passes_it_through) { + const sourcemeta::core::URITemplate uri_template{"{var:4}"}; + const auto result = uri_template.expand( + single_value("café!"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "café"); +} + +TEST(multiple_variables_mix_raw_and_encoded_rules) { + const sourcemeta::core::URITemplate uri_template{"{x,y}"}; + const auto result = uri_template.expand( + [](const std::string_view name) -> sourcemeta::core::URITemplateValue { + if (name == "x") { + return std::make_tuple(std::string_view{"café"}, std::nullopt, false); + } else if (name == "y") { + return std::make_tuple(std::string_view{"a b"}, std::nullopt, false); + } else { + return std::nullopt; + } + }, + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "café,a%20b"); +} + +TEST(list_values_pass_through) { + const sourcemeta::core::URITemplate uri_template{"{var}"}; + std::size_t call_count = 0; + const auto result = uri_template.expand( + [&call_count]( + const std::string_view name) -> sourcemeta::core::URITemplateValue { + if (name == "var") { + call_count++; + if (call_count == 1) { + return std::make_tuple(std::string_view{"café"}, std::nullopt, + true); + } else { + return std::make_tuple(std::string_view{"menü"}, std::nullopt, + false); + } + } + return std::nullopt; + }, + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "café,menü"); +} + +TEST(list_explode_path_values_pass_through) { + const sourcemeta::core::URITemplate uri_template{"{/var*}"}; + std::size_t call_count = 0; + const auto result = uri_template.expand( + [&call_count]( + const std::string_view name) -> sourcemeta::core::URITemplateValue { + if (name == "var") { + call_count++; + if (call_count == 1) { + return std::make_tuple(std::string_view{"café"}, std::nullopt, + true); + } else { + return std::make_tuple(std::string_view{"東京"}, std::nullopt, + false); + } + } + return std::nullopt; + }, + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "/café/東京"); +} + +TEST(object_explode_keys_and_values_pass_through) { + const sourcemeta::core::URITemplate uri_template{"{?var*}"}; + std::size_t call_count = 0; + const auto result = uri_template.expand( + [&call_count]( + const std::string_view name) -> sourcemeta::core::URITemplateValue { + if (name == "var") { + call_count++; + if (call_count == 1) { + return std::make_tuple(std::string_view{"café"}, + std::optional{"clé"}, + true); + } else { + return std::make_tuple(std::string_view{"menü"}, + std::optional{"öl"}, + false); + } + } + return std::nullopt; + }, + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "?clé=café&öl=menü"); +} + +TEST(map_container_overload_passes_through) { + const sourcemeta::core::URITemplate uri_template{"/results/{var}"}; + const std::map variables{{"var", "café"}}; + const auto result = uri_template.expand( + variables, sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "/results/café"); +} + +TEST(map_container_overload_default_mode_still_encodes) { + const sourcemeta::core::URITemplate uri_template{"/results/{var}"}; + const std::map variables{{"var", "café"}}; + const auto result = uri_template.expand(variables); + + EXPECT_EQ(result, "/results/caf%C3%A9"); +} + +TEST(unordered_map_container_overload_passes_through) { + const sourcemeta::core::URITemplate uri_template{"{+var}"}; + const std::unordered_map variables{ + {"var", "https://example.com/café"}}; + const auto result = uri_template.expand( + variables, sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "https://example.com/café"); +} + +TEST(fragment_existing_triplets_are_not_decoded) { + const sourcemeta::core::URITemplate uri_template{"{#var}"}; + const auto result = + uri_template.expand(single_value("caf%C3%A9"), + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "#caf%C3%A9"); +} + +TEST(query_still_encodes_ampersand_in_value) { + const sourcemeta::core::URITemplate uri_template{"{?var}"}; + const auto result = uri_template.expand( + single_value("a&café"), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "?var=a%26café"); +} + +TEST(missing_variable_expands_to_nothing) { + const sourcemeta::core::URITemplate uri_template{"/a/{var}"}; + const auto result = uri_template.expand( + [](const std::string_view) -> sourcemeta::core::URITemplateValue { + return std::nullopt; + }, + sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "/a/"); +} + +TEST(empty_value_expands_to_nothing) { + const sourcemeta::core::URITemplate uri_template{"/a/{var}"}; + const auto result = uri_template.expand( + single_value(""), sourcemeta::core::URITemplateExpansionMode::IRI); + + EXPECT_EQ(result, "/a/"); +} diff --git a/test/uritemplate/uritemplate_expand_test.cc b/test/uritemplate/uritemplate_expand_uri_test.cc similarity index 100% rename from test/uritemplate/uritemplate_expand_test.cc rename to test/uritemplate/uritemplate_expand_uri_test.cc