Add IRI support to URI Template expansion

src/core/uritemplate/helpers.h

@@ -89,11 +89,26 @@ inline auto append_percent_encoded(std::string &output, const char character)
   output += HEX_DIGITS[byte & 0x0F];
 }
 
-inline auto percent_encode(std::string &output, const std::string_view input)
-    -> void {
+// In the IRI mode, the bytes of internationalized characters pass through
+// rather than becoming percent encoded triplets. This is an extension beyond
+// RFC 6570, whose result always takes the URI syntax (Section 1.1: "Although
+// the URI syntax is used for the result, the template string is allowed to
+// contain the broader set of characters that can be found in
+// Internationalized Resource Identifier (IRI) references"). Nothing is ever
+// decoded, and whether the passed through codepoints are valid in an IRI is
+// left to the caller
+inline auto passes_through_unencoded(const char character,
+                                     const URITemplateExpansionMode mode)
+    -> bool {
+  return mode == URITemplateExpansionMode::IRI &&
+         static_cast<unsigned char>(character) >= 0x80;
+}
+
+inline auto percent_encode(std::string &output, const std::string_view input,
+                           const URITemplateExpansionMode mode) -> void {
   output.reserve(output.size() + input.size() * 3);
   for (const char character : input) {
-    if (is_unreserved(character)) {
+    if (is_unreserved(character) || passes_through_unencoded(character, mode)) {
       output += character;
     } else {
       append_percent_encoded(output, character);
@@ -102,11 +117,14 @@ inline auto percent_encode(std::string &output, const std::string_view input)
 }
 
 inline auto percent_encode_reserved(std::string &output,
-                                    const std::string_view input) -> void {
+                                    const std::string_view input,
+                                    const URITemplateExpansionMode mode)
+    -> void {
   output.reserve(output.size() + input.size() * 3);
   for (std::size_t index = 0; index < input.size(); ++index) {
     const char character = input[index];
     if (is_unreserved(character) || is_reserved(character) ||
+        passes_through_unencoded(character, mode) ||
         (character == '%' && index + 2 < input.size() &&
          is_hex_digit(input[index + 1]) && is_hex_digit(input[index + 2]))) {
       output += character;
@@ -117,11 +135,12 @@ inline auto percent_encode_reserved(std::string &output,
 }
 
 template <typename T>
-inline auto encode(std::string &output, const std::string_view input) -> void {
+inline auto encode(std::string &output, const std::string_view input,
+                   const URITemplateExpansionMode mode) -> void {
   if constexpr (T::allow_reserved) {
-    percent_encode_reserved(output, input);
+    percent_encode_reserved(output, input, mode);
   } else {
-    percent_encode(output, input);
+    percent_encode(output, input, mode);
   }
 }
 
@@ -399,7 +418,8 @@ template <typename T>
 auto expand_expression(
     std::string &result,
     const std::vector<URITemplateVariableSpecification> &variables,
-    const std::function<URITemplateValue(std::string_view)> &callback) -> void {
+    const std::function<URITemplateValue(std::string_view)> &callback,
+    const URITemplateExpansionMode mode) -> void {
   bool first_var = true;
 
   for (const auto &variable : variables) {
@@ -435,14 +455,14 @@ auto expand_expression(
         }
 
         if (object_key.has_value()) {
-          encode<T>(result, object_key.value());
+          encode<T>(result, object_key.value(), mode);
           if (actual_value.empty()) {
             if constexpr (has_empty_suffix<T>::value) {
               result += T::empty_suffix;
             }
           } else {
             result += '=';
-            encode<T>(result, actual_value);
+            encode<T>(result, actual_value, mode);
           }
         } else if constexpr (T::named) {
           result += variable.name;
@@ -452,10 +472,10 @@ auto expand_expression(
             }
           } else {
             result += '=';
-            encode<T>(result, actual_value);
+            encode<T>(result, actual_value, mode);
           }
         } else {
-          encode<T>(result, actual_value);
+          encode<T>(result, actual_value, mode);
         }
       } else {
         // An associative-array pair always contributes its key, so it is never
@@ -476,10 +496,10 @@ auto expand_expression(
 
         if (!first_value || !value_empty || has_more) {
           if (object_key.has_value()) {
-            encode<T>(result, object_key.value());
+            encode<T>(result, object_key.value(), mode);
             result += ',';
           }
-          encode<T>(result, actual_value);
+          encode<T>(result, actual_value, mode);
         }
       }
 

src/core/uritemplate/include/sourcemeta/core/uritemplate.h

@@ -37,6 +37,22 @@ namespace sourcemeta::core {
 using URITemplateValue = std::optional<
     std::tuple<std::string_view, std::optional<std::string_view>, bool>>;
 
+/// @ingroup uritemplate
+/// The alphabet that expansion encodes variable values against
+enum class URITemplateExpansionMode : std::uint8_t {
+  /// Encode against the URI alphabet as defined by RFC 6570
+  URI,
+  /// Encode against the IRI alphabet, passing internationalized characters
+  /// through unencoded rather than percent encoding them. RFC 6570 Section
+  /// 1.1 acknowledges that "a URI Template is also an IRI template", but
+  /// only by expanding to a URI and converting the result per RFC 3987
+  /// Section 3.2, a conversion that also decodes percent encoded triplets
+  /// already present in variable values, whereas this mode never decodes
+  /// anything. Passed through characters are not validated against the
+  /// RFC 3987 grammar, which is left to the caller
+  IRI
+};
+
 /// @ingroup uritemplate
 /// The result of parsing a token: the token and how many characters were
 /// consumed
@@ -95,14 +111,18 @@ class SOURCEMETA_CORE_URITEMPLATE_EXPORT URITemplate {
 
   /// Expand the template by looking up variable values via a callback.
   /// The callback is called repeatedly for composite values
-  [[nodiscard]] auto expand(
-      const std::function<URITemplateValue(std::string_view)> &callback) const
+  [[nodiscard]] auto
+  expand(const std::function<URITemplateValue(std::string_view)> &callback,
+         URITemplateExpansionMode mode = URITemplateExpansionMode::URI) const
       -> std::string;
 
   /// Expand the template using an associative container (string values only)
   template <typename Container,
             typename = std::void_t<typename Container::key_type>>
-  [[nodiscard]] auto expand(const Container &variables) const -> std::string {
+  [[nodiscard]] auto expand(
+      const Container &variables,
+      const URITemplateExpansionMode mode = URITemplateExpansionMode::URI) const
+      -> std::string {
     return this->expand(
         [&variables](const std::string_view name) -> URITemplateValue {
           const auto iterator{find_variable(variables, name)};
@@ -112,7 +132,8 @@ class SOURCEMETA_CORE_URITEMPLATE_EXPORT URITemplate {
             return std::make_tuple(std::string_view{iterator->second},
                                    std::nullopt, false);
           }
-        });
+        },
+        mode);
   }
 
 private:

src/core/uritemplate/uritemplate.cc

@@ -109,18 +109,18 @@ auto URITemplate::end() const noexcept
 }
 
 auto URITemplate::expand(
-    const std::function<URITemplateValue(std::string_view name)> &callback)
-    const -> std::string {
+    const std::function<URITemplateValue(std::string_view name)> &callback,
+    const URITemplateExpansionMode mode) const -> std::string {
   std::string result;
 
   for (const auto &token : this->tokens_) {
     std::visit(
-        [&result, &callback](const auto &expansion) -> void {
+        [&result, &callback, mode](const auto &expansion) -> void {
           using T = std::decay_t<decltype(expansion)>;
           if constexpr (std::is_same_v<T, URITemplateTokenLiteral>) {
             result += expansion.value;
           } else {
-            expand_expression<T>(result, expansion.variables, callback);
+            expand_expression<T>(result, expansion.variables, callback, mode);
           }
         },
         token);

test/uritemplate/CMakeLists.txt

@@ -4,7 +4,8 @@ sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME uritemplate
     uritemplate_test.cc
     uritemplate_parse_test.cc
     uritemplate_parse_error_test.cc
-    uritemplate_expand_test.cc
+    uritemplate_expand_uri_test.cc
+    uritemplate_expand_iri_test.cc
     uritemplate_router_test.cc
     uritemplate_router_view_test.cc)