From 2f7fe60ed91214b06718648bcbcd26643a9cb691 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 21:27:52 +0000 Subject: [PATCH 01/19] refactor(ast): Share borrowed parser across constructors --- src/ast/container.rs | 11 +++++++---- src/convert.rs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ast/container.rs b/src/ast/container.rs index ca4271a..cca5ac9 100644 --- a/src/ast/container.rs +++ b/src/ast/container.rs @@ -22,7 +22,7 @@ impl Ast { /// ``` #[must_use] pub fn from_owned(value: &(impl AsRef + ?Sized)) -> Self { - Self::from_value(value.as_ref()) + Self::parse(value.as_ref()) } /// Build an AST tree from Gemtext @@ -35,7 +35,7 @@ impl Ast { #[must_use] #[allow(clippy::needless_pass_by_value)] pub fn from_string(value: impl Into) -> Self { - Self::from_value(&value.into()) + Self::parse(&value.into()) } /// Build an AST tree from a value @@ -47,10 +47,13 @@ impl Ast { /// ``` #[must_use] pub fn from_value(value: &(impl ToString + ?Sized)) -> Self { + Self::parse(&value.to_string()) + } + + fn parse(source: &str) -> Self { let mut ast = vec![]; let mut in_preformatted = false; let mut in_list = false; - let source = value.to_string(); let mut lines = source.lines(); // Iterate over all lines in the Gemtext `source` @@ -85,7 +88,7 @@ impl Ast { /// // the original Gemtext. /// assert_eq!( /// germ::ast::Ast::from_nodes( - /// germ::gemini_to_ast!("=> / Home").inner().to_vec() + /// germ::ast::Ast::from_string("=> / Home").inner().to_vec() /// ) /// .to_gemtext(), /// "=> / Home" diff --git a/src/convert.rs b/src/convert.rs index c2a94d9..5e0574f 100644 --- a/src/convert.rs +++ b/src/convert.rs @@ -53,5 +53,5 @@ pub fn from_string( source: &(impl ToString + ?Sized), target: &Target, ) -> String { - from_ast(&Ast::from_owned(&source.to_string()), target) + from_ast(&Ast::from_value(source), target) } From 113b8e50493494eb3d909552a5ca77a1ea1d7573 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 21:28:05 +0000 Subject: [PATCH 02/19] chore(examples): Stop tracking generated conversions --- examples/convert.html | 6 ------ examples/convert.md | 28 ---------------------------- examples/html.rs | 5 +---- examples/markdown.rs | 6 ++---- 4 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 examples/convert.html delete mode 100644 examples/convert.md diff --git a/examples/convert.html b/examples/convert.html deleted file mode 100644 index cdfde66..0000000 --- a/examples/convert.html +++ /dev/null @@ -1,6 +0,0 @@ -
Here goes the pre-formatted text.
-
-This continues the pre-formatted text on a new line after a blank line.
-

This is a heading

This is some text.

This is more text after a blank line.

  • This is a single list item.
    • -
  • This is the next list item.
  • This is a new list.
    • -
  • This is the next item on the new list.

This is a sub-heading

This is a blockquote.

This is a sub-sub-heading.

This is a link to GemRest
/somewhere

That was a link without text.

\ No newline at end of file diff --git a/examples/convert.md b/examples/convert.md deleted file mode 100644 index ea553ee..0000000 --- a/examples/convert.md +++ /dev/null @@ -1,28 +0,0 @@ -```This is alt-text -Here goes the pre-formatted text. - -This continues the pre-formatted text on a new line after a blank line. -``` - -# This is a heading - -This is some text. - -This is more text after a blank line. - -- This is a single list item. -- This is the next list item. - -- This is a new list. -- This is the next item on the new list. - -## This is a sub-heading - -> This is a blockquote. - -### This is a sub-sub-heading. - -[This is a link to GemRest](gemini://gem.rest/) - - -That was a link without text. diff --git a/examples/html.rs b/examples/html.rs index 802bd8e..1532d20 100644 --- a/examples/html.rs +++ b/examples/html.rs @@ -2,13 +2,10 @@ //! HTML. fn main() { - // Convert the Gemtext to HTML let html = germ::convert::from_string( germ::EXAMPLE_GEMTEXT, &germ::convert::Target::HTML, ); - // Write the HTML to a file - std::fs::write("examples/convert.html", html) - .expect("could not write to file"); + println!("{html}"); } diff --git a/examples/markdown.rs b/examples/markdown.rs index 64cfcd4..47c1bdd 100644 --- a/examples/markdown.rs +++ b/examples/markdown.rs @@ -2,12 +2,10 @@ //! Markdown. fn main() { - // Convert the Gemtext to Markdown - let html = germ::convert::from_string( + let markdown = germ::convert::from_string( germ::EXAMPLE_GEMTEXT, &germ::convert::Target::Markdown, ); - // Write the Markdown to a file - std::fs::write("examples/convert.md", html).expect("could not write to file"); + println!("{markdown}"); } From 28da9515aa09594138580fb48ebd320487e3847a Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 21:28:22 +0000 Subject: [PATCH 03/19] fix(features): Gate optional modules, tests and examples --- Cargo.toml | 32 ++++++++++++++++++++++++++++++++ src/lib.rs | 2 +- tests/ast.rs | 12 +++++++----- tests/convert.rs | 10 ++++++---- tests/meta.rs | 2 ++ tests/node.rs | 2 ++ tests/quick.rs | 2 ++ tests/status.rs | 2 ++ 8 files changed, 54 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 85775e5..d6d226c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,38 @@ request = ["rustls", "url", "anyhow", "tokio", "tokio-rustls"] quick = [] example-gemtext = [] +[[example]] +name = "ast" +required-features = ["ast", "example-gemtext"] + +[[example]] +name = "ast_to_gemtext" +required-features = ["ast", "example-gemtext"] + +[[example]] +name = "html" +required-features = ["convert", "example-gemtext"] + +[[example]] +name = "markdown" +required-features = ["convert", "example-gemtext"] + +[[example]] +name = "meta" +required-features = ["meta"] + +[[example]] +name = "request" +required-features = ["request"] + +[[example]] +name = "request_blocking" +required-features = ["blocking"] + +[[example]] +name = "request_blocking_to_gemtext_from_ast" +required-features = ["ast", "blocking"] + [dependencies] anyhow = { version = "1.0.98", optional = true } # `Result` rustls = { version = "0.21.12", features = [ diff --git a/src/lib.rs b/src/lib.rs index 5503a7d..ec5b146 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,7 @@ #[cfg(feature = "convert")] pub mod convert; -#[cfg(feature = "request")] pub mod request; +#[cfg(any(feature = "request", feature = "blocking"))] pub mod request; #[cfg(feature = "meta")] pub mod meta; diff --git a/tests/ast.rs b/tests/ast.rs index 317272b..9c7abbf 100644 --- a/tests/ast.rs +++ b/tests/ast.rs @@ -1,9 +1,9 @@ +#![cfg(feature = "ast")] + #[cfg(test)] mod test { - use germ::{ - EXAMPLE_GEMTEXT, - ast::{Ast, Node}, - }; + #[cfg(feature = "example-gemtext")] use germ::EXAMPLE_GEMTEXT; + use germ::ast::{Ast, Node}; #[test] fn build_multi_line_list_with_text() { @@ -38,6 +38,7 @@ mod test { ); } + #[cfg(feature = "example-gemtext")] #[test] fn gemtext_to_ast_then_ast_to_gemtext() { assert_eq!( @@ -48,6 +49,7 @@ mod test { ); } + #[cfg(all(feature = "example-gemtext", feature = "macros"))] #[test] fn gemtext_to_ast_then_ast_to_gemtext_macro_expression() { assert_eq!( @@ -61,7 +63,7 @@ mod test { #[test] fn gemtext_to_ast_then_node_to_ast_to_gemtext() { assert_eq!( - Ast::from_nodes(germ::gemini_to_ast!("=> / Home").inner().to_vec()) + Ast::from_nodes(Ast::from_string("=> / Home").inner().to_vec()) .to_gemtext(), "=> / Home" ); diff --git a/tests/convert.rs b/tests/convert.rs index 871f590..0e47a8d 100644 --- a/tests/convert.rs +++ b/tests/convert.rs @@ -1,9 +1,9 @@ +#![cfg(feature = "convert")] + #[cfg(test)] mod test { - use germ::{ - convert::{Target, from_string}, - gemini_to_html, gemini_to_md, - }; + use germ::convert::{Target, from_string}; + #[cfg(feature = "macros")] use germ::{gemini_to_html, gemini_to_md}; #[test] fn convert_from_string_to_html_single_line() { @@ -15,6 +15,7 @@ mod test { assert_eq!(from_string("hi\n# hi", &Target::HTML), "

hi

hi

",); } + #[cfg(feature = "macros")] #[test] fn convert_from_string_to_html_single_link_macro_expression() { assert_eq!( @@ -41,6 +42,7 @@ mod test { ); } + #[cfg(feature = "macros")] #[test] fn convert_from_string_to_markdown_single_macro_expression() { assert_eq!(gemini_to_md!("=> /to hello !"), "[hello !](/to)\n",); diff --git a/tests/meta.rs b/tests/meta.rs index 13db158..842fc3e 100644 --- a/tests/meta.rs +++ b/tests/meta.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "meta")] + #[cfg(test)] mod test { use germ::meta::Meta; diff --git a/tests/node.rs b/tests/node.rs index 0faf8c3..5931804 100644 --- a/tests/node.rs +++ b/tests/node.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "ast")] + #[cfg(test)] mod test { #[test] diff --git a/tests/quick.rs b/tests/quick.rs index b52ea17..e3d5055 100644 --- a/tests/quick.rs +++ b/tests/quick.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "quick")] + #[cfg(test)] mod test { use germ::quick::{self, heading}; diff --git a/tests/status.rs b/tests/status.rs index ff9f25f..82b78bb 100644 --- a/tests/status.rs +++ b/tests/status.rs @@ -1,3 +1,5 @@ +#![cfg(any(feature = "request", feature = "blocking"))] + #[cfg(test)] mod test { use germ::request::Status; From 54ee4e0b94fcb43423623f62aed1473984365af6 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 21:40:03 +0000 Subject: [PATCH 04/19] refactor(status): Derive default status --- src/request/status.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/request/status.rs b/src/request/status.rs index 94600bf..86dfb85 100644 --- a/src/request/status.rs +++ b/src/request/status.rs @@ -10,10 +10,11 @@ use std::{fmt, fmt::Formatter}; /// assert_eq!(Status::from(10), Status::Input); /// assert_eq!(i32::from(Status::Input), 10); /// ``` -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] pub enum Status { Input, SensitiveInput, + #[default] Success, TemporaryRedirect, PermanentRedirect, @@ -33,10 +34,6 @@ pub enum Status { Unsupported, } -impl Default for Status { - fn default() -> Self { Self::Success } -} - impl From for i32 { fn from(n: Status) -> Self { match n { From 3e23d4ec2015e86d22dc804690cc5a84df9c1377 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 21:40:11 +0000 Subject: [PATCH 05/19] refactor(examples): Simplify request result handling --- examples/request.rs | 36 +++++++++--------------------------- examples/request_blocking.rs | 36 +++++++++--------------------------- 2 files changed, 18 insertions(+), 54 deletions(-) diff --git a/examples/request.rs b/examples/request.rs index 8979fcd..9958f82 100644 --- a/examples/request.rs +++ b/examples/request.rs @@ -2,33 +2,15 @@ //! request to a Gemini capsule. #[tokio::main] -async fn main() { - // Form a valid URL to a Gemini capsule - let url = url::Url::parse("gemini://fuwn.me").unwrap(); - // Perform a non-blocking request to the Gemini capsule - let request = germ::request::request(&url).await; +async fn main() -> anyhow::Result<()> { + let url = url::Url::parse("gemini://fuwn.me")?; + let response = germ::request::request(&url).await?; - match request { - // If the request was successful, print a debug view of the response - Ok(response) => { - // Print the status of the response - println!("{:?}", response.status()); + println!("{:?}", response.status()); + println!("{}", response.meta()); + println!("{:?}", response.content()); + println!("{:?}", response.size()); + println!("{:?}", response.suite()); - // Print the meta string of the response - // - // More detailed meta usage can be found in the `meta` example - println!("{}", response.meta()); - - // Print the content of the response, if present - println!("{:?}", response.content()); - - // Print the size of the response - println!("{:?}", response.size()); - - // Print a debug view of the SSL suite used - println!("{:?}", response.suite()); - } - // If the request was unsuccessful, do nothing - Err(_) => {} - } + Ok(()) } diff --git a/examples/request_blocking.rs b/examples/request_blocking.rs index 72cedd3..a4dd5fe 100644 --- a/examples/request_blocking.rs +++ b/examples/request_blocking.rs @@ -1,33 +1,15 @@ //! This example demonstrates Germ's capabilities for performing a blocking //! request to a Gemini capsule. -fn main() { - // Form a valid URL to a Gemini capsule - let url = url::Url::parse("gemini://fuwn.me").unwrap(); - // Perform a blocking request to the Gemini capsule - let request = germ::request::blocking::request(&url); +fn main() -> anyhow::Result<()> { + let url = url::Url::parse("gemini://fuwn.me")?; + let response = germ::request::blocking::request(&url)?; - match request { - // If the request was successful, print a debug view of the response - Ok(response) => { - // Print the status of the response - println!("{:?}", response.status()); + println!("{:?}", response.status()); + println!("{}", response.meta()); + println!("{:?}", response.content()); + println!("{:?}", response.size()); + println!("{:?}", response.suite()); - // Print the meta string of the response - // - // More detailed meta usage can be found in the `meta` example - println!("{}", response.meta()); - - // Print the content of the response, if present - println!("{:?}", response.content()); - - // Print the size of the response - println!("{:?}", response.size()); - - // Print a debug view of the SSL suite used - println!("{:?}", response.suite()); - } - // If the request was unsuccessful, do nothing - Err(_) => {} - } + Ok(()) } From cc83595e2898c5e8ff9e43bc6ba2cc060e671f11 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 21:40:25 +0000 Subject: [PATCH 06/19] ci: Run tests and Clippy across feature sets --- .github/workflows/check.yaml | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 257dfc4..8eb1e20 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -2,11 +2,7 @@ name: Check โœ… on: workflow_dispatch: push: - paths: - - "*" pull_request: - paths: - - "*" env: CARGO_TERM_COLOR: always jobs: @@ -14,17 +10,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout ๐Ÿ›’ - uses: actions/checkout@v3 - - name: Toolchain ๐Ÿงฐ - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - components: rustfmt, clippy - override: true - - name: Check โœ… - uses: actions-rs/cargo@v1 - continue-on-error: false - with: - command: check - args: --verbose + uses: actions/checkout@v7 + - name: Test ๐Ÿงช + run: | + cargo test + cargo test --all-features + cargo test --no-default-features + for feature in ast blocking convert example-gemtext macros meta quick request; do + cargo test --no-default-features --features "$feature" + done + - name: Clippy ๐Ÿ“Ž + run: cargo clippy --all-features --all-targets -- -D warnings From bb373d34af21808f273895608a926ec8ce15eec0 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 23:04:12 +0000 Subject: [PATCH 07/19] fix(convert): Escape HTML and reject unsafe link targets --- src/convert.rs | 3 +- src/convert/html.rs | 104 +++++++++++++++++++++++++++++++++----------- tests/convert.rs | 89 ++++++++++++++++++++++++++++++++++++- 3 files changed, 168 insertions(+), 28 deletions(-) diff --git a/src/convert.rs b/src/convert.rs index 5e0574f..ec5ccc8 100644 --- a/src/convert.rs +++ b/src/convert.rs @@ -10,7 +10,8 @@ mod markdown; /// Different targets to convert Gemtext to #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum Target { - /// Convert Gemtext to HTML + /// Convert Gemtext to HTML. Links outside the Gemini, Gopher, HTTP, HTTPS, + /// mailto, and FTP schemes are rendered as text. HTML, /// Convert Gemtext to Markdown Markdown, diff --git a/src/convert/html.rs b/src/convert/html.rs index 5774954..a75186a 100644 --- a/src/convert/html.rs +++ b/src/convert/html.rs @@ -1,51 +1,103 @@ -use {crate::ast::Node, std::fmt::Write}; +use crate::ast::Node; + +fn push_escaped(html: &mut String, value: &str) { + for character in value.chars() { + match character { + '&' => html.push_str("&"), + '<' => html.push_str("<"), + '>' => html.push_str(">"), + '"' => html.push_str("""), + '\'' => html.push_str("'"), + _ => html.push(character), + } + } +} + +fn safe_link_target(target: &str) -> bool { + if target.is_empty() + || target.chars().any(|character| { + character.is_whitespace() || character.is_control() || character == '\\' + }) + { + return false; + } + + match target.find([':', '/', '?', '#']) { + Some(index) if target.as_bytes()[index] == b':' => matches!( + target[..index].to_ascii_lowercase().as_str(), + "gemini" | "gopher" | "http" | "https" | "mailto" | "ftp" + ), + _ => true, + } +} pub fn convert(source: &[Node]) -> String { let mut html = String::new(); - // Since we have an AST tree of the Gemtext, it is very easy to convert from - // this AST tree to an alternative markup format. for node in source { match node { Node::Text(text) => { - let _ = write!(&mut html, "

{text}

"); + html.push_str("

"); + push_escaped(&mut html, text); + html.push_str("

"); } + Node::Link { to, text } => { - let _ = write!( - &mut html, - "{}
", - to, - text.clone().unwrap_or_else(|| to.clone()) - ); + let label = text.as_deref().unwrap_or(to); + + if safe_link_target(to) { + html.push_str(""); + push_escaped(&mut html, label); + html.push_str("
"); + } else { + push_escaped(&mut html, label); + html.push_str("
"); + } } + Node::Heading { level, text } => { - let _ = write!( - &mut html, - "<{}>{}", - match level { - 1 => "h1", - 2 => "h2", - 3 => "h3", - _ => "p", - }, - text - ); + let tag = match level { + 1 => "h1", + 2 => "h2", + 3 => "h3", + _ => "p", + }; + + html.push('<'); + html.push_str(tag); + html.push('>'); + push_escaped(&mut html, text); + html.push_str("'); } + Node::List(items) => { - let _ = write!(&mut html, "
    "); + html.push_str("
      "); for item in items { - let _ = write!(&mut html, "
    • {item}
      • "); + html.push_str("
    • "); + push_escaped(&mut html, item); + html.push_str("
      • "); } - let _ = write!(&mut html, "
    "); + html.push_str("
"); } + Node::Blockquote(text) => { - let _ = write!(&mut html, "
{text}
"); + html.push_str("
"); + push_escaped(&mut html, text); + html.push_str("
"); } + Node::PreformattedText { text, .. } => { - let _ = write!(&mut html, "
{text}
"); + html.push_str("
");
+        push_escaped(&mut html, text);
+        html.push_str("
"); } + Node::Whitespace => {} } } diff --git a/tests/convert.rs b/tests/convert.rs index 0e47a8d..3c5dfe5 100644 --- a/tests/convert.rs +++ b/tests/convert.rs @@ -2,7 +2,10 @@ #[cfg(test)] mod test { - use germ::convert::{Target, from_string}; + use germ::{ + ast::{Ast, Node}, + convert::{Target, from_ast, from_string}, + }; #[cfg(feature = "macros")] use germ::{gemini_to_html, gemini_to_md}; #[test] @@ -15,6 +18,90 @@ mod test { assert_eq!(from_string("hi\n# hi", &Target::HTML), "

hi

hi

",); } + #[test] + fn html_escapes_content_in_every_node() { + let hostile = "<&>\"'".to_owned(); + let ast = Ast::from_nodes(vec![ + Node::Text(hostile.clone()), + Node::Heading { level: 1, text: hostile.clone() }, + Node::List(vec![hostile.clone()]), + Node::Blockquote(hostile.clone()), + Node::PreformattedText { alt_text: None, text: hostile }, + ]); + + assert_eq!( + from_ast(&ast, &Target::HTML), + "

<&>"'

<&>"'
  • <&>"'
<&>&\ + quot;'
<&>"'
", + ); + } + + #[test] + fn html_escapes_link_targets_and_labels() { + let ast = Ast::from_nodes(vec![Node::Link { + to: "/path?x=\"<&>".to_owned(), + text: Some("".to_owned()), + }]); + + assert_eq!( + from_ast(&ast, &Target::HTML), + "<script>alert('x'\ + )</script>
", + ); + } + + #[test] + fn html_renders_unsafe_links_as_text() { + for target in [ + "javascript:alert(1)", + "JaVaScRiPt:alert(1)", + "data:text/html,", + "java\nscript:alert(1)", + "vbscript:msgbox(1)", + "sftp://example.com/file", + ] { + let ast = Ast::from_nodes(vec![Node::Link { + to: target.to_owned(), + text: Some("safe label".to_owned()), + }]); + + assert_eq!(from_ast(&ast, &Target::HTML), "safe label
"); + } + } + + #[test] + fn html_escapes_raw_gemtext() { + assert_eq!( + from_string( + "\n=> javascript:alert(2) click", + &Target::HTML + ), + "

<script>alert(1)</script>

click
", + ); + } + + #[test] + fn html_keeps_safe_relative_and_absolute_links() { + for target in [ + "/local", + "relative", + "#anchor", + "gemini://example.com", + "https://example.com", + ] { + let ast = Ast::from_nodes(vec![Node::Link { + to: target.to_owned(), + text: None, + }]); + + assert_eq!( + from_ast(&ast, &Target::HTML), + format!("{target}
"), + ); + } + } + #[cfg(feature = "macros")] #[test] fn convert_from_string_to_html_single_link_macro_expression() { From 2f6ba432aba73e946bcdde7649034fdb615137a8 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Wed, 23 Sep 2026 23:04:44 +0000 Subject: [PATCH 08/19] fix(request): Verify server certificates --- Cargo.toml | 12 +-- README.md | 19 ++++- src/request.rs | 7 +- src/request/blocking.rs | 44 +++++++--- src/request/non_blocking.rs | 74 +++++++++-------- src/request/trust.rs | 32 ++++++++ src/request/verifier.rs | 28 ------- tests/request_tls.rs | 157 ++++++++++++++++++++++++++++++++++++ 8 files changed, 283 insertions(+), 90 deletions(-) create mode 100644 src/request/trust.rs delete mode 100644 src/request/verifier.rs create mode 100644 tests/request_tls.rs diff --git a/Cargo.toml b/Cargo.toml index d6d226c..0b39a1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,12 +17,12 @@ rust-version = "1.83.0" [features] ast = [] -blocking = ["rustls", "url", "anyhow"] +blocking = ["rustls", "webpki-roots", "url", "anyhow"] convert = ["ast"] default = ["ast", "convert", "meta", "request"] macros = ["ast", "convert"] meta = [] -request = ["rustls", "url", "anyhow", "tokio", "tokio-rustls"] +request = ["rustls", "webpki-roots", "url", "anyhow", "tokio", "tokio-rustls"] quick = [] example-gemtext = [] @@ -60,9 +60,8 @@ required-features = ["ast", "blocking"] [dependencies] anyhow = { version = "1.0.98", optional = true } # `Result` -rustls = { version = "0.21.12", features = [ - "dangerous_configuration", -], optional = true } # TLS +rustls = { version = "0.21.12", optional = true } # TLS +webpki-roots = { version = "1.0.9", optional = true } # CA certificates tokio-rustls = { version = "0.24.1", optional = true } # Non-blocking TLS tokio = { version = "1.45.1", optional = true, default-features = false, features = [ "net", @@ -71,3 +70,6 @@ tokio = { version = "1.45.1", optional = true, default-features = false, feature "macros", ] } # Non-blocking I/O url = { version = "2.5.4", optional = true } # URL Validation + +[dev-dependencies] +rcgen = "=0.12.1" diff --git a/README.md b/README.md index 2f20c0a..927fa69 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ The Ultimate Gemini Toolkit Germ is a toolkit for the Gemini protocol which aims to have a little something -for everyone. At the moment, Germ has **ZERO** dependencies (unless you use the -`request` feature), and Germ will continue to try its hardest to have as few -dependencies as possible. +for everyone. At the moment, Germ has **ZERO** dependencies unless you use the +`request` or `blocking` feature, and Germ will continue to try its hardest to +have as few dependencies as possible. ## Features @@ -54,6 +54,19 @@ features = ["ast"] # Enable the features you would like to use! | `macros` | Macros to aid with various Germ-related functionalities | | `quick` | Tiny functions to create valid Gemtext elements from structured input | +Gemini requests verify server certificates against Mozilla CA roots by default. +For a self-signed capsule, add its certificate in DER form to a +`rustls::RootCertStore`, then pass that store to +`request::blocking::request_with_roots` or +`request::non_blocking::request_with_roots`. The store is a trust anchor, not an +exact certificate fingerprint pin. Save and review the certificate separately +before trusting it; the default request functions do not remember certificates +between calls. + +HTML conversion escapes Gemtext content. It makes relative links and Gemini, +Gopher, HTTP, HTTPS, mailto, and FTP links clickable; other schemes are shown +as text. + ### Examples Thoroughly commented examples can be found within the diff --git a/src/request.rs b/src/request.rs index 1d76a9d..c5d9508 100644 --- a/src/request.rs +++ b/src/request.rs @@ -2,12 +2,13 @@ mod response; mod status; -mod verifier; +mod trust; #[cfg(feature = "blocking")] pub mod blocking; #[cfg(feature = "request")] pub mod non_blocking; #[cfg(feature = "request")] pub use non_blocking::request; -pub(crate) use verifier::GermVerifier; -pub use {response::Response, status::Status}; +pub use { + response::Response, status::Status, trust::default_root_certificates, +}; diff --git a/src/request/blocking.rs b/src/request/blocking.rs index 5e2947a..c82ce68 100644 --- a/src/request/blocking.rs +++ b/src/request/blocking.rs @@ -1,6 +1,9 @@ use { - crate::request::{GermVerifier, Response}, - std::io::{Read, Write}, + crate::request::{Response, default_root_certificates, trust::client_config}, + std::{ + io::{Read, Write}, + sync::Arc, + }, }; /// Make a request to a Gemini server. The `url` **should** be prefixed with a @@ -18,19 +21,34 @@ use { /// ``` /// /// # Errors -/// - May error if the URL is invalid -/// - May error if the TLS write fails -/// - May error if the TLS read fails +/// +/// Returns an error if the URL is invalid, the server certificate is invalid +/// or untrusted, or TLS I/O fails. pub fn request(url: &url::Url) -> anyhow::Result { - let config = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(std::sync::Arc::new(GermVerifier::new())) - .with_no_client_auth(); + request_with_roots(url, &default_root_certificates()) +} + +/// Make a request using the supplied trusted certificate authorities. +/// +/// The supplied roots replace the default Mozilla roots. To keep those roots, +/// start with `default_root_certificates()` before adding a capsule's DER +/// certificate with `RootCertStore::add`. Added certificates become trust +/// anchors, not exact fingerprint pins. The server name and certificate +/// validity period are still checked. +/// +/// # Errors +/// +/// Returns an error if the URL is invalid, the server certificate is invalid +/// or untrusted, or TLS I/O fails. +pub fn request_with_roots( + url: &url::Url, + roots: &rustls::RootCertStore, +) -> anyhow::Result { let domain = url .domain() .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; let mut connection = rustls::ClientConnection::new( - std::sync::Arc::new(config), + Arc::new(client_config(roots.clone())), domain.try_into()?, )?; let mut stream = std::net::TcpStream::connect(format!( @@ -42,9 +60,9 @@ pub fn request(url: &url::Url) -> anyhow::Result { tls.write_all(format!("{url}\r\n").as_bytes())?; - let mut plain_text = Vec::new(); + let mut response_bytes = Vec::new(); - tls.read_to_end(&mut plain_text)?; + tls.read_to_end(&mut response_bytes)?; - Ok(Response::new(&plain_text, tls.conn.negotiated_cipher_suite())) + Ok(Response::new(&response_bytes, tls.conn.negotiated_cipher_suite())) } diff --git a/src/request/non_blocking.rs b/src/request/non_blocking.rs index dd59e6b..36fc568 100644 --- a/src/request/non_blocking.rs +++ b/src/request/non_blocking.rs @@ -1,9 +1,10 @@ use { - crate::request::Response, + crate::request::{Response, default_root_certificates, trust::client_config}, + std::sync::Arc, tokio::io::{AsyncReadExt, AsyncWriteExt}, }; -/// Make a request to a Gemini server +/// Make a request to a Gemini server. /// /// The `url` **should** be prefixed with a scheme (e.g. "gemini://"). /// @@ -23,47 +24,44 @@ use { /// /// # Errors /// -/// - May error if the URL is invalid -/// - May error if the server is unreachable -/// - May error if the TLS write fails -/// - May error if the TLS read fails +/// Returns an error if the URL is invalid, the server is unreachable, its +/// certificate is invalid or untrusted, or TLS I/O fails. pub async fn request(url: &url::Url) -> anyhow::Result { - let mut tls = tokio_rustls::TlsConnector::from(std::sync::Arc::new( - rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(std::sync::Arc::new( - crate::request::GermVerifier::new(), - )) - .with_no_client_auth(), - )) - .connect( - rustls::ServerName::try_from( - url - .domain() - .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?, - )?, - tokio::net::TcpStream::connect(format!( - "{}:{}", - url - .domain() - .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?, - url.port().unwrap_or(1965) - )) - .await?, - ) - .await?; + request_with_roots(url, &default_root_certificates()).await +} + +/// Make a request using the supplied trusted certificate authorities. +/// +/// The supplied roots replace the default Mozilla roots. To keep those roots, +/// start with `default_root_certificates()` before adding a capsule's DER +/// certificate with `RootCertStore::add`. Added certificates become trust +/// anchors, not exact fingerprint pins. The server name and certificate +/// validity period are still checked. +/// +/// # Errors +/// +/// Returns an error if the URL is invalid, the server is unreachable, its +/// certificate is invalid or untrusted, or TLS I/O fails. +pub async fn request_with_roots( + url: &url::Url, + roots: &rustls::RootCertStore, +) -> anyhow::Result { + let domain = url + .domain() + .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; + let server_name = rustls::ServerName::try_from(domain)?; + let address = format!("{}:{}", domain, url.port().unwrap_or(1965)); + let stream = tokio::net::TcpStream::connect(address).await?; + let connector = + tokio_rustls::TlsConnector::from(Arc::new(client_config(roots.clone()))); + let mut tls = connector.connect(server_name, stream).await?; let cipher_suite = tls.get_mut().1.negotiated_cipher_suite(); tls.write_all(format!("{url}\r\n").as_bytes()).await?; - Ok(Response::new( - &{ - let mut plain_text = Vec::new(); + let mut response_bytes = Vec::new(); - tls.read_to_end(&mut plain_text).await?; + tls.read_to_end(&mut response_bytes).await?; - plain_text - }, - cipher_suite, - )) + Ok(Response::new(&response_bytes, cipher_suite)) } diff --git a/src/request/trust.rs b/src/request/trust.rs new file mode 100644 index 0000000..837b832 --- /dev/null +++ b/src/request/trust.rs @@ -0,0 +1,32 @@ +use rustls::{ClientConfig, OwnedTrustAnchor, RootCertStore}; + +/// Return the Mozilla CA roots used by the default request functions. +/// +/// Callers can add a certificate to this store before passing it to +/// `request_with_roots` to trust a selected self-signed Gemini capsule. +#[must_use] +pub fn default_root_certificates() -> RootCertStore { + let mut roots = RootCertStore::empty(); + + roots.add_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.iter().map( + |anchor| { + OwnedTrustAnchor::from_subject_spki_name_constraints( + anchor.subject.as_ref().to_vec(), + anchor.subject_public_key_info.as_ref().to_vec(), + anchor + .name_constraints + .as_ref() + .map(|constraints| constraints.as_ref().to_vec()), + ) + }, + )); + + roots +} + +pub fn client_config(roots: RootCertStore) -> ClientConfig { + ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(roots) + .with_no_client_auth() +} diff --git a/src/request/verifier.rs b/src/request/verifier.rs deleted file mode 100644 index 9fc54a6..0000000 --- a/src/request/verifier.rs +++ /dev/null @@ -1,28 +0,0 @@ -use { - rustls::{ - Certificate, - client::{self, ServerCertVerified}, - }, - std::time::SystemTime, -}; - -#[allow(clippy::module_name_repetitions)] -pub struct GermVerifier; - -impl GermVerifier { - pub const fn new() -> Self { Self {} } -} - -impl client::ServerCertVerifier for GermVerifier { - fn verify_server_cert( - &self, - _end_entity: &Certificate, - _intermediates: &[Certificate], - _server_name: &client::ServerName, - _scts: &mut dyn Iterator, - _ocsp_response: &[u8], - _now: SystemTime, - ) -> Result { - Ok(ServerCertVerified::assertion()) - } -} diff --git a/tests/request_tls.rs b/tests/request_tls.rs new file mode 100644 index 0000000..fa473e3 --- /dev/null +++ b/tests/request_tls.rs @@ -0,0 +1,157 @@ +#![cfg(any(feature = "blocking", feature = "request"))] + +use { + rustls::{ + Certificate, CertificateError, PrivateKey, ServerConfig, ServerConnection, + StreamOwned, + }, + std::{ + io::{self, Read, Write}, + net::TcpListener, + sync::Arc, + thread::{self, JoinHandle}, + time::Duration, + }, + url::Url, +}; + +fn assert_certificate_error(error: &anyhow::Error, expected: CertificateError) { + let actual = error + .downcast_ref::() + .and_then(io::Error::get_ref) + .and_then(|cause| cause.downcast_ref::()); + + assert!( + matches!(actual, Some(rustls::Error::InvalidCertificate(reason)) if reason == &expected), + "Unexpected TLS error: {error:?}" + ); +} + +fn assert_handshake_rejected(server: JoinHandle>>) { + let error = server.join().unwrap().unwrap_err(); + + assert_ne!(error.kind(), io::ErrorKind::TimedOut); +} + +fn spawn_tls_server( + certificate_name: &str, +) -> (Url, Certificate, JoinHandle>>) { + let certificate = + rcgen::generate_simple_self_signed(vec![certificate_name.to_owned()]) + .unwrap(); + let certificate_der = Certificate(certificate.serialize_der().unwrap()); + let private_key = PrivateKey(certificate.serialize_private_key_der()); + let config = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(vec![certificate_der.clone()], private_key) + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = Url::parse(&format!( + "gemini://localhost:{}/", + listener.local_addr().unwrap().port() + )) + .unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + + let connection = ServerConnection::new(Arc::new(config)).unwrap(); + let mut tls = StreamOwned::new(connection, stream); + let mut request_buffer = [0; 1024]; + let bytes_read = tls.read(&mut request_buffer)?; + + if bytes_read > 0 { + tls.write_all(b"20 text/gemini\r\nhello").unwrap(); + tls.conn.send_close_notify(); + tls.flush().unwrap(); + } + + Ok(request_buffer[..bytes_read].to_vec()) + }); + + (url, certificate_der, server) +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_request_rejects_untrusted_certificate_before_sending_url() { + let (url, _, server) = spawn_tls_server("localhost"); + let error = germ::request::blocking::request(&url).unwrap_err(); + + assert_certificate_error(&error, CertificateError::UnknownIssuer); + assert_handshake_rejected(server); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_request_accepts_explicitly_trusted_certificate() { + let (url, certificate, server) = spawn_tls_server("localhost"); + let mut roots = rustls::RootCertStore::empty(); + + roots.add(&certificate).unwrap(); + + let response = + germ::request::blocking::request_with_roots(&url, &roots).unwrap(); + + assert_eq!(response.content().as_deref(), Some("hello")); + assert_eq!(server.join().unwrap().unwrap(), format!("{url}\r\n").as_bytes()); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_request_rejects_wrong_hostname() { + let (url, certificate, server) = spawn_tls_server("elsewhere.test"); + let mut roots = rustls::RootCertStore::empty(); + + roots.add(&certificate).unwrap(); + + let error = + germ::request::blocking::request_with_roots(&url, &roots).unwrap_err(); + + assert_certificate_error(&error, CertificateError::NotValidForName); + assert_handshake_rejected(server); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_request_rejects_untrusted_certificate_before_sending_url() { + let (url, _, server) = spawn_tls_server("localhost"); + let error = germ::request::request(&url).await.unwrap_err(); + + assert_certificate_error(&error, CertificateError::UnknownIssuer); + assert_handshake_rejected(server); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_request_accepts_explicitly_trusted_certificate() { + let (url, certificate, server) = spawn_tls_server("localhost"); + let mut roots = rustls::RootCertStore::empty(); + + roots.add(&certificate).unwrap(); + + let response = germ::request::non_blocking::request_with_roots(&url, &roots) + .await + .unwrap(); + + assert_eq!(response.content().as_deref(), Some("hello")); + assert_eq!(server.join().unwrap().unwrap(), format!("{url}\r\n").as_bytes()); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_request_rejects_wrong_hostname() { + let (url, certificate, server) = spawn_tls_server("elsewhere.test"); + let mut roots = rustls::RootCertStore::empty(); + + roots.add(&certificate).unwrap(); + + let error = germ::request::non_blocking::request_with_roots(&url, &roots) + .await + .unwrap_err(); + + assert_certificate_error(&error, CertificateError::NotValidForName); + assert_handshake_rejected(server); +} From 4a48056986c5b5bca56a59e791095ff36cef8568 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 00:07:07 +0000 Subject: [PATCH 09/19] fix(request): Validate and bound Gemini exchanges --- Cargo.toml | 1 + README.md | 7 ++ src/request.rs | 128 +++++++++++++++++++++++++++- src/request/blocking.rs | 144 +++++++++++++++++++++++++++----- src/request/non_blocking.rs | 83 ++++++++++++------ src/request/response.rs | 112 +++++++++++++++++++------ tests/request_tls.rs | 162 ++++++++++++++++++++++++++++++++++-- 7 files changed, 556 insertions(+), 81 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0b39a1e..836c5ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ tokio = { version = "1.45.1", optional = true, default-features = false, feature "io-util", "rt-multi-thread", "macros", + "time", ] } # Non-blocking I/O url = { version = "2.5.4", optional = true } # URL Validation diff --git a/README.md b/README.md index 927fa69..afc0d54 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,13 @@ exact certificate fingerprint pin. Save and review the certificate separately before trusting it; the default request functions do not remember certificates between calls. +Requests time out after 30 seconds and have a 16 MiB response limit, including +the header. To change these limits or the trusted roots, set the fields of +`request::RequestOptions` and pass it to +`request::blocking::request_with_options` or +`request::non_blocking::request_with_options`. The blocking system DNS lookup +may outlast the timeout before it returns. + HTML conversion escapes Gemtext content. It makes relative links and Gemini, Gopher, HTTP, HTTPS, mailto, and FTP links clickable; other schemes are shown as text. diff --git a/src/request.rs b/src/request.rs index c5d9508..e6fde36 100644 --- a/src/request.rs +++ b/src/request.rs @@ -1,4 +1,8 @@ -//! Make Gemini requests and get sane, structured results +//! Make Gemini requests and get structured results. + +use std::time::Duration; + +const MAX_REQUEST_URL_BYTES: usize = 1024; mod response; mod status; @@ -12,3 +16,125 @@ mod trust; pub use { response::Response, status::Status, trust::default_root_certificates, }; + +/// Configure both blocking and async Gemini requests. +#[derive(Clone, Debug)] +pub struct RequestOptions { + /// Set trusted certificate authorities; defaults to the Mozilla CA roots. + pub root_certificates: rustls::RootCertStore, + /// Limit elapsed time for the request; defaults to 30 seconds. + pub timeout: Duration, + /// Limit response bytes, including the header; defaults to 16 MiB. + pub max_response_bytes: usize, +} + +impl RequestOptions { + const fn with_roots(root_certificates: rustls::RootCertStore) -> Self { + Self { + root_certificates, + timeout: Duration::from_secs(30), + max_response_bytes: 16 * 1024 * 1024, + } + } +} + +impl Default for RequestOptions { + fn default() -> Self { Self::with_roots(default_root_certificates()) } +} + +pub(crate) fn request_line(url: &url::Url) -> anyhow::Result { + if url.scheme() == "gemini" { + anyhow::ensure!( + url.username().is_empty() && url.password().is_none(), + "Gemini URLs cannot contain user information" + ); + } + + let mut request_url = url.clone(); + + request_url.set_fragment(None); + + let url_bytes = request_url.as_str().as_bytes(); + + anyhow::ensure!( + url_bytes.len() <= MAX_REQUEST_URL_BYTES, + "Gemini URL exceeds {MAX_REQUEST_URL_BYTES} bytes" + ); + anyhow::ensure!( + !url_bytes.contains(&b'\r') && !url_bytes.contains(&b'\n'), + "Gemini URL contains a line break" + ); + + Ok(format!("{request_url}\r\n")) +} + +#[cfg(test)] +mod tests { + use { + super::{MAX_REQUEST_URL_BYTES, request_line}, + url::Url, + }; + + #[test] + fn request_line_omits_fragment() { + let url = Url::parse("gemini://example.com/path?query#fragment").unwrap(); + + assert_eq!( + request_line(&url).unwrap(), + "gemini://example.com/path?query\r\n" + ); + } + + #[test] + fn request_line_enforces_url_byte_limit() { + let prefix = "gemini://example.com/"; + let accepted = Url::parse(&format!( + "{prefix}{}", + "a".repeat(MAX_REQUEST_URL_BYTES - prefix.len()) + )) + .unwrap(); + let rejected = Url::parse(&format!( + "{prefix}{}", + "a".repeat(MAX_REQUEST_URL_BYTES + 1 - prefix.len()) + )) + .unwrap(); + + assert_eq!( + request_line(&accepted).unwrap().len(), + MAX_REQUEST_URL_BYTES + b"\r\n".len() + ); + assert!(request_line(&rejected).is_err()); + } + + #[test] + fn request_line_counts_encoded_unicode_bytes() { + let prefix = "gemini://example.com/"; + let encoded_character_bytes = "%C3%A9".len(); + let accepted = Url::parse(&format!( + "{prefix}{}รฉ", + "a" + .repeat(MAX_REQUEST_URL_BYTES - prefix.len() - encoded_character_bytes) + )) + .unwrap(); + let rejected = Url::parse(&format!( + "{prefix}{}รฉ", + "a".repeat( + MAX_REQUEST_URL_BYTES + 1 - prefix.len() - encoded_character_bytes + ) + )) + .unwrap(); + + assert_eq!( + request_line(&accepted).unwrap().len(), + MAX_REQUEST_URL_BYTES + b"\r\n".len() + ); + assert!(request_line(&rejected).is_err()); + } + + #[test] + fn request_line_rejects_user_information() { + let url = Url::parse("gemini://user:secret@example.com/").unwrap(); + + assert!(request_line(&url).is_err()); + } +} diff --git a/src/request/blocking.rs b/src/request/blocking.rs index c82ce68..f9b0742 100644 --- a/src/request/blocking.rs +++ b/src/request/blocking.rs @@ -1,31 +1,86 @@ use { - crate::request::{Response, default_root_certificates, trust::client_config}, + crate::request::{ + RequestOptions, Response, request_line, trust::client_config, + }, std::{ - io::{Read, Write}, + io::{self, Read, Write}, + net::{TcpStream, ToSocketAddrs}, sync::Arc, + time::{Duration, Instant}, }, }; +// Recompute the timeout before each TLS read and write so partial I/O shares +// one request deadline. +struct DeadlineStream { + stream: TcpStream, + deadline: Instant, +} + +impl Read for DeadlineStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let remaining_duration = remaining_time(self.deadline)?; + + self.stream.set_read_timeout(Some(remaining_duration))?; + + self.stream.read(buffer).map_err(normalize_timeout) + } +} + +impl Write for DeadlineStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + let remaining_duration = remaining_time(self.deadline)?; + + self.stream.set_write_timeout(Some(remaining_duration))?; + + self.stream.write(buffer).map_err(normalize_timeout) + } + + fn flush(&mut self) -> io::Result<()> { self.stream.flush() } +} + +fn remaining_time(deadline: Instant) -> io::Result { + deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or_else(timeout_error) +} + +fn timeout_error() -> io::Error { + io::Error::new(io::ErrorKind::TimedOut, "Gemini request timed out") +} + +fn normalize_timeout(error: io::Error) -> io::Error { + if matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut) + { + timeout_error() + } else { + error + } +} + /// Make a request to a Gemini server. The `url` **should** be prefixed with a /// scheme (e.g. "gemini://"). /// /// # Example /// -/// ```rust -/// match germ::request::blocking::request( -/// &url::Url::parse("gemini://fuwn.me").unwrap(), -/// ) { -/// Ok(response) => println!("{:?}", response), -/// Err(_) => {} +/// ```rust,no_run +/// fn main() -> anyhow::Result<()> { +/// let url = url::Url::parse("gemini://fuwn.me")?; +/// let response = germ::request::blocking::request(&url)?; +/// +/// println!("{response:?}"); +/// +/// Ok(()) /// } /// ``` /// /// # Errors /// -/// Returns an error if the URL is invalid, the server certificate is invalid -/// or untrusted, or TLS I/O fails. +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. pub fn request(url: &url::Url) -> anyhow::Result { - request_with_roots(url, &default_root_certificates()) + request_with_options(url, &RequestOptions::default()) } /// Make a request using the supplied trusted certificate authorities. @@ -34,35 +89,78 @@ pub fn request(url: &url::Url) -> anyhow::Result { /// start with `default_root_certificates()` before adding a capsule's DER /// certificate with `RootCertStore::add`. Added certificates become trust /// anchors, not exact fingerprint pins. The server name and certificate -/// validity period are still checked. +/// validity period are still checked. Default time and size limits apply. /// /// # Errors /// -/// Returns an error if the URL is invalid, the server certificate is invalid -/// or untrusted, or TLS I/O fails. +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. pub fn request_with_roots( url: &url::Url, roots: &rustls::RootCertStore, ) -> anyhow::Result { + let options = RequestOptions::with_roots(roots.clone()); + + request_with_options(url, &options) +} + +/// Make a request with the supplied certificate, time, and size limits. +/// +/// The timeout covers connection, TLS, sending, and receiving. Hostname lookup +/// uses the system resolver and may outlast the timeout before returning. +/// The response limit includes the header and rejects an oversized response. +/// +/// # Errors +/// +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. +pub fn request_with_options( + url: &url::Url, + options: &RequestOptions, +) -> anyhow::Result { + let request_line = request_line(url)?; let domain = url .domain() .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; + let deadline = Instant::now() + .checked_add(options.timeout) + .ok_or_else(|| anyhow::anyhow!("Gemini request timeout is too large"))?; + let mut connection_error = + io::Error::new(io::ErrorKind::AddrNotAvailable, "No server address found"); + let mut connected_stream = None; + + for address in (domain, url.port().unwrap_or(1965)).to_socket_addrs()? { + match TcpStream::connect_timeout(&address, remaining_time(deadline)?) { + Ok(stream) => { + connected_stream = Some(stream); + + break; + } + Err(error) => connection_error = error, + } + } + + let mut stream = DeadlineStream { + stream: connected_stream.ok_or(connection_error)?, + deadline, + }; let mut connection = rustls::ClientConnection::new( - Arc::new(client_config(roots.clone())), + Arc::new(client_config(options.root_certificates.clone())), domain.try_into()?, )?; - let mut stream = std::net::TcpStream::connect(format!( - "{}:{}", - domain, - url.port().unwrap_or(1965) - ))?; let mut tls = rustls::Stream::new(&mut connection, &mut stream); - tls.write_all(format!("{url}\r\n").as_bytes())?; + tls.write_all(request_line.as_bytes())?; let mut response_bytes = Vec::new(); + let read_limit_bytes = options.max_response_bytes.saturating_add(1) as u64; - tls.read_to_end(&mut response_bytes)?; + Read::take(&mut tls, read_limit_bytes).read_to_end(&mut response_bytes)?; + anyhow::ensure!( + response_bytes.len() <= options.max_response_bytes, + "Gemini response exceeds {} bytes", + options.max_response_bytes + ); - Ok(Response::new(&response_bytes, tls.conn.negotiated_cipher_suite())) + Response::parse(&response_bytes, tls.conn.negotiated_cipher_suite()) } diff --git a/src/request/non_blocking.rs b/src/request/non_blocking.rs index 36fc568..8ee6d46 100644 --- a/src/request/non_blocking.rs +++ b/src/request/non_blocking.rs @@ -1,5 +1,7 @@ use { - crate::request::{Response, default_root_certificates, trust::client_config}, + crate::request::{ + RequestOptions, Response, request_line, trust::client_config, + }, std::sync::Arc, tokio::io::{AsyncReadExt, AsyncWriteExt}, }; @@ -10,24 +12,24 @@ use { /// /// # Example /// -/// ```rust +/// ```rust,no_run /// #[tokio::main] -/// async fn main() { -/// match germ::request::request(&url::Url::parse("gemini://fuwn.me").unwrap()) -/// .await -/// { -/// Ok(response) => println!("{:?}", response), -/// Err(_) => {} -/// } +/// async fn main() -> anyhow::Result<()> { +/// let url = url::Url::parse("gemini://fuwn.me")?; +/// let response = germ::request::request(&url).await?; +/// +/// println!("{response:?}"); +/// +/// Ok(()) /// } /// ``` /// /// # Errors /// -/// Returns an error if the URL is invalid, the server is unreachable, its -/// certificate is invalid or untrusted, or TLS I/O fails. +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. pub async fn request(url: &url::Url) -> anyhow::Result { - request_with_roots(url, &default_root_certificates()).await + request_with_options(url, &RequestOptions::default()).await } /// Make a request using the supplied trusted certificate authorities. @@ -36,32 +38,63 @@ pub async fn request(url: &url::Url) -> anyhow::Result { /// start with `default_root_certificates()` before adding a capsule's DER /// certificate with `RootCertStore::add`. Added certificates become trust /// anchors, not exact fingerprint pins. The server name and certificate -/// validity period are still checked. +/// validity period are still checked. Default time and size limits apply. /// /// # Errors /// -/// Returns an error if the URL is invalid, the server is unreachable, its -/// certificate is invalid or untrusted, or TLS I/O fails. +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. pub async fn request_with_roots( url: &url::Url, roots: &rustls::RootCertStore, ) -> anyhow::Result { + let options = RequestOptions::with_roots(roots.clone()); + + request_with_options(url, &options).await +} + +/// Make a request with the supplied certificate, time, and size limits. +/// +/// The timeout covers connection, TLS, sending, and receiving. The response +/// limit includes the header and rejects an oversized response. +/// +/// # Errors +/// +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. +pub async fn request_with_options( + url: &url::Url, + options: &RequestOptions, +) -> anyhow::Result { + let request_line = request_line(url)?; let domain = url .domain() .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; let server_name = rustls::ServerName::try_from(domain)?; - let address = format!("{}:{}", domain, url.port().unwrap_or(1965)); - let stream = tokio::net::TcpStream::connect(address).await?; - let connector = - tokio_rustls::TlsConnector::from(Arc::new(client_config(roots.clone()))); - let mut tls = connector.connect(server_name, stream).await?; - let cipher_suite = tls.get_mut().1.negotiated_cipher_suite(); - tls.write_all(format!("{url}\r\n").as_bytes()).await?; + tokio::time::timeout(options.timeout, async { + let address = (domain, url.port().unwrap_or(1965)); + let stream = tokio::net::TcpStream::connect(address).await?; + let connector = tokio_rustls::TlsConnector::from(Arc::new(client_config( + options.root_certificates.clone(), + ))); + let mut tls = connector.connect(server_name, stream).await?; + let cipher_suite = tls.get_mut().1.negotiated_cipher_suite(); + + tls.write_all(request_line.as_bytes()).await?; - let mut response_bytes = Vec::new(); + let mut response_bytes = Vec::new(); + let read_limit_bytes = options.max_response_bytes.saturating_add(1) as u64; - tls.read_to_end(&mut response_bytes).await?; + tls.take(read_limit_bytes).read_to_end(&mut response_bytes).await?; + anyhow::ensure!( + response_bytes.len() <= options.max_response_bytes, + "Gemini response exceeds {} bytes", + options.max_response_bytes + ); - Ok(Response::new(&response_bytes, cipher_suite)) + Response::parse(&response_bytes, cipher_suite) + }) + .await + .map_err(|_| anyhow::anyhow!("Gemini request timed out"))? } diff --git a/src/request/response.rs b/src/request/response.rs index b1a5cbc..9b23c52 100644 --- a/src/request/response.rs +++ b/src/request/response.rs @@ -1,5 +1,7 @@ use {crate::request::Status, rustls::SupportedCipherSuite, std::borrow::Cow}; +const MAX_META_BYTES: usize = 1024; + #[derive(Debug, Clone, PartialEq)] pub struct Response { status: Status, @@ -10,34 +12,54 @@ pub struct Response { } impl Response { - pub(crate) fn new(data: &[u8], suite: Option) -> Self { + pub(crate) fn parse( + response_bytes: &[u8], + cipher_suite: Option, + ) -> anyhow::Result { let delimiter = b"\r\n"; - let header_end = data + let header_end = response_bytes .windows(delimiter.len()) .position(|window| window == delimiter) - .map_or(data.len(), |pos| pos + delimiter.len()); - let header_bytes = &data[..header_end]; - let header_cow = String::from_utf8_lossy(header_bytes); - let header_trimmed = header_cow.trim_end(); - let content_bytes = if header_end < data.len() { - Some(data[header_end..].to_vec()) - } else { - None - }; - let (status_string, meta_string) = if header_trimmed.len() >= 2 { - header_trimmed.split_at(2) - } else { - (header_trimmed, "") - }; - let status_code = status_string.parse::().unwrap_or(0); - - Self { + .ok_or_else(|| anyhow::anyhow!("Gemini response header has no CRLF"))?; + let header_bytes = &response_bytes[..header_end]; + + anyhow::ensure!( + header_bytes.len() >= 3 + && header_bytes[0].is_ascii_digit() + && header_bytes[1].is_ascii_digit() + && header_bytes[2] == b' ', + "Gemini response header must begin with two digits and a space" + ); + + let meta_bytes = &header_bytes[3..]; + + anyhow::ensure!( + meta_bytes.len() <= MAX_META_BYTES, + "Gemini response meta exceeds {MAX_META_BYTES} bytes" + ); + anyhow::ensure!( + !meta_bytes.starts_with(&[0xef, 0xbb, 0xbf]), + "Gemini response meta begins with a byte order mark" + ); + anyhow::ensure!( + !meta_bytes.contains(&b'\r') && !meta_bytes.contains(&b'\n'), + "Gemini response meta contains a line break" + ); + + let meta = std::str::from_utf8(meta_bytes)?.to_owned(); + let status_code = i32::from(header_bytes[0] - b'0') * 10 + + i32::from(header_bytes[1] - b'0'); + let body_start = header_end + delimiter.len(); + let content = (body_start < response_bytes.len()) + .then(|| response_bytes[body_start..].to_vec()); + + Ok(Self { status: Status::from(status_code), - meta: meta_string.trim_start().to_string(), - content: content_bytes, - size: data.len(), - suite, - } + meta, + content, + size: response_bytes.len(), + suite: cipher_suite, + }) } #[must_use] @@ -47,9 +69,9 @@ impl Response { #[must_use] pub fn meta(&self) -> Cow<'_, str> { Cow::Borrowed(&self.meta) } - /// This associated function assumes that the content is valid UTF-8. + /// Return the body as text, replacing invalid UTF-8 sequences. /// - /// If you want to handle data bytes directly, use + /// To inspect the original bytes, use /// [`Response::content_bytes`]. #[must_use] pub fn content(&self) -> Option { @@ -68,3 +90,41 @@ impl Response { #[must_use] pub const fn suite(&self) -> &Option { &self.suite } } + +#[cfg(test)] +mod tests { + use super::{MAX_META_BYTES, Response}; + + #[test] + fn parses_header_and_body_without_changing_meta() { + let response = Response::parse(b"20 text/gemini\r\nhello", None).unwrap(); + + assert_eq!(response.meta(), " text/gemini"); + assert_eq!(response.content_bytes(), Some(b"hello".as_slice())); + assert_eq!(*response.size(), 22); + } + + #[test] + fn rejects_malformed_headers() { + for header in [ + b"\xc3\xa9 text/gemini\r\n".as_slice(), + b"2x text/gemini\r\n", + b"20text/gemini\r\n", + b"20 text/gemini", + b"20 bad\nmeta\r\n", + b"20 \xff\r\n", + b"20 \xef\xbb\xbfmeta\r\n", + ] { + assert!(Response::parse(header, None).is_err(), "{header:?}"); + } + } + + #[test] + fn enforces_meta_byte_limit() { + let accepted = format!("20 {}\r\n", "a".repeat(MAX_META_BYTES)); + let rejected = format!("20 {}\r\n", "a".repeat(MAX_META_BYTES + 1)); + + assert!(Response::parse(accepted.as_bytes(), None).is_ok()); + assert!(Response::parse(rejected.as_bytes(), None).is_err()); + } +} diff --git a/tests/request_tls.rs b/tests/request_tls.rs index fa473e3..84fbf0a 100644 --- a/tests/request_tls.rs +++ b/tests/request_tls.rs @@ -35,6 +35,18 @@ fn assert_handshake_rejected(server: JoinHandle>>) { fn spawn_tls_server( certificate_name: &str, +) -> (Url, Certificate, JoinHandle>>) { + spawn_tls_server_with_response( + certificate_name, + b"20 text/gemini\r\nhello".to_vec(), + Duration::ZERO, + ) +} + +fn spawn_tls_server_with_response( + certificate_name: &str, + response_bytes: Vec, + response_delay: Duration, ) -> (Url, Certificate, JoinHandle>>) { let certificate = rcgen::generate_simple_self_signed(vec![certificate_name.to_owned()]) @@ -63,9 +75,10 @@ fn spawn_tls_server( let bytes_read = tls.read(&mut request_buffer)?; if bytes_read > 0 { - tls.write_all(b"20 text/gemini\r\nhello").unwrap(); + thread::sleep(response_delay); + tls.write_all(&response_bytes)?; tls.conn.send_close_notify(); - tls.flush().unwrap(); + tls.flush()?; } Ok(request_buffer[..bytes_read].to_vec()) @@ -74,6 +87,19 @@ fn spawn_tls_server( (url, certificate_der, server) } +fn options_with_certificate( + certificate: &Certificate, +) -> germ::request::RequestOptions { + let mut options = germ::request::RequestOptions { + root_certificates: rustls::RootCertStore::empty(), + ..Default::default() + }; + + options.root_certificates.add(certificate).unwrap(); + + options +} + #[cfg(feature = "blocking")] #[test] fn blocking_request_rejects_untrusted_certificate_before_sending_url() { @@ -87,16 +113,20 @@ fn blocking_request_rejects_untrusted_certificate_before_sending_url() { #[cfg(feature = "blocking")] #[test] fn blocking_request_accepts_explicitly_trusted_certificate() { - let (url, certificate, server) = spawn_tls_server("localhost"); + let (mut url, certificate, server) = spawn_tls_server("localhost"); let mut roots = rustls::RootCertStore::empty(); + url.set_fragment(Some("not-sent")); roots.add(&certificate).unwrap(); let response = germ::request::blocking::request_with_roots(&url, &roots).unwrap(); assert_eq!(response.content().as_deref(), Some("hello")); - assert_eq!(server.join().unwrap().unwrap(), format!("{url}\r\n").as_bytes()); + assert_eq!( + server.join().unwrap().unwrap(), + format!("gemini://localhost:{}/\r\n", url.port().unwrap()).as_bytes() + ); } #[cfg(feature = "blocking")] @@ -114,6 +144,64 @@ fn blocking_request_rejects_wrong_hostname() { assert_handshake_rejected(server); } +#[cfg(feature = "blocking")] +#[test] +fn blocking_request_rejects_oversized_response() { + let response_bytes = b"20 text/gemini\r\nhello".to_vec(); + let (url, certificate, server) = spawn_tls_server_with_response( + "localhost", + response_bytes.clone(), + Duration::ZERO, + ); + let mut options = options_with_certificate(&certificate); + + options.max_response_bytes = response_bytes.len() - 1; + + let error = + germ::request::blocking::request_with_options(&url, &options).unwrap_err(); + + assert!(error.to_string().contains("response exceeds")); + assert!(server.join().is_ok()); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_request_times_out_waiting_for_response() { + let (url, certificate, server) = spawn_tls_server_with_response( + "localhost", + b"20 text/gemini\r\nhello".to_vec(), + Duration::from_millis(200), + ); + let mut options = options_with_certificate(&certificate); + + options.timeout = Duration::from_millis(50); + + let error = + germ::request::blocking::request_with_options(&url, &options).unwrap_err(); + + assert_eq!( + error.downcast_ref::().unwrap().kind(), + io::ErrorKind::TimedOut + ); + assert!(server.join().is_ok()); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_request_rejects_malformed_header() { + let (url, certificate, server) = spawn_tls_server_with_response( + "localhost", + b"\xc3\xa9 text/gemini\r\nhello".to_vec(), + Duration::ZERO, + ); + let options = options_with_certificate(&certificate); + let error = + germ::request::blocking::request_with_options(&url, &options).unwrap_err(); + + assert!(error.to_string().contains("two digits and a space")); + assert!(server.join().is_ok()); +} + #[cfg(feature = "request")] #[tokio::test] async fn async_request_rejects_untrusted_certificate_before_sending_url() { @@ -127,9 +215,10 @@ async fn async_request_rejects_untrusted_certificate_before_sending_url() { #[cfg(feature = "request")] #[tokio::test] async fn async_request_accepts_explicitly_trusted_certificate() { - let (url, certificate, server) = spawn_tls_server("localhost"); + let (mut url, certificate, server) = spawn_tls_server("localhost"); let mut roots = rustls::RootCertStore::empty(); + url.set_fragment(Some("not-sent")); roots.add(&certificate).unwrap(); let response = germ::request::non_blocking::request_with_roots(&url, &roots) @@ -137,7 +226,10 @@ async fn async_request_accepts_explicitly_trusted_certificate() { .unwrap(); assert_eq!(response.content().as_deref(), Some("hello")); - assert_eq!(server.join().unwrap().unwrap(), format!("{url}\r\n").as_bytes()); + assert_eq!( + server.join().unwrap().unwrap(), + format!("gemini://localhost:{}/\r\n", url.port().unwrap()).as_bytes() + ); } #[cfg(feature = "request")] @@ -155,3 +247,61 @@ async fn async_request_rejects_wrong_hostname() { assert_certificate_error(&error, CertificateError::NotValidForName); assert_handshake_rejected(server); } + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_request_rejects_oversized_response() { + let response_bytes = b"20 text/gemini\r\nhello".to_vec(); + let (url, certificate, server) = spawn_tls_server_with_response( + "localhost", + response_bytes.clone(), + Duration::ZERO, + ); + let mut options = options_with_certificate(&certificate); + + options.max_response_bytes = response_bytes.len() - 1; + + let error = germ::request::non_blocking::request_with_options(&url, &options) + .await + .unwrap_err(); + + assert!(error.to_string().contains("response exceeds")); + assert!(server.join().is_ok()); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_request_times_out_waiting_for_response() { + let (url, certificate, server) = spawn_tls_server_with_response( + "localhost", + b"20 text/gemini\r\nhello".to_vec(), + Duration::from_millis(200), + ); + let mut options = options_with_certificate(&certificate); + + options.timeout = Duration::from_millis(50); + + let error = germ::request::non_blocking::request_with_options(&url, &options) + .await + .unwrap_err(); + + assert_eq!(error.to_string(), "Gemini request timed out"); + assert!(server.join().is_ok()); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_request_rejects_malformed_header() { + let (url, certificate, server) = spawn_tls_server_with_response( + "localhost", + b"\xc3\xa9 text/gemini\r\nhello".to_vec(), + Duration::ZERO, + ); + let options = options_with_certificate(&certificate); + let error = germ::request::non_blocking::request_with_options(&url, &options) + .await + .unwrap_err(); + + assert!(error.to_string().contains("two digits and a space")); + assert!(server.join().is_ok()); +} From 060d519f68a45c9fe6a6724c623d9c730b3baaaa Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 01:06:59 +0000 Subject: [PATCH 10/19] refactor(ast): Parse Gemtext lines in one pass --- src/ast/container.rs | 421 +++++++++++++++++++++------------------- src/ast/node.rs | 2 +- src/convert/markdown.rs | 28 ++- tests/ast.rs | 93 +++++++-- tests/convert.rs | 24 +++ 5 files changed, 337 insertions(+), 231 deletions(-) diff --git a/src/ast/container.rs b/src/ast/container.rs index cca5ac9..fe85097 100644 --- a/src/ast/container.rs +++ b/src/ast/container.rs @@ -1,6 +1,6 @@ -use {super::Node, std::fmt::Write}; +use super::Node; -/// An AST structure which contains an AST tree +/// A parsed Gemtext document. /// /// # Example /// @@ -9,11 +9,14 @@ use {super::Node, std::fmt::Write}; /// ``` #[derive(Debug, Clone, PartialEq, Eq)] pub struct Ast { - inner: Vec, + inner: Vec, + append_final_line_ending: bool, + line_ending: &'static str, + unclosed_preformatted: bool, } impl Ast { - /// Build an AST tree from Gemtext + /// Parses Gemtext into an AST. /// /// # Example /// @@ -25,7 +28,7 @@ impl Ast { Self::parse(value.as_ref()) } - /// Build an AST tree from Gemtext + /// Parses Gemtext into an AST. /// /// # Example /// @@ -38,7 +41,7 @@ impl Ast { Self::parse(&value.into()) } - /// Build an AST tree from a value + /// Parses the string representation of a value into an AST. /// /// # Example /// @@ -51,39 +54,101 @@ impl Ast { } fn parse(source: &str) -> Self { - let mut ast = vec![]; - let mut in_preformatted = false; - let mut in_list = false; - let mut lines = source.lines(); - - // Iterate over all lines in the Gemtext `source` - while let Some(line) = lines.next() { - // Evaluate the Gemtext line and append its AST node to the `ast` tree - ast.append(&mut Self::evaluate( - line, - &mut lines, - &mut in_preformatted, - &mut in_list, - )); - } + let mut line_ending = None; + let mut nodes = Vec::new(); + let mut list_items = Vec::new(); + let mut preformatted_alt_text: Option = None; + let mut preformatted_lines = Vec::new(); + let mut unclosed_preformatted = false; + + for line_with_ending in source.split_inclusive('\n') { + let line = if let Some(line) = line_with_ending.strip_suffix("\r\n") { + line_ending.get_or_insert("\r\n"); + + line + } else if let Some(line) = line_with_ending.strip_suffix('\n') { + line_ending.get_or_insert("\n"); + + line + } else { + line_with_ending + }; + + if preformatted_alt_text.is_some() { + if line.starts_with("```") { + let alt_text = preformatted_alt_text.take().unwrap(); + let text = if preformatted_lines.is_empty() { + String::new() + } else { + let mut text = preformatted_lines.join(line_ending.unwrap_or("\n")); + + text.push_str(line_ending.unwrap_or("\n")); - if source.ends_with('\n') { - if let Some(last) = ast.last() { - if !matches!(last, Node::Whitespace) { - ast.push(Node::Whitespace); + text + }; + + nodes.push(Node::PreformattedText { + alt_text: (!alt_text.is_empty()).then_some(alt_text), + text, + }); + preformatted_lines.clear(); + } else { + preformatted_lines.push(line); } + + continue; + } + + if let Some(alt_text) = line.strip_prefix("```") { + flush_list(&mut nodes, &mut list_items); + + preformatted_alt_text = Some(alt_text.to_owned()); + + continue; + } + + if let Some(item) = line.strip_prefix("* ") { + list_items.push(item.to_owned()); + + continue; + } + + flush_list(&mut nodes, &mut list_items); + nodes.push(parse_line(line)); + } + + flush_list(&mut nodes, &mut list_items); + + if let Some(alt_text) = preformatted_alt_text { + unclosed_preformatted = true; + + let mut text = preformatted_lines.join(line_ending.unwrap_or("\n")); + + if source.ends_with('\n') && !preformatted_lines.is_empty() { + text.push_str(line_ending.unwrap_or("\n")); } + + nodes.push(Node::PreformattedText { + alt_text: (!alt_text.is_empty()).then_some(alt_text), + text, + }); } - Self { inner: ast } + Self { + inner: nodes, + append_final_line_ending: source.ends_with('\n') + && (!unclosed_preformatted || preformatted_lines.is_empty()), + line_ending: line_ending.unwrap_or("\n"), + unclosed_preformatted, + } } - /// Build an AST tree from a [`Vec`] of [`Node`]s + /// Builds an AST from a [`Vec`] of [`Node`]s. /// /// # Example /// /// ```rust - /// // This assertion converts the Gemtext "=> / Home\n" to an AST tree of one + /// // This assertion converts the Gemtext "=> / Home" to an AST tree of one /// // node, then converts the AST tree back to Gemtext, and compares it against /// // the original Gemtext. /// assert_eq!( @@ -95,62 +160,110 @@ impl Ast { /// ); /// ``` #[must_use] - pub const fn from_nodes(nodes: Vec) -> Self { Self { inner: nodes } } + pub const fn from_nodes(nodes: Vec) -> Self { + Self { + inner: nodes, + append_final_line_ending: false, + line_ending: "\n", + unclosed_preformatted: false, + } + } + /// Serialises the document as Gemtext. #[must_use] pub fn to_gemtext(&self) -> String { let mut gemtext = String::new(); - for node in &self.inner { + for (index, node) in self.inner.iter().enumerate() { + if index > 0 { + gemtext.push_str(self.line_ending); + } + match node { - Node::Text(text) => { - let _ = writeln!(&mut gemtext, "{text}"); - } + Node::Text(text) => gemtext.push_str(text), + Node::Link { to, text } => { - let _ = writeln!( - &mut gemtext, - "=> {}{}", - to, - text.clone().map_or_else(String::new, |text| format!(" {text}")), - ); + gemtext.push_str("=>"); + + if !to.is_empty() { + gemtext.push(' '); + gemtext.push_str(to); + } + + if let Some(text) = text { + gemtext.push(' '); + gemtext.push_str(text); + } } + Node::Heading { level, text } => { - let _ = writeln!(&mut gemtext, "{} {}", "#".repeat(*level), text); + gemtext.push_str(&"#".repeat(*level)); + + if !text.is_empty() { + gemtext.push(' '); + gemtext.push_str(text); + } } + Node::List(items) => { - let _ = writeln!( - &mut gemtext, - "{}", - items - .iter() - .map(|i| format!("* {i}")) - .collect::>() - .join("\n"), - ); + for (item_index, item) in items.iter().enumerate() { + if item_index > 0 { + gemtext.push_str(self.line_ending); + } + + gemtext.push_str("* "); + gemtext.push_str(item); + } } + Node::Blockquote(text) => { - let _ = writeln!(&mut gemtext, "> {text}"); + gemtext.push('>'); + + if !text.is_empty() { + gemtext.push(' '); + gemtext.push_str(text); + } } + Node::PreformattedText { alt_text, text } => { - let _ = writeln!( - &mut gemtext, - "```{}\n{}```", - alt_text.clone().unwrap_or_default(), - text - ); + let is_unclosed_last = + index + 1 == self.inner.len() && self.unclosed_preformatted; + + gemtext.push_str("```"); + + if let Some(alt_text) = alt_text { + gemtext.push_str(alt_text); + } + + if is_unclosed_last { + if !text.is_empty() { + gemtext.push_str(self.line_ending); + gemtext.push_str(text); + } + } else { + gemtext.push_str(self.line_ending); + gemtext.push_str(text); + + if !text.is_empty() && !text.ends_with('\n') { + gemtext.push_str(self.line_ending); + } + + gemtext.push_str("```"); + } } - Node::Whitespace => gemtext.push('\n'), + + Node::Whitespace => {} } } - if gemtext.ends_with('\n') { - gemtext.pop(); + if self.append_final_line_ending { + gemtext.push_str(self.line_ending); } gemtext } - /// The actual AST of `Ast` + /// Returns the parsed nodes. /// /// # Example /// @@ -160,158 +273,56 @@ impl Ast { /// ``` #[must_use] pub const fn inner(&self) -> &Vec { &self.inner } +} - #[allow(clippy::too_many_lines)] - fn evaluate( - line: &str, - lines: &mut std::str::Lines<'_>, - in_preformatted: &mut bool, - in_list: &mut bool, - ) -> Vec { - let mut preformatted = String::new(); - let mut alt_text = String::new(); - let mut nodes = vec![]; - let mut line = line; - let mut list_items = vec![]; - - // Enter a not-so-infinite loop as sometimes, we may need to stay in an - // evaluation loop, e.g., multiline contexts: preformatted text, lists, etc. - loop { - // Match the first character of the Gemtext line to understand the line - // type - match line.get(0..1).unwrap_or("") { - "=" if !*in_preformatted => { - // If the Gemtext line starts with an "=" ("=>"), it is a link line, - // so splitting it up should be easy enough. - let line = line.get(2..).unwrap_or(""); - let mut split = line.split_whitespace(); - - nodes.push(Node::Link { - to: split.next().unwrap_or_default().to_string(), - text: { - let rest: Vec<&str> = split.collect(); - - if rest.is_empty() { None } else { Some(rest.join(" ")) } - }, - }); - - break; - } - "#" if !*in_preformatted => { - // If the Gemtext line starts with an "#", it is a heading, so let's - // find out how deep it goes. - let level = - line.trim_start().chars().take_while(|&c| c == '#').count(); - - nodes.push(Node::Heading { - level, - // Here, the text after the heading markers is safely extracted. - // `chars().skip()` is used to safely handle UTF-8 boundaries. - text: line - .chars() - .skip(level) - .collect::() - .trim_start() - .to_string(), - }); - - break; - } - "*" if !*in_preformatted => { - // If the Gemtext line starts with an asterisk, it is a list item, so - // let's enter a list context. - if !*in_list { - *in_list = true; - } - - list_items.push(line.get(1..).unwrap_or("").trim_start().to_string()); - - if let Some(next_line) = lines.next() { - line = next_line; - } else { - break; - } - } - ">" if !*in_preformatted => { - // If the Gemtext line starts with an ">", it is a blockquote, so - // let's just clip off the line identifier. - nodes.push(Node::Blockquote( - line.get(1..).unwrap_or("").trim_start().to_string(), - )); - - break; - } - "`" => { - // If the Gemtext line starts with a backtick, it's a preformatted - // toggle, so let's enter a preformatted text context. - *in_preformatted = !*in_preformatted; - - if *in_preformatted { - alt_text = line.get(3..).unwrap_or("").to_string(); - - if let Some(next_line) = lines.next() { - line = next_line; - } else { - break; - } - } else { - nodes.push(Node::PreformattedText { - alt_text: if alt_text.is_empty() { None } else { Some(alt_text) }, - text: preformatted, - }); - - break; - } - } - "" if !*in_preformatted => { - if line.is_empty() { - // If the line has nothing on it, it is a whitespace line, as long - // as we aren't in a preformatted line context. - nodes.push(Node::Whitespace); - } else { - nodes.push(Node::Text(line.to_string())); - } - - break; - } - // This as a catchall. It does a number of things. - _ => { - if *in_preformatted { - // If we are in a preformatted line context, add the line to the - // preformatted blocks content and increment the line. - let _ = writeln!(&mut preformatted, "{line}"); - - if let Some(next_line) = lines.next() { - line = next_line; - } else { - break; - } - } else { - // If we are in a list item and hit a catchall, that must mean that - // we encountered a line which is not a list line, so - // let's stop adding items to the list context. - if *in_list { - *in_list = false; +fn flush_list(nodes: &mut Vec, items: &mut Vec) { + if !items.is_empty() { + nodes.push(Node::List(std::mem::take(items))); + } +} - nodes.push(Node::Text(line.to_string())); +fn after_optional_whitespace(text: &str) -> &str { + match text.chars().next() { + Some(character) if character.is_whitespace() => + &text[character.len_utf8()..], + _ => text, + } +} - break; - } +fn parse_line(line: &str) -> Node { + if let Some(link_content) = line.strip_prefix("=>") { + return parse_link(link_content); + } - nodes.push(Node::Text(line.to_string())); + if line.starts_with('#') { + let level = line.bytes().take_while(|byte| *byte == b'#').count(); - break; - } - } - } + if level <= 3 { + return Node::Heading { + level, + text: after_optional_whitespace(&line[level..]).to_owned(), + }; } + } - if !list_items.is_empty() { - nodes.reverse(); - nodes.push(Node::List(list_items)); - nodes.reverse(); - } + if let Some(quote) = line.strip_prefix('>') { + return Node::Blockquote(after_optional_whitespace(quote).to_owned()); + } + + if line.is_empty() { Node::Whitespace } else { Node::Text(line.to_owned()) } +} - nodes +fn parse_link(link_content: &str) -> Node { + let target_and_label = link_content.trim_start_matches(char::is_whitespace); + let target_end = target_and_label + .find(char::is_whitespace) + .unwrap_or(target_and_label.len()); + let target = &target_and_label[..target_end]; + let label = + target_and_label[target_end..].trim_start_matches(char::is_whitespace); + + Node::Link { + to: target.to_owned(), + text: (!label.is_empty()).then(|| label.to_owned()), } } diff --git a/src/ast/node.rs b/src/ast/node.rs index af00e8a..c806095 100644 --- a/src/ast/node.rs +++ b/src/ast/node.rs @@ -150,7 +150,7 @@ pub enum Node { /// ``` text: String, }, - /// A whitespace line, a line which contains nothing but whitespace. + /// An empty line. Whitespace, } diff --git a/src/convert/markdown.rs b/src/convert/markdown.rs index d77434e..38635c4 100644 --- a/src/convert/markdown.rs +++ b/src/convert/markdown.rs @@ -3,17 +3,17 @@ use {crate::ast::Node, std::fmt::Write}; pub fn convert(source: &[Node]) -> String { let mut markdown = String::new(); - // Since we have an AST tree of the Gemtext, it is very easy to convert from - // this AST tree to an alternative markup format. for node in source { match node { Node::Text(text) => { let _ = writeln!(&mut markdown, "{text}"); } + Node::Link { to, text } => markdown.push_str(&text.clone().map_or_else( || format!("<{to}>\n"), |text| format!("[{text}]({to})\n"), )), + Node::Heading { level, text } => { let _ = writeln!( &mut markdown, @@ -27,21 +27,33 @@ pub fn convert(source: &[Node]) -> String { text ); } + Node::List(items) => for item in items { let _ = writeln!(&mut markdown, "- {item}"); }, + Node::Blockquote(text) => { let _ = writeln!(&mut markdown, "> {text}"); } + Node::PreformattedText { alt_text, text } => { - let _ = writeln!( - &mut markdown, - "```{}\n{}```", - alt_text.clone().unwrap_or_default(), - text - ); + markdown.push_str("```"); + + if let Some(alt_text) = alt_text { + markdown.push_str(alt_text); + } + + markdown.push('\n'); + markdown.push_str(text); + + if !text.is_empty() && !text.ends_with('\n') { + markdown.push('\n'); + } + + markdown.push_str("```\n"); } + Node::Whitespace => markdown.push('\n'), } } diff --git a/tests/ast.rs b/tests/ast.rs index 9c7abbf..a5a3e8b 100644 --- a/tests/ast.rs +++ b/tests/ast.rs @@ -41,12 +41,7 @@ mod test { #[cfg(feature = "example-gemtext")] #[test] fn gemtext_to_ast_then_ast_to_gemtext() { - assert_eq!( - Ast::from_string(EXAMPLE_GEMTEXT).to_gemtext(), - // `to_gemtext` appends a newline to all responses, so let's make sure we - // account for that. - EXAMPLE_GEMTEXT - ); + assert_eq!(Ast::from_string(EXAMPLE_GEMTEXT).to_gemtext(), EXAMPLE_GEMTEXT); } #[cfg(all(feature = "example-gemtext", feature = "macros"))] @@ -54,8 +49,6 @@ mod test { fn gemtext_to_ast_then_ast_to_gemtext_macro_expression() { assert_eq!( germ::gemini_to_ast!(EXAMPLE_GEMTEXT).to_gemtext(), - // `to_gemtext` appends a newline to all responses, so let's make sure we - // account for that. EXAMPLE_GEMTEXT ); } @@ -85,7 +78,6 @@ mod test { #[test] fn build_heading_with_unicode_and_edge_cases() { - // Unicode characters let ast = Ast::from_string("# Hello, ไธ–็•Œ!"); assert_eq!(ast.inner().len(), 1); @@ -97,7 +89,6 @@ mod test { panic!("Expected heading node"); } - // Only hashes let ast = Ast::from_string("###"); assert_eq!(ast.inner().len(), 1); @@ -109,16 +100,84 @@ mod test { panic!("Expected heading node"); } - // Many hashes let ast = Ast::from_string("########## Very Deep Heading"); - assert_eq!(ast.inner().len(), 1); + assert_eq!(ast.inner(), &vec![Node::Text( + "########## Very Deep Heading".to_string() + )]); + } - if let Node::Heading { level, text } = ast.inner().first().unwrap() { - assert_eq!(level, &10); - assert_eq!(text, "Very Deep Heading"); - } else { - panic!("Expected heading node"); + #[test] + fn recognises_only_exact_line_markers() { + let ast = Ast::from_string( + "*star\n=text\n`tick\n#### too deep\n* item\n* indented", + ); + + assert_eq!(ast.inner(), &vec![ + Node::Text("*star".to_string()), + Node::Text("=text".to_string()), + Node::Text("`tick".to_string()), + Node::Text("#### too deep".to_string()), + Node::List(vec!["item".to_string(), " indented".to_string()]), + ]); + } + + #[test] + fn preserves_link_label_spacing() { + let source = "=> /path label with spaces"; + let ast = Ast::from_string(source); + + assert_eq!(ast.inner(), &vec![Node::Link { + to: "/path".to_string(), + text: Some("label with spaces".to_string()), + }]); + assert_eq!(ast.to_gemtext(), source); + } + + #[test] + fn preserves_unclosed_preformatted_content() { + let source = "```alt\n* item\n=> /path\n# heading"; + let ast = Ast::from_string(source); + + assert_eq!(ast.inner(), &vec![Node::PreformattedText { + alt_text: Some("alt".to_string()), + text: "* item\n=> /path\n# heading".to_string(), + }]); + assert_eq!(ast.to_gemtext(), source); + + let source_with_final_newline = "```alt\ncode\n"; + let ast = Ast::from_string(source_with_final_newline); + + assert_eq!(ast.inner(), &vec![Node::PreformattedText { + alt_text: Some("alt".to_string()), + text: "code\n".to_string(), + }]); + assert_eq!(ast.to_gemtext(), source_with_final_newline); + } + + #[test] + fn preserves_canonical_line_round_trips() { + for source in [ + "", + "plain", + "plain\n", + "plain\n\n", + "\n", + "\n\n", + "* item\n* indented\nnext", + "* item\n\n", + "# heading\n###\n> quoted", + "```\n```\n", + "```\ncode\n```", + "```\ncode\n```\n\n", + "```\ncode\n", + "```", + "```\n", + "```\n\n", + "plain\r\n\r\n* item\r\n", + "```\r\ncode\r\n```\r\n", + ] { + assert_eq!(Ast::from_string(source).to_gemtext(), source, "{source:?}"); } } } diff --git a/tests/convert.rs b/tests/convert.rs index 3c5dfe5..317970a 100644 --- a/tests/convert.rs +++ b/tests/convert.rs @@ -129,6 +129,30 @@ mod test { ); } + #[test] + fn renderers_keep_unclosed_preformatted_content() { + assert_eq!( + from_string("```alt\n* item\n# heading", &Target::HTML), + "
* item\n# heading
" + ); + assert_eq!( + from_string("```alt\n* item\n# heading", &Target::Markdown), + "```alt\n* item\n# heading\n```\n" + ); + assert_eq!( + from_string("```alt\ncode\n", &Target::HTML), + "
code\n
" + ); + } + + #[test] + fn markdown_separates_a_fenced_block_from_following_text() { + assert_eq!( + from_string("```\ncode\n```\nafter", &Target::Markdown), + "```\ncode\n```\nafter\n" + ); + } + #[cfg(feature = "macros")] #[test] fn convert_from_string_to_markdown_single_macro_expression() { From 7454cd9c3bd89eab249114fc1fb75e661bbaf7a1 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 03:32:58 +0000 Subject: [PATCH 11/19] fix(quick): Put link URLs before labels --- src/quick.rs | 35 ++++++++++++++++++++++++----------- tests/quick.rs | 6 +++--- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/quick.rs b/src/quick.rs index 76dd402..0b2a87f 100644 --- a/src/quick.rs +++ b/src/quick.rs @@ -27,20 +27,33 @@ pub fn list_item(text: &(impl ToString + ?Sized)) -> String { #[must_use] pub fn list_items(items: &[&(impl ToString + ?Sized)]) -> String { - items - .iter() - .map(|item| list_item(&item.to_string())) - .collect::>() - .join("\n") + let mut gemtext = String::new(); + + for (index, item) in items.iter().enumerate() { + if index > 0 { + gemtext.push('\n'); + } + + gemtext.push_str("* "); + gemtext.push_str(&item.to_string()); + } + + gemtext } +/// Builds a link line with a URL and an optional label. #[must_use] -pub fn link(text: &(impl ToString + ?Sized), location: Option<&str>) -> String { - format!( - "=> {}{}", - text.to_string(), - location.map_or_else(String::new, |l| format!(" {l}")) - ) +pub fn link(url: &(impl ToString + ?Sized), label: Option<&str>) -> String { + let mut gemtext = String::from("=> "); + + gemtext.push_str(&url.to_string()); + + if let Some(label) = label { + gemtext.push(' '); + gemtext.push_str(label); + } + + gemtext } #[must_use] diff --git a/tests/quick.rs b/tests/quick.rs index e3d5055..ac793e5 100644 --- a/tests/quick.rs +++ b/tests/quick.rs @@ -32,10 +32,10 @@ mod test { #[test] fn link_variants() { - assert_eq!(quick::link("Soup", None), "=> Soup"); + assert_eq!(quick::link("gemini://soup.com", None), "=> gemini://soup.com"); assert_eq!( - quick::link("Soup", Some("gemini://soup.com")), - "=> Soup gemini://soup.com" + quick::link("gemini://soup.com", Some("Soup")), + "=> gemini://soup.com Soup" ); } From e73ca286726d238d2d34fff04f3d29e7ef9925c4 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 03:32:59 +0000 Subject: [PATCH 12/19] fix(meta): Parse quoted MIME parameters --- examples/meta.rs | 13 ++--- src/meta.rs | 134 ++++++++++++++++++++++++++++++++++++++--------- tests/meta.rs | 33 +++++++++++- 3 files changed, 142 insertions(+), 38 deletions(-) diff --git a/examples/meta.rs b/examples/meta.rs index 056102f..62b3a4e 100644 --- a/examples/meta.rs +++ b/examples/meta.rs @@ -2,19 +2,12 @@ //! sections. fn main() { - // Parse Gemini meta section into a structured meta representation - let meta = germ::meta::Meta::from_string("text/gemini; hi=2; hi2=string=2"); + let meta = germ::meta::Meta::from_string( + r#"text/gemini; charset=utf-8; title="two; parts""#, + ); - // Debug view of the structured meta representation println!("{:?}", meta); - - // Convert the structured meta representation back to a string, identical to - // the original meta section println!("{}", meta); - - // The MIME type of the meta section println!("{}", meta.mime()); - - // A debug view of the parameters of the meta section println!("{:?}", meta.parameters()); } diff --git a/src/meta.rs b/src/meta.rs index 75452c2..e0d4f09 100644 --- a/src/meta.rs +++ b/src/meta.rs @@ -1,12 +1,60 @@ use std::{borrow::Cow, collections::HashMap, fmt::Display}; -/// Structure-ize a Gemini response's meta section into it's mime type and it's -/// parameters. +const MIME_SPECIAL_CHARACTERS: &[u8] = b"()<>@,;:\\\"/[]?="; + +fn is_mime_token(value: &str) -> bool { + !value.is_empty() + && value.bytes().all(|byte| { + byte.is_ascii_graphic() && !MIME_SPECIAL_CHARACTERS.contains(&byte) + }) +} + +fn parse_parameter(parameter: &str, parameters: &mut HashMap) { + let Some((name, value)) = parameter.trim().split_once('=') else { + return; + }; + let name = name.trim(); + let value = value.trim(); + + if !is_mime_token(name) { + return; + } + + let value = if let Some(quoted) = value.strip_prefix('"') { + let Some(quoted) = quoted.strip_suffix('"') else { + return; + }; + let mut unescaped = String::with_capacity(quoted.len()); + let mut characters = quoted.chars(); + + while let Some(character) = characters.next() { + if character == '\\' { + let Some(escaped) = characters.next() else { + return; + }; + + unescaped.push(escaped); + } else if character == '"' { + return; + } else { + unescaped.push(character); + } + } + + unescaped + } else { + value.to_owned() + }; + + parameters.insert(name.to_owned(), value); +} + +/// Structures a Gemini response's MIME type and parameters. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct Meta { - /// The mime type of a Gemini response + /// The MIME type of a Gemini response. mime: String, - /// The parameters of a Gemini response + /// The parameters of a Gemini response. parameters: HashMap, } @@ -17,16 +65,32 @@ impl Display for Meta { if !self.parameters.is_empty() { write!(f, "; ")?; - let mut parameters: Vec<_> = self.parameters.iter().collect(); + let mut sorted_parameters: Vec<_> = self.parameters.iter().collect(); - parameters.sort_by(|a, b| a.0.cmp(b.0)); + sorted_parameters.sort_by(|left, right| left.0.cmp(right.0)); - for (i, (key, value)) in parameters.iter().enumerate() { - if i > 0 { + for (index, (key, value)) in sorted_parameters.iter().enumerate() { + if index > 0 { write!(f, "; ")?; } - write!(f, "{key}={value}")?; + write!(f, "{key}=")?; + + if is_mime_token(value) { + write!(f, "{value}")?; + } else { + write!(f, "\"")?; + + for character in value.chars() { + if matches!(character, '"' | '\\') { + write!(f, "\\")?; + } + + write!(f, "{character}")?; + } + + write!(f, "\"")?; + } } } @@ -35,17 +99,18 @@ impl Display for Meta { } impl Meta { - /// Create a new `Meta` + /// Creates an empty `Meta`. /// /// # Example /// /// ```rust - /// let mut meta = germ::meta::Meta::new(); + /// let meta = germ::meta::Meta::new(); + /// assert!(meta.parameters().is_empty()); /// ``` #[must_use] pub fn new() -> Self { Self::default() } - /// Create a `Meta` from a string + /// Parses a MIME type and its semicolon-separated parameters. /// /// # Example /// @@ -57,27 +122,44 @@ impl Meta { /// ``` #[must_use] pub fn from_string<'a>(meta: impl Into>) -> Self { - let meta = meta.into().to_string(); - let mut metas = meta.split(';'); - let mime = metas.next().unwrap_or("").to_string(); + let meta = meta.into(); + let (mime, parameters_source) = meta.split_once(';').unwrap_or((&meta, "")); + let mime = mime.to_owned(); let mut parameters = HashMap::new(); + let mut parameter_start = 0; + let mut quoted = false; + let mut escaped = false; - for parameter in metas { - let trimmed = parameter.trim_start(); + for (index, character) in parameters_source.char_indices() { + if escaped { + escaped = false; - // Only parse parameters containing '=' as those without are malformed - // according to RFC 2045 - if let Some(equal_pos) = trimmed.find('=') { - let (key, value) = trimmed.split_at(equal_pos); + continue; + } + + match character { + '\\' if quoted => escaped = true, + '"' => quoted = !quoted, + ';' if !quoted => { + parse_parameter( + ¶meters_source[parameter_start..index], + &mut parameters, + ); - parameters.insert(key.to_string(), value[1..].to_string()); + parameter_start = index + 1; + } + _ => {} } } + if !quoted { + parse_parameter(¶meters_source[parameter_start..], &mut parameters); + } + Self { mime, parameters } } - /// Obtain non-mutable access to the mime of the `Meta` + /// Returns the MIME type. /// /// # Example /// @@ -91,7 +173,7 @@ impl Meta { #[must_use] pub fn mime(&self) -> Cow<'_, str> { Cow::Borrowed(&self.mime) } - /// Obtain mutable access to the mime of the `Meta` + /// Returns mutable access to the MIME type. /// /// # Example /// @@ -102,7 +184,7 @@ impl Meta { /// ``` pub const fn mime_mut(&mut self) -> &mut String { &mut self.mime } - /// Obtain non-mutable access to the parameters of the `Meta` + /// Returns the parameters. /// /// # Example /// @@ -119,7 +201,7 @@ impl Meta { &self.parameters } - /// Obtain mutable access to the parameters of the `Meta` + /// Returns mutable access to the parameters. /// /// # Example /// diff --git a/tests/meta.rs b/tests/meta.rs index 842fc3e..5dce058 100644 --- a/tests/meta.rs +++ b/tests/meta.rs @@ -24,7 +24,7 @@ mod test { *meta.mime_mut() = "text/gemini".to_string(); *meta.parameters_mut() = parameters; - assert_eq!(meta.to_string(), "text/gemini; hi=2; hi2=string=2"); + assert_eq!(meta.to_string(), "text/gemini; hi=2; hi2=\"string=2\""); } #[test] @@ -36,7 +36,7 @@ mod test { #[test] fn meta_to_string_with_parameters() { - let original_string = "text/gemini; hi=2; hi2=string=2"; + let original_string = "text/gemini; hi=2; hi2=\"string=2\""; assert_eq!(Meta::from_string(original_string).to_string(), original_string); } @@ -86,4 +86,33 @@ mod test { assert_eq!(meta.parameters().get("another"), Some(&"test".to_string())); assert_eq!(meta.parameters().get("malformed"), None); } + + #[test] + fn parses_quoted_parameter_values() { + let source = r#"text/gemini; title="part one; part two"; path="a\"b\\c"; charset=utf-8"#; + let meta = Meta::from_string(source); + + assert_eq!( + meta.parameters().get("title").map(String::as_str), + Some("part one; part two") + ); + assert_eq!( + meta.parameters().get("path").map(String::as_str), + Some("a\"b\\c") + ); + assert_eq!( + meta.to_string(), + r#"text/gemini; charset=utf-8; path="a\"b\\c"; title="part one; part two""# + ); + } + + #[test] + fn skips_unclosed_quoted_parameters() { + let meta = Meta::from_string( + "text/gemini; valid=yes; broken=\"unfinished; other=lost", + ); + + assert_eq!(meta.parameters().len(), 1); + assert_eq!(meta.parameters().get("valid").map(String::as_str), Some("yes")); + } } From 4d667ba6b41e0f3139323598085611384829e4cc Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 03:32:59 +0000 Subject: [PATCH 13/19] fix(request): Preserve unknown Gemini status codes --- src/request.rs | 4 +++- src/request/response.rs | 14 +++++++++++++- src/request/status.rs | 43 +++++++++++++++++++++++++++++++++++------ tests/status.rs | 24 ++++++++++++++++++++++- 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/request.rs b/src/request.rs index e6fde36..f852daa 100644 --- a/src/request.rs +++ b/src/request.rs @@ -14,7 +14,9 @@ mod trust; #[cfg(feature = "request")] pub use non_blocking::request; pub use { - response::Response, status::Status, trust::default_root_certificates, + response::Response, + status::{Status, StatusCategory}, + trust::default_root_certificates, }; /// Configure both blocking and async Gemini requests. diff --git a/src/request/response.rs b/src/request/response.rs index 9b23c52..cf92878 100644 --- a/src/request/response.rs +++ b/src/request/response.rs @@ -93,7 +93,10 @@ impl Response { #[cfg(test)] mod tests { - use super::{MAX_META_BYTES, Response}; + use { + super::{MAX_META_BYTES, Response}, + crate::request::{Status, StatusCategory}, + }; #[test] fn parses_header_and_body_without_changing_meta() { @@ -127,4 +130,13 @@ mod tests { assert!(Response::parse(accepted.as_bytes(), None).is_ok()); assert!(Response::parse(rejected.as_bytes(), None).is_err()); } + + #[test] + fn retains_unknown_response_status() { + let response = Response::parse(b"29 text/gemini\r\nbody", None).unwrap(); + + assert_eq!(*response.status(), Status::Unknown(29)); + assert_eq!(response.status().category(), Some(StatusCategory::Success)); + assert_eq!(response.content_bytes(), Some(b"body".as_slice())); + } } diff --git a/src/request/status.rs b/src/request/status.rs index 86dfb85..0d30332 100644 --- a/src/request/status.rs +++ b/src/request/status.rs @@ -1,6 +1,6 @@ use std::{fmt, fmt::Formatter}; -/// Simple Gemini status reporting +/// A Gemini response status. /// /// # Examples /// @@ -31,12 +31,42 @@ pub enum Status { ClientCertificateRequired, CertificateNotAuthorised, CertificateNotValid, + /// A code without a named variant. + Unknown(i32), + /// A legacy status without a known code. Unsupported, } +/// The response category indicated by the first status digit. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum StatusCategory { + Input, + Success, + Redirect, + TemporaryFailure, + PermanentFailure, + ClientCertificateRequired, +} + +impl Status { + /// Returns the category for codes in the Gemini status range. + #[must_use] + pub fn category(self) -> Option { + match i32::from(self) { + 10..=19 => Some(StatusCategory::Input), + 20..=29 => Some(StatusCategory::Success), + 30..=39 => Some(StatusCategory::Redirect), + 40..=49 => Some(StatusCategory::TemporaryFailure), + 50..=59 => Some(StatusCategory::PermanentFailure), + 60..=69 => Some(StatusCategory::ClientCertificateRequired), + _ => None, + } + } +} + impl From for i32 { - fn from(n: Status) -> Self { - match n { + fn from(status: Status) -> Self { + match status { Status::Input => 10, Status::SensitiveInput => 11, Status::Success => 20, @@ -55,14 +85,15 @@ impl From for i32 { Status::ClientCertificateRequired => 60, Status::CertificateNotAuthorised => 61, Status::CertificateNotValid => 62, + Status::Unknown(code) => code, Status::Unsupported => 0, } } } impl From for Status { - fn from(n: i32) -> Self { - match n { + fn from(code: i32) -> Self { + match code { 10 => Self::Input, 11 => Self::SensitiveInput, 20 => Self::Success, @@ -81,7 +112,7 @@ impl From for Status { 60 => Self::ClientCertificateRequired, 61 => Self::CertificateNotAuthorised, 62 => Self::CertificateNotValid, - _ => Self::Unsupported, + _ => Self::Unknown(code), } } } diff --git a/tests/status.rs b/tests/status.rs index 82b78bb..a79d6bb 100644 --- a/tests/status.rs +++ b/tests/status.rs @@ -2,7 +2,7 @@ #[cfg(test)] mod test { - use germ::request::Status; + use germ::request::{Status, StatusCategory}; #[test] fn status_from_i32() { @@ -14,6 +14,28 @@ mod test { assert_eq!(i32::from(Status::Input), 10); } + #[test] + fn unknown_codes_keep_their_value_and_category() { + for (code, category) in [ + (19, StatusCategory::Input), + (29, StatusCategory::Success), + (39, StatusCategory::Redirect), + (49, StatusCategory::TemporaryFailure), + (58, StatusCategory::PermanentFailure), + (69, StatusCategory::ClientCertificateRequired), + ] { + let status = Status::from(code); + + assert_eq!(status, Status::Unknown(code)); + assert_eq!(i32::from(status), code); + assert_eq!(status.category(), Some(category)); + } + + assert_eq!(Status::from(79).category(), None); + assert_eq!(i32::from(Status::from(79)), 79); + assert_eq!(Status::Unsupported.category(), None); + } + #[cfg(feature = "blocking")] #[test] fn invalid_url_handling() { From 1127009c30af7baf8b02efa35a28801120cf4cc1 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 03:33:00 +0000 Subject: [PATCH 14/19] fix(convert): Escape Markdown links --- src/convert.rs | 28 +++++++++++++++++---- src/convert/html.rs | 20 +-------------- src/convert/markdown.rs | 55 +++++++++++++++++++++++++++++++++++++---- tests/convert.rs | 32 ++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/src/convert.rs b/src/convert.rs index ec5ccc8..d516790 100644 --- a/src/convert.rs +++ b/src/convert.rs @@ -7,17 +7,35 @@ mod markdown; #[cfg(feature = "macros")] mod macros; -/// Different targets to convert Gemtext to +fn safe_link_target(target: &str) -> bool { + if target.is_empty() + || target.chars().any(|character| { + character.is_whitespace() || character.is_control() || character == '\\' + }) + { + return false; + } + + match target.find([':', '/', '?', '#']) { + Some(index) if target.as_bytes()[index] == b':' => matches!( + target[..index].to_ascii_lowercase().as_str(), + "gemini" | "gopher" | "http" | "https" | "mailto" | "ftp" + ), + _ => true, + } +} + +/// The available Gemtext conversion targets. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum Target { - /// Convert Gemtext to HTML. Links outside the Gemini, Gopher, HTTP, HTTPS, + /// Converts Gemtext to HTML. Links outside the Gemini, Gopher, HTTP, HTTPS, /// mailto, and FTP schemes are rendered as text. HTML, - /// Convert Gemtext to Markdown + /// Converts Gemtext to Markdown with the same link scheme policy as HTML. Markdown, } -/// Convert AST'd Gemtext into an alternative markup format. +/// Converts a Gemtext AST into another markup format. /// /// # Example /// @@ -37,7 +55,7 @@ pub fn from_ast(source: &Ast, target: &Target) -> String { } } -/// Convert raw Gemtext into an alternative markup format. +/// Converts raw Gemtext into another markup format. /// /// # Example /// diff --git a/src/convert/html.rs b/src/convert/html.rs index a75186a..e865989 100644 --- a/src/convert/html.rs +++ b/src/convert/html.rs @@ -1,4 +1,4 @@ -use crate::ast::Node; +use {super::safe_link_target, crate::ast::Node}; fn push_escaped(html: &mut String, value: &str) { for character in value.chars() { @@ -13,24 +13,6 @@ fn push_escaped(html: &mut String, value: &str) { } } -fn safe_link_target(target: &str) -> bool { - if target.is_empty() - || target.chars().any(|character| { - character.is_whitespace() || character.is_control() || character == '\\' - }) - { - return false; - } - - match target.find([':', '/', '?', '#']) { - Some(index) if target.as_bytes()[index] == b':' => matches!( - target[..index].to_ascii_lowercase().as_str(), - "gemini" | "gopher" | "http" | "https" | "mailto" | "ftp" - ), - _ => true, - } -} - pub fn convert(source: &[Node]) -> String { let mut html = String::new(); diff --git a/src/convert/markdown.rs b/src/convert/markdown.rs index 38635c4..ca1a887 100644 --- a/src/convert/markdown.rs +++ b/src/convert/markdown.rs @@ -1,4 +1,39 @@ -use {crate::ast::Node, std::fmt::Write}; +use {super::safe_link_target, crate::ast::Node, std::fmt::Write}; + +fn push_escaped_label(markdown: &mut String, label: &str) { + for character in label.chars() { + if character.is_control() { + let _ = write!(markdown, "&#{};", u32::from(character)); + + continue; + } + + if matches!( + character, + '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '&' | '|' | '~' + ) { + markdown.push('\\'); + } + + markdown.push(character); + } +} + +fn push_escaped_destination(markdown: &mut String, target: &str) { + for character in target.chars() { + match character { + '(' | ')' => { + markdown.push('\\'); + markdown.push(character); + } + '<' => markdown.push_str("%3C"), + '>' => markdown.push_str("%3E"), + '"' => markdown.push_str("%22"), + '&' => markdown.push_str("&"), + _ => markdown.push(character), + } + } +} pub fn convert(source: &[Node]) -> String { let mut markdown = String::new(); @@ -9,10 +44,20 @@ pub fn convert(source: &[Node]) -> String { let _ = writeln!(&mut markdown, "{text}"); } - Node::Link { to, text } => markdown.push_str(&text.clone().map_or_else( - || format!("<{to}>\n"), - |text| format!("[{text}]({to})\n"), - )), + Node::Link { to, text } => { + let label = text.as_deref().unwrap_or(to); + + if safe_link_target(to) { + markdown.push('['); + push_escaped_label(&mut markdown, label); + markdown.push_str("]("); + push_escaped_destination(&mut markdown, to); + markdown.push_str(")\n"); + } else { + push_escaped_label(&mut markdown, label); + markdown.push('\n'); + } + } Node::Heading { level, text } => { let _ = writeln!( diff --git a/tests/convert.rs b/tests/convert.rs index 317970a..4b9e8fd 100644 --- a/tests/convert.rs +++ b/tests/convert.rs @@ -129,6 +129,38 @@ mod test { ); } + #[test] + fn markdown_escapes_link_labels_and_destinations() { + assert_eq!( + from_string( + "=> /docs/(a)?x=1&next= A [link] *wow* ©", + &Target::Markdown + ), + "[A \\[link\\] \\*wow\\* \ + \\©](/docs/\\(a\\)?x=1&next=%3Ctag%3E)\n" + ); + assert_eq!( + from_string("=> /local", &Target::Markdown), + "[/local](/local)\n" + ); + } + + #[test] + fn markdown_renders_unsafe_links_as_text() { + let ast = Ast::from_nodes(vec![ + Node::Link { + to: "javascript:alert(1)".to_owned(), + text: Some("click".to_owned()), + }, + Node::Link { + to: "/bad\nline".to_owned(), + text: Some("evil\nlabel".to_owned()), + }, + ]); + + assert_eq!(from_ast(&ast, &Target::Markdown), "click\nevil label\n"); + } + #[test] fn renderers_keep_unclosed_preformatted_content() { assert_eq!( From e579d570d1db03e07f6a226e47b96f27e0e0e40d Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 03:33:00 +0000 Subject: [PATCH 15/19] docs: Correct feature guidance and simplify examples --- README.md | 73 +++++------ examples/ast.rs | 6 +- examples/ast_to_gemtext.rs | 8 +- .../request_blocking_to_gemtext_from_ast.rs | 117 +++++++----------- 4 files changed, 78 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index afc0d54..0cc127b 100644 --- a/README.md +++ b/README.md @@ -4,41 +4,23 @@ [![docs.rs](https://docs.rs/germ/badge.svg)](https://docs.rs/germ) [![github.com](https://github.com/gemrest/germ/actions/workflows/check.yaml/badge.svg?branch=main)](https://github.com/gemrest/germ/actions/workflows/check.yaml) -The Ultimate Gemini Toolkit - -Germ is a toolkit for the Gemini protocol which aims to have a little something -for everyone. At the moment, Germ has **ZERO** dependencies unless you use the -`request` or `blocking` feature, and Germ will continue to try its hardest to -have as few dependencies as possible. - -## Features - -- AST builder to easily construct and manipulate AST trees from raw Gemtext -- Converters to easily convert from Gemtext to markup formats such as HTML or - Markdown -- Blocking and non-blocking request suite -- Structured meta section manipulation -- And more! - -Check out the rest of the features in the Features section under Usage +Germ parses and generates Gemtext, converts it to HTML or Markdown, handles +response metadata, and makes Gemini requests. Its default features include +the request client and its network and TLS dependencies. Disable default +features to use `ast`, `convert`, `meta`, or `quick` without normal dependencies. ## Usage -Current version: -[![crates.io](https://img.shields.io/crates/v/germ.svg)](https://crates.io/crates/germ) - ```toml -# Cargo.toml +[dependencies] +germ = "0.4.8" +``` + +To select features without the default request client: +```toml [dependencies] -# To enable only the base (default) features: ast, convert, meta, request -# germ = "*" # Use current version show above! - -# To enable only certain features -[dependencies.germ] -version = "*" # Use current version show above! -default-features = false -features = ["ast"] # Enable the features you would like to use! +germ = { version = "0.4.8", default-features = false, features = ["ast"] } ``` ### Features @@ -47,12 +29,12 @@ features = ["ast"] # Enable the features you would like to use! | ---------- | --------------------------------------------------------------------- | | `default` | `ast`, `convert`, `meta`, `request` | | `ast` | Construct AST trees from raw Gemtext | -| `blocking` | Blocking equivalent of `request` | -| `convert` | Convert Gemtext to markup formats such as HTML or Markdown | -| `request` | Make Gemini requests, get sane, structured results | -| `meta` | Structure-ise a Gemini response's meta section | -| `macros` | Macros to aid with various Germ-related functionalities | -| `quick` | Tiny functions to create valid Gemtext elements from structured input | +| `blocking` | Make blocking Gemini requests | +| `convert` | Convert Gemtext to HTML or Markdown | +| `request` | Make asynchronous Gemini requests | +| `meta` | Parse and format Gemini response metadata | +| `macros` | Generate AST and converted output with macros | +| `quick` | Build individual Gemtext lines | Gemini requests verify server certificates against Mozilla CA roots by default. For a self-signed capsule, add its certificate in DER form to a @@ -70,17 +52,26 @@ the header. To change these limits or the trusted roots, set the fields of `request::non_blocking::request_with_options`. The blocking system DNS lookup may outlast the timeout before it returns. -HTML conversion escapes Gemtext content. It makes relative links and Gemini, -Gopher, HTTP, HTTPS, mailto, and FTP links clickable; other schemes are shown -as text. +HTML conversion escapes Gemtext content. HTML and Markdown conversion emit +links for relative targets and Gemini, Gopher, HTTP, HTTPS, mailto, and FTP +URLs; other schemes are shown as text. Markdown conversion also escapes link +labels and destinations. + +`Meta` parses quoted MIME parameters and quotes values as needed when formatting +them. + +`quick::link(url, label)` takes the URL first and an optional label second. + +Unknown Gemini status codes retain their numeric value. Use +`Status::category()` to inspect the response category indicated by the first +digit. ### Examples -Thoroughly commented examples can be found within the +Runnable examples can be found within the [`examples/`](https://github.com/gemrest/germ/tree/main/examples) directory. -Examples can be run by name using the example just task. -(e.g., `just example ast_to_gemtext`) +Run one with `just example ast_to_gemtext`. ## License diff --git a/examples/ast.rs b/examples/ast.rs index d7fb91f..1b8d033 100644 --- a/examples/ast.rs +++ b/examples/ast.rs @@ -2,13 +2,9 @@ //! abstract syntax tree. fn main() { - // Parse `EXAMPLE_GEMTEXT` into an abstract syntax tree let ast = germ::ast::Ast::from_string(germ::EXAMPLE_GEMTEXT); - // Get the nodes of the abstract syntax tree - let ast_nodes = ast.inner(); - // Print the abstract syntax tree nodes, one-by-one - for node in ast_nodes { + for node in ast.inner() { println!("{:?}", node); } } diff --git a/examples/ast_to_gemtext.rs b/examples/ast_to_gemtext.rs index 7bf4206..8c31487 100644 --- a/examples/ast_to_gemtext.rs +++ b/examples/ast_to_gemtext.rs @@ -1,12 +1,8 @@ //! This example converts Gemtext into an abstract syntax tree and then back -//! into Gemtext, demonstrating both Germ's parsing and generation capabilities. +//! into Gemtext. fn main() { - // Parse `EXAMPLE_GEMTEXT` into an abstract syntax tree let ast = germ::ast::Ast::from_string(germ::EXAMPLE_GEMTEXT); - // Convert the abstract syntax tree back to Gemtext - let gemtext = ast.to_gemtext(); - // Print the Gemtext, identical to `EXAMPLE_GEMTEXT` - println!("{gemtext}"); + print!("{}", ast.to_gemtext()); } diff --git a/examples/request_blocking_to_gemtext_from_ast.rs b/examples/request_blocking_to_gemtext_from_ast.rs index 3c0ff12..e5bdef7 100644 --- a/examples/request_blocking_to_gemtext_from_ast.rs +++ b/examples/request_blocking_to_gemtext_from_ast.rs @@ -1,94 +1,63 @@ -//! This example demonstrates a chain of Germ's capabilities by fetching a -//! Gemini capsule, parsing the response content into an abstract syntax tree, -//! and converting the abstract syntax tree back to Gemtext, identical to the -//! Gemini response content. +//! Fetches a Gemtext response and checks its AST round trip. use std::env; -fn main() { - // Try to obtain a URL from the command line arguments or use the default one +fn main() -> anyhow::Result<()> { let url_string = - env::args().nth(1).unwrap_or_else(|| "gemini://fuwn.me/".to_string()); - // Form a valid URL to a Gemini capsule - let url = match url::Url::parse(&url_string) { - Ok(url) => url, - Err(error) => { - eprintln!("Error parsing URL '{}': {}", url_string, error); - std::process::exit(1); - } - }; + env::args().nth(1).unwrap_or_else(|| "gemini://fuwn.me/".to_owned()); + let url = url::Url::parse(&url_string)?; + let response = germ::request::blocking::request(&url)?; - // Perform a blocking request to the Gemini capsule - let request = germ::request::blocking::request(&url); + anyhow::ensure!( + response.status().category() + == Some(germ::request::StatusCategory::Success), + "Gemini response was not successful" + ); - match request { - // If the request was successful: - Ok(response) => { - // Obtain the content of the Gemini response - let response_content = - &*response.content().clone().unwrap_or_else(|| "".to_string()); - // Parse the Gemini response content into an abstract syntax tree - let ast = germ::ast::Ast::from_string(response_content); - // Convert the abstract syntax tree back to Gemtext, identical to the - // Gemini response content, constructed from the parsed abstract syntax - // tree - let gemtext = ast.to_gemtext(); + let response_meta = response.meta(); + let mime_type = response_meta.split(';').next().unwrap_or_default().trim(); - // Print the Gemtext - println!("{}", gemtext); + anyhow::ensure!( + mime_type.eq_ignore_ascii_case("text/gemini"), + "Gemini response is not Gemtext" + ); - // Check if the response content and reconstruction are identical - if response_content == gemtext { - println!( - "\nValidation: Response content and reconstruction are identical" - ); - } else { - println!("\nValidation: Response content and reconstruction differ"); - print_diff(response_content, &gemtext); - } - } - // If the request was unsuccessful, print an error message and exit - Err(error) => { - eprintln!("Error fetching '{}': {}", url_string, error); - std::process::exit(1); - } - } -} + let response_bytes = response.content_bytes().unwrap_or_default(); + let original = std::str::from_utf8(response_bytes)?; + let reconstructed = germ::ast::Ast::from_owned(original).to_gemtext(); -fn print_diff(original: &str, reconstructed: &str) { - use std::io::{self, Write}; + print!("{reconstructed}"); - let mut stdout = io::stdout(); - let original_lines = original.lines().collect::>(); - let reconstructed_lines = reconstructed.lines().collect::>(); - let max_lines = original_lines.len().max(reconstructed_lines.len()); - let mut has_printed_diff = false; + if original != reconstructed { + print_diff(original, &reconstructed); + anyhow::bail!("Gemtext reconstruction differs from the response"); + } - for i in 0..max_lines { - let original_line = original_lines.get(i).unwrap_or(&""); - let reconstructed_line = reconstructed_lines.get(i).unwrap_or(&""); + eprintln!("Gemtext reconstruction matches the response"); - if original_line != reconstructed_line { - if has_printed_diff { - let _ = writeln!(stdout); - } + Ok(()) +} + +fn print_diff(original: &str, reconstructed: &str) { + let mut original_lines = original.split('\n'); + let mut reconstructed_lines = reconstructed.split('\n'); + let mut line_number = 1; - let _ = writeln!(stdout, "Line {}:", i + 1); - let _ = writeln!(stdout, " Original: '{}'", original_line); - let _ = writeln!(stdout, " Reconstructed: '{}'", reconstructed_line); + loop { + let original_line = original_lines.next(); + let reconstructed_line = reconstructed_lines.next(); - has_printed_diff = true; + if original_line.is_none() && reconstructed_line.is_none() { + break; } - } - if original_lines.len() != reconstructed_lines.len() { - if has_printed_diff { - let _ = writeln!(stdout); + if original_line != reconstructed_line { + eprintln!( + "Line {line_number}: original {original_line:?}, reconstructed \ + {reconstructed_line:?}" + ); } - let _ = writeln!(stdout, "Length difference:"); - let _ = writeln!(stdout, " Original: {} lines", original_lines.len()); - let _ = - writeln!(stdout, " Reconstructed: {} lines", reconstructed_lines.len()); + line_number += 1; } } From 12c44ea3c3f22603da817c69ec380cf64e1c31fb Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 06:09:04 +0000 Subject: [PATCH 16/19] feat(request): Add caller-owned TOFU certificate trust --- Cargo.toml | 2 +- README.md | 13 ++ src/request.rs | 8 +- src/request/blocking.rs | 62 +++++++++- src/request/non_blocking.rs | 61 +++++++++- src/request/trust.rs | 194 +++++++++++++++++++++++++++++- tests/request_tls.rs | 231 ++++++++++++++++++++++++++++++++++++ 7 files changed, 556 insertions(+), 15 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 836c5ce..100e13a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ required-features = ["ast", "blocking"] [dependencies] anyhow = { version = "1.0.98", optional = true } # `Result` -rustls = { version = "0.21.12", optional = true } # TLS +rustls = { version = "0.21.12", optional = true, features = ["dangerous_configuration"] } # TLS webpki-roots = { version = "1.0.9", optional = true } # CA certificates tokio-rustls = { version = "0.24.1", optional = true } # Non-blocking TLS tokio = { version = "1.45.1", optional = true, default-features = false, features = [ diff --git a/README.md b/README.md index 0cc127b..4cefc53 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,19 @@ the header. To change these limits or the trusted roots, set the fields of `request::non_blocking::request_with_options`. The blocking system DNS lookup may outlast the timeout before it returns. +For trust on first use, implement `request::CertificateStore` with durable +storage keyed by hostname and port, then pass it to +`blocking::request_with_tofu` or `non_blocking::request_with_tofu` alongside +`RequestOptions`. These methods verify the presented certificate and complete +the TLS handshake before saving a first-use certificate or checking an existing +one, and before sending the URL. They reject an unexpected certificate change +until the stored certificate expires. An expired certificate is replaced after +the new one passes validation. The store must persist each update before +returning; callers can remove a pin deliberately when changing trust early. +Store methods run synchronously, including in the async request path. +TOFU uses the timeout and response size limits from `RequestOptions` but +ignores its `root_certificates` field. + HTML conversion escapes Gemtext content. HTML and Markdown conversion emit links for relative targets and Gemini, Gopher, HTTP, HTTPS, mailto, and FTP URLs; other schemes are shown as text. Markdown conversion also escapes link diff --git a/src/request.rs b/src/request.rs index f852daa..21ededb 100644 --- a/src/request.rs +++ b/src/request.rs @@ -3,6 +3,7 @@ use std::time::Duration; const MAX_REQUEST_URL_BYTES: usize = 1024; +const DEFAULT_GEMINI_PORT: u16 = 1965; mod response; mod status; @@ -16,13 +17,14 @@ mod trust; pub use { response::Response, status::{Status, StatusCategory}, - trust::default_root_certificates, + trust::{CertificateStore, default_root_certificates}, }; -/// Configure both blocking and async Gemini requests. +/// Configures time and size limits for all requests and roots for CA trust. #[derive(Clone, Debug)] pub struct RequestOptions { - /// Set trusted certificate authorities; defaults to the Mozilla CA roots. + /// Sets trusted certificate authorities for CA-verified requests. + /// Defaults to Mozilla roots; TOFU requests ignore this field. pub root_certificates: rustls::RootCertStore, /// Limit elapsed time for the request; defaults to 30 seconds. pub timeout: Duration, diff --git a/src/request/blocking.rs b/src/request/blocking.rs index f9b0742..8fde6e5 100644 --- a/src/request/blocking.rs +++ b/src/request/blocking.rs @@ -1,6 +1,7 @@ use { crate::request::{ - RequestOptions, Response, request_line, trust::client_config, + DEFAULT_GEMINI_PORT, RequestOptions, Response, request_line, + trust::{CertificateStore, check_tofu, client_config, tofu_client_config}, }, std::{ io::{self, Read, Write}, @@ -117,11 +118,56 @@ pub fn request_with_roots( pub fn request_with_options( url: &url::Url, options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + client_config(options.root_certificates.clone()), + |_, _, _| Ok(()), + ) +} + +/// Makes a request using a caller-owned trust-on-first-use certificate store. +/// +/// The first verified certificate is saved before the request URL is sent. +/// Later requests require the same certificate. An expired certificate may be +/// replaced; an unexpected change before expiry returns an error. The store +/// must persist the certificate for this policy to work across process runs. +/// TOFU uses the timeout and response size limits in `options` and ignores +/// `options.root_certificates`. +/// +/// # Errors +/// +/// Returns an error for TLS or pin validation, store failure, timeout, +/// oversized response, or malformed response. +pub fn request_with_tofu( + url: &url::Url, + store: &mut dyn CertificateStore, + options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + tofu_client_config(), + |certificate, host, port| check_tofu(store, host, port, certificate), + ) +} + +fn request_with_config( + url: &url::Url, + options: &RequestOptions, + config: rustls::ClientConfig, + check_certificate: impl FnOnce( + Option<&rustls::Certificate>, + &str, + u16, + ) -> anyhow::Result<()>, ) -> anyhow::Result { let request_line = request_line(url)?; let domain = url .domain() .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; + let port = url.port().unwrap_or(DEFAULT_GEMINI_PORT); let deadline = Instant::now() .checked_add(options.timeout) .ok_or_else(|| anyhow::anyhow!("Gemini request timeout is too large"))?; @@ -129,7 +175,7 @@ pub fn request_with_options( io::Error::new(io::ErrorKind::AddrNotAvailable, "No server address found"); let mut connected_stream = None; - for address in (domain, url.port().unwrap_or(1965)).to_socket_addrs()? { + for address in (domain, port).to_socket_addrs()? { match TcpStream::connect_timeout(&address, remaining_time(deadline)?) { Ok(stream) => { connected_stream = Some(stream); @@ -144,10 +190,16 @@ pub fn request_with_options( stream: connected_stream.ok_or(connection_error)?, deadline, }; - let mut connection = rustls::ClientConnection::new( - Arc::new(client_config(options.root_certificates.clone())), - domain.try_into()?, + let mut connection = + rustls::ClientConnection::new(Arc::new(config), domain.try_into()?)?; + + connection.complete_io(&mut stream)?; + check_certificate( + connection.peer_certificates().and_then(|chain| chain.first()), + domain, + port, )?; + let mut tls = rustls::Stream::new(&mut connection, &mut stream); tls.write_all(request_line.as_bytes())?; diff --git a/src/request/non_blocking.rs b/src/request/non_blocking.rs index 8ee6d46..26f5a4a 100644 --- a/src/request/non_blocking.rs +++ b/src/request/non_blocking.rs @@ -1,6 +1,7 @@ use { crate::request::{ - RequestOptions, Response, request_line, trust::client_config, + DEFAULT_GEMINI_PORT, RequestOptions, Response, request_line, + trust::{CertificateStore, check_tofu, client_config, tofu_client_config}, }, std::sync::Arc, tokio::io::{AsyncReadExt, AsyncWriteExt}, @@ -65,22 +66,72 @@ pub async fn request_with_roots( pub async fn request_with_options( url: &url::Url, options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + client_config(options.root_certificates.clone()), + |_, _, _| Ok(()), + ) + .await +} + +/// Makes a request using a caller-owned trust-on-first-use certificate store. +/// +/// The first verified certificate is saved before the request URL is sent. +/// Later requests require the same certificate. An expired certificate may be +/// replaced; an unexpected change before expiry returns an error. The store +/// must persist the certificate for this policy to work across process runs. +/// TOFU uses the timeout and response size limits in `options` and ignores +/// `options.root_certificates`. +/// +/// # Errors +/// +/// Returns an error for TLS or pin validation, store failure, timeout, +/// oversized response, or malformed response. +pub async fn request_with_tofu( + url: &url::Url, + store: &mut dyn CertificateStore, + options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + tofu_client_config(), + |certificate, host, port| check_tofu(store, host, port, certificate), + ) + .await +} + +async fn request_with_config( + url: &url::Url, + options: &RequestOptions, + config: rustls::ClientConfig, + check_certificate: impl FnOnce( + Option<&rustls::Certificate>, + &str, + u16, + ) -> anyhow::Result<()>, ) -> anyhow::Result { let request_line = request_line(url)?; let domain = url .domain() .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; let server_name = rustls::ServerName::try_from(domain)?; + let port = url.port().unwrap_or(DEFAULT_GEMINI_PORT); tokio::time::timeout(options.timeout, async { - let address = (domain, url.port().unwrap_or(1965)); + let address = (domain, port); let stream = tokio::net::TcpStream::connect(address).await?; - let connector = tokio_rustls::TlsConnector::from(Arc::new(client_config( - options.root_certificates.clone(), - ))); + let connector = tokio_rustls::TlsConnector::from(Arc::new(config)); let mut tls = connector.connect(server_name, stream).await?; let cipher_suite = tls.get_mut().1.negotiated_cipher_suite(); + check_certificate( + tls.get_ref().1.peer_certificates().and_then(|chain| chain.first()), + domain, + port, + )?; tls.write_all(request_line.as_bytes()).await?; let mut response_bytes = Vec::new(); diff --git a/src/request/trust.rs b/src/request/trust.rs index 837b832..33c06fb 100644 --- a/src/request/trust.rs +++ b/src/request/trust.rs @@ -1,4 +1,123 @@ -use rustls::{ClientConfig, OwnedTrustAnchor, RootCertStore}; +use { + rustls::{ + Certificate, CertificateError, ClientConfig, Error, OwnedTrustAnchor, + RootCertStore, ServerName, + client::{ServerCertVerified, ServerCertVerifier, WebPkiVerifier}, + }, + std::{sync::Arc, time::SystemTime}, +}; + +/// Stores exact server certificates by hostname and port across requests. +/// +/// Implementations should persist updates before returning from `save` and +/// coordinate concurrent writers to avoid silently replacing a changed pin. +/// Store methods run synchronously, including during async requests. +pub trait CertificateStore: Send { + /// Loads the previously trusted DER certificate, if any. + /// + /// # Errors + /// + /// Returns an error when the store cannot read the certificate. + fn load( + &mut self, + hostname: &str, + port: u16, + ) -> anyhow::Result>>; + + /// Saves the DER certificate as the trusted certificate for this server. + /// + /// # Errors + /// + /// Returns an error when the store cannot persist the certificate. + fn save( + &mut self, + hostname: &str, + port: u16, + certificate: &[u8], + ) -> anyhow::Result<()>; +} + +#[derive(Debug)] +struct TrustOnFirstUseVerifier; + +impl ServerCertVerifier for TrustOnFirstUseVerifier { + fn verify_server_cert( + &self, + end_entity: &Certificate, + intermediates: &[Certificate], + server_name: &ServerName, + scts: &mut dyn Iterator, + ocsp_response: &[u8], + now: SystemTime, + ) -> Result { + let mut roots = RootCertStore::empty(); + + roots.add(end_entity)?; + + // The presented leaf is the temporary trust anchor; rustls still checks + // its validity, server name, and handshake signature. + WebPkiVerifier::new(roots, None).verify_server_cert( + end_entity, + intermediates, + server_name, + scts, + ocsp_response, + now, + ) + } +} + +pub fn tofu_client_config() -> ClientConfig { + ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(Arc::new(TrustOnFirstUseVerifier)) + .with_no_client_auth() +} + +pub fn check_tofu( + store: &mut dyn CertificateStore, + hostname: &str, + port: u16, + certificate: Option<&Certificate>, +) -> anyhow::Result<()> { + let certificate = certificate.ok_or_else(|| { + anyhow::anyhow!("Gemini server did not provide a certificate") + })?; + + match store.load(hostname, port)? { + Some(previous) if previous == certificate.0 => Ok(()), + Some(previous) if previous_certificate_expired(&previous, hostname) => + store.save(hostname, port, &certificate.0), + Some(_) => + anyhow::bail!("Gemini server certificate changed for {hostname}:{port}"), + None => store.save(hostname, port, &certificate.0), + } +} + +fn previous_certificate_expired(certificate: &[u8], hostname: &str) -> bool { + let certificate = Certificate(certificate.to_vec()); + let Ok(server_name) = ServerName::try_from(hostname) else { + return false; + }; + let mut roots = RootCertStore::empty(); + + if roots.add(&certificate).is_err() { + return false; + } + + // Revalidate the stored certificate to detect expiry without parsing its + // validity dates separately. + let result = WebPkiVerifier::new(roots, None).verify_server_cert( + &certificate, + &[], + &server_name, + &mut std::iter::empty(), + &[], + SystemTime::now(), + ); + + matches!(result, Err(Error::InvalidCertificate(CertificateError::Expired))) +} /// Return the Mozilla CA roots used by the default request functions. /// @@ -30,3 +149,76 @@ pub fn client_config(roots: RootCertStore) -> ClientConfig { .with_root_certificates(roots) .with_no_client_auth() } + +#[cfg(test)] +mod tests { + use { + super::{CertificateStore, check_tofu}, + rustls::Certificate, + std::collections::HashMap, + }; + + #[derive(Default)] + struct MemoryCertificates(HashMap<(String, u16), Vec>); + + impl CertificateStore for MemoryCertificates { + fn load( + &mut self, + hostname: &str, + port: u16, + ) -> anyhow::Result>> { + Ok(self.0.get(&(hostname.to_owned(), port)).cloned()) + } + + fn save( + &mut self, + hostname: &str, + port: u16, + certificate: &[u8], + ) -> anyhow::Result<()> { + self.0.insert((hostname.to_owned(), port), certificate.to_vec()); + + Ok(()) + } + } + + #[test] + fn pins_are_scoped_to_hostname_and_port() { + let certificate = Certificate(vec![1, 2, 3]); + let other_certificate = Certificate(vec![4, 5, 6]); + let mut store = MemoryCertificates::default(); + + check_tofu(&mut store, "one.test", 1965, Some(&certificate)).unwrap(); + check_tofu(&mut store, "one.test", 1965, Some(&certificate)).unwrap(); + check_tofu(&mut store, "one.test", 1966, Some(&other_certificate)).unwrap(); + check_tofu(&mut store, "two.test", 1965, Some(&other_certificate)).unwrap(); + assert_eq!(store.0.len(), 3); + assert!( + check_tofu(&mut store, "one.test", 1965, Some(&other_certificate)) + .is_err() + ); + } + + #[test] + fn replaces_an_expired_certificate() { + let mut parameters = + rcgen::CertificateParams::new(vec!["localhost".to_owned()]); + + parameters.not_before = rcgen::date_time_ymd(2018, 1, 1); + parameters.not_after = rcgen::date_time_ymd(2019, 1, 1); + + let expired_certificate = rcgen::Certificate::from_params(parameters) + .unwrap() + .serialize_der() + .unwrap(); + let replacement = Certificate(vec![7, 8, 9]); + let mut store = MemoryCertificates::default(); + + store.0.insert(("localhost".to_owned(), 1965), expired_certificate); + check_tofu(&mut store, "localhost", 1965, Some(&replacement)).unwrap(); + assert_eq!( + store.0.get(&("localhost".to_owned(), 1965)), + Some(&replacement.0) + ); + } +} diff --git a/tests/request_tls.rs b/tests/request_tls.rs index 84fbf0a..063f8e6 100644 --- a/tests/request_tls.rs +++ b/tests/request_tls.rs @@ -6,6 +6,7 @@ use { StreamOwned, }, std::{ + collections::HashMap, io::{self, Read, Write}, net::TcpListener, sync::Arc, @@ -15,6 +16,34 @@ use { url::Url, }; +#[derive(Default)] +struct MemoryCertificates { + certificates: HashMap<(String, u16), Vec>, + reject_save: bool, +} + +impl germ::request::CertificateStore for MemoryCertificates { + fn load( + &mut self, + hostname: &str, + port: u16, + ) -> anyhow::Result>> { + Ok(self.certificates.get(&(hostname.to_owned(), port)).cloned()) + } + + fn save( + &mut self, + hostname: &str, + port: u16, + certificate: &[u8], + ) -> anyhow::Result<()> { + anyhow::ensure!(!self.reject_save, "Certificate store rejected save"); + self.certificates.insert((hostname.to_owned(), port), certificate.to_vec()); + + Ok(()) + } +} + fn assert_certificate_error(error: &anyhow::Error, expected: CertificateError) { let actual = error .downcast_ref::() @@ -33,6 +62,16 @@ fn assert_handshake_rejected(server: JoinHandle>>) { assert_ne!(error.kind(), io::ErrorKind::TimedOut); } +fn assert_no_request_sent(server: JoinHandle>>) { + match server.join().unwrap() { + Ok(bytes) => assert!(bytes.is_empty()), + Err(error) => assert!(matches!( + error.kind(), + io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset + )), + } +} + fn spawn_tls_server( certificate_name: &str, ) -> (Url, Certificate, JoinHandle>>) { @@ -100,6 +139,198 @@ fn options_with_certificate( options } +#[cfg(feature = "blocking")] +#[test] +fn blocking_tofu_remembers_first_certificate() { + let (url, certificate, server) = spawn_tls_server("localhost"); + let mut store = MemoryCertificates::default(); + let response = germ::request::blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .unwrap(); + + assert_eq!(response.content().as_deref(), Some("hello")); + assert_eq!( + store.certificates.get(&("localhost".to_owned(), url.port().unwrap())), + Some(&certificate.0) + ); + assert!(server.join().unwrap().is_ok()); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_tofu_accepts_a_previously_pinned_certificate() { + let (url, certificate, server) = spawn_tls_server("localhost"); + let mut store = MemoryCertificates::default(); + + store + .certificates + .insert(("localhost".to_owned(), url.port().unwrap()), certificate.0); + + let response = germ::request::blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .unwrap(); + + assert_eq!(response.content().as_deref(), Some("hello")); + assert!(server.join().unwrap().is_ok()); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_tofu_rejects_changed_certificate_before_sending_url() { + let (url, certificate, server) = spawn_tls_server("localhost"); + let different_certificate = + rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]) + .unwrap() + .serialize_der() + .unwrap(); + let mut store = MemoryCertificates::default(); + + store.certificates.insert( + ("localhost".to_owned(), url.port().unwrap()), + different_certificate.clone(), + ); + + let error = germ::request::blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .unwrap_err(); + + assert!(error.to_string().contains("certificate changed")); + assert_ne!(different_certificate, certificate.0); + assert_eq!( + store.certificates.get(&("localhost".to_owned(), url.port().unwrap())), + Some(&different_certificate) + ); + assert_no_request_sent(server); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_tofu_rejects_store_failure_before_sending_url() { + let (url, _, server) = spawn_tls_server("localhost"); + let mut store = + MemoryCertificates { reject_save: true, ..Default::default() }; + let error = germ::request::blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .unwrap_err(); + + assert!(error.to_string().contains("rejected save")); + assert_no_request_sent(server); +} + +#[cfg(feature = "blocking")] +#[test] +fn blocking_tofu_rejects_wrong_hostname() { + let (url, _, server) = spawn_tls_server("elsewhere.test"); + let mut store = MemoryCertificates::default(); + let error = germ::request::blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .unwrap_err(); + + assert_certificate_error(&error, CertificateError::NotValidForName); + assert!(store.certificates.is_empty()); + assert_handshake_rejected(server); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_tofu_remembers_first_certificate() { + let (url, certificate, server) = spawn_tls_server("localhost"); + let mut store = MemoryCertificates::default(); + let response = germ::request::non_blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .await + .unwrap(); + + assert_eq!(response.content().as_deref(), Some("hello")); + assert_eq!( + store.certificates.get(&("localhost".to_owned(), url.port().unwrap())), + Some(&certificate.0) + ); + assert!(server.join().unwrap().is_ok()); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_tofu_rejects_changed_certificate_before_sending_url() { + let (url, _, server) = spawn_tls_server("localhost"); + let different_certificate = + rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]) + .unwrap() + .serialize_der() + .unwrap(); + let mut store = MemoryCertificates::default(); + + store.certificates.insert( + ("localhost".to_owned(), url.port().unwrap()), + different_certificate, + ); + + let error = germ::request::non_blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("certificate changed")); + assert_no_request_sent(server); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_tofu_rejects_store_failure_before_sending_url() { + let (url, _, server) = spawn_tls_server("localhost"); + let mut store = + MemoryCertificates { reject_save: true, ..Default::default() }; + let error = germ::request::non_blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("rejected save")); + assert_no_request_sent(server); +} + +#[cfg(feature = "request")] +#[tokio::test] +async fn async_tofu_rejects_wrong_hostname() { + let (url, _, server) = spawn_tls_server("elsewhere.test"); + let mut store = MemoryCertificates::default(); + let error = germ::request::non_blocking::request_with_tofu( + &url, + &mut store, + &Default::default(), + ) + .await + .unwrap_err(); + + assert_certificate_error(&error, CertificateError::NotValidForName); + assert!(store.certificates.is_empty()); + assert_handshake_rejected(server); +} + #[cfg(feature = "blocking")] #[test] fn blocking_request_rejects_untrusted_certificate_before_sending_url() { From 2689cf10e7d7821c6d0de417537a33496fd9f084 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 09:47:43 +0000 Subject: [PATCH 17/19] fix(examples): Check live Gemtext round trips by AST nodes --- .../request_blocking_to_gemtext_from_ast.rs | 57 ++++++++++++++++--- justfile | 2 +- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/examples/request_blocking_to_gemtext_from_ast.rs b/examples/request_blocking_to_gemtext_from_ast.rs index e5bdef7..1a20b6a 100644 --- a/examples/request_blocking_to_gemtext_from_ast.rs +++ b/examples/request_blocking_to_gemtext_from_ast.rs @@ -1,12 +1,46 @@ //! Fetches a Gemtext response and checks its AST round trip. +//! This example's TOFU certificate store lasts only for one run. -use std::env; +use { + germ::request::{CertificateStore, RequestOptions}, + std::{collections::HashMap, env}, +}; + +#[derive(Default)] +struct MemoryCertificates(HashMap<(String, u16), Vec>); + +impl CertificateStore for MemoryCertificates { + fn load( + &mut self, + hostname: &str, + port: u16, + ) -> anyhow::Result>> { + Ok(self.0.get(&(hostname.to_owned(), port)).cloned()) + } + + fn save( + &mut self, + hostname: &str, + port: u16, + certificate: &[u8], + ) -> anyhow::Result<()> { + self.0.insert((hostname.to_owned(), port), certificate.to_vec()); + + Ok(()) + } +} fn main() -> anyhow::Result<()> { - let url_string = - env::args().nth(1).unwrap_or_else(|| "gemini://fuwn.me/".to_owned()); + let url_string = env::args() + .nth(1) + .unwrap_or_else(|| "gemini://geminiprotocol.net/".to_owned()); let url = url::Url::parse(&url_string)?; - let response = germ::request::blocking::request(&url)?; + let mut certificates = MemoryCertificates::default(); + let response = germ::request::blocking::request_with_tofu( + &url, + &mut certificates, + &RequestOptions::default(), + )?; anyhow::ensure!( response.status().category() @@ -24,16 +58,23 @@ fn main() -> anyhow::Result<()> { let response_bytes = response.content_bytes().unwrap_or_default(); let original = std::str::from_utf8(response_bytes)?; - let reconstructed = germ::ast::Ast::from_owned(original).to_gemtext(); + let ast = germ::ast::Ast::from_string(original); + let reconstructed = ast.to_gemtext(); + let reparsed = germ::ast::Ast::from_string(&reconstructed); - print!("{reconstructed}"); + anyhow::ensure!( + ast.inner() == reparsed.inner(), + "Gemtext reconstruction changed the parsed nodes" + ); if original != reconstructed { + eprintln!("Gemtext source formatting was normalised:"); print_diff(original, &reconstructed); - anyhow::bail!("Gemtext reconstruction differs from the response"); + } else { + eprintln!("Gemtext reconstruction matches the response exactly"); } - eprintln!("Gemtext reconstruction matches the response"); + print!("{reconstructed}"); Ok(()) } diff --git a/justfile b/justfile index e93776d..43a29bd 100644 --- a/justfile +++ b/justfile @@ -20,5 +20,5 @@ docs: example example: cargo run --example {{ example }} --all-features -diff url="gemini://fuwn.me/": +diff url="gemini://geminiprotocol.net/": cargo run --example request_blocking_to_gemtext_from_ast --all-features -- {{ url }} From 605eca27d52c7d7b81c4398d4eb009956c9c23b4 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Thu, 24 Sep 2026 09:47:52 +0000 Subject: [PATCH 18/19] fix(convert): Escape Markdown text and protect code fences --- src/convert/macros.rs | 13 +++---- src/convert/markdown.rs | 86 +++++++++++++++++++++++++++-------------- tests/convert.rs | 57 +++++++++++++++++++++++++-- 3 files changed, 116 insertions(+), 40 deletions(-) diff --git a/src/convert/macros.rs b/src/convert/macros.rs index c8866f6..8274e63 100644 --- a/src/convert/macros.rs +++ b/src/convert/macros.rs @@ -1,13 +1,13 @@ -/// Convert Gemtext into HTML +/// Converts Gemtext into HTML. /// /// # Examples /// /// ```rust -/// // Using a value /// assert_eq!( /// germ::gemini_to_html!("=> /to hello !"), /// "hello !
", /// ); +/// ``` #[macro_export] macro_rules! gemini_to_html { ($gemini:expr) => { @@ -24,16 +24,13 @@ macro_rules! gemini_to_html { }; } -/// Convert Gemtext into Markdown +/// Converts Gemtext into Markdown. /// /// # Examples /// /// ```rust -/// assert_eq!( -/// // Using a value -/// germ::gemini_to_md!("=> /to hello !"), -/// "[hello !](/to)\n", -/// ); +/// assert_eq!(germ::gemini_to_md!("=> /to hello !"), "[hello \\!](/to)\n",); +/// ``` #[macro_export] macro_rules! gemini_to_md { ($gemini:expr) => { diff --git a/src/convert/markdown.rs b/src/convert/markdown.rs index ca1a887..cc7db2b 100644 --- a/src/convert/markdown.rs +++ b/src/convert/markdown.rs @@ -1,7 +1,13 @@ use {super::safe_link_target, crate::ast::Node, std::fmt::Write}; -fn push_escaped_label(markdown: &mut String, label: &str) { - for character in label.chars() { +fn push_escaped_text(markdown: &mut String, text: &str) { + for (byte_index, character) in text.char_indices() { + if character == ' ' && (byte_index == 0 || byte_index + 1 == text.len()) { + markdown.push_str(" "); + + continue; + } + if character.is_control() { let _ = write!(markdown, "&#{};", u32::from(character)); @@ -10,7 +16,24 @@ fn push_escaped_label(markdown: &mut String, label: &str) { if matches!( character, - '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '&' | '|' | '~' + '\\' + | '`' + | '*' + | '_' + | '[' + | ']' + | '<' + | '>' + | '&' + | '|' + | '~' + | '!' + | '#' + | '+' + | '-' + | '=' + | '.' + | ')' ) { markdown.push('\\'); } @@ -41,52 +64,59 @@ pub fn convert(source: &[Node]) -> String { for node in source { match node { Node::Text(text) => { - let _ = writeln!(&mut markdown, "{text}"); + push_escaped_text(&mut markdown, text); + markdown.push('\n'); } - Node::Link { to, text } => { let label = text.as_deref().unwrap_or(to); if safe_link_target(to) { markdown.push('['); - push_escaped_label(&mut markdown, label); + push_escaped_text(&mut markdown, label); markdown.push_str("]("); push_escaped_destination(&mut markdown, to); markdown.push_str(")\n"); } else { - push_escaped_label(&mut markdown, label); + push_escaped_text(&mut markdown, label); markdown.push('\n'); } } - Node::Heading { level, text } => { - let _ = writeln!( - &mut markdown, - "{} {}", - match level { - 1 => "#", - 2 => "##", - 3 => "###", - _ => "", - }, - text - ); + markdown.push_str(match level { + 1 => "# ", + 2 => "## ", + 3 => "### ", + _ => " ", + }); + push_escaped_text(&mut markdown, text); + markdown.push('\n'); } - Node::List(items) => for item in items { - let _ = writeln!(&mut markdown, "- {item}"); + markdown.push_str("- "); + push_escaped_text(&mut markdown, item); + markdown.push('\n'); }, - Node::Blockquote(text) => { - let _ = writeln!(&mut markdown, "> {text}"); + markdown.push_str("> "); + push_escaped_text(&mut markdown, text); + markdown.push('\n'); } - Node::PreformattedText { alt_text, text } => { - markdown.push_str("```"); + let longest_tilde_run = text + .split(|character| character != '~') + .map(str::len) + .max() + .unwrap_or(0); + let fence = "~".repeat((longest_tilde_run + 1).max(3)); + + markdown.push_str(&fence); if let Some(alt_text) = alt_text { - markdown.push_str(alt_text); + markdown.push(' '); + markdown.extend( + alt_text.chars().filter(|character| !character.is_control()), + ); } markdown.push('\n'); @@ -96,9 +126,9 @@ pub fn convert(source: &[Node]) -> String { markdown.push('\n'); } - markdown.push_str("```\n"); + markdown.push_str(&fence); + markdown.push('\n'); } - Node::Whitespace => markdown.push('\n'), } } diff --git a/tests/convert.rs b/tests/convert.rs index 4b9e8fd..a0797e4 100644 --- a/tests/convert.rs +++ b/tests/convert.rs @@ -125,7 +125,56 @@ mod test { fn convert_from_string_to_markdown_single_link() { assert_eq!( from_string("=> /to hello !", &Target::Markdown), - "[hello !](/to)\n", + "[hello \\!](/to)\n", + ); + } + + #[test] + fn markdown_escapes_literal_text_in_every_node() { + let literal = "