diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 257dfc4..b9ae053 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -2,11 +2,7 @@ name: Check βœ… on: workflow_dispatch: push: - paths: - - "*" pull_request: - paths: - - "*" env: CARGO_TERM_COLOR: always jobs: @@ -14,17 +10,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout πŸ›’ - uses: actions/checkout@v3 - - name: Toolchain 🧰 - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - components: rustfmt, clippy - override: true - - name: Check βœ… - uses: actions-rs/cargo@v1 - continue-on-error: false - with: - command: check - args: --verbose + uses: actions/checkout@v7 + - name: Test πŸ§ͺ + run: | + cargo test + cargo test --all-features + cargo test --no-default-features + for feature in ast blocking convert macros meta quick request; do + cargo test --no-default-features --features "$feature" + done + - name: Clippy πŸ“Ž + run: cargo clippy --all-features --all-targets -- -D warnings diff --git a/Cargo.toml b/Cargo.toml index 85775e5..5730729 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "germ" -version = "0.4.8" +version = "0.5.0" authors = ["Fuwn "] edition = "2021" description = "The Ultimate Gemini Toolkit." @@ -17,25 +17,59 @@ rust-version = "1.83.0" [features] ast = [] -blocking = ["rustls", "url", "anyhow"] +blocking = ["rustls", "webpki-roots", "url", "anyhow"] convert = ["ast"] default = ["ast", "convert", "meta", "request"] macros = ["ast", "convert"] meta = [] -request = ["rustls", "url", "anyhow", "tokio", "tokio-rustls"] +request = ["rustls", "webpki-roots", "url", "anyhow", "tokio", "tokio-rustls"] quick = [] -example-gemtext = [] + +[[example]] +name = "ast" +required-features = ["ast"] + +[[example]] +name = "ast_to_gemtext" +required-features = ["ast"] + +[[example]] +name = "html" +required-features = ["convert"] + +[[example]] +name = "markdown" +required-features = ["convert"] + +[[example]] +name = "meta" +required-features = ["meta"] + +[[example]] +name = "request" +required-features = ["request"] + +[[example]] +name = "request_blocking" +required-features = ["blocking"] + +[[example]] +name = "request_blocking_to_gemtext_from_ast" +required-features = ["ast", "blocking"] [dependencies] anyhow = { version = "1.0.98", optional = true } # `Result` -rustls = { version = "0.21.12", features = [ - "dangerous_configuration", -], optional = true } # TLS +rustls = { version = "0.21.12", optional = true, features = ["dangerous_configuration"] } # TLS +webpki-roots = { version = "1.0.9", optional = true } # CA certificates tokio-rustls = { version = "0.24.1", optional = true } # Non-blocking TLS tokio = { version = "1.45.1", optional = true, default-features = false, features = [ "net", "io-util", "rt-multi-thread", "macros", + "time", ] } # Non-blocking I/O url = { version = "2.5.4", optional = true } # URL Validation + +[dev-dependencies] +rcgen = "=0.12.1" diff --git a/README.md b/README.md index 2f20c0a..8847552 100644 --- a/README.md +++ b/README.md @@ -4,41 +4,23 @@ [![docs.rs](https://docs.rs/germ/badge.svg)](https://docs.rs/germ) [![github.com](https://github.com/gemrest/germ/actions/workflows/check.yaml/badge.svg?branch=main)](https://github.com/gemrest/germ/actions/workflows/check.yaml) -The Ultimate Gemini Toolkit - -Germ is a toolkit for the Gemini protocol which aims to have a little something -for everyone. At the moment, Germ has **ZERO** dependencies (unless you use the -`request` feature), and Germ will continue to try its hardest to have as few -dependencies as possible. - -## Features - -- AST builder to easily construct and manipulate AST trees from raw Gemtext -- Converters to easily convert from Gemtext to markup formats such as HTML or - Markdown -- Blocking and non-blocking request suite -- Structured meta section manipulation -- And more! - -Check out the rest of the features in the Features section under Usage +Germ parses and generates Gemtext, converts it to HTML or Markdown, handles +response metadata, and makes Gemini requests. Its default features include +the request client and its network and TLS dependencies. Disable default +features to use `ast`, `convert`, `meta`, or `quick` without normal dependencies. ## Usage -Current version: -[![crates.io](https://img.shields.io/crates/v/germ.svg)](https://crates.io/crates/germ) - ```toml -# Cargo.toml +[dependencies] +germ = "0.5.0" +``` + +To select features without the default request client: +```toml [dependencies] -# To enable only the base (default) features: ast, convert, meta, request -# germ = "*" # Use current version show above! - -# To enable only certain features -[dependencies.germ] -version = "*" # Use current version show above! -default-features = false -features = ["ast"] # Enable the features you would like to use! +germ = { version = "0.5.0", default-features = false, features = ["ast"] } ``` ### Features @@ -47,20 +29,73 @@ features = ["ast"] # Enable the features you would like to use! | ---------- | --------------------------------------------------------------------- | | `default` | `ast`, `convert`, `meta`, `request` | | `ast` | Construct AST trees from raw Gemtext | -| `blocking` | Blocking equivalent of `request` | -| `convert` | Convert Gemtext to markup formats such as HTML or Markdown | -| `request` | Make Gemini requests, get sane, structured results | -| `meta` | Structure-ise a Gemini response's meta section | -| `macros` | Macros to aid with various Germ-related functionalities | -| `quick` | Tiny functions to create valid Gemtext elements from structured input | +| `blocking` | Make blocking Gemini requests | +| `convert` | Convert Gemtext to HTML or Markdown | +| `request` | Make asynchronous Gemini requests | +| `meta` | Parse and format Gemini response metadata | +| `macros` | Generate AST and converted output with macros | +| `quick` | Build individual Gemtext lines | + +Gemini requests verify server certificates against Mozilla CA roots by default. +For a self-signed capsule, add its certificate in DER form to a +`rustls::RootCertStore`, then pass that store to +`request::blocking::request_with_roots` or +`request::non_blocking::request_with_roots`. The store is a trust anchor, not an +exact certificate fingerprint pin. Save and review the certificate separately +before trusting it; the default request functions do not remember certificates +between calls. + +Requests time out after 30 seconds and have a 16 MiB response limit, including +the header. To change these limits or the trusted roots, set the fields of +`request::RequestOptions` and pass it to +`request::blocking::request_with_options` or +`request::non_blocking::request_with_options`. The blocking system DNS lookup +may outlast the timeout before it returns. + +For trust on first use, implement `request::CertificateStore` with durable +storage keyed by hostname and port, then pass it to +`blocking::request_with_tofu` or `non_blocking::request_with_tofu` alongside +`RequestOptions`. These methods verify the presented certificate and complete +the TLS handshake before saving a first-use certificate or checking an existing +one, and before sending the URL. They reject an unexpected certificate change +until the stored certificate expires. An expired certificate is replaced after +the new one passes validation. The store must persist each update before +returning; callers can remove a pin deliberately when changing trust early. +Store methods run synchronously, including in the async request path. +TOFU uses the timeout and response size limits from `RequestOptions` but +ignores its `root_certificates` field. + +`Ast::to_gemtext()` serialises parsed nodes and retains a consistent LF or CRLF +style and a final line ending. Mixed line endings and optional spaces after +Gemtext markers can be normalised, so exact source round trips are not +guaranteed. + +HTML conversion escapes Gemtext content. HTML and Markdown conversion emit +links for relative targets and Gemini, Gopher, HTTP, HTTPS, mailto, and FTP +URLs; other schemes are shown as text. Markdown conversion escapes literal +text, link labels, and destinations, and chooses a code fence that does not +close inside preformatted content. + +`Meta` parses quoted MIME parameters and quotes values as needed when formatting +them. + +`quick::link(url, label)` takes the URL first and an optional label second. + +Unknown Gemini status codes retain their numeric value. Use +`Status::category()` to inspect the response category indicated by the first +digit. ### Examples -Thoroughly commented examples can be found within the +Runnable examples can be found within the [`examples/`](https://github.com/gemrest/germ/tree/main/examples) directory. -Examples can be run by name using the example just task. -(e.g., `just example ast_to_gemtext`) +Run one with `just example ast_to_gemtext`. + +`just diff` fetches `gemini://geminiprotocol.net/` with a process-local TOFU +certificate store. The certificate pin is not retained between runs. The +example reports source formatting changes and checks that serialising and +reparsing preserves the parsed nodes. ## License diff --git a/examples/ast.rs b/examples/ast.rs index d7fb91f..984dd1a 100644 --- a/examples/ast.rs +++ b/examples/ast.rs @@ -2,13 +2,9 @@ //! abstract syntax tree. fn main() { - // Parse `EXAMPLE_GEMTEXT` into an abstract syntax tree - let ast = germ::ast::Ast::from_string(germ::EXAMPLE_GEMTEXT); - // Get the nodes of the abstract syntax tree - let ast_nodes = ast.inner(); + let ast = germ::ast::Ast::from_string(include_str!("example.gmi")); - // Print the abstract syntax tree nodes, one-by-one - for node in ast_nodes { + for node in ast.inner() { println!("{:?}", node); } } diff --git a/examples/ast_to_gemtext.rs b/examples/ast_to_gemtext.rs index 7bf4206..cc1695f 100644 --- a/examples/ast_to_gemtext.rs +++ b/examples/ast_to_gemtext.rs @@ -1,12 +1,8 @@ //! This example converts Gemtext into an abstract syntax tree and then back -//! into Gemtext, demonstrating both Germ's parsing and generation capabilities. +//! into Gemtext. fn main() { - // Parse `EXAMPLE_GEMTEXT` into an abstract syntax tree - let ast = germ::ast::Ast::from_string(germ::EXAMPLE_GEMTEXT); - // Convert the abstract syntax tree back to Gemtext - let gemtext = ast.to_gemtext(); + let ast = germ::ast::Ast::from_string(include_str!("example.gmi")); - // Print the Gemtext, identical to `EXAMPLE_GEMTEXT` - println!("{gemtext}"); + print!("{}", ast.to_gemtext()); } diff --git a/examples/convert.html b/examples/convert.html deleted file mode 100644 index cdfde66..0000000 --- a/examples/convert.html +++ /dev/null @@ -1,6 +0,0 @@ -
Here goes the pre-formatted text.
-
-This continues the pre-formatted text on a new line after a blank line.
-

This is a heading

This is some text.

This is more text after a blank line.

This is a sub-heading

This is a blockquote.

This is a sub-sub-heading.

This is a link to GemRest
/somewhere

That was a link without text.

\ No newline at end of file diff --git a/examples/convert.md b/examples/convert.md deleted file mode 100644 index ea553ee..0000000 --- a/examples/convert.md +++ /dev/null @@ -1,28 +0,0 @@ -```This is alt-text -Here goes the pre-formatted text. - -This continues the pre-formatted text on a new line after a blank line. -``` - -# This is a heading - -This is some text. - -This is more text after a blank line. - -- This is a single list item. -- This is the next list item. - -- This is a new list. -- This is the next item on the new list. - -## This is a sub-heading - -> This is a blockquote. - -### This is a sub-sub-heading. - -[This is a link to GemRest](gemini://gem.rest/) - - -That was a link without text. diff --git a/examples/example.gmi b/examples/example.gmi new file mode 100644 index 0000000..a2f61a9 --- /dev/null +++ b/examples/example.gmi @@ -0,0 +1,42 @@ +```This is alt-text +Here goes the pre-formatted text. + +This continues the pre-formatted text on a new line after a blank line. +``` + +# This is a heading + +This is some text. + +This is more text after a blank line. + +* This is a single list item. +* This is the next list item. + +* This is a new list. +* This is the next item on the new list. + +## This is a sub-heading + +> This is a blockquote. + +### This is a sub-sub-heading. + +=> gemini://gem.rest/ This is a link to GemRest +=> /somewhere + +```This is a preformatted block containing inner Gemtext. +=> gemini://fuwn.me/ This is a link. + +* This is a list item. + +> This is a blockquote. + +# This is a heading. + +## This is a sub-heading. + +### This is a sub-sub-heading. +``` + +That was a link without text. diff --git a/examples/html.rs b/examples/html.rs index 802bd8e..0dbca21 100644 --- a/examples/html.rs +++ b/examples/html.rs @@ -2,13 +2,10 @@ //! HTML. fn main() { - // Convert the Gemtext to HTML let html = germ::convert::from_string( - germ::EXAMPLE_GEMTEXT, + include_str!("example.gmi"), &germ::convert::Target::HTML, ); - // Write the HTML to a file - std::fs::write("examples/convert.html", html) - .expect("could not write to file"); + println!("{html}"); } diff --git a/examples/markdown.rs b/examples/markdown.rs index 64cfcd4..1e4fe1a 100644 --- a/examples/markdown.rs +++ b/examples/markdown.rs @@ -2,12 +2,10 @@ //! Markdown. fn main() { - // Convert the Gemtext to Markdown - let html = germ::convert::from_string( - germ::EXAMPLE_GEMTEXT, + let markdown = germ::convert::from_string( + include_str!("example.gmi"), &germ::convert::Target::Markdown, ); - // Write the Markdown to a file - std::fs::write("examples/convert.md", html).expect("could not write to file"); + println!("{markdown}"); } diff --git a/examples/meta.rs b/examples/meta.rs index 056102f..62b3a4e 100644 --- a/examples/meta.rs +++ b/examples/meta.rs @@ -2,19 +2,12 @@ //! sections. fn main() { - // Parse Gemini meta section into a structured meta representation - let meta = germ::meta::Meta::from_string("text/gemini; hi=2; hi2=string=2"); + let meta = germ::meta::Meta::from_string( + r#"text/gemini; charset=utf-8; title="two; parts""#, + ); - // Debug view of the structured meta representation println!("{:?}", meta); - - // Convert the structured meta representation back to a string, identical to - // the original meta section println!("{}", meta); - - // The MIME type of the meta section println!("{}", meta.mime()); - - // A debug view of the parameters of the meta section println!("{:?}", meta.parameters()); } diff --git a/examples/request.rs b/examples/request.rs index 8979fcd..9958f82 100644 --- a/examples/request.rs +++ b/examples/request.rs @@ -2,33 +2,15 @@ //! request to a Gemini capsule. #[tokio::main] -async fn main() { - // Form a valid URL to a Gemini capsule - let url = url::Url::parse("gemini://fuwn.me").unwrap(); - // Perform a non-blocking request to the Gemini capsule - let request = germ::request::request(&url).await; +async fn main() -> anyhow::Result<()> { + let url = url::Url::parse("gemini://fuwn.me")?; + let response = germ::request::request(&url).await?; - match request { - // If the request was successful, print a debug view of the response - Ok(response) => { - // Print the status of the response - println!("{:?}", response.status()); + println!("{:?}", response.status()); + println!("{}", response.meta()); + println!("{:?}", response.content()); + println!("{:?}", response.size()); + println!("{:?}", response.suite()); - // Print the meta string of the response - // - // More detailed meta usage can be found in the `meta` example - println!("{}", response.meta()); - - // Print the content of the response, if present - println!("{:?}", response.content()); - - // Print the size of the response - println!("{:?}", response.size()); - - // Print a debug view of the SSL suite used - println!("{:?}", response.suite()); - } - // If the request was unsuccessful, do nothing - Err(_) => {} - } + Ok(()) } diff --git a/examples/request_blocking.rs b/examples/request_blocking.rs index 72cedd3..a4dd5fe 100644 --- a/examples/request_blocking.rs +++ b/examples/request_blocking.rs @@ -1,33 +1,15 @@ //! This example demonstrates Germ's capabilities for performing a blocking //! request to a Gemini capsule. -fn main() { - // Form a valid URL to a Gemini capsule - let url = url::Url::parse("gemini://fuwn.me").unwrap(); - // Perform a blocking request to the Gemini capsule - let request = germ::request::blocking::request(&url); +fn main() -> anyhow::Result<()> { + let url = url::Url::parse("gemini://fuwn.me")?; + let response = germ::request::blocking::request(&url)?; - match request { - // If the request was successful, print a debug view of the response - Ok(response) => { - // Print the status of the response - println!("{:?}", response.status()); + println!("{:?}", response.status()); + println!("{}", response.meta()); + println!("{:?}", response.content()); + println!("{:?}", response.size()); + println!("{:?}", response.suite()); - // Print the meta string of the response - // - // More detailed meta usage can be found in the `meta` example - println!("{}", response.meta()); - - // Print the content of the response, if present - println!("{:?}", response.content()); - - // Print the size of the response - println!("{:?}", response.size()); - - // Print a debug view of the SSL suite used - println!("{:?}", response.suite()); - } - // If the request was unsuccessful, do nothing - Err(_) => {} - } + Ok(()) } diff --git a/examples/request_blocking_to_gemtext_from_ast.rs b/examples/request_blocking_to_gemtext_from_ast.rs index 3c0ff12..1a20b6a 100644 --- a/examples/request_blocking_to_gemtext_from_ast.rs +++ b/examples/request_blocking_to_gemtext_from_ast.rs @@ -1,94 +1,104 @@ -//! This example demonstrates a chain of Germ's capabilities by fetching a -//! Gemini capsule, parsing the response content into an abstract syntax tree, -//! and converting the abstract syntax tree back to Gemtext, identical to the -//! Gemini response content. - -use std::env; - -fn main() { - // Try to obtain a URL from the command line arguments or use the default one - let url_string = - env::args().nth(1).unwrap_or_else(|| "gemini://fuwn.me/".to_string()); - // Form a valid URL to a Gemini capsule - let url = match url::Url::parse(&url_string) { - Ok(url) => url, - Err(error) => { - eprintln!("Error parsing URL '{}': {}", url_string, error); - std::process::exit(1); - } - }; - - // Perform a blocking request to the Gemini capsule - let request = germ::request::blocking::request(&url); - - match request { - // If the request was successful: - Ok(response) => { - // Obtain the content of the Gemini response - let response_content = - &*response.content().clone().unwrap_or_else(|| "".to_string()); - // Parse the Gemini response content into an abstract syntax tree - let ast = germ::ast::Ast::from_string(response_content); - // Convert the abstract syntax tree back to Gemtext, identical to the - // Gemini response content, constructed from the parsed abstract syntax - // tree - let gemtext = ast.to_gemtext(); - - // Print the Gemtext - println!("{}", gemtext); - - // Check if the response content and reconstruction are identical - if response_content == gemtext { - println!( - "\nValidation: Response content and reconstruction are identical" - ); - } else { - println!("\nValidation: Response content and reconstruction differ"); - print_diff(response_content, &gemtext); - } - } - // If the request was unsuccessful, print an error message and exit - Err(error) => { - eprintln!("Error fetching '{}': {}", url_string, error); - std::process::exit(1); - } +//! Fetches a Gemtext response and checks its AST round trip. +//! This example's TOFU certificate store lasts only for one run. + +use { + germ::request::{CertificateStore, RequestOptions}, + std::{collections::HashMap, env}, +}; + +#[derive(Default)] +struct MemoryCertificates(HashMap<(String, u16), Vec>); + +impl CertificateStore for MemoryCertificates { + fn load( + &mut self, + hostname: &str, + port: u16, + ) -> anyhow::Result>> { + Ok(self.0.get(&(hostname.to_owned(), port)).cloned()) + } + + fn save( + &mut self, + hostname: &str, + port: u16, + certificate: &[u8], + ) -> anyhow::Result<()> { + self.0.insert((hostname.to_owned(), port), certificate.to_vec()); + + Ok(()) } } -fn print_diff(original: &str, reconstructed: &str) { - use std::io::{self, Write}; +fn main() -> anyhow::Result<()> { + let url_string = env::args() + .nth(1) + .unwrap_or_else(|| "gemini://geminiprotocol.net/".to_owned()); + let url = url::Url::parse(&url_string)?; + let mut certificates = MemoryCertificates::default(); + let response = germ::request::blocking::request_with_tofu( + &url, + &mut certificates, + &RequestOptions::default(), + )?; - let mut stdout = io::stdout(); - let original_lines = original.lines().collect::>(); - let reconstructed_lines = reconstructed.lines().collect::>(); - let max_lines = original_lines.len().max(reconstructed_lines.len()); - let mut has_printed_diff = false; + anyhow::ensure!( + response.status().category() + == Some(germ::request::StatusCategory::Success), + "Gemini response was not successful" + ); - for i in 0..max_lines { - let original_line = original_lines.get(i).unwrap_or(&""); - let reconstructed_line = reconstructed_lines.get(i).unwrap_or(&""); + let response_meta = response.meta(); + let mime_type = response_meta.split(';').next().unwrap_or_default().trim(); - if original_line != reconstructed_line { - if has_printed_diff { - let _ = writeln!(stdout); - } + anyhow::ensure!( + mime_type.eq_ignore_ascii_case("text/gemini"), + "Gemini response is not Gemtext" + ); - let _ = writeln!(stdout, "Line {}:", i + 1); - let _ = writeln!(stdout, " Original: '{}'", original_line); - let _ = writeln!(stdout, " Reconstructed: '{}'", reconstructed_line); + let response_bytes = response.content_bytes().unwrap_or_default(); + let original = std::str::from_utf8(response_bytes)?; + let ast = germ::ast::Ast::from_string(original); + let reconstructed = ast.to_gemtext(); + let reparsed = germ::ast::Ast::from_string(&reconstructed); - has_printed_diff = true; - } + anyhow::ensure!( + ast.inner() == reparsed.inner(), + "Gemtext reconstruction changed the parsed nodes" + ); + + if original != reconstructed { + eprintln!("Gemtext source formatting was normalised:"); + print_diff(original, &reconstructed); + } else { + eprintln!("Gemtext reconstruction matches the response exactly"); } - if original_lines.len() != reconstructed_lines.len() { - if has_printed_diff { - let _ = writeln!(stdout); + print!("{reconstructed}"); + + Ok(()) +} + +fn print_diff(original: &str, reconstructed: &str) { + let mut original_lines = original.split('\n'); + let mut reconstructed_lines = reconstructed.split('\n'); + let mut line_number = 1; + + loop { + let original_line = original_lines.next(); + let reconstructed_line = reconstructed_lines.next(); + + if original_line.is_none() && reconstructed_line.is_none() { + break; + } + + if original_line != reconstructed_line { + eprintln!( + "Line {line_number}: original {original_line:?}, reconstructed \ + {reconstructed_line:?}" + ); } - let _ = writeln!(stdout, "Length difference:"); - let _ = writeln!(stdout, " Original: {} lines", original_lines.len()); - let _ = - writeln!(stdout, " Reconstructed: {} lines", reconstructed_lines.len()); + line_number += 1; } } diff --git a/justfile b/justfile index e93776d..43a29bd 100644 --- a/justfile +++ b/justfile @@ -20,5 +20,5 @@ docs: example example: cargo run --example {{ example }} --all-features -diff url="gemini://fuwn.me/": +diff url="gemini://geminiprotocol.net/": cargo run --example request_blocking_to_gemtext_from_ast --all-features -- {{ url }} diff --git a/src/ast/container.rs b/src/ast/container.rs index ca4271a..b0dad49 100644 --- a/src/ast/container.rs +++ b/src/ast/container.rs @@ -1,6 +1,6 @@ -use {super::Node, std::fmt::Write}; +use super::Node; -/// An AST structure which contains an AST tree +/// A parsed Gemtext document. /// /// # Example /// @@ -9,11 +9,14 @@ use {super::Node, std::fmt::Write}; /// ``` #[derive(Debug, Clone, PartialEq, Eq)] pub struct Ast { - inner: Vec, + inner: Vec, + append_final_line_ending: bool, + line_ending: &'static str, + unclosed_preformatted: bool, } impl Ast { - /// Build an AST tree from Gemtext + /// Parses Gemtext into an AST. /// /// # Example /// @@ -21,133 +24,223 @@ impl Ast { /// let _ = germ::ast::Ast::from_string(r#"=> gemini://gem.rest/ GemRest"#); /// ``` #[must_use] - pub fn from_owned(value: &(impl AsRef + ?Sized)) -> Self { - Self::from_value(value.as_ref()) + pub fn from_string(value: impl AsRef) -> Self { + Self::parse(value.as_ref()) } - /// Build an AST tree from Gemtext - /// - /// # Example - /// - /// ```rust - /// let _ = germ::ast::Ast::from_string(r#"=> gemini://gem.rest/ GemRest"#); - /// ``` - #[must_use] - #[allow(clippy::needless_pass_by_value)] - pub fn from_string(value: impl Into) -> Self { - Self::from_value(&value.into()) - } + fn parse(source: &str) -> Self { + let mut line_ending = None; + let mut nodes = Vec::new(); + let mut list_items = Vec::new(); + let mut preformatted_alt_text: Option = None; + let mut preformatted_lines = Vec::new(); + let mut unclosed_preformatted = false; + + for line_with_ending in source.split_inclusive('\n') { + let line = if let Some(line) = line_with_ending.strip_suffix("\r\n") { + line_ending.get_or_insert("\r\n"); + + line + } else if let Some(line) = line_with_ending.strip_suffix('\n') { + line_ending.get_or_insert("\n"); + + line + } else { + line_with_ending + }; + + if preformatted_alt_text.is_some() { + if line.starts_with("```") { + let alt_text = preformatted_alt_text.take().unwrap(); + let text = if preformatted_lines.is_empty() { + String::new() + } else { + let mut text = preformatted_lines.join(line_ending.unwrap_or("\n")); - /// Build an AST tree from a value - /// - /// # Example - /// - /// ```rust - /// let _ = germ::ast::Ast::from_value(r#"=> gemini://gem.rest/ GemRest"#); - /// ``` - #[must_use] - pub fn from_value(value: &(impl ToString + ?Sized)) -> Self { - let mut ast = vec![]; - let mut in_preformatted = false; - let mut in_list = false; - let source = value.to_string(); - let mut lines = source.lines(); - - // Iterate over all lines in the Gemtext `source` - while let Some(line) = lines.next() { - // Evaluate the Gemtext line and append its AST node to the `ast` tree - ast.append(&mut Self::evaluate( - line, - &mut lines, - &mut in_preformatted, - &mut in_list, - )); - } + text.push_str(line_ending.unwrap_or("\n")); + + text + }; - if source.ends_with('\n') { - if let Some(last) = ast.last() { - if !matches!(last, Node::Whitespace) { - ast.push(Node::Whitespace); + nodes.push(Node::PreformattedText { + alt_text: (!alt_text.is_empty()).then_some(alt_text), + text, + }); + preformatted_lines.clear(); + } else { + preformatted_lines.push(line); } + + continue; + } + + if let Some(alt_text) = line.strip_prefix("```") { + flush_list(&mut nodes, &mut list_items); + + preformatted_alt_text = Some(alt_text.to_owned()); + + continue; + } + + if let Some(item) = line.strip_prefix("* ") { + list_items.push(item.to_owned()); + + continue; } + + flush_list(&mut nodes, &mut list_items); + nodes.push(parse_line(line)); + } + + flush_list(&mut nodes, &mut list_items); + + if let Some(alt_text) = preformatted_alt_text { + unclosed_preformatted = true; + + let mut text = preformatted_lines.join(line_ending.unwrap_or("\n")); + + if source.ends_with('\n') && !preformatted_lines.is_empty() { + text.push_str(line_ending.unwrap_or("\n")); + } + + nodes.push(Node::PreformattedText { + alt_text: (!alt_text.is_empty()).then_some(alt_text), + text, + }); } - Self { inner: ast } + Self { + inner: nodes, + append_final_line_ending: source.ends_with('\n') + && (!unclosed_preformatted || preformatted_lines.is_empty()), + line_ending: line_ending.unwrap_or("\n"), + unclosed_preformatted, + } } - /// Build an AST tree from a [`Vec`] of [`Node`]s + /// Builds an AST from a [`Vec`] of [`Node`]s. /// /// # Example /// /// ```rust - /// // This assertion converts the Gemtext "=> / Home\n" to an AST tree of one + /// // This assertion converts the Gemtext "=> / Home" to an AST tree of one /// // node, then converts the AST tree back to Gemtext, and compares it against /// // the original Gemtext. /// assert_eq!( /// germ::ast::Ast::from_nodes( - /// germ::gemini_to_ast!("=> / Home").inner().to_vec() + /// germ::ast::Ast::from_string("=> / Home").inner().to_vec() /// ) /// .to_gemtext(), /// "=> / Home" /// ); /// ``` #[must_use] - pub const fn from_nodes(nodes: Vec) -> Self { Self { inner: nodes } } + pub const fn from_nodes(nodes: Vec) -> Self { + Self { + inner: nodes, + append_final_line_ending: false, + line_ending: "\n", + unclosed_preformatted: false, + } + } + /// Serialises parsed nodes as Gemtext. A consistent line ending style and + /// final line ending are retained, but mixed line endings and optional + /// spaces after line markers can be normalised. #[must_use] pub fn to_gemtext(&self) -> String { let mut gemtext = String::new(); - for node in &self.inner { + for (index, node) in self.inner.iter().enumerate() { + if index > 0 { + gemtext.push_str(self.line_ending); + } + match node { - Node::Text(text) => { - let _ = writeln!(&mut gemtext, "{text}"); - } + Node::Text(text) => gemtext.push_str(text), + Node::Link { to, text } => { - let _ = writeln!( - &mut gemtext, - "=> {}{}", - to, - text.clone().map_or_else(String::new, |text| format!(" {text}")), - ); + gemtext.push_str("=>"); + + if !to.is_empty() { + gemtext.push(' '); + gemtext.push_str(to); + } + + if let Some(text) = text { + gemtext.push(' '); + gemtext.push_str(text); + } } + Node::Heading { level, text } => { - let _ = writeln!(&mut gemtext, "{} {}", "#".repeat(*level), text); + gemtext.push_str(&"#".repeat(*level)); + + if !text.is_empty() { + gemtext.push(' '); + gemtext.push_str(text); + } } + Node::List(items) => { - let _ = writeln!( - &mut gemtext, - "{}", - items - .iter() - .map(|i| format!("* {i}")) - .collect::>() - .join("\n"), - ); + for (item_index, item) in items.iter().enumerate() { + if item_index > 0 { + gemtext.push_str(self.line_ending); + } + + gemtext.push_str("* "); + gemtext.push_str(item); + } } + Node::Blockquote(text) => { - let _ = writeln!(&mut gemtext, "> {text}"); + gemtext.push('>'); + + if !text.is_empty() { + gemtext.push(' '); + gemtext.push_str(text); + } } + Node::PreformattedText { alt_text, text } => { - let _ = writeln!( - &mut gemtext, - "```{}\n{}```", - alt_text.clone().unwrap_or_default(), - text - ); + let is_unclosed_last = + index + 1 == self.inner.len() && self.unclosed_preformatted; + + gemtext.push_str("```"); + + if let Some(alt_text) = alt_text { + gemtext.push_str(alt_text); + } + + if is_unclosed_last { + if !text.is_empty() { + gemtext.push_str(self.line_ending); + gemtext.push_str(text); + } + } else { + gemtext.push_str(self.line_ending); + gemtext.push_str(text); + + if !text.is_empty() && !text.ends_with('\n') { + gemtext.push_str(self.line_ending); + } + + gemtext.push_str("```"); + } } - Node::Whitespace => gemtext.push('\n'), + + Node::Whitespace => {} } } - if gemtext.ends_with('\n') { - gemtext.pop(); + if self.append_final_line_ending { + gemtext.push_str(self.line_ending); } gemtext } - /// The actual AST of `Ast` + /// Returns the parsed nodes. /// /// # Example /// @@ -157,158 +250,56 @@ impl Ast { /// ``` #[must_use] pub const fn inner(&self) -> &Vec { &self.inner } +} - #[allow(clippy::too_many_lines)] - fn evaluate( - line: &str, - lines: &mut std::str::Lines<'_>, - in_preformatted: &mut bool, - in_list: &mut bool, - ) -> Vec { - let mut preformatted = String::new(); - let mut alt_text = String::new(); - let mut nodes = vec![]; - let mut line = line; - let mut list_items = vec![]; - - // Enter a not-so-infinite loop as sometimes, we may need to stay in an - // evaluation loop, e.g., multiline contexts: preformatted text, lists, etc. - loop { - // Match the first character of the Gemtext line to understand the line - // type - match line.get(0..1).unwrap_or("") { - "=" if !*in_preformatted => { - // If the Gemtext line starts with an "=" ("=>"), it is a link line, - // so splitting it up should be easy enough. - let line = line.get(2..).unwrap_or(""); - let mut split = line.split_whitespace(); - - nodes.push(Node::Link { - to: split.next().unwrap_or_default().to_string(), - text: { - let rest: Vec<&str> = split.collect(); - - if rest.is_empty() { None } else { Some(rest.join(" ")) } - }, - }); - - break; - } - "#" if !*in_preformatted => { - // If the Gemtext line starts with an "#", it is a heading, so let's - // find out how deep it goes. - let level = - line.trim_start().chars().take_while(|&c| c == '#').count(); - - nodes.push(Node::Heading { - level, - // Here, the text after the heading markers is safely extracted. - // `chars().skip()` is used to safely handle UTF-8 boundaries. - text: line - .chars() - .skip(level) - .collect::() - .trim_start() - .to_string(), - }); - - break; - } - "*" if !*in_preformatted => { - // If the Gemtext line starts with an asterisk, it is a list item, so - // let's enter a list context. - if !*in_list { - *in_list = true; - } - - list_items.push(line.get(1..).unwrap_or("").trim_start().to_string()); - - if let Some(next_line) = lines.next() { - line = next_line; - } else { - break; - } - } - ">" if !*in_preformatted => { - // If the Gemtext line starts with an ">", it is a blockquote, so - // let's just clip off the line identifier. - nodes.push(Node::Blockquote( - line.get(1..).unwrap_or("").trim_start().to_string(), - )); - - break; - } - "`" => { - // If the Gemtext line starts with a backtick, it's a preformatted - // toggle, so let's enter a preformatted text context. - *in_preformatted = !*in_preformatted; - - if *in_preformatted { - alt_text = line.get(3..).unwrap_or("").to_string(); - - if let Some(next_line) = lines.next() { - line = next_line; - } else { - break; - } - } else { - nodes.push(Node::PreformattedText { - alt_text: if alt_text.is_empty() { None } else { Some(alt_text) }, - text: preformatted, - }); - - break; - } - } - "" if !*in_preformatted => { - if line.is_empty() { - // If the line has nothing on it, it is a whitespace line, as long - // as we aren't in a preformatted line context. - nodes.push(Node::Whitespace); - } else { - nodes.push(Node::Text(line.to_string())); - } - - break; - } - // This as a catchall. It does a number of things. - _ => { - if *in_preformatted { - // If we are in a preformatted line context, add the line to the - // preformatted blocks content and increment the line. - let _ = writeln!(&mut preformatted, "{line}"); - - if let Some(next_line) = lines.next() { - line = next_line; - } else { - break; - } - } else { - // If we are in a list item and hit a catchall, that must mean that - // we encountered a line which is not a list line, so - // let's stop adding items to the list context. - if *in_list { - *in_list = false; +fn flush_list(nodes: &mut Vec, items: &mut Vec) { + if !items.is_empty() { + nodes.push(Node::List(std::mem::take(items))); + } +} - nodes.push(Node::Text(line.to_string())); +fn after_optional_whitespace(text: &str) -> &str { + match text.chars().next() { + Some(character) if character.is_whitespace() => + &text[character.len_utf8()..], + _ => text, + } +} - break; - } +fn parse_line(line: &str) -> Node { + if let Some(link_content) = line.strip_prefix("=>") { + return parse_link(link_content); + } - nodes.push(Node::Text(line.to_string())); + if line.starts_with('#') { + let level = line.bytes().take_while(|byte| *byte == b'#').count(); - break; - } - } - } + if level <= 3 { + return Node::Heading { + level, + text: after_optional_whitespace(&line[level..]).to_owned(), + }; } + } - if !list_items.is_empty() { - nodes.reverse(); - nodes.push(Node::List(list_items)); - nodes.reverse(); - } + if let Some(quote) = line.strip_prefix('>') { + return Node::Blockquote(after_optional_whitespace(quote).to_owned()); + } + + if line.is_empty() { Node::Whitespace } else { Node::Text(line.to_owned()) } +} - nodes +fn parse_link(link_content: &str) -> Node { + let target_and_label = link_content.trim_start_matches(char::is_whitespace); + let target_end = target_and_label + .find(char::is_whitespace) + .unwrap_or(target_and_label.len()); + let target = &target_and_label[..target_end]; + let label = + target_and_label[target_end..].trim_start_matches(char::is_whitespace); + + Node::Link { + to: target.to_owned(), + text: (!label.is_empty()).then(|| label.to_owned()), } } diff --git a/src/ast/macros.rs b/src/ast/macros.rs index 6f91ac4..66e81f1 100644 --- a/src/ast/macros.rs +++ b/src/ast/macros.rs @@ -1,15 +1,13 @@ -/// Convert Gemtext an `Ast` +/// Converts Gemtext into an `Ast`. /// /// # Examples /// /// ```rust -/// // Using a value /// assert_eq!( /// germ::gemini_to_ast!("=> / A link!").to_gemtext(), -/// // `to_gemtext` appends a newline to all responses, so let's make sure we -/// // account for that. /// "=> / A link!", /// ); +/// ``` #[macro_export] macro_rules! gemini_to_ast { ($gemini:expr) => { diff --git a/src/ast/node.rs b/src/ast/node.rs index af00e8a..c806095 100644 --- a/src/ast/node.rs +++ b/src/ast/node.rs @@ -150,7 +150,7 @@ pub enum Node { /// ``` text: String, }, - /// A whitespace line, a line which contains nothing but whitespace. + /// An empty line. Whitespace, } diff --git a/src/convert.rs b/src/convert.rs index c2a94d9..dc6c787 100644 --- a/src/convert.rs +++ b/src/convert.rs @@ -7,16 +7,35 @@ mod markdown; #[cfg(feature = "macros")] mod macros; -/// Different targets to convert Gemtext to +fn safe_link_target(target: &str) -> bool { + if target.is_empty() + || target.chars().any(|character| { + character.is_whitespace() || character.is_control() || character == '\\' + }) + { + return false; + } + + match target.find([':', '/', '?', '#']) { + Some(index) if target.as_bytes()[index] == b':' => matches!( + target[..index].to_ascii_lowercase().as_str(), + "gemini" | "gopher" | "http" | "https" | "mailto" | "ftp" + ), + _ => true, + } +} + +/// The available Gemtext conversion targets. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum Target { - /// Convert Gemtext to HTML + /// Converts Gemtext to HTML. Links outside the Gemini, Gopher, HTTP, HTTPS, + /// mailto, and FTP schemes are rendered as text. HTML, - /// Convert Gemtext to Markdown + /// Converts Gemtext to Markdown with the same link scheme policy as HTML. Markdown, } -/// Convert AST'd Gemtext into an alternative markup format. +/// Converts a Gemtext AST into another markup format. /// /// # Example /// @@ -36,7 +55,7 @@ pub fn from_ast(source: &Ast, target: &Target) -> String { } } -/// Convert raw Gemtext into an alternative markup format. +/// Converts raw Gemtext into another markup format. /// /// # Example /// @@ -49,9 +68,6 @@ pub fn from_ast(source: &Ast, target: &Target) -> String { /// ); /// ``` #[must_use] -pub fn from_string( - source: &(impl ToString + ?Sized), - target: &Target, -) -> String { - from_ast(&Ast::from_owned(&source.to_string()), target) +pub fn from_string(source: impl AsRef, target: &Target) -> String { + from_ast(&Ast::from_string(source), target) } diff --git a/src/convert/html.rs b/src/convert/html.rs index 5774954..e865989 100644 --- a/src/convert/html.rs +++ b/src/convert/html.rs @@ -1,51 +1,85 @@ -use {crate::ast::Node, std::fmt::Write}; +use {super::safe_link_target, crate::ast::Node}; + +fn push_escaped(html: &mut String, value: &str) { + for character in value.chars() { + match character { + '&' => html.push_str("&"), + '<' => html.push_str("<"), + '>' => html.push_str(">"), + '"' => html.push_str("""), + '\'' => html.push_str("'"), + _ => html.push(character), + } + } +} pub fn convert(source: &[Node]) -> String { let mut html = String::new(); - // Since we have an AST tree of the Gemtext, it is very easy to convert from - // this AST tree to an alternative markup format. for node in source { match node { Node::Text(text) => { - let _ = write!(&mut html, "

{text}

"); + html.push_str("

"); + push_escaped(&mut html, text); + html.push_str("

"); } + Node::Link { to, text } => { - let _ = write!( - &mut html, - "{}
", - to, - text.clone().unwrap_or_else(|| to.clone()) - ); + let label = text.as_deref().unwrap_or(to); + + if safe_link_target(to) { + html.push_str(""); + push_escaped(&mut html, label); + html.push_str("
"); + } else { + push_escaped(&mut html, label); + html.push_str("
"); + } } + Node::Heading { level, text } => { - let _ = write!( - &mut html, - "<{}>{}", - match level { - 1 => "h1", - 2 => "h2", - 3 => "h3", - _ => "p", - }, - text - ); + let tag = match level { + 1 => "h1", + 2 => "h2", + 3 => "h3", + _ => "p", + }; + + html.push('<'); + html.push_str(tag); + html.push('>'); + push_escaped(&mut html, text); + html.push_str("'); } + Node::List(items) => { - let _ = write!(&mut html, "
    "); + html.push_str("
      "); for item in items { - let _ = write!(&mut html, "
    • {item}
    • "); + html.push_str("
    • "); + push_escaped(&mut html, item); + html.push_str("
    • "); } - let _ = write!(&mut html, "
    "); + html.push_str("
"); } + Node::Blockquote(text) => { - let _ = write!(&mut html, "
{text}
"); + html.push_str("
"); + push_escaped(&mut html, text); + html.push_str("
"); } + Node::PreformattedText { text, .. } => { - let _ = write!(&mut html, "
{text}
"); + html.push_str("
");
+        push_escaped(&mut html, text);
+        html.push_str("
"); } + Node::Whitespace => {} } } diff --git a/src/convert/macros.rs b/src/convert/macros.rs index c8866f6..8274e63 100644 --- a/src/convert/macros.rs +++ b/src/convert/macros.rs @@ -1,13 +1,13 @@ -/// Convert Gemtext into HTML +/// Converts Gemtext into HTML. /// /// # Examples /// /// ```rust -/// // Using a value /// assert_eq!( /// germ::gemini_to_html!("=> /to hello !"), /// "hello !
", /// ); +/// ``` #[macro_export] macro_rules! gemini_to_html { ($gemini:expr) => { @@ -24,16 +24,13 @@ macro_rules! gemini_to_html { }; } -/// Convert Gemtext into Markdown +/// Converts Gemtext into Markdown. /// /// # Examples /// /// ```rust -/// assert_eq!( -/// // Using a value -/// germ::gemini_to_md!("=> /to hello !"), -/// "[hello !](/to)\n", -/// ); +/// assert_eq!(germ::gemini_to_md!("=> /to hello !"), "[hello \\!](/to)\n",); +/// ``` #[macro_export] macro_rules! gemini_to_md { ($gemini:expr) => { diff --git a/src/convert/markdown.rs b/src/convert/markdown.rs index d77434e..cc7db2b 100644 --- a/src/convert/markdown.rs +++ b/src/convert/markdown.rs @@ -1,46 +1,133 @@ -use {crate::ast::Node, std::fmt::Write}; +use {super::safe_link_target, crate::ast::Node, std::fmt::Write}; + +fn push_escaped_text(markdown: &mut String, text: &str) { + for (byte_index, character) in text.char_indices() { + if character == ' ' && (byte_index == 0 || byte_index + 1 == text.len()) { + markdown.push_str(" "); + + continue; + } + + if character.is_control() { + let _ = write!(markdown, "&#{};", u32::from(character)); + + continue; + } + + if matches!( + character, + '\\' + | '`' + | '*' + | '_' + | '[' + | ']' + | '<' + | '>' + | '&' + | '|' + | '~' + | '!' + | '#' + | '+' + | '-' + | '=' + | '.' + | ')' + ) { + markdown.push('\\'); + } + + markdown.push(character); + } +} + +fn push_escaped_destination(markdown: &mut String, target: &str) { + for character in target.chars() { + match character { + '(' | ')' => { + markdown.push('\\'); + markdown.push(character); + } + '<' => markdown.push_str("%3C"), + '>' => markdown.push_str("%3E"), + '"' => markdown.push_str("%22"), + '&' => markdown.push_str("&"), + _ => markdown.push(character), + } + } +} pub fn convert(source: &[Node]) -> String { let mut markdown = String::new(); - // Since we have an AST tree of the Gemtext, it is very easy to convert from - // this AST tree to an alternative markup format. for node in source { match node { Node::Text(text) => { - let _ = writeln!(&mut markdown, "{text}"); + push_escaped_text(&mut markdown, text); + markdown.push('\n'); + } + Node::Link { to, text } => { + let label = text.as_deref().unwrap_or(to); + + if safe_link_target(to) { + markdown.push('['); + push_escaped_text(&mut markdown, label); + markdown.push_str("]("); + push_escaped_destination(&mut markdown, to); + markdown.push_str(")\n"); + } else { + push_escaped_text(&mut markdown, label); + markdown.push('\n'); + } } - Node::Link { to, text } => markdown.push_str(&text.clone().map_or_else( - || format!("<{to}>\n"), - |text| format!("[{text}]({to})\n"), - )), Node::Heading { level, text } => { - let _ = writeln!( - &mut markdown, - "{} {}", - match level { - 1 => "#", - 2 => "##", - 3 => "###", - _ => "", - }, - text - ); + markdown.push_str(match level { + 1 => "# ", + 2 => "## ", + 3 => "### ", + _ => " ", + }); + push_escaped_text(&mut markdown, text); + markdown.push('\n'); } Node::List(items) => for item in items { - let _ = writeln!(&mut markdown, "- {item}"); + markdown.push_str("- "); + push_escaped_text(&mut markdown, item); + markdown.push('\n'); }, Node::Blockquote(text) => { - let _ = writeln!(&mut markdown, "> {text}"); + markdown.push_str("> "); + push_escaped_text(&mut markdown, text); + markdown.push('\n'); } Node::PreformattedText { alt_text, text } => { - let _ = writeln!( - &mut markdown, - "```{}\n{}```", - alt_text.clone().unwrap_or_default(), - text - ); + let longest_tilde_run = text + .split(|character| character != '~') + .map(str::len) + .max() + .unwrap_or(0); + let fence = "~".repeat((longest_tilde_run + 1).max(3)); + + markdown.push_str(&fence); + + if let Some(alt_text) = alt_text { + markdown.push(' '); + markdown.extend( + alt_text.chars().filter(|character| !character.is_control()), + ); + } + + markdown.push('\n'); + markdown.push_str(text); + + if !text.is_empty() && !text.ends_with('\n') { + markdown.push('\n'); + } + + markdown.push_str(&fence); + markdown.push('\n'); } Node::Whitespace => markdown.push('\n'), } diff --git a/src/lib.rs b/src/lib.rs index 5503a7d..b4ea1f0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,52 +15,8 @@ #[cfg(feature = "convert")] pub mod convert; -#[cfg(feature = "request")] pub mod request; +#[cfg(any(feature = "request", feature = "blocking"))] pub mod request; #[cfg(feature = "meta")] pub mod meta; #[cfg(feature = "quick")] pub mod quick; - -#[cfg(feature = "example-gemtext")] -pub const EXAMPLE_GEMTEXT: &str = r"```This is alt-text -Here goes the pre-formatted text. - -This continues the pre-formatted text on a new line after a blank line. -``` - -# This is a heading - -This is some text. - -This is more text after a blank line. - -* This is a single list item. -* This is the next list item. - -* This is a new list. -* This is the next item on the new list. - -## This is a sub-heading - -> This is a blockquote. - -### This is a sub-sub-heading. - -=> gemini://gem.rest/ This is a link to GemRest -=> /somewhere - -```This is a preformatted block containing inner Gemtext. -=> gemini://fuwn.me/ This is a link. - -* This is a list item. - -> This is a blockquote. - -# This is a heading. - -## This is a sub-heading. - -### This is a sub-sub-heading. -``` - -That was a link without text."; diff --git a/src/meta.rs b/src/meta.rs index 75452c2..e0d4f09 100644 --- a/src/meta.rs +++ b/src/meta.rs @@ -1,12 +1,60 @@ use std::{borrow::Cow, collections::HashMap, fmt::Display}; -/// Structure-ize a Gemini response's meta section into it's mime type and it's -/// parameters. +const MIME_SPECIAL_CHARACTERS: &[u8] = b"()<>@,;:\\\"/[]?="; + +fn is_mime_token(value: &str) -> bool { + !value.is_empty() + && value.bytes().all(|byte| { + byte.is_ascii_graphic() && !MIME_SPECIAL_CHARACTERS.contains(&byte) + }) +} + +fn parse_parameter(parameter: &str, parameters: &mut HashMap) { + let Some((name, value)) = parameter.trim().split_once('=') else { + return; + }; + let name = name.trim(); + let value = value.trim(); + + if !is_mime_token(name) { + return; + } + + let value = if let Some(quoted) = value.strip_prefix('"') { + let Some(quoted) = quoted.strip_suffix('"') else { + return; + }; + let mut unescaped = String::with_capacity(quoted.len()); + let mut characters = quoted.chars(); + + while let Some(character) = characters.next() { + if character == '\\' { + let Some(escaped) = characters.next() else { + return; + }; + + unescaped.push(escaped); + } else if character == '"' { + return; + } else { + unescaped.push(character); + } + } + + unescaped + } else { + value.to_owned() + }; + + parameters.insert(name.to_owned(), value); +} + +/// Structures a Gemini response's MIME type and parameters. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct Meta { - /// The mime type of a Gemini response + /// The MIME type of a Gemini response. mime: String, - /// The parameters of a Gemini response + /// The parameters of a Gemini response. parameters: HashMap, } @@ -17,16 +65,32 @@ impl Display for Meta { if !self.parameters.is_empty() { write!(f, "; ")?; - let mut parameters: Vec<_> = self.parameters.iter().collect(); + let mut sorted_parameters: Vec<_> = self.parameters.iter().collect(); - parameters.sort_by(|a, b| a.0.cmp(b.0)); + sorted_parameters.sort_by(|left, right| left.0.cmp(right.0)); - for (i, (key, value)) in parameters.iter().enumerate() { - if i > 0 { + for (index, (key, value)) in sorted_parameters.iter().enumerate() { + if index > 0 { write!(f, "; ")?; } - write!(f, "{key}={value}")?; + write!(f, "{key}=")?; + + if is_mime_token(value) { + write!(f, "{value}")?; + } else { + write!(f, "\"")?; + + for character in value.chars() { + if matches!(character, '"' | '\\') { + write!(f, "\\")?; + } + + write!(f, "{character}")?; + } + + write!(f, "\"")?; + } } } @@ -35,17 +99,18 @@ impl Display for Meta { } impl Meta { - /// Create a new `Meta` + /// Creates an empty `Meta`. /// /// # Example /// /// ```rust - /// let mut meta = germ::meta::Meta::new(); + /// let meta = germ::meta::Meta::new(); + /// assert!(meta.parameters().is_empty()); /// ``` #[must_use] pub fn new() -> Self { Self::default() } - /// Create a `Meta` from a string + /// Parses a MIME type and its semicolon-separated parameters. /// /// # Example /// @@ -57,27 +122,44 @@ impl Meta { /// ``` #[must_use] pub fn from_string<'a>(meta: impl Into>) -> Self { - let meta = meta.into().to_string(); - let mut metas = meta.split(';'); - let mime = metas.next().unwrap_or("").to_string(); + let meta = meta.into(); + let (mime, parameters_source) = meta.split_once(';').unwrap_or((&meta, "")); + let mime = mime.to_owned(); let mut parameters = HashMap::new(); + let mut parameter_start = 0; + let mut quoted = false; + let mut escaped = false; - for parameter in metas { - let trimmed = parameter.trim_start(); + for (index, character) in parameters_source.char_indices() { + if escaped { + escaped = false; - // Only parse parameters containing '=' as those without are malformed - // according to RFC 2045 - if let Some(equal_pos) = trimmed.find('=') { - let (key, value) = trimmed.split_at(equal_pos); + continue; + } + + match character { + '\\' if quoted => escaped = true, + '"' => quoted = !quoted, + ';' if !quoted => { + parse_parameter( + ¶meters_source[parameter_start..index], + &mut parameters, + ); - parameters.insert(key.to_string(), value[1..].to_string()); + parameter_start = index + 1; + } + _ => {} } } + if !quoted { + parse_parameter(¶meters_source[parameter_start..], &mut parameters); + } + Self { mime, parameters } } - /// Obtain non-mutable access to the mime of the `Meta` + /// Returns the MIME type. /// /// # Example /// @@ -91,7 +173,7 @@ impl Meta { #[must_use] pub fn mime(&self) -> Cow<'_, str> { Cow::Borrowed(&self.mime) } - /// Obtain mutable access to the mime of the `Meta` + /// Returns mutable access to the MIME type. /// /// # Example /// @@ -102,7 +184,7 @@ impl Meta { /// ``` pub const fn mime_mut(&mut self) -> &mut String { &mut self.mime } - /// Obtain non-mutable access to the parameters of the `Meta` + /// Returns the parameters. /// /// # Example /// @@ -119,7 +201,7 @@ impl Meta { &self.parameters } - /// Obtain mutable access to the parameters of the `Meta` + /// Returns mutable access to the parameters. /// /// # Example /// diff --git a/src/quick.rs b/src/quick.rs index 76dd402..0b2a87f 100644 --- a/src/quick.rs +++ b/src/quick.rs @@ -27,20 +27,33 @@ pub fn list_item(text: &(impl ToString + ?Sized)) -> String { #[must_use] pub fn list_items(items: &[&(impl ToString + ?Sized)]) -> String { - items - .iter() - .map(|item| list_item(&item.to_string())) - .collect::>() - .join("\n") + let mut gemtext = String::new(); + + for (index, item) in items.iter().enumerate() { + if index > 0 { + gemtext.push('\n'); + } + + gemtext.push_str("* "); + gemtext.push_str(&item.to_string()); + } + + gemtext } +/// Builds a link line with a URL and an optional label. #[must_use] -pub fn link(text: &(impl ToString + ?Sized), location: Option<&str>) -> String { - format!( - "=> {}{}", - text.to_string(), - location.map_or_else(String::new, |l| format!(" {l}")) - ) +pub fn link(url: &(impl ToString + ?Sized), label: Option<&str>) -> String { + let mut gemtext = String::from("=> "); + + gemtext.push_str(&url.to_string()); + + if let Some(label) = label { + gemtext.push(' '); + gemtext.push_str(label); + } + + gemtext } #[must_use] diff --git a/src/request.rs b/src/request.rs index 1d76a9d..21ededb 100644 --- a/src/request.rs +++ b/src/request.rs @@ -1,13 +1,144 @@ -//! Make Gemini requests and get sane, structured results +//! Make Gemini requests and get structured results. + +use std::time::Duration; + +const MAX_REQUEST_URL_BYTES: usize = 1024; +const DEFAULT_GEMINI_PORT: u16 = 1965; mod response; mod status; -mod verifier; +mod trust; #[cfg(feature = "blocking")] pub mod blocking; #[cfg(feature = "request")] pub mod non_blocking; #[cfg(feature = "request")] pub use non_blocking::request; -pub(crate) use verifier::GermVerifier; -pub use {response::Response, status::Status}; +pub use { + response::Response, + status::{Status, StatusCategory}, + trust::{CertificateStore, default_root_certificates}, +}; + +/// Configures time and size limits for all requests and roots for CA trust. +#[derive(Clone, Debug)] +pub struct RequestOptions { + /// Sets trusted certificate authorities for CA-verified requests. + /// Defaults to Mozilla roots; TOFU requests ignore this field. + pub root_certificates: rustls::RootCertStore, + /// Limit elapsed time for the request; defaults to 30 seconds. + pub timeout: Duration, + /// Limit response bytes, including the header; defaults to 16 MiB. + pub max_response_bytes: usize, +} + +impl RequestOptions { + const fn with_roots(root_certificates: rustls::RootCertStore) -> Self { + Self { + root_certificates, + timeout: Duration::from_secs(30), + max_response_bytes: 16 * 1024 * 1024, + } + } +} + +impl Default for RequestOptions { + fn default() -> Self { Self::with_roots(default_root_certificates()) } +} + +pub(crate) fn request_line(url: &url::Url) -> anyhow::Result { + if url.scheme() == "gemini" { + anyhow::ensure!( + url.username().is_empty() && url.password().is_none(), + "Gemini URLs cannot contain user information" + ); + } + + let mut request_url = url.clone(); + + request_url.set_fragment(None); + + let url_bytes = request_url.as_str().as_bytes(); + + anyhow::ensure!( + url_bytes.len() <= MAX_REQUEST_URL_BYTES, + "Gemini URL exceeds {MAX_REQUEST_URL_BYTES} bytes" + ); + anyhow::ensure!( + !url_bytes.contains(&b'\r') && !url_bytes.contains(&b'\n'), + "Gemini URL contains a line break" + ); + + Ok(format!("{request_url}\r\n")) +} + +#[cfg(test)] +mod tests { + use { + super::{MAX_REQUEST_URL_BYTES, request_line}, + url::Url, + }; + + #[test] + fn request_line_omits_fragment() { + let url = Url::parse("gemini://example.com/path?query#fragment").unwrap(); + + assert_eq!( + request_line(&url).unwrap(), + "gemini://example.com/path?query\r\n" + ); + } + + #[test] + fn request_line_enforces_url_byte_limit() { + let prefix = "gemini://example.com/"; + let accepted = Url::parse(&format!( + "{prefix}{}", + "a".repeat(MAX_REQUEST_URL_BYTES - prefix.len()) + )) + .unwrap(); + let rejected = Url::parse(&format!( + "{prefix}{}", + "a".repeat(MAX_REQUEST_URL_BYTES + 1 - prefix.len()) + )) + .unwrap(); + + assert_eq!( + request_line(&accepted).unwrap().len(), + MAX_REQUEST_URL_BYTES + b"\r\n".len() + ); + assert!(request_line(&rejected).is_err()); + } + + #[test] + fn request_line_counts_encoded_unicode_bytes() { + let prefix = "gemini://example.com/"; + let encoded_character_bytes = "%C3%A9".len(); + let accepted = Url::parse(&format!( + "{prefix}{}Γ©", + "a" + .repeat(MAX_REQUEST_URL_BYTES - prefix.len() - encoded_character_bytes) + )) + .unwrap(); + let rejected = Url::parse(&format!( + "{prefix}{}Γ©", + "a".repeat( + MAX_REQUEST_URL_BYTES + 1 - prefix.len() - encoded_character_bytes + ) + )) + .unwrap(); + + assert_eq!( + request_line(&accepted).unwrap().len(), + MAX_REQUEST_URL_BYTES + b"\r\n".len() + ); + assert!(request_line(&rejected).is_err()); + } + + #[test] + fn request_line_rejects_user_information() { + let url = Url::parse("gemini://user:secret@example.com/").unwrap(); + + assert!(request_line(&url).is_err()); + } +} diff --git a/src/request/blocking.rs b/src/request/blocking.rs index 5e2947a..8fde6e5 100644 --- a/src/request/blocking.rs +++ b/src/request/blocking.rs @@ -1,50 +1,218 @@ use { - crate::request::{GermVerifier, Response}, - std::io::{Read, Write}, + crate::request::{ + DEFAULT_GEMINI_PORT, RequestOptions, Response, request_line, + trust::{CertificateStore, check_tofu, client_config, tofu_client_config}, + }, + std::{ + io::{self, Read, Write}, + net::{TcpStream, ToSocketAddrs}, + sync::Arc, + time::{Duration, Instant}, + }, }; +// Recompute the timeout before each TLS read and write so partial I/O shares +// one request deadline. +struct DeadlineStream { + stream: TcpStream, + deadline: Instant, +} + +impl Read for DeadlineStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let remaining_duration = remaining_time(self.deadline)?; + + self.stream.set_read_timeout(Some(remaining_duration))?; + + self.stream.read(buffer).map_err(normalize_timeout) + } +} + +impl Write for DeadlineStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + let remaining_duration = remaining_time(self.deadline)?; + + self.stream.set_write_timeout(Some(remaining_duration))?; + + self.stream.write(buffer).map_err(normalize_timeout) + } + + fn flush(&mut self) -> io::Result<()> { self.stream.flush() } +} + +fn remaining_time(deadline: Instant) -> io::Result { + deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or_else(timeout_error) +} + +fn timeout_error() -> io::Error { + io::Error::new(io::ErrorKind::TimedOut, "Gemini request timed out") +} + +fn normalize_timeout(error: io::Error) -> io::Error { + if matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut) + { + timeout_error() + } else { + error + } +} + /// Make a request to a Gemini server. The `url` **should** be prefixed with a /// scheme (e.g. "gemini://"). /// /// # Example /// -/// ```rust -/// match germ::request::blocking::request( -/// &url::Url::parse("gemini://fuwn.me").unwrap(), -/// ) { -/// Ok(response) => println!("{:?}", response), -/// Err(_) => {} +/// ```rust,no_run +/// fn main() -> anyhow::Result<()> { +/// let url = url::Url::parse("gemini://fuwn.me")?; +/// let response = germ::request::blocking::request(&url)?; +/// +/// println!("{response:?}"); +/// +/// Ok(()) /// } /// ``` /// /// # Errors -/// - May error if the URL is invalid -/// - May error if the TLS write fails -/// - May error if the TLS read fails +/// +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. pub fn request(url: &url::Url) -> anyhow::Result { - let config = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(std::sync::Arc::new(GermVerifier::new())) - .with_no_client_auth(); + request_with_options(url, &RequestOptions::default()) +} + +/// Make a request using the supplied trusted certificate authorities. +/// +/// The supplied roots replace the default Mozilla roots. To keep those roots, +/// start with `default_root_certificates()` before adding a capsule's DER +/// certificate with `RootCertStore::add`. Added certificates become trust +/// anchors, not exact fingerprint pins. The server name and certificate +/// validity period are still checked. Default time and size limits apply. +/// +/// # Errors +/// +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. +pub fn request_with_roots( + url: &url::Url, + roots: &rustls::RootCertStore, +) -> anyhow::Result { + let options = RequestOptions::with_roots(roots.clone()); + + request_with_options(url, &options) +} + +/// Make a request with the supplied certificate, time, and size limits. +/// +/// The timeout covers connection, TLS, sending, and receiving. Hostname lookup +/// uses the system resolver and may outlast the timeout before returning. +/// The response limit includes the header and rejects an oversized response. +/// +/// # Errors +/// +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. +pub fn request_with_options( + url: &url::Url, + options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + client_config(options.root_certificates.clone()), + |_, _, _| Ok(()), + ) +} + +/// Makes a request using a caller-owned trust-on-first-use certificate store. +/// +/// The first verified certificate is saved before the request URL is sent. +/// Later requests require the same certificate. An expired certificate may be +/// replaced; an unexpected change before expiry returns an error. The store +/// must persist the certificate for this policy to work across process runs. +/// TOFU uses the timeout and response size limits in `options` and ignores +/// `options.root_certificates`. +/// +/// # Errors +/// +/// Returns an error for TLS or pin validation, store failure, timeout, +/// oversized response, or malformed response. +pub fn request_with_tofu( + url: &url::Url, + store: &mut dyn CertificateStore, + options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + tofu_client_config(), + |certificate, host, port| check_tofu(store, host, port, certificate), + ) +} + +fn request_with_config( + url: &url::Url, + options: &RequestOptions, + config: rustls::ClientConfig, + check_certificate: impl FnOnce( + Option<&rustls::Certificate>, + &str, + u16, + ) -> anyhow::Result<()>, +) -> anyhow::Result { + let request_line = request_line(url)?; let domain = url .domain() .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; - let mut connection = rustls::ClientConnection::new( - std::sync::Arc::new(config), - domain.try_into()?, - )?; - let mut stream = std::net::TcpStream::connect(format!( - "{}:{}", + let port = url.port().unwrap_or(DEFAULT_GEMINI_PORT); + let deadline = Instant::now() + .checked_add(options.timeout) + .ok_or_else(|| anyhow::anyhow!("Gemini request timeout is too large"))?; + let mut connection_error = + io::Error::new(io::ErrorKind::AddrNotAvailable, "No server address found"); + let mut connected_stream = None; + + for address in (domain, port).to_socket_addrs()? { + match TcpStream::connect_timeout(&address, remaining_time(deadline)?) { + Ok(stream) => { + connected_stream = Some(stream); + + break; + } + Err(error) => connection_error = error, + } + } + + let mut stream = DeadlineStream { + stream: connected_stream.ok_or(connection_error)?, + deadline, + }; + let mut connection = + rustls::ClientConnection::new(Arc::new(config), domain.try_into()?)?; + + connection.complete_io(&mut stream)?; + check_certificate( + connection.peer_certificates().and_then(|chain| chain.first()), domain, - url.port().unwrap_or(1965) - ))?; + port, + )?; + let mut tls = rustls::Stream::new(&mut connection, &mut stream); - tls.write_all(format!("{url}\r\n").as_bytes())?; + tls.write_all(request_line.as_bytes())?; - let mut plain_text = Vec::new(); + let mut response_bytes = Vec::new(); + let read_limit_bytes = options.max_response_bytes.saturating_add(1) as u64; - tls.read_to_end(&mut plain_text)?; + Read::take(&mut tls, read_limit_bytes).read_to_end(&mut response_bytes)?; + anyhow::ensure!( + response_bytes.len() <= options.max_response_bytes, + "Gemini response exceeds {} bytes", + options.max_response_bytes + ); - Ok(Response::new(&plain_text, tls.conn.negotiated_cipher_suite())) + Response::parse(&response_bytes, tls.conn.negotiated_cipher_suite()) } diff --git a/src/request/non_blocking.rs b/src/request/non_blocking.rs index dd59e6b..26f5a4a 100644 --- a/src/request/non_blocking.rs +++ b/src/request/non_blocking.rs @@ -1,69 +1,151 @@ use { - crate::request::Response, + crate::request::{ + DEFAULT_GEMINI_PORT, RequestOptions, Response, request_line, + trust::{CertificateStore, check_tofu, client_config, tofu_client_config}, + }, + std::sync::Arc, tokio::io::{AsyncReadExt, AsyncWriteExt}, }; -/// Make a request to a Gemini server +/// Make a request to a Gemini server. /// /// The `url` **should** be prefixed with a scheme (e.g. "gemini://"). /// /// # Example /// -/// ```rust +/// ```rust,no_run /// #[tokio::main] -/// async fn main() { -/// match germ::request::request(&url::Url::parse("gemini://fuwn.me").unwrap()) -/// .await -/// { -/// Ok(response) => println!("{:?}", response), -/// Err(_) => {} -/// } +/// async fn main() -> anyhow::Result<()> { +/// let url = url::Url::parse("gemini://fuwn.me")?; +/// let response = germ::request::request(&url).await?; +/// +/// println!("{response:?}"); +/// +/// Ok(()) /// } /// ``` /// /// # Errors /// -/// - May error if the URL is invalid -/// - May error if the server is unreachable -/// - May error if the TLS write fails -/// - May error if the TLS read fails +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. pub async fn request(url: &url::Url) -> anyhow::Result { - let mut tls = tokio_rustls::TlsConnector::from(std::sync::Arc::new( - rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(std::sync::Arc::new( - crate::request::GermVerifier::new(), - )) - .with_no_client_auth(), - )) - .connect( - rustls::ServerName::try_from( - url - .domain() - .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?, - )?, - tokio::net::TcpStream::connect(format!( - "{}:{}", - url - .domain() - .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?, - url.port().unwrap_or(1965) - )) - .await?, + request_with_options(url, &RequestOptions::default()).await +} + +/// Make a request using the supplied trusted certificate authorities. +/// +/// The supplied roots replace the default Mozilla roots. To keep those roots, +/// start with `default_root_certificates()` before adding a capsule's DER +/// certificate with `RootCertStore::add`. Added certificates become trust +/// anchors, not exact fingerprint pins. The server name and certificate +/// validity period are still checked. Default time and size limits apply. +/// +/// # Errors +/// +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. +pub async fn request_with_roots( + url: &url::Url, + roots: &rustls::RootCertStore, +) -> anyhow::Result { + let options = RequestOptions::with_roots(roots.clone()); + + request_with_options(url, &options).await +} + +/// Make a request with the supplied certificate, time, and size limits. +/// +/// The timeout covers connection, TLS, sending, and receiving. The response +/// limit includes the header and rejects an oversized response. +/// +/// # Errors +/// +/// Returns an error for an invalid URL, failed connection or TLS validation, +/// elapsed timeout, oversized response, or malformed response header. +pub async fn request_with_options( + url: &url::Url, + options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + client_config(options.root_certificates.clone()), + |_, _, _| Ok(()), + ) + .await +} + +/// Makes a request using a caller-owned trust-on-first-use certificate store. +/// +/// The first verified certificate is saved before the request URL is sent. +/// Later requests require the same certificate. An expired certificate may be +/// replaced; an unexpected change before expiry returns an error. The store +/// must persist the certificate for this policy to work across process runs. +/// TOFU uses the timeout and response size limits in `options` and ignores +/// `options.root_certificates`. +/// +/// # Errors +/// +/// Returns an error for TLS or pin validation, store failure, timeout, +/// oversized response, or malformed response. +pub async fn request_with_tofu( + url: &url::Url, + store: &mut dyn CertificateStore, + options: &RequestOptions, +) -> anyhow::Result { + request_with_config( + url, + options, + tofu_client_config(), + |certificate, host, port| check_tofu(store, host, port, certificate), ) - .await?; - let cipher_suite = tls.get_mut().1.negotiated_cipher_suite(); + .await +} + +async fn request_with_config( + url: &url::Url, + options: &RequestOptions, + config: rustls::ClientConfig, + check_certificate: impl FnOnce( + Option<&rustls::Certificate>, + &str, + u16, + ) -> anyhow::Result<()>, +) -> anyhow::Result { + let request_line = request_line(url)?; + let domain = url + .domain() + .ok_or_else(|| anyhow::anyhow!("Invalid URL: missing domain"))?; + let server_name = rustls::ServerName::try_from(domain)?; + let port = url.port().unwrap_or(DEFAULT_GEMINI_PORT); + + tokio::time::timeout(options.timeout, async { + let address = (domain, port); + let stream = tokio::net::TcpStream::connect(address).await?; + let connector = tokio_rustls::TlsConnector::from(Arc::new(config)); + let mut tls = connector.connect(server_name, stream).await?; + let cipher_suite = tls.get_mut().1.negotiated_cipher_suite(); - tls.write_all(format!("{url}\r\n").as_bytes()).await?; + check_certificate( + tls.get_ref().1.peer_certificates().and_then(|chain| chain.first()), + domain, + port, + )?; + tls.write_all(request_line.as_bytes()).await?; - Ok(Response::new( - &{ - let mut plain_text = Vec::new(); + let mut response_bytes = Vec::new(); + let read_limit_bytes = options.max_response_bytes.saturating_add(1) as u64; - tls.read_to_end(&mut plain_text).await?; + tls.take(read_limit_bytes).read_to_end(&mut response_bytes).await?; + anyhow::ensure!( + response_bytes.len() <= options.max_response_bytes, + "Gemini response exceeds {} bytes", + options.max_response_bytes + ); - plain_text - }, - cipher_suite, - )) + Response::parse(&response_bytes, cipher_suite) + }) + .await + .map_err(|_| anyhow::anyhow!("Gemini request timed out"))? } diff --git a/src/request/response.rs b/src/request/response.rs index b1a5cbc..cf92878 100644 --- a/src/request/response.rs +++ b/src/request/response.rs @@ -1,5 +1,7 @@ use {crate::request::Status, rustls::SupportedCipherSuite, std::borrow::Cow}; +const MAX_META_BYTES: usize = 1024; + #[derive(Debug, Clone, PartialEq)] pub struct Response { status: Status, @@ -10,34 +12,54 @@ pub struct Response { } impl Response { - pub(crate) fn new(data: &[u8], suite: Option) -> Self { + pub(crate) fn parse( + response_bytes: &[u8], + cipher_suite: Option, + ) -> anyhow::Result { let delimiter = b"\r\n"; - let header_end = data + let header_end = response_bytes .windows(delimiter.len()) .position(|window| window == delimiter) - .map_or(data.len(), |pos| pos + delimiter.len()); - let header_bytes = &data[..header_end]; - let header_cow = String::from_utf8_lossy(header_bytes); - let header_trimmed = header_cow.trim_end(); - let content_bytes = if header_end < data.len() { - Some(data[header_end..].to_vec()) - } else { - None - }; - let (status_string, meta_string) = if header_trimmed.len() >= 2 { - header_trimmed.split_at(2) - } else { - (header_trimmed, "") - }; - let status_code = status_string.parse::().unwrap_or(0); - - Self { + .ok_or_else(|| anyhow::anyhow!("Gemini response header has no CRLF"))?; + let header_bytes = &response_bytes[..header_end]; + + anyhow::ensure!( + header_bytes.len() >= 3 + && header_bytes[0].is_ascii_digit() + && header_bytes[1].is_ascii_digit() + && header_bytes[2] == b' ', + "Gemini response header must begin with two digits and a space" + ); + + let meta_bytes = &header_bytes[3..]; + + anyhow::ensure!( + meta_bytes.len() <= MAX_META_BYTES, + "Gemini response meta exceeds {MAX_META_BYTES} bytes" + ); + anyhow::ensure!( + !meta_bytes.starts_with(&[0xef, 0xbb, 0xbf]), + "Gemini response meta begins with a byte order mark" + ); + anyhow::ensure!( + !meta_bytes.contains(&b'\r') && !meta_bytes.contains(&b'\n'), + "Gemini response meta contains a line break" + ); + + let meta = std::str::from_utf8(meta_bytes)?.to_owned(); + let status_code = i32::from(header_bytes[0] - b'0') * 10 + + i32::from(header_bytes[1] - b'0'); + let body_start = header_end + delimiter.len(); + let content = (body_start < response_bytes.len()) + .then(|| response_bytes[body_start..].to_vec()); + + Ok(Self { status: Status::from(status_code), - meta: meta_string.trim_start().to_string(), - content: content_bytes, - size: data.len(), - suite, - } + meta, + content, + size: response_bytes.len(), + suite: cipher_suite, + }) } #[must_use] @@ -47,9 +69,9 @@ impl Response { #[must_use] pub fn meta(&self) -> Cow<'_, str> { Cow::Borrowed(&self.meta) } - /// This associated function assumes that the content is valid UTF-8. + /// Return the body as text, replacing invalid UTF-8 sequences. /// - /// If you want to handle data bytes directly, use + /// To inspect the original bytes, use /// [`Response::content_bytes`]. #[must_use] pub fn content(&self) -> Option { @@ -68,3 +90,53 @@ impl Response { #[must_use] pub const fn suite(&self) -> &Option { &self.suite } } + +#[cfg(test)] +mod tests { + use { + super::{MAX_META_BYTES, Response}, + crate::request::{Status, StatusCategory}, + }; + + #[test] + fn parses_header_and_body_without_changing_meta() { + let response = Response::parse(b"20 text/gemini\r\nhello", None).unwrap(); + + assert_eq!(response.meta(), " text/gemini"); + assert_eq!(response.content_bytes(), Some(b"hello".as_slice())); + assert_eq!(*response.size(), 22); + } + + #[test] + fn rejects_malformed_headers() { + for header in [ + b"\xc3\xa9 text/gemini\r\n".as_slice(), + b"2x text/gemini\r\n", + b"20text/gemini\r\n", + b"20 text/gemini", + b"20 bad\nmeta\r\n", + b"20 \xff\r\n", + b"20 \xef\xbb\xbfmeta\r\n", + ] { + assert!(Response::parse(header, None).is_err(), "{header:?}"); + } + } + + #[test] + fn enforces_meta_byte_limit() { + let accepted = format!("20 {}\r\n", "a".repeat(MAX_META_BYTES)); + let rejected = format!("20 {}\r\n", "a".repeat(MAX_META_BYTES + 1)); + + assert!(Response::parse(accepted.as_bytes(), None).is_ok()); + assert!(Response::parse(rejected.as_bytes(), None).is_err()); + } + + #[test] + fn retains_unknown_response_status() { + let response = Response::parse(b"29 text/gemini\r\nbody", None).unwrap(); + + assert_eq!(*response.status(), Status::Unknown(29)); + assert_eq!(response.status().category(), Some(StatusCategory::Success)); + assert_eq!(response.content_bytes(), Some(b"body".as_slice())); + } +} diff --git a/src/request/status.rs b/src/request/status.rs index 94600bf..0d30332 100644 --- a/src/request/status.rs +++ b/src/request/status.rs @@ -1,6 +1,6 @@ use std::{fmt, fmt::Formatter}; -/// Simple Gemini status reporting +/// A Gemini response status. /// /// # Examples /// @@ -10,10 +10,11 @@ use std::{fmt, fmt::Formatter}; /// assert_eq!(Status::from(10), Status::Input); /// assert_eq!(i32::from(Status::Input), 10); /// ``` -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] pub enum Status { Input, SensitiveInput, + #[default] Success, TemporaryRedirect, PermanentRedirect, @@ -30,16 +31,42 @@ pub enum Status { ClientCertificateRequired, CertificateNotAuthorised, CertificateNotValid, + /// A code without a named variant. + Unknown(i32), + /// A legacy status without a known code. Unsupported, } -impl Default for Status { - fn default() -> Self { Self::Success } +/// The response category indicated by the first status digit. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum StatusCategory { + Input, + Success, + Redirect, + TemporaryFailure, + PermanentFailure, + ClientCertificateRequired, +} + +impl Status { + /// Returns the category for codes in the Gemini status range. + #[must_use] + pub fn category(self) -> Option { + match i32::from(self) { + 10..=19 => Some(StatusCategory::Input), + 20..=29 => Some(StatusCategory::Success), + 30..=39 => Some(StatusCategory::Redirect), + 40..=49 => Some(StatusCategory::TemporaryFailure), + 50..=59 => Some(StatusCategory::PermanentFailure), + 60..=69 => Some(StatusCategory::ClientCertificateRequired), + _ => None, + } + } } impl From for i32 { - fn from(n: Status) -> Self { - match n { + fn from(status: Status) -> Self { + match status { Status::Input => 10, Status::SensitiveInput => 11, Status::Success => 20, @@ -58,14 +85,15 @@ impl From for i32 { Status::ClientCertificateRequired => 60, Status::CertificateNotAuthorised => 61, Status::CertificateNotValid => 62, + Status::Unknown(code) => code, Status::Unsupported => 0, } } } impl From for Status { - fn from(n: i32) -> Self { - match n { + fn from(code: i32) -> Self { + match code { 10 => Self::Input, 11 => Self::SensitiveInput, 20 => Self::Success, @@ -84,7 +112,7 @@ impl From for Status { 60 => Self::ClientCertificateRequired, 61 => Self::CertificateNotAuthorised, 62 => Self::CertificateNotValid, - _ => Self::Unsupported, + _ => Self::Unknown(code), } } } diff --git a/src/request/trust.rs b/src/request/trust.rs new file mode 100644 index 0000000..33c06fb --- /dev/null +++ b/src/request/trust.rs @@ -0,0 +1,224 @@ +use { + rustls::{ + Certificate, CertificateError, ClientConfig, Error, OwnedTrustAnchor, + RootCertStore, ServerName, + client::{ServerCertVerified, ServerCertVerifier, WebPkiVerifier}, + }, + std::{sync::Arc, time::SystemTime}, +}; + +/// Stores exact server certificates by hostname and port across requests. +/// +/// Implementations should persist updates before returning from `save` and +/// coordinate concurrent writers to avoid silently replacing a changed pin. +/// Store methods run synchronously, including during async requests. +pub trait CertificateStore: Send { + /// Loads the previously trusted DER certificate, if any. + /// + /// # Errors + /// + /// Returns an error when the store cannot read the certificate. + fn load( + &mut self, + hostname: &str, + port: u16, + ) -> anyhow::Result>>; + + /// Saves the DER certificate as the trusted certificate for this server. + /// + /// # Errors + /// + /// Returns an error when the store cannot persist the certificate. + fn save( + &mut self, + hostname: &str, + port: u16, + certificate: &[u8], + ) -> anyhow::Result<()>; +} + +#[derive(Debug)] +struct TrustOnFirstUseVerifier; + +impl ServerCertVerifier for TrustOnFirstUseVerifier { + fn verify_server_cert( + &self, + end_entity: &Certificate, + intermediates: &[Certificate], + server_name: &ServerName, + scts: &mut dyn Iterator, + ocsp_response: &[u8], + now: SystemTime, + ) -> Result { + let mut roots = RootCertStore::empty(); + + roots.add(end_entity)?; + + // The presented leaf is the temporary trust anchor; rustls still checks + // its validity, server name, and handshake signature. + WebPkiVerifier::new(roots, None).verify_server_cert( + end_entity, + intermediates, + server_name, + scts, + ocsp_response, + now, + ) + } +} + +pub fn tofu_client_config() -> ClientConfig { + ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(Arc::new(TrustOnFirstUseVerifier)) + .with_no_client_auth() +} + +pub fn check_tofu( + store: &mut dyn CertificateStore, + hostname: &str, + port: u16, + certificate: Option<&Certificate>, +) -> anyhow::Result<()> { + let certificate = certificate.ok_or_else(|| { + anyhow::anyhow!("Gemini server did not provide a certificate") + })?; + + match store.load(hostname, port)? { + Some(previous) if previous == certificate.0 => Ok(()), + Some(previous) if previous_certificate_expired(&previous, hostname) => + store.save(hostname, port, &certificate.0), + Some(_) => + anyhow::bail!("Gemini server certificate changed for {hostname}:{port}"), + None => store.save(hostname, port, &certificate.0), + } +} + +fn previous_certificate_expired(certificate: &[u8], hostname: &str) -> bool { + let certificate = Certificate(certificate.to_vec()); + let Ok(server_name) = ServerName::try_from(hostname) else { + return false; + }; + let mut roots = RootCertStore::empty(); + + if roots.add(&certificate).is_err() { + return false; + } + + // Revalidate the stored certificate to detect expiry without parsing its + // validity dates separately. + let result = WebPkiVerifier::new(roots, None).verify_server_cert( + &certificate, + &[], + &server_name, + &mut std::iter::empty(), + &[], + SystemTime::now(), + ); + + matches!(result, Err(Error::InvalidCertificate(CertificateError::Expired))) +} + +/// Return the Mozilla CA roots used by the default request functions. +/// +/// Callers can add a certificate to this store before passing it to +/// `request_with_roots` to trust a selected self-signed Gemini capsule. +#[must_use] +pub fn default_root_certificates() -> RootCertStore { + let mut roots = RootCertStore::empty(); + + roots.add_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.iter().map( + |anchor| { + OwnedTrustAnchor::from_subject_spki_name_constraints( + anchor.subject.as_ref().to_vec(), + anchor.subject_public_key_info.as_ref().to_vec(), + anchor + .name_constraints + .as_ref() + .map(|constraints| constraints.as_ref().to_vec()), + ) + }, + )); + + roots +} + +pub fn client_config(roots: RootCertStore) -> ClientConfig { + ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(roots) + .with_no_client_auth() +} + +#[cfg(test)] +mod tests { + use { + super::{CertificateStore, check_tofu}, + rustls::Certificate, + std::collections::HashMap, + }; + + #[derive(Default)] + struct MemoryCertificates(HashMap<(String, u16), Vec>); + + impl CertificateStore for MemoryCertificates { + fn load( + &mut self, + hostname: &str, + port: u16, + ) -> anyhow::Result>> { + Ok(self.0.get(&(hostname.to_owned(), port)).cloned()) + } + + fn save( + &mut self, + hostname: &str, + port: u16, + certificate: &[u8], + ) -> anyhow::Result<()> { + self.0.insert((hostname.to_owned(), port), certificate.to_vec()); + + Ok(()) + } + } + + #[test] + fn pins_are_scoped_to_hostname_and_port() { + let certificate = Certificate(vec![1, 2, 3]); + let other_certificate = Certificate(vec![4, 5, 6]); + let mut store = MemoryCertificates::default(); + + check_tofu(&mut store, "one.test", 1965, Some(&certificate)).unwrap(); + check_tofu(&mut store, "one.test", 1965, Some(&certificate)).unwrap(); + check_tofu(&mut store, "one.test", 1966, Some(&other_certificate)).unwrap(); + check_tofu(&mut store, "two.test", 1965, Some(&other_certificate)).unwrap(); + assert_eq!(store.0.len(), 3); + assert!( + check_tofu(&mut store, "one.test", 1965, Some(&other_certificate)) + .is_err() + ); + } + + #[test] + fn replaces_an_expired_certificate() { + let mut parameters = + rcgen::CertificateParams::new(vec!["localhost".to_owned()]); + + parameters.not_before = rcgen::date_time_ymd(2018, 1, 1); + parameters.not_after = rcgen::date_time_ymd(2019, 1, 1); + + let expired_certificate = rcgen::Certificate::from_params(parameters) + .unwrap() + .serialize_der() + .unwrap(); + let replacement = Certificate(vec![7, 8, 9]); + let mut store = MemoryCertificates::default(); + + store.0.insert(("localhost".to_owned(), 1965), expired_certificate); + check_tofu(&mut store, "localhost", 1965, Some(&replacement)).unwrap(); + assert_eq!( + store.0.get(&("localhost".to_owned(), 1965)), + Some(&replacement.0) + ); + } +} diff --git a/src/request/verifier.rs b/src/request/verifier.rs deleted file mode 100644 index 9fc54a6..0000000 --- a/src/request/verifier.rs +++ /dev/null @@ -1,28 +0,0 @@ -use { - rustls::{ - Certificate, - client::{self, ServerCertVerified}, - }, - std::time::SystemTime, -}; - -#[allow(clippy::module_name_repetitions)] -pub struct GermVerifier; - -impl GermVerifier { - pub const fn new() -> Self { Self {} } -} - -impl client::ServerCertVerifier for GermVerifier { - fn verify_server_cert( - &self, - _end_entity: &Certificate, - _intermediates: &[Certificate], - _server_name: &client::ServerName, - _scts: &mut dyn Iterator, - _ocsp_response: &[u8], - _now: SystemTime, - ) -> Result { - Ok(ServerCertVerified::assertion()) - } -} diff --git a/tests/ast.rs b/tests/ast.rs index 317272b..7d7b47d 100644 --- a/tests/ast.rs +++ b/tests/ast.rs @@ -1,9 +1,10 @@ +#![cfg(feature = "ast")] + #[cfg(test)] mod test { - use germ::{ - EXAMPLE_GEMTEXT, - ast::{Ast, Node}, - }; + use germ::ast::{Ast, Node}; + + const EXAMPLE_GEMTEXT: &str = include_str!("../examples/example.gmi"); #[test] fn build_multi_line_list_with_text() { @@ -40,20 +41,14 @@ mod test { #[test] fn gemtext_to_ast_then_ast_to_gemtext() { - assert_eq!( - Ast::from_string(EXAMPLE_GEMTEXT).to_gemtext(), - // `to_gemtext` appends a newline to all responses, so let's make sure we - // account for that. - EXAMPLE_GEMTEXT - ); + assert_eq!(Ast::from_string(EXAMPLE_GEMTEXT).to_gemtext(), EXAMPLE_GEMTEXT); } + #[cfg(feature = "macros")] #[test] fn gemtext_to_ast_then_ast_to_gemtext_macro_expression() { assert_eq!( germ::gemini_to_ast!(EXAMPLE_GEMTEXT).to_gemtext(), - // `to_gemtext` appends a newline to all responses, so let's make sure we - // account for that. EXAMPLE_GEMTEXT ); } @@ -61,7 +56,7 @@ mod test { #[test] fn gemtext_to_ast_then_node_to_ast_to_gemtext() { assert_eq!( - Ast::from_nodes(germ::gemini_to_ast!("=> / Home").inner().to_vec()) + Ast::from_nodes(Ast::from_string("=> / Home").inner().to_vec()) .to_gemtext(), "=> / Home" ); @@ -83,7 +78,6 @@ mod test { #[test] fn build_heading_with_unicode_and_edge_cases() { - // Unicode characters let ast = Ast::from_string("# Hello, δΈ–η•Œ!"); assert_eq!(ast.inner().len(), 1); @@ -95,7 +89,6 @@ mod test { panic!("Expected heading node"); } - // Only hashes let ast = Ast::from_string("###"); assert_eq!(ast.inner().len(), 1); @@ -107,16 +100,92 @@ mod test { panic!("Expected heading node"); } - // Many hashes let ast = Ast::from_string("########## Very Deep Heading"); - assert_eq!(ast.inner().len(), 1); + assert_eq!(ast.inner(), &vec![Node::Text( + "########## Very Deep Heading".to_string() + )]); + } - if let Node::Heading { level, text } = ast.inner().first().unwrap() { - assert_eq!(level, &10); - assert_eq!(text, "Very Deep Heading"); - } else { - panic!("Expected heading node"); + #[test] + fn recognises_only_exact_line_markers() { + let ast = Ast::from_string( + "*star\n=text\n`tick\n#### too deep\n* item\n* indented", + ); + + assert_eq!(ast.inner(), &vec![ + Node::Text("*star".to_string()), + Node::Text("=text".to_string()), + Node::Text("`tick".to_string()), + Node::Text("#### too deep".to_string()), + Node::List(vec!["item".to_string(), " indented".to_string()]), + ]); + } + + #[test] + fn preserves_link_label_spacing() { + let source = "=> /path label with spaces"; + let ast = Ast::from_string(source); + + assert_eq!(ast.inner(), &vec![Node::Link { + to: "/path".to_string(), + text: Some("label with spaces".to_string()), + }]); + assert_eq!(ast.to_gemtext(), source); + } + + #[test] + fn preserves_unclosed_preformatted_content() { + let source = "```alt\n* item\n=> /path\n# heading"; + let ast = Ast::from_string(source); + + assert_eq!(ast.inner(), &vec![Node::PreformattedText { + alt_text: Some("alt".to_string()), + text: "* item\n=> /path\n# heading".to_string(), + }]); + assert_eq!(ast.to_gemtext(), source); + + let source_with_final_newline = "```alt\ncode\n"; + let ast = Ast::from_string(source_with_final_newline); + + assert_eq!(ast.inner(), &vec![Node::PreformattedText { + alt_text: Some("alt".to_string()), + text: "code\n".to_string(), + }]); + assert_eq!(ast.to_gemtext(), source_with_final_newline); + } + + #[test] + fn preserves_canonical_line_round_trips() { + for source in [ + "", + "plain", + "plain\n", + "plain\n\n", + "\n", + "\n\n", + "* item\n* indented\nnext", + "* item\n\n", + "# heading\n###\n> quoted", + "```\n```\n", + "```\ncode\n```", + "```\ncode\n```\n\n", + "```\ncode\n", + "```", + "```\n", + "```\n\n", + "plain\r\n\r\n* item\r\n", + "```\r\ncode\r\n```\r\n", + ] { + assert_eq!(Ast::from_string(source).to_gemtext(), source, "{source:?}"); } } + + #[test] + fn normalises_mixed_line_endings_and_optional_marker_spaces() { + let source = "# Heading\r\n=> /path label\n> quote\r\n"; + let expected = "# Heading\r\n=> /path label\r\n> quote\r\n"; + + assert_eq!(Ast::from_string(source).to_gemtext(), expected); + } } diff --git a/tests/convert.rs b/tests/convert.rs index 871f590..a0797e4 100644 --- a/tests/convert.rs +++ b/tests/convert.rs @@ -1,9 +1,12 @@ +#![cfg(feature = "convert")] + #[cfg(test)] mod test { use germ::{ - convert::{Target, from_string}, - gemini_to_html, gemini_to_md, + ast::{Ast, Node}, + convert::{Target, from_ast, from_string}, }; + #[cfg(feature = "macros")] use germ::{gemini_to_html, gemini_to_md}; #[test] fn convert_from_string_to_html_single_line() { @@ -15,6 +18,91 @@ mod test { assert_eq!(from_string("hi\n# hi", &Target::HTML), "

hi

hi

",); } + #[test] + fn html_escapes_content_in_every_node() { + let hostile = "<&>\"'".to_owned(); + let ast = Ast::from_nodes(vec![ + Node::Text(hostile.clone()), + Node::Heading { level: 1, text: hostile.clone() }, + Node::List(vec![hostile.clone()]), + Node::Blockquote(hostile.clone()), + Node::PreformattedText { alt_text: None, text: hostile }, + ]); + + assert_eq!( + from_ast(&ast, &Target::HTML), + "

<&>"'

<&>"'
  • <&>"'
<&>&\ + quot;'
<&>"'
", + ); + } + + #[test] + fn html_escapes_link_targets_and_labels() { + let ast = Ast::from_nodes(vec![Node::Link { + to: "/path?x=\"<&>".to_owned(), + text: Some("".to_owned()), + }]); + + assert_eq!( + from_ast(&ast, &Target::HTML), + "<script>alert('x'\ + )</script>
", + ); + } + + #[test] + fn html_renders_unsafe_links_as_text() { + for target in [ + "javascript:alert(1)", + "JaVaScRiPt:alert(1)", + "data:text/html,", + "java\nscript:alert(1)", + "vbscript:msgbox(1)", + "sftp://example.com/file", + ] { + let ast = Ast::from_nodes(vec![Node::Link { + to: target.to_owned(), + text: Some("safe label".to_owned()), + }]); + + assert_eq!(from_ast(&ast, &Target::HTML), "safe label
"); + } + } + + #[test] + fn html_escapes_raw_gemtext() { + assert_eq!( + from_string( + "\n=> javascript:alert(2) click", + &Target::HTML + ), + "

<script>alert(1)</script>

click
", + ); + } + + #[test] + fn html_keeps_safe_relative_and_absolute_links() { + for target in [ + "/local", + "relative", + "#anchor", + "gemini://example.com", + "https://example.com", + ] { + let ast = Ast::from_nodes(vec![Node::Link { + to: target.to_owned(), + text: None, + }]); + + assert_eq!( + from_ast(&ast, &Target::HTML), + format!("{target}
"), + ); + } + } + + #[cfg(feature = "macros")] #[test] fn convert_from_string_to_html_single_link_macro_expression() { assert_eq!( @@ -37,12 +125,118 @@ mod test { fn convert_from_string_to_markdown_single_link() { assert_eq!( from_string("=> /to hello !", &Target::Markdown), - "[hello !](/to)\n", + "[hello \\!](/to)\n", + ); + } + + #[test] + fn markdown_escapes_literal_text_in_every_node() { + let literal = "